#include "interpolate.h"

#include "logger.h"

#include <array>
#include <charconv>
#include <stdexcept>

#include "asyncpp/generator.h"
#include "kblib/build.h"
#include "srell/srell.hpp"

using namespace std::literals;

auto datafile::parse_nodename(std::string_view name)
    -> std::tuple<std::string, int, bool> {
	using Ret = std::tuple<std::string, int, bool>;
	auto split = name.find('|');
	if (split == std::string_view::npos) {
		return Ret{name, 0, false};
	}
	auto primary_part = name.substr(0, split);
	auto args_spec = name.substr(split);

	return Ret{primary_part, std::count(args_spec.begin(), args_spec.end(), '|'),
	           kblib::ends_with(name, "..."sv)};
}

auto datafile::find_node(std::string_view name) -> node* {
	try {
		auto split = name.find('|');
		if (split == std::string_view::npos) {
			return find_node(name, 0);
		}
		auto primary_part = name.substr(0, split);
		auto args_spec = name.substr(split);

		auto nlist = nodes.find(primary_part);

		if (nlist == nodes.end()) {
			return nullptr;
		}

		auto argc = std::count(args_spec.begin(), args_spec.end(), '|');

		for (auto&& n : nlist->second) {
			if (n.declared_argc == argc
			    or (n.variadic and n.declared_argc <= argc)) {
				return &n;
			}
		}
		return nullptr;
	} catch (const std::out_of_range& e) {
		log_err("In ", kblib::quoted(name), ", error ", e.what());
		throw;
	}
}

auto datafile::find_node(std::string_view name) const -> const node* {
	try {
		auto split = name.find('|');
		if (split == std::string_view::npos) {
			return find_node(name, 0);
		}
		auto primary_part = name.substr(0, split);
		auto args_spec = name.substr(split);

		return find_node(primary_part,
		                 std::count(args_spec.begin(), args_spec.end(), '|'));
	} catch (const std::out_of_range& e) {
		log_err("In ", kblib::quoted(name), ", error ", e.what());
		throw;
	}
}

auto format_path_impl(const path_t& path, std::string& out) -> void {
	out.push_back('[');
	out += kblib::to_string<62>(path.branch);
	for (const path_t& c : path.children) {
		format_path_impl(c, out);
	}
	out.push_back(']');
}

auto format_path(const path_t& path) -> std::string {
	std::string out = "+"s + kblib::to_string<62>(path.branch);
	for (const path_t& c : path.children) {
		format_path_impl(c, out);
	}
	return out;
}

static auto mod_string(std::string_view in, std::string_view mods)
    -> std::string {
	using namespace kblib::literals;
	switch (kblib::FNV32a(mods)) {
	case ""_fnv32:
	default:
		return std::string(in);
	case "q"_fnv32:
		return kblib::quoted(in);
	case "h"_fnv32:
		return kblib::concat("<td>", kblib::html_encode(in), "</td>");
	case "Hh"_fnv32:
	case "hH"_fnv32:
	case "H"_fnv32:
		return kblib::concat("<th>", kblib::html_encode(in), "</th>");
	case "e"_fnv32:
		return kblib::escapify(in);
	case "u"_fnv32:
		return kblib::url_encode(in);
	}
}

auto format(const word_data_t& data, std::string_view fmt, const path_t* path)
    -> std::string {
	std::string out;
	for (const auto& [text, spec, mods] : basic_parser(fmt)) {
		if (not text.empty()) {
			out += text;
		}
		if (spec) {
			if (*spec == "%all%") {
				bool htmlMode = (mods == "h");
				out += std::accumulate(
				    data.begin(), data.end(), std::string{},
				    [&, mods = mods](
				        std::string out,
				        const std::pair<channelID, std::string>& ch) mutable {
					    if (ch.first == "val"_ch or ch.first == "path"_ch) {
						    return out;
					    } else {
						    if (htmlMode) {
							    out += "</tr><tr>";
						    } else {
							    out += "\t";
						    }
						    return out += mod_string(ch.second, mods);
					    }
				    });
			} else if (*spec == "path") {
				if (path != nullptr) {
					out += format_path(*path);
				} else if (auto check = kblib::get_check(data, "path"_ch)) {
					out += mod_string(check.it->second, mods);
				}
			} else if (auto check = kblib::get_check(data, ch(*spec))) {
				out += mod_string(check.it->second, mods);
			}
		}
	}
	return out;
}

auto freqs_of(const template_machine& bm, const node& n,
              const std::vector<std::string>& args, const counters& c, int argc)
    -> small_vector<double, 16> {
	return kblib::build<small_vector<double, 16>>(
	    n.freqs.begin(), n.freqs.end(), [&](auto v) {
		    return kblib::visit2(
		        v,
		        // Negative frequencies are set to zero
		        [](double f) { return std::max(f, 0.); },
		        [&](const tstr_ops& v) {
			        return std::max(
			            kblib::lexical_cast<double>(bm.eval(v, args, c, argc)),
			            0.);
		        });
	    });
}

struct fixed_branch {
	std::string val;
	double freq;
	vmap<channelID, std::string> other_channels;
};

auto apply_args_lazy(const template_machine& bm, const node& n,
                     const std::vector<std::string>& args, const counters& c,
                     int argc, std::size_t selection) -> fixed_branch {
	const auto& valsel = n.vals[selection];
	const auto& ocsel = n.other_channels[selection];
	return fixed_branch{
	    bm.eval(valsel, args, c, argc),
	    kblib::visit2(
	        n.freqs[selection], [](double f) { return std::max(f, 0.); },
	        [&](const tstr_ops& v) {
		        return std::max(
		            kblib::lexical_cast<double>(bm.eval(v, args, c, argc)), 0.);
	        }),
	    [&] {
		    auto x = kblib::build<small_vector<
		        std::pair<channelID, std::string>,
		        4>>(ocsel.begin(), ocsel.end(), [&](const auto& alt) {
			    return std::pair{alt.first, bm.eval(alt.second, args, c, argc)};
		    });
		    return vmap<channelID, std::string>{x.begin(), x.end()};
	    }()};
}

auto chooseFrom(
    const datafile& data, RandomGenerator& rng, counters c, const node& n,
    const std::vector<std::string>& args,
    const std::variant<std::monostate, flist_t, ilist_t>& freq_override)
    -> Word {
	// Algorithm:
	// 1. Determine branch to take
	//  1. Apply args to node frequencies
	//  2. Apply frequency override
	//  3. categorically choose branch
	// 2. Apply args to branch
	// 3. Interpolate noderefs into branch's val

	if (n.freqs.size() != n.vals.size()
	    or n.freqs.size() != n.other_channels.size()) {
		log_err("Invariant failure: Node's raw and computed data disagree in "
		        "number of branches.",
		        "\nnode: ", n.name, "\nvalsize: ", n.vals.size(),
		        "\nfreqsize: ", n.freqs.size(),
		        "\nocsize: ", n.other_channels.size());
		std::terminate();
	}

	auto computed_freqs = [&] {
		auto raw_freqs = freqs_of(data.bm, n, args, c, n.declared_argc);
		return kblib::visit2(
		    freq_override, [&](std::monostate /*unused*/) { return raw_freqs; },
		    [&](const flist_t& fl) {
			    std::copy_n(fl.begin(), std::min(fl.size(), raw_freqs.size()),
			                raw_freqs.begin());
			    return raw_freqs;
		    },
		    [&](const ilist_t& il) {
			    decltype(raw_freqs) new_freqs(raw_freqs.size(), 0.0);
			    for (const auto& [idx, f] : il) {
				    if (f) {
					    // std::clog << ':' << *f;
					    new_freqs[idx] = *f;
				    } else {
					    new_freqs[idx] = raw_freqs[idx];
				    }
			    }
			    return new_freqs;
		    });
	}();

	std::discrete_distribution<std::size_t> dist(computed_freqs.begin(),
	                                             computed_freqs.end());
	auto selection = dist(rng);

	auto branch
	    = apply_args_lazy(data.bm, n, args, c, n.declared_argc, selection);

	double probability
	    = computed_freqs[selection]
	      / std::max(kblib::sum(computed_freqs.begin(), computed_freqs.end()),
	                 std::numeric_limits<double>::epsilon());

	Word ret{{}, probability, {static_cast<int>(selection), {}}};
	auto include_ocs = [&] {
		for (const auto& ch : branch.other_channels) {
			ret.data[ch.first] += ch.second;
		}
	};
	if (*c.expansions >= c.expansionsLimit) {
		log_warn("expansion limit reached.");
		ret.val() = branch.val;
		include_ocs();
		return ret;
	} else if (c.depth >= c.depthLimit) {
		log_warn("depth limit reached.");
		ret.val() = branch.val;
		include_ocs();
		return ret;
	}
	if (branch.val.empty()) {
		include_ocs();
		return ret;
	}
	for (auto&& e : fparse(data, branch.val)) {
		kblib::visit2(
		    e,
		    [&](const std::string& s) {
			    ret.val() += s;
			    include_ocs();
		    },
		    [&](const noderef& nr) {
			    if (not nr.source) {
				    assert(false);
			    }
			    const auto* onode
			        = (*nr.source)->find_node(nr.name, nr.args.size());
			    if (not onode) {
				    log_err("Node not found: ", kblib::quoted(nr.name), " (with ",
				            std::to_string(nr.args.size()), " arguments).");
				    std::abort();
				    throw wordgen_error(ec::internal, "", "\n", branch.val);
			    }
			    auto tmp = chooseFrom(**nr.source, rng, incr(c), *onode, nr.args,
			                          nr.freq_override);
			    for (auto&& [ch, val] : tmp.data) {
				    ret.data[ch] += val;
			    }
			    ret.freq *= tmp.freq;
			    ret.path.children.push_back(tmp.path);
		    },
		    [&](wordgen_error& e) { throw std::move(e); });
	}
	return ret;
}

static auto include_ocs(const fixed_branch& branch, Word& ret) -> void {
	for (const auto& ch : branch.other_channels) {
		ret.data[ch.first] += ch.second;
	}
};

static auto enumerate_helper(const datafile& data, counters c,
                             bool include_zeros, Word so_far, const node& node,
                             const fixed_branch& branch, std::size_t skip_to)
    -> asyncpp::generator<Word> {
	std::size_t pos{};
	for (auto&& e : fparse(data, branch.val)) {
		if (pos < skip_to) {
			++pos;
			continue;
		}
		assert(pos >= skip_to);
		if (const std::string* s = std::get_if<std::string>(&e)) {
			so_far.val() += *s;
			include_ocs(branch, so_far);
		} else if (auto* pnr = std::get_if<noderef>(&e)) {
			auto& nr = *pnr;
			if (not nr.source) {
				assert(false);
			}
			const auto* onode = (*nr.source)->find_node(nr.name, nr.args.size());
			if (not onode) {
				log_err("Node not found: ", kblib::quoted(nr.name), " (with ",
				        std::to_string(nr.args.size()), " arguments).");
				std::abort();
				throw wordgen_error(ec::internal, "", "\n", branch.val);
			}
			log_info("Building on[B]: ", kblib::quoted(so_far.val()));
			log_info("Recursing at position ", pos);
			for (const auto& word :
			     enumerate(**nr.source, incr(c), *onode, nr.args, nr.freq_override,
			               include_zeros)) {
				Word ret = so_far;
				for (auto&& [ch, val] : word.data) {
					ret.data[ch] += val;
				}
				ret.freq *= word.freq;
				ret.path.children.push_back(word.path);
				log_info("Building on[C]: ", kblib::quoted(ret.val()));
				log_info("Recursing at position ", pos);
				for (auto word : enumerate_helper(data, c, include_zeros, ret, node,
				                                  branch, pos + 1)) {
					log_info("Yielding word: ", kblib::quoted(word.val()));
					co_yield word;
				}
			}
			co_return;
		} else {
			throw std::move(std::get<wordgen_error>(e));
		}
		++pos;
	}
	log_info("Yielding word: ", kblib::quoted(so_far.val()));
	co_yield so_far;
}

auto enumerate(
    const datafile& data, counters c, const node& n,
    std::vector<std::string> args,
    const std::variant<std::monostate, flist_t, ilist_t> freq_override,
    bool include_zeros) -> asyncpp::generator<Word> {
	// Algorithm:
	// 1. Calculate branch frequencies:
	//  1. Apply args to node frequencies
	//  2. Apply frequency override
	// 2. For each branch (if included):
	//  1. Apply args to branch
	//	 2. For each fragment:
	//   1. if literal, add to current
	//   2. else if noderef, recurse with helper
	//   3. else if error, throw

	if (n.freqs.size() != n.vals.size()
	    or n.freqs.size() != n.other_channels.size()) {
		log_err("valsize: ", std::to_string(n.vals.size()),
		        "\nfreqsize: ", std::to_string(n.freqs.size()),
		        "\nocsize: ", n.other_channels.size());
		std::terminate();
	}

	// std::clog<<"node: "<<n.name;
	auto computed_freqs = [&] {
		auto raw_freqs = freqs_of(data.bm, n, args, c, n.declared_argc);
		return kblib::visit2(
		    freq_override, [&](std::monostate /*unused*/) { return raw_freqs; },
		    [&](const flist_t& fl) {
			    std::copy_n(fl.begin(), std::min(fl.size(), raw_freqs.size()),
			                raw_freqs.begin());
			    return raw_freqs;
		    },
		    [&](const ilist_t& il) {
			    decltype(raw_freqs) new_freqs(raw_freqs.size(), 0.0);
			    // std::clog<<'!';
			    for (const auto& [idx, f] : il) {
				    // std::clog<<' '<<idx;
				    if (f) {
					    // std::clog << ':' << *f;
					    new_freqs[idx] = *f;
				    } else {
					    new_freqs[idx] = raw_freqs[idx];
				    }
			    }
			    return new_freqs;
		    });
	}();

	log_info("Searching node ", kblib::quoted(n.name));

	for (auto selection : kblib::range(computed_freqs.size())) {
		if (not include_zeros and computed_freqs[selection] == 0) {
			continue;
		}
		auto branch
		    = apply_args_lazy(data.bm, n, args, c, n.declared_argc, selection);

		log_info("Searching branch ", selection, ": ", kblib::quoted(branch.val));

		double probability
		    = computed_freqs[selection]
		      / std::max(kblib::sum(computed_freqs.begin(), computed_freqs.end()),
		                 std::numeric_limits<double>::epsilon());

		Word ret{{}, probability, {static_cast<int>(selection), {}}};

		if (*c.expansions >= c.expansionsLimit) {
			log_warn("expansion limit reached.");
			ret.val() = branch.val;
			include_ocs(branch, ret);
			co_yield ret;
		} else if (c.depth >= c.depthLimit) {
			log_warn("depth limit reached.");
			ret.val() = branch.val;
			include_ocs(branch, ret);
			co_yield ret;
		} else if (branch.val.empty()) {
			include_ocs(branch, ret);
			co_yield ret;
		} else {
			for (auto&& word :
			     enumerate_helper(data, c, include_zeros, ret, n, branch, 0)) {
				log_info("Yielding word: ", kblib::quoted(word.val()),
				         " from node ", kblib::quoted(n.name));
				co_yield word;
			}
		}
	}
}

auto basic_parser(std::string_view input)
    -> asyncpp::generator<basic_parse_token> {
	using namespace std::literals;
	enum {
		start,
		backslash,
		lbe,
		rbe,
		lb,
		backslashlb,
		mod,
		backslashmod,
	} s
	    = start;
	std::string text;
	std::string spec;
	std::string mods;
	for (const auto& c : input) {
		switch (s) {
		case start:
			switch (c) {
			default:
				text.push_back(c);
				break;
			case '\\':
				s = backslash;
				break;
			case '{':
				s = lbe;
				break;
			case '}':
				s = rbe;
				break;
			}
			break;
		case backslash:
			if (c == 't') {
				text.push_back('\t');
			} else if (c == 'n') {
				text.push_back('\n');
			} else if (c == 'v') {
				text.push_back('\v');
			} else if (c == 'f') {
				text.push_back('\f');
			} else if (c == 'r') {
				text.push_back('\r');
			} else if (c == '0') {
				text.push_back('\0');
			} else {
				text.push_back(c);
			}
			s = start;
			break;
		case backslashlb:
			if (c == 't') {
				text.push_back('\t');
			} else if (c == 'n') {
				text.push_back('\n');
			} else if (c == 'v') {
				text.push_back('\v');
			} else if (c == 'f') {
				text.push_back('\f');
			} else if (c == 'r') {
				text.push_back('\r');
			} else if (c == '0') {
				text.push_back('\0');
			} else {
				text.push_back(c);
			}
			s = lb;
			break;
		case backslashmod:
			if (c == 't') {
				mods.push_back('\t');
			} else if (c == 'n') {
				mods.push_back('\n');
			} else if (c == 'v') {
				mods.push_back('\v');
			} else if (c == 'f') {
				mods.push_back('\f');
			} else if (c == 'r') {
				mods.push_back('\r');
			} else if (c == '0') {
				mods.push_back('\0');
			} else {
				mods.push_back(c);
			}
			s = mod;
			break;
		case lbe:
			if (c == '{') {
				text.push_back(c);
				s = start;
			} else if (c == '}') {
				co_yield {text, "", ""};
				text.clear();
				s = start;
			} else if (c == '!') {
				s = mod;
			} else {
				spec.push_back(c);
				s = lb;
			}
			break;
		case rbe:
			if (c == '}') {
				text.push_back(c);
				s = start;
			} else {
				parse_error(ec::t_internal, input, &c);
			}
			break;
		case lb:
			if (c == '}') {
				co_yield {text, spec, ""};
				text.clear();
				spec.clear();
				s = start;
			} else if (c == '\\') {
				s = backslashlb;
			} else if (c == '{') {
				parse_error(ec::t_internal, input, &c);
			} else if (c == '!') {
				s = mod;
			} else {
				spec.push_back(c);
			}
			break;
		case mod:
			if (c == '}') {
				co_yield {text, spec, mods};
				text.clear();
				spec.clear();
				mods.clear();
				s = start;
			} else if (c == '\\') {
				s = backslashmod;
			} else if (c == '{') {
				parse_error(ec::t_internal, input, &c);
			} else {
				mods.push_back(c);
			}
		}
	}
	if (s == lb or s == backslash or s == backslashlb or s == rbe) {
		parse_error(ec::t_internal, input, input.end() - 1);
	}
	co_yield {text, std::nullopt, ""};
}

struct parse_result {
	std::string literal;
	std::optional<std::string> name;
	std::variant<std::monostate, flist_t, ilist_t> freq_override;
	std::vector<std::string> args;
	std::string domain;
};

auto refParseSM(std::string refstr) -> asyncpp::generator<parse_result>;
auto refParseRE(std::string refstr) -> asyncpp::generator<parse_result>;

[[nodiscard]] auto fparse(const datafile& domain, const std::string& s)
    -> asyncpp::generator<std::variant<std::string, noderef, wordgen_error>> {
	/*
	   format:
	      string(escape('{}', '\\', '{', '{', '}', '}'))
	      "{" name ["|" args] [(":" flist) | ("!" ilist)] "}"
	   name:
	      string(escape('{}|:!', '\\'))
	   args:
	      simple-string ["|" args]
	   simple-string:
	      string(escape('{}|:!', '\\'))
	   flist:
	      [f-pre] [fsep] double (fsep double)*
	   f-pre:
	      [fsep] ("+" | "*" | "=")
	   fsep:
	      ignore(re('[^0-9.]*[^0-9.+*=]'))
	   ilist:
	      [isep] iref (isep iref)*
	   iref:
	      int [":" double]
	   isep:
	      ignore(re('[^0-9.:]+'))
	*/
	// std::clog<<kblib::quoted(s)<<'\n';
	std::optional<wordgen_error> r;
	try {
		for (const parse_result& val : refParseSM(s)) {
			if (not val.literal.empty()) {
				co_yield val.literal;
			}
			if (val.name) {
				co_yield noderef{domain.lookup_domain(val.domain), *val.name,
				                 val.args, val.freq_override};
			}
		}
	} catch (wordgen_error& e) {
		// co_yield std::move(e);
		// Apparently you can't co_yield from a handler.
		r.emplace(std::move(e));
	}
	if (r) {
		co_yield* r;
	}
}

auto scan_flist(const std::string& input) -> flist_t {
	log_debug("flist: ", input);

	const static thread_local srell::regex fsep{
	    R"__([^\d.+-]+([-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?))__",
	    srell::regex_constants::dotall};
	flist_t ret;

	for (srell::sregex_iterator it{input.begin(), input.end(), fsep}, end{};
	     it != end; ++it) {
		ret.push_back(std::stod((*it)[1]));
	}
	return ret;
#if 0
	std::size_t pos = 0;
	while (pos < input.size()) {
		auto oldpos = pos;
		srell::smatch m;
		if (srell::regex_search(input.begin()+pos, input.end(), m, fsep)) {
			pos += m[0].length();
			// pos = m[0].second - input.begin();
		}
		try {
			ret.push_back(std::stod(input, &pos));
		} catch (const std::invalid_argument& e) {
			log_err("Failed to parse flist: ", kblib::quoted(input), "[", std::to_string(pos), "]");
			throw;
		}
		assert(pos > oldpos);
	}
	return ret;
#endif
}

auto scan_ilist(const std::string& input) -> ilist_t {
	log_debug("ilist: ", input);

	const static thread_local srell::regex isep{
	    R"__([^\d.:+=-]+(\d+)(?::([-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?))?)__",
	    srell::regex_constants::dotall};
	ilist_t ret;
	for (srell::sregex_iterator it{input.begin(), input.end(), isep}, end{};
	     it != end; ++it) {
		ret.emplace_back(std::stoi((*it)[1]), [&]() -> std::optional<double> {
			if ((*it)[2].length() > 0) {
				return std::stod((*it)[2]);
			} else {
				return std::nullopt;
			}
		}());
	}
	return ret;
#if 0
	std::size_t pos = 0;
	while (pos < input.size()) {
		auto opos = pos;
		srell::smatch m;
		if (srell::regex_search(input.begin()+pos, input.end(), m, isep)) {
			pos = m[0].second - input.begin();
		}
		try {
			ret.emplace_back(std::stoi(input, &pos), std::nullopt);
		} catch (const std::invalid_argument& e) {
			log_err("Failed to parse ilist index: ", kblib::quoted(input), "[", std::to_string(pos), "]");
			throw;
		}
		if (input[pos] == ':') {
			++pos;
			try {
				ret.back().second = std::stod(input, &pos);
			}  catch (const std::invalid_argument& e) {
				log_err("Failed to parse ilist freq: ", kblib::quoted(input), "[", std::to_string(pos), "]");
				throw;
			}
		}
		//must make progress
		assert(pos > opos);
	}
	return ret;
#endif
}

// Basically copy Formatter().parse() but treat ! differently:
// Only one of : or ! is allowed to follow the refname, and |
// is used to separate arguments. Also, the subtle points of
// {{ and }} interpretation are surely different.

// The two functions in this file should be equivalent in behavior.

auto refParseSM(std::string refstr) -> asyncpp::generator<parse_result> {

	log_debug(refstr);

	enum class s {
		literal,
		esc,
		lb,
		rb,
		name,
		flist,
		ilist,
		args,
	};

	std::string lit;
	std::string domain;
	std::string name;
	std::string flist;
	std::string ilist;
	enum class fo { none, flist, ilist } f = fo::none;
	std::vector<std::string> args;

	s state = s::literal;
	s ostate = s::literal;

	auto prep_fo = [&]() -> std::variant<std::monostate, flist_t, ilist_t> {
		// std::clog<<kblib::etoi(f)<<'\n';
		assert(kblib::etoi(f) >= 0 and kblib::etoi(f) <= kblib::etoi(fo::ilist));
		switch (f) {
		case fo::none:
			return std::monostate{};
		case fo::flist:
			return scan_flist(flist);
		case fo::ilist:
			return scan_ilist(ilist);
		default:
			assert(false);
		}
	};

	auto push_char = [&](char c, s state) {
		if (state == s::literal) {
			lit += c;
		} else if (state == s::name) {
			name += c;
		} else if (state == s::flist) {
			flist += c;
		} else if (state == s::ilist) {
			ilist += c;
		} else if (state == s::args) {
			args.back() += c;
		}
	};

	for (char c : refstr) {
		if (state == s::literal) {
			if (c == '{') {
				ostate = state;
				state = s::lb;
			} else if (c == '}') {
				ostate = state;
				state = s::rb;
			} else if (c == '\\') {
				ostate = state;
				state = s::esc;
			} else {
				lit += c;
			}
		} else if (state == s::lb) {
			if (c == '{') {
				//				if (ostate == s::literal) {
				//					lit += c;
				//				} else if (ostate == s::name) {
				//					name += c;
				//				} else if (ostate == s::flist) {
				//					flist += c;
				//				} else if (ostate == s::ilist) {
				//					ilist += c;
				//				} else if (ostate == s::args) {
				//					args.back() += c;
				//				}
				push_char(c, ostate);
				state = ostate;
			} else if (c == '}') {
				if (ostate == s::literal) {
					state = s::rb;
					ostate = s::name;
				} else {
					throw wordgen_error(
					    ec::lb_in_name, kblib::quoted(refstr), "\n",
					    "state=" + std::to_string(kblib::etoi(state)) + " ostate="
					        + std::to_string(kblib::etoi(ostate)) + '\n');
				}
			} else if (c == ':') {
				state = s::flist;
				f = fo::flist;
				flist.clear();
				flist.push_back(c);
			} else if (c == '!') {
				state = s::ilist;
				f = fo::ilist;
				ilist.clear();
				ilist.push_back(c);
			} else if (c == '|') {
				state = s::args;
				args.emplace_back();
			} else if (c == '\\') {
				ostate = state;
				state = s::esc;
			} else {
				state = s::name;
				name = c;
			}
		} else if (state == s::rb) {
			if (c == '}') {
				//				if (ostate == s::literal) {
				//					lit += c;
				//				} else if (ostate == s::name) {
				//					name += c;
				//				} else if (ostate == s::flist) {
				//					flist += c;
				//				} else if (ostate == s::ilist) {
				//					ilist += c;
				//				} else if (ostate == s::args) {
				//					args.back() += c;
				//				}
				push_char(c, ostate);
				state = ostate;
			} else {
				if (ostate == s::literal) {
					throw wordgen_error(
					    ec::extra_rb, kblib::quoted(refstr), "\n",
					    "state=" + std::to_string(kblib::etoi(state)) + " ostate="
					        + std::to_string(kblib::etoi(ostate)) + '\n');
				} else if (c == '{') {
					co_yield parse_result{lit, name, prep_fo(), args, domain};
					state = s::lb;
					f = fo::none;
					lit.clear();
					name.clear();
					flist.clear();
					ilist.clear();
					args.clear();
				} else {
					co_yield parse_result{lit, name, prep_fo(), args, domain};
					state = s::literal;
					f = fo::none;
					lit = c;
					name.clear();
					flist.clear();
					ilist.clear();
					args.clear();
				}
			}
		} else if (state == s::name) {
			if (c == ':') {
				state = s::flist;
				f = fo::flist;
				flist.clear();
				flist.push_back(c);
			} else if (c == '!') {
				state = s::ilist;
				f = fo::ilist;
				ilist.clear();
				ilist.push_back(c);
			} else if (c == '|') {
				state = s::args;
				args.emplace_back();
			} else if (c == '\\') {
				ostate = state;
				state = s::esc;
			} else if (c == '}') {
				ostate = state;
				state = s::rb;
			} else if (c == '{') {
				ostate = state;
				state = s::lb;
			} else {
				name += c;
			}
		} else if (state == s::flist) {
			if (c == '}') {
				ostate = state;
				state = s::rb;
			} else if (c == '{') {
				ostate = state;
				state = s::lb;
			} else {
				flist += c;
			}
		} else if (state == s::ilist) {
			if (c == '}') {
				ostate = state;
				state = s::rb;
			} else if (c == '{') {
				ostate = state;
				state = s::lb;
			} else {
				ilist += c;
			}
		} else if (state == s::args) {
			if (c == ':') {
				state = s::flist;
				flist.clear();
			} else if (c == '!') {
				state = s::ilist;
				ilist.clear();
			} else if (c == '|') {
				args.emplace_back();
			} else if (c == '}') {
				ostate = state;
				state = s::rb;
			} else if (c == '{') {
				ostate = state;
				state = s::lb;
			} else if (c == '\\') {
				ostate = state;
				state = s::esc;
			} else {
				args.back() += c;
			}
		} else if (state == s::esc) {
			if (R"(\"':!|{})"sv.find(c) != std::string_view::npos) {
				//				if (ostate == s::literal) {
				//					lit += c;
				//				} else if (ostate == s::name) {
				//					name += c;
				//				} else if (ostate == s::flist) {
				//					flist += c;
				//				} else if (ostate == s::ilist) {
				//					ilist += c;
				//				} else if (ostate == s::args) {
				//					args.back() += c;
				//				}
				push_char(c, ostate);
				state = ostate;
			} else {
				throw wordgen_error(ec::illegal_escape, "\\"s + c, "\n",
				                    "in: " + kblib::quoted(refstr) + '\n');
			}
		}
	}
	if (state == s::literal) {
		co_yield parse_result{lit, std::nullopt, prep_fo(), args, domain};
	} else if (state == s::rb and ostate != s::literal) {
		co_yield parse_result{lit, name, prep_fo(), args, domain};
	} else if (state == s::rb and ostate == s::literal) {
		throw wordgen_error(ec::extra_rb, kblib::quoted(refstr), "\n",
		                    "state=" + std::to_string(kblib::etoi(state))
		                        + " ostate=" + std::to_string(kblib::etoi(ostate))
		                        + '\n');
	} else if (state == s::esc and ostate == s::literal) {
		// If a string ends with \, just treat it as a normal character.
		co_yield parse_result{lit + '\\', std::nullopt, prep_fo(), args, domain};
	} else {
		throw wordgen_error(ec::unterminated_noderef, kblib::quoted(refstr), "\n",
		                    "state=" + std::to_string(kblib::etoi(state))
		                        + " ostate=" + std::to_string(kblib::etoi(ostate))
		                        + '\n');
	}
}

// TODO(killerbee): error handling
// bug: includes | in arguments after expansion
auto refParseRE(std::string refstr) -> asyncpp::generator<parse_result> {
	const static thread_local srell::regex ref{
	    R"__((?=.)((?:[^{}]|\\[{}]|\{\{|\}\})*)(?:\{([^%{}|:!]%)?((?:[^{}|:!]|\\[{}|:!])*)((?:\|(?:[^{}|:!]|\\[{}|:!])*)*)((:[^{}|\d.+*=]*(?:\+|\*|=)?[^{}|\d.-]*[\d.-]+(?:[^{}|\d.-]+[\d.-]+)*[^{}|\d.-]*)|(![^{}|\d.:+=-]*\d+(?::[\d.-]+)?(?:[^{}|\d.:+=-]+\d+(?::[\d.-]+))*[^{}|\d.:+=-]*))?(\}))?)__",
	    srell::regex_constants::dotall};
	const static thread_local srell::regex arg{
	    R"__((?<=\|)(?:[^{}|:!]|\\[{}|:!])*)__", srell::regex_constants::dotall};

	for (srell::sregex_iterator it{refstr.begin(), refstr.end(), ref}, end{};
	     it != end; ++it) {
		auto&& match = *it;
		co_yield parse_result{
		    match[1],
		    match[8].length() ? std::optional<std::string>{match[3]}
		                      : std::nullopt,
		    [&]() -> std::variant<std::monostate, flist_t, ilist_t> {
			    if (match[6].length()) {
				    return scan_flist(match[6].str().substr(1));
			    } else if (match[7].length()) {
				    return scan_ilist(match[7].str().substr(1));
			    } else {
				    return std::monostate{};
			    }
		    }(),
		    kblib::build_copy<std::vector<std::string>>(
		        srell::sregex_token_iterator{match[4].first, match[4].second,
		                                     arg},
		        srell::sregex_token_iterator{}),
		    match[2]};
	}
}

auto test_parser(std::vector<std::pair<std::string, parse_result>> tests)
    -> bool {
	throw tests;
}

auto datafile::ch_db() -> std::unordered_map<channelID, std::string>& {
	static std::unordered_map<channelID, std::string> db;
	return db;
}