#include <stdexcept>
#include <istream>
#include <iomanip>
// #include <sstream>
#include <stack>
#include <deque>

#include <unicode/schriter.h>
#include <unicode/uchar.h>

#include "FRC.h"

#include "grammar.h"
#include "word.h"

extern randutils::random_generator<pcg64> randgen;

using namespace std::string_literals;

Grammar::channelID Grammar::chI(const icu::UnicodeString& channel)
{
	for (size_t i = 0; i != channelNames.size(); ++i) {
		if (channelNames[i] == channel) {
			return i;
		}
	}
	channelNames.push_back(channel);
	channelDescs.push_back(channel);
	// std::clog<<__LINE__<<" "<<"Added channel "<<std::hex<<channel<<" as "<<channelNames.size()-1<<std::endl;
	return channelNames.size()-1;
}

Grammar::channelID Grammar::chI(const icu::UnicodeString& channel) const
{
	for (size_t i = 0; i != channelNames.size(); ++i) {
		if (channelNames[i] == channel) {
			return i;
		}
	}
	throw std::out_of_range("Could not find channel "s + channel);
}

const icu::UnicodeString& Grammar::chN(Grammar::channelID channel) const
{
	return channelNames.at(channel);
}

Grammar::channelID Grammar::nameChannel(const icu::UnicodeString& channel, const icu::UnicodeString& desc)
{
	for (size_t i = 0; i != channelNames.size(); ++i) {
		if (channelNames[i] == channel) {
			channelDescs[i] = desc;
			return i;
		}
	}
	channelNames.push_back(channel);
	channelDescs.push_back(desc);
	// std::clog<<__LINE__<<" "<<"Added channel "<<channel<<':'<<desc<<" as "<<channelNames.size()-1<<std::endl;
	return channelNames.size()-1;
}

Grammar::channelID Grammar::nameChannel(Grammar::channelID channel, const icu::UnicodeString& desc)
{
	channelDescs.at(channel) = desc;
	return channel;
}

std::string Grammar::printAllChannelNames() const
{
	std::string ret;
	for (channelID i = 0; i != channelNames.size(); ++i) {
		ret += toStr(i) + ": " + toUTF8(channelNames[i]) + ": " + toUTF8(channelNames[i]) + "\n";
	}
	ret.pop_back();
	return ret;
}

Word Grammar::genWord(Grammar::Node const * node, complexity c) const
{
	return node->selectFrom(nullptr, c);
}

namespace a_call {
	using ArgsType = Grammar::ArgsType;
	static icu::UnicodeString null(const ArgsType& a)
	{
		return u"";
	}
	static icu::UnicodeString escapePipe(const icu::UnicodeString& val);
	static icu::UnicodeString condense(int s, const ArgsType& a);
	class param {
	private:
		class rest_t {};
	public:
		param(const icu::UnicodeString& val)
		{
			UErrorCode status = U_ZERO_ERROR;
			icu::RegexMatcher ref{u"#(.+)", val, 0, status};
			icu::RegexMatcher fun{u"(.+)\\((.+)\\)", val, 0, status};
			#if _DEBUG
			// if (U_FAILURE(status))
				// throw std::make_pair(status, __LINE__);
			#endif
			if (ref.matches(status)) {
				p = pFromStr(int, val.tempSubString(1));
			} else if (fun.matches(status)) {
				p = Grammar::ArgRef(val);
			} else if (val == u"...") {
				p = rest_t();
			} else {
				p = val;
			}
			if (U_FAILURE(status))
				throw std::make_pair(status, __LINE__);
		}
		bool isRest() const
			{return boost::get<rest_t>(&p);}
		struct p_visitor : public boost::static_visitor<icu::UnicodeString> {
			int s;
			PtrToConst<ArgsType> a;
			p_visitor(int _s, const ArgsType& _a) : s(_s), a(&_a) {;}
			icu::UnicodeString operator()(const icu::UnicodeString& v) const {
				return v;
			}
			icu::UnicodeString operator()(int v) const {
				return a->at(v);
			}
			icu::UnicodeString operator()([[maybe_unused]] rest_t v) const {
				return condense(s, *a);
			}
			icu::UnicodeString operator()(const Grammar::ArgRef& v) const {
				return v(*a);
			}
		};
		icu::UnicodeString getVal(int s, const ArgsType& a) const
			{
#ifdef _DEBUG
				std::clog<<__LINE__<<" "<<"argC = "<<s<<"; size = "<<a.size()<<std::endl;
#endif
				return boost::apply_visitor(p_visitor(s, a), p);}
		static std::pair<ArgsType::const_iterator,ArgsType::const_iterator>
			getRest(int s, const ArgsType& a)
			{
#ifdef _DEBUG
				// std::clog<<__LINE__<<" "<<"argC = "<<s<<std::endl;
#endif
				return std::make_pair(a.begin() + s, a.end());}
	private:
		boost::variant<icu::UnicodeString, int, rest_t, Grammar::ArgRef> p;
	};
	static std::vector<param> extractParams(const icu::UnicodeString& val, int argC, bool argV)
	{
		std::vector<param> ret;
		icu::UnicodeString tmp;
		StringCharacterIterator iter{val};
		bool inEscape = false;
		for (iter.setToStart(); iter.hasNext(); ) {
			UChar32 c=iter.next32PostInc();
			if (!inEscape) {
				if (c == U'\\') {
					inEscape = true;
				} else if (c == U'|') {
					ret.emplace_back(tmp);
					tmp.remove();
				} else {
					tmp += c;
				}
			} else {
				tmp += c;
				inEscape = false;
			}
		}
		ret.emplace_back(tmp);
		return ret;
	}
	static icu::UnicodeString simple(int v, const ArgsType& a)
		{return a.at(v);}
	static constexpr double& add(double& l, double r)
		{return l += r;}
	static constexpr double& sub(double& l, double r)
		{return l -= r;}
	static constexpr double& mul(double& l, double r)
		{return l *= r;}
	static constexpr double& div(double& l, double r)
		{return l /= r;}
	static icu::UnicodeString c_mather(const std::vector<param>& s,
		int c, double accum, double&(*op)(double&,double), const ArgsType& a)
		{
#ifdef _DEBUG
			std::clog<<__LINE__<<" "<<"c="<<c<<" i="<<accum<<std::endl;
#endif
			for (const auto& i : s) {
			if (i.isRest()) {
				auto j = i.getRest(c, a);
				accum = std::accumulate(
					j.first, j.second,
					accum,
					[op](double a, icu::UnicodeString b)
						{return op(a, pFromStr(double, b));}
				);
			} else 
				op(accum, pFromStr(double, i.getVal(c, a)));
		}
		return fromUTF8(toStr(accum));}
	static icu::UnicodeString len(int s, const ArgsType& a)
		{return fromUTF8(toStr(a.size()-s));}
	static icu::UnicodeString len1([[maybe_unused]] const ArgsType& a)
		{return u"1";}
	static icu::UnicodeString escapePipe(const icu::UnicodeString& val)
	{
		icu::UnicodeString ret;
		StringCharacterIterator iter{val};
		for (iter.setToStart(); iter.hasNext(); ) {
			UChar32 c=iter.next32PostInc();
			if (c == U'\\') {
				ret += u"\\\\";
			} else if (c == U'|') {
				ret += u"\\|";
			} else {
				ret += c;
			}
		}
		return ret;
	}
	static icu::UnicodeString condense(int s, const ArgsType& a)
	{
		icu::UnicodeString ret;
		if (a.size() > s) {
			ret = a[s];
			for (int i = s + 1; i != a.size(); ++i) {
				ret += u'|';
				ret += escapePipe(a[i]);
			}
		}
		return ret;
	}
	//Returns the substituted param at the position specified by the first param
	static icu::UnicodeString select(const std::vector<param>& p, int s, const ArgsType& a)
		{return p.at(1+fromStr<unsigned>(p.at(0).getVal(s, a))).getVal(s, a);}
	//Substitutes arguments into param list, then finds the position of the param equal to the first param
	static icu::UnicodeString which(const std::vector<param>& p, int s, const ArgsType& a)
	{
		//Fill pvals with the substituted values of the parameter list
		std::vector<icu::UnicodeString> pvals;
		std::transform(
			p.begin(), p.end(),
			std::back_inserter(pvals),
			[&a, &s](const param& p){return p.getVal(s, a);}
		);
		return fromUTF8(toStr(
			std::find(pvals.begin()+1, pvals.end(), pvals.at(0))
			- (pvals.begin()+1)
		));
	}
	static icu::UnicodeString lesser(const std::vector<param>& p, int s, const ArgsType& a)
		{return (p[0].getVal(s, a) < p[1].getVal(s, a)) ? p[2].getVal(s, a) : p[3].getVal(s, a);}
	static icu::UnicodeString greater(const std::vector<param>& p, int s, const ArgsType& a)
		{return (p[0].getVal(s, a) > p[1].getVal(s, a)) ? p[2].getVal(s, a) : p[3].getVal(s, a);}
	static icu::UnicodeString equal(const std::vector<param>& p, int s, const ArgsType& a)
		{return (p[0].getVal(s, a) == p[1].getVal(s, a)) ? p[2].getVal(s, a) : p[3].getVal(s, a);}
	
	static icu::UnicodeString concat(const std::vector<param>& p, int s, const ArgsType& a)
	{
		return std::accumulate(
			p.begin(), p.end(),
			icu::UnicodeString{},
			[&a, &s](const icu::UnicodeString& acc, const param& p)
				{return acc+p.getVal(s, a);}
		);
	}
	
	static icu::UnicodeString repeat(const std::vector<param>& p, int s, const ArgsType& a)
	{
		return ::repeat(p[0].getVal(s, a), pFromStr(int, p[1].getVal(s, a)));
	}
	
	template<typename val>
	inline val pop(std::stack<val>& s)
		{val ret = s.top(); s.pop(); return ret;}
	
	static icu::UnicodeString math(const std::vector<param>& p, int s, const ArgsType& a)
	{
		//Fill pvals with the substituted values of the parameter list (flattening ...)
		std::vector<icu::UnicodeString> pvals;
		std::for_each(
			p.begin(), p.end(),
			[&a, &s, &pvals](const param& p) {
				if (p.isRest()) {
					std::transform(
						std::next(a.begin(), s), a.end(),
						std::back_inserter(pvals),
						[&a, &s](const icu::UnicodeString& s){return s;}
					);
				} else {
					pvals.push_back(p.getVal(s, a));
				}
			}
		);
		const std::array<char32_t,5> ops = {u'+',u'-',u'*',u'/',u'.'};
		std::stack<char> opstack;
		std::stack<icu::UnicodeString> valstack;
		bool pending = false;
		
		for (auto& p : pvals) {
			decltype(&ops[0]) op;
			if (p.countChar32() == 1
				&& (op = std::find(ops.begin(), ops.end(),
					p.char32At(0))) != ops.end()) {
				opstack.push(*op);
			} else {
				if (pending) {
					while(valstack.size()) {
						auto v1 = pop(valstack);
						auto op = pop(opstack);
#define APPLYOP(OP) \
p=fromUTF8(toStr(pFromStr(double, v1) OP pFromStr(double, p)))
						if (op == '+') APPLYOP(+);
						else if (op == '-') APPLYOP(-);
						else if (op == '*') APPLYOP(*);
						else if (op == '/') APPLYOP(/);
						else if (op == '.') p=v1+p;
					}
				}
				valstack.push(p);
				pending = true;
			}
		}
		return pop(valstack);
	}
};


Grammar::ArgRef::ArgRef(const icu::UnicodeString& val, int argC, bool argV)
{
	if (!val.length()) {
		call = a_call::null;
		return;
	}
	UErrorCode status = U_ZERO_ERROR;
	icu::RegexMatcher num{u"(\\d+)|(\\.\\.\\.)", val, 0, status};
	icu::RegexMatcher fun{u"(.+)\\((.+)\\)", val, 0, status};
	#if _DEBUG
	if (U_FAILURE(status))
		throw std::make_pair(status, __LINE__);
	#endif
	status = U_ZERO_ERROR;
	
	bool isSimple = num.matches(status);
	if (U_FAILURE(status))
		throw std::make_pair(status, __LINE__);
	if (isSimple) {
		if (val == u"...") {
			call = std::bind(a_call::condense, argC, std::placeholders::_1);
		} else {
			call = std::bind(a_call::simple, fromStr<int>(val), std::placeholders::_1);
		}
	} else {
		bool isFunc = fun.matches(0, status);
		if (U_FAILURE(status)) {
			std::cerr<<val<<std::endl;
			throw std::make_pair(status, __LINE__);
		}
		if (isFunc) {
			icu::UnicodeString fn{fun.group(1, status)}, ps{fun.group(2, status)};
#ifdef _DEBUG
			// std::clog<<__LINE__<<" "<<"fn="<<fn<<" c="<<argC<<std::endl;
#endif
			if (fn == u"len") {
				if (ps == u"...") {
					call = std::bind(a_call::len, argC, std::placeholders::_1);
				} else {
					call = a_call::len1;
				}
			} else if (fn == u"+") {
				call = std::bind(
					a_call::c_mather,
					a_call::extractParams(ps, argC, argV),
					argC, 0.0, a_call::add, std::placeholders::_1
				);
			} else if (fn == u"-") {
				call = std::bind(
					a_call::c_mather,
					a_call::extractParams(ps, argC, argV),
					argC, 0.0, a_call::sub, std::placeholders::_1
				);
			} else if (fn == u"*") {
				call = std::bind(
					a_call::c_mather,
					a_call::extractParams(ps, argC, argV),
					argC, 1.0, a_call::mul, std::placeholders::_1
				);
			} else if (fn == u"/") {
				call = std::bind(
					a_call::c_mather,
					a_call::extractParams(ps, argC, argV),
					argC, 1.0, a_call::div, std::placeholders::_1
				);
			} else if (fn == u"?") {
				call = std::bind(
					a_call::select,
					a_call::extractParams(ps, argC, argV),
					argC, std::placeholders::_1
				);
			} else if (fn == u"which") {
				call = std::bind(
					a_call::which,
					a_call::extractParams(ps, argC, argV),
					argC, std::placeholders::_1
				);
			} else if (fn == u"lt") {
				auto p = a_call::extractParams(ps, argC, argV);
				if (p.size() != 4) {
					throw std::runtime_error(
						"in: <"
						+toUTF8(val)
						+">: lt requires exactly 4 parameters (got "
						+toStr(p.size())
						+")"
					);
				}
				call = std::bind(a_call::lesser, p, argC, std::placeholders::_1);
			} else if (fn == u"gt") {
				auto p = a_call::extractParams(ps, argC, argV);
				if (p.size() != 4) {
					throw std::runtime_error(
						"in: <"
						+toUTF8(val)
						+">: gt requires exactly 4 parameters (got "
						+toStr(p.size())
						+")"
					);
				}
				call = std::bind(a_call::greater, p, argC, std::placeholders::_1);
			} else if (fn == u"=" || fn == u"eq") {
				auto p = a_call::extractParams(ps, argC, argV);
				if (p.size() != 4) {
					throw std::runtime_error(
						"in: <"
						+toUTF8(val)
						+">: "+toUTF8(fn)
						+" requires exactly 4 parameters (got "
						+toStr(p.size())
						+")"
					);
				}
				call = std::bind(a_call::equal, p, argC, std::placeholders::_1);
			} else if (fn == u".") {
				call = std::bind(
					a_call::concat,
					a_call::extractParams(ps, argC, argV),
					argC, std::placeholders::_1
				);
			} else if (fn == u".*") {
				call = std::bind(
					a_call::repeat,
					a_call::extractParams(ps, argC, argV),
					argC, std::placeholders::_1
				);
			} else if (fn == u"math" || fn == u"calc") {
				call = std::bind(
					a_call::math,
					a_call::extractParams(ps, argC, argV),
					argC, std::placeholders::_1
				);
			} else {
					throw std::runtime_error(
						"in: <"
						+toUTF8(val)
						+">: no match for function '"+toUTF8(fn)
						+"'"
					);
			}
		} else {
			throw std::runtime_error(
				"<"
				+toUTF8(val)
				+"> is not a valid argref"
			);
		}
	}
}

Grammar::ArgStr Grammar::parseArgs(const icu::UnicodeString& val, int argC, bool argV)
{
	Grammar::ArgStr ret;
	std::pair<icu::UnicodeString,std::tuple<icu::UnicodeString, int, bool>> tmp
		= std::make_pair(u"", std::make_tuple(u"", argC, argV));
	StringCharacterIterator iter{val};
	enum states {
		lit,
		litesc,
		ref,
		refesc
	} state = lit;
	for (iter.setToStart(); iter.hasNext(); ) {
		UChar32 c=iter.next32PostInc();
		switch (state) {
		case lit:
			if (c == U'<') {
				state = ref;
			} else if (c == U'\\') {
				state = litesc;
			} else {
				tmp.first += c;
			}
			break;
		case litesc:
			tmp.first += c;
			state = lit;
			break;
		case ref:
			if (c == U'>') {
				state = lit;
				ret.emplace_back(tmp);
				tmp = std::make_pair(u"", std::make_tuple(u"", argC, argV));
			} else if (c == U'\\') {
				state = refesc;
			} else {
				std::get<0>(tmp.second) += c;
			}
			break;
		case refesc:
			std::get<0>(tmp.second) += c;
			state = ref;
		}
	}
	switch (state) {
	case lit:
		if (tmp.first.length())
			ret.emplace_back(tmp);
		return ret;
	case litesc:
		throw std::runtime_error(
			"Unterminated escape reference in '"
			+ toUTF8(val)
			+ "' at end (index "
			+ toStr(val.countChar32(0,iter.getIndex()))
			+ ")");
	case ref:
		throw std::runtime_error(
			"Unterminated argument reference in '"
			+ toUTF8(val)
			+ "' at end (index "
			+ toStr(val.countChar32(0,iter.getIndex()))
			+ ")");
	case refesc:
		throw std::runtime_error(
			"Unterminated escape reference in '"
			+ toUTF8(val)
			+ "' at end (index "
			+ toStr(val.countChar32(0,iter.getIndex()))
			+ ")");
	}
}

icu::UnicodeString Grammar::applyArgs(const Grammar::ArgStr& val, const Grammar::ArgsType& args)
{
	icu::UnicodeString ret;
	if (val.empty()) {
		return ret;
	} else {
		for (auto& i : val) {
			ret.append(i.first);
			ret.append(i.second(args));
		}
		return ret;
	}
}

Grammar::ParsedString::ParsedString()
: argCount(0), varArgs(false)
{
	return;
}

Grammar::ParsedString::ParsedString(const icu::UnicodeString& str, int argC, bool argV)
: argCount(argC), varArgs(argV)
{
#ifdef _DEBUG
	// std::clog<<__LINE__<<" "<<"Parsing "<<str<<std::endl;
#endif
	StringCharacterIterator iter{str};
	UChar32 c;
	enum {
		lit, litlb, litrb, litesc,
		node, nodelb, noderb, nodeesc
	} state{lit};
	icu::UnicodeString litStr, refStr;
	for (iter.setToStart(); iter.hasNext(); ) {
		c=iter.next32PostInc();
		switch (state) {
		case lit:
			switch (c) {
				case U'{':
					state = litlb;
					break;
				case U'}':
					state = litrb;
					break;
				case U'\\':
					state = litesc;
					break;
				default:
					litStr += c;
			}
			break;
		case litlb:
			switch (c) {
				case U'{':
					litStr += c;
					state = lit;
					break;
				case U'}':
#ifdef _DEBUG
					// std::clog<<__LINE__<<" "<<litStr<<"{"<<refStr<<"}"<<std::endl;
#endif
					elements.push_back(std::make_pair(litStr, NodeRef{u"", argC, argV}));
					litStr = u"";
					state = lit;
					break;
				case U'\\':
					state = nodeesc;
					break;
				default:
					state = node;
					refStr += c;
			}
			break;
		case litrb:
			switch (c) {
				case U'}':
					litStr += c;
					state = lit;
					break;
				default:
					throw std::runtime_error(
						"Single '}' encountered in '"
						+ toUTF8(str)
						+ "' at index "
						+ toStr(str.countChar32(0,iter.getIndex())));
			}
			break;
		case litesc:
			switch (c) {
				case U'<':
				case U'>':
					refStr += u'\\';
					[[fallthrough]];
				case U':':
				case U'!':
				case U'|':
				case U'{':
				case U'}':
				case U'\\':
					litStr += c;
					state = lit;
					break;
				default:
					throw std::runtime_error(
						"Illegal escape in '"
						+ toUTF8(str)
						+ "' at index "
						+ toStr(str.countChar32(0,iter.getIndex())));
			}
			break;
		case node:
			switch (c) {
				case U'{':
					state = nodelb;
					break;
				case U'}':
					state = noderb;
					break;
				case U'\\':
					state = nodeesc;
					break;
				default:
					refStr += c;
			}
			break;
		case nodelb:
			switch (c) {
				case U'{':
					refStr += c;
					state = node;
					break;
				default:
					throw std::runtime_error(
						"Single '{' encountered in '"
						+ toUTF8(str)
						+ "' at index "
						+ toStr(str.countChar32(0,iter.getIndex())));
			}
			break;
		case noderb:
			switch (c) {
				case U'}':
					refStr += c;
					state = node;
					break;
				case U'{':
#ifdef _DEBUG
					// std::clog<<__LINE__<<" "<<litStr<<"{"<<refStr<<"}"<<std::endl;
#endif
					elements.push_back(std::make_pair(litStr, NodeRef{refStr, argC, argV}));
					litStr = u"";
					refStr = u"";
					state = node;
					break;
				case U'\\':
#ifdef _DEBUG
					// std::clog<<__LINE__<<" "<<litStr<<"{"<<refStr<<"}"<<std::endl;
#endif
					elements.push_back(std::make_pair(litStr, NodeRef{refStr, argC, argV}));
					litStr = u"";
					refStr = u"";
					state = litesc;
					break;
				default:
#ifdef _DEBUG
					// std::clog<<__LINE__<<" "<<litStr<<"{"<<refStr<<"}"<<std::endl;
#endif
					elements.push_back(std::make_pair(litStr, NodeRef{refStr, argC, argV}));
					litStr = c;
					refStr = u"";
					state = lit;
			}
			break;
		case nodeesc:
			switch (c) {
				case U':':
				case U'!':
				case U'|':
				case U'\\':
				case U'<':
				case U'>':
					refStr += u'\\';
					[[fallthrough]];
				case U'{':
				case U'}':
					refStr += c;
					state = node;
					break;
				default:
					throw std::runtime_error(
						"Illegal escape in '"
						+ toUTF8(str)
						+ "' at index "
						+ toStr(str.countChar32(0,iter.getIndex())));
			}
			break;
		}
	}
	switch (state) {
	case lit:
#ifdef _DEBUG
		// std::clog<<__LINE__<<" "<<litStr<<"\\0"<<std::endl;
#endif
		elements.push_back(std::make_pair(litStr, NodeRef{}));
		break;
	case noderb:
#ifdef _DEBUG
		// std::clog<<__LINE__<<" "<<litStr<<"{"<<refStr<<"}\\0"<<std::endl;
#endif
		elements.push_back(std::make_pair(litStr, NodeRef{refStr, argC, argV}));
		break;
	case node:
	case litlb:
	case nodelb:
		throw std::runtime_error(
			"Unterminated reference in '"
			+ toUTF8(str)
			+ "' at end (index "
			+ toStr(str.countChar32(0,iter.getIndex()))
			+ ")");
	case litrb:
		throw std::runtime_error(
			"Single '}' encountered in '"
			+ toUTF8(str)
			+ "' at end (index "
			+ toStr(str.countChar32(0,iter.getIndex()))
			+ ")");
	case litesc:
	case nodeesc:
		throw std::runtime_error(
			"Illegal escape in '"
			+ toUTF8(str)
			+ "' at end (index "
			+ toStr(str.countChar32(0,iter.getIndex()))
			+ ")");
	}
}

Grammar::ParsedString Grammar::ParsedString::sub(const Grammar::ArgsType& args, int argC, bool argV) const
{
	Grammar::ParsedString out;
	for (auto& p : elements) {
		Grammar::ParsedString tmp(applyArgs(parseArgs(p.first, argC, argV), args));
		std::move(tmp.begin(), tmp.end(), std::back_inserter(out.elements));
		auto tmp2 = p.second.sub(args);
		if (!out.elements.back().second) {
			out.elements.back().second = tmp2;
		} else {
			out.elements.emplace_back(u"", std::move(tmp2));
		}
	}
	return out;
}

static std::vector<std::pair<int, double>> _parseIlist(const icu::UnicodeString& in);

static Grammar::ArgsType _parseArgs(const icu::UnicodeString& in);

Grammar::ParsedString::NodeRef::NodeRef(const icu::UnicodeString& val, int argC, bool argV)
: isRef(true), argCount(argC), varArgs(argV)
{
	// "ref"; "ref\\:"; "ref:1 1"; "ref!0:1 2:2"; "ref|1";
	// "ref|1:1 1"; "ref|1!0:1 2:2";
	StringCharacterIterator iter{val};
	icu::UnicodeString t_name, t_flist, t_ilist, t_args;
	UChar32 c;
	enum class states {
		name,
		args,
		flist,
		ilist,
	} state = states::name;
	bool inEscape = false;
	bool inaref = false;
	for (iter.setToStart(); iter.hasNext(); ) {
		c=iter.next32PostInc();
		if (!inEscape) {
			if (inaref) {
				switch (state) {
				case states::name:
					switch (c) {
					case U'\\':
						inEscape = true;
						break;
					case U'>':
						inaref = false;
						[[fallthrough]];
					default:
						t_name += c;
					}
					break;
				case states::args:
					switch (c) {
					case U'\\':
						inEscape = true;
						break;
					case U'>':
						inaref = false;
						[[fallthrough]];
					default:
						t_args += c;
					}
					break;
				case states::flist:
					if (c == U'\\') {
						inEscape = true;
					} else if (c == U'>') {
						inaref = false;
						t_flist += c;
					} else {
						t_flist += c;
					}
					break;
				case states::ilist:
					if (c == U'\\') {
						inEscape = true;
					} else if (c == U'>') {
						inaref = false;
						t_ilist += c;
					} else {
						t_ilist += c;
					}
				}
			} else {
				switch (state) {
				case states::name:
					switch (c) {
					case U'\\':
						inEscape = true;
						break;
					case U':':
						state = states::flist;
						break;
					case U'!':
						state = states::ilist;
						break;
					case U'|':
						state = states::args;
						break;
					case U'<':
						inaref = true;
						[[fallthrough]];
					default:
						t_name += c;
					}
					break;
				case states::args:
					switch (c) {
					case U'\\':
						inEscape = true;
						break;
					case U':':
						state = states::flist;
						break;
					case U'!':
						state = states::ilist;
						break;
					case U'<':
						inaref = true;
						[[fallthrough]];
					default:
						t_args += c;
					}
					break;
				case states::flist:
					if (c == U'\\') {
						inEscape = true;
					} else if (c == U'<') {
						inaref = true;
						t_flist += c;
					} else {
						t_flist += c;
					}
					break;
				case states::ilist:
					if (c == U'\\') {
						inEscape = true;
					} else if (c == U'<') {
						inaref = true;
						t_ilist += c;
					} else {
						t_ilist += c;
					}
				}
			}
		} else {
			switch (state) {
			case states::name:
				if (c == U'<') {
					t_name += u"\\<";
				} else if (c == U'>') {
					t_name += u"\\>";
				} else {
					t_name += c;
				}
				break;
			case states::args:
				if (c == U'|') {
					t_args += u"\\|";
				} else if (c == U'<') {
					t_args += u"\\<";
				} else if (c == U'>') {
					t_args += u"\\>";
				} else {
					t_args += c;
				}
				break;
			case states::flist:
				if (c == U'<') {
					t_flist += u"\\<";
				} else if (c == U'>') {
					t_flist += u"\\>";
				} else {
					t_flist += c;
				}
				break;
			case states::ilist:
				if (c == U'<') {
					t_ilist += u"\\<";
				} else if (c == U'>') {
					t_ilist += u"\\>";
				} else {
					t_ilist += c;
				}
			}
			inEscape = false;
		}
	}
	if (argCount) {
		name = parseArgs(t_name, argC, argV);
		flist = parseArgs(t_flist, argC, argV);
		ilist = parseArgs(t_ilist, argC, argV);
		args = parseArgs(t_args, argC, argV);
	} else {
		name = t_name;
		icu::UnicodeString fv;
		flist = listToVec<double>(toUTF8(t_flist));
		ilist = _parseIlist(t_ilist);
		args = _parseArgs(t_args);
	}
}

static std::vector<icu::UnicodeString> splitWords(const icu::UnicodeString& str)
{
	std::vector<icu::UnicodeString> ret;
	icu::UnicodeString tmp;
	StringCharacterIterator iter{str};
	for (iter.setToStart(); iter.hasNext(); ) {
		UChar32 c=iter.next32PostInc();
		if (u_isspace(c)) {
			if (tmp.length()) {
				ret.push_back(std::move(tmp));
				tmp = u"";
			}
		} else {
			tmp += c;
		}
	}
	if (tmp.length()) {
		ret.push_back(std::move(tmp));
	}
	return ret;
}

static std::pair<int, double> parseIndex(const icu::UnicodeString& str)
{
	// "0" "0:1.0";
	auto sp = str.indexOf(u':');
	if (sp == -1) {
		return std::make_pair(
			pFromStr(int, str),
			std::numeric_limits<double>::quiet_NaN()
		);
	} else {
		return std::make_pair(
			pFromStr(int, str.tempSubStringBetween(0, sp)),
			pFromStr(double, str.tempSubStringBetween(sp+1))
		);
	}
}

static std::vector<std::pair<int, double>> _parseIlist(const icu::UnicodeString& in)
{
	std::vector<std::pair<int, double>> ret;
	auto w = splitWords(in);
	std::transform(w.begin(), w.end(), std::back_inserter(ret), parseIndex);
	return ret;
}

static Grammar::ArgsType _parseArgs(const icu::UnicodeString& in)
{
	Grammar::ArgsType ret;
	icu::UnicodeString tmp;
	StringCharacterIterator iter{in};
	
	bool inEscape = false, unsubmitted = false;
	for (iter.setToStart(); iter.hasNext(); ) {
		UChar32 c=iter.next32PostInc();
		if (inEscape) {
			tmp += c;
			unsubmitted = true;
			inEscape = false;
		} else {
			if (c == U'|') {
				if (unsubmitted) {
					ret.push_back(std::move(tmp));
					tmp = u"";
					unsubmitted = false;
				}
			} else if (c == U'\\') {
				inEscape = true;
			} else {
				tmp += c;
				unsubmitted = true;
			}
		}
		
	}
	if (unsubmitted) {
		ret.push_back(std::move(tmp));
	}
	return ret;
}

Grammar::ParsedString::NodeRef Grammar::ParsedString::NodeRef::sub(const Grammar::ArgsType& args) const
{
	auto aa = [&args](auto v){
		return Grammar::applyArgs(boost::get<Grammar::ArgStr>(v), args);
	};
	return NodeRef {
		aa(this->name),
		_parseArgs(aa(this->args)),
		listToVec<double>(toUTF8(aa(this->flist))),
		_parseIlist(aa(this->ilist))
	};
}

Grammar::ParsedString::NodeRef::operator bool() const noexcept
{
	return this->isRef;
}

Path Grammar::ParsedString::NodeRef::firstOf(const Grammar& g, bool incZero, complexity c) const
{
	if (this->isRef) {
		
	}
	throw __LINE__;
}

Path Grammar::ParsedString::NodeRef::firstOf(const Grammar& g, bool incZero, const ArgsType& args, complexity c) const
{
	throw __LINE__;
}

Word Grammar::ParsedString::NodeRef::chooseOf(const Grammar& g, Path* select, complexity c) const
{
	try {
		auto& a = boost::get<Grammar::ArgsType>(args);
		const auto& n = this->getName();
		if (a.empty()) {
			if (g.cnodes.count(n) && checkRefStyle() == nr_None) {
				return g.cnodes.at(n).selectFrom(select, c);
			} else if (g.nodes.count(n)) {
				if (checkRefStyle() == nr_None) {
					return g.nodes.at(n).selectFrom(select, c);
				} else if (checkRefStyle() == nr_flist) {
					return g.nodes.at(n)
						.cloneWithFreqList(
							boost::get<std::vector<double>>(flist)
						).selectFrom(select, c);
				} else if (checkRefStyle() == nr_ilist) {
					return g.nodes.at(n)
						.cloneWithIndexList(
							boost::get<std::vector<std::pair<int,double>>>(
								ilist)
						).selectFrom(select, c);
				}
			} else {
				throw std::runtime_error("No node '"+toUTF8(n)+"' found");
			}
		} else {
			const Grammar::tNode *tn = nullptr;
			if (g.tnodes.count(std::forward_as_tuple(
				n + icu::UnicodeString(
					a.size(), U'|', a.size()
				), a.size(), false
			))) {
				tn = &g.tnodes.at(std::forward_as_tuple(
					n + icu::UnicodeString(
						a.size(), U'|', a.size()
					), a.size(), false)
				);
			} else {
				for (int ac = a.size(); ac; --ac) {
					if (g.tnodes.count(std::forward_as_tuple(
						n + icu::UnicodeString(
							ac, U'|', ac
						) + u"...", ac, true
					))) {
						tn = &g.tnodes.at(std::forward_as_tuple(
							n + icu::UnicodeString(
								ac, U'|', ac
							) + u"...", ac, true
						));
						goto success;
					}
				}
				throw std::runtime_error("No tnode matching '"+toUTF8(n)+"' (with "+toStr(a.size())+" args) found");
			}
success:
			if (checkRefStyle() == nr_None) {
				return tn->selectFrom(a, select, c);
			} else if (checkRefStyle() == nr_flist) {
				return tn->applyArgs(a)
					.cloneWithFreqList(
						boost::get<std::vector<double>>(flist)
					).selectFrom(select, c);
			} else if (checkRefStyle() == nr_ilist) {
				return tn->applyArgs(a)
					.cloneWithIndexList(
						boost::get<std::vector<std::pair<int,double>>>(
							ilist)
					).selectFrom(select, c);
			}
		}
	} catch (boost::bad_get& e) {
		throw _needArgs();
	} catch (std::out_of_range& e) {
		std::cerr<<"Threw std::out_of_range: "<<e.what()
			<<" while expanding a reference to "<<this->getName()<<std::endl;
		throw;
	}
}

Word Grammar::ParsedString::NodeRef::chooseOf(const Grammar& g, const ArgsType& args, Path* select, complexity c) const
{
	throw __LINE__;
}

using refstyle = Grammar::ParsedString::NodeRef::refstyle;

struct nr_check {
	bool operator()(const std::vector<double>& fl) const {
		return fl.size();
	}
	bool operator()(const std::vector<std::pair<int,double>>& fl) const {
		return fl.size();
	}
	bool operator()(const Grammar::ArgStr& fl) const {
		return fl.size();
	}
};

refstyle Grammar::ParsedString::NodeRef::checkRefStyle() const
{
	if (boost::apply_visitor(nr_check{}, flist)) {
		return nr_flist;
	} else if (boost::apply_visitor(nr_check{}, ilist)) {
		return nr_ilist;
	} else {
		return nr_None;
	}
}

const icu::UnicodeString& Grammar::ParsedString::NodeRef::getName() const
{
	return boost::get<icu::UnicodeString>(name);
}

icu::UnicodeString Grammar::ParsedString::NodeRef::getName(const ArgsType& args) const
{
	return Grammar::applyArgs(boost::get<Grammar::ArgStr>(name), args);
}

Grammar::Node::Node(const Grammar::nodeData& data, Grammar& g)
: source(g)
{
	for (auto& branch : data) {
		channelID cV = g.chI(u"val"), cF = g.chI(u"freq");
		Grammar::Node::Branch b;
		if (branch.count(cV)) {
			b.val = branch.at(cV);
		}
		if (branch.count(cF)) {
			b.freq = pFromStr(double, branch.at(cF));
		} else {
			b.freq = 1.0;
		}
		for (auto& c : branch) {
			b.other_channels.insert(
				b.other_channels.end(),
				c
			);
		}
		branches.push_back(std::move(b));
	}
}

Word Grammar::Node::selectFrom(Path* select, complexity c) const
{
	Word out{source, {}};
	if (branches.empty()) {
		return out;
	}
	if (select) {
		throw __LINE__;
	} else {
		double sum = std::accumulate(
			branches.begin(), branches.end(), 0.0,
			[](double a, Grammar::Node::Branch b){
				return (b.freq >= 0) ? a+b.freq : a;
			}
		);
		double choose = randgen.uniform(0.0, sum);
		int stop = 0;
#ifdef _DEBUG
		// std::clog<<__LINE__<<" n "<<"("<<choose
			// <<" / "<<sum<<") ";
#endif
		for (; stop != branches.size(); ++stop) {
			choose -= branches[stop].freq;
			if (choose <= 0) {
#ifdef _DEBUG
				// std::clog<<"Chose "<<stop+1
					// <<" / "<<branches.size()<<std::endl;
#endif
				goto chose;
			}
		}
#ifdef _DEBUG
		// std::clog<<choose<<" : Chose "<<stop+1
			// <<" / "<<branches.size()<<std::endl;
#endif
		throw std::logic_error("Chose outside of range");
chose:
		if (!c) {
			return raw(stop);
		}
		const Grammar::Node::Branch& b = branches[stop];
		out.f() = b.freq/sum;
		out.p().append(stop);
		if (b.val.null()) {
			out.append(b.other_channels, true);
		} else {
			for (const auto& p : b.val) {
				if (p.first.length()) {
					out.append(p.first, b.other_channels);
				}
				if (p.second) {
					out.append(p.second.chooseOf(source, select, c.incr()));
				}
			}
		}
	}
	return out;
}

Path Grammar::Node::first(complexity c, bool incZero) const
{
	int i = 0;
	for (; i != branches.size(); ++i) {
		if (branches[i].freq > 0.0 || (incZero && branches[i].freq == 0.0)) {
			break;
		}
	}
	Path tmp;
	for (auto& i : branches[i].val) {
		if (i.second) {
			tmp.append(i.second.firstOf(source, incZero, c));
		}
	}
	return tmp;
}

Grammar::cNode::cNode(const std::map<channelID, icu::UnicodeString>& branch, int p, Grammar& g)
: source(g), pathNum(p)
{
	channelID cV = g.chI(u"val"), cF = g.chI(u"freq");
	if (branch.count(cV)) {
		b.val = branch.at(cV);
	}
	if (branch.count(cF)) {
		b.freq = pFromStr(double, branch.at(cF));
	} else {
		b.freq = 1.0;
	}
	for (auto& c : branch) {
		b.other_channels.insert(
			b.other_channels.end(),
			c
		);
	}
}

Word Grammar::cNode::selectFrom(Path* select, complexity c) const
{
	Word out{source, {}};
	if (select) {
		throw __LINE__;
	} else {
		if (!c) {
			return raw();
		}
		out.f() = 1;
		out.p().append(pathNum);
		if (b.val.null()) {
			out.append(b.other_channels, true);
		} else {
			for (const auto& p : b.val) {
				if (p.first.length()) {
					out.append(p.first, b.other_channels);
				}
				if (p.second) {
					out.append(p.second.chooseOf(source, select, c.incr()));
				}
			}
		}
	}
	return out;
}

Path Grammar::cNode::first(complexity c) const
{
	Path tmp;
	for (auto& i : b.val) {
		if (i.second) {
			tmp.append(i.second.firstOf(source, false, c));
		}
	}
	return tmp;
}

Word Grammar::cNode::raw() const
{
	std::map<Grammar::channelID, icu::UnicodeString> data;
	for (const auto& c : b.other_channels) {
		data[c.first] = c.second;
	}
	return Word{source, data};
}

Grammar::wNode Grammar::Node::cloneWithFreqList(Grammar::freqlist flist) const
{
	if (flist.size() > branches.size()) {
		flist.resize(branches.size());
	} else if (flist.size() > branches.size()) {
		std::transform(
			branches.begin()+flist.size(),
			branches.end(),
			std::back_inserter(flist),
			[](const Grammar::Node::Branch& b)
			{return b.freq;}
		);
	}
	return {this, flist};
}

Grammar::wNode Grammar::Node::cloneWithIndexList(Grammar::indexlist ilist) const
{
	if (ilist.empty()) {
		return {&source.nullNode(), Grammar::freqlist{}};
	}
	std::for_each(
		ilist.begin(),
		ilist.end(),
		[this](std::pair<int,double>& p)
		{if (std::isnan(p.second))
			p.second = this->branches[p.first].freq;}
	);
	return {this, ilist};
}

Word Grammar::Node::raw(unsigned b) const
{
	std::map<Grammar::channelID, icu::UnicodeString> data;
	for (const auto& c : branches.at(b).other_channels) {
		data[c.first] = c.second;
	}
	return Word{source, data};
}

Grammar::wNode::~wNode()
{
	switch(which) {
		case _fl:
			flist.~freqlist();
			break;
		case _il:
			ilist.~indexlist();
	}
}

Word Grammar::wNode::selectFrom(Path* select, complexity c) const
{
	int stop = 0;
	double sum = 0.0;
	if (orig->branches.empty()) {
		return {orig->source, {}};
	}
	switch(which) {
	case _fl:
#ifdef _DEBUG
		// std::clog<<__LINE__<<" "<<"fl\t";
#endif
		assert(flist.size());
		if (select) {
			throw __LINE__;
		} else {
			sum = std::accumulate(
				flist.begin(), flist.end(), 0.0,
				[](double a, double b){
					return (b >= 0) ? a+b : a;
				}
			);
			double choose = randgen.uniform(0.0, sum);
#ifdef _DEBUG
					// std::clog<<"("<<choose
						// <<" / "<<sum<<") ";
#endif
			for (; stop != orig->branches.size(); ++stop) {
				choose -= flist[stop];
				if (choose <= 0) {
#ifdef _DEBUG
					// std::clog<<"Chose "<<stop+1
						// <<" / "<<orig->branches.size()<<std::endl;
#endif
					goto chose;
				}
			}
			throw std::logic_error("Chose outside of range");
		}
		break;
	case _il:
#ifdef _DEBUG
		// std::clog<<__LINE__<<" "<<"il\t";
#endif
		assert(ilist.size());
		if (select) {
			throw __LINE__;
		} else if (ilist.size() > 1) {
			sum = std::accumulate(
				ilist.begin(), ilist.end(), 0.0,
				[](double a, std::pair<int,double> b){
					return (b.second >= 0) ? a+b.second : a;
				}
			);
			double choose = randgen.uniform(0.0, sum);
#ifdef _DEBUG
					// std::clog<<"("<<choose
						// <<" / "<<sum<<") ";
#endif
			for (; stop != orig->branches.size(); ++stop) {
				choose -= ilist[stop].second;
				if (choose <= 0) {
#ifdef _DEBUG
					// std::clog<<"Chose "<<stop+1
						// <<" / "<<orig->branches.size()<<std::endl;
#endif
					stop = ilist[stop].first;
					goto chose;
				}
			}
			throw std::logic_error("Chose outside of range");
		} else {
			stop = ilist[0].first;
			goto chose;
		}
	}
chose:
	if (!c) {
		return orig->raw(stop);
	}
	Word out{orig->source, {}};
	const Grammar::Node::Branch& b = orig->branches.at(stop);
	out.f() = b.freq/sum;
	out.p().append(stop);
	if (b.val.null()) {
		out.append(b.other_channels, true);
	} else {
		for (const auto& p : b.val) {
			if (p.first.length()) {
				out.append(p.first, b.other_channels);
			}
			if (p.second) {
				out.append(p.second.chooseOf(orig->source, select, c.incr()));
			}
		}
	}
	return out;
}

Path Grammar::wNode::first(complexity c, bool incZero) const
{
	throw __LINE__;
}

Grammar::tNode::tNode(const Grammar::nodeData& data, Grammar& g, int c, bool vararg)
: source(g), argc(c), varargs(vararg)
{
	const channelID cV = g.chI(u"val"), cF = g.chI(u"freq");
	for (auto& branch : data) {
		Grammar::tNode::Branch b;
		if (branch.count(cV)) {
#ifdef _DEBUG
			// std::clog<<"  - val: "<<branch.at(cV)<<std::endl;
#endif
			b.val = {branch.at(cV), c, vararg};
		}
#ifdef _DEBUG
		// else {
			// std::clog<<"    freq: 1"<<std::endl;
		// }
#endif
		if (branch.count(cF)) {
#ifdef _DEBUG
			// std::clog<<"    freq: "<<branch.at(cF)<<std::endl;
#endif
			b.freq = Grammar::parseArgs(branch.at(cF), argc, varargs);
		} else {
#ifdef _DEBUG
			// std::clog<<"    freq: 1"<<std::endl;
#endif
			b.freq = Grammar::parseArgs(u"1", argc, varargs);
		}
		for (auto& c : branch) {
			b.other_channels.insert(
				b.other_channels.end(),
				{c.first, Grammar::parseArgs(c.second, argc, varargs)}
			);
		}
		branches.push_back(std::move(b));
	}
}

Grammar::Node Grammar::tNode::applyArgs(const Grammar::ArgsType& args) const
{
	Grammar::Node ret(Grammar::nodeData{}, source);
	for (auto& b : branches) {
		Grammar::Node::Branch tb{b.val.sub(args, argc, varargs),1.0,{}};
		try {
			tb.freq = pFromStr(double, Grammar::applyArgs(b.freq, args));
		} catch (std::runtime_error& e) {
			std::cerr<<e.what()<<std::endl;
			throw __LINE__;
		}
		for (auto& c : b.other_channels) {
			tb.other_channels[c.first] = Grammar::applyArgs(c.second, args);
		}
		ret.branches.push_back(tb);
	}
	return ret;
}

Word Grammar::tNode::selectFrom(const Grammar::ArgsType& args, Path* select, complexity c) const
{
	return applyArgs(args).selectFrom(select, c);
}

inline double _applyToFreq(const Grammar::ArgStr& f, const Grammar::ArgsType& args)
{
	return pFromStr(double, Grammar::applyArgs(f, args));
}

Path Grammar::tNode::first(const Grammar::ArgsType& args, complexity c, bool incZero) const
{
	int i = 0;
	for (; i != branches.size(); ++i) {
		double f = _applyToFreq(branches[i].freq, args);
		if (f > 0.0 || (incZero && f == 0.0)) {
			break;
		}
	}
	Path tmp;
	for (auto& i : branches[i].val.sub(args, argc, varargs)) {
		if (i.second) {
			tmp.append(i.second.firstOf(source, incZero, args, c));
		}
	}
	return tmp;
}