From 4f9e993c8b6e5e57e2df31bfa9bedf67d64bfa69 Mon Sep 17 00:00:00 2001
From: Jim Pivarski <jpivarski@users.noreply.github.com>
Date: Fri, 12 Jan 2024 10:47:34 -0600
Subject: [PATCH] chore: restore files that generated the type-parser (#2926)

* chore: restore files that generated the type-parser

* style: pre-commit fixes

* Revert "style: pre-commit fixes"

pre-commit should not be allowed to modify auto-generated code. Or any of these files; they're for historical reference.

This reverts commit fd96676f0089be718379a3c3286259d418a6051d.

* Prevent linters from touching these files.

* style: pre-commit fixes

* Including this diff is useless if pre-commit is going to keep changing it.

* Move to studies/type-parser-1.5.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 studies/type-parser-1.5/generated_parser.py | 2571 +++++++++++++++++++
 studies/type-parser-1.5/parser.py           |  322 +++
 studies/type-parser-1.5/type-grammar.lark   |   82 +
 3 files changed, 2975 insertions(+)
 create mode 100644 studies/type-parser-1.5/generated_parser.py
 create mode 100644 studies/type-parser-1.5/parser.py
 create mode 100644 studies/type-parser-1.5/type-grammar.lark

diff --git a/studies/type-parser-1.5/generated_parser.py b/studies/type-parser-1.5/generated_parser.py
new file mode 100644
index 0000000000..5aa0f8c820
--- /dev/null
+++ b/studies/type-parser-1.5/generated_parser.py
@@ -0,0 +1,2571 @@
+# flake8: noqa
+# fmt: off
+# pylint: skip-file
+
+# The file was automatically generated by Lark v0.11.1
+__version__ = "0.11.1"
+
+#
+#
+#   Lark Stand-alone Generator Tool
+# ----------------------------------
+# Generates a stand-alone LALR(1) parser with a standard lexer
+#
+# Git:    https://github.com/erezsh/lark
+# Author: Erez Shinan (erezshin@gmail.com)
+#
+#
+#    >>> LICENSE
+#
+#    This tool and its generated code use a separate license from Lark,
+#    and are subject to the terms of the Mozilla Public License, v. 2.0.
+#    If a copy of the MPL was not distributed with this
+#    file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+#    If you wish to purchase a commercial license for this tool and its
+#    generated code, you may contact me via email or otherwise.
+#
+#    If MPL2 is incompatible with your free or open-source project,
+#    contact me and we'll work it out.
+#
+#
+
+from io import open
+
+
+
+class LarkError(Exception):
+    pass
+
+
+class GrammarError(LarkError):
+    pass
+
+
+class ParseError(LarkError):
+    pass
+
+
+class LexError(LarkError):
+    pass
+
+
+class UnexpectedEOF(ParseError):
+    def __init__(self, expected):
+        self.expected = expected
+
+        message = ("Unexpected end-of-input. Expected one of: \n\t* %s\n" % '\n\t* '.join(x.name for x in self.expected))
+        super(UnexpectedEOF, self).__init__(message)
+
+
+class UnexpectedInput(LarkError):
+    #--
+    pos_in_stream = None
+
+    def get_context(self, text, span=40):
+        #--
+        pos = self.pos_in_stream
+        start = max(pos - span, 0)
+        end = pos + span
+        if not isinstance(text, bytes):
+            before = text[start:pos].rsplit('\n', 1)[-1]
+            after = text[pos:end].split('\n', 1)[0]
+            return before + after + '\n' + ' ' * len(before.expandtabs()) + '^\n'
+        else:
+            before = text[start:pos].rsplit(b'\n', 1)[-1]
+            after = text[pos:end].split(b'\n', 1)[0]
+            return (before + after + b'\n' + b' ' * len(before.expandtabs()) + b'^\n').decode("ascii", "backslashreplace")
+
+    def match_examples(self, parse_fn, examples, token_type_match_fallback=False, use_accepts=False):
+        #--
+        assert self.state is not None, "Not supported for this exception"
+
+        if isinstance(examples, dict):
+            examples = examples.items()
+
+        candidate = (None, False)
+        for i, (label, example) in enumerate(examples):
+            assert not isinstance(example, STRING_TYPE)
+
+            for j, malformed in enumerate(example):
+                try:
+                    parse_fn(malformed)
+                except UnexpectedInput as ut:
+                    if ut.state == self.state:
+                        if use_accepts and ut.accepts != self.accepts:
+                            logger.debug("Different accepts with same state[%d]: %s != %s at example [%s][%s]" %
+                                        (self.state, self.accepts, ut.accepts, i, j))
+                            continue
+                        try:
+                            if ut.token == self.token:  ##
+
+                                logger.debug("Exact Match at example [%s][%s]" % (i, j))
+                                return label
+
+                            if token_type_match_fallback:
+                                ##
+
+                                if (ut.token.type == self.token.type) and not candidate[-1]:
+                                    logger.debug("Token Type Fallback at example [%s][%s]" % (i, j))
+                                    candidate = label, True
+
+                        except AttributeError:
+                            pass
+                        if not candidate[0]:
+                            logger.debug("Same State match at example [%s][%s]" % (i, j))
+                            candidate = label, False
+
+        return candidate[0]
+
+
+class UnexpectedCharacters(LexError, UnexpectedInput):
+    def __init__(self, seq, lex_pos, line, column, allowed=None, considered_tokens=None, state=None, token_history=None):
+        self.line = line
+        self.column = column
+        self.pos_in_stream = lex_pos
+        self.state = state
+
+        self.allowed = allowed
+        self.considered_tokens = considered_tokens
+
+        if isinstance(seq, bytes):
+            _s = seq[lex_pos:lex_pos+1].decode("ascii", "backslashreplace")
+        else:
+            _s = seq[lex_pos]
+
+        message = "No terminal defined for %r at line %d col %d" % (_s, line, column)
+        message += '\n\n' + self.get_context(seq)
+        if allowed:
+            message += '\nExpecting: %s\n' % allowed
+        if token_history:
+            message += '\nPrevious tokens: %s\n' % ', '.join(repr(t) for t in token_history)
+
+        super(UnexpectedCharacters, self).__init__(message)
+
+
+class UnexpectedToken(ParseError, UnexpectedInput):
+    #--
+    def __init__(self, token, expected, considered_rules=None, state=None, puppet=None):
+        self.line = getattr(token, 'line', '?')
+        self.column = getattr(token, 'column', '?')
+        self.pos_in_stream = getattr(token, 'pos_in_stream', None)
+        self.state = state
+
+        self.token = token
+        self.expected = expected     ##
+
+        self.considered_rules = considered_rules
+        self.puppet = puppet
+
+        ##
+
+        ##
+
+        self.accepts = puppet and puppet.accepts()
+
+        message = ("Unexpected token %r at line %s, column %s.\n"
+                   "Expected one of: \n\t* %s\n"
+                   % (token, self.line, self.column, '\n\t* '.join(self.accepts or self.expected)))
+
+        super(UnexpectedToken, self).__init__(message)
+
+
+class VisitError(LarkError):
+    #--
+    def __init__(self, rule, obj, orig_exc):
+        self.obj = obj
+        self.orig_exc = orig_exc
+
+        message = 'Error trying to process rule "%s":\n\n%s' % (rule, orig_exc)
+        super(VisitError, self).__init__(message)
+
+import sys, re
+import logging
+logger = logging.getLogger("lark")
+logger.addHandler(logging.StreamHandler())
+##
+
+logger.setLevel(logging.CRITICAL)
+
+
+def classify(seq, key=None, value=None):
+    d = {}
+    for item in seq:
+        k = key(item) if (key is not None) else item
+        v = value(item) if (value is not None) else item
+        if k in d:
+            d[k].append(v)
+        else:
+            d[k] = [v]
+    return d
+
+
+def _deserialize(data, namespace, memo):
+    if isinstance(data, dict):
+        if '__type__' in data:  ##
+
+            class_ = namespace[data['__type__']]
+            return class_.deserialize(data, memo)
+        elif '@' in data:
+            return memo[data['@']]
+        return {key:_deserialize(value, namespace, memo) for key, value in data.items()}
+    elif isinstance(data, list):
+        return [_deserialize(value, namespace, memo) for value in data]
+    return data
+
+
+class Serialize(object):
+    #--
+
+    def memo_serialize(self, types_to_memoize):
+        memo = SerializeMemoizer(types_to_memoize)
+        return self.serialize(memo), memo.serialize()
+
+    def serialize(self, memo=None):
+        if memo and memo.in_types(self):
+            return {'@': memo.memoized.get(self)}
+
+        fields = getattr(self, '__serialize_fields__')
+        res = {f: _serialize(getattr(self, f), memo) for f in fields}
+        res['__type__'] = type(self).__name__
+        postprocess = getattr(self, '_serialize', None)
+        if postprocess:
+            postprocess(res, memo)
+        return res
+
+    @classmethod
+    def deserialize(cls, data, memo):
+        namespace = getattr(cls, '__serialize_namespace__', {})
+        namespace = {c.__name__:c for c in namespace}
+
+        fields = getattr(cls, '__serialize_fields__')
+
+        if '@' in data:
+            return memo[data['@']]
+
+        inst = cls.__new__(cls)
+        for f in fields:
+            try:
+                setattr(inst, f, _deserialize(data[f], namespace, memo))
+            except KeyError as e:
+                raise KeyError("Cannot find key for class", cls, e)
+        postprocess = getattr(inst, '_deserialize', None)
+        if postprocess:
+            postprocess()
+        return inst
+
+
+class SerializeMemoizer(Serialize):
+    #--
+
+    __serialize_fields__ = 'memoized',
+
+    def __init__(self, types_to_memoize):
+        self.types_to_memoize = tuple(types_to_memoize)
+        self.memoized = Enumerator()
+
+    def in_types(self, value):
+        return isinstance(value, self.types_to_memoize)
+
+    def serialize(self):
+        return _serialize(self.memoized.reversed(), None)
+
+    @classmethod
+    def deserialize(cls, data, namespace, memo):
+        return _deserialize(data, namespace, memo)
+
+
+try:
+    STRING_TYPE = basestring
+except NameError:   ##
+
+    STRING_TYPE = str
+
+
+import types
+from functools import wraps, partial
+from contextlib import contextmanager
+
+Str = type(u'')
+try:
+    classtype = types.ClassType  ##
+
+except AttributeError:
+    classtype = type    ##
+
+
+
+def smart_decorator(f, create_decorator):
+    if isinstance(f, types.FunctionType):
+        return wraps(f)(create_decorator(f, True))
+
+    elif isinstance(f, (classtype, type, types.BuiltinFunctionType)):
+        return wraps(f)(create_decorator(f, False))
+
+    elif isinstance(f, types.MethodType):
+        return wraps(f)(create_decorator(f.__func__, True))
+
+    elif isinstance(f, partial):
+        ##
+
+        return wraps(f.func)(create_decorator(lambda *args, **kw: f(*args[1:], **kw), True))
+
+    else:
+        return create_decorator(f.__func__.__call__, True)
+
+
+try:
+    import regex
+except ModuleNotFoundError:
+    regex = None
+
+import sre_parse
+import sre_constants
+categ_pattern = re.compile(r'\\p{[A-Za-z_]+}')
+
+def get_regexp_width(expr):
+    if regex:
+        ##
+
+        ##
+
+        ##
+
+        regexp_final = re.sub(categ_pattern, 'A', expr)
+    else:
+        if re.search(categ_pattern, expr):
+            raise ImportError('`regex` module must be installed in order to use Unicode categories.', expr)
+        regexp_final = expr
+    try:
+        return [int(x) for x in sre_parse.parse(regexp_final).getwidth()]
+    except sre_constants.error:
+        raise ValueError(expr)
+
+
+from collections import OrderedDict
+
+
+class Meta:
+    def __init__(self):
+        self.empty = True
+
+
+class Tree(object):
+    #--
+    def __init__(self, data, children, meta=None):
+        self.data = data
+        self.children = children
+        self._meta = meta
+
+    @property
+    def meta(self):
+        if self._meta is None:
+            self._meta = Meta()
+        return self._meta
+
+    def __repr__(self):
+        return 'Tree(%r, %r)' % (self.data, self.children)
+
+    def _pretty_label(self):
+        return self.data
+
+    def _pretty(self, level, indent_str):
+        if len(self.children) == 1 and not isinstance(self.children[0], Tree):
+            return [indent_str*level, self._pretty_label(), '\t', '%s' % (self.children[0],), '\n']
+
+        l = [indent_str*level, self._pretty_label(), '\n']
+        for n in self.children:
+            if isinstance(n, Tree):
+                l += n._pretty(level+1, indent_str)
+            else:
+                l += [indent_str*(level+1), '%s' % (n,), '\n']
+
+        return l
+
+    def pretty(self, indent_str='  '):
+        #--
+        return ''.join(self._pretty(0, indent_str))
+
+    def __eq__(self, other):
+        try:
+            return self.data == other.data and self.children == other.children
+        except AttributeError:
+            return False
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __hash__(self):
+        return hash((self.data, tuple(self.children)))
+
+    def iter_subtrees(self):
+        #--
+        queue = [self]
+        subtrees = OrderedDict()
+        for subtree in queue:
+            subtrees[id(subtree)] = subtree
+            queue += [c for c in reversed(subtree.children)
+                      if isinstance(c, Tree) and id(c) not in subtrees]
+
+        del queue
+        return reversed(list(subtrees.values()))
+
+    def find_pred(self, pred):
+        #--
+        return filter(pred, self.iter_subtrees())
+
+    def find_data(self, data):
+        #--
+        return self.find_pred(lambda t: t.data == data)
+
+
+from inspect import getmembers, getmro
+
+
+class Discard(Exception):
+    #--
+    pass
+
+##
+
+
+
+class _Decoratable:
+    #--
+
+    @classmethod
+    def _apply_decorator(cls, decorator, **kwargs):
+        mro = getmro(cls)
+        assert mro[0] is cls
+        libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)}
+        for name, value in getmembers(cls):
+
+            ##
+
+            if name.startswith('_') or (name in libmembers and name not in cls.__dict__):
+                continue
+            if not callable(value):
+                continue
+
+            ##
+
+            if hasattr(cls.__dict__[name], 'vargs_applied') or hasattr(value, 'vargs_applied'):
+                continue
+
+            static = isinstance(cls.__dict__[name], (staticmethod, classmethod))
+            setattr(cls, name, decorator(value, static=static, **kwargs))
+        return cls
+
+    def __class_getitem__(cls, _):
+        return cls
+
+
+class Transformer(_Decoratable):
+    #--
+    __visit_tokens__ = True   ##
+
+
+    def __init__(self,  visit_tokens=True):
+        self.__visit_tokens__ = visit_tokens
+
+    def _call_userfunc(self, tree, new_children=None):
+        ##
+
+        children = new_children if new_children is not None else tree.children
+        try:
+            f = getattr(self, tree.data)
+        except AttributeError:
+            return self.__default__(tree.data, children, tree.meta)
+        else:
+            try:
+                wrapper = getattr(f, 'visit_wrapper', None)
+                if wrapper is not None:
+                    return f.visit_wrapper(f, tree.data, children, tree.meta)
+                else:
+                    return f(children)
+            except (GrammarError, Discard):
+                raise
+            except Exception as e:
+                raise VisitError(tree.data, tree, e)
+
+    def _call_userfunc_token(self, token):
+        try:
+            f = getattr(self, token.type)
+        except AttributeError:
+            return self.__default_token__(token)
+        else:
+            try:
+                return f(token)
+            except (GrammarError, Discard):
+                raise
+            except Exception as e:
+                raise VisitError(token.type, token, e)
+
+    def _transform_children(self, children):
+        for c in children:
+            try:
+                if isinstance(c, Tree):
+                    yield self._transform_tree(c)
+                elif self.__visit_tokens__ and isinstance(c, Token):
+                    yield self._call_userfunc_token(c)
+                else:
+                    yield c
+            except Discard:
+                pass
+
+    def _transform_tree(self, tree):
+        children = list(self._transform_children(tree.children))
+        return self._call_userfunc(tree, children)
+
+    def transform(self, tree):
+        #--
+        return self._transform_tree(tree)
+
+    def __mul__(self, other):
+        #--
+        return TransformerChain(self, other)
+
+    def __default__(self, data, children, meta):
+        #--
+        return Tree(data, children, meta)
+
+    def __default_token__(self, token):
+        #--
+        return token
+
+
+class InlineTransformer(Transformer):   ##
+
+    def _call_userfunc(self, tree, new_children=None):
+        ##
+
+        children = new_children if new_children is not None else tree.children
+        try:
+            f = getattr(self, tree.data)
+        except AttributeError:
+            return self.__default__(tree.data, children, tree.meta)
+        else:
+            return f(*children)
+
+
+class TransformerChain(object):
+    def __init__(self, *transformers):
+        self.transformers = transformers
+
+    def transform(self, tree):
+        for t in self.transformers:
+            tree = t.transform(tree)
+        return tree
+
+    def __mul__(self, other):
+        return TransformerChain(*self.transformers + (other,))
+
+
+class Transformer_InPlace(Transformer):
+    #--
+    def _transform_tree(self, tree):           ##
+
+        return self._call_userfunc(tree)
+
+    def transform(self, tree):
+        for subtree in tree.iter_subtrees():
+            subtree.children = list(self._transform_children(subtree.children))
+
+        return self._transform_tree(tree)
+
+
+class Transformer_NonRecursive(Transformer):
+    #--
+
+    def transform(self, tree):
+        ##
+
+        rev_postfix = []
+        q = [tree]
+        while q:
+            t = q.pop()
+            rev_postfix.append(t)
+            if isinstance(t, Tree):
+                q += t.children
+
+        ##
+
+        stack = []
+        for x in reversed(rev_postfix):
+            if isinstance(x, Tree):
+                size = len(x.children)
+                if size:
+                    args = stack[-size:]
+                    del stack[-size:]
+                else:
+                    args = []
+                stack.append(self._call_userfunc(x, args))
+            else:
+                stack.append(x)
+
+        t ,= stack  ##
+
+        return t
+
+
+class Transformer_InPlaceRecursive(Transformer):
+    #--
+    def _transform_tree(self, tree):
+        tree.children = list(self._transform_children(tree.children))
+        return self._call_userfunc(tree)
+
+
+##
+
+
+class VisitorBase:
+    def _call_userfunc(self, tree):
+        return getattr(self, tree.data, self.__default__)(tree)
+
+    def __default__(self, tree):
+        #--
+        return tree
+
+    def __class_getitem__(cls, _):
+        return cls
+
+
+class Visitor(VisitorBase):
+    #--
+
+    def visit(self, tree):
+        #--
+        for subtree in tree.iter_subtrees():
+            self._call_userfunc(subtree)
+        return tree
+
+    def visit_topdown(self,tree):
+        #--
+        for subtree in tree.iter_subtrees_topdown():
+            self._call_userfunc(subtree)
+        return tree
+
+
+class Visitor_Recursive(VisitorBase):
+    #--
+
+    def visit(self, tree):
+        #--
+        for child in tree.children:
+            if isinstance(child, Tree):
+                self.visit(child)
+
+        self._call_userfunc(tree)
+        return tree
+
+    def visit_topdown(self,tree):
+        #--
+        self._call_userfunc(tree)
+
+        for child in tree.children:
+            if isinstance(child, Tree):
+                self.visit_topdown(child)
+
+        return tree
+
+
+def visit_children_decor(func):
+    #--
+    @wraps(func)
+    def inner(cls, tree):
+        values = cls.visit_children(tree)
+        return func(cls, values)
+    return inner
+
+
+class Interpreter(_Decoratable):
+    #--
+
+    def visit(self, tree):
+        f = getattr(self, tree.data)
+        wrapper = getattr(f, 'visit_wrapper', None)
+        if wrapper is not None:
+            return f.visit_wrapper(f, tree.data, tree.children, tree.meta)
+        else:
+            return f(tree)
+
+    def visit_children(self, tree):
+        return [self.visit(child) if isinstance(child, Tree) else child
+                for child in tree.children]
+
+    def __getattr__(self, name):
+        return self.__default__
+
+    def __default__(self, tree):
+        return self.visit_children(tree)
+
+
+##
+
+
+def _apply_decorator(obj, decorator, **kwargs):
+    try:
+        _apply = obj._apply_decorator
+    except AttributeError:
+        return decorator(obj, **kwargs)
+    else:
+        return _apply(decorator, **kwargs)
+
+
+def _inline_args__func(func):
+    @wraps(func)
+    def create_decorator(_f, with_self):
+        if with_self:
+            def f(self, children):
+                return _f(self, *children)
+        else:
+            def f(self, children):
+                return _f(*children)
+        return f
+
+    return smart_decorator(func, create_decorator)
+
+
+def inline_args(obj):   ##
+
+    return _apply_decorator(obj, _inline_args__func)
+
+
+def _visitor_args_func_dec(func, visit_wrapper=None, static=False):
+    def create_decorator(_f, with_self):
+        if with_self:
+            def f(self, *args, **kwargs):
+                return _f(self, *args, **kwargs)
+        else:
+            def f(self, *args, **kwargs):
+                return _f(*args, **kwargs)
+        return f
+
+    if static:
+        f = wraps(func)(create_decorator(func, False))
+    else:
+        f = smart_decorator(func, create_decorator)
+    f.vargs_applied = True
+    f.visit_wrapper = visit_wrapper
+    return f
+
+
+def _vargs_inline(f, _data, children, _meta):
+    return f(*children)
+def _vargs_meta_inline(f, _data, children, meta):
+    return f(meta, *children)
+def _vargs_meta(f, _data, children, meta):
+    return f(children, meta)   ##
+
+def _vargs_tree(f, data, children, meta):
+    return f(Tree(data, children, meta))
+
+
+def v_args(inline=False, meta=False, tree=False, wrapper=None):
+    #--
+    if tree and (meta or inline):
+        raise ValueError("Visitor functions cannot combine 'tree' with 'meta' or 'inline'.")
+
+    func = None
+    if meta:
+        if inline:
+            func = _vargs_meta_inline
+        else:
+            func = _vargs_meta
+    elif inline:
+        func = _vargs_inline
+    elif tree:
+        func = _vargs_tree
+
+    if wrapper is not None:
+        if func is not None:
+            raise ValueError("Cannot use 'wrapper' along with 'tree', 'meta' or 'inline'.")
+        func = wrapper
+
+    def _visitor_args_dec(obj):
+        return _apply_decorator(obj, _visitor_args_func_dec, visit_wrapper=func)
+    return _visitor_args_dec
+
+
+
+class Indenter:
+    def __init__(self):
+        self.paren_level = None
+        self.indent_level = None
+        assert self.tab_len > 0
+
+    def handle_NL(self, token):
+        if self.paren_level > 0:
+            return
+
+        yield token
+
+        indent_str = token.rsplit('\n', 1)[1] ##
+
+        indent = indent_str.count(' ') + indent_str.count('\t') * self.tab_len
+
+        if indent > self.indent_level[-1]:
+            self.indent_level.append(indent)
+            yield Token.new_borrow_pos(self.INDENT_type, indent_str, token)
+        else:
+            while indent < self.indent_level[-1]:
+                self.indent_level.pop()
+                yield Token.new_borrow_pos(self.DEDENT_type, indent_str, token)
+
+            assert indent == self.indent_level[-1], '%s != %s' % (indent, self.indent_level[-1])
+
+    def _process(self, stream):
+        for token in stream:
+            if token.type == self.NL_type:
+                for t in self.handle_NL(token):
+                    yield t
+            else:
+                yield token
+
+            if token.type in self.OPEN_PAREN_types:
+                self.paren_level += 1
+            elif token.type in self.CLOSE_PAREN_types:
+                self.paren_level -= 1
+                assert self.paren_level >= 0
+
+        while len(self.indent_level) > 1:
+            self.indent_level.pop()
+            yield Token(self.DEDENT_type, '')
+
+        assert self.indent_level == [0], self.indent_level
+
+    def process(self, stream):
+        self.paren_level = 0
+        self.indent_level = [0]
+        return self._process(stream)
+
+    ##
+
+    @property
+    def always_accept(self):
+        return (self.NL_type,)
+
+
+
+class Symbol(Serialize):
+    __slots__ = ('name',)
+
+    is_term = NotImplemented
+
+    def __init__(self, name):
+        self.name = name
+
+    def __eq__(self, other):
+        assert isinstance(other, Symbol), other
+        return self.is_term == other.is_term and self.name == other.name
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __hash__(self):
+        return hash(self.name)
+
+    def __repr__(self):
+        return '%s(%r)' % (type(self).__name__, self.name)
+
+    fullrepr = property(__repr__)
+
+
+class Terminal(Symbol):
+    __serialize_fields__ = 'name', 'filter_out'
+
+    is_term = True
+
+    def __init__(self, name, filter_out=False):
+        self.name = name
+        self.filter_out = filter_out
+
+    @property
+    def fullrepr(self):
+        return '%s(%r, %r)' % (type(self).__name__, self.name, self.filter_out)
+
+
+class NonTerminal(Symbol):
+    __serialize_fields__ = 'name',
+
+    is_term = False
+
+
+class RuleOptions(Serialize):
+    __serialize_fields__ = 'keep_all_tokens', 'expand1', 'priority', 'template_source', 'empty_indices'
+
+    def __init__(self, keep_all_tokens=False, expand1=False, priority=None, template_source=None, empty_indices=()):
+        self.keep_all_tokens = keep_all_tokens
+        self.expand1 = expand1
+        self.priority = priority
+        self.template_source = template_source
+        self.empty_indices = empty_indices
+
+    def __repr__(self):
+        return 'RuleOptions(%r, %r, %r, %r)' % (
+            self.keep_all_tokens,
+            self.expand1,
+            self.priority,
+            self.template_source
+        )
+
+
+class Rule(Serialize):
+    #--
+    __slots__ = ('origin', 'expansion', 'alias', 'options', 'order', '_hash')
+
+    __serialize_fields__ = 'origin', 'expansion', 'order', 'alias', 'options'
+    __serialize_namespace__ = Terminal, NonTerminal, RuleOptions
+
+    def __init__(self, origin, expansion, order=0, alias=None, options=None):
+        self.origin = origin
+        self.expansion = expansion
+        self.alias = alias
+        self.order = order
+        self.options = options or RuleOptions()
+        self._hash = hash((self.origin, tuple(self.expansion)))
+
+    def _deserialize(self):
+        self._hash = hash((self.origin, tuple(self.expansion)))
+
+    def __str__(self):
+        return '<%s : %s>' % (self.origin.name, ' '.join(x.name for x in self.expansion))
+
+    def __repr__(self):
+        return 'Rule(%r, %r, %r, %r)' % (self.origin, self.expansion, self.alias, self.options)
+
+    def __hash__(self):
+        return self._hash
+
+    def __eq__(self, other):
+        if not isinstance(other, Rule):
+            return False
+        return self.origin == other.origin and self.expansion == other.expansion
+
+
+
+from copy import copy
+
+
+class Pattern(Serialize):
+
+    def __init__(self, value, flags=()):
+        self.value = value
+        self.flags = frozenset(flags)
+
+    def __repr__(self):
+        return repr(self.to_regexp())
+
+    ##
+
+    def __hash__(self):
+        return hash((type(self), self.value, self.flags))
+
+    def __eq__(self, other):
+        return type(self) == type(other) and self.value == other.value and self.flags == other.flags
+
+    def to_regexp(self):
+        raise NotImplementedError()
+
+    def _get_flags(self, value):
+        for f in self.flags:
+            value = f'(?{f}:{value})'
+        return value
+
+
+class PatternStr(Pattern):
+    __serialize_fields__ = 'value', 'flags'
+
+    type = "str"
+
+    def to_regexp(self):
+        return self._get_flags(re.escape(self.value))
+
+    @property
+    def min_width(self):
+        return len(self.value)
+    max_width = min_width
+
+
+class PatternRE(Pattern):
+    __serialize_fields__ = 'value', 'flags', '_width'
+
+    type = "re"
+
+    def to_regexp(self):
+        return self._get_flags(self.value)
+
+    _width = None
+    def _get_width(self):
+        if self._width is None:
+            self._width = get_regexp_width(self.to_regexp())
+        return self._width
+
+    @property
+    def min_width(self):
+        return self._get_width()[0]
+
+    @property
+    def max_width(self):
+        return self._get_width()[1]
+
+
+class TerminalDef(Serialize):
+    __serialize_fields__ = 'name', 'pattern', 'priority'
+    __serialize_namespace__ = PatternStr, PatternRE
+
+    def __init__(self, name, pattern, priority=1):
+        assert isinstance(pattern, Pattern), pattern
+        self.name = name
+        self.pattern = pattern
+        self.priority = priority
+
+    def __repr__(self):
+        return '%s(%r, %r)' % (type(self).__name__, self.name, self.pattern)
+
+
+class Token(Str):
+    #--
+    __slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos')
+
+    def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None, end_line=None, end_column=None, end_pos=None):
+        try:
+            self = super(Token, cls).__new__(cls, value)
+        except UnicodeDecodeError:
+            value = value.decode('latin1')
+            self = super(Token, cls).__new__(cls, value)
+
+        self.type = type_
+        self.pos_in_stream = pos_in_stream
+        self.value = value
+        self.line = line
+        self.column = column
+        self.end_line = end_line
+        self.end_column = end_column
+        self.end_pos = end_pos
+        return self
+
+    def update(self, type_=None, value=None):
+        return Token.new_borrow_pos(
+            type_ if type_ is not None else self.type,
+            value if value is not None else self.value,
+            self
+        )
+
+    @classmethod
+    def new_borrow_pos(cls, type_, value, borrow_t):
+        return cls(type_, value, borrow_t.pos_in_stream, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos)
+
+    def __reduce__(self):
+        return (self.__class__, (self.type, self.value, self.pos_in_stream, self.line, self.column))
+
+    def __repr__(self):
+        return 'Token(%r, %r)' % (self.type, self.value)
+
+    def __deepcopy__(self, memo):
+        return Token(self.type, self.value, self.pos_in_stream, self.line, self.column)
+
+    def __eq__(self, other):
+        if isinstance(other, Token) and self.type != other.type:
+            return False
+
+        return Str.__eq__(self, other)
+
+    __hash__ = Str.__hash__
+
+
+class LineCounter:
+    __slots__ = 'char_pos', 'line', 'column', 'line_start_pos', 'newline_char'
+
+    def __init__(self, newline_char):
+        self.newline_char = newline_char
+        self.char_pos = 0
+        self.line = 1
+        self.column = 1
+        self.line_start_pos = 0
+
+    def feed(self, token, test_newline=True):
+        #--
+        if test_newline:
+            newlines = token.count(self.newline_char)
+            if newlines:
+                self.line += newlines
+                self.line_start_pos = self.char_pos + token.rindex(self.newline_char) + 1
+
+        self.char_pos += len(token)
+        self.column = self.char_pos - self.line_start_pos + 1
+
+
+class UnlessCallback:
+    def __init__(self, mres):
+        self.mres = mres
+
+    def __call__(self, t):
+        for mre, type_from_index in self.mres:
+            m = mre.match(t.value)
+            if m:
+                t.type = type_from_index[m.lastindex]
+                break
+        return t
+
+
+class CallChain:
+    def __init__(self, callback1, callback2, cond):
+        self.callback1 = callback1
+        self.callback2 = callback2
+        self.cond = cond
+
+    def __call__(self, t):
+        t2 = self.callback1(t)
+        return self.callback2(t) if self.cond(t2) else t2
+
+
+def _create_unless(terminals, g_regex_flags, re_, use_bytes):
+    tokens_by_type = classify(terminals, lambda t: type(t.pattern))
+    assert len(tokens_by_type) <= 2, tokens_by_type.keys()
+    embedded_strs = set()
+    callback = {}
+    for retok in tokens_by_type.get(PatternRE, []):
+        unless = []
+        for strtok in tokens_by_type.get(PatternStr, []):
+            if strtok.priority > retok.priority:
+                continue
+            s = strtok.pattern.value
+            m = re_.match(retok.pattern.to_regexp(), s, g_regex_flags)
+            if m and m.group(0) == s:
+                unless.append(strtok)
+                if strtok.pattern.flags <= retok.pattern.flags:
+                    embedded_strs.add(strtok)
+        if unless:
+            callback[retok.name] = UnlessCallback(build_mres(unless, g_regex_flags, re_, match_whole=True, use_bytes=use_bytes))
+
+    terminals = [t for t in terminals if t not in embedded_strs]
+    return terminals, callback
+
+
+def _build_mres(terminals, max_size, g_regex_flags, match_whole, re_, use_bytes):
+    ##
+
+    ##
+
+    ##
+
+    postfix = '$' if match_whole else ''
+    mres = []
+    while terminals:
+        pattern = u'|'.join(u'(?P<%s>%s)' % (t.name, t.pattern.to_regexp() + postfix) for t in terminals[:max_size])
+        if use_bytes:
+            pattern = pattern.encode('latin-1')
+        try:
+            mre = re_.compile(pattern, g_regex_flags)
+        except AssertionError:  ##
+
+            return _build_mres(terminals, max_size//2, g_regex_flags, match_whole, re_, use_bytes)
+
+        mres.append((mre, {i: n for n, i in mre.groupindex.items()}))
+        terminals = terminals[max_size:]
+    return mres
+
+
+def build_mres(terminals, g_regex_flags, re_, use_bytes, match_whole=False):
+    return _build_mres(terminals, len(terminals), g_regex_flags, match_whole, re_, use_bytes)
+
+
+def _regexp_has_newline(r):
+    #--
+    return '\n' in r or '\\n' in r or '\\s' in r or '[^' in r or ('(?s' in r and '.' in r)
+
+
+class Lexer(object):
+    #--
+    lex = NotImplemented
+
+    def make_lexer_state(self, text):
+        line_ctr = LineCounter(b'\n' if isinstance(text, bytes) else '\n')
+        return LexerState(text, line_ctr)
+
+
+class TraditionalLexer(Lexer):
+
+    def __init__(self, conf):
+        terminals = list(conf.tokens)
+        assert all(isinstance(t, TerminalDef) for t in terminals), terminals
+
+        self.re = conf.re_module
+
+        if not conf.skip_validation:
+            ##
+
+            for t in terminals:
+                try:
+                    self.re.compile(t.pattern.to_regexp(), conf.g_regex_flags)
+                except self.re.error:
+                    raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern))
+
+                if t.pattern.min_width == 0:
+                    raise LexError("Lexer does not allow zero-width terminals. (%s: %s)" % (t.name, t.pattern))
+
+            assert set(conf.ignore) <= {t.name for t in terminals}
+
+        ##
+
+        self.newline_types = frozenset(t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp()))
+        self.ignore_types = frozenset(conf.ignore)
+
+        terminals.sort(key=lambda x: (-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name))
+        self.terminals = terminals
+        self.user_callbacks = conf.callbacks
+        self.g_regex_flags = conf.g_regex_flags
+        self.use_bytes = conf.use_bytes
+
+        self._mres = None
+
+    def _build(self):
+        terminals, self.callback = _create_unless(self.terminals, self.g_regex_flags, self.re, self.use_bytes)
+        assert all(self.callback.values())
+
+        for type_, f in self.user_callbacks.items():
+            if type_ in self.callback:
+                ##
+
+                self.callback[type_] = CallChain(self.callback[type_], f, lambda t: t.type == type_)
+            else:
+                self.callback[type_] = f
+
+        self._mres = build_mres(terminals, self.g_regex_flags, self.re, self.use_bytes)
+
+    @property
+    def mres(self):
+        if self._mres is None:
+            self._build()
+        return self._mres
+
+    def match(self, text, pos):
+        for mre, type_from_index in self.mres:
+            m = mre.match(text, pos)
+            if m:
+                return m.group(0), type_from_index[m.lastindex]
+
+    def lex(self, state, _parser_state):
+        with suppress(EOFError):
+            while True:
+                yield self.next_token(state)
+
+    def next_token(self, lex_state):
+        line_ctr = lex_state.line_ctr
+        while line_ctr.char_pos < len(lex_state.text):
+            res = self.match(lex_state.text, line_ctr.char_pos)
+            if not res:
+                allowed = {v for m, tfi in self.mres for v in tfi.values()} - self.ignore_types
+                if not allowed:
+                    allowed = {"<END-OF-FILE>"}
+                raise UnexpectedCharacters(lex_state.text, line_ctr.char_pos, line_ctr.line, line_ctr.column,
+                                           allowed=allowed, token_history=lex_state.last_token and [lex_state.last_token])
+
+            value, type_ = res
+
+            if type_ not in self.ignore_types:
+                t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
+                line_ctr.feed(value, type_ in self.newline_types)
+                t.end_line = line_ctr.line
+                t.end_column = line_ctr.column
+                t.end_pos = line_ctr.char_pos
+                if t.type in self.callback:
+                    t = self.callback[t.type](t)
+                    if not isinstance(t, Token):
+                        raise ValueError("Callbacks must return a token (returned %r)" % t)
+                lex_state.last_token = t
+                return t
+            else:
+                if type_ in self.callback:
+                    t2 = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
+                    self.callback[type_](t2)
+                line_ctr.feed(value, type_ in self.newline_types)
+
+        ##
+
+        raise EOFError(self)
+
+
+class LexerState:
+    __slots__ = 'text', 'line_ctr', 'last_token'
+
+    def __init__(self, text, line_ctr, last_token=None):
+        self.text = text
+        self.line_ctr = line_ctr
+        self.last_token = last_token
+
+    def __copy__(self):
+        return type(self)(self.text, copy(self.line_ctr), self.last_token)
+
+
+class ContextualLexer(Lexer):
+
+    def __init__(self, conf, states, always_accept=()):
+        terminals = list(conf.tokens)
+        tokens_by_name = {}
+        for t in terminals:
+            assert t.name not in tokens_by_name, t
+            tokens_by_name[t.name] = t
+
+        trad_conf = copy(conf)
+        trad_conf.tokens = terminals
+
+        lexer_by_tokens = {}
+        self.lexers = {}
+        for state, accepts in states.items():
+            key = frozenset(accepts)
+            try:
+                lexer = lexer_by_tokens[key]
+            except KeyError:
+                accepts = set(accepts) | set(conf.ignore) | set(always_accept)
+                state_tokens = [tokens_by_name[n] for n in accepts if n and n in tokens_by_name]
+                lexer_conf = copy(trad_conf)
+                lexer_conf.tokens = state_tokens
+                lexer = TraditionalLexer(lexer_conf)
+                lexer_by_tokens[key] = lexer
+
+            self.lexers[state] = lexer
+
+        assert trad_conf.tokens is terminals
+        self.root_lexer = TraditionalLexer(trad_conf)
+
+    def make_lexer_state(self, text):
+        return self.root_lexer.make_lexer_state(text)
+
+    def lex(self, lexer_state, parser_state):
+        try:
+            while True:
+                lexer = self.lexers[parser_state.position]
+                yield lexer.next_token(lexer_state)
+        except EOFError:
+            pass
+        except UnexpectedCharacters as e:
+            ##
+
+            ##
+
+            token = self.root_lexer.next_token(lexer_state)
+            raise UnexpectedToken(token, e.allowed, state=parser_state.position)
+
+
+class LexerThread:
+    #--
+
+    def __init__(self, lexer, text):
+        self.lexer = lexer
+        self.state = lexer.make_lexer_state(text)
+
+    def lex(self, parser_state):
+        return self.lexer.lex(self.state, parser_state)
+
+
+
+class LexerConf(Serialize):
+    __serialize_fields__ = 'tokens', 'ignore', 'g_regex_flags', 'use_bytes'
+    __serialize_namespace__ = TerminalDef,
+
+    def __init__(self, tokens, re_module, ignore=(), postlex=None, callbacks=None, g_regex_flags=0, skip_validation=False, use_bytes=False):
+        self.tokens = tokens    ##
+
+        self.ignore = ignore
+        self.postlex = postlex
+        self.callbacks = callbacks or {}
+        self.g_regex_flags = g_regex_flags
+        self.re_module = re_module
+        self.skip_validation = skip_validation
+        self.use_bytes = use_bytes
+
+
+from functools import partial, wraps
+from itertools import repeat, product
+
+
+class ExpandSingleChild:
+    def __init__(self, node_builder):
+        self.node_builder = node_builder
+
+    def __call__(self, children):
+        if len(children) == 1:
+            return children[0]
+        else:
+            return self.node_builder(children)
+
+
+class PropagatePositions:
+    def __init__(self, node_builder):
+        self.node_builder = node_builder
+
+    def __call__(self, children):
+        res = self.node_builder(children)
+
+        ##
+
+        if isinstance(res, Tree):
+            res_meta = res.meta
+            for c in children:
+                if isinstance(c, Tree):
+                    child_meta = c.meta
+                    if not child_meta.empty:
+                        res_meta.line = child_meta.line
+                        res_meta.column = child_meta.column
+                        res_meta.start_pos = child_meta.start_pos
+                        res_meta.empty = False
+                        break
+                elif isinstance(c, Token):
+                    res_meta.line = c.line
+                    res_meta.column = c.column
+                    res_meta.start_pos = c.pos_in_stream
+                    res_meta.empty = False
+                    break
+
+            for c in reversed(children):
+                if isinstance(c, Tree):
+                    child_meta = c.meta
+                    if not child_meta.empty:
+                        res_meta.end_line = child_meta.end_line
+                        res_meta.end_column = child_meta.end_column
+                        res_meta.end_pos = child_meta.end_pos
+                        res_meta.empty = False
+                        break
+                elif isinstance(c, Token):
+                    res_meta.end_line = c.end_line
+                    res_meta.end_column = c.end_column
+                    res_meta.end_pos = c.end_pos
+                    res_meta.empty = False
+                    break
+
+        return res
+
+
+class ChildFilter:
+    def __init__(self, to_include, append_none, node_builder):
+        self.node_builder = node_builder
+        self.to_include = to_include
+        self.append_none = append_none
+
+    def __call__(self, children):
+        filtered = []
+
+        for i, to_expand, add_none in self.to_include:
+            if add_none:
+                filtered += [None] * add_none
+            if to_expand:
+                filtered += children[i].children
+            else:
+                filtered.append(children[i])
+
+        if self.append_none:
+            filtered += [None] * self.append_none
+
+        return self.node_builder(filtered)
+
+
+class ChildFilterLALR(ChildFilter):
+    #--
+
+    def __call__(self, children):
+        filtered = []
+        for i, to_expand, add_none in self.to_include:
+            if add_none:
+                filtered += [None] * add_none
+            if to_expand:
+                if filtered:
+                    filtered += children[i].children
+                else:   ##
+
+                    filtered = children[i].children
+            else:
+                filtered.append(children[i])
+
+        if self.append_none:
+            filtered += [None] * self.append_none
+
+        return self.node_builder(filtered)
+
+
+class ChildFilterLALR_NoPlaceholders(ChildFilter):
+    #--
+    def __init__(self, to_include, node_builder):
+        self.node_builder = node_builder
+        self.to_include = to_include
+
+    def __call__(self, children):
+        filtered = []
+        for i, to_expand in self.to_include:
+            if to_expand:
+                if filtered:
+                    filtered += children[i].children
+                else:   ##
+
+                    filtered = children[i].children
+            else:
+                filtered.append(children[i])
+        return self.node_builder(filtered)
+
+
+def _should_expand(sym):
+    return not sym.is_term and sym.name.startswith('_')
+
+
+def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices):
+    ##
+
+    if _empty_indices:
+        assert _empty_indices.count(False) == len(expansion)
+        s = ''.join(str(int(b)) for b in _empty_indices)
+        empty_indices = [len(ones) for ones in s.split('0')]
+        assert len(empty_indices) == len(expansion)+1, (empty_indices, len(expansion))
+    else:
+        empty_indices = [0] * (len(expansion)+1)
+
+    to_include = []
+    nones_to_add = 0
+    for i, sym in enumerate(expansion):
+        nones_to_add += empty_indices[i]
+        if keep_all_tokens or not (sym.is_term and sym.filter_out):
+            to_include.append((i, _should_expand(sym), nones_to_add))
+            nones_to_add = 0
+
+    nones_to_add += empty_indices[len(expansion)]
+
+    if _empty_indices or len(to_include) < len(expansion) or any(to_expand for i, to_expand,_ in to_include):
+        if _empty_indices or ambiguous:
+            return partial(ChildFilter if ambiguous else ChildFilterLALR, to_include, nones_to_add)
+        else:
+            ##
+
+            return partial(ChildFilterLALR_NoPlaceholders, [(i, x) for i,x,_ in to_include])
+
+
+class AmbiguousExpander:
+    #--
+    def __init__(self, to_expand, tree_class, node_builder):
+        self.node_builder = node_builder
+        self.tree_class = tree_class
+        self.to_expand = to_expand
+
+    def __call__(self, children):
+        def _is_ambig_tree(t):
+            return hasattr(t, 'data') and t.data == '_ambig'
+
+        ##
+
+        ##
+
+        ##
+
+        ##
+
+        ambiguous = []
+        for i, child in enumerate(children):
+            if _is_ambig_tree(child):
+                if i in self.to_expand:
+                    ambiguous.append(i)
+
+                to_expand = [j for j, grandchild in enumerate(child.children) if _is_ambig_tree(grandchild)]
+                child.expand_kids_by_index(*to_expand)
+
+        if not ambiguous:
+            return self.node_builder(children)
+
+        expand = [iter(child.children) if i in ambiguous else repeat(child) for i, child in enumerate(children)]
+        return self.tree_class('_ambig', [self.node_builder(list(f[0])) for f in product(zip(*expand))])
+
+
+def maybe_create_ambiguous_expander(tree_class, expansion, keep_all_tokens):
+    to_expand = [i for i, sym in enumerate(expansion)
+                 if keep_all_tokens or ((not (sym.is_term and sym.filter_out)) and _should_expand(sym))]
+    if to_expand:
+        return partial(AmbiguousExpander, to_expand, tree_class)
+
+
+class AmbiguousIntermediateExpander:
+    #--
+
+    def __init__(self, tree_class, node_builder):
+        self.node_builder = node_builder
+        self.tree_class = tree_class
+
+    def __call__(self, children):
+        def _is_iambig_tree(child):
+            return hasattr(child, 'data') and child.data == '_iambig'
+
+        def _collapse_iambig(children):
+            #--
+
+            ##
+
+            ##
+
+            if children and _is_iambig_tree(children[0]):
+                iambig_node = children[0]
+                result = []
+                for grandchild in iambig_node.children:
+                    collapsed = _collapse_iambig(grandchild.children)
+                    if collapsed:
+                        for child in collapsed:
+                            child.children += children[1:]
+                        result += collapsed
+                    else:
+                        new_tree = self.tree_class('_inter', grandchild.children + children[1:])
+                        result.append(new_tree)
+                return result
+
+        collapsed = _collapse_iambig(children)
+        if collapsed:
+            processed_nodes = [self.node_builder(c.children) for c in collapsed]
+            return self.tree_class('_ambig', processed_nodes)
+
+        return self.node_builder(children)
+
+
+def ptb_inline_args(func):
+    @wraps(func)
+    def f(children):
+        return func(*children)
+    return f
+
+
+def inplace_transformer(func):
+    @wraps(func)
+    def f(children):
+        ##
+
+        tree = Tree(func.__name__, children)
+        return func(tree)
+    return f
+
+
+def apply_visit_wrapper(func, name, wrapper):
+    if wrapper is _vargs_meta or wrapper is _vargs_meta_inline:
+        raise NotImplementedError("Meta args not supported for internal transformer")
+
+    @wraps(func)
+    def f(children):
+        return wrapper(func, name, children, None)
+    return f
+
+
+class ParseTreeBuilder:
+    def __init__(self, rules, tree_class, propagate_positions=False, ambiguous=False, maybe_placeholders=False):
+        self.tree_class = tree_class
+        self.propagate_positions = propagate_positions
+        self.ambiguous = ambiguous
+        self.maybe_placeholders = maybe_placeholders
+
+        self.rule_builders = list(self._init_builders(rules))
+
+    def _init_builders(self, rules):
+        for rule in rules:
+            options = rule.options
+            keep_all_tokens = options.keep_all_tokens
+            expand_single_child = options.expand1
+
+            wrapper_chain = list(filter(None, [
+                (expand_single_child and not rule.alias) and ExpandSingleChild,
+                maybe_create_child_filter(rule.expansion, keep_all_tokens, self.ambiguous, options.empty_indices if self.maybe_placeholders else None),
+                self.propagate_positions and PropagatePositions,
+                self.ambiguous and maybe_create_ambiguous_expander(self.tree_class, rule.expansion, keep_all_tokens),
+                self.ambiguous and partial(AmbiguousIntermediateExpander, self.tree_class)
+            ]))
+
+            yield rule, wrapper_chain
+
+    def create_callback(self, transformer=None):
+        callbacks = {}
+
+        for rule, wrapper_chain in self.rule_builders:
+
+            user_callback_name = rule.alias or rule.options.template_source or rule.origin.name
+            try:
+                f = getattr(transformer, user_callback_name)
+                ##
+
+                wrapper = getattr(f, 'visit_wrapper', None)
+                if wrapper is not None:
+                    f = apply_visit_wrapper(f, user_callback_name, wrapper)
+                else:
+                    if isinstance(transformer, InlineTransformer):
+                        f = ptb_inline_args(f)
+                    elif isinstance(transformer, Transformer_InPlace):
+                        f = inplace_transformer(f)
+            except AttributeError:
+                f = partial(self.tree_class, user_callback_name)
+
+            for w in wrapper_chain:
+                f = w(f)
+
+            if rule in callbacks:
+                raise GrammarError("Rule '%s' already exists" % (rule,))
+
+            callbacks[rule] = f
+
+        return callbacks
+
+
+
+class LALR_Parser(object):
+    def __init__(self, parser_conf, debug=False):
+        analysis = LALR_Analyzer(parser_conf, debug=debug)
+        analysis.compute_lalr()
+        callbacks = parser_conf.callbacks
+
+        self._parse_table = analysis.parse_table
+        self.parser_conf = parser_conf
+        self.parser = _Parser(analysis.parse_table, callbacks, debug)
+
+    @classmethod
+    def deserialize(cls, data, memo, callbacks, debug=False):
+        inst = cls.__new__(cls)
+        inst._parse_table = IntParseTable.deserialize(data, memo)
+        inst.parser = _Parser(inst._parse_table, callbacks, debug)
+        return inst
+
+    def serialize(self, memo):
+        return self._parse_table.serialize(memo)
+
+    def parse(self, *args):
+        return self.parser.parse(*args)
+
+
+class ParseConf:
+    __slots__ = 'parse_table', 'callbacks', 'start', 'start_state', 'end_state', 'states'
+
+    def __init__(self, parse_table, callbacks, start):
+        self.parse_table = parse_table
+
+        self.start_state = self.parse_table.start_states[start]
+        self.end_state = self.parse_table.end_states[start]
+        self.states = self.parse_table.states
+
+        self.callbacks = callbacks
+        self.start = start
+
+
+class ParserState:
+    __slots__ = 'parse_conf', 'lexer', 'state_stack', 'value_stack'
+
+    def __init__(self, parse_conf, lexer, state_stack=None, value_stack=None):
+        self.parse_conf = parse_conf
+        self.lexer = lexer
+        self.state_stack = state_stack or [self.parse_conf.start_state]
+        self.value_stack = value_stack or []
+
+    @property
+    def position(self):
+        return self.state_stack[-1]
+
+    def __copy__(self):
+        return type(self)(
+            self.parse_conf,
+            self.lexer, ##
+
+            copy(self.state_stack),
+            deepcopy(self.value_stack),
+        )
+
+    def copy(self):
+        return copy(self)
+
+    def feed_token(self, token, is_end=False):
+        state_stack = self.state_stack
+        value_stack = self.value_stack
+        states = self.parse_conf.states
+        end_state = self.parse_conf.end_state
+        callbacks = self.parse_conf.callbacks
+
+        while True:
+            state = state_stack[-1]
+            try:
+                action, arg = states[state][token.type]
+            except KeyError:
+                expected = {s for s in states[state].keys() if s.isupper()}
+                raise UnexpectedToken(token, expected, state=state, puppet=None)
+
+            assert arg != end_state
+
+            if action is Shift:
+                ##
+
+                assert not is_end
+                state_stack.append(arg)
+                value_stack.append(token)
+                return
+            else:
+                ##
+
+                rule = arg
+                size = len(rule.expansion)
+                if size:
+                    s = value_stack[-size:]
+                    del state_stack[-size:]
+                    del value_stack[-size:]
+                else:
+                    s = []
+
+                value = callbacks[rule](s)
+
+                _action, new_state = states[state_stack[-1]][rule.origin.name]
+                assert _action is Shift
+                state_stack.append(new_state)
+                value_stack.append(value)
+
+                if is_end and state_stack[-1] == end_state:
+                    return value_stack[-1]
+
+class _Parser:
+    def __init__(self, parse_table, callbacks, debug=False):
+        self.parse_table = parse_table
+        self.callbacks = callbacks
+        self.debug = debug
+
+    def parse(self, lexer, start, value_stack=None, state_stack=None):
+        parse_conf = ParseConf(self.parse_table, self.callbacks, start)
+        parser_state = ParserState(parse_conf, lexer, state_stack, value_stack)
+        return self.parse_from_state(parser_state)
+
+    def parse_from_state(self, state):
+        ##
+
+        try:
+            token = None
+            for token in state.lexer.lex(state):
+                state.feed_token(token)
+
+            token = Token.new_borrow_pos('$END', '', token) if token else Token('$END', '', 0, 1, 1)
+            return state.feed_token(token, True)
+        except UnexpectedInput as e:
+            try:
+                e.puppet = ParserPuppet(self, state, state.lexer)
+            except NameError:
+                pass
+            raise e
+        except Exception as e:
+            if self.debug:
+                print("")
+                print("STATE STACK DUMP")
+                print("----------------")
+                for i, s in enumerate(state.state_stack):
+                    print('%d)' % i , s)
+                print("")
+
+            raise
+
+
+class Action:
+    def __init__(self, name):
+        self.name = name
+    def __str__(self):
+        return self.name
+    def __repr__(self):
+        return str(self)
+
+Shift = Action('Shift')
+Reduce = Action('Reduce')
+
+
+class ParseTable:
+    def __init__(self, states, start_states, end_states):
+        self.states = states
+        self.start_states = start_states
+        self.end_states = end_states
+
+    def serialize(self, memo):
+        tokens = Enumerator()
+        rules = Enumerator()
+
+        states = {
+            state: {tokens.get(token): ((1, arg.serialize(memo)) if action is Reduce else (0, arg))
+                    for token, (action, arg) in actions.items()}
+            for state, actions in self.states.items()
+        }
+
+        return {
+            'tokens': tokens.reversed(),
+            'states': states,
+            'start_states': self.start_states,
+            'end_states': self.end_states,
+        }
+
+    @classmethod
+    def deserialize(cls, data, memo):
+        tokens = data['tokens']
+        states = {
+            state: {tokens[token]: ((Reduce, Rule.deserialize(arg, memo)) if action==1 else (Shift, arg))
+                    for token, (action, arg) in actions.items()}
+            for state, actions in data['states'].items()
+        }
+        return cls(states, data['start_states'], data['end_states'])
+
+
+class IntParseTable(ParseTable):
+
+    @classmethod
+    def from_ParseTable(cls, parse_table):
+        enum = list(parse_table.states)
+        state_to_idx = {s:i for i,s in enumerate(enum)}
+        int_states = {}
+
+        for s, la in parse_table.states.items():
+            la = {k:(v[0], state_to_idx[v[1]]) if v[0] is Shift else v
+                  for k,v in la.items()}
+            int_states[ state_to_idx[s] ] = la
+
+
+        start_states = {start:state_to_idx[s] for start, s in parse_table.start_states.items()}
+        end_states = {start:state_to_idx[s] for start, s in parse_table.end_states.items()}
+        return cls(int_states, start_states, end_states)
+
+
+
+def get_frontend(parser, lexer):
+    if parser=='lalr':
+        if lexer is None:
+            raise ValueError('The LALR parser requires use of a lexer')
+        elif lexer == 'standard':
+            return LALR_TraditionalLexer
+        elif lexer == 'contextual':
+            return LALR_ContextualLexer
+        elif issubclass(lexer, Lexer):
+            class CustomLexerWrapper(Lexer):
+                def __init__(self, lexer_conf):
+                    self.lexer = lexer(lexer_conf)
+                def lex(self, lexer_state, parser_state):
+                    return self.lexer.lex(lexer_state.text)
+
+            class LALR_CustomLexerWrapper(LALR_WithLexer):
+                def __init__(self, lexer_conf, parser_conf, options=None):
+                    super(LALR_CustomLexerWrapper, self).__init__(lexer_conf, parser_conf, options=options)
+                def init_lexer(self):
+                    future_interface = getattr(lexer, '__future_interface__', False)
+                    if future_interface:
+                        self.lexer = lexer(self.lexer_conf)
+                    else:
+                        self.lexer = CustomLexerWrapper(self.lexer_conf)
+
+            return LALR_CustomLexerWrapper
+        else:
+            raise ValueError('Unknown lexer: %s' % lexer)
+    elif parser=='earley':
+        if lexer=='standard':
+            return Earley
+        elif lexer=='dynamic':
+            return XEarley
+        elif lexer=='dynamic_complete':
+            return XEarley_CompleteLex
+        elif lexer=='contextual':
+            raise ValueError('The Earley parser does not support the contextual parser')
+        else:
+            raise ValueError('Unknown lexer: %s' % lexer)
+    elif parser == 'cyk':
+        if lexer == 'standard':
+            return CYK
+        else:
+            raise ValueError('CYK parser requires using standard parser.')
+    else:
+        raise ValueError('Unknown parser: %s' % parser)
+
+
+class _ParserFrontend(Serialize):
+    def _parse(self, start, input, *args):
+        if start is None:
+            start = self.start
+            if len(start) > 1:
+                raise ValueError("Lark initialized with more than 1 possible start rule. Must specify which start rule to parse", start)
+            start ,= start
+        return self.parser.parse(input, start, *args)
+
+
+def _get_lexer_callbacks(transformer, terminals):
+    result = {}
+    for terminal in terminals:
+        callback = getattr(transformer, terminal.name, None)
+        if callback is not None:
+            result[terminal.name] = callback
+    return result
+
+class PostLexConnector:
+    def __init__(self, lexer, postlexer):
+        self.lexer = lexer
+        self.postlexer = postlexer
+
+    def make_lexer_state(self, text):
+        return self.lexer.make_lexer_state(text)
+
+    def lex(self, lexer_state, parser_state):
+        i = self.lexer.lex(lexer_state, parser_state)
+        return self.postlexer.process(i)
+
+
+class WithLexer(_ParserFrontend):
+    lexer = None
+    parser = None
+    lexer_conf = None
+    start = None
+
+    __serialize_fields__ = 'parser', 'lexer_conf', 'start'
+    __serialize_namespace__ = LexerConf,
+
+    def __init__(self, lexer_conf, parser_conf, options=None):
+        self.lexer_conf = lexer_conf
+        self.start = parser_conf.start
+        self.postlex = lexer_conf.postlex
+
+    @classmethod
+    def deserialize(cls, data, memo, callbacks, options):
+        inst = super(WithLexer, cls).deserialize(data, memo)
+
+        inst.postlex = options.postlex
+        inst.parser = LALR_Parser.deserialize(inst.parser, memo, callbacks, options.debug)
+
+        terminals = [item for item in memo.values() if isinstance(item, TerminalDef)]
+        inst.lexer_conf.callbacks = _get_lexer_callbacks(options.transformer, terminals)
+        inst.lexer_conf.re_module = regex if options.regex else re
+        inst.lexer_conf.use_bytes = options.use_bytes
+        inst.lexer_conf.g_regex_flags = options.g_regex_flags
+        inst.lexer_conf.skip_validation = True
+        inst.init_lexer()
+
+        return inst
+
+    def _serialize(self, data, memo):
+        data['parser'] = data['parser'].serialize(memo)
+
+    def make_lexer(self, text):
+        lexer = self.lexer
+        if self.postlex:
+            lexer = PostLexConnector(self.lexer, self.postlex)
+        return LexerThread(lexer, text)
+
+    def parse(self, text, start=None):
+        return self._parse(start, self.make_lexer(text))
+
+    def init_traditional_lexer(self):
+        self.lexer = TraditionalLexer(self.lexer_conf)
+
+class LALR_WithLexer(WithLexer):
+    def __init__(self, lexer_conf, parser_conf, options=None):
+        debug = options.debug if options else False
+        self.parser = LALR_Parser(parser_conf, debug=debug)
+        WithLexer.__init__(self, lexer_conf, parser_conf, options)
+
+        self.init_lexer()
+
+    def init_lexer(self, **kw):
+        raise NotImplementedError()
+
+class LALR_TraditionalLexer(LALR_WithLexer):
+    def init_lexer(self):
+        self.init_traditional_lexer()
+
+class LALR_ContextualLexer(LALR_WithLexer):
+    def init_lexer(self):
+        states = {idx:list(t.keys()) for idx, t in self.parser._parse_table.states.items()}
+        always_accept = self.postlex.always_accept if self.postlex else ()
+        self.lexer = ContextualLexer(self.lexer_conf, states, always_accept=always_accept)
+
+
+
+
+class LarkOptions(Serialize):
+    #--
+    OPTIONS_DOC = """
+    **===  General Options  ===**
+
+    start
+            The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start")
+    debug
+            Display debug information and extra warnings. Use only when debugging (default: False)
+            When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed.
+    transformer
+            Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster)
+    propagate_positions
+            Propagates (line, column, end_line, end_column) attributes into all tree branches.
+    maybe_placeholders
+            When True, the ``[]`` operator returns ``None`` when not matched.
+
+            When ``False``,  ``[]`` behaves like the ``?`` operator, and returns no value at all.
+            (default= ``False``. Recommended to set to ``True``)
+    cache
+            Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now.
+
+            - When ``False``, does nothing (default)
+            - When ``True``, caches to a temporary file in the local directory
+            - When given a string, caches to the path pointed by the string
+    regex
+            When True, uses the ``regex`` module instead of the stdlib ``re``.
+    g_regex_flags
+            Flags that are applied to all terminals (both regex and strings)
+    keep_all_tokens
+            Prevent the tree builder from automagically removing "punctuation" tokens (default: False)
+    tree_class
+            Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``.
+
+    **=== Algorithm Options ===**
+
+    parser
+            Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley").
+            (there is also a "cyk" option for legacy)
+    lexer
+            Decides whether or not to use a lexer stage
+
+            - "auto" (default): Choose for me based on the parser
+            - "standard": Use a standard lexer
+            - "contextual": Stronger lexer (only works with parser="lalr")
+            - "dynamic": Flexible and powerful (only with parser="earley")
+            - "dynamic_complete": Same as dynamic, but tries *every* variation of tokenizing possible.
+    ambiguity
+            Decides how to handle ambiguity in the parse. Only relevant if parser="earley"
+
+            - "resolve": The parser will automatically choose the simplest derivation
+              (it chooses consistently: greedy for tokens, non-greedy for rules)
+            - "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
+            - "forest": The parser will return the root of the shared packed parse forest.
+
+    **=== Misc. / Domain Specific Options ===**
+
+    postlex
+            Lexer post-processing (Default: None) Only works with the standard and contextual lexers.
+    priority
+            How priorities should be evaluated - auto, none, normal, invert (Default: auto)
+    lexer_callbacks
+            Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
+    use_bytes
+            Accept an input of type ``bytes`` instead of ``str`` (Python 3 only).
+    edit_terminals
+            A callback for editing the terminals before parse.
+    import_paths
+            A List of either paths or loader functions to specify from where grammars are imported
+    source_path
+            Override the source of from where the grammar was loaded. Useful for relative imports and unconventional grammar loading
+
+    **=== End Options ===**
+    """
+    if __doc__:
+        __doc__ += OPTIONS_DOC
+
+
+    ##
+
+    ##
+
+    ##
+
+    ##
+
+    ##
+
+    ##
+
+    ##
+
+    ##
+
+    _defaults = {
+        'debug': False,
+        'keep_all_tokens': False,
+        'tree_class': None,
+        'cache': False,
+        'postlex': None,
+        'parser': 'earley',
+        'lexer': 'auto',
+        'transformer': None,
+        'start': 'start',
+        'priority': 'auto',
+        'ambiguity': 'auto',
+        'regex': False,
+        'propagate_positions': False,
+        'lexer_callbacks': {},
+        'maybe_placeholders': False,
+        'edit_terminals': None,
+        'g_regex_flags': 0,
+        'use_bytes': False,
+        'import_paths': [],
+        'source_path': None,
+    }
+
+    def __init__(self, options_dict):
+        o = dict(options_dict)
+
+        options = {}
+        for name, default in self._defaults.items():
+            if name in o:
+                value = o.pop(name)
+                if isinstance(default, bool) and name not in ('cache', 'use_bytes'):
+                    value = bool(value)
+            else:
+                value = default
+
+            options[name] = value
+
+        if isinstance(options['start'], STRING_TYPE):
+            options['start'] = [options['start']]
+
+        self.__dict__['options'] = options
+
+        assert self.parser in ('earley', 'lalr', 'cyk', None)
+
+        if self.parser == 'earley' and self.transformer:
+            raise ValueError('Cannot specify an embedded transformer when using the Earley algorithm.'
+                             'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)')
+
+        if o:
+            raise ValueError("Unknown options: %s" % o.keys())
+
+    def __getattr__(self, name):
+        try:
+            return self.options[name]
+        except KeyError as e:
+            raise AttributeError(e)
+
+    def __setattr__(self, name, value):
+        assert name in self.options
+        self.options[name] = value
+
+    def serialize(self, memo):
+        return self.options
+
+    @classmethod
+    def deserialize(cls, data, memo):
+        return cls(data)
+
+
+##
+
+##
+
+_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'}
+
+_VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None)
+_VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest')
+
+
+class Lark(Serialize):
+    #--
+    def __init__(self, grammar, **options):
+        self.options = LarkOptions(options)
+
+        ##
+
+        use_regex = self.options.regex
+        if use_regex:
+            if regex:
+                re_module = regex
+            else:
+                raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.')
+        else:
+            re_module = re
+
+        ##
+
+        if self.options.source_path is None:
+            try:
+                self.source_path = grammar.name
+            except AttributeError:
+                self.source_path = '<string>'
+        else:
+            self.source_path = self.options.source_path
+
+        ##
+
+        try:
+            read = grammar.read
+        except AttributeError:
+            pass
+        else:
+            grammar = read()
+
+        assert isinstance(grammar, STRING_TYPE)
+        self.source_grammar = grammar
+        if self.options.use_bytes:
+            if not isascii(grammar):
+                raise ValueError("Grammar must be ascii only, when use_bytes=True")
+
+        cache_fn = None
+        if self.options.cache:
+            if self.options.parser != 'lalr':
+                raise NotImplementedError("cache only works with parser='lalr' for now")
+            if isinstance(self.options.cache, STRING_TYPE):
+                cache_fn = self.options.cache
+            else:
+                if self.options.cache is not True:
+                    raise ValueError("cache argument must be bool or str")
+                unhashable = ('transformer', 'postlex', 'lexer_callbacks', 'edit_terminals')
+                from . import __version__
+                options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable)
+                s = grammar + options_str + __version__
+                md5 = hashlib.md5(s.encode()).hexdigest()
+                cache_fn = tempfile.gettempdir() + '/.lark_cache_%s.tmp' % md5
+
+            if FS.exists(cache_fn):
+                logger.debug('Loading grammar from cache: %s', cache_fn)
+                ##
+
+                for name in (set(options) - _LOAD_ALLOWED_OPTIONS):
+                    del options[name]
+                with FS.open(cache_fn, 'rb') as f:
+                    self._load(f, **options)
+                return
+
+        if self.options.lexer == 'auto':
+            if self.options.parser == 'lalr':
+                self.options.lexer = 'contextual'
+            elif self.options.parser == 'earley':
+                self.options.lexer = 'dynamic'
+            elif self.options.parser == 'cyk':
+                self.options.lexer = 'standard'
+            else:
+                assert False, self.options.parser
+        lexer = self.options.lexer
+        assert lexer in ('standard', 'contextual', 'dynamic', 'dynamic_complete') or issubclass(lexer, Lexer)
+
+        if self.options.ambiguity == 'auto':
+            if self.options.parser == 'earley':
+                self.options.ambiguity = 'resolve'
+        else:
+            disambig_parsers = ['earley', 'cyk']
+            assert self.options.parser in disambig_parsers, (
+                'Only %s supports disambiguation right now') % ', '.join(disambig_parsers)
+
+        if self.options.priority == 'auto':
+            self.options.priority = 'normal'
+
+        if self.options.priority not in _VALID_PRIORITY_OPTIONS:
+            raise ValueError("invalid priority option: %r. Must be one of %r" % (self.options.priority, _VALID_PRIORITY_OPTIONS))
+        assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"'
+        if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS:
+            raise ValueError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS))
+
+        ##
+
+        self.grammar = load_grammar(grammar, self.source_path, self.options.import_paths, self.options.keep_all_tokens)
+
+        if self.options.postlex is not None:
+            terminals_to_keep = set(self.options.postlex.always_accept)
+        else:
+            terminals_to_keep = set()
+
+        ##
+
+        self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start, terminals_to_keep)
+
+        if self.options.edit_terminals:
+            for t in self.terminals:
+                self.options.edit_terminals(t)
+
+        self._terminals_dict = {t.name: t for t in self.terminals}
+
+        ##
+
+        ##
+
+        if self.options.priority == 'invert':
+            for rule in self.rules:
+                if rule.options.priority is not None:
+                    rule.options.priority = -rule.options.priority
+        ##
+
+        ##
+
+        ##
+
+        elif self.options.priority is None:
+            for rule in self.rules:
+                if rule.options.priority is not None:
+                    rule.options.priority = None
+
+        ##
+
+        lexer_callbacks = (_get_lexer_callbacks(self.options.transformer, self.terminals)
+                           if self.options.transformer
+                           else {})
+        lexer_callbacks.update(self.options.lexer_callbacks)
+
+        self.lexer_conf = LexerConf(self.terminals, re_module, self.ignore_tokens, self.options.postlex, lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes)
+
+        if self.options.parser:
+            self.parser = self._build_parser()
+        elif lexer:
+            self.lexer = self._build_lexer()
+
+        if cache_fn:
+            logger.debug('Saving grammar to cache: %s', cache_fn)
+            with FS.open(cache_fn, 'wb') as f:
+                self.save(f)
+
+    if __doc__:
+        __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC
+
+    __serialize_fields__ = 'parser', 'rules', 'options'
+
+    def _build_lexer(self):
+        return TraditionalLexer(self.lexer_conf)
+
+    def _prepare_callbacks(self):
+        self.parser_class = get_frontend(self.options.parser, self.options.lexer)
+        self._callbacks = None
+        ##
+
+        if self.options.ambiguity != 'forest':
+            self._parse_tree_builder = ParseTreeBuilder(
+                    self.rules,
+                    self.options.tree_class or Tree,
+                    self.options.propagate_positions,
+                    self.options.parser != 'lalr' and self.options.ambiguity == 'explicit',
+                    self.options.maybe_placeholders
+                )
+            self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
+
+    def _build_parser(self):
+        self._prepare_callbacks()
+        parser_conf = ParserConf(self.rules, self._callbacks, self.options.start)
+        return self.parser_class(self.lexer_conf, parser_conf, options=self.options)
+
+    def save(self, f):
+        #--
+        data, m = self.memo_serialize([TerminalDef, Rule])
+        pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+    @classmethod
+    def load(cls, f):
+        #--
+        inst = cls.__new__(cls)
+        return inst._load(f)
+
+    def _load(self, f, **kwargs):
+        if isinstance(f, dict):
+            d = f
+        else:
+            d = pickle.load(f)
+        memo = d['memo']
+        data = d['data']
+
+        assert memo
+        memo = SerializeMemoizer.deserialize(memo, {'Rule': Rule, 'TerminalDef': TerminalDef}, {})
+        options = dict(data['options'])
+        if (set(kwargs) - _LOAD_ALLOWED_OPTIONS) & set(LarkOptions._defaults):
+            raise ValueError("Some options are not allowed when loading a Parser: {}"
+                             .format(set(kwargs) - _LOAD_ALLOWED_OPTIONS))
+        options.update(kwargs)
+        self.options = LarkOptions.deserialize(options, memo)
+        self.rules = [Rule.deserialize(r, memo) for r in data['rules']]
+        self.source_path = '<deserialized>'
+        self._prepare_callbacks()
+        self.parser = self.parser_class.deserialize(
+            data['parser'],
+            memo,
+            self._callbacks,
+            self.options,  ##
+
+        )
+        self.terminals = self.parser.lexer_conf.tokens
+        self._terminals_dict = {t.name: t for t in self.terminals}
+        return self
+
+    @classmethod
+    def _load_from_dict(cls, data, memo, **kwargs):
+        inst = cls.__new__(cls)
+        return inst._load({'data': data, 'memo': memo}, **kwargs)
+
+    @classmethod
+    def open(cls, grammar_filename, rel_to=None, **options):
+        #--
+        if rel_to:
+            basepath = os.path.dirname(rel_to)
+            grammar_filename = os.path.join(basepath, grammar_filename)
+        with open(grammar_filename, encoding='utf8') as f:
+            return cls(f, **options)
+
+    @classmethod
+    def open_from_package(cls, package, grammar_path, search_paths=("",), **options):
+        #--
+        package = FromPackageLoader(package, search_paths)
+        full_path, text = package(None, grammar_path)
+        options.setdefault('source_path', full_path)
+        options.setdefault('import_paths', [])
+        options['import_paths'].append(package)
+        return cls(text, **options)
+
+    def __repr__(self):
+        return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source_path, self.options.parser, self.options.lexer)
+
+
+    def lex(self, text):
+        #--
+        if not hasattr(self, 'lexer'):
+            self.lexer = self._build_lexer()
+        stream = self.lexer.lex(text)
+        if self.options.postlex:
+            return self.options.postlex.process(stream)
+        return stream
+
+    def get_terminal(self, name):
+        #--
+        return self._terminals_dict[name]
+
+    def parse(self, text, start=None, on_error=None):
+        #--
+
+        try:
+            return self.parser.parse(text, start=start)
+        except UnexpectedInput as e:
+            if on_error is None:
+                raise
+
+            while True:
+                if isinstance(e, UnexpectedCharacters):
+                    s = e.puppet.lexer_state.state
+                    p = s.line_ctr.char_pos
+
+                if not on_error(e):
+                    raise e
+
+                if isinstance(e, UnexpectedCharacters):
+                    ##
+
+                    if p == s.line_ctr.char_pos:
+                        s.line_ctr.feed(s.text[p:p+1])
+
+                try:
+                    return e.puppet.resume_parse()
+                except UnexpectedToken as e2:
+                    if isinstance(e, UnexpectedToken) and e.token.type == e2.token.type == '$END' and e.puppet == e2.puppet:
+                        ##
+
+                        raise e2
+                    e = e2
+                except UnexpectedCharacters as e2:
+                    e = e2
+
+    @property
+    def source(self):
+        warn("Lark.source attribute has been renamed to Lark.source_path", DeprecationWarning)
+        return self.source_path
+
+    @source.setter
+    def source(self, value):
+        self.source_path = value
+
+    @property
+    def grammar_source(self):
+        warn("Lark.grammar_source attribute has been renamed to Lark.source_grammar", DeprecationWarning)
+        return self.source_grammar
+
+    @grammar_source.setter
+    def grammar_source(self, value):
+        self.source_grammar = value
+
+
+
+import pickle, zlib, base64
+DATA = (
+{'parser': {'parser': {'tokens': {0: 'RSQB', 1: 'COMMA', 2: 'option_parm', 3: 'option_highlevel', 4: 'VAR', 5: 'record_highlevel', 6: 'predefined_typestr', 7: 'LPAR', 8: 'LSQB', 9: 'record_tuple', 10: 'UNKNOWN', 11: 'HARDCODED', 12: 'UNQUOTED_STRING', 13: 'listtype', 14: 'input', 15: 'list_single', 16: 'UNION', 17: 'STRUCT', 18: 'option_single', 19: 'QMARK', 20: 'regular_outparm', 21: 'union_parm', 22: 'regular_inparm', 23: 'LBRACE', 24: 'uniontype', 25: 'TUPLE', 26: 'TYPE', 27: 'record', 28: 'unknown', 29: 'list_parm', 30: 'optiontype', 31: 'union_single', 32: 'CATEGORICAL', 33: 'record_dict', 34: 'categories', 35: 'regular', 36: 'number', 37: 'OPTION', 38: 'record_tuple_param', 39: 'record_struct', 40: 'SIGNED_NUMBER', 41: 'primitive', 42: 'PARAMETERS', 43: 'def_option', 44: '$END', 45: 'RBRACE', 46: 'RPAR', 47: 'STAR', 48: '__union_single_star_0', 49: 'start', 50: '__record_struct_star_2', 51: 'COLON', 52: '__ANON_1', 53: 'string', 54: 'options', 55: '__record_dict_star_1', 56: 'TRUE', 57: 'NULL', 58: 'list_obj', 59: 'FALSE', 60: 'dict_obj', 61: 'json', 62: '__list_obj_star_3', 63: 'pair', 64: '__ANON_0', 65: '__dict_obj_star_4', 66: 'EQUAL'}, 'states': {0: {0: (1, {'@': 29}), 1: (1, {'@': 29})}, 1: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 14: (0, 163), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 2: {1: (0, 3)}, 3: {42: (0, 193), 43: (0, 113)}, 4: {0: (1, {'@': 30}), 1: (1, {'@': 30}), 44: (1, {'@': 30}), 45: (1, {'@': 30}), 8: (1, {'@': 30}), 46: (1, {'@': 30})}, 5: {47: (0, 182)}, 6: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 14: (0, 13), 19: (0, 142), 18: (0, 143), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 7: {8: (0, 101)}, 8: {0: (1, {'@': 31}), 1: (1, {'@': 31}), 44: (1, {'@': 31}), 45: (1, {'@': 31}), 8: (1, {'@': 31}), 46: (1, {'@': 31})}, 9: {0: (1, {'@': 32}), 1: (1, {'@': 32}), 44: (1, {'@': 32}), 45: (1, {'@': 32}), 8: (1, {'@': 32}), 46: (1, {'@': 32})}, 10: {0: (0, 77), 1: (0, 179)}, 11: {42: (0, 193), 43: (0, 93)}, 12: {45: (1, {'@': 33}), 1: (1, {'@': 33})}, 13: {48: (0, 10), 1: (0, 164), 0: (0, 83)}, 14: {45: (1, {'@': 34}), 1: (1, {'@': 34})}, 15: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 14: (0, 153), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 49: (0, 65), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 16: {0: (1, {'@': 35}), 1: (1, {'@': 35}), 44: (1, {'@': 35}), 45: (1, {'@': 35}), 8: (1, {'@': 35}), 46: (1, {'@': 35})}, 17: {0: (1, {'@': 36}), 1: (1, {'@': 36}), 44: (1, {'@': 36}), 45: (1, {'@': 36}), 8: (1, {'@': 36}), 46: (1, {'@': 36})}, 18: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 14: (0, 72), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 19: {0: (1, {'@': 37}), 1: (1, {'@': 37}), 44: (1, {'@': 37}), 45: (1, {'@': 37}), 8: (1, {'@': 37}), 46: (1, {'@': 37})}, 20: {45: (1, {'@': 38}), 1: (1, {'@': 38}), 0: (1, {'@': 38})}, 21: {0: (0, 75), 1: (0, 33), 50: (0, 104)}, 22: {0: (1, {'@': 39}), 1: (1, {'@': 39}), 44: (1, {'@': 39}), 45: (1, {'@': 39}), 8: (1, {'@': 39}), 46: (1, {'@': 39})}, 23: {8: (0, 1)}, 24: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 14: (0, 145), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 25: {0: (1, {'@': 40}), 1: (1, {'@': 40}), 44: (1, {'@': 40}), 45: (1, {'@': 40}), 8: (1, {'@': 40}), 46: (1, {'@': 40})}, 26: {0: (1, {'@': 41}), 1: (1, {'@': 41}), 44: (1, {'@': 41}), 45: (1, {'@': 41}), 8: (1, {'@': 41}), 46: (1, {'@': 41})}, 27: {51: (0, 140)}, 28: {0: (1, {'@': 42}), 1: (1, {'@': 42}), 45: (1, {'@': 42})}, 29: {0: (1, {'@': 43}), 1: (1, {'@': 43}), 44: (1, {'@': 43}), 45: (1, {'@': 43}), 8: (1, {'@': 43}), 46: (1, {'@': 43})}, 30: {47: (1, {'@': 44}), 0: (1, {'@': 44}), 1: (1, {'@': 44}), 45: (1, {'@': 44})}, 31: {0: (0, 107)}, 32: {0: (1, {'@': 45}), 1: (1, {'@': 45}), 45: (1, {'@': 45})}, 33: {52: (0, 61), 53: (0, 67)}, 34: {0: (1, {'@': 46}), 1: (1, {'@': 46}), 45: (1, {'@': 46})}, 35: {54: (0, 118), 8: (0, 183), 0: (1, {'@': 47}), 1: (1, {'@': 47}), 44: (1, {'@': 47}), 45: (1, {'@': 47}), 46: (1, {'@': 47})}, 36: {8: (0, 18)}, 37: {55: (0, 81), 45: (0, 45), 1: (0, 115)}, 38: {0: (1, {'@': 48}), 1: (1, {'@': 48}), 44: (1, {'@': 48}), 45: (1, {'@': 48}), 8: (1, {'@': 48}), 46: (1, {'@': 48})}, 39: {0: (1, {'@': 49}), 1: (1, {'@': 49}), 45: (1, {'@': 49})}, 40: {51: (0, 68)}, 41: {0: (0, 43)}, 42: {0: (1, {'@': 50}), 1: (1, {'@': 50}), 45: (1, {'@': 50})}, 43: {0: (1, {'@': 51}), 1: (1, {'@': 51}), 44: (1, {'@': 51}), 45: (1, {'@': 51}), 8: (1, {'@': 51}), 46: (1, {'@': 51})}, 44: {0: (1, {'@': 52}), 1: (1, {'@': 52}), 45: (1, {'@': 52})}, 45: {0: (1, {'@': 53}), 1: (1, {'@': 53}), 44: (1, {'@': 53}), 45: (1, {'@': 53}), 8: (1, {'@': 53}), 46: (1, {'@': 53})}, 46: {0: (1, {'@': 54}), 1: (1, {'@': 54})}, 47: {0: (1, {'@': 55}), 1: (1, {'@': 55}), 44: (1, {'@': 55}), 45: (1, {'@': 55}), 8: (1, {'@': 55}), 46: (1, {'@': 55})}, 48: {0: (1, {'@': 56}), 1: (1, {'@': 56}), 44: (1, {'@': 56}), 45: (1, {'@': 56}), 8: (1, {'@': 56}), 46: (1, {'@': 56})}, 49: {1: (0, 98)}, 50: {23: (0, 173), 56: (0, 66), 57: (0, 34), 58: (0, 42), 59: (0, 121), 60: (0, 39), 8: (0, 125), 61: (0, 46), 52: (0, 61), 53: (0, 100), 36: (0, 28), 40: (0, 30)}, 51: {47: (0, 122)}, 52: {8: (0, 6)}, 53: {0: (1, {'@': 57}), 1: (1, {'@': 57}), 44: (1, {'@': 57}), 45: (1, {'@': 57}), 8: (1, {'@': 57}), 46: (1, {'@': 57})}, 54: {23: (0, 173), 56: (0, 66), 61: (0, 12), 57: (0, 34), 58: (0, 42), 59: (0, 121), 60: (0, 39), 8: (0, 125), 52: (0, 61), 53: (0, 100), 36: (0, 28), 40: (0, 30)}, 55: {47: (0, 24)}, 56: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 14: (0, 37), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 57: {53: (0, 27), 52: (0, 61)}, 58: {0: (1, {'@': 58}), 1: (1, {'@': 58}), 45: (1, {'@': 58})}, 59: {0: (1, {'@': 59}), 1: (1, {'@': 59}), 44: (1, {'@': 59}), 45: (1, {'@': 59}), 8: (1, {'@': 59}), 46: (1, {'@': 59})}, 60: {53: (0, 88), 52: (0, 61)}, 61: {0: (1, {'@': 60}), 1: (1, {'@': 60}), 45: (1, {'@': 60}), 51: (1, {'@': 60})}, 62: {0: (1, {'@': 61}), 1: (1, {'@': 61}), 44: (1, {'@': 61}), 45: (1, {'@': 61}), 8: (1, {'@': 61}), 46: (1, {'@': 61})}, 63: {62: (0, 117), 1: (0, 50), 0: (0, 44)}, 64: {48: (0, 160), 1: (0, 164), 46: (0, 167)}, 65: {}, 66: {0: (1, {'@': 62}), 1: (1, {'@': 62}), 45: (1, {'@': 62})}, 67: {0: (1, {'@': 63}), 1: (1, {'@': 63})}, 68: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 14: (0, 20), 41: (0, 48)}, 69: {0: (0, 59)}, 70: {45: (1, {'@': 64}), 1: (1, {'@': 64})}, 71: {0: (1, {'@': 65}), 1: (1, {'@': 65}), 44: (1, {'@': 65}), 45: (1, {'@': 65}), 8: (1, {'@': 65}), 46: (1, {'@': 65})}, 72: {48: (0, 186), 1: (0, 164), 0: (0, 82)}, 73: {0: (1, {'@': 66}), 1: (1, {'@': 66}), 45: (1, {'@': 66})}, 74: {0: (1, {'@': 67}), 1: (1, {'@': 67}), 44: (1, {'@': 67}), 45: (1, {'@': 67}), 8: (1, {'@': 67}), 46: (1, {'@': 67})}, 75: {1: (0, 52)}, 76: {0: (1, {'@': 68}), 1: (1, {'@': 68}), 44: (1, {'@': 68}), 45: (1, {'@': 68}), 8: (1, {'@': 68}), 46: (1, {'@': 68})}, 77: {1: (0, 169)}, 78: {51: (0, 56)}, 79: {42: (0, 193), 43: (0, 41)}, 80: {0: (1, {'@': 69}), 1: (1, {'@': 69}), 44: (1, {'@': 69}), 45: (1, {'@': 69}), 8: (1, {'@': 69}), 46: (1, {'@': 69})}, 81: {45: (0, 99), 1: (0, 57)}, 82: {1: (0, 79)}, 83: {1: (0, 94)}, 84: {0: (0, 138), 1: (0, 179)}, 85: {36: (0, 110), 40: (0, 30), 4: (0, 5)}, 86: {0: (1, {'@': 70}), 1: (1, {'@': 70}), 44: (1, {'@': 70}), 45: (1, {'@': 70}), 8: (1, {'@': 70}), 46: (1, {'@': 70})}, 87: {0: (0, 123), 55: (0, 106), 1: (0, 115)}, 88: {0: (1, {'@': 71}), 1: (1, {'@': 71})}, 89: {0: (1, {'@': 72}), 1: (1, {'@': 72}), 44: (1, {'@': 72}), 45: (1, {'@': 72}), 8: (1, {'@': 72}), 46: (1, {'@': 72})}, 90: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 14: (0, 64), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 91: {8: (0, 170)}, 92: {48: (0, 84), 1: (0, 164), 0: (0, 156)}, 93: {0: (0, 62)}, 94: {42: (0, 193), 43: (0, 189)}, 95: {54: (0, 26), 8: (0, 183), 0: (1, {'@': 73}), 1: (1, {'@': 73}), 44: (1, {'@': 73}), 45: (1, {'@': 73}), 46: (1, {'@': 73})}, 96: {0: (0, 71)}, 97: {63: (0, 14), 53: (0, 102), 52: (0, 61)}, 98: {8: (0, 181)}, 99: {0: (1, {'@': 74}), 1: (1, {'@': 74}), 44: (1, {'@': 74}), 45: (1, {'@': 74}), 8: (1, {'@': 74}), 46: (1, {'@': 74})}, 100: {0: (1, {'@': 75}), 1: (1, {'@': 75}), 45: (1, {'@': 75})}, 101: {64: (0, 126)}, 102: {51: (0, 54)}, 103: {0: (0, 89)}, 104: {0: (0, 49), 1: (0, 60)}, 105: {0: (1, {'@': 76})}, 106: {0: (0, 168), 1: (0, 57)}, 107: {0: (1, {'@': 77}), 1: (1, {'@': 77}), 44: (1, {'@': 77}), 45: (1, {'@': 77}), 8: (1, {'@': 77}), 46: (1, {'@': 77})}, 108: {0: (1, {'@': 78}), 1: (1, {'@': 78}), 44: (1, {'@': 78}), 45: (1, {'@': 78}), 8: (1, {'@': 78}), 46: (1, {'@': 78})}, 109: {0: (0, 9)}, 110: {47: (0, 161)}, 111: {42: (0, 193), 43: (0, 109)}, 112: {52: (0, 61), 63: (0, 70), 53: (0, 102)}, 113: {0: (0, 185)}, 114: {0: (1, {'@': 79}), 1: (1, {'@': 79}), 44: (1, {'@': 79}), 45: (1, {'@': 79}), 8: (1, {'@': 79}), 46: (1, {'@': 79})}, 115: {53: (0, 40), 52: (0, 61)}, 116: {1: (0, 112), 65: (0, 120), 45: (0, 73)}, 117: {0: (0, 137), 1: (0, 174)}, 118: {0: (1, {'@': 80}), 1: (1, {'@': 80}), 44: (1, {'@': 80}), 45: (1, {'@': 80}), 8: (1, {'@': 80}), 46: (1, {'@': 80})}, 119: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 14: (0, 159), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 43: (0, 31), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 42: (0, 193), 41: (0, 48)}, 120: {45: (0, 58), 1: (0, 97)}, 121: {0: (1, {'@': 81}), 1: (1, {'@': 81}), 45: (1, {'@': 81})}, 122: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 14: (0, 154), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 123: {0: (1, {'@': 82}), 1: (1, {'@': 82}), 44: (1, {'@': 82}), 45: (1, {'@': 82}), 8: (1, {'@': 82}), 46: (1, {'@': 82})}, 124: {0: (1, {'@': 83}), 1: (1, {'@': 83}), 45: (1, {'@': 83})}, 125: {23: (0, 173), 61: (0, 63), 56: (0, 66), 57: (0, 34), 58: (0, 42), 59: (0, 121), 0: (0, 32), 60: (0, 39), 8: (0, 125), 52: (0, 61), 53: (0, 100), 36: (0, 28), 40: (0, 30)}, 126: {66: (0, 166)}, 127: {42: (0, 193), 43: (0, 96)}, 128: {0: (0, 150)}, 129: {54: (0, 108), 8: (0, 183), 0: (1, {'@': 84}), 1: (1, {'@': 84}), 44: (1, {'@': 84}), 45: (1, {'@': 84}), 46: (1, {'@': 84})}, 130: {42: (0, 193), 43: (0, 69)}, 131: {42: (0, 193), 43: (0, 103)}, 132: {0: (1, {'@': 85}), 1: (1, {'@': 85}), 44: (1, {'@': 85}), 45: (1, {'@': 85}), 8: (1, {'@': 85}), 46: (1, {'@': 85})}, 133: {23: (0, 173), 60: (0, 105)}, 134: {0: (0, 76)}, 135: {0: (1, {'@': 86}), 1: (1, {'@': 86}), 44: (1, {'@': 86}), 45: (1, {'@': 86}), 8: (1, {'@': 86}), 46: (1, {'@': 86})}, 136: {8: (0, 155)}, 137: {0: (1, {'@': 87}), 1: (1, {'@': 87}), 45: (1, {'@': 87})}, 138: {1: (0, 131)}, 139: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 14: (0, 87), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 140: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 14: (0, 157), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 141: {0: (1, {'@': 88}), 1: (1, {'@': 88}), 44: (1, {'@': 88}), 45: (1, {'@': 88}), 8: (1, {'@': 88}), 46: (1, {'@': 88})}, 142: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 14: (0, 95), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 143: {0: (1, {'@': 89}), 1: (1, {'@': 89}), 44: (1, {'@': 89}), 45: (1, {'@': 89}), 8: (1, {'@': 89}), 46: (1, {'@': 89})}, 144: {1: (0, 11)}, 145: {0: (1, {'@': 90}), 1: (1, {'@': 90}), 44: (1, {'@': 90}), 45: (1, {'@': 90}), 8: (1, {'@': 90}), 46: (1, {'@': 90})}, 146: {8: (0, 191)}, 147: {0: (1, {'@': 91}), 1: (1, {'@': 91}), 44: (1, {'@': 91}), 45: (1, {'@': 91}), 8: (1, {'@': 91}), 46: (1, {'@': 91})}, 148: {1: (0, 111)}, 149: {0: (1, {'@': 92}), 1: (1, {'@': 92}), 44: (1, {'@': 92}), 45: (1, {'@': 92}), 8: (1, {'@': 92}), 46: (1, {'@': 92})}, 150: {0: (1, {'@': 93}), 1: (1, {'@': 93}), 44: (1, {'@': 93}), 45: (1, {'@': 93}), 8: (1, {'@': 93}), 46: (1, {'@': 93})}, 151: {0: (1, {'@': 94}), 1: (1, {'@': 94}), 44: (1, {'@': 94}), 45: (1, {'@': 94}), 8: (1, {'@': 94}), 46: (1, {'@': 94})}, 152: {1: (0, 171), 48: (0, 180), 0: (0, 184)}, 153: {44: (1, {'@': 95})}, 154: {0: (1, {'@': 96}), 1: (1, {'@': 96}), 44: (1, {'@': 96}), 45: (1, {'@': 96}), 8: (1, {'@': 96}), 46: (1, {'@': 96})}, 155: {52: (0, 61), 53: (0, 190)}, 156: {1: (0, 130)}, 157: {45: (1, {'@': 97}), 1: (1, {'@': 97}), 0: (1, {'@': 97})}, 158: {0: (1, {'@': 98}), 1: (1, {'@': 98}), 44: (1, {'@': 98}), 45: (1, {'@': 98}), 8: (1, {'@': 98}), 46: (1, {'@': 98})}, 159: {0: (1, {'@': 99}), 1: (1, {'@': 99}), 46: (1, {'@': 99})}, 160: {46: (0, 177), 1: (0, 179)}, 161: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 14: (0, 148), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 162: {0: (1, {'@': 100}), 1: (1, {'@': 100}), 44: (1, {'@': 100}), 45: (1, {'@': 100}), 8: (1, {'@': 100}), 46: (1, {'@': 100})}, 163: {0: (0, 178), 1: (0, 127)}, 164: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 37: (0, 23), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 38: (0, 74), 39: (0, 8), 40: (0, 30), 14: (0, 187), 41: (0, 48)}, 165: {44: (1, {'@': 101}), 8: (1, {'@': 101}), 46: (1, {'@': 101}), 1: (1, {'@': 101}), 45: (1, {'@': 101}), 0: (1, {'@': 101})}, 166: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 14: (0, 176), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 167: {0: (1, {'@': 102}), 1: (1, {'@': 102}), 44: (1, {'@': 102}), 45: (1, {'@': 102}), 8: (1, {'@': 102}), 46: (1, {'@': 102})}, 168: {0: (1, {'@': 103}), 1: (1, {'@': 103}), 44: (1, {'@': 103}), 45: (1, {'@': 103}), 8: (1, {'@': 103}), 46: (1, {'@': 103})}, 169: {42: (0, 193), 43: (0, 128)}, 170: {53: (0, 21), 52: (0, 61)}, 171: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 37: (0, 23), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 43: (0, 134), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 38: (0, 74), 39: (0, 8), 40: (0, 30), 42: (0, 193), 14: (0, 187), 41: (0, 48)}, 172: {0: (1, {'@': 104}), 1: (1, {'@': 104}), 44: (1, {'@': 104}), 45: (1, {'@': 104}), 8: (1, {'@': 104}), 46: (1, {'@': 104})}, 173: {63: (0, 116), 52: (0, 61), 53: (0, 102), 45: (0, 124)}, 174: {23: (0, 173), 56: (0, 66), 61: (0, 0), 57: (0, 34), 58: (0, 42), 59: (0, 121), 60: (0, 39), 8: (0, 125), 52: (0, 61), 53: (0, 100), 36: (0, 28), 40: (0, 30)}, 175: {8: (0, 36)}, 176: {0: (0, 188)}, 177: {0: (1, {'@': 105}), 1: (1, {'@': 105}), 44: (1, {'@': 105}), 45: (1, {'@': 105}), 8: (1, {'@': 105}), 46: (1, {'@': 105})}, 178: {0: (1, {'@': 106}), 1: (1, {'@': 106}), 44: (1, {'@': 106}), 45: (1, {'@': 106}), 8: (1, {'@': 106}), 46: (1, {'@': 106})}, 179: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 14: (0, 159), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 180: {1: (0, 119), 0: (0, 4)}, 181: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 14: (0, 92), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 182: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 14: (0, 144), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 183: {42: (0, 193), 43: (0, 194)}, 184: {0: (1, {'@': 107}), 1: (1, {'@': 107}), 44: (1, {'@': 107}), 45: (1, {'@': 107}), 8: (1, {'@': 107}), 46: (1, {'@': 107})}, 185: {0: (1, {'@': 108}), 1: (1, {'@': 108}), 44: (1, {'@': 108}), 45: (1, {'@': 108}), 8: (1, {'@': 108}), 46: (1, {'@': 108})}, 186: {0: (0, 2), 1: (0, 179)}, 187: {0: (1, {'@': 109}), 1: (1, {'@': 109}), 46: (1, {'@': 109})}, 188: {0: (1, {'@': 110}), 1: (1, {'@': 110}), 44: (1, {'@': 110}), 45: (1, {'@': 110}), 8: (1, {'@': 110}), 46: (1, {'@': 110})}, 189: {0: (0, 149)}, 190: {51: (0, 139)}, 191: {2: (0, 22), 3: (0, 47), 4: (0, 55), 5: (0, 29), 6: (0, 114), 7: (0, 90), 8: (0, 85), 9: (0, 38), 10: (0, 35), 11: (0, 151), 12: (0, 136), 13: (0, 158), 15: (0, 162), 16: (0, 146), 17: (0, 192), 18: (0, 143), 19: (0, 142), 20: (0, 147), 21: (0, 132), 22: (0, 141), 23: (0, 195), 24: (0, 172), 25: (0, 175), 26: (0, 129), 14: (0, 152), 27: (0, 135), 28: (0, 86), 29: (0, 25), 30: (0, 19), 31: (0, 80), 32: (0, 7), 33: (0, 53), 34: (0, 17), 35: (0, 16), 36: (0, 51), 37: (0, 23), 38: (0, 74), 39: (0, 8), 40: (0, 30), 41: (0, 48)}, 192: {8: (0, 91)}, 193: {66: (0, 133)}, 194: {0: (0, 165)}, 195: {52: (0, 61), 53: (0, 78)}}, 'start_states': {'start': 15}, 'end_states': {'start': 65}}, 'lexer_conf': {'tokens': [{'@': 0}, {'@': 1}, {'@': 2}, {'@': 3}, {'@': 4}, {'@': 5}, {'@': 6}, {'@': 7}, {'@': 8}, {'@': 9}, {'@': 10}, {'@': 11}, {'@': 12}, {'@': 13}, {'@': 14}, {'@': 15}, {'@': 16}, {'@': 17}, {'@': 18}, {'@': 19}, {'@': 20}, {'@': 21}, {'@': 22}, {'@': 23}, {'@': 24}, {'@': 25}, {'@': 26}, {'@': 27}, {'@': 28}], 'ignore': ['WS'], 'g_regex_flags': 0, 'use_bytes': False, '__type__': 'LexerConf'}, 'start': ['start'], '__type__': 'LALR_ContextualLexer'}, 'rules': [{'@': 95}, {'@': 86}, {'@': 70}, {'@': 56}, {'@': 37}, {'@': 35}, {'@': 98}, {'@': 104}, {'@': 79}, {'@': 36}, {'@': 48}, {'@': 67}, {'@': 57}, {'@': 31}, {'@': 43}, {'@': 78}, {'@': 84}, {'@': 80}, {'@': 47}, {'@': 89}, {'@': 39}, {'@': 55}, {'@': 88}, {'@': 91}, {'@': 100}, {'@': 40}, {'@': 69}, {'@': 85}, {'@': 110}, {'@': 94}, {'@': 41}, {'@': 73}, {'@': 65}, {'@': 106}, {'@': 30}, {'@': 107}, {'@': 77}, {'@': 68}, {'@': 61}, {'@': 90}, {'@': 105}, {'@': 102}, {'@': 108}, {'@': 51}, {'@': 74}, {'@': 53}, {'@': 72}, {'@': 59}, {'@': 93}, {'@': 92}, {'@': 103}, {'@': 82}, {'@': 96}, {'@': 32}, {'@': 101}, {'@': 76}, {'@': 49}, {'@': 50}, {'@': 75}, {'@': 42}, {'@': 62}, {'@': 81}, {'@': 46}, {'@': 87}, {'@': 52}, {'@': 45}, {'@': 58}, {'@': 66}, {'@': 83}, {'@': 33}, {'@': 60}, {'@': 44}, {'@': 109}, {'@': 111}, {'@': 99}, {'@': 112}, {'@': 38}, {'@': 113}, {'@': 97}, {'@': 114}, {'@': 63}, {'@': 115}, {'@': 71}, {'@': 116}, {'@': 54}, {'@': 29}, {'@': 64}, {'@': 34}], 'options': {'debug': False, 'keep_all_tokens': False, 'tree_class': None, 'cache': False, 'postlex': None, 'parser': 'lalr', 'lexer': 'contextual', 'transformer': None, 'start': ['start'], 'priority': 'normal', 'ambiguity': 'auto', 'regex': False, 'propagate_positions': False, 'lexer_callbacks': {}, 'maybe_placeholders': False, 'edit_terminals': None, 'g_regex_flags': 0, 'use_bytes': False, 'import_paths': [], 'source_path': None}, '__type__': 'Lark'}
+)
+MEMO = (
+{0: {'name': 'HARDCODED', 'pattern': {'value': '(?:(?:(?:string|char)|bytes)|byte)', 'flags': [], '_width': [4, 6], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 1: {'name': 'TYPE', 'pattern': {'value': '(?:(?:(?:(?:(?:(?:(?:(?:(?:(?:(?:(?:int8|int16)|int32)|int64)|int128)|uint8)|uint16)|uint32)|uint64)|uint128)|float32)|float64)|bool)', 'flags': [], '_width': [4, 7], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 2: {'name': 'UNQUOTED_STRING', 'pattern': {'value': '[a-zA-Z]+', 'flags': [], '_width': [1, 4294967295], '__type__': 'PatternRE'}, 'priority': -1, '__type__': 'TerminalDef'}, 3: {'name': 'SIGNED_NUMBER', 'pattern': {'value': '(?:(?:\\+|\\-))?(?:(?:(?:[0-9])+(?:e|E)(?:(?:\\+|\\-))?(?:[0-9])+|(?:(?:[0-9])+\\.(?:(?:[0-9])+)?|\\.(?:[0-9])+)(?:(?:e|E)(?:(?:\\+|\\-))?(?:[0-9])+)?)|(?:[0-9])+)', 'flags': [], '_width': [1, 4294967295], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 4: {'name': 'WS', 'pattern': {'value': '(?:[ \t\x0c\r\n])+', 'flags': [], '_width': [1, 4294967295], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 5: {'name': 'UNKNOWN', 'pattern': {'value': 'unknown', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 6: {'name': 'CATEGORICAL', 'pattern': {'value': 'categorical', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 7: {'name': 'LSQB', 'pattern': {'value': '[', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 8: {'name': '__ANON_0', 'pattern': {'value': 'type', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 9: {'name': 'EQUAL', 'pattern': {'value': '=', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 10: {'name': 'RSQB', 'pattern': {'value': ']', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 11: {'name': 'QMARK', 'pattern': {'value': '?', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 12: {'name': 'OPTION', 'pattern': {'value': 'option', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 13: {'name': 'COMMA', 'pattern': {'value': ',', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 14: {'name': 'UNION', 'pattern': {'value': 'union', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 15: {'name': 'VAR', 'pattern': {'value': 'var', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 16: {'name': 'STAR', 'pattern': {'value': '*', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 17: {'name': 'LPAR', 'pattern': {'value': '(', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 18: {'name': 'RPAR', 'pattern': {'value': ')', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 19: {'name': 'TUPLE', 'pattern': {'value': 'tuple', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 20: {'name': 'COLON', 'pattern': {'value': ':', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 21: {'name': 'LBRACE', 'pattern': {'value': '{', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 22: {'name': 'RBRACE', 'pattern': {'value': '}', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 23: {'name': 'STRUCT', 'pattern': {'value': 'struct', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 24: {'name': 'PARAMETERS', 'pattern': {'value': 'parameters', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 25: {'name': 'TRUE', 'pattern': {'value': 'true', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 26: {'name': 'FALSE', 'pattern': {'value': 'false', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 27: {'name': 'NULL', 'pattern': {'value': 'null', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 28: {'name': '__ANON_1', 'pattern': {'value': '((?:"(?:[^"\n\r\\\\]|(?:\\\\u[0-9a-fA-F]{4})|(?:\\\\["bfnrt]))*")|(?:\\\\\\\'(?:[^\\\\\\\'\n\r\\\\]|(?:\\\\u[0-9a-fA-F]{4})|(?:\\\\[\\\'bfnrt]))*")*\\\\\\\')', 'flags': [], '_width': [2, 4294967295], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 29: {'origin': {'name': '__list_obj_star_3', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__list_obj_star_3', '__type__': 'NonTerminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'json', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 30: {'origin': {'name': 'union_single', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'UNION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': '__union_single_star_0', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 31: {'origin': {'name': 'record', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'record_struct', '__type__': 'NonTerminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 32: {'origin': {'name': 'regular_outparm', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'number', '__type__': 'NonTerminal'}, {'name': 'STAR', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 33: {'origin': {'name': 'pair', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'string', '__type__': 'NonTerminal'}, {'name': 'COLON', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'json', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 34: {'origin': {'name': '__dict_obj_star_4', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__dict_obj_star_4', '__type__': 'NonTerminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'pair', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 35: {'origin': {'name': 'input', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'regular', '__type__': 'NonTerminal'}], 'order': 4, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 36: {'origin': {'name': 'input', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'categories', '__type__': 'NonTerminal'}], 'order': 8, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 37: {'origin': {'name': 'input', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'optiontype', '__type__': 'NonTerminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 38: {'origin': {'name': '__record_dict_star_1', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}, {'name': 'COLON', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 39: {'origin': {'name': 'optiontype', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'option_parm', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 40: {'origin': {'name': 'listtype', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'list_parm', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 41: {'origin': {'name': 'option_single', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'QMARK', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 42: {'origin': {'name': 'json', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'number', '__type__': 'NonTerminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 43: {'origin': {'name': 'record', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'record_highlevel', '__type__': 'NonTerminal'}], 'order': 4, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 44: {'origin': {'name': 'number', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'SIGNED_NUMBER', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 45: {'origin': {'name': 'list_obj', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [False, True, False], '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 46: {'origin': {'name': 'json', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'NULL', 'filter_out': True, '__type__': 'Terminal'}], 'order': 6, 'alias': 'null', 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 47: {'origin': {'name': 'unknown', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'UNKNOWN', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [False, True], '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 48: {'origin': {'name': 'record', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'record_tuple', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 49: {'origin': {'name': 'json', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'dict_obj', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 50: {'origin': {'name': 'json', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'list_obj', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 51: {'origin': {'name': 'record_tuple_param', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'TUPLE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 52: {'origin': {'name': 'list_obj', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'json', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 53: {'origin': {'name': 'record_dict', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}, {'name': 'COLON', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 54: {'origin': {'name': '__list_obj_star_3', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'json', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 55: {'origin': {'name': 'optiontype', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'option_highlevel', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 56: {'origin': {'name': 'input', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'primitive', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 57: {'origin': {'name': 'record', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'record_dict', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 58: {'origin': {'name': 'dict_obj', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'pair', '__type__': 'NonTerminal'}, {'name': '__dict_obj_star_4', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 59: {'origin': {'name': 'record_struct', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'STRUCT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}, {'name': '__record_struct_star_2', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 60: {'origin': {'name': 'string', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__ANON_1', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 61: {'origin': {'name': 'list_parm', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'VAR', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'STAR', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 62: {'origin': {'name': 'json', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'TRUE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 4, 'alias': 'true', 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 63: {'origin': {'name': '__record_struct_star_2', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 64: {'origin': {'name': '__dict_obj_star_4', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'pair', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 65: {'origin': {'name': 'option_parm', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OPTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 66: {'origin': {'name': 'dict_obj', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'pair', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 67: {'origin': {'name': 'record', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'record_tuple_param', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 68: {'origin': {'name': 'union_parm', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'UNION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 69: {'origin': {'name': 'uniontype', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'union_single', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 70: {'origin': {'name': 'input', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'unknown', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 71: {'origin': {'name': '__record_struct_star_2', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__record_struct_star_2', '__type__': 'NonTerminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 72: {'origin': {'name': 'record_struct', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'STRUCT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}, {'name': '__record_struct_star_2', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': '__union_single_star_0', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 73: {'origin': {'name': 'option_single', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'QMARK', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [False, False, True], '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 74: {'origin': {'name': 'record_dict', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}, {'name': 'COLON', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': '__record_dict_star_1', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 75: {'origin': {'name': 'json', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'string', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 76: {'origin': {'name': 'def_option', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'PARAMETERS', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'EQUAL', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'dict_obj', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 77: {'origin': {'name': 'union_parm', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'UNION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': '__union_single_star_0', '__type__': 'NonTerminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 78: {'origin': {'name': 'primitive', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'TYPE', 'filter_out': False, '__type__': 'Terminal'}, {'name': 'options', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 79: {'origin': {'name': 'input', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'predefined_typestr', '__type__': 'NonTerminal'}], 'order': 7, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 80: {'origin': {'name': 'unknown', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'UNKNOWN', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'options', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 81: {'origin': {'name': 'json', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'FALSE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 5, 'alias': 'false', 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 82: {'origin': {'name': 'record_highlevel', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'UNQUOTED_STRING', 'filter_out': False, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}, {'name': 'COLON', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 83: {'origin': {'name': 'dict_obj', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [False, True, False], '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 84: {'origin': {'name': 'primitive', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'TYPE', 'filter_out': False, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [False, True], '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 85: {'origin': {'name': 'uniontype', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'union_parm', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 86: {'origin': {'name': 'input', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'record', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 87: {'origin': {'name': 'list_obj', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'json', '__type__': 'NonTerminal'}, {'name': '__list_obj_star_3', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 88: {'origin': {'name': 'regular', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'regular_inparm', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 89: {'origin': {'name': 'optiontype', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'option_single', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 90: {'origin': {'name': 'list_single', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'VAR', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'STAR', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 91: {'origin': {'name': 'regular', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'regular_outparm', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 92: {'origin': {'name': 'record_struct', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'STRUCT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 93: {'origin': {'name': 'record_struct', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'STRUCT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': '__union_single_star_0', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 94: {'origin': {'name': 'predefined_typestr', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'HARDCODED', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 95: {'origin': {'name': 'start', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'input', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 96: {'origin': {'name': 'regular_inparm', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'number', '__type__': 'NonTerminal'}, {'name': 'STAR', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 97: {'origin': {'name': '__record_dict_star_1', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__record_dict_star_1', '__type__': 'NonTerminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}, {'name': 'COLON', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 98: {'origin': {'name': 'input', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'listtype', '__type__': 'NonTerminal'}], 'order': 5, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 99: {'origin': {'name': '__union_single_star_0', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__union_single_star_0', '__type__': 'NonTerminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 100: {'origin': {'name': 'listtype', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'list_single', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 101: {'origin': {'name': 'options', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 102: {'origin': {'name': 'record_tuple', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LPAR', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'RPAR', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 103: {'origin': {'name': 'record_highlevel', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'UNQUOTED_STRING', 'filter_out': False, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'string', '__type__': 'NonTerminal'}, {'name': 'COLON', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': '__record_dict_star_1', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 104: {'origin': {'name': 'input', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'uniontype', '__type__': 'NonTerminal'}], 'order': 6, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 105: {'origin': {'name': 'record_tuple', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LPAR', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': '__union_single_star_0', '__type__': 'NonTerminal'}, {'name': 'RPAR', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 106: {'origin': {'name': 'option_highlevel', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OPTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 107: {'origin': {'name': 'union_single', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'UNION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 108: {'origin': {'name': 'record_tuple_param', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'TUPLE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': '__union_single_star_0', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'def_option', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 109: {'origin': {'name': '__union_single_star_0', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'COMMA', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 110: {'origin': {'name': 'categories', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'CATEGORICAL', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': '__ANON_0', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'EQUAL', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'input', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 111: {'origin': {'name': '__union_single_star_0', '__type__': 'NonTerminal'}, 'expansion': [], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [True], '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 112: {'origin': {'name': '__union_single_star_0', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__union_single_star_0', '__type__': 'NonTerminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [False, True], '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 113: {'origin': {'name': '__record_dict_star_1', '__type__': 'NonTerminal'}, 'expansion': [], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [True], '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 114: {'origin': {'name': '__record_dict_star_1', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__record_dict_star_1', '__type__': 'NonTerminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [False, True], '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 115: {'origin': {'name': '__record_struct_star_2', '__type__': 'NonTerminal'}, 'expansion': [], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [True], '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 116: {'origin': {'name': '__record_struct_star_2', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__record_struct_star_2', '__type__': 'NonTerminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': [False, True], '__type__': 'RuleOptions'}, '__type__': 'Rule'}}
+)
+Shift = 0
+Reduce = 1
+def Lark_StandAlone(**kwargs):
+  return Lark._load_from_dict(DATA, MEMO, **kwargs)
diff --git a/studies/type-parser-1.5/parser.py b/studies/type-parser-1.5/parser.py
new file mode 100644
index 0000000000..dec387e86a
--- /dev/null
+++ b/studies/type-parser-1.5/parser.py
@@ -0,0 +1,322 @@
+# flake8: noqa
+# fmt: off
+# pylint: skip-file
+
+# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
+
+# v2: keep this file, but drop the two *_v1 functions
+
+import awkward as ak
+
+from awkward._typeparser.generated_parser import Lark_StandAlone, Transformer
+
+
+class TreeToJson(Transformer):
+    def string(self, s):
+        (s,) = s
+        return s[1:-1]
+
+    def number(self, n):
+        (n,) = n
+        if "." in n:
+            return float(n)
+        else:
+            return int(n)
+
+    list_obj = list
+    pair = tuple
+    dict_obj = dict
+
+    def null(self, s):
+        return None
+
+    def true(self, s):
+        return True
+
+    def false(self, s):
+        return False
+
+
+def toast_v1(ptnode, highlevel, categorical):
+    if ptnode.__class__.__name__ == "Token":
+        return ptnode.value
+
+    elif ptnode.data == "start":
+        return toast_v1(ptnode.children[0], highlevel, categorical)
+
+    elif ptnode.data == "input":
+        assert len(ptnode.children) == 1
+        return toast_v1(ptnode.children[0], highlevel, categorical)
+
+    elif ptnode.data == "predefined_typestr":
+        if ptnode.children[0] == "string":
+            parms = {"__array__": "string"}
+            if categorical:
+                parms.update({"__categorical__": True})
+                categorical = False
+            return ak.types.ListType(
+                ak.types.PrimitiveType(
+                    "uint8", parameters={"__array__": "char"}, typestr="char"
+                ),
+                parameters=parms,
+                typestr="string",
+            )
+        elif ptnode.children[0] == "char":
+            parms = {"__array__": "char"}
+            if categorical:
+                parms.update({"__categorical__": True})
+                categorical = False
+            return ak.types.PrimitiveType("uint8", parameters=parms, typestr="char")
+        elif ptnode.children[0] == "byte":
+            parms = {"__array__": "byte"}
+            if categorical:
+                parms.update({"__categorical__": True})
+                categorical = False
+            return ak.types.PrimitiveType("uint8", parameters=parms, typestr="byte")
+        elif ptnode.children[0] == "bytes":
+            parms = {"__array__": "bytestring"}
+            if categorical:
+                parms.update({"__categorical__": True})
+                categorical = False
+            return ak.types.ListType(
+                ak.types.PrimitiveType(
+                    "uint8", parameters={"__array__": "byte"}, typestr="byte"
+                ),
+                parameters=parms,
+                typestr="bytes",
+            )
+        else:
+            raise AssertionError(f"unhandled typestring {ptnode.children[0]}")
+
+    elif ptnode.data == "primitive":
+        if len(ptnode.children) == 1:
+            parms = {}
+            if categorical:
+                parms.update({"__categorical__": True})
+                categorical = False
+            return ak.types.PrimitiveType(
+                toast_v1(ptnode.children[0], highlevel, False), parameters=parms
+            )
+        elif len(ptnode.children) == 2:
+            parms = toast_v1(ptnode.children[1], highlevel, False)
+            if categorical:
+                parms.update({"__categorical__": True})
+                categorical = False
+            return ak.types.PrimitiveType(
+                toast_v1(ptnode.children[0], highlevel, categorical),
+                parms,
+            )
+        else:
+            raise AssertionError("unhandled PrimitiveType node")
+
+    elif ptnode.data == "categories":
+        assert highlevel is True
+        return toast_v1(ptnode.children[0], highlevel, True)
+
+    elif ptnode.data == "unknown":
+        if len(ptnode.children) == 0:
+            parms = {}
+            if categorical:
+                parms.update({"__categorical__": True})
+                categorical = False
+            return ak.types.UnknownType(parameters=parms)
+        elif len(ptnode.children) == 1:
+            parms = toast_v1(ptnode.children[0], highlevel, False)
+            if categorical:
+                parms.update({"__categorical__": True})
+                categorical = False
+            return ak.types.UnknownType(parameters=parms)
+        else:
+            raise AssertionError("unhandled UnknownType node")
+
+    elif ptnode.data == "listtype":
+        return toast_v1(ptnode.children[0], highlevel, categorical)
+
+    elif ptnode.data == "list_single":
+        parms = {}
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        return ak.types.ListType(
+            toast_v1(ptnode.children[0], highlevel, False), parameters=parms
+        )
+
+    elif ptnode.data == "list_parm":
+        parms = toast_v1(ptnode.children[1], highlevel, False)
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        return ak.types.ListType(
+            toast_v1(ptnode.children[0], highlevel, categorical), parms
+        )
+
+    elif ptnode.data == "uniontype":
+        return toast_v1(ptnode.children[0], highlevel, categorical)
+
+    elif ptnode.data == "union_single":
+        parms = {}
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        content_list = []
+        for node in ptnode.children:
+            content_list.append(toast_v1(node, highlevel, False))
+        return ak.types.UnionType(content_list, parameters=parms)
+
+    elif ptnode.data == "union_parm":
+        parms = toast_v1(ptnode.children[-1], highlevel, False)
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        content_list = []
+        for node in ptnode.children[:-1]:
+            content_list.append(toast_v1(node, highlevel, False))
+        return ak.types.UnionType(content_list, parms)
+
+    elif ptnode.data == "optiontype":
+        return toast_v1(ptnode.children[0], highlevel, categorical)
+
+    elif ptnode.data == "option_single":
+        parms = {}
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        return ak.types.OptionType(
+            toast_v1(ptnode.children[0], highlevel, False), parameters=parms
+        )
+
+    elif ptnode.data == "option_parm":
+        parms = toast_v1(ptnode.children[1], highlevel, False)
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        return ak.types.OptionType(
+            toast_v1(ptnode.children[0], highlevel, False),
+            parameters=parms,
+        )
+
+    elif ptnode.data == "option_highlevel":
+        assert highlevel
+        parms = {}
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        return ak.types.OptionType(
+            toast_v1(ptnode.children[0], highlevel, False), parameters=parms
+        )
+
+    elif ptnode.data == "record":
+        return toast_v1(ptnode.children[0], highlevel, categorical)
+
+    elif ptnode.data == "record_tuple":
+        parms = {}
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        content_list = []
+        for node in ptnode.children:
+            content_list.append(toast_v1(node, highlevel, categorical))
+        return ak.types.RecordType(tuple(content_list), parameters=parms)
+
+    elif ptnode.data == "record_dict":
+        parms = {}
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        content_types = []
+        content_keys = []
+        for i in range(0, len(ptnode.children), 2):
+            content_keys.append(ptnode.children[i])
+            content_types.append(
+                toast_v1(ptnode.children[i + 1], highlevel, categorical)
+            )
+        return ak.types.RecordType(content_types, content_keys, parameters=parms)
+
+    elif ptnode.data == "record_tuple_param":
+        parms = toast_v1(ptnode.children[-1], highlevel, False)
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        content_list = []
+        for node in ptnode.children[:-1]:
+            content_list.append(toast_v1(node, highlevel, False))
+        return ak.types.RecordType(tuple(content_list), parameters=parms)
+
+    elif ptnode.data == "record_struct":
+        parms = toast_v1(ptnode.children[-1], highlevel, False)
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        content_list = []
+        content_keys = []
+        for node in ptnode.children[:-1]:
+            if isinstance(node, str):
+                content_keys.append(node)
+            else:
+                content_list.append(toast_v1(node, highlevel, False))
+        return ak.types.RecordType(
+            tuple(content_list),
+            keys=content_keys,
+            parameters=parms,
+        )
+
+    elif ptnode.data == "record_highlevel":
+        assert highlevel
+        parms = {"__record__": ptnode.children[0]}
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        content_list = []
+        content_keys = []
+        for node in ptnode.children[1:]:
+            if isinstance(node, str):
+                content_keys.append(node)
+            else:
+                content_list.append(toast_v1(node, highlevel, False))
+        return ak.types.RecordType(
+            tuple(content_list),
+            keys=content_keys,
+            parameters=parms,
+        )
+
+    elif ptnode.data == "regular":
+        assert (len(ptnode.children)) == 1
+        return toast_v1(ptnode.children[0], highlevel, categorical)
+
+    elif ptnode.data == "regular_inparm":
+        assert len(ptnode.children) == 2
+        if highlevel:
+            return ak.types.ArrayType(
+                toast_v1(ptnode.children[1], highlevel, categorical), ptnode.children[0]
+            )
+        return ak.types.RegularType(
+            toast_v1(ptnode.children[1], highlevel, categorical), ptnode.children[0]
+        )
+
+    elif ptnode.data == "regular_outparm":
+        assert len(ptnode.children) == 3
+        parms = toast_v1(ptnode.children[2], highlevel, False)
+        if categorical:
+            parms.update({"__categorical__": True})
+            categorical = False
+        return ak.types.RegularType(
+            toast_v1(ptnode.children[1], highlevel, False),
+            ptnode.children[0],
+            parms,
+        )
+
+    elif ptnode.data == "def_option":
+        assert len(ptnode.children) == 1
+        return ptnode.children[0]
+
+    elif ptnode.data == "options":
+        assert len(ptnode.children) == 1
+        return toast_v1(ptnode.children[0], highlevel, categorical)
+
+    else:
+        raise AssertionError("unhandled node")
+
+
+def from_datashape_v1(typestr, highlevel=False):
+    parseobj = Lark_StandAlone(transformer=TreeToJson())
+    return toast_v1(parseobj.parse(typestr), highlevel, False)
diff --git a/studies/type-parser-1.5/type-grammar.lark b/studies/type-parser-1.5/type-grammar.lark
new file mode 100644
index 0000000000..436ca681b8
--- /dev/null
+++ b/studies/type-parser-1.5/type-grammar.lark
@@ -0,0 +1,82 @@
+start: input
+
+input: record
+     | unknown
+     | primitive
+     | optiontype
+     | regular
+     | listtype
+     | uniontype
+     | predefined_typestr
+     | categories
+
+record: record_tuple
+      | record_tuple_param
+      | record_dict
+      | record_struct
+      | record_highlevel
+primitive: TYPE [options]
+unknown: "unknown" [options]
+optiontype: option_single
+          | option_parm
+          | option_highlevel
+regular: regular_inparm
+       | regular_outparm
+listtype: list_single
+        | list_parm
+uniontype: union_single
+         | union_parm
+categories: "categorical" "[" "type" "=" input "]"
+predefined_typestr: HARDCODED
+
+HARDCODED: "string"
+         | "char"
+         | "bytes"
+         | "byte"
+option_single: "?" input [options]
+option_parm: "option" "[" input "," def_option "]"
+option_highlevel: "option" "[" input "]"
+union_single: "union" "[" input [ "," input]* "]"
+union_parm: "union" "[" input [ "," input]* "," def_option "]"
+list_parm: "[" "var" "*" input "," def_option "]"
+list_single: "var" "*" input
+record_tuple: "(" input ["," input]* ")"
+record_tuple_param: "tuple" "[" "[" input ["," input]* "]" "," def_option "]"
+record_dict: "{" string ":" input ["," string ":" input]* "}"
+record_struct: "struct" "[" "[" string ["," string]* "]" "," "[" input ["," input]* "]" "," def_option "]"
+record_highlevel: UNQUOTED_STRING "[" string ":" input ["," string ":" input]* "]"
+regular_inparm: number "*" input
+regular_outparm: "[" number "*" input "," def_option "]"
+options: "[" def_option "]"
+def_option: "parameters" "=" dict_obj
+TYPE: "int8"
+    | "int16"
+    | "int32"
+    | "int64"
+    | "int128"
+    | "uint8"
+    | "uint16"
+    | "uint32"
+    | "uint64"
+    | "uint128"
+    | "float32"
+    | "float64"
+    | "bool"
+?json: dict_obj
+     | list_obj
+     | string
+     | number
+     | "true"             -> true
+     | "false"            -> false
+     | "null"             -> null
+list_obj: "[" [json ("," json)*] "]"
+dict_obj: "{" [pair ("," pair)*] "}"
+pair: string ":" json
+string: /((?:"(?:[^"\n\r\\]|(?:\\u[0-9a-fA-F]{4})|(?:\\["bfnrt]))*")|(?:\\\'(?:[^\\\'\n\r\\]|(?:\\u[0-9a-fA-F]{4})|(?:\\[\'bfnrt]))*")*\\\')/
+UNQUOTED_STRING.-1: /[a-zA-Z]+/
+number: SIGNED_NUMBER
+
+%import common.SIGNED_NUMBER
+%import common.WS
+
+%ignore WS