parser: add unimplemented nodes to AST instead of raising exceptions (fixes #88)

bashlex/ast.py

@@ -92,6 +92,11 @@ def visit(self, n):
             dochild = self._visitnode(n, n.command)
             if dochild is None or dochild:
                 self.visit(n.command)
+        elif k == 'unimplemented':
+            dochild = self._visitnode(n, n.parts)
+            if dochild is None or dochild:
+                for child in n.parts:
+                    self.visit(child)
         else:
             raise ValueError('unknown node kind %r' % k)
         self.visitnodeend(n)
@@ -144,6 +149,8 @@ def visitcase(self, node, parts):
         pass
     def visitpattern(self, node, parts):
         pass
+    def visitunimplemented(self, node, parts):
+        pass
 
 
 def _dump(tree, indent='  '):

bashlex/parser.py

@@ -13,6 +13,10 @@ def _partsspan(parts):
 )
 
 def handleNotImplemented(p, type):
+    if p.context._proceedonerror:
+        parts = _makeparts(p)
+        p[0] = ast.node(kind='unimplemented', parts=parts, pos=_partsspan(parts))
+        return
     if len(p) == 2:
         raise NotImplementedError('type = {%s}, token = {%s}' % (type, p[1]))
     else:
@@ -611,16 +615,16 @@ def get_correction_rightparen_states():
 yaccparser.action[states[1]]['RIGHT_PAREN'] = -148
 yaccparser.action[states[2]]['RIGHT_PAREN'] = -154
 
-def parsesingle(s, strictmode=True, expansionlimit=None, convertpos=False):
+def parsesingle(s, strictmode=True, expansionlimit=None, convertpos=False, proceedonerror=False):
     '''like parse, but only consumes a single top level node, e.g. parsing
     'a\nb' will only return a node for 'a', leaving b unparsed'''
-    p = _parser(s, strictmode=strictmode, expansionlimit=expansionlimit)
+    p = _parser(s, strictmode=strictmode, expansionlimit=expansionlimit, proceedonerror=proceedonerror)
     tree = p.parse()
     if convertpos:
         ast.posconverter(s).visit(tree)
     return tree
 
-def parse(s, strictmode=True, expansionlimit=None, convertpos=False):
+def parse(s, strictmode=True, expansionlimit=None, convertpos=False, proceedonerror=False):
     '''parse the input string, returning a list of nodes
 
     top level node kinds are:
@@ -638,8 +642,10 @@ def parse(s, strictmode=True, expansionlimit=None, convertpos=False):
 
     expansionlimit is used to limit the amount of recursive parsing done due to
     command substitutions found during word expansion.
+
+    when proceedonerror set, the parser will return AST nodes for unimplemented features, etc. (e.g., rather than throwing a NotImplementedError)
     '''
-    p = _parser(s, strictmode=strictmode, expansionlimit=expansionlimit)
+    p = _parser(s, strictmode=strictmode, expansionlimit=expansionlimit, proceedonerror=proceedonerror)
     parts = [p.parse()]
 
     class endfinder(ast.nodevisitor):
@@ -653,7 +659,7 @@ def visitheredoc(self, node, value):
     ef.visit(parts[-1])
     index = max(parts[-1].pos[1], ef.end) + 1
     while index < len(s):
-        part = _parser(s[index:], strictmode=strictmode).parse()
+        part = _parser(s[index:], strictmode=strictmode, proceedonerror=proceedonerror).parse()
 
         if not isinstance(part, ast.node):
             break
@@ -698,12 +704,14 @@ class _parser(object):
     when we're in the middle of parsing. as a hack, we shove it into the
     YaccProduction context attribute to make it accessible.
     '''
-    def __init__(self, s, strictmode=True, expansionlimit=None, tokenizerargs=None):
+    def __init__(self, s, strictmode=True, expansionlimit=None, tokenizerargs=None,
+                 proceedonerror=None):
         assert expansionlimit is None or isinstance(expansionlimit, int)
 
         self.s = s
         self._strictmode = strictmode
         self._expansionlimit = expansionlimit
+        self._proceedonerror = proceedonerror
 
         if tokenizerargs is None:
             tokenizerargs = {}

tests/test_parser.py

@@ -91,6 +91,9 @@ def patternnode(s, *parts):
 def functionnode(s, name, body, *parts):
     return ast.node(kind='function', name=name, body=body, parts=list(parts), s=s)
 
+def unimplementednode(s, *parts):
+    return ast.node(kind='unimplemented', parts=list(parts), s=s)
+
 class test_parser(unittest.TestCase):
 
     def setUp(self):
@@ -1237,3 +1240,13 @@ def test_case_clause_sequence(self):
           )
         )
       )
+
+    def test_unimplemented(self):
+      s = 'coproc echo'
+      self.assertASTEquals(s,
+              unimplementednode(s,
+                  reservedwordnode('coproc', 'coproc'),
+                  wordnode('echo', 'echo')),
+              proceedonerror=True)
+      with self.assertRaises(NotImplementedError):
+          parse(s, proceedonerror=False)