Improve error reporting

lib/nearley.js

@@ -210,19 +210,34 @@
         // so the culprit is index-1
         var buffer = this.buffer;
         if (typeof buffer === 'string') {
+            var lines = buffer
+                .split("\n")
+                .slice(
+                    Math.max(0, this.line - 5), 
+                    this.line
+                );
+
             var nextLineBreak = buffer.indexOf('\n', this.index);
             if (nextLineBreak === -1) nextLineBreak = buffer.length;
-            var line = buffer.substring(this.lastLineBreak, nextLineBreak)
             var col = this.index - this.lastLineBreak;
+            var lastLineDigits = String(this.line).length;
             message += " at line " + this.line + " col " + col + ":\n\n";
-            message += "  " + line + "\n"
-            message += "  " + Array(col).join(" ") + "^"
+            message += lines
+                .map(function(line, i) {
+                    return pad(this.line - lines.length + i + 1, lastLineDigits) + " " + line;
+                }, this)
+                .join("\n");
+            message += "\n" + pad("", lastLineDigits + col) + "^\n";
             return message;
         } else {
             return message + " at index " + (this.index - 1);
         }
-    }
 
+        function pad(n, length) {
+            var s = String(n);
+            return Array(length - s.length + 1).join(" ") + s;
+        }
+    }
 
     function Parser(rules, start, options) {
         if (rules instanceof Grammar) {
@@ -266,7 +281,22 @@
         lexer.reset(chunk, this.lexerState);
 
         var token;
-        while (token = lexer.next()) {
+        while (true) {
+            try {
+                token = lexer.next();
+                if (!token) {
+                    break;
+                }
+            } catch (e) {
+                // Create the next column so that the error reporter
+                // can display the correctly predicted states.
+                var nextColumn = new Column(this.grammar, this.current + 1);
+                this.table.push(nextColumn);
+                var err = new Error(this.reportLexerError(e));
+                err.offset = this.current;
+                err.token = e.token;
+                throw err;
+            }
             // We add new states to table[current+1]
             var column = this.table[this.current];
 
@@ -334,39 +364,63 @@
         return this;
     };
 
+    Parser.prototype.reportLexerError = function(lexerError) {
+        var tokenDisplay, lexerMessage;
+        // Planning to add a token property to moo's thrown error
+        // even on erroring tokens to be used in error display below
+        var token = lexerError.token;
+        if (token) {
+            tokenDisplay = "input " + JSON.stringify(token.text[0]) + " (lexer error)";
+            lexerMessage = this.lexer.formatError(token, "Syntax error");
+        } else {
+            tokenDisplay = "input (lexer error)";
+            lexerMessage = lexerError.message;
+        }
+        return this.reportErrorCommon(lexerMessage, tokenDisplay);
+    };
+
     Parser.prototype.reportError = function(token) {
-        var lines = [];
         var tokenDisplay = (token.type ? token.type + " token: " : "") + JSON.stringify(token.value !== undefined ? token.value : token);
-        lines.push(this.lexer.formatError(token, "Syntax error"));
-        lines.push('Unexpected ' + tokenDisplay + '. Instead, I was expecting to see one of the following:\n');
+        var lexerMessage = this.lexer.formatError(token, "Syntax error");
+        return this.reportErrorCommon(lexerMessage, tokenDisplay);
+    };
+
+    Parser.prototype.reportErrorCommon = function(lexerMessage, tokenDisplay) {
+        var lines = [];
+        lines.push(lexerMessage);
         var lastColumnIndex = this.table.length - 2;
-        var lastColumn = this.table[lastColumnIndex];
+        var lastColumn = this.table[lastColumnIndex];    
         var expectantStates = lastColumn.states
             .filter(function(state) {
                 var nextSymbol = state.rule.symbols[state.dot];
                 return nextSymbol && typeof nextSymbol !== "string";
             });
 
-        // Display a "state stack" for each expectant state
-        // - which shows you how this state came to be, step by step.
-        // If there is more than one derivation, we only display the first one.
-        var stateStacks = expectantStates
-            .map(function(state) {
-                return this.buildFirstStateStack(state, []) || [state];
+        if (expectantStates.length === 0) {
+            lines.push('Unexpected ' + tokenDisplay + '. I did not expect any more input. Here is the state of my parse table:\n');
+            this.displayStateStack(lastColumn.states, lines);
+        } else {
+            lines.push('Unexpected ' + tokenDisplay + '. Instead, I was expecting to see one of the following:\n');
+            // Display a "state stack" for each expectant state
+            // - which shows you how this state came to be, step by step.
+            // If there is more than one derivation, we only display the first one.
+            var stateStacks = expectantStates
+                .map(function(state) {
+                    return this.buildFirstStateStack(state, []) || [state];
+                }, this);
+            // Display each state that is expecting a terminal symbol next.
+            stateStacks.forEach(function(stateStack) {
+                var state = stateStack[0];
+                var nextSymbol = state.rule.symbols[state.dot];
+                var symbolDisplay = this.getSymbolDisplay(nextSymbol);
+                lines.push('A ' + symbolDisplay + ' based on:');
+                this.displayStateStack(stateStack, lines);
             }, this);
-        // Display each state that is expecting a terminal symbol next.
-        stateStacks.forEach(function(stateStack) {
-            var state = stateStack[0];
-            var nextSymbol = state.rule.symbols[state.dot];
-            var symbolDisplay = this.getSymbolDisplay(nextSymbol);
-            lines.push('A ' + symbolDisplay + ' based on:');
-            this.displayStateStack(stateStack, lines);
-        }, this);
-
+        }
         lines.push("");
         return lines.join("\n");
-    };
-
+    }
+    
     Parser.prototype.displayStateStack = function(stateStack, lines) {
         var lastDisplay;
         var sameDisplayCount = 0;
@@ -377,7 +431,7 @@
                 sameDisplayCount++;
             } else {
                 if (sameDisplayCount > 0) {
-                    lines.push('    ⬆ ︎' + sameDisplayCount + ' more lines identical to this');
+                    lines.push('    ^ ' + sameDisplayCount + ' more lines identical to this');
                 }
                 sameDisplayCount = 0;
                 lines.push('    ' + display);