JS: allow lexer to continue after unexpected character, see #118

input.go

@@ -92,7 +92,7 @@ func (z *Input) Err() error {
 func (z *Input) PeekErr(pos int) error {
 	if z.err != nil {
 		return z.err
-	} else if z.pos+pos >= len(z.buf)-1 {
+	} else if len(z.buf)-1 <= z.pos+pos {
 		return io.EOF
 	}
 	return nil
@@ -109,11 +109,11 @@ func (z *Input) Peek(pos int) byte {
 func (z *Input) PeekRune(pos int) (rune, int) {
 	// from unicode/utf8
 	c := z.Peek(pos)
-	if c < 0xC0 || z.Peek(pos+1) == 0 {
+	if c < 0xC0 || len(z.buf)-1-z.pos < 2 {
 		return rune(c), 1
-	} else if c < 0xE0 || z.Peek(pos+2) == 0 {
+	} else if c < 0xE0 || len(z.buf)-1-z.pos < 3 {
 		return rune(c&0x1F)<<6 | rune(z.Peek(pos+1)&0x3F), 2
-	} else if c < 0xF0 || z.Peek(pos+3) == 0 {
+	} else if c < 0xF0 || len(z.buf)-1-z.pos < 4 {
 		return rune(c&0x0F)<<12 | rune(z.Peek(pos+1)&0x3F)<<6 | rune(z.Peek(pos+2)&0x3F), 3
 	}
 	return rune(c&0x07)<<18 | rune(z.Peek(pos+1)&0x3F)<<12 | rune(z.Peek(pos+2)&0x3F)<<6 | rune(z.Peek(pos+3)&0x3F), 4
@@ -124,6 +124,20 @@ func (z *Input) Move(n int) {
 	z.pos += n
 }
 
+// MoveRune advances the position by the length of the current rune.
+func (z *Input) MoveRune() {
+	c := z.Peek(0)
+	if c < 0xC0 || len(z.buf)-1-z.pos < 2 {
+		z.pos++
+	} else if c < 0xE0 || len(z.buf)-1-z.pos < 3 {
+		z.pos += 2
+	} else if c < 0xF0 || len(z.buf)-1-z.pos < 4 {
+		z.pos += 3
+	} else {
+		z.pos += 4
+	}
+}
+
 // Pos returns a mark to which can be rewinded.
 func (z *Input) Pos() int {
 	return z.pos - z.start

js/lex.go

@@ -221,6 +221,7 @@ func (l *Lexer) Next() (TokenType, []byte) {
 
 	r, _ := l.r.PeekRune(0)
 	l.err = parse.NewErrorLexer(l.r, "unexpected %s", parse.Printable(r))
+	l.r.MoveRune() // allow to continue after error
 	return ErrorToken, l.r.Shift()
 }
 

js/lex_test.go

@@ -275,6 +275,16 @@ func TestLexerErrors(t *testing.T) {
 	l.Next()
 	l.RegExp()
 	test.T(t, l.Err().(*parse.Error).Message, "unexpected EOF or newline")
+
+	// see #118
+	l = NewLexer(parse.NewInputString("\uFFFDa"))
+	tt, data := l.Next()
+	test.T(t, tt, ErrorToken)
+	test.T(t, string(data), "�")
+	test.T(t, l.Err().(*parse.Error).Message, "unexpected �")
+	tt, data = l.Next()
+	test.T(t, tt, IdentifierToken)
+	test.T(t, string(data), "a")
 }
 
 ////////////////////////////////////////////////////////////////