From d06fea95b1d7c4a959dea8d9119b00ccfd0bc33e Mon Sep 17 00:00:00 2001 From: Anton Medvedev Date: Mon, 2 Mar 2020 14:56:50 +0300 Subject: [PATCH] Fix lexer to emit correct location with range operator --- cmd/exe/main.go | 50 +++++++++++++++++++++---------------- optimizer/optimizer_test.go | 9 +++---- parser/lexer/lexer_test.go | 16 ++++++++++++ parser/lexer/state.go | 10 +++++++- 4 files changed, 58 insertions(+), 27 deletions(-) diff --git a/cmd/exe/main.go b/cmd/exe/main.go index ed7e89dca..1624cb328 100644 --- a/cmd/exe/main.go +++ b/cmd/exe/main.go @@ -19,13 +19,14 @@ import ( ) var ( - bytecode bool - debug bool - run bool - past bool - dot bool - repl bool - opt bool + bytecode bool + debug bool + run bool + past bool + dot bool + repl bool + opt bool + typeCheck bool ) func init() { @@ -36,6 +37,7 @@ func init() { flag.BoolVar(&dot, "dot", false, "dot format") flag.BoolVar(&repl, "repl", false, "start repl") flag.BoolVar(&opt, "opt", true, "do optimization") + flag.BoolVar(&typeCheck, "type", true, "do a type check") } func main() { @@ -85,13 +87,15 @@ func printAst() { tree, err := parser.Parse(input()) check(err) - _, err = checker.Check(tree, &conf.Config{ - AllowUndefinedVariables: true, - }) - check(err) + if typeCheck { + _, err = checker.Check(tree, &conf.Config{ + AllowUndefinedVariables: true, + }) + check(err) - if opt { - optimizer.Optimize(&tree.Node) + if opt { + optimizer.Optimize(&tree.Node) + } } if !dot { @@ -105,11 +109,13 @@ func printDisassemble() { tree, err := parser.Parse(input()) check(err) - _, err = checker.Check(tree, nil) - check(err) + if typeCheck { + _, err = checker.Check(tree, nil) + check(err) - if opt { - optimizer.Optimize(&tree.Node) + if opt { + optimizer.Optimize(&tree.Node) + } } program, err := compiler.Compile(tree, nil) @@ -122,11 +128,13 @@ func runProgram() { tree, err := parser.Parse(input()) check(err) - _, err = checker.Check(tree, nil) - check(err) + if typeCheck { + _, err = checker.Check(tree, nil) + check(err) - if opt { - optimizer.Optimize(&tree.Node) + if opt { + optimizer.Optimize(&tree.Node) + } } program, err := compiler.Compile(tree, nil) diff --git a/optimizer/optimizer_test.go b/optimizer/optimizer_test.go index c51f9f907..f07e11b64 100644 --- a/optimizer/optimizer_test.go +++ b/optimizer/optimizer_test.go @@ -8,7 +8,6 @@ import ( "github.com/antonmedv/expr/internal/conf" "github.com/antonmedv/expr/optimizer" "github.com/antonmedv/expr/parser" - "github.com/sanity-io/litter" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -24,7 +23,7 @@ func TestOptimize_constant_folding(t *testing.T) { Index: &ast.IntegerNode{Value: 0}, } - assert.Equal(t, litter.Sdump(expected), litter.Sdump(tree.Node)) + assert.Equal(t, ast.Dump(expected), ast.Dump(tree.Node)) } func TestOptimize_in_array(t *testing.T) { @@ -44,7 +43,7 @@ func TestOptimize_in_array(t *testing.T) { Right: &ast.ConstantNode{Value: optimizer.Map{1: {}, 2: {}, 3: {}}}, } - assert.Equal(t, litter.Sdump(expected), litter.Sdump(tree.Node)) + assert.Equal(t, ast.Dump(expected), ast.Dump(tree.Node)) } func TestOptimize_in_range(t *testing.T) { @@ -74,7 +73,7 @@ func TestOptimize_in_range(t *testing.T) { }, } - assert.Equal(t, litter.Sdump(expected), litter.Sdump(tree.Node)) + assert.Equal(t, ast.Dump(expected), ast.Dump(tree.Node)) } func TestOptimize_const_range(t *testing.T) { @@ -87,5 +86,5 @@ func TestOptimize_const_range(t *testing.T) { Value: []int{-1, 0, 1}, } - assert.Equal(t, litter.Sdump(expected), litter.Sdump(tree.Node)) + assert.Equal(t, ast.Dump(expected), ast.Dump(tree.Node)) } diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go index 68065993a..35c8673a9 100644 --- a/parser/lexer/lexer_test.go +++ b/parser/lexer/lexer_test.go @@ -3,6 +3,7 @@ package lexer_test import ( "fmt" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "strings" "testing" @@ -117,6 +118,21 @@ func TestLex(t *testing.T) { } } +func TestLex_location(t *testing.T) { + source := file.NewSource("1..2 3..4") + tokens, err := Lex(source) + require.NoError(t, err) + require.Equal(t, []Token{ + {Location: file.Location{Line: 1, Column: 0}, Kind: Number, Value: "1"}, + {Location: file.Location{Line: 1, Column: 1}, Kind: Operator, Value: ".."}, + {Location: file.Location{Line: 1, Column: 3}, Kind: Number, Value: "2"}, + {Location: file.Location{Line: 1, Column: 5}, Kind: Number, Value: "3"}, + {Location: file.Location{Line: 1, Column: 6}, Kind: Operator, Value: ".."}, + {Location: file.Location{Line: 1, Column: 8}, Kind: Number, Value: "4"}, + {Location: file.Location{Line: 1, Column: 8}, Kind: EOF, Value: ""}, + }, tokens) +} + const errorTests = ` "\xQA" invalid char escape (1:5) diff --git a/parser/lexer/state.go b/parser/lexer/state.go index 81ac76713..bc18e95fb 100644 --- a/parser/lexer/state.go +++ b/parser/lexer/state.go @@ -67,10 +67,18 @@ func (l *lexer) scanNumber() bool { } } l.acceptRun(digits) + end := l.end + loc := l.loc + prev := l.prev if l.accept(".") { // Lookup for .. operator: if after dot there is another dot (1..2), it maybe a range operator. if l.peek() == '.' { - l.backup() + // We can't backup() here, as it would require two backups, + // and backup() func supports only one for now. So, save and + // restore it here. + l.end = end + l.loc = loc + l.prev = prev return true } l.acceptRun(digits)