From 6d446f724bb86d06b3a0f59311b2906b9db67710 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Tue, 10 Dec 2019 10:09:48 -0500 Subject: [PATCH] ast: New parser implementation This commit replaces the existing PEG generated parser with a parser implemented by hand. The new parser is more efficient (avoiding old problems with pathological input cases like {{{{{{{{{}}}}}}}} and deeply-nested composites in general) and offers better opportunities for improved error reporting (which has been improved already but there is still room to grow.) During the test process of implementing the new parser, we identified a few issues that were present in the old parser. Those issues are fixed by this commit. Fixes #1251 Fixes #501 Fixes #2198 Fixes #2199 Fixes #2200 Fixes #2201 Fixes #2202 Fixes #2203 Co-authored-by: Torin Sandall Co-authored-by: Patrick East Signed-off-by: Torin Sandall Signed-off-by: Patrick East --- CHANGELOG.md | 91 + ast/check_test.go | 28 +- ast/compile_test.go | 30 +- ast/internal/scanner/scanner.go | 387 ++ ast/internal/scanner/scanner_test.go | 205 + ast/internal/tokens/tokens.go | 138 + ast/location/location.go | 90 + ast/location/location_test.go | 87 + ast/parser.go | 5965 ++++------------- ast/parser_ext.go | 107 +- ast/parser_internal.go | 620 -- ast/parser_test.go | 661 +- ast/rego.peg | 311 - ast/term.go | 79 +- ast/term_test.go | 80 - cmd/bench_test.go | 4 +- docs/devel/DEVELOPMENT.md | 21 +- format/format.go | 18 +- format/format_test.go | 2 +- format/testfiles/test.rego.formatted | 20 +- go.mod | 2 - go.sum | 9 - internal/presentation/presentation_test.go | 2 +- main.go | 5 +- server/server_test.go | 2 +- tools.go | 1 - topdown/bits_test.go | 4 +- topdown/parse_test.go | 2 +- topdown/topdown_test.go | 2 +- vendor/github.com/mna/pigeon/.editorconfig | 6 - vendor/github.com/mna/pigeon/.gitattributes | 2 - vendor/github.com/mna/pigeon/.gitignore | 22 - vendor/github.com/mna/pigeon/.travis.yml | 12 - vendor/github.com/mna/pigeon/CONTRIBUTING.md | 33 - vendor/github.com/mna/pigeon/LICENSE | 12 - vendor/github.com/mna/pigeon/Makefile | 177 - vendor/github.com/mna/pigeon/README.md | 146 - vendor/github.com/mna/pigeon/TODO | 3 - vendor/github.com/mna/pigeon/ast/ast.go | 662 -- .../github.com/mna/pigeon/ast/ast_optimize.go | 454 -- vendor/github.com/mna/pigeon/ast/ast_walk.go | 87 - .../github.com/mna/pigeon/builder/builder.go | 812 --- .../pigeon/builder/generated_static_code.go | 1442 ---- .../generated_static_code_range_table.go | 21 - .../mna/pigeon/builder/static_code.go | 1457 ---- .../pigeon/builder/static_code_range_table.go | 24 - vendor/github.com/mna/pigeon/doc.go | 594 -- vendor/github.com/mna/pigeon/main.go | 294 - vendor/github.com/mna/pigeon/pigeon.go | 4418 ------------ .../github.com/mna/pigeon/reserved_words.go | 71 - .../github.com/mna/pigeon/unicode_classes.go | 200 - vendor/golang.org/x/tools/imports/forward.go | 62 - vendor/modules.txt | 5 - 53 files changed, 2831 insertions(+), 17158 deletions(-) create mode 100644 ast/internal/scanner/scanner.go create mode 100644 ast/internal/scanner/scanner_test.go create mode 100644 ast/internal/tokens/tokens.go create mode 100644 ast/location/location.go create mode 100644 ast/location/location_test.go delete mode 100644 ast/parser_internal.go delete mode 100644 ast/rego.peg delete mode 100644 vendor/github.com/mna/pigeon/.editorconfig delete mode 100644 vendor/github.com/mna/pigeon/.gitattributes delete mode 100644 vendor/github.com/mna/pigeon/.gitignore delete mode 100644 vendor/github.com/mna/pigeon/.travis.yml delete mode 100644 vendor/github.com/mna/pigeon/CONTRIBUTING.md delete mode 100644 vendor/github.com/mna/pigeon/LICENSE delete mode 100644 vendor/github.com/mna/pigeon/Makefile delete mode 100644 vendor/github.com/mna/pigeon/README.md delete mode 100644 vendor/github.com/mna/pigeon/TODO delete mode 100644 vendor/github.com/mna/pigeon/ast/ast.go delete mode 100644 vendor/github.com/mna/pigeon/ast/ast_optimize.go delete mode 100644 vendor/github.com/mna/pigeon/ast/ast_walk.go delete mode 100644 vendor/github.com/mna/pigeon/builder/builder.go delete mode 100644 vendor/github.com/mna/pigeon/builder/generated_static_code.go delete mode 100644 vendor/github.com/mna/pigeon/builder/generated_static_code_range_table.go delete mode 100644 vendor/github.com/mna/pigeon/builder/static_code.go delete mode 100644 vendor/github.com/mna/pigeon/builder/static_code_range_table.go delete mode 100644 vendor/github.com/mna/pigeon/doc.go delete mode 100644 vendor/github.com/mna/pigeon/main.go delete mode 100644 vendor/github.com/mna/pigeon/pigeon.go delete mode 100644 vendor/github.com/mna/pigeon/reserved_words.go delete mode 100644 vendor/github.com/mna/pigeon/unicode_classes.go delete mode 100644 vendor/golang.org/x/tools/imports/forward.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a550db154..2f8e31ae74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,97 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +### New Parser + +The next minor release includes a new parser implementation that resolves a number +of existing issues with the old parser. As part of implementing the new parser +a small number of backwards incompatible changes have been made. + +#### Backwards Compatibility + +The new parser contains a small number of backwards incompatible changes that +correct questionable behaviour from the old parser. These changes affect +a very small number of actual policies and we feel confident in the decision to +break backwards compatibility here. + +- Numbers no longer lose-precision [#501](https://github.com/open-policy-agent/opa/issues/501) +- Leading commas do not cause objects to lose values [#2198](https://github.com/open-policy-agent/opa/issues/2198) +- Rules wrapped with braces no longer parse [#2199](https://github.com/open-policy-agent/opa/issues/2199) +- Rule names can no longer contain dots/hyphens [#2200](https://github.com/open-policy-agent/opa/issues/2200) +- Object comprehensions now have priority over logical OR in all cases [#2201](https://github.com/open-policy-agent/opa/issues/2201) + +In addition there are a few small changes backwards incompatible changes in APIs: + +- The `message` field on `rego_parse_error` objects contains a human-readable description + of the parse error. The old parser would often report "no match found" to indicate + the input contained invalid syntax. The new parser has slightly more specific + errors. If you integrated with OPA and implemented error handling based on the + content of these human-readable error message strings, your integration may be affected. +- The `github.com/open-policy-agent/opa/format#Bytes` function has been removed (it was unused.) + +#### Benchmark Results + +The output below shows the Go `benchstat` result for master (5a5d2a42) compared to the new parser. + +``` +name old time/op new time/op delta +ParseModuleRulesBase/1-16 210µs ± 1% 4µs ± 1% -98.02% (p=0.008 n=5+5) +ParseModuleRulesBase/10-16 1.39ms ± 1% 0.03ms ± 0% -97.93% (p=0.008 n=5+5) +ParseModuleRulesBase/100-16 13.5ms ± 1% 0.3ms ± 1% -97.93% (p=0.008 n=5+5) +ParseModuleRulesBase/1000-16 148ms ± 5% 3ms ± 6% -97.77% (p=0.008 n=5+5) +ParseStatementBasicCall-16 141µs ± 5% 3µs ± 1% -97.92% (p=0.008 n=5+5) +ParseStatementMixedJSON-16 9.06ms ± 2% 0.07ms ± 1% -99.19% (p=0.008 n=5+5) +ParseStatementSimpleArray/1-16 131µs ± 6% 2µs ± 1% -98.10% (p=0.008 n=5+5) +ParseStatementSimpleArray/10-16 499µs ± 6% 7µs ± 2% -98.54% (p=0.008 n=5+5) +ParseStatementSimpleArray/100-16 4.00ms ± 2% 0.06ms ± 4% -98.58% (p=0.008 n=5+5) +ParseStatementSimpleArray/1000-16 42.0ms ± 3% 0.5ms ± 4% -98.70% (p=0.008 n=5+5) +ParseStatementNestedObjects/1x1-16 233µs ± 6% 4µs ± 3% -98.49% (p=0.008 n=5+5) +ParseStatementNestedObjects/5x1-16 514µs ± 0% 9µs ± 4% -98.33% (p=0.008 n=5+5) +ParseStatementNestedObjects/10x1-16 911µs ± 5% 14µs ± 5% -98.46% (p=0.008 n=5+5) +ParseStatementNestedObjects/1x5-16 4.24ms ± 1% 0.01ms ± 1% -99.82% (p=0.016 n=4+5) +ParseStatementNestedObjects/1x10-16 138ms ± 1% 0ms ± 1% -99.99% (p=0.008 n=5+5) +ParseStatementNestedObjects/5x5-16 714ms ± 0% 5ms ± 5% -99.26% (p=0.016 n=4+5) +ParseBasicABACModule-16 3.12ms ± 3% 0.04ms ± 4% -98.63% (p=0.008 n=5+5) + +name old alloc/op new alloc/op delta +ParseModuleRulesBase/1-16 99.2kB ± 0% 5.7kB ± 0% -94.30% (p=0.008 n=5+5) +ParseModuleRulesBase/10-16 600kB ± 0% 29kB ± 0% -95.16% (p=0.008 n=5+5) +ParseModuleRulesBase/100-16 5.72MB ± 0% 0.27MB ± 0% -95.34% (p=0.008 n=5+5) +ParseModuleRulesBase/1000-16 58.0MB ± 0% 2.7MB ± 0% -95.42% (p=0.008 n=5+5) +ParseStatementBasicCall-16 70.2kB ± 0% 5.0kB ± 0% -92.82% (p=0.008 n=5+5) +ParseStatementMixedJSON-16 3.64MB ± 0% 0.06MB ± 0% -98.34% (p=0.008 n=5+5) +ParseStatementSimpleArray/1-16 63.7kB ± 0% 4.8kB ± 0% -92.42% (p=0.008 n=5+5) +ParseStatementSimpleArray/10-16 205kB ± 0% 8kB ± 0% -96.00% (p=0.008 n=5+5) +ParseStatementSimpleArray/100-16 1.64MB ± 0% 0.05MB ± 0% -97.19% (p=0.008 n=5+5) +ParseStatementSimpleArray/1000-16 16.5MB ± 0% 0.4MB ± 0% -97.50% (p=0.008 n=5+5) +ParseStatementNestedObjects/1x1-16 98.6kB ± 0% 5.7kB ± 0% -94.22% (p=0.008 n=5+5) +ParseStatementNestedObjects/5x1-16 224kB ± 0% 9kB ± 0% -96.05% (p=0.008 n=5+5) +ParseStatementNestedObjects/10x1-16 381kB ± 0% 13kB ± 0% -96.63% (p=0.008 n=5+5) +ParseStatementNestedObjects/1x5-16 1.76MB ± 0% 0.01MB ± 0% -99.38% (p=0.008 n=5+5) +ParseStatementNestedObjects/1x10-16 56.2MB ± 0% 0.0MB ± 0% -99.97% (p=0.008 n=5+5) +ParseStatementNestedObjects/5x5-16 280MB ± 0% 4MB ± 0% -98.67% (p=0.008 n=5+5) +ParseBasicABACModule-16 1.27MB ± 0% 0.04MB ± 0% -97.08% (p=0.008 n=5+5) + +name old allocs/op new allocs/op delta +ParseModuleRulesBase/1-16 2.28k ± 0% 0.07k ± 0% -96.75% (p=0.008 n=5+5) +ParseModuleRulesBase/10-16 16.1k ± 0% 0.5k ± 0% -96.59% (p=0.008 n=5+5) +ParseModuleRulesBase/100-16 159k ± 0% 5k ± 0% -96.64% (p=0.008 n=5+5) +ParseModuleRulesBase/1000-16 1.62M ± 0% 0.05M ± 0% -96.72% (p=0.008 n=5+5) +ParseStatementBasicCall-16 1.36k ± 0% 0.05k ± 0% -96.25% (p=0.008 n=5+5) +ParseStatementMixedJSON-16 105k ± 0% 1k ± 0% ~ (p=0.079 n=4+5) +ParseStatementSimpleArray/1-16 1.34k ± 0% 0.04k ± 0% -97.09% (p=0.008 n=5+5) +ParseStatementSimpleArray/10-16 5.49k ± 0% 0.12k ± 0% -97.90% (p=0.008 n=5+5) +ParseStatementSimpleArray/100-16 47.8k ± 0% 0.8k ± 0% ~ (p=0.079 n=4+5) +ParseStatementSimpleArray/1000-16 481k ± 0% 8k ± 0% -98.33% (p=0.008 n=5+5) +ParseStatementNestedObjects/1x1-16 2.38k ± 0% 0.05k ± 0% -97.82% (p=0.008 n=5+5) +ParseStatementNestedObjects/5x1-16 6.02k ± 0% 0.12k ± 0% -97.94% (p=0.008 n=5+5) +ParseStatementNestedObjects/10x1-16 10.6k ± 0% 0.2k ± 0% -98.01% (p=0.008 n=5+5) +ParseStatementNestedObjects/1x5-16 51.2k ± 0% 0.1k ± 0% ~ (p=0.079 n=4+5) +ParseStatementNestedObjects/1x10-16 1.66M ± 0% 0.00M ± 0% -99.99% (p=0.008 n=5+5) +ParseStatementNestedObjects/5x5-16 8.16M ± 0% 0.07M ± 0% -99.13% (p=0.008 n=5+5) +ParseBasicABACModule-16 36.5k ± 0% 0.7k ± 0% -98.09% (p=0.008 n=5+5) +``` + ## 0.18.0 ### Features diff --git a/ast/check_test.go b/ast/check_test.go index 0d73b4732a..78a2170ad6 100644 --- a/ast/check_test.go +++ b/ast/check_test.go @@ -137,7 +137,7 @@ func TestCheckInference(t *testing.T) { obj[i] = v1; arr[j] = v2; set[v3]; - obj = {"foo": "bar"} + obj = {"foo": "bar"}; arr = [1]; set = {1,2,3} `, map[Var]types.Type{ @@ -574,19 +574,19 @@ func TestCheckMatchErrors(t *testing.T) { note string query string }{ - {"null", "{ null = true }"}, - {"boolean", "{ true = null }"}, - {"number", "{ 1 = null }"}, - {"string", `{ "hello" = null }`}, - {"array", "{[1,2,3] = null}"}, - {"array-nested", `{[1,2,3] = [1,2,"3"]}`}, - {"array-nested-2", `{[1,2] = [1,2,3]}`}, - {"array-dynamic", `{ [ true | true ] = [x | a = [1, "foo"]; x = a[_]] }`}, - {"object", `{{"a": 1, "b": 2} = null}`}, - {"object-nested", `{ {"a": 1, "b": "2"} = {"a": 1, "b": 2} }`}, - {"object-nested-2", `{ {"a": 1} = {"a": 1, "b": "2"} }`}, - {"set", "{{1,2,3} = null}"}, - {"any", `{x = ["str", 1]; x[_] = null}`}, + {"null", "null = true"}, + {"boolean", "true = null"}, + {"number", "1 = null"}, + {"string", `"hello" = null`}, + {"array", "[1,2,3] = null"}, + {"array-nested", `[1,2,3] = [1,2,"3"]`}, + {"array-nested-2", `[1,2] = [1,2,3]`}, + {"array-dynamic", `[ true | true ] = [x | a = [1, "foo"]; x = a[_]]`}, + {"object", `{"a": 1, "b": 2} = null`}, + {"object-nested", `{"a": 1, "b": "2"} = {"a": 1, "b": 2}`}, + {"object-nested-2", `{"a": 1} = {"a": 1, "b": "2"}`}, + {"set", "{1,2,3} = null"}, + {"any", `x = ["str", 1]; x[_] = null`}, } for _, tc := range tests { test.Subtest(t, tc.note, func(t *testing.T) { diff --git a/ast/compile_test.go b/ast/compile_test.go index 9be1760434..7fc1f06811 100644 --- a/ast/compile_test.go +++ b/ast/compile_test.go @@ -2118,27 +2118,27 @@ func TestCompilerRewriteDynamicTerms(t *testing.T) { input string expected string }{ - {`arr { [str] }`, `{__local0__ = data.test.str; [__local0__]}`}, - {`arr2 { [[str]] }`, `{__local0__ = data.test.str; [[__local0__]]}`}, - {`obj { {"x": str} }`, `{__local0__ = data.test.str; {"x": __local0__}}`}, - {`obj2 { {"x": {"y": str}} }`, `{__local0__ = data.test.str; {"x": {"y": __local0__}}}`}, - {`set { {str} }`, `{__local0__ = data.test.str; {__local0__}}`}, - {`set2 { {{str}} }`, `{__local0__ = data.test.str; {{__local0__}}}`}, - {`ref { str[str] }`, `{__local0__ = data.test.str; data.test.str[__local0__]}`}, - {`ref2 { str[str[str]] }`, `{__local0__ = data.test.str; __local1__ = data.test.str[__local0__]; data.test.str[__local1__]}`}, + {`arr { [str] }`, `__local0__ = data.test.str; [__local0__]`}, + {`arr2 { [[str]] }`, `__local0__ = data.test.str; [[__local0__]]`}, + {`obj { {"x": str} }`, `__local0__ = data.test.str; {"x": __local0__}`}, + {`obj2 { {"x": {"y": str}} }`, `__local0__ = data.test.str; {"x": {"y": __local0__}}`}, + {`set { {str} }`, `__local0__ = data.test.str; {__local0__}`}, + {`set2 { {{str}} }`, `__local0__ = data.test.str; {{__local0__}}`}, + {`ref { str[str] }`, `__local0__ = data.test.str; data.test.str[__local0__]`}, + {`ref2 { str[str[str]] }`, `__local0__ = data.test.str; __local1__ = data.test.str[__local0__]; data.test.str[__local1__]`}, {`arr_compr { [1 | [str]] }`, `[1 | __local0__ = data.test.str; [__local0__]]`}, {`arr_compr2 { [1 | [1 | [str]]] }`, `[1 | [1 | __local0__ = data.test.str; [__local0__]]]`}, {`set_compr { {1 | [str]} }`, `{1 | __local0__ = data.test.str; [__local0__]}`}, {`set_compr2 { {1 | {1 | [str]}} }`, `{1 | {1 | __local0__ = data.test.str; [__local0__]}}`}, {`obj_compr { {"a": "b" | [str]} }`, `{"a": "b" | __local0__ = data.test.str; [__local0__]}`}, {`obj_compr2 { {"a": "b" | {"a": "b" | [str]}} }`, `{"a": "b" | {"a": "b" | __local0__ = data.test.str; [__local0__]}}`}, - {`equality { str = str }`, `{data.test.str = data.test.str}`}, - {`equality2 { [str] = [str] }`, `{__local0__ = data.test.str; __local1__ = data.test.str; [__local0__] = [__local1__]}`}, - {`call { startswith(str, "") }`, `{__local0__ = data.test.str; startswith(__local0__, "")}`}, - {`call2 { count([str], n) }`, `{__local0__ = data.test.str; count([__local0__], n)}`}, - {`eq_with { [str] = [1] with input as 1 }`, `{__local0__ = data.test.str with input as 1; [__local0__] = [1] with input as 1}`}, - {`term_with { [[str]] with input as 1 }`, `{__local0__ = data.test.str with input as 1; [[__local0__]] with input as 1}`}, - {`call_with { count(str) with input as 1 }`, `{__local0__ = data.test.str with input as 1; count(__local0__) with input as 1}`}, + {`equality { str = str }`, `data.test.str = data.test.str`}, + {`equality2 { [str] = [str] }`, `__local0__ = data.test.str; __local1__ = data.test.str; [__local0__] = [__local1__]`}, + {`call { startswith(str, "") }`, `__local0__ = data.test.str; startswith(__local0__, "")`}, + {`call2 { count([str], n) }`, `__local0__ = data.test.str; count([__local0__], n)`}, + {`eq_with { [str] = [1] with input as 1 }`, `__local0__ = data.test.str with input as 1; [__local0__] = [1] with input as 1`}, + {`term_with { [[str]] with input as 1 }`, `__local0__ = data.test.str with input as 1; [[__local0__]] with input as 1`}, + {`call_with { count(str) with input as 1 }`, `__local0__ = data.test.str with input as 1; count(__local0__) with input as 1`}, } for _, tc := range tests { diff --git a/ast/internal/scanner/scanner.go b/ast/internal/scanner/scanner.go new file mode 100644 index 0000000000..8223e59e6a --- /dev/null +++ b/ast/internal/scanner/scanner.go @@ -0,0 +1,387 @@ +// Copyright 2020 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package scanner + +import ( + "fmt" + "io" + "io/ioutil" + "unicode" + "unicode/utf8" + + "github.com/open-policy-agent/opa/ast/internal/tokens" +) + +const bom = 0xFEFF + +// Scanner is used to tokenize an input stream of +// Rego source code. +type Scanner struct { + offset int + row int + col int + bs []byte + curr rune + width int + errors []Error + filename string +} + +// Error represents a scanner error. +type Error struct { + Pos Position + Message string +} + +// Position represents a point in the scanned source code. +type Position struct { + Offset int // start offset in bytes + End int // end offset in bytes + Row int // line number computed in bytes + Col int // column number computed in bytes +} + +// New returns an initialized scanner that will scan +// through the source code provided by the io.Reader. +func New(r io.Reader) (*Scanner, error) { + + bs, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + s := &Scanner{ + offset: 0, + row: 1, + col: 0, + bs: bs, + curr: -1, + width: 0, + } + + s.next() + + if s.curr == bom { + s.next() + } + + return s, nil +} + +// Bytes returns the raw bytes for the full source +// which the scanner has read in. +func (s *Scanner) Bytes() []byte { + return s.bs +} + +// String returns a human readable string of the current scanner state. +func (s *Scanner) String() string { + return fmt.Sprintf("", s.curr, s.offset, len(s.bs)) +} + +// Scan will increment the scanners position in the source +// code until the next token is found. The token, starting position +// of the token, string literal, and any errors encountered are +// returned. A token will always be returned, the caller must check +// for any errors before using the other values. +func (s *Scanner) Scan() (tokens.Token, Position, string, []Error) { + + pos := Position{Offset: s.offset - s.width, Row: s.row, Col: s.col} + var tok tokens.Token + var lit string + + if s.isWhitespace() { + lit = string(s.curr) + s.next() + tok = tokens.Whitespace + } else if isLetter(s.curr) { + lit = s.scanIdentifier() + tok = tokens.Keyword(lit) + } else if isDecimal(s.curr) { + lit = s.scanNumber() + tok = tokens.Number + } else { + ch := s.curr + s.next() + switch ch { + case -1: + tok = tokens.EOF + case '#': + lit = s.scanComment() + tok = tokens.Comment + case '"': + lit = s.scanString() + tok = tokens.String + case '`': + lit = s.scanRawString() + tok = tokens.String + case '[': + tok = tokens.LBrack + case ']': + tok = tokens.RBrack + case '{': + tok = tokens.LBrace + case '}': + tok = tokens.RBrace + case '(': + tok = tokens.LParen + case ')': + tok = tokens.RParen + case ',': + tok = tokens.Comma + case ':': + if s.curr == '=' { + s.next() + tok = tokens.Assign + } else { + tok = tokens.Colon + } + case '+': + tok = tokens.Add + case '-': + tok = tokens.Sub + case '*': + tok = tokens.Mul + case '/': + tok = tokens.Quo + case '%': + tok = tokens.Rem + case '&': + tok = tokens.And + case '|': + tok = tokens.Or + case '=': + if s.curr == '=' { + s.next() + tok = tokens.Equal + } else { + tok = tokens.Unify + } + case '>': + if s.curr == '=' { + s.next() + tok = tokens.Gte + } else { + tok = tokens.Gt + } + case '<': + if s.curr == '=' { + s.next() + tok = tokens.Lte + } else { + tok = tokens.Lt + } + case '!': + if s.curr == '=' { + s.next() + tok = tokens.Neq + } else { + s.error("illegal ! character") + } + case ';': + tok = tokens.Semicolon + case '.': + tok = tokens.Dot + } + } + + pos.End = s.offset - s.width + errs := s.errors + s.errors = nil + + return tok, pos, lit, errs +} + +func (s *Scanner) scanIdentifier() string { + start := s.offset - 1 + for isLetter(s.curr) || isDigit(s.curr) { + s.next() + } + return string(s.bs[start : s.offset-1]) +} + +func (s *Scanner) scanNumber() string { + + start := s.offset - 1 + + if s.curr != '.' { + for isDecimal(s.curr) { + s.next() + } + } + + if s.curr == '.' { + s.next() + var found bool + for isDecimal(s.curr) { + s.next() + found = true + } + if !found { + s.error("expected fraction") + } + } + + if lower(s.curr) == 'e' { + s.next() + if s.curr == '+' || s.curr == '-' { + s.next() + } + var found bool + for isDecimal(s.curr) { + s.next() + found = true + } + if !found { + s.error("expected exponent") + } + } + + // Scan any digits following the decimals to get the + // entire invalid number/identifier. + // Example: 0a2b should be a single invalid number "0a2b" + // rather than a number "0", followed by identifier "a2b". + if isLetter(s.curr) { + s.error("illegal number format") + for isLetter(s.curr) || isDigit(s.curr) { + s.next() + } + } + + return string(s.bs[start : s.offset-1]) +} + +func (s *Scanner) scanString() string { + start := s.literalStart() + for { + ch := s.curr + + if ch == '\n' || ch < 0 { + s.error("non-terminated string") + break + } + + s.next() + + if ch == '"' { + break + } + + if ch == '\\' { + switch s.curr { + case '\\', '"', '/', 'b', 'f', 'n', 'r', 't': + s.next() + case 'u': + s.next() + s.next() + s.next() + s.next() + default: + s.error("illegal escape sequence") + } + } + } + + return string(s.bs[start : s.offset-1]) +} + +func (s *Scanner) scanRawString() string { + start := s.literalStart() + for { + ch := s.curr + s.next() + if ch == '`' { + break + } else if ch < 0 { + s.error("non-terminated string") + break + } + } + return string(s.bs[start : s.offset-1]) +} + +func (s *Scanner) scanComment() string { + start := s.literalStart() + for s.curr != '\n' && s.curr != -1 { + s.next() + } + end := s.offset - 1 + // Trim carriage returns that precede the newline + if s.offset > 1 && s.bs[s.offset-2] == '\r' { + end = end - 1 + } + return string(s.bs[start:end]) +} + +func (s *Scanner) next() { + + if s.offset >= len(s.bs) { + s.curr = -1 + s.offset = len(s.bs) + 1 + return + } + + s.curr = rune(s.bs[s.offset]) + s.width = 1 + + if s.curr == 0 { + s.error("illegal null character") + } else if s.curr >= utf8.RuneSelf { + s.curr, s.width = utf8.DecodeRune(s.bs[s.offset:]) + if s.curr == utf8.RuneError && s.width == 1 { + s.error("illegal utf-8 character") + } else if s.curr == bom && s.offset > 0 { + s.error("illegal byte-order mark") + } + } + + s.offset += s.width + + if s.curr == '\n' { + s.row++ + s.col = 0 + } else { + s.col++ + } +} + +func (s *Scanner) peek(i int) rune { + if s.offset+i < len(s.bs) { + return rune(s.bs[s.offset+i]) + } + return 0 +} + +func (s *Scanner) literalStart() int { + // The current offset is at the first character past the literal delimiter (#, ", `, etc.) + // Need to subtract width of first character (plus one for the delimiter). + return s.offset - (s.width + 1) +} + +// From the Go scanner (src/go/scanner/scanner.go) + +func isLetter(ch rune) bool { + return 'a' <= lower(ch) && lower(ch) <= 'z' || ch == '_' +} + +func isDigit(ch rune) bool { + return isDecimal(ch) || ch >= utf8.RuneSelf && unicode.IsDigit(ch) +} + +func isDecimal(ch rune) bool { return '0' <= ch && ch <= '9' } + +func lower(ch rune) rune { return ('a' - 'A') | ch } // returns lower-case ch iff ch is ASCII letter + +func (s *Scanner) isWhitespace() bool { + return s.curr == ' ' || s.curr == '\t' || s.curr == '\n' || s.curr == '\r' +} + +func (s *Scanner) error(reason string) { + s.errors = append(s.errors, Error{Pos: Position{ + Offset: s.offset, + Row: s.row, + Col: s.col, + }, Message: reason}) +} diff --git a/ast/internal/scanner/scanner_test.go b/ast/internal/scanner/scanner_test.go new file mode 100644 index 0000000000..0e1d3b0883 --- /dev/null +++ b/ast/internal/scanner/scanner_test.go @@ -0,0 +1,205 @@ +// Copyright 2020 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package scanner + +import ( + "bytes" + "testing" + + "github.com/open-policy-agent/opa/ast/internal/tokens" +) + +func TestPositions(t *testing.T) { + tests := []struct { + note string + input string + wantOffset int + wantEnd int + }{ + { + note: "symbol", + input: "(", + wantOffset: 0, + wantEnd: 1, + }, + { + note: "ident", + input: "foo", + wantOffset: 0, + wantEnd: 3, + }, + { + note: "number", + input: "100", + wantOffset: 0, + wantEnd: 3, + }, + { + note: "string", + input: `"foo"`, + wantOffset: 0, + wantEnd: 5, + }, + { + note: "string - wide char", + input: `"foo÷"`, + wantOffset: 0, + wantEnd: 7, + }, + { + note: "comment", + input: `# foo`, + wantOffset: 0, + wantEnd: 5, + }, + { + note: "newline", + input: "foo\n", + wantOffset: 0, + wantEnd: 3, + }, + { + note: "invalid number", + input: "0xDEADBEEF", + wantOffset: 0, + wantEnd: 10, + }, + { + note: "invalid identifier", + input: "0.1e12a1b2c3d", + wantOffset: 0, + wantEnd: 13, + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + s, err := New(bytes.NewBufferString(tc.input)) + if err != nil { + t.Fatal(err) + } + _, pos, _, _ := s.Scan() + if pos.Offset != tc.wantOffset { + t.Fatalf("want offset %d but got %d", tc.wantOffset, pos.Offset) + } + if pos.End != tc.wantEnd { + t.Fatalf("want end %d but got %d", tc.wantEnd, pos.End) + } + }) + } +} + +func TestLiterals(t *testing.T) { + + tests := []struct { + note string + input string + wantRow int + wantOffset int + wantTok tokens.Token + wantLit string + }{ + { + note: "ascii chars", + input: `"hello world"`, + wantRow: 1, + wantOffset: 0, + wantTok: tokens.String, + wantLit: `"hello world"`, + }, + { + note: "wide chars", + input: `"¡¡¡foo, bar!!!"`, + wantRow: 1, + wantOffset: 0, + wantTok: tokens.String, + wantLit: `"¡¡¡foo, bar!!!"`, + }, + { + note: "raw strings", + input: "`foo`", + wantRow: 1, + wantOffset: 0, + wantTok: tokens.String, + wantLit: "`foo`", + }, + { + note: "raw strings - wide chars", + input: "`¡¡¡foo, bar!!!`", + wantRow: 1, + wantOffset: 0, + wantTok: tokens.String, + wantLit: "`¡¡¡foo, bar!!!`", + }, + { + note: "comments", + input: "# foo", + wantRow: 1, + wantOffset: 0, + wantTok: tokens.Comment, + wantLit: "# foo", + }, + { + note: "comments - wide chars", + input: "#¡foo", + wantRow: 1, + wantOffset: 0, + wantTok: tokens.Comment, + wantLit: "#¡foo", + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + s, err := New(bytes.NewBufferString(tc.input)) + if err != nil { + t.Fatal(err) + } + tok, pos, lit, errs := s.Scan() + if pos.Row != tc.wantRow { + t.Errorf("Expected row %d but got %d", tc.wantRow, pos.Row) + } + if pos.Offset != tc.wantOffset { + t.Errorf("Expected offset %d but got %d", tc.wantOffset, pos.Offset) + } + if tok != tc.wantTok { + t.Errorf("Expected token %v but got %v", tc.wantTok, tok) + } + if lit != tc.wantLit { + t.Errorf("Expected literal %v but got %v", tc.wantLit, lit) + } + if len(errs) > 0 { + t.Fatal("Unexpected error(s):", errs) + } + }) + } + +} + +func TestIllegalTokens(t *testing.T) { + + tests := []struct { + input string + wantErr bool + }{ + {input: `墳`}, + {input: `0e`, wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.input, func(t *testing.T) { + s, err := New(bytes.NewBufferString(tc.input)) + if err != nil { + t.Fatal(err) + } + tok, _, _, errs := s.Scan() + if !tc.wantErr && tok != tokens.Illegal { + t.Fatalf("expected illegal token on %q but got %v", tc.input, tok) + } else if tc.wantErr && len(errs) == 0 { + t.Fatalf("expected errors on %q but got %v", tc.input, tok) + } + }) + } +} diff --git a/ast/internal/tokens/tokens.go b/ast/internal/tokens/tokens.go new file mode 100644 index 0000000000..29229a5a3a --- /dev/null +++ b/ast/internal/tokens/tokens.go @@ -0,0 +1,138 @@ +// Copyright 2020 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package tokens + +// Token represents a single Rego source code token +// for use by the Parser. +type Token int + +func (t Token) String() string { + if t < 0 || int(t) >= len(strings) { + return "unknown" + } + return strings[t] +} + +// All tokens must be defined here +const ( + Illegal Token = iota + EOF + Whitespace + Ident + Comment + + Package + Import + As + Default + Else + Not + Some + With + Null + True + False + + Number + String + + LBrack + RBrack + LBrace + RBrace + LParen + RParen + Comma + Colon + + Add + Sub + Mul + Quo + Rem + And + Or + Unify + Equal + Assign + Neq + Gt + Lt + Gte + Lte + Dot + Semicolon +) + +var strings = [...]string{ + Illegal: "illegal", + EOF: "eof", + Whitespace: "whitespace", + Comment: "comment", + Ident: "ident", + Package: "package", + Import: "import", + As: "as", + Default: "default", + Else: "else", + Not: "not", + Some: "some", + With: "with", + Null: "null", + True: "true", + False: "false", + Number: "number", + String: "string", + LBrack: "[", + RBrack: "]", + LBrace: "{", + RBrace: "}", + LParen: "(", + RParen: ")", + Comma: ",", + Colon: ":", + Add: "plus", + Sub: "minus", + Mul: "mul", + Quo: "div", + Rem: "rem", + And: "and", + Or: "or", + Unify: "eq", + Equal: "equal", + Assign: "assign", + Neq: "neq", + Gt: "gt", + Lt: "lt", + Gte: "gte", + Lte: "lte", + Dot: ".", + Semicolon: ";", +} + +var keywords = map[string]Token{ + "package": Package, + "import": Import, + "as": As, + "default": Default, + "else": Else, + "not": Not, + "some": Some, + "with": With, + "null": Null, + "true": True, + "false": False, +} + +// Keyword will return a token for the passed in +// literal value. If the value is a Rego keyword +// then the appropriate token is returned. Everything +// else is an Ident. +func Keyword(lit string) Token { + if tok, ok := keywords[lit]; ok { + return tok + } + return Ident +} diff --git a/ast/location/location.go b/ast/location/location.go new file mode 100644 index 0000000000..13ae6e35d7 --- /dev/null +++ b/ast/location/location.go @@ -0,0 +1,90 @@ +// Package location defines locations in Rego source code. +package location + +import ( + "bytes" + "fmt" + + "github.com/pkg/errors" +) + +// Location records a position in source code +type Location struct { + Text []byte `json:"-"` // The original text fragment from the source. + File string `json:"file"` // The name of the source file (which may be empty). + Row int `json:"row"` // The line in the source. + Col int `json:"col"` // The column in the row. + Offset int `json:"-"` // The byte offset for the location in the source. +} + +// NewLocation returns a new Location object. +func NewLocation(text []byte, file string, row int, col int) *Location { + return &Location{Text: text, File: file, Row: row, Col: col} +} + +// Equal checks if two locations are equal to each other. +func (loc *Location) Equal(other *Location) bool { + return bytes.Equal(loc.Text, other.Text) && + loc.File == other.File && + loc.Row == other.Row && + loc.Col == other.Col +} + +// Errorf returns a new error value with a message formatted to include the location +// info (e.g., line, column, filename, etc.) +func (loc *Location) Errorf(f string, a ...interface{}) error { + return errors.New(loc.Format(f, a...)) +} + +// Wrapf returns a new error value that wraps an existing error with a message formatted +// to include the location info (e.g., line, column, filename, etc.) +func (loc *Location) Wrapf(err error, f string, a ...interface{}) error { + return errors.Wrap(err, loc.Format(f, a...)) +} + +// Format returns a formatted string prefixed with the location information. +func (loc *Location) Format(f string, a ...interface{}) string { + if len(loc.File) > 0 { + f = fmt.Sprintf("%v:%v: %v", loc.File, loc.Row, f) + } else { + f = fmt.Sprintf("%v:%v: %v", loc.Row, loc.Col, f) + } + return fmt.Sprintf(f, a...) +} + +func (loc *Location) String() string { + if len(loc.File) > 0 { + return fmt.Sprintf("%v:%v", loc.File, loc.Row) + } + if len(loc.Text) > 0 { + return string(loc.Text) + } + return fmt.Sprintf("%v:%v", loc.Row, loc.Col) +} + +// Compare returns -1, 0, or 1 to indicate if this loc is less than, equal to, +// or greater than the other. Comparison is performed on the file, row, and +// column of the Location (but not on the text.) Nil locations are greater than +// non-nil locations. +func (loc *Location) Compare(other *Location) int { + if loc == nil && other == nil { + return 0 + } else if loc == nil { + return 1 + } else if other == nil { + return -1 + } else if loc.File < other.File { + return -1 + } else if loc.File > other.File { + return 1 + } else if loc.Row < other.Row { + return -1 + } else if loc.Row > other.Row { + return 1 + } else if loc.Col < other.Col { + return -1 + } else if loc.Col > other.Col { + return 1 + } + return 0 +} diff --git a/ast/location/location_test.go b/ast/location/location_test.go new file mode 100644 index 0000000000..a6e54c62d1 --- /dev/null +++ b/ast/location/location_test.go @@ -0,0 +1,87 @@ +package location + +import ( + "testing" + + "github.com/open-policy-agent/opa/util" +) + +func TestLocationCompare(t *testing.T) { + + tests := []struct { + a string + b string + exp int + }{ + { + a: "", + b: "", + exp: 0, + }, + { + a: "", + b: `{"file": "a", "row": 1, "col": 1}`, + exp: 1, + }, + { + a: `{"file": "a", "row": 1, "col": 1}`, + b: "", + exp: -1, + }, + { + a: `{"file": "a", "row": 1, "col": 1}`, + b: `{"file": "a", "row": 1, "col": 1}`, + exp: 0, + }, + { + a: `{"file": "a", "row": 1, "col": 1}`, + b: `{"file": "b", "row": 1, "col": 1}`, + exp: -1, + }, + { + a: `{"file": "b", "row": 1, "col": 1}`, + b: `{"file": "a", "row": 1, "col": 1}`, + exp: 1, + }, + { + a: `{"file": "a", "row": 1, "col": 1}`, + b: `{"file": "a", "row": 2, "col": 1}`, + exp: -1, + }, + { + a: `{"file": "a", "row": 2, "col": 1}`, + b: `{"file": "a", "row": 1, "col": 1}`, + exp: 1, + }, + { + a: `{"file": "a", "row": 1, "col": 1}`, + b: `{"file": "a", "row": 1, "col": 2}`, + exp: -1, + }, + { + a: `{"file": "a", "row": 1, "col": 2}`, + b: `{"file": "a", "row": 1, "col": 1}`, + exp: 1, + }, + } + + unmarshal := func(s string) *Location { + if s != "" { + var loc Location + if err := util.Unmarshal([]byte(s), &loc); err != nil { + t.Fatal(err) + } + return &loc + } + return nil + } + + for _, tc := range tests { + locA := unmarshal(tc.a) + locB := unmarshal(tc.b) + result := locA.Compare(locB) + if tc.exp != result { + t.Fatalf("Expected %v but got %v for %v.Compare(%v)", tc.exp, result, locA, locB) + } + } +} diff --git a/ast/parser.go b/ast/parser.go index d2236de7e9..f02c0bb0bd 100644 --- a/ast/parser.go +++ b/ast/parser.go @@ -1,5150 +1,1483 @@ -// Code generated by pigeon; DO NOT EDIT. +// Copyright 2020 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. package ast import ( - "bytes" - "errors" + "encoding/json" "fmt" "io" - "io/ioutil" - "math" - "os" - "sort" - "strconv" - "strings" - "unicode" - "unicode/utf8" -) + "math/big" -var g = &grammar{ - rules: []*rule{ - { - name: "Program", - pos: position{line: 5, col: 1, offset: 17}, - expr: &actionExpr{ - pos: position{line: 5, col: 12, offset: 28}, - run: (*parser).callonProgram1, - expr: &seqExpr{ - pos: position{line: 5, col: 12, offset: 28}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 5, col: 12, offset: 28}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 5, col: 14, offset: 30}, - label: "vals", - expr: &zeroOrOneExpr{ - pos: position{line: 5, col: 19, offset: 35}, - expr: &seqExpr{ - pos: position{line: 5, col: 20, offset: 36}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 5, col: 20, offset: 36}, - name: "Stmt", - }, - &zeroOrMoreExpr{ - pos: position{line: 5, col: 25, offset: 41}, - expr: &seqExpr{ - pos: position{line: 5, col: 26, offset: 42}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 5, col: 26, offset: 42}, - name: "ws", - }, - &ruleRefExpr{ - pos: position{line: 5, col: 29, offset: 45}, - name: "Stmt", - }, - }, - }, - }, - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 5, col: 38, offset: 54}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 5, col: 40, offset: 56}, - name: "EOF", - }, - }, - }, - }, - }, - { - name: "Stmt", - pos: position{line: 9, col: 1, offset: 97}, - expr: &actionExpr{ - pos: position{line: 9, col: 9, offset: 105}, - run: (*parser).callonStmt1, - expr: &labeledExpr{ - pos: position{line: 9, col: 9, offset: 105}, - label: "val", - expr: &choiceExpr{ - pos: position{line: 9, col: 14, offset: 110}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 9, col: 14, offset: 110}, - name: "Package", - }, - &ruleRefExpr{ - pos: position{line: 9, col: 24, offset: 120}, - name: "Import", - }, - &ruleRefExpr{ - pos: position{line: 9, col: 33, offset: 129}, - name: "Rules", - }, - &ruleRefExpr{ - pos: position{line: 9, col: 41, offset: 137}, - name: "Body", - }, - &ruleRefExpr{ - pos: position{line: 9, col: 48, offset: 144}, - name: "Comment", - }, - }, - }, - }, - }, - }, - { - name: "Package", - pos: position{line: 13, col: 1, offset: 178}, - expr: &actionExpr{ - pos: position{line: 13, col: 12, offset: 189}, - run: (*parser).callonPackage1, - expr: &seqExpr{ - pos: position{line: 13, col: 12, offset: 189}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 13, col: 12, offset: 189}, - val: "package", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 13, col: 22, offset: 199}, - name: "ws", - }, - &labeledExpr{ - pos: position{line: 13, col: 25, offset: 202}, - label: "val", - expr: &choiceExpr{ - pos: position{line: 13, col: 30, offset: 207}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 13, col: 30, offset: 207}, - name: "Ref", - }, - &ruleRefExpr{ - pos: position{line: 13, col: 36, offset: 213}, - name: "Var", - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "Import", - pos: position{line: 17, col: 1, offset: 271}, - expr: &actionExpr{ - pos: position{line: 17, col: 11, offset: 281}, - run: (*parser).callonImport1, - expr: &seqExpr{ - pos: position{line: 17, col: 11, offset: 281}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 17, col: 11, offset: 281}, - val: "import", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 17, col: 20, offset: 290}, - name: "ws", - }, - &labeledExpr{ - pos: position{line: 17, col: 23, offset: 293}, - label: "path", - expr: &choiceExpr{ - pos: position{line: 17, col: 29, offset: 299}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 17, col: 29, offset: 299}, - name: "Ref", - }, - &ruleRefExpr{ - pos: position{line: 17, col: 35, offset: 305}, - name: "Var", - }, - }, - }, - }, - &labeledExpr{ - pos: position{line: 17, col: 40, offset: 310}, - label: "alias", - expr: &zeroOrOneExpr{ - pos: position{line: 17, col: 46, offset: 316}, - expr: &seqExpr{ - pos: position{line: 17, col: 47, offset: 317}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 17, col: 47, offset: 317}, - name: "ws", - }, - &litMatcher{ - pos: position{line: 17, col: 50, offset: 320}, - val: "as", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 17, col: 55, offset: 325}, - name: "ws", - }, - &ruleRefExpr{ - pos: position{line: 17, col: 58, offset: 328}, - name: "Var", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "Rules", - pos: position{line: 21, col: 1, offset: 394}, - expr: &choiceExpr{ - pos: position{line: 21, col: 10, offset: 403}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 21, col: 10, offset: 403}, - name: "DefaultRules", - }, - &ruleRefExpr{ - pos: position{line: 21, col: 25, offset: 418}, - name: "NormalRules", - }, - }, - }, - }, - { - name: "DefaultRules", - pos: position{line: 23, col: 1, offset: 431}, - expr: &actionExpr{ - pos: position{line: 23, col: 17, offset: 447}, - run: (*parser).callonDefaultRules1, - expr: &seqExpr{ - pos: position{line: 23, col: 17, offset: 447}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 23, col: 17, offset: 447}, - val: "default", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 23, col: 27, offset: 457}, - name: "ws", - }, - &labeledExpr{ - pos: position{line: 23, col: 30, offset: 460}, - label: "name", - expr: &ruleRefExpr{ - pos: position{line: 23, col: 35, offset: 465}, - name: "Var", - }, - }, - &ruleRefExpr{ - pos: position{line: 23, col: 39, offset: 469}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 23, col: 41, offset: 471}, - label: "operator", - expr: &choiceExpr{ - pos: position{line: 23, col: 52, offset: 482}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 23, col: 52, offset: 482}, - val: ":=", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 23, col: 59, offset: 489}, - val: "=", - ignoreCase: false, - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 23, col: 65, offset: 495}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 23, col: 67, offset: 497}, - label: "value", - expr: &ruleRefExpr{ - pos: position{line: 23, col: 73, offset: 503}, - name: "Term", - }, - }, - }, - }, - }, - }, - { - name: "NormalRules", - pos: position{line: 27, col: 1, offset: 583}, - expr: &actionExpr{ - pos: position{line: 27, col: 16, offset: 598}, - run: (*parser).callonNormalRules1, - expr: &seqExpr{ - pos: position{line: 27, col: 16, offset: 598}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 27, col: 16, offset: 598}, - label: "head", - expr: &choiceExpr{ - pos: position{line: 27, col: 22, offset: 604}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 27, col: 22, offset: 604}, - name: "PartialRuleHead", - }, - &ruleRefExpr{ - pos: position{line: 27, col: 40, offset: 622}, - name: "RuleHead", - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 27, col: 50, offset: 632}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 27, col: 52, offset: 634}, - label: "rest", - expr: &seqExpr{ - pos: position{line: 27, col: 58, offset: 640}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 27, col: 58, offset: 640}, - name: "NonEmptyBraceEnclosedBody", - }, - &zeroOrMoreExpr{ - pos: position{line: 27, col: 84, offset: 666}, - expr: &seqExpr{ - pos: position{line: 27, col: 86, offset: 668}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 27, col: 86, offset: 668}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 27, col: 88, offset: 670}, - name: "RuleExt", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "PartialRuleHead", - pos: position{line: 31, col: 1, offset: 739}, - expr: &actionExpr{ - pos: position{line: 31, col: 20, offset: 758}, - run: (*parser).callonPartialRuleHead1, - expr: &seqExpr{ - pos: position{line: 31, col: 20, offset: 758}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 31, col: 20, offset: 758}, - label: "name", - expr: &ruleRefExpr{ - pos: position{line: 31, col: 25, offset: 763}, - name: "Var", - }, - }, - &labeledExpr{ - pos: position{line: 31, col: 29, offset: 767}, - label: "args", - expr: &seqExpr{ - pos: position{line: 31, col: 36, offset: 774}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 31, col: 36, offset: 774}, - name: "_", - }, - &litMatcher{ - pos: position{line: 31, col: 38, offset: 776}, - val: "(", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 31, col: 42, offset: 780}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 31, col: 44, offset: 782}, - name: "Args", - }, - &ruleRefExpr{ - pos: position{line: 31, col: 49, offset: 787}, - name: "_", - }, - &litMatcher{ - pos: position{line: 31, col: 51, offset: 789}, - val: ")", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 31, col: 55, offset: 793}, - name: "_", - }, - }, - }, - }, - &labeledExpr{ - pos: position{line: 31, col: 59, offset: 797}, - label: "value", - expr: &zeroOrOneExpr{ - pos: position{line: 31, col: 65, offset: 803}, - expr: &seqExpr{ - pos: position{line: 31, col: 67, offset: 805}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 31, col: 67, offset: 805}, - name: "_", - }, - &choiceExpr{ - pos: position{line: 31, col: 71, offset: 809}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 31, col: 71, offset: 809}, - val: ":=", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 31, col: 78, offset: 816}, - val: "=", - ignoreCase: false, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 31, col: 84, offset: 822}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 31, col: 86, offset: 824}, - name: "ExprTerm", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "RuleHead", - pos: position{line: 35, col: 1, offset: 909}, - expr: &actionExpr{ - pos: position{line: 35, col: 13, offset: 921}, - run: (*parser).callonRuleHead1, - expr: &seqExpr{ - pos: position{line: 35, col: 13, offset: 921}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 35, col: 13, offset: 921}, - label: "name", - expr: &ruleRefExpr{ - pos: position{line: 35, col: 18, offset: 926}, - name: "Var", - }, - }, - &labeledExpr{ - pos: position{line: 35, col: 22, offset: 930}, - label: "key", - expr: &zeroOrOneExpr{ - pos: position{line: 35, col: 26, offset: 934}, - expr: &seqExpr{ - pos: position{line: 35, col: 28, offset: 936}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 35, col: 28, offset: 936}, - name: "_", - }, - &litMatcher{ - pos: position{line: 35, col: 30, offset: 938}, - val: "[", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 35, col: 34, offset: 942}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 35, col: 36, offset: 944}, - name: "ExprTerm", - }, - &ruleRefExpr{ - pos: position{line: 35, col: 45, offset: 953}, - name: "_", - }, - &litMatcher{ - pos: position{line: 35, col: 47, offset: 955}, - val: "]", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 35, col: 51, offset: 959}, - name: "_", - }, - }, - }, - }, - }, - &labeledExpr{ - pos: position{line: 35, col: 56, offset: 964}, - label: "value", - expr: &zeroOrOneExpr{ - pos: position{line: 35, col: 62, offset: 970}, - expr: &seqExpr{ - pos: position{line: 35, col: 64, offset: 972}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 35, col: 64, offset: 972}, - name: "_", - }, - &choiceExpr{ - pos: position{line: 35, col: 68, offset: 976}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 35, col: 68, offset: 976}, - val: ":=", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 35, col: 75, offset: 983}, - val: "=", - ignoreCase: false, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 35, col: 81, offset: 989}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 35, col: 83, offset: 991}, - name: "ExprTerm", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "Args", - pos: position{line: 39, col: 1, offset: 1075}, - expr: &actionExpr{ - pos: position{line: 39, col: 9, offset: 1083}, - run: (*parser).callonArgs1, - expr: &labeledExpr{ - pos: position{line: 39, col: 9, offset: 1083}, - label: "list", - expr: &ruleRefExpr{ - pos: position{line: 39, col: 14, offset: 1088}, - name: "ExprTermList", - }, - }, - }, - }, - { - name: "Else", - pos: position{line: 43, col: 1, offset: 1132}, - expr: &actionExpr{ - pos: position{line: 43, col: 9, offset: 1140}, - run: (*parser).callonElse1, - expr: &seqExpr{ - pos: position{line: 43, col: 9, offset: 1140}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 43, col: 9, offset: 1140}, - val: "else", - ignoreCase: false, - }, - &labeledExpr{ - pos: position{line: 43, col: 16, offset: 1147}, - label: "value", - expr: &zeroOrOneExpr{ - pos: position{line: 43, col: 22, offset: 1153}, - expr: &seqExpr{ - pos: position{line: 43, col: 24, offset: 1155}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 43, col: 24, offset: 1155}, - name: "_", - }, - &litMatcher{ - pos: position{line: 43, col: 26, offset: 1157}, - val: "=", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 43, col: 30, offset: 1161}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 43, col: 32, offset: 1163}, - name: "Term", - }, - }, - }, - }, - }, - &labeledExpr{ - pos: position{line: 43, col: 40, offset: 1171}, - label: "body", - expr: &seqExpr{ - pos: position{line: 43, col: 47, offset: 1178}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 43, col: 47, offset: 1178}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 43, col: 49, offset: 1180}, - name: "NonEmptyBraceEnclosedBody", - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "RuleDup", - pos: position{line: 47, col: 1, offset: 1269}, - expr: &actionExpr{ - pos: position{line: 47, col: 12, offset: 1280}, - run: (*parser).callonRuleDup1, - expr: &labeledExpr{ - pos: position{line: 47, col: 12, offset: 1280}, - label: "b", - expr: &ruleRefExpr{ - pos: position{line: 47, col: 14, offset: 1282}, - name: "NonEmptyBraceEnclosedBody", - }, - }, - }, - }, - { - name: "RuleExt", - pos: position{line: 51, col: 1, offset: 1378}, - expr: &choiceExpr{ - pos: position{line: 51, col: 12, offset: 1389}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 51, col: 12, offset: 1389}, - name: "Else", - }, - &ruleRefExpr{ - pos: position{line: 51, col: 19, offset: 1396}, - name: "RuleDup", - }, - }, - }, - }, - { - name: "Body", - pos: position{line: 53, col: 1, offset: 1405}, - expr: &choiceExpr{ - pos: position{line: 53, col: 9, offset: 1413}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 53, col: 9, offset: 1413}, - name: "NonWhitespaceBody", - }, - &ruleRefExpr{ - pos: position{line: 53, col: 29, offset: 1433}, - name: "BraceEnclosedBody", - }, - }, - }, - }, - { - name: "NonEmptyBraceEnclosedBody", - pos: position{line: 55, col: 1, offset: 1452}, - expr: &actionExpr{ - pos: position{line: 55, col: 30, offset: 1481}, - run: (*parser).callonNonEmptyBraceEnclosedBody1, - expr: &seqExpr{ - pos: position{line: 55, col: 30, offset: 1481}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 55, col: 30, offset: 1481}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 55, col: 34, offset: 1485}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 55, col: 36, offset: 1487}, - label: "val", - expr: &zeroOrOneExpr{ - pos: position{line: 55, col: 40, offset: 1491}, - expr: &ruleRefExpr{ - pos: position{line: 55, col: 40, offset: 1491}, - name: "WhitespaceBody", - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 55, col: 56, offset: 1507}, - name: "_", - }, - &litMatcher{ - pos: position{line: 55, col: 58, offset: 1509}, - val: "}", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "BraceEnclosedBody", - pos: position{line: 62, col: 1, offset: 1621}, - expr: &actionExpr{ - pos: position{line: 62, col: 22, offset: 1642}, - run: (*parser).callonBraceEnclosedBody1, - expr: &seqExpr{ - pos: position{line: 62, col: 22, offset: 1642}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 62, col: 22, offset: 1642}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 62, col: 26, offset: 1646}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 62, col: 28, offset: 1648}, - label: "val", - expr: &zeroOrOneExpr{ - pos: position{line: 62, col: 32, offset: 1652}, - expr: &ruleRefExpr{ - pos: position{line: 62, col: 32, offset: 1652}, - name: "WhitespaceBody", - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 62, col: 48, offset: 1668}, - name: "_", - }, - &litMatcher{ - pos: position{line: 62, col: 50, offset: 1670}, - val: "}", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "WhitespaceBody", - pos: position{line: 66, col: 1, offset: 1737}, - expr: &actionExpr{ - pos: position{line: 66, col: 19, offset: 1755}, - run: (*parser).callonWhitespaceBody1, - expr: &seqExpr{ - pos: position{line: 66, col: 19, offset: 1755}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 66, col: 19, offset: 1755}, - label: "head", - expr: &ruleRefExpr{ - pos: position{line: 66, col: 24, offset: 1760}, - name: "Literal", - }, - }, - &labeledExpr{ - pos: position{line: 66, col: 32, offset: 1768}, - label: "tail", - expr: &zeroOrMoreExpr{ - pos: position{line: 66, col: 37, offset: 1773}, - expr: &seqExpr{ - pos: position{line: 66, col: 38, offset: 1774}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 66, col: 38, offset: 1774}, - name: "WhitespaceLiteralSeparator", - }, - &ruleRefExpr{ - pos: position{line: 66, col: 65, offset: 1801}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 66, col: 67, offset: 1803}, - name: "Literal", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "NonWhitespaceBody", - pos: position{line: 70, col: 1, offset: 1853}, - expr: &actionExpr{ - pos: position{line: 70, col: 22, offset: 1874}, - run: (*parser).callonNonWhitespaceBody1, - expr: &seqExpr{ - pos: position{line: 70, col: 22, offset: 1874}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 70, col: 22, offset: 1874}, - label: "head", - expr: &ruleRefExpr{ - pos: position{line: 70, col: 27, offset: 1879}, - name: "Literal", - }, - }, - &labeledExpr{ - pos: position{line: 70, col: 35, offset: 1887}, - label: "tail", - expr: &zeroOrMoreExpr{ - pos: position{line: 70, col: 40, offset: 1892}, - expr: &seqExpr{ - pos: position{line: 70, col: 42, offset: 1894}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 70, col: 42, offset: 1894}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 70, col: 44, offset: 1896}, - name: "NonWhitespaceLiteralSeparator", - }, - &ruleRefExpr{ - pos: position{line: 70, col: 74, offset: 1926}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 70, col: 76, offset: 1928}, - name: "Literal", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "WhitespaceLiteralSeparator", - pos: position{line: 74, col: 1, offset: 1978}, - expr: &seqExpr{ - pos: position{line: 74, col: 31, offset: 2008}, - exprs: []interface{}{ - &zeroOrMoreExpr{ - pos: position{line: 74, col: 31, offset: 2008}, - expr: &charClassMatcher{ - pos: position{line: 74, col: 31, offset: 2008}, - val: "[ \\t]", - chars: []rune{' ', '\t'}, - ignoreCase: false, - inverted: false, - }, - }, - &choiceExpr{ - pos: position{line: 74, col: 39, offset: 2016}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 74, col: 40, offset: 2017}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 74, col: 40, offset: 2017}, - name: "NonWhitespaceLiteralSeparator", - }, - &zeroOrOneExpr{ - pos: position{line: 74, col: 70, offset: 2047}, - expr: &ruleRefExpr{ - pos: position{line: 74, col: 70, offset: 2047}, - name: "Comment", - }, - }, - }, - }, - &seqExpr{ - pos: position{line: 74, col: 83, offset: 2060}, - exprs: []interface{}{ - &zeroOrOneExpr{ - pos: position{line: 74, col: 83, offset: 2060}, - expr: &ruleRefExpr{ - pos: position{line: 74, col: 83, offset: 2060}, - name: "Comment", - }, - }, - &charClassMatcher{ - pos: position{line: 74, col: 92, offset: 2069}, - val: "[\\r\\n]", - chars: []rune{'\r', '\n'}, - ignoreCase: false, - inverted: false, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "NonWhitespaceLiteralSeparator", - pos: position{line: 76, col: 1, offset: 2079}, - expr: &litMatcher{ - pos: position{line: 76, col: 34, offset: 2112}, - val: ";", - ignoreCase: false, - }, - }, - { - name: "Literal", - pos: position{line: 78, col: 1, offset: 2117}, - expr: &choiceExpr{ - pos: position{line: 78, col: 12, offset: 2128}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 78, col: 12, offset: 2128}, - name: "TermExpr", - }, - &ruleRefExpr{ - pos: position{line: 78, col: 23, offset: 2139}, - name: "SomeDecl", - }, - }, - }, - }, - { - name: "SomeDecl", - pos: position{line: 80, col: 1, offset: 2149}, - expr: &actionExpr{ - pos: position{line: 80, col: 13, offset: 2161}, - run: (*parser).callonSomeDecl1, - expr: &seqExpr{ - pos: position{line: 80, col: 13, offset: 2161}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 80, col: 13, offset: 2161}, - val: "some", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 80, col: 20, offset: 2168}, - name: "ws", - }, - &labeledExpr{ - pos: position{line: 80, col: 23, offset: 2171}, - label: "symbols", - expr: &ruleRefExpr{ - pos: position{line: 80, col: 31, offset: 2179}, - name: "SomeDeclList", - }, - }, - }, - }, - }, - }, - { - name: "SomeDeclList", - pos: position{line: 84, col: 1, offset: 2257}, - expr: &actionExpr{ - pos: position{line: 84, col: 17, offset: 2273}, - run: (*parser).callonSomeDeclList1, - expr: &seqExpr{ - pos: position{line: 84, col: 17, offset: 2273}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 84, col: 17, offset: 2273}, - label: "head", - expr: &ruleRefExpr{ - pos: position{line: 84, col: 22, offset: 2278}, - name: "Var", - }, - }, - &labeledExpr{ - pos: position{line: 84, col: 26, offset: 2282}, - label: "rest", - expr: &zeroOrMoreExpr{ - pos: position{line: 84, col: 31, offset: 2287}, - expr: &seqExpr{ - pos: position{line: 84, col: 33, offset: 2289}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 84, col: 33, offset: 2289}, - name: "_", - }, - &litMatcher{ - pos: position{line: 84, col: 35, offset: 2291}, - val: ",", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 84, col: 39, offset: 2295}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 84, col: 41, offset: 2297}, - name: "Var", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "TermExpr", - pos: position{line: 88, col: 1, offset: 2351}, - expr: &actionExpr{ - pos: position{line: 88, col: 13, offset: 2363}, - run: (*parser).callonTermExpr1, - expr: &seqExpr{ - pos: position{line: 88, col: 13, offset: 2363}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 88, col: 13, offset: 2363}, - label: "negated", - expr: &zeroOrOneExpr{ - pos: position{line: 88, col: 21, offset: 2371}, - expr: &ruleRefExpr{ - pos: position{line: 88, col: 21, offset: 2371}, - name: "NotKeyword", - }, - }, - }, - &labeledExpr{ - pos: position{line: 88, col: 33, offset: 2383}, - label: "value", - expr: &ruleRefExpr{ - pos: position{line: 88, col: 39, offset: 2389}, - name: "LiteralExpr", - }, - }, - &labeledExpr{ - pos: position{line: 88, col: 51, offset: 2401}, - label: "with", - expr: &zeroOrOneExpr{ - pos: position{line: 88, col: 56, offset: 2406}, - expr: &ruleRefExpr{ - pos: position{line: 88, col: 56, offset: 2406}, - name: "WithKeywordList", - }, - }, - }, - }, - }, - }, - }, - { - name: "LiteralExpr", - pos: position{line: 92, col: 1, offset: 2473}, - expr: &actionExpr{ - pos: position{line: 92, col: 16, offset: 2488}, - run: (*parser).callonLiteralExpr1, - expr: &seqExpr{ - pos: position{line: 92, col: 16, offset: 2488}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 92, col: 16, offset: 2488}, - label: "lhs", - expr: &ruleRefExpr{ - pos: position{line: 92, col: 20, offset: 2492}, - name: "ExprTerm", - }, - }, - &labeledExpr{ - pos: position{line: 92, col: 29, offset: 2501}, - label: "rest", - expr: &zeroOrOneExpr{ - pos: position{line: 92, col: 34, offset: 2506}, - expr: &seqExpr{ - pos: position{line: 92, col: 36, offset: 2508}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 92, col: 36, offset: 2508}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 92, col: 38, offset: 2510}, - name: "LiteralExprOperator", - }, - &ruleRefExpr{ - pos: position{line: 92, col: 58, offset: 2530}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 92, col: 60, offset: 2532}, - name: "ExprTerm", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "LiteralExprOperator", - pos: position{line: 96, col: 1, offset: 2606}, - expr: &actionExpr{ - pos: position{line: 96, col: 24, offset: 2629}, - run: (*parser).callonLiteralExprOperator1, - expr: &labeledExpr{ - pos: position{line: 96, col: 24, offset: 2629}, - label: "val", - expr: &choiceExpr{ - pos: position{line: 96, col: 30, offset: 2635}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 96, col: 30, offset: 2635}, - val: ":=", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 96, col: 37, offset: 2642}, - val: "=", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - { - name: "NotKeyword", - pos: position{line: 100, col: 1, offset: 2710}, - expr: &actionExpr{ - pos: position{line: 100, col: 15, offset: 2724}, - run: (*parser).callonNotKeyword1, - expr: &labeledExpr{ - pos: position{line: 100, col: 15, offset: 2724}, - label: "val", - expr: &zeroOrOneExpr{ - pos: position{line: 100, col: 19, offset: 2728}, - expr: &seqExpr{ - pos: position{line: 100, col: 20, offset: 2729}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 100, col: 20, offset: 2729}, - val: "not", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 100, col: 26, offset: 2735}, - name: "ws", - }, - }, - }, - }, - }, - }, - }, - { - name: "WithKeywordList", - pos: position{line: 104, col: 1, offset: 2772}, - expr: &actionExpr{ - pos: position{line: 104, col: 20, offset: 2791}, - run: (*parser).callonWithKeywordList1, - expr: &seqExpr{ - pos: position{line: 104, col: 20, offset: 2791}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 104, col: 20, offset: 2791}, - name: "ws", - }, - &labeledExpr{ - pos: position{line: 104, col: 23, offset: 2794}, - label: "head", - expr: &ruleRefExpr{ - pos: position{line: 104, col: 28, offset: 2799}, - name: "WithKeyword", - }, - }, - &labeledExpr{ - pos: position{line: 104, col: 40, offset: 2811}, - label: "rest", - expr: &zeroOrMoreExpr{ - pos: position{line: 104, col: 45, offset: 2816}, - expr: &seqExpr{ - pos: position{line: 104, col: 47, offset: 2818}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 104, col: 47, offset: 2818}, - name: "ws", - }, - &ruleRefExpr{ - pos: position{line: 104, col: 50, offset: 2821}, - name: "WithKeyword", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "WithKeyword", - pos: position{line: 108, col: 1, offset: 2884}, - expr: &actionExpr{ - pos: position{line: 108, col: 16, offset: 2899}, - run: (*parser).callonWithKeyword1, - expr: &seqExpr{ - pos: position{line: 108, col: 16, offset: 2899}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 108, col: 16, offset: 2899}, - val: "with", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 108, col: 23, offset: 2906}, - name: "ws", - }, - &labeledExpr{ - pos: position{line: 108, col: 26, offset: 2909}, - label: "target", - expr: &ruleRefExpr{ - pos: position{line: 108, col: 33, offset: 2916}, - name: "ExprTerm", - }, - }, - &ruleRefExpr{ - pos: position{line: 108, col: 42, offset: 2925}, - name: "ws", - }, - &litMatcher{ - pos: position{line: 108, col: 45, offset: 2928}, - val: "as", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 108, col: 50, offset: 2933}, - name: "ws", - }, - &labeledExpr{ - pos: position{line: 108, col: 53, offset: 2936}, - label: "value", - expr: &ruleRefExpr{ - pos: position{line: 108, col: 59, offset: 2942}, - name: "ExprTerm", - }, - }, - }, - }, - }, - }, - { - name: "ExprTerm", - pos: position{line: 112, col: 1, offset: 3018}, - expr: &actionExpr{ - pos: position{line: 112, col: 13, offset: 3030}, - run: (*parser).callonExprTerm1, - expr: &seqExpr{ - pos: position{line: 112, col: 13, offset: 3030}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 112, col: 13, offset: 3030}, - label: "lhs", - expr: &ruleRefExpr{ - pos: position{line: 112, col: 17, offset: 3034}, - name: "RelationExpr", - }, - }, - &labeledExpr{ - pos: position{line: 112, col: 30, offset: 3047}, - label: "rest", - expr: &zeroOrMoreExpr{ - pos: position{line: 112, col: 35, offset: 3052}, - expr: &seqExpr{ - pos: position{line: 112, col: 37, offset: 3054}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 112, col: 37, offset: 3054}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 112, col: 39, offset: 3056}, - name: "RelationOperator", - }, - &ruleRefExpr{ - pos: position{line: 112, col: 56, offset: 3073}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 112, col: 58, offset: 3075}, - name: "RelationExpr", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "ExprTermPairList", - pos: position{line: 116, col: 1, offset: 3151}, - expr: &actionExpr{ - pos: position{line: 116, col: 21, offset: 3171}, - run: (*parser).callonExprTermPairList1, - expr: &seqExpr{ - pos: position{line: 116, col: 21, offset: 3171}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 116, col: 21, offset: 3171}, - label: "head", - expr: &zeroOrOneExpr{ - pos: position{line: 116, col: 26, offset: 3176}, - expr: &ruleRefExpr{ - pos: position{line: 116, col: 26, offset: 3176}, - name: "ExprTermPair", - }, - }, - }, - &labeledExpr{ - pos: position{line: 116, col: 40, offset: 3190}, - label: "tail", - expr: &zeroOrMoreExpr{ - pos: position{line: 116, col: 45, offset: 3195}, - expr: &seqExpr{ - pos: position{line: 116, col: 47, offset: 3197}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 116, col: 47, offset: 3197}, - name: "_", - }, - &litMatcher{ - pos: position{line: 116, col: 49, offset: 3199}, - val: ",", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 116, col: 53, offset: 3203}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 116, col: 55, offset: 3205}, - name: "ExprTermPair", - }, - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 116, col: 71, offset: 3221}, - name: "_", - }, - &zeroOrOneExpr{ - pos: position{line: 116, col: 73, offset: 3223}, - expr: &litMatcher{ - pos: position{line: 116, col: 73, offset: 3223}, - val: ",", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - { - name: "ExprTermList", - pos: position{line: 120, col: 1, offset: 3277}, - expr: &actionExpr{ - pos: position{line: 120, col: 17, offset: 3293}, - run: (*parser).callonExprTermList1, - expr: &seqExpr{ - pos: position{line: 120, col: 17, offset: 3293}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 120, col: 17, offset: 3293}, - label: "head", - expr: &zeroOrOneExpr{ - pos: position{line: 120, col: 22, offset: 3298}, - expr: &ruleRefExpr{ - pos: position{line: 120, col: 22, offset: 3298}, - name: "ExprTerm", - }, - }, - }, - &labeledExpr{ - pos: position{line: 120, col: 32, offset: 3308}, - label: "tail", - expr: &zeroOrMoreExpr{ - pos: position{line: 120, col: 37, offset: 3313}, - expr: &seqExpr{ - pos: position{line: 120, col: 39, offset: 3315}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 120, col: 39, offset: 3315}, - name: "_", - }, - &litMatcher{ - pos: position{line: 120, col: 41, offset: 3317}, - val: ",", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 120, col: 45, offset: 3321}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 120, col: 47, offset: 3323}, - name: "ExprTerm", - }, - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 120, col: 59, offset: 3335}, - name: "_", - }, - &zeroOrOneExpr{ - pos: position{line: 120, col: 61, offset: 3337}, - expr: &litMatcher{ - pos: position{line: 120, col: 61, offset: 3337}, - val: ",", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - { - name: "ExprTermPair", - pos: position{line: 124, col: 1, offset: 3388}, - expr: &actionExpr{ - pos: position{line: 124, col: 17, offset: 3404}, - run: (*parser).callonExprTermPair1, - expr: &seqExpr{ - pos: position{line: 124, col: 17, offset: 3404}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 124, col: 17, offset: 3404}, - label: "key", - expr: &ruleRefExpr{ - pos: position{line: 124, col: 21, offset: 3408}, - name: "ExprTerm", - }, - }, - &ruleRefExpr{ - pos: position{line: 124, col: 30, offset: 3417}, - name: "_", - }, - &litMatcher{ - pos: position{line: 124, col: 32, offset: 3419}, - val: ":", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 124, col: 36, offset: 3423}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 124, col: 38, offset: 3425}, - label: "value", - expr: &ruleRefExpr{ - pos: position{line: 124, col: 44, offset: 3431}, - name: "ExprTerm", - }, - }, - }, - }, - }, - }, - { - name: "RelationOperator", - pos: position{line: 128, col: 1, offset: 3485}, - expr: &actionExpr{ - pos: position{line: 128, col: 21, offset: 3505}, - run: (*parser).callonRelationOperator1, - expr: &labeledExpr{ - pos: position{line: 128, col: 21, offset: 3505}, - label: "val", - expr: &choiceExpr{ - pos: position{line: 128, col: 26, offset: 3510}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 128, col: 26, offset: 3510}, - val: "==", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 128, col: 33, offset: 3517}, - val: "!=", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 128, col: 40, offset: 3524}, - val: "<=", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 128, col: 47, offset: 3531}, - val: ">=", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 128, col: 54, offset: 3538}, - val: ">", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 128, col: 60, offset: 3544}, - val: "<", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - { - name: "RelationExpr", - pos: position{line: 132, col: 1, offset: 3611}, - expr: &actionExpr{ - pos: position{line: 132, col: 17, offset: 3627}, - run: (*parser).callonRelationExpr1, - expr: &seqExpr{ - pos: position{line: 132, col: 17, offset: 3627}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 132, col: 17, offset: 3627}, - label: "lhs", - expr: &ruleRefExpr{ - pos: position{line: 132, col: 21, offset: 3631}, - name: "BitwiseOrExpr", - }, - }, - &labeledExpr{ - pos: position{line: 132, col: 35, offset: 3645}, - label: "rest", - expr: &zeroOrMoreExpr{ - pos: position{line: 132, col: 40, offset: 3650}, - expr: &seqExpr{ - pos: position{line: 132, col: 42, offset: 3652}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 132, col: 42, offset: 3652}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 132, col: 44, offset: 3654}, - name: "BitwiseOrOperator", - }, - &ruleRefExpr{ - pos: position{line: 132, col: 62, offset: 3672}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 132, col: 64, offset: 3674}, - name: "BitwiseOrExpr", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "BitwiseOrOperator", - pos: position{line: 136, col: 1, offset: 3750}, - expr: &actionExpr{ - pos: position{line: 136, col: 22, offset: 3771}, - run: (*parser).callonBitwiseOrOperator1, - expr: &labeledExpr{ - pos: position{line: 136, col: 22, offset: 3771}, - label: "val", - expr: &litMatcher{ - pos: position{line: 136, col: 26, offset: 3775}, - val: "|", - ignoreCase: false, - }, - }, - }, - }, - { - name: "BitwiseOrExpr", - pos: position{line: 140, col: 1, offset: 3841}, - expr: &actionExpr{ - pos: position{line: 140, col: 18, offset: 3858}, - run: (*parser).callonBitwiseOrExpr1, - expr: &seqExpr{ - pos: position{line: 140, col: 18, offset: 3858}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 140, col: 18, offset: 3858}, - label: "lhs", - expr: &ruleRefExpr{ - pos: position{line: 140, col: 22, offset: 3862}, - name: "BitwiseAndExpr", - }, - }, - &labeledExpr{ - pos: position{line: 140, col: 37, offset: 3877}, - label: "rest", - expr: &zeroOrMoreExpr{ - pos: position{line: 140, col: 42, offset: 3882}, - expr: &seqExpr{ - pos: position{line: 140, col: 44, offset: 3884}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 140, col: 44, offset: 3884}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 140, col: 46, offset: 3886}, - name: "BitwiseAndOperator", - }, - &ruleRefExpr{ - pos: position{line: 140, col: 65, offset: 3905}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 140, col: 67, offset: 3907}, - name: "BitwiseAndExpr", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "BitwiseAndOperator", - pos: position{line: 144, col: 1, offset: 3984}, - expr: &actionExpr{ - pos: position{line: 144, col: 23, offset: 4006}, - run: (*parser).callonBitwiseAndOperator1, - expr: &labeledExpr{ - pos: position{line: 144, col: 23, offset: 4006}, - label: "val", - expr: &litMatcher{ - pos: position{line: 144, col: 27, offset: 4010}, - val: "&", - ignoreCase: false, - }, - }, - }, - }, - { - name: "BitwiseAndExpr", - pos: position{line: 148, col: 1, offset: 4076}, - expr: &actionExpr{ - pos: position{line: 148, col: 19, offset: 4094}, - run: (*parser).callonBitwiseAndExpr1, - expr: &seqExpr{ - pos: position{line: 148, col: 19, offset: 4094}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 148, col: 19, offset: 4094}, - label: "lhs", - expr: &ruleRefExpr{ - pos: position{line: 148, col: 23, offset: 4098}, - name: "ArithExpr", - }, - }, - &labeledExpr{ - pos: position{line: 148, col: 33, offset: 4108}, - label: "rest", - expr: &zeroOrMoreExpr{ - pos: position{line: 148, col: 38, offset: 4113}, - expr: &seqExpr{ - pos: position{line: 148, col: 40, offset: 4115}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 148, col: 40, offset: 4115}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 148, col: 42, offset: 4117}, - name: "ArithOperator", - }, - &ruleRefExpr{ - pos: position{line: 148, col: 56, offset: 4131}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 148, col: 58, offset: 4133}, - name: "ArithExpr", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "ArithOperator", - pos: position{line: 152, col: 1, offset: 4205}, - expr: &actionExpr{ - pos: position{line: 152, col: 18, offset: 4222}, - run: (*parser).callonArithOperator1, - expr: &labeledExpr{ - pos: position{line: 152, col: 18, offset: 4222}, - label: "val", - expr: &choiceExpr{ - pos: position{line: 152, col: 23, offset: 4227}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 152, col: 23, offset: 4227}, - val: "+", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 152, col: 29, offset: 4233}, - val: "-", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - { - name: "ArithExpr", - pos: position{line: 156, col: 1, offset: 4300}, - expr: &actionExpr{ - pos: position{line: 156, col: 14, offset: 4313}, - run: (*parser).callonArithExpr1, - expr: &seqExpr{ - pos: position{line: 156, col: 14, offset: 4313}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 156, col: 14, offset: 4313}, - label: "lhs", - expr: &ruleRefExpr{ - pos: position{line: 156, col: 18, offset: 4317}, - name: "FactorExpr", - }, - }, - &labeledExpr{ - pos: position{line: 156, col: 29, offset: 4328}, - label: "rest", - expr: &zeroOrMoreExpr{ - pos: position{line: 156, col: 34, offset: 4333}, - expr: &seqExpr{ - pos: position{line: 156, col: 36, offset: 4335}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 156, col: 36, offset: 4335}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 156, col: 38, offset: 4337}, - name: "FactorOperator", - }, - &ruleRefExpr{ - pos: position{line: 156, col: 53, offset: 4352}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 156, col: 55, offset: 4354}, - name: "FactorExpr", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "FactorOperator", - pos: position{line: 160, col: 1, offset: 4428}, - expr: &actionExpr{ - pos: position{line: 160, col: 19, offset: 4446}, - run: (*parser).callonFactorOperator1, - expr: &labeledExpr{ - pos: position{line: 160, col: 19, offset: 4446}, - label: "val", - expr: &choiceExpr{ - pos: position{line: 160, col: 24, offset: 4451}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 160, col: 24, offset: 4451}, - val: "*", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 160, col: 30, offset: 4457}, - val: "/", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 160, col: 36, offset: 4463}, - val: "%", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - { - name: "FactorExpr", - pos: position{line: 164, col: 1, offset: 4529}, - expr: &choiceExpr{ - pos: position{line: 164, col: 15, offset: 4543}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 164, col: 15, offset: 4543}, - run: (*parser).callonFactorExpr2, - expr: &seqExpr{ - pos: position{line: 164, col: 17, offset: 4545}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 164, col: 17, offset: 4545}, - val: "(", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 164, col: 21, offset: 4549}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 164, col: 23, offset: 4551}, - label: "expr", - expr: &ruleRefExpr{ - pos: position{line: 164, col: 28, offset: 4556}, - name: "ExprTerm", - }, - }, - &ruleRefExpr{ - pos: position{line: 164, col: 37, offset: 4565}, - name: "_", - }, - &litMatcher{ - pos: position{line: 164, col: 39, offset: 4567}, - val: ")", - ignoreCase: false, - }, - }, - }, - }, - &actionExpr{ - pos: position{line: 166, col: 5, offset: 4600}, - run: (*parser).callonFactorExpr10, - expr: &labeledExpr{ - pos: position{line: 166, col: 5, offset: 4600}, - label: "term", - expr: &ruleRefExpr{ - pos: position{line: 166, col: 10, offset: 4605}, - name: "Term", - }, - }, - }, - }, - }, - }, - { - name: "Call", - pos: position{line: 170, col: 1, offset: 4636}, - expr: &actionExpr{ - pos: position{line: 170, col: 9, offset: 4644}, - run: (*parser).callonCall1, - expr: &seqExpr{ - pos: position{line: 170, col: 9, offset: 4644}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 170, col: 9, offset: 4644}, - label: "operator", - expr: &choiceExpr{ - pos: position{line: 170, col: 19, offset: 4654}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 170, col: 19, offset: 4654}, - name: "Ref", - }, - &ruleRefExpr{ - pos: position{line: 170, col: 25, offset: 4660}, - name: "Var", - }, - }, - }, - }, - &litMatcher{ - pos: position{line: 170, col: 30, offset: 4665}, - val: "(", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 170, col: 34, offset: 4669}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 170, col: 36, offset: 4671}, - label: "args", - expr: &ruleRefExpr{ - pos: position{line: 170, col: 41, offset: 4676}, - name: "ExprTermList", - }, - }, - &ruleRefExpr{ - pos: position{line: 170, col: 54, offset: 4689}, - name: "_", - }, - &litMatcher{ - pos: position{line: 170, col: 56, offset: 4691}, - val: ")", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "Term", - pos: position{line: 174, col: 1, offset: 4756}, - expr: &actionExpr{ - pos: position{line: 174, col: 9, offset: 4764}, - run: (*parser).callonTerm1, - expr: &seqExpr{ - pos: position{line: 174, col: 9, offset: 4764}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 174, col: 9, offset: 4764}, - label: "val", - expr: &choiceExpr{ - pos: position{line: 174, col: 15, offset: 4770}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 174, col: 15, offset: 4770}, - name: "Comprehension", - }, - &ruleRefExpr{ - pos: position{line: 174, col: 31, offset: 4786}, - name: "Composite", - }, - &ruleRefExpr{ - pos: position{line: 174, col: 43, offset: 4798}, - name: "Scalar", - }, - &ruleRefExpr{ - pos: position{line: 174, col: 52, offset: 4807}, - name: "Call", - }, - &ruleRefExpr{ - pos: position{line: 174, col: 59, offset: 4814}, - name: "Var", - }, - }, - }, - }, - &labeledExpr{ - pos: position{line: 174, col: 65, offset: 4820}, - label: "refs", - expr: &zeroOrMoreExpr{ - pos: position{line: 174, col: 70, offset: 4825}, - expr: &ruleRefExpr{ - pos: position{line: 174, col: 70, offset: 4825}, - name: "RefOperand", - }, - }, - }, - }, - }, - }, - }, - { - name: "TermPair", - pos: position{line: 178, col: 1, offset: 4892}, - expr: &actionExpr{ - pos: position{line: 178, col: 13, offset: 4904}, - run: (*parser).callonTermPair1, - expr: &seqExpr{ - pos: position{line: 178, col: 13, offset: 4904}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 178, col: 13, offset: 4904}, - label: "key", - expr: &ruleRefExpr{ - pos: position{line: 178, col: 17, offset: 4908}, - name: "Term", - }, - }, - &ruleRefExpr{ - pos: position{line: 178, col: 22, offset: 4913}, - name: "_", - }, - &litMatcher{ - pos: position{line: 178, col: 24, offset: 4915}, - val: ":", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 178, col: 28, offset: 4919}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 178, col: 30, offset: 4921}, - label: "value", - expr: &ruleRefExpr{ - pos: position{line: 178, col: 36, offset: 4927}, - name: "Term", - }, - }, - }, - }, - }, - }, - { - name: "Comprehension", - pos: position{line: 182, col: 1, offset: 4977}, - expr: &choiceExpr{ - pos: position{line: 182, col: 18, offset: 4994}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 182, col: 18, offset: 4994}, - name: "ArrayComprehension", - }, - &ruleRefExpr{ - pos: position{line: 182, col: 39, offset: 5015}, - name: "ObjectComprehension", - }, - &ruleRefExpr{ - pos: position{line: 182, col: 61, offset: 5037}, - name: "SetComprehension", - }, - }, - }, - }, - { - name: "ArrayComprehension", - pos: position{line: 184, col: 1, offset: 5055}, - expr: &actionExpr{ - pos: position{line: 184, col: 23, offset: 5077}, - run: (*parser).callonArrayComprehension1, - expr: &seqExpr{ - pos: position{line: 184, col: 23, offset: 5077}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 184, col: 23, offset: 5077}, - val: "[", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 184, col: 27, offset: 5081}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 184, col: 29, offset: 5083}, - label: "head", - expr: &ruleRefExpr{ - pos: position{line: 184, col: 34, offset: 5088}, - name: "Term", - }, - }, - &ruleRefExpr{ - pos: position{line: 184, col: 39, offset: 5093}, - name: "_", - }, - &litMatcher{ - pos: position{line: 184, col: 41, offset: 5095}, - val: "|", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 184, col: 45, offset: 5099}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 184, col: 47, offset: 5101}, - label: "body", - expr: &ruleRefExpr{ - pos: position{line: 184, col: 52, offset: 5106}, - name: "WhitespaceBody", - }, - }, - &ruleRefExpr{ - pos: position{line: 184, col: 67, offset: 5121}, - name: "_", - }, - &litMatcher{ - pos: position{line: 184, col: 69, offset: 5123}, - val: "]", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "ObjectComprehension", - pos: position{line: 188, col: 1, offset: 5198}, - expr: &actionExpr{ - pos: position{line: 188, col: 24, offset: 5221}, - run: (*parser).callonObjectComprehension1, - expr: &seqExpr{ - pos: position{line: 188, col: 24, offset: 5221}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 188, col: 24, offset: 5221}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 188, col: 28, offset: 5225}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 188, col: 30, offset: 5227}, - label: "head", - expr: &ruleRefExpr{ - pos: position{line: 188, col: 35, offset: 5232}, - name: "TermPair", - }, - }, - &ruleRefExpr{ - pos: position{line: 188, col: 45, offset: 5242}, - name: "_", - }, - &litMatcher{ - pos: position{line: 188, col: 47, offset: 5244}, - val: "|", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 188, col: 51, offset: 5248}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 188, col: 53, offset: 5250}, - label: "body", - expr: &ruleRefExpr{ - pos: position{line: 188, col: 58, offset: 5255}, - name: "WhitespaceBody", - }, - }, - &ruleRefExpr{ - pos: position{line: 188, col: 73, offset: 5270}, - name: "_", - }, - &litMatcher{ - pos: position{line: 188, col: 75, offset: 5272}, - val: "}", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "SetComprehension", - pos: position{line: 192, col: 1, offset: 5348}, - expr: &actionExpr{ - pos: position{line: 192, col: 21, offset: 5368}, - run: (*parser).callonSetComprehension1, - expr: &seqExpr{ - pos: position{line: 192, col: 21, offset: 5368}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 192, col: 21, offset: 5368}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 192, col: 25, offset: 5372}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 192, col: 27, offset: 5374}, - label: "head", - expr: &ruleRefExpr{ - pos: position{line: 192, col: 32, offset: 5379}, - name: "Term", - }, - }, - &ruleRefExpr{ - pos: position{line: 192, col: 37, offset: 5384}, - name: "_", - }, - &litMatcher{ - pos: position{line: 192, col: 39, offset: 5386}, - val: "|", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 192, col: 43, offset: 5390}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 192, col: 45, offset: 5392}, - label: "body", - expr: &ruleRefExpr{ - pos: position{line: 192, col: 50, offset: 5397}, - name: "WhitespaceBody", - }, - }, - &ruleRefExpr{ - pos: position{line: 192, col: 65, offset: 5412}, - name: "_", - }, - &litMatcher{ - pos: position{line: 192, col: 67, offset: 5414}, - val: "}", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "Composite", - pos: position{line: 196, col: 1, offset: 5487}, - expr: &choiceExpr{ - pos: position{line: 196, col: 14, offset: 5500}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 196, col: 14, offset: 5500}, - name: "Object", - }, - &ruleRefExpr{ - pos: position{line: 196, col: 23, offset: 5509}, - name: "Array", - }, - &ruleRefExpr{ - pos: position{line: 196, col: 31, offset: 5517}, - name: "Set", - }, - }, - }, - }, - { - name: "Scalar", - pos: position{line: 198, col: 1, offset: 5522}, - expr: &choiceExpr{ - pos: position{line: 198, col: 11, offset: 5532}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 198, col: 11, offset: 5532}, - name: "Number", - }, - &ruleRefExpr{ - pos: position{line: 198, col: 20, offset: 5541}, - name: "String", - }, - &ruleRefExpr{ - pos: position{line: 198, col: 29, offset: 5550}, - name: "Bool", - }, - &ruleRefExpr{ - pos: position{line: 198, col: 36, offset: 5557}, - name: "Null", - }, - }, - }, - }, - { - name: "Object", - pos: position{line: 200, col: 1, offset: 5563}, - expr: &actionExpr{ - pos: position{line: 200, col: 11, offset: 5573}, - run: (*parser).callonObject1, - expr: &seqExpr{ - pos: position{line: 200, col: 11, offset: 5573}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 200, col: 11, offset: 5573}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 200, col: 15, offset: 5577}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 200, col: 17, offset: 5579}, - label: "list", - expr: &ruleRefExpr{ - pos: position{line: 200, col: 22, offset: 5584}, - name: "ExprTermPairList", - }, - }, - &ruleRefExpr{ - pos: position{line: 200, col: 39, offset: 5601}, - name: "_", - }, - &litMatcher{ - pos: position{line: 200, col: 41, offset: 5603}, - val: "}", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "Array", - pos: position{line: 204, col: 1, offset: 5660}, - expr: &actionExpr{ - pos: position{line: 204, col: 10, offset: 5669}, - run: (*parser).callonArray1, - expr: &seqExpr{ - pos: position{line: 204, col: 10, offset: 5669}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 204, col: 10, offset: 5669}, - val: "[", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 204, col: 14, offset: 5673}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 204, col: 16, offset: 5675}, - label: "list", - expr: &ruleRefExpr{ - pos: position{line: 204, col: 21, offset: 5680}, - name: "ExprTermList", - }, - }, - &ruleRefExpr{ - pos: position{line: 204, col: 34, offset: 5693}, - name: "_", - }, - &litMatcher{ - pos: position{line: 204, col: 36, offset: 5695}, - val: "]", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "Set", - pos: position{line: 208, col: 1, offset: 5751}, - expr: &choiceExpr{ - pos: position{line: 208, col: 8, offset: 5758}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 208, col: 8, offset: 5758}, - name: "SetEmpty", - }, - &ruleRefExpr{ - pos: position{line: 208, col: 19, offset: 5769}, - name: "SetNonEmpty", - }, - }, - }, - }, - { - name: "SetEmpty", - pos: position{line: 210, col: 1, offset: 5782}, - expr: &actionExpr{ - pos: position{line: 210, col: 13, offset: 5794}, - run: (*parser).callonSetEmpty1, - expr: &seqExpr{ - pos: position{line: 210, col: 13, offset: 5794}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 210, col: 13, offset: 5794}, - val: "set(", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 210, col: 20, offset: 5801}, - name: "_", - }, - &litMatcher{ - pos: position{line: 210, col: 22, offset: 5803}, - val: ")", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "SetNonEmpty", - pos: position{line: 215, col: 1, offset: 5880}, - expr: &actionExpr{ - pos: position{line: 215, col: 16, offset: 5895}, - run: (*parser).callonSetNonEmpty1, - expr: &seqExpr{ - pos: position{line: 215, col: 16, offset: 5895}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 215, col: 16, offset: 5895}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 215, col: 20, offset: 5899}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 215, col: 22, offset: 5901}, - label: "list", - expr: &ruleRefExpr{ - pos: position{line: 215, col: 27, offset: 5906}, - name: "ExprTermList", - }, - }, - &ruleRefExpr{ - pos: position{line: 215, col: 40, offset: 5919}, - name: "_", - }, - &litMatcher{ - pos: position{line: 215, col: 42, offset: 5921}, - val: "}", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "Ref", - pos: position{line: 219, col: 1, offset: 5975}, - expr: &actionExpr{ - pos: position{line: 219, col: 8, offset: 5982}, - run: (*parser).callonRef1, - expr: &seqExpr{ - pos: position{line: 219, col: 8, offset: 5982}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 219, col: 8, offset: 5982}, - label: "head", - expr: &choiceExpr{ - pos: position{line: 219, col: 14, offset: 5988}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 219, col: 14, offset: 5988}, - name: "Composite", - }, - &ruleRefExpr{ - pos: position{line: 219, col: 26, offset: 6000}, - name: "Var", - }, - }, - }, - }, - &labeledExpr{ - pos: position{line: 219, col: 31, offset: 6005}, - label: "rest", - expr: &oneOrMoreExpr{ - pos: position{line: 219, col: 36, offset: 6010}, - expr: &ruleRefExpr{ - pos: position{line: 219, col: 36, offset: 6010}, - name: "RefOperand", - }, - }, - }, - }, - }, - }, - }, - { - name: "RefOperand", - pos: position{line: 223, col: 1, offset: 6078}, - expr: &choiceExpr{ - pos: position{line: 223, col: 15, offset: 6092}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 223, col: 15, offset: 6092}, - name: "RefOperandDot", - }, - &ruleRefExpr{ - pos: position{line: 223, col: 31, offset: 6108}, - name: "RefOperandCanonical", - }, - }, - }, - }, - { - name: "RefOperandDot", - pos: position{line: 225, col: 1, offset: 6129}, - expr: &actionExpr{ - pos: position{line: 225, col: 18, offset: 6146}, - run: (*parser).callonRefOperandDot1, - expr: &seqExpr{ - pos: position{line: 225, col: 18, offset: 6146}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 225, col: 18, offset: 6146}, - val: ".", - ignoreCase: false, - }, - &labeledExpr{ - pos: position{line: 225, col: 22, offset: 6150}, - label: "val", - expr: &ruleRefExpr{ - pos: position{line: 225, col: 26, offset: 6154}, - name: "Var", - }, - }, - }, - }, - }, - }, - { - name: "RefOperandCanonical", - pos: position{line: 229, col: 1, offset: 6217}, - expr: &actionExpr{ - pos: position{line: 229, col: 24, offset: 6240}, - run: (*parser).callonRefOperandCanonical1, - expr: &seqExpr{ - pos: position{line: 229, col: 24, offset: 6240}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 229, col: 24, offset: 6240}, - val: "[", - ignoreCase: false, - }, - &labeledExpr{ - pos: position{line: 229, col: 28, offset: 6244}, - label: "val", - expr: &ruleRefExpr{ - pos: position{line: 229, col: 32, offset: 6248}, - name: "ExprTerm", - }, - }, - &litMatcher{ - pos: position{line: 229, col: 41, offset: 6257}, - val: "]", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "Var", - pos: position{line: 233, col: 1, offset: 6286}, - expr: &actionExpr{ - pos: position{line: 233, col: 8, offset: 6293}, - run: (*parser).callonVar1, - expr: &labeledExpr{ - pos: position{line: 233, col: 8, offset: 6293}, - label: "val", - expr: &ruleRefExpr{ - pos: position{line: 233, col: 12, offset: 6297}, - name: "VarChecked", - }, - }, - }, - }, - { - name: "VarChecked", - pos: position{line: 237, col: 1, offset: 6352}, - expr: &seqExpr{ - pos: position{line: 237, col: 15, offset: 6366}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 237, col: 15, offset: 6366}, - label: "val", - expr: &ruleRefExpr{ - pos: position{line: 237, col: 19, offset: 6370}, - name: "VarUnchecked", - }, - }, - ¬CodeExpr{ - pos: position{line: 237, col: 32, offset: 6383}, - run: (*parser).callonVarChecked4, - }, - }, - }, - }, - { - name: "VarUnchecked", - pos: position{line: 241, col: 1, offset: 6448}, - expr: &actionExpr{ - pos: position{line: 241, col: 17, offset: 6464}, - run: (*parser).callonVarUnchecked1, - expr: &seqExpr{ - pos: position{line: 241, col: 17, offset: 6464}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 241, col: 17, offset: 6464}, - name: "VarStart", - }, - &zeroOrMoreExpr{ - pos: position{line: 241, col: 26, offset: 6473}, - expr: &ruleRefExpr{ - pos: position{line: 241, col: 26, offset: 6473}, - name: "VarChar", - }, - }, - }, - }, - }, - }, - { - name: "Number", - pos: position{line: 245, col: 1, offset: 6534}, - expr: &actionExpr{ - pos: position{line: 245, col: 11, offset: 6544}, - run: (*parser).callonNumber1, - expr: &seqExpr{ - pos: position{line: 245, col: 11, offset: 6544}, - exprs: []interface{}{ - &zeroOrOneExpr{ - pos: position{line: 245, col: 11, offset: 6544}, - expr: &litMatcher{ - pos: position{line: 245, col: 11, offset: 6544}, - val: "-", - ignoreCase: false, - }, - }, - &choiceExpr{ - pos: position{line: 245, col: 18, offset: 6551}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 245, col: 18, offset: 6551}, - name: "Float", - }, - &ruleRefExpr{ - pos: position{line: 245, col: 26, offset: 6559}, - name: "Integer", - }, - }, - }, - }, - }, - }, - }, - { - name: "Float", - pos: position{line: 249, col: 1, offset: 6624}, - expr: &choiceExpr{ - pos: position{line: 249, col: 10, offset: 6633}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 249, col: 10, offset: 6633}, - name: "ExponentFloat", - }, - &ruleRefExpr{ - pos: position{line: 249, col: 26, offset: 6649}, - name: "PointFloat", - }, - }, - }, - }, - { - name: "ExponentFloat", - pos: position{line: 251, col: 1, offset: 6661}, - expr: &seqExpr{ - pos: position{line: 251, col: 18, offset: 6678}, - exprs: []interface{}{ - &choiceExpr{ - pos: position{line: 251, col: 20, offset: 6680}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 251, col: 20, offset: 6680}, - name: "PointFloat", - }, - &ruleRefExpr{ - pos: position{line: 251, col: 33, offset: 6693}, - name: "Integer", - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 251, col: 43, offset: 6703}, - name: "Exponent", - }, - }, - }, - }, - { - name: "PointFloat", - pos: position{line: 253, col: 1, offset: 6713}, - expr: &seqExpr{ - pos: position{line: 253, col: 15, offset: 6727}, - exprs: []interface{}{ - &zeroOrOneExpr{ - pos: position{line: 253, col: 15, offset: 6727}, - expr: &ruleRefExpr{ - pos: position{line: 253, col: 15, offset: 6727}, - name: "Integer", - }, - }, - &ruleRefExpr{ - pos: position{line: 253, col: 24, offset: 6736}, - name: "Fraction", - }, - }, - }, - }, - { - name: "Fraction", - pos: position{line: 255, col: 1, offset: 6746}, - expr: &seqExpr{ - pos: position{line: 255, col: 13, offset: 6758}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 255, col: 13, offset: 6758}, - val: ".", - ignoreCase: false, - }, - &oneOrMoreExpr{ - pos: position{line: 255, col: 17, offset: 6762}, - expr: &ruleRefExpr{ - pos: position{line: 255, col: 17, offset: 6762}, - name: "DecimalDigit", - }, - }, - }, - }, - }, - { - name: "Exponent", - pos: position{line: 257, col: 1, offset: 6777}, - expr: &seqExpr{ - pos: position{line: 257, col: 13, offset: 6789}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 257, col: 13, offset: 6789}, - val: "e", - ignoreCase: true, - }, - &zeroOrOneExpr{ - pos: position{line: 257, col: 18, offset: 6794}, - expr: &charClassMatcher{ - pos: position{line: 257, col: 18, offset: 6794}, - val: "[+-]", - chars: []rune{'+', '-'}, - ignoreCase: false, - inverted: false, - }, - }, - &oneOrMoreExpr{ - pos: position{line: 257, col: 24, offset: 6800}, - expr: &ruleRefExpr{ - pos: position{line: 257, col: 24, offset: 6800}, - name: "DecimalDigit", - }, - }, - }, - }, - }, - { - name: "Integer", - pos: position{line: 259, col: 1, offset: 6815}, - expr: &choiceExpr{ - pos: position{line: 259, col: 12, offset: 6826}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 259, col: 12, offset: 6826}, - val: "0", - ignoreCase: false, - }, - &seqExpr{ - pos: position{line: 259, col: 20, offset: 6834}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 259, col: 20, offset: 6834}, - name: "NonZeroDecimalDigit", - }, - &zeroOrMoreExpr{ - pos: position{line: 259, col: 40, offset: 6854}, - expr: &ruleRefExpr{ - pos: position{line: 259, col: 40, offset: 6854}, - name: "DecimalDigit", - }, - }, - }, - }, - }, - }, - }, - { - name: "String", - pos: position{line: 261, col: 1, offset: 6871}, - expr: &choiceExpr{ - pos: position{line: 261, col: 11, offset: 6881}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 261, col: 11, offset: 6881}, - name: "QuotedString", - }, - &ruleRefExpr{ - pos: position{line: 261, col: 26, offset: 6896}, - name: "RawString", - }, - }, - }, - }, - { - name: "QuotedString", - pos: position{line: 263, col: 1, offset: 6907}, - expr: &choiceExpr{ - pos: position{line: 263, col: 17, offset: 6923}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 263, col: 17, offset: 6923}, - run: (*parser).callonQuotedString2, - expr: &seqExpr{ - pos: position{line: 263, col: 17, offset: 6923}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 263, col: 17, offset: 6923}, - val: "\"", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 263, col: 21, offset: 6927}, - expr: &ruleRefExpr{ - pos: position{line: 263, col: 21, offset: 6927}, - name: "Char", - }, - }, - &litMatcher{ - pos: position{line: 263, col: 27, offset: 6933}, - val: "\"", - ignoreCase: false, - }, - }, - }, - }, - &actionExpr{ - pos: position{line: 265, col: 5, offset: 6993}, - run: (*parser).callonQuotedString8, - expr: &seqExpr{ - pos: position{line: 265, col: 5, offset: 6993}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 265, col: 5, offset: 6993}, - val: "\"", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 265, col: 9, offset: 6997}, - expr: &ruleRefExpr{ - pos: position{line: 265, col: 9, offset: 6997}, - name: "Char", - }, - }, - ¬Expr{ - pos: position{line: 265, col: 15, offset: 7003}, - expr: &litMatcher{ - pos: position{line: 265, col: 16, offset: 7004}, - val: "\"", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "RawString", - pos: position{line: 269, col: 1, offset: 7084}, - expr: &actionExpr{ - pos: position{line: 269, col: 14, offset: 7097}, - run: (*parser).callonRawString1, - expr: &seqExpr{ - pos: position{line: 269, col: 14, offset: 7097}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 269, col: 14, offset: 7097}, - val: "`", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 269, col: 18, offset: 7101}, - expr: &charClassMatcher{ - pos: position{line: 269, col: 18, offset: 7101}, - val: "[^`]", - chars: []rune{'`'}, - ignoreCase: false, - inverted: true, - }, - }, - &litMatcher{ - pos: position{line: 269, col: 24, offset: 7107}, - val: "`", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "Bool", - pos: position{line: 273, col: 1, offset: 7169}, - expr: &actionExpr{ - pos: position{line: 273, col: 9, offset: 7177}, - run: (*parser).callonBool1, - expr: &seqExpr{ - pos: position{line: 273, col: 9, offset: 7177}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 273, col: 9, offset: 7177}, - label: "val", - expr: &choiceExpr{ - pos: position{line: 273, col: 14, offset: 7182}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 273, col: 14, offset: 7182}, - val: "true", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 273, col: 23, offset: 7191}, - val: "false", - ignoreCase: false, - }, - }, - }, - }, - ¬Expr{ - pos: position{line: 273, col: 32, offset: 7200}, - expr: &ruleRefExpr{ - pos: position{line: 273, col: 33, offset: 7201}, - name: "VarChar", - }, - }, - }, - }, - }, - }, - { - name: "Null", - pos: position{line: 277, col: 1, offset: 7262}, - expr: &actionExpr{ - pos: position{line: 277, col: 9, offset: 7270}, - run: (*parser).callonNull1, - expr: &seqExpr{ - pos: position{line: 277, col: 9, offset: 7270}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 277, col: 9, offset: 7270}, - val: "null", - ignoreCase: false, - }, - ¬Expr{ - pos: position{line: 277, col: 16, offset: 7277}, - expr: &ruleRefExpr{ - pos: position{line: 277, col: 17, offset: 7278}, - name: "VarChar", - }, - }, - }, - }, - }, - }, - { - name: "VarStart", - pos: position{line: 281, col: 1, offset: 7331}, - expr: &ruleRefExpr{ - pos: position{line: 281, col: 13, offset: 7343}, - name: "AsciiLetter", - }, - }, - { - name: "VarChar", - pos: position{line: 283, col: 1, offset: 7356}, - expr: &choiceExpr{ - pos: position{line: 283, col: 12, offset: 7367}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 283, col: 12, offset: 7367}, - name: "AsciiLetter", - }, - &ruleRefExpr{ - pos: position{line: 283, col: 26, offset: 7381}, - name: "DecimalDigit", - }, - }, - }, - }, - { - name: "AsciiLetter", - pos: position{line: 285, col: 1, offset: 7395}, - expr: &charClassMatcher{ - pos: position{line: 285, col: 16, offset: 7410}, - val: "[A-Za-z_]", - chars: []rune{'_'}, - ranges: []rune{'A', 'Z', 'a', 'z'}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "Char", - pos: position{line: 287, col: 1, offset: 7421}, - expr: &choiceExpr{ - pos: position{line: 287, col: 9, offset: 7429}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 287, col: 11, offset: 7431}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 287, col: 11, offset: 7431}, - expr: &ruleRefExpr{ - pos: position{line: 287, col: 12, offset: 7432}, - name: "EscapedChar", - }, - }, - &anyMatcher{ - line: 287, col: 24, offset: 7444, - }, - }, - }, - &seqExpr{ - pos: position{line: 287, col: 32, offset: 7452}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 287, col: 32, offset: 7452}, - val: "\\", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 287, col: 37, offset: 7457}, - name: "EscapeSequence", - }, - }, - }, - }, - }, - }, - { - name: "EscapedChar", - pos: position{line: 289, col: 1, offset: 7475}, - expr: &charClassMatcher{ - pos: position{line: 289, col: 16, offset: 7490}, - val: "[\\x00-\\x1f\"\\\\]", - chars: []rune{'"', '\\'}, - ranges: []rune{'\x00', '\x1f'}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "EscapeSequence", - pos: position{line: 291, col: 1, offset: 7506}, - expr: &choiceExpr{ - pos: position{line: 291, col: 19, offset: 7524}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 291, col: 19, offset: 7524}, - name: "SingleCharEscape", - }, - &ruleRefExpr{ - pos: position{line: 291, col: 38, offset: 7543}, - name: "UnicodeEscape", - }, - }, - }, - }, - { - name: "SingleCharEscape", - pos: position{line: 293, col: 1, offset: 7558}, - expr: &charClassMatcher{ - pos: position{line: 293, col: 21, offset: 7578}, - val: "[ \" \\\\ / b f n r t ]", - chars: []rune{' ', '"', ' ', '\\', ' ', '/', ' ', 'b', ' ', 'f', ' ', 'n', ' ', 'r', ' ', 't', ' '}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "UnicodeEscape", - pos: position{line: 295, col: 1, offset: 7600}, - expr: &seqExpr{ - pos: position{line: 295, col: 18, offset: 7617}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 295, col: 18, offset: 7617}, - val: "u", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 295, col: 22, offset: 7621}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 295, col: 31, offset: 7630}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 295, col: 40, offset: 7639}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 295, col: 49, offset: 7648}, - name: "HexDigit", - }, - }, - }, - }, - { - name: "DecimalDigit", - pos: position{line: 297, col: 1, offset: 7658}, - expr: &charClassMatcher{ - pos: position{line: 297, col: 17, offset: 7674}, - val: "[0-9]", - ranges: []rune{'0', '9'}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "NonZeroDecimalDigit", - pos: position{line: 299, col: 1, offset: 7681}, - expr: &charClassMatcher{ - pos: position{line: 299, col: 24, offset: 7704}, - val: "[1-9]", - ranges: []rune{'1', '9'}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "HexDigit", - pos: position{line: 301, col: 1, offset: 7711}, - expr: &charClassMatcher{ - pos: position{line: 301, col: 13, offset: 7723}, - val: "[0-9a-fA-F]", - ranges: []rune{'0', '9', 'a', 'f', 'A', 'F'}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "ws", - displayName: "\"whitespace\"", - pos: position{line: 303, col: 1, offset: 7736}, - expr: &oneOrMoreExpr{ - pos: position{line: 303, col: 20, offset: 7755}, - expr: &charClassMatcher{ - pos: position{line: 303, col: 20, offset: 7755}, - val: "[ \\t\\r\\n]", - chars: []rune{' ', '\t', '\r', '\n'}, - ignoreCase: false, - inverted: false, - }, - }, - }, - { - name: "_", - displayName: "\"whitespace\"", - pos: position{line: 305, col: 1, offset: 7767}, - expr: &zeroOrMoreExpr{ - pos: position{line: 305, col: 19, offset: 7785}, - expr: &choiceExpr{ - pos: position{line: 305, col: 21, offset: 7787}, - alternatives: []interface{}{ - &charClassMatcher{ - pos: position{line: 305, col: 21, offset: 7787}, - val: "[ \\t\\r\\n]", - chars: []rune{' ', '\t', '\r', '\n'}, - ignoreCase: false, - inverted: false, - }, - &ruleRefExpr{ - pos: position{line: 305, col: 33, offset: 7799}, - name: "Comment", - }, - }, - }, - }, - }, - { - name: "Comment", - pos: position{line: 307, col: 1, offset: 7811}, - expr: &actionExpr{ - pos: position{line: 307, col: 12, offset: 7822}, - run: (*parser).callonComment1, - expr: &seqExpr{ - pos: position{line: 307, col: 12, offset: 7822}, - exprs: []interface{}{ - &zeroOrMoreExpr{ - pos: position{line: 307, col: 12, offset: 7822}, - expr: &charClassMatcher{ - pos: position{line: 307, col: 12, offset: 7822}, - val: "[ \\t]", - chars: []rune{' ', '\t'}, - ignoreCase: false, - inverted: false, - }, - }, - &litMatcher{ - pos: position{line: 307, col: 19, offset: 7829}, - val: "#", - ignoreCase: false, - }, - &labeledExpr{ - pos: position{line: 307, col: 23, offset: 7833}, - label: "text", - expr: &zeroOrMoreExpr{ - pos: position{line: 307, col: 28, offset: 7838}, - expr: &charClassMatcher{ - pos: position{line: 307, col: 28, offset: 7838}, - val: "[^\\r\\n]", - chars: []rune{'\r', '\n'}, - ignoreCase: false, - inverted: true, - }, - }, - }, - }, - }, - }, - }, - { - name: "EOF", - pos: position{line: 311, col: 1, offset: 7885}, - expr: ¬Expr{ - pos: position{line: 311, col: 8, offset: 7892}, - expr: &anyMatcher{ - line: 311, col: 9, offset: 7893, - }, - }, - }, - }, -} - -func (c *current) onProgram1(vals interface{}) (interface{}, error) { - return makeProgram(c, vals) -} - -func (p *parser) callonProgram1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onProgram1(stack["vals"]) -} - -func (c *current) onStmt1(val interface{}) (interface{}, error) { - return val, nil -} - -func (p *parser) callonStmt1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onStmt1(stack["val"]) -} - -func (c *current) onPackage1(val interface{}) (interface{}, error) { - return makePackage(currentLocation(c), val) -} - -func (p *parser) callonPackage1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onPackage1(stack["val"]) -} - -func (c *current) onImport1(path, alias interface{}) (interface{}, error) { - return makeImport(currentLocation(c), path, alias) -} - -func (p *parser) callonImport1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onImport1(stack["path"], stack["alias"]) -} - -func (c *current) onDefaultRules1(name, operator, value interface{}) (interface{}, error) { - return makeDefaultRule(currentLocation(c), name, operator, value) -} - -func (p *parser) callonDefaultRules1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onDefaultRules1(stack["name"], stack["operator"], stack["value"]) -} - -func (c *current) onNormalRules1(head, rest interface{}) (interface{}, error) { - return makeRule(currentLocation(c), head, rest) -} - -func (p *parser) callonNormalRules1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNormalRules1(stack["head"], stack["rest"]) -} - -func (c *current) onPartialRuleHead1(name, args, value interface{}) (interface{}, error) { - return makeRuleHead(currentLocation(c), name, args, nil, value) -} - -func (p *parser) callonPartialRuleHead1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onPartialRuleHead1(stack["name"], stack["args"], stack["value"]) -} - -func (c *current) onRuleHead1(name, key, value interface{}) (interface{}, error) { - return makeRuleHead(currentLocation(c), name, nil, key, value) -} - -func (p *parser) callonRuleHead1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRuleHead1(stack["name"], stack["key"], stack["value"]) -} - -func (c *current) onArgs1(list interface{}) (interface{}, error) { - return makeArgs(list) -} - -func (p *parser) callonArgs1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onArgs1(stack["list"]) -} - -func (c *current) onElse1(value, body interface{}) (interface{}, error) { - return makeRuleExt(currentLocation(c), value, body) -} + "github.com/open-policy-agent/opa/ast/internal/scanner" + "github.com/open-policy-agent/opa/ast/internal/tokens" + "github.com/open-policy-agent/opa/ast/location" +) -func (p *parser) callonElse1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onElse1(stack["value"], stack["body"]) +// Note: This state is kept isolated from the parser so that we +// can do efficient shallow copies of these values when doing a +// save() and restore(). +type state struct { + s *scanner.Scanner + lastEnd int + skippedNL bool + tok tokens.Token + tokEnd int + lit string + loc Location + errors Errors + comments []*Comment + wildcard int } -func (c *current) onRuleDup1(b interface{}) (interface{}, error) { - return ruleExt{loc: currentLocation(c), body: b.(Body)}, nil +func (s *state) String() string { + return fmt.Sprintf("", s.s, s.tok, s.lit, s.loc, len(s.errors), len(s.comments)) } -func (p *parser) callonRuleDup1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRuleDup1(stack["b"]) +func (s *state) Loc() *location.Location { + cpy := s.loc + return &cpy } -func (c *current) onNonEmptyBraceEnclosedBody1(val interface{}) (interface{}, error) { - if val == nil { - return NewBody(), fmt.Errorf("found empty body") +func (s *state) Text(offset, end int) []byte { + bs := s.s.Bytes() + if offset >= 0 && offset < len(bs) { + if end >= offset && end <= len(bs) { + return bs[offset:end] + } } - return val, nil -} - -func (p *parser) callonNonEmptyBraceEnclosedBody1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNonEmptyBraceEnclosedBody1(stack["val"]) -} - -func (c *current) onBraceEnclosedBody1(val interface{}) (interface{}, error) { - return makeBraceEnclosedBody(currentLocation(c), val) -} - -func (p *parser) callonBraceEnclosedBody1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onBraceEnclosedBody1(stack["val"]) -} - -func (c *current) onWhitespaceBody1(head, tail interface{}) (interface{}, error) { - return makeBody(head, tail, 2) -} - -func (p *parser) callonWhitespaceBody1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onWhitespaceBody1(stack["head"], stack["tail"]) -} - -func (c *current) onNonWhitespaceBody1(head, tail interface{}) (interface{}, error) { - return makeBody(head, tail, 3) -} - -func (p *parser) callonNonWhitespaceBody1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNonWhitespaceBody1(stack["head"], stack["tail"]) -} - -func (c *current) onSomeDecl1(symbols interface{}) (interface{}, error) { - return makeSomeDeclLiteral(currentLocation(c), symbols) -} - -func (p *parser) callonSomeDecl1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSomeDecl1(stack["symbols"]) -} - -func (c *current) onSomeDeclList1(head, rest interface{}) (interface{}, error) { - return makeSomeDeclSymbols(head, rest) -} - -func (p *parser) callonSomeDeclList1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSomeDeclList1(stack["head"], stack["rest"]) -} - -func (c *current) onTermExpr1(negated, value, with interface{}) (interface{}, error) { - return makeLiteral(negated, value, with) -} - -func (p *parser) callonTermExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onTermExpr1(stack["negated"], stack["value"], stack["with"]) -} - -func (c *current) onLiteralExpr1(lhs, rest interface{}) (interface{}, error) { - return makeLiteralExpr(currentLocation(c), lhs, rest) -} - -func (p *parser) callonLiteralExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onLiteralExpr1(stack["lhs"], stack["rest"]) -} - -func (c *current) onLiteralExprOperator1(val interface{}) (interface{}, error) { - return makeInfixOperator(currentLocation(c), c.text) -} - -func (p *parser) callonLiteralExprOperator1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onLiteralExprOperator1(stack["val"]) + return nil } -func (c *current) onNotKeyword1(val interface{}) (interface{}, error) { - return val != nil, nil +// Parser is used to parse Rego statements. +type Parser struct { + r io.Reader + s *state } -func (p *parser) callonNotKeyword1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNotKeyword1(stack["val"]) -} - -func (c *current) onWithKeywordList1(head, rest interface{}) (interface{}, error) { - return makeWithKeywordList(head, rest) -} - -func (p *parser) callonWithKeywordList1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onWithKeywordList1(stack["head"], stack["rest"]) -} - -func (c *current) onWithKeyword1(target, value interface{}) (interface{}, error) { - return makeWithKeyword(currentLocation(c), target, value) -} - -func (p *parser) callonWithKeyword1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onWithKeyword1(stack["target"], stack["value"]) -} - -func (c *current) onExprTerm1(lhs, rest interface{}) (interface{}, error) { - return makeExprTerm(currentLocation(c), lhs, rest) -} - -func (p *parser) callonExprTerm1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onExprTerm1(stack["lhs"], stack["rest"]) -} - -func (c *current) onExprTermPairList1(head, tail interface{}) (interface{}, error) { - return makeExprTermPairList(head, tail) -} - -func (p *parser) callonExprTermPairList1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onExprTermPairList1(stack["head"], stack["tail"]) -} - -func (c *current) onExprTermList1(head, tail interface{}) (interface{}, error) { - return makeExprTermList(head, tail) -} - -func (p *parser) callonExprTermList1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onExprTermList1(stack["head"], stack["tail"]) -} - -func (c *current) onExprTermPair1(key, value interface{}) (interface{}, error) { - return makeExprTermPair(key, value) -} - -func (p *parser) callonExprTermPair1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onExprTermPair1(stack["key"], stack["value"]) -} - -func (c *current) onRelationOperator1(val interface{}) (interface{}, error) { - return makeInfixOperator(currentLocation(c), c.text) -} - -func (p *parser) callonRelationOperator1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRelationOperator1(stack["val"]) -} - -func (c *current) onRelationExpr1(lhs, rest interface{}) (interface{}, error) { - return makeExprTerm(currentLocation(c), lhs, rest) -} - -func (p *parser) callonRelationExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRelationExpr1(stack["lhs"], stack["rest"]) +// NewParser creates and initializes a Parser. +func NewParser() *Parser { + p := &Parser{s: &state{}} + return p } -func (c *current) onBitwiseOrOperator1(val interface{}) (interface{}, error) { - return makeInfixOperator(currentLocation(c), c.text) +// WithFilename provides the filename for Location details +// on parsed statements. +func (p *Parser) WithFilename(filename string) *Parser { + p.s.loc.File = filename + return p } -func (p *parser) callonBitwiseOrOperator1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onBitwiseOrOperator1(stack["val"]) +// WithReader provides the io.Reader that the parser will +// use as its source. +func (p *Parser) WithReader(r io.Reader) *Parser { + p.r = r + return p } -func (c *current) onBitwiseOrExpr1(lhs, rest interface{}) (interface{}, error) { - return makeExprTerm(currentLocation(c), lhs, rest) -} +// Parse will read the Rego source and parse statements and +// comments as they are found. Any errors encountered while +// parsing will be accumulated and returned as a list of Errors. +func (p *Parser) Parse() ([]Statement, []*Comment, Errors) { -func (p *parser) callonBitwiseOrExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onBitwiseOrExpr1(stack["lhs"], stack["rest"]) -} + var err error + p.s.s, err = scanner.New(p.r) + if err != nil { + return nil, nil, Errors{ + &Error{ + Code: ParseErr, + Message: err.Error(), + Location: nil, + }, + } + } -func (c *current) onBitwiseAndOperator1(val interface{}) (interface{}, error) { - return makeInfixOperator(currentLocation(c), c.text) -} + // read the first token to initialize the parser + p.scan() -func (p *parser) callonBitwiseAndOperator1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onBitwiseAndOperator1(stack["val"]) -} + var stmts []Statement -func (c *current) onBitwiseAndExpr1(lhs, rest interface{}) (interface{}, error) { - return makeExprTerm(currentLocation(c), lhs, rest) -} + // Read from the scanner until the last token is reached or no statements + // can be parsed. Attempt to parse package statements, import statements, + // rule statements, and then body/query statements (in that order). If a + // statement cannot be parsed, restore the parser state before trying the + // next type of statement. If a statement can be parsed, continue from that + // point trying to parse packages, imports, etc. in the same order. + for p.s.tok != tokens.EOF { -func (p *parser) callonBitwiseAndExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onBitwiseAndExpr1(stack["lhs"], stack["rest"]) -} + s := p.save() -func (c *current) onArithOperator1(val interface{}) (interface{}, error) { - return makeInfixOperator(currentLocation(c), c.text) -} + if pkg := p.parsePackage(); pkg != nil { + stmts = append(stmts, pkg) + continue + } else if len(p.s.errors) > 0 { + break + } -func (p *parser) callonArithOperator1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onArithOperator1(stack["val"]) -} + p.restore(s) + s = p.save() -func (c *current) onArithExpr1(lhs, rest interface{}) (interface{}, error) { - return makeExprTerm(currentLocation(c), lhs, rest) -} + if imp := p.parseImport(); imp != nil { + stmts = append(stmts, imp) + continue + } else if len(p.s.errors) > 0 { + break + } -func (p *parser) callonArithExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onArithExpr1(stack["lhs"], stack["rest"]) -} + p.restore(s) + s = p.save() -func (c *current) onFactorOperator1(val interface{}) (interface{}, error) { - return makeInfixOperator(currentLocation(c), c.text) -} + if rules := p.parseRules(); rules != nil { + for i := range rules { + stmts = append(stmts, rules[i]) + } + continue + } else if len(p.s.errors) > 0 { + break + } -func (p *parser) callonFactorOperator1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onFactorOperator1(stack["val"]) -} + p.restore(s) + s = p.save() -func (c *current) onFactorExpr2(expr interface{}) (interface{}, error) { - return expr, nil -} + if body := p.parseQuery(true, tokens.EOF); body != nil { + stmts = append(stmts, body) + continue + } -func (p *parser) callonFactorExpr2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onFactorExpr2(stack["expr"]) -} + break + } -func (c *current) onFactorExpr10(term interface{}) (interface{}, error) { - return term, nil + return stmts, p.s.comments, p.s.errors } -func (p *parser) callonFactorExpr10() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onFactorExpr10(stack["term"]) -} +func (p *Parser) parsePackage() *Package { -func (c *current) onCall1(operator, args interface{}) (interface{}, error) { - return makeCall(currentLocation(c), operator, args) -} + var pkg Package + pkg.SetLoc(p.s.Loc()) -func (p *parser) callonCall1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onCall1(stack["operator"], stack["args"]) -} + if p.s.tok != tokens.Package { + return nil + } -func (c *current) onTerm1(val, refs interface{}) (interface{}, error) { - return makeRef(currentLocation(c), val, refs) -} + p.scan() + if p.s.tok != tokens.Ident { + p.illegalToken() + return nil + } -func (p *parser) callonTerm1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onTerm1(stack["val"], stack["refs"]) -} + term := p.parseTerm() -func (c *current) onTermPair1(key, value interface{}) (interface{}, error) { - return makeExprTermPair(key, value) -} + if term != nil { + switch v := term.Value.(type) { + case Var: + pkg.Path = Ref{ + DefaultRootDocument.Copy().SetLocation(term.Location), + StringTerm(string(v)).SetLocation(term.Location), + } + case Ref: + pkg.Path = make(Ref, len(v)+1) + pkg.Path[0] = DefaultRootDocument.Copy().SetLocation(v[0].Location) + first, ok := v[0].Value.(Var) + if !ok { + p.errorf(v[0].Location, "unexpected %v token: expecting var", TypeName(v[0].Value)) + return nil + } + pkg.Path[1] = StringTerm(string(first)).SetLocation(v[0].Location) + for i := 2; i < len(pkg.Path); i++ { + switch v[i-1].Value.(type) { + case String: + pkg.Path[i] = v[i-1] + default: + p.errorf(v[i-1].Location, "unexpected %v token: expecting string", TypeName(v[i-1].Value)) + return nil + } + } + default: + p.illegalToken() + return nil + } + } -func (p *parser) callonTermPair1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onTermPair1(stack["key"], stack["value"]) -} + if pkg.Path == nil { + if len(p.s.errors) == 0 { + p.error(p.s.Loc(), "expected path") + } + return nil + } -func (c *current) onArrayComprehension1(head, body interface{}) (interface{}, error) { - return makeArrayComprehension(currentLocation(c), head, body) + return &pkg } -func (p *parser) callonArrayComprehension1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onArrayComprehension1(stack["head"], stack["body"]) -} +func (p *Parser) parseImport() *Import { -func (c *current) onObjectComprehension1(head, body interface{}) (interface{}, error) { - return makeObjectComprehension(currentLocation(c), head, body) -} + var imp Import + imp.SetLoc(p.s.Loc()) -func (p *parser) callonObjectComprehension1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onObjectComprehension1(stack["head"], stack["body"]) -} + if p.s.tok != tokens.Import { + return nil + } -func (c *current) onSetComprehension1(head, body interface{}) (interface{}, error) { - return makeSetComprehension(currentLocation(c), head, body) -} + p.scan() + if p.s.tok != tokens.Ident { + p.error(p.s.Loc(), "expected ident") + return nil + } -func (p *parser) callonSetComprehension1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSetComprehension1(stack["head"], stack["body"]) -} + term := p.parseTerm() + if term != nil { + switch v := term.Value.(type) { + case Var: + imp.Path = RefTerm(term).SetLocation(term.Location) + case Ref: + for i := 1; i < len(v); i++ { + if _, ok := v[i].Value.(String); !ok { + p.errorf(v[i].Location, "unexpected %v token: expecting string", TypeName(v[i].Value)) + return nil + } + } + imp.Path = term + } + } -func (c *current) onObject1(list interface{}) (interface{}, error) { - return makeObject(currentLocation(c), list) -} + if imp.Path == nil { + p.error(p.s.Loc(), "expected path") + return nil + } -func (p *parser) callonObject1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onObject1(stack["list"]) -} + path := imp.Path.Value.(Ref) -func (c *current) onArray1(list interface{}) (interface{}, error) { - return makeArray(currentLocation(c), list) -} + if !RootDocumentNames.Contains(path[0]) { + p.errorf(imp.Path.Location, "unexpected import path, must begin with one of: %v, got: %v", RootDocumentNames, path[0]) + return nil + } -func (p *parser) callonArray1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onArray1(stack["list"]) -} + if p.s.tok == tokens.As { + p.scan() -func (c *current) onSetEmpty1() (interface{}, error) { - var empty []*Term - return makeSet(currentLocation(c), empty) -} + if p.s.tok != tokens.Ident { + p.illegal("expected var") + return nil + } -func (p *parser) callonSetEmpty1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSetEmpty1() -} + alias := p.parseTerm() -func (c *current) onSetNonEmpty1(list interface{}) (interface{}, error) { - return makeSet(currentLocation(c), list) -} + v, ok := alias.Value.(Var) + if !ok { + p.illegal("expected var") + return nil + } + imp.Alias = v + } -func (p *parser) callonSetNonEmpty1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSetNonEmpty1(stack["list"]) + return &imp } -func (c *current) onRef1(head, rest interface{}) (interface{}, error) { - return makeRef(currentLocation(c), head, rest) -} +func (p *Parser) parseRules() []*Rule { -func (p *parser) callonRef1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRef1(stack["head"], stack["rest"]) -} + var rule Rule + rule.SetLoc(p.s.Loc()) -func (c *current) onRefOperandDot1(val interface{}) (interface{}, error) { - return makeRefOperandDot(currentLocation(c), val) -} + if p.s.tok == tokens.Default { + p.scan() + rule.Default = true + } -func (p *parser) callonRefOperandDot1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRefOperandDot1(stack["val"]) -} + if p.s.tok != tokens.Ident { + return nil + } -func (c *current) onRefOperandCanonical1(val interface{}) (interface{}, error) { - return val, nil -} + if rule.Head = p.parseHead(rule.Default); rule.Head == nil { + return nil + } -func (p *parser) callonRefOperandCanonical1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRefOperandCanonical1(stack["val"]) -} + if rule.Default { + if !p.validateDefaultRuleValue(&rule) { + return nil + } -func (c *current) onVar1(val interface{}) (interface{}, error) { - return val.([]interface{})[0], nil -} + rule.Body = NewBody(NewExpr(BooleanTerm(true).SetLocation(rule.Location)).SetLocation(rule.Location)) + return []*Rule{&rule} + } -func (p *parser) callonVar1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onVar1(stack["val"]) -} + if p.s.tok == tokens.LBrace { + p.scan() + if rule.Body = p.parseBody(tokens.RBrace); rule.Body == nil { + return nil + } + p.scan() + } else { + return nil + } -func (c *current) onVarChecked4(val interface{}) (bool, error) { - return IsKeyword(string(val.(*Term).Value.(Var))), nil -} + if p.s.tok == tokens.Else { -func (p *parser) callonVarChecked4() (bool, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onVarChecked4(stack["val"]) -} + if rule.Head.Assign { + p.error(p.s.Loc(), "else keyword cannot be used on rule declared with := operator") + return nil + } -func (c *current) onVarUnchecked1() (interface{}, error) { - return makeVar(currentLocation(c), c.text) -} + if rule.Head.Key != nil { + p.error(p.s.Loc(), "else keyword cannot be used on partial rules") + return nil + } -func (p *parser) callonVarUnchecked1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onVarUnchecked1() -} + if rule.Else = p.parseElse(rule.Head); rule.Else == nil { + return nil + } + } -func (c *current) onNumber1() (interface{}, error) { - return makeNumber(currentLocation(c), c.text) -} + var rules []*Rule -func (p *parser) callonNumber1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNumber1() -} + rules = append(rules, &rule) -func (c *current) onQuotedString2() (interface{}, error) { - return makeString(currentLocation(c), c.text) -} + for p.s.tok == tokens.LBrace { -func (p *parser) callonQuotedString2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onQuotedString2() -} + if rule.Else != nil { + p.error(p.s.Loc(), "expected else keyword") + return nil + } -func (c *current) onQuotedString8() (interface{}, error) { - return makeNonterminatedString(currentLocation(c), string(c.text)) -} + loc := p.s.Loc() -func (p *parser) callonQuotedString8() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onQuotedString8() -} + p.scan() + var next Rule -func (c *current) onRawString1() (interface{}, error) { - return makeRawString(currentLocation(c), c.text) -} + if next.Body = p.parseBody(tokens.RBrace); next.Body == nil { + return nil + } -func (p *parser) callonRawString1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRawString1() -} + next.Head = rule.Head.Copy() + setLocRecursive(next.Head, loc) + next.SetLoc(loc) + rules = append(rules, &next) + p.scan() + } -func (c *current) onBool1(val interface{}) (interface{}, error) { - return makeBool(currentLocation(c), c.text) + return rules } -func (p *parser) callonBool1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onBool1(stack["val"]) -} +func (p *Parser) parseElse(head *Head) *Rule { -func (c *current) onNull1() (interface{}, error) { - return makeNull(currentLocation(c)) -} + var rule Rule + rule.SetLoc(p.s.Loc()) + p.scan() -func (p *parser) callonNull1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNull1() -} + switch p.s.tok { + case tokens.LBrace: + p.scan() -func (c *current) onComment1(text interface{}) (interface{}, error) { - return makeComments(c, text) -} + if rule.Body = p.parseBody(tokens.RBrace); rule.Body == nil { + return nil + } -func (p *parser) callonComment1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onComment1(stack["text"]) -} + p.scan() -var ( - // errNoRule is returned when the grammar to parse has no rule. - errNoRule = errors.New("grammar has no rule") + if p.s.tok == tokens.Else { + if rule.Else = p.parseElse(head); rule.Else == nil { + return nil + } + } + rule.Head = head.Copy() + rule.Head.Value = BooleanTerm(true) + setLocRecursive(rule.Head, rule.Location) + return &rule + + case tokens.Unify: + p.scan() + rule.Head = head.Copy() + rule.Head.Value = p.parseTermRelation() + setLocRecursive(rule.Head, rule.Location) + + if rule.Head.Value == nil { + return nil + } - // errInvalidEntrypoint is returned when the specified entrypoint rule - // does not exit. - errInvalidEntrypoint = errors.New("invalid entrypoint") + if p.s.tok != tokens.LBrace { + return &rule + } - // errInvalidEncoding is returned when the source is not properly - // utf8-encoded. - errInvalidEncoding = errors.New("invalid encoding") + p.scan() - // errMaxExprCnt is used to signal that the maximum number of - // expressions have been parsed. - errMaxExprCnt = errors.New("max number of expresssions parsed") -) + if rule.Body = p.parseBody(tokens.RBrace); rule.Body == nil { + return nil + } -// Option is a function that can set an option on the parser. It returns -// the previous setting as an Option. -type Option func(*parser) Option - -// MaxExpressions creates an Option to stop parsing after the provided -// number of expressions have been parsed, if the value is 0 then the parser will -// parse for as many steps as needed (possibly an infinite number). -// -// The default for maxExprCnt is 0. -func MaxExpressions(maxExprCnt uint64) Option { - return func(p *parser) Option { - oldMaxExprCnt := p.maxExprCnt - p.maxExprCnt = maxExprCnt - return MaxExpressions(oldMaxExprCnt) - } -} + p.scan() -// Entrypoint creates an Option to set the rule name to use as entrypoint. -// The rule name must have been specified in the -alternate-entrypoints -// if generating the parser with the -optimize-grammar flag, otherwise -// it may have been optimized out. Passing an empty string sets the -// entrypoint to the first rule in the grammar. -// -// The default is to start parsing at the first rule in the grammar. -func Entrypoint(ruleName string) Option { - return func(p *parser) Option { - oldEntrypoint := p.entrypoint - p.entrypoint = ruleName - if ruleName == "" { - p.entrypoint = g.rules[0].name + if p.s.tok == tokens.Else { + if rule.Else = p.parseElse(head); rule.Else == nil { + return nil + } } - return Entrypoint(oldEntrypoint) - } -} -// Statistics adds a user provided Stats struct to the parser to allow -// the user to process the results after the parsing has finished. -// Also the key for the "no match" counter is set. -// -// Example usage: -// -// input := "input" -// stats := Stats{} -// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) -// if err != nil { -// log.Panicln(err) -// } -// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") -// if err != nil { -// log.Panicln(err) -// } -// fmt.Println(string(b)) -// -func Statistics(stats *Stats, choiceNoMatch string) Option { - return func(p *parser) Option { - oldStats := p.Stats - p.Stats = stats - oldChoiceNoMatch := p.choiceNoMatch - p.choiceNoMatch = choiceNoMatch - if p.Stats.ChoiceAltCnt == nil { - p.Stats.ChoiceAltCnt = make(map[string]map[string]int) - } - return Statistics(oldStats, oldChoiceNoMatch) + return &rule + default: + p.illegal("expected else value term or rule body") + return nil } } -// Debug creates an Option to set the debug flag to b. When set to true, -// debugging information is printed to stdout while parsing. -// -// The default is false. -func Debug(b bool) Option { - return func(p *parser) Option { - old := p.debug - p.debug = b - return Debug(old) - } -} +func (p *Parser) parseHead(defaultRule bool) *Head { -// Memoize creates an Option to set the memoize flag to b. When set to true, -// the parser will cache all results so each expression is evaluated only -// once. This guarantees linear parsing time even for pathological cases, -// at the expense of more memory and slower times for typical cases. -// -// The default is false. -func Memoize(b bool) Option { - return func(p *parser) Option { - old := p.memoize - p.memoize = b - return Memoize(old) - } -} + var head Head + head.SetLoc(p.s.Loc()) -// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. -// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) -// by character class matchers and is matched by the any matcher. -// The returned matched value, c.text and c.offset are NOT affected. -// -// The default is false. -func AllowInvalidUTF8(b bool) Option { - return func(p *parser) Option { - old := p.allowInvalidUTF8 - p.allowInvalidUTF8 = b - return AllowInvalidUTF8(old) + if term := p.parseVar(); term != nil { + if v, ok := term.Value.(Var); ok { + head.Name = v + } } -} - -// Recover creates an Option to set the recover flag to b. When set to -// true, this causes the parser to recover from panics and convert it -// to an error. Setting it to false can be useful while debugging to -// access the full stack trace. -// -// The default is true. -func Recover(b bool) Option { - return func(p *parser) Option { - old := p.recover - p.recover = b - return Recover(old) + if head.Name == "" { + p.illegal("expected rule head name") } -} -// GlobalStore creates an Option to set a key to a certain value in -// the globalStore. -func GlobalStore(key string, value interface{}) Option { - return func(p *parser) Option { - old := p.cur.globalStore[key] - p.cur.globalStore[key] = value - return GlobalStore(key, old) - } -} + p.scan() -// InitState creates an Option to set a key to a certain value in -// the global "state" store. -func InitState(key string, value interface{}) Option { - return func(p *parser) Option { - old := p.cur.state[key] - p.cur.state[key] = value - return InitState(key, old) + if p.s.tok == tokens.LParen { + p.scan() + if p.s.tok != tokens.RParen { + head.Args = p.parseTermList(tokens.RParen, nil) + if head.Args == nil { + return nil + } + } + p.scan() } -} -// ParseFile parses the file identified by filename. -func ParseFile(filename string, opts ...Option) (i interface{}, err error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer func() { - if closeErr := f.Close(); closeErr != nil { - err = closeErr + if p.s.tok == tokens.LBrack { + p.scan() + head.Key = p.parseTermRelation() + if head.Key == nil { + p.illegal("expected rule key term (e.g., %s[] { ... })", head.Name) } - }() - return ParseReader(filename, f, opts...) -} - -// ParseReader parses the data from r using filename as information in the -// error messages. -func ParseReader(filename string, r io.Reader, opts ...Option) (interface{}, error) { - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err + if p.s.tok != tokens.RBrack { + p.illegal("non-terminated rule key") + } + p.scan() } - return Parse(filename, b, opts...) -} - -// Parse parses the data from b using filename as information in the -// error messages. -func Parse(filename string, b []byte, opts ...Option) (interface{}, error) { - return newParser(filename, b, opts...).parse(g) -} - -// position records a position in the text. -type position struct { - line, col, offset int -} - -func (p position) String() string { - return fmt.Sprintf("%d:%d [%d]", p.line, p.col, p.offset) -} - -// savepoint stores all state required to go back to this point in the -// parser. -type savepoint struct { - position - rn rune - w int -} - -type current struct { - pos position // start position of the match - text []byte // raw text of the match - - // state is a store for arbitrary key,value pairs that the user wants to be - // tied to the backtracking of the parser. - // This is always rolled back if a parsing rule fails. - state storeDict - - // globalStore is a general store for the user to store arbitrary key-value - // pairs that they need to manage and that they do not want tied to the - // backtracking of the parser. This is only modified by the user and never - // rolled back by the parser. It is always up to the user to keep this in a - // consistent state. - globalStore storeDict -} - -type storeDict map[string]interface{} - -// the AST types... - -type grammar struct { - pos position - rules []*rule -} - -type rule struct { - pos position - name string - displayName string - expr interface{} -} - -type choiceExpr struct { - pos position - alternatives []interface{} -} - -type actionExpr struct { - pos position - expr interface{} - run func(*parser) (interface{}, error) -} + if p.s.tok == tokens.Unify { + p.scan() + head.Value = p.parseTermRelation() + if head.Value == nil { + p.illegal("expected rule value term (e.g., %s[] { ... })", head.Name) + } + } else if p.s.tok == tokens.Assign { + + if defaultRule { + p.error(p.s.Loc(), "default rules must use = operator (not := operator)") + return nil + } else if head.Key != nil { + p.error(p.s.Loc(), "partial rules must use = operator (not := operator)") + return nil + } else if len(head.Args) > 0 { + p.error(p.s.Loc(), "functions must use = operator (not := operator)") + return nil + } -type recoveryExpr struct { - pos position - expr interface{} - recoverExpr interface{} - failureLabel []string -} + p.scan() + head.Assign = true + head.Value = p.parseTermRelation() + if head.Value == nil { + p.illegal("expected rule value term (e.g., %s := { ... })", head.Name) + } + } -type seqExpr struct { - pos position - exprs []interface{} -} + if head.Value == nil && head.Key == nil { + head.Value = BooleanTerm(true).SetLocation(head.Location) + } -type throwExpr struct { - pos position - label string + return &head } -type labeledExpr struct { - pos position - label string - expr interface{} +func (p *Parser) parseBody(end tokens.Token) Body { + return p.parseQuery(false, end) } -type expr struct { - pos position - expr interface{} -} +func (p *Parser) parseQuery(requireSemi bool, end tokens.Token) Body { + body := Body{} -type andExpr expr -type notExpr expr -type zeroOrOneExpr expr -type zeroOrMoreExpr expr -type oneOrMoreExpr expr + if p.s.tok == end { + p.error(p.s.Loc(), "found empty body") + return nil + } -type ruleRefExpr struct { - pos position - name string -} + for { -type stateCodeExpr struct { - pos position - run func(*parser) error -} + expr := p.parseLiteral() + if expr == nil { + return nil + } -type andCodeExpr struct { - pos position - run func(*parser) (bool, error) -} + body.Append(expr) -type notCodeExpr struct { - pos position - run func(*parser) (bool, error) -} + if p.s.tok == tokens.Semicolon { + p.scan() + continue + } -type litMatcher struct { - pos position - val string - ignoreCase bool -} + if p.s.tok == end || requireSemi { + return body + } -type charClassMatcher struct { - pos position - val string - basicLatinChars [128]bool - chars []rune - ranges []rune - classes []*unicode.RangeTable - ignoreCase bool - inverted bool + if !p.s.skippedNL { + // If there was already an error then don't pile this one on + if len(p.s.errors) == 0 { + p.illegal(`expected \n or %s or %s`, tokens.Semicolon, end) + } + return nil + } + } } -type anyMatcher position +func (p *Parser) parseLiteral() (expr *Expr) { -// errList cumulates the errors found by the parser. -type errList []error + offset := p.s.loc.Offset + loc := p.s.Loc() -func (e *errList) add(err error) { - *e = append(*e, err) -} - -func (e errList) err() error { - if len(e) == 0 { - return nil - } - e.dedupe() - return e -} - -func (e *errList) dedupe() { - var cleaned []error - set := make(map[string]bool) - for _, err := range *e { - if msg := err.Error(); !set[msg] { - set[msg] = true - cleaned = append(cleaned, err) + defer func() { + if expr != nil { + loc.Text = p.s.Text(offset, p.s.lastEnd) + expr.SetLoc(loc) } - } - *e = cleaned -} + }() -func (e errList) Error() string { - switch len(e) { - case 0: - return "" - case 1: - return e[0].Error() + var negated bool + switch p.s.tok { + case tokens.Some: + return p.parseSome() + case tokens.Not: + p.scan() + negated = true + fallthrough default: - var buf bytes.Buffer - - for i, err := range e { - if i > 0 { - buf.WriteRune('\n') + expr := p.parseExpr() + if expr != nil { + expr.Negated = negated + if p.s.tok == tokens.With { + if expr.With = p.parseWith(); expr.With == nil { + return nil + } } - buf.WriteString(err.Error()) + return expr } - return buf.String() + return nil } } -// parserError wraps an error with a prefix indicating the rule in which -// the error occurred. The original error is stored in the Inner field. -type parserError struct { - Inner error - pos position - prefix string - expected []string -} +func (p *Parser) parseWith() []*With { -// Error returns the error message. -func (p *parserError) Error() string { - return p.prefix + ": " + p.Inner.Error() -} + withs := []*With{} -// newParser creates a parser with the specified input source and options. -func newParser(filename string, b []byte, opts ...Option) *parser { - stats := Stats{ - ChoiceAltCnt: make(map[string]map[string]int), - } + for { - p := &parser{ - filename: filename, - errs: new(errList), - data: b, - pt: savepoint{position: position{line: 1}}, - recover: true, - cur: current{ - state: make(storeDict), - globalStore: make(storeDict), - }, - maxFailPos: position{col: 1, line: 1}, - maxFailExpected: make([]string, 0, 20), - Stats: &stats, - // start rule is rule [0] unless an alternate entrypoint is specified - entrypoint: g.rules[0].name, - } - p.setOptions(opts) + // NOTE(tsandall): location is not being set correctly on with + // statements. need special test case for this. - if p.maxExprCnt == 0 { - p.maxExprCnt = math.MaxUint64 - } + var with With + p.scan() - return p -} + if p.s.tok != tokens.Ident { + p.illegal("expected ident") + return nil + } -// setOptions applies the options to the parser. -func (p *parser) setOptions(opts []Option) { - for _, opt := range opts { - opt(p) - } -} + if with.Target = p.parseTerm(); with.Target == nil { + return nil + } -type resultTuple struct { - v interface{} - b bool - end savepoint -} + switch with.Target.Value.(type) { + case Ref, Var: + break + default: + p.illegal("expected with target path") + } -const choiceNoMatch = -1 + if p.s.tok != tokens.As { + p.illegal("expected as keyword") + return nil + } -// Stats stores some statistics, gathered during parsing -type Stats struct { - // ExprCnt counts the number of expressions processed during parsing - // This value is compared to the maximum number of expressions allowed - // (set by the MaxExpressions option). - ExprCnt uint64 + p.scan() - // ChoiceAltCnt is used to count for each ordered choice expression, - // which alternative is used how may times. - // These numbers allow to optimize the order of the ordered choice expression - // to increase the performance of the parser - // - // The outer key of ChoiceAltCnt is composed of the name of the rule as well - // as the line and the column of the ordered choice. - // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. - // For each alternative the number of matches are counted. If an ordered choice does not - // match, a special counter is incremented. The name of this counter is set with - // the parser option Statistics. - // For an alternative to be included in ChoiceAltCnt, it has to match at least once. - ChoiceAltCnt map[string]map[string]int -} + if with.Value = p.parseTermRelation(); with.Value == nil { + return nil + } -type parser struct { - filename string - pt savepoint - cur current + withs = append(withs, &with) - data []byte - errs *errList + if p.s.tok != tokens.With { + break + } + } - depth int - recover bool - debug bool + return withs +} - memoize bool - // memoization table for the packrat algorithm: - // map[offset in source] map[expression or rule] {value, match} - memo map[int]map[interface{}]resultTuple +func (p *Parser) parseSome() *Expr { - // rules table, maps the rule identifier to the rule node - rules map[string]*rule - // variables stack, map of label to value - vstack []map[string]interface{} - // rule stack, allows identification of the current rule in errors - rstack []*rule + decl := &SomeDecl{} + decl.SetLoc(p.s.Loc()) - // parse fail - maxFailPos position - maxFailExpected []string - maxFailInvertExpected bool + for { - // max number of expressions to be parsed - maxExprCnt uint64 - // entrypoint for the parser - entrypoint string + p.scan() - allowInvalidUTF8 bool + switch p.s.tok { + case tokens.Ident: + } - *Stats + if p.s.tok != tokens.Ident { + p.illegal("expected var") + return nil + } - choiceNoMatch string - // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse - recoveryStack []map[string]interface{} -} + decl.Symbols = append(decl.Symbols, p.parseVar()) -// push a variable set on the vstack. -func (p *parser) pushV() { - if cap(p.vstack) == len(p.vstack) { - // create new empty slot in the stack - p.vstack = append(p.vstack, nil) - } else { - // slice to 1 more - p.vstack = p.vstack[:len(p.vstack)+1] - } + p.scan() - // get the last args set - m := p.vstack[len(p.vstack)-1] - if m != nil && len(m) == 0 { - // empty map, all good - return + if p.s.tok != tokens.Comma { + break + } } - m = make(map[string]interface{}) - p.vstack[len(p.vstack)-1] = m + return NewExpr(decl).SetLocation(decl.Location) } -// pop a variable set from the vstack. -func (p *parser) popV() { - // if the map is not empty, clear it - m := p.vstack[len(p.vstack)-1] - if len(m) > 0 { - // GC that map - p.vstack[len(p.vstack)-1] = nil - } - p.vstack = p.vstack[:len(p.vstack)-1] -} +func (p *Parser) parseExpr() *Expr { -// push a recovery expression with its labels to the recoveryStack -func (p *parser) pushRecovery(labels []string, expr interface{}) { - if cap(p.recoveryStack) == len(p.recoveryStack) { - // create new empty slot in the stack - p.recoveryStack = append(p.recoveryStack, nil) - } else { - // slice to 1 more - p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] - } + lhs := p.parseTermRelation() - m := make(map[string]interface{}, len(labels)) - for _, fl := range labels { - m[fl] = expr + if lhs == nil { + return nil } - p.recoveryStack[len(p.recoveryStack)-1] = m -} -// pop a recovery expression from the recoveryStack -func (p *parser) popRecovery() { - // GC that map - p.recoveryStack[len(p.recoveryStack)-1] = nil - - p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] -} + if op := p.parseTermOp(tokens.Assign, tokens.Unify); op != nil { + if rhs := p.parseTermRelation(); rhs != nil { + return NewExpr([]*Term{op, lhs, rhs}) + } + return nil + } -func (p *parser) print(prefix, s string) string { - if !p.debug { - return s + // NOTE(tsandall): the top-level call term is converted to an expr because + // the evaluator does not support the call term type (nested calls are + // rewritten by the compiler.) + if call, ok := lhs.Value.(Call); ok { + return NewExpr([]*Term(call)) } - fmt.Printf("%s %d:%d:%d: %s [%#U]\n", - prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) - return s + return NewExpr(lhs) } -func (p *parser) in(s string) string { - p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) +// parseTermRelation consumes the next term from the input and returns it. If a +// term cannot be parsed the return value is nil and error will be recorded. The +// scanner will be advanced to the next token before returning. +func (p *Parser) parseTermRelation() *Term { + return p.parseTermRelationRec(nil, p.s.loc.Offset) } -func (p *parser) out(s string) string { - p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) +func (p *Parser) parseTermRelationRec(lhs *Term, offset int) *Term { + if lhs == nil { + lhs = p.parseTermOr(nil, offset) + } + if lhs != nil { + if op := p.parseTermOp(tokens.Equal, tokens.Neq, tokens.Lt, tokens.Gt, tokens.Lte, tokens.Gte); op != nil { + if rhs := p.parseTermOr(nil, p.s.loc.Offset); rhs != nil { + call := p.setLoc(CallTerm(op, lhs, rhs), lhs.Location, offset, p.s.lastEnd) + switch p.s.tok { + case tokens.Equal, tokens.Neq, tokens.Lt, tokens.Gt, tokens.Lte, tokens.Gte: + return p.parseTermRelationRec(call, offset) + default: + return call + } + } + } + } + return lhs } -func (p *parser) addErr(err error) { - p.addErrAt(err, p.pt.position, []string{}) +func (p *Parser) parseTermOr(lhs *Term, offset int) *Term { + if lhs == nil { + lhs = p.parseTermAnd(nil, offset) + } + if lhs != nil { + if op := p.parseTermOp(tokens.Or); op != nil { + if rhs := p.parseTermAnd(nil, p.s.loc.Offset); rhs != nil { + call := p.setLoc(CallTerm(op, lhs, rhs), lhs.Location, offset, p.s.lastEnd) + switch p.s.tok { + case tokens.Or: + return p.parseTermOr(call, offset) + default: + return call + } + } + } + return lhs + } + return nil } -func (p *parser) addErrAt(err error, pos position, expected []string) { - var buf bytes.Buffer - if p.filename != "" { - buf.WriteString(p.filename) - } - if buf.Len() > 0 { - buf.WriteString(":") +func (p *Parser) parseTermAnd(lhs *Term, offset int) *Term { + if lhs == nil { + lhs = p.parseTermArith(nil, offset) } - buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) - if len(p.rstack) > 0 { - if buf.Len() > 0 { - buf.WriteString(": ") - } - rule := p.rstack[len(p.rstack)-1] - if rule.displayName != "" { - buf.WriteString("rule " + rule.displayName) - } else { - buf.WriteString("rule " + rule.name) + if lhs != nil { + if op := p.parseTermOp(tokens.And); op != nil { + if rhs := p.parseTermArith(nil, p.s.loc.Offset); rhs != nil { + call := p.setLoc(CallTerm(op, lhs, rhs), lhs.Location, offset, p.s.lastEnd) + switch p.s.tok { + case tokens.And: + return p.parseTermAnd(call, offset) + default: + return call + } + } } + return lhs } - pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} - p.errs.add(pe) + return nil } -func (p *parser) failAt(fail bool, pos position, want string) { - // process fail if parsing fails and not inverted or parsing succeeds and invert is set - if fail == p.maxFailInvertExpected { - if pos.offset < p.maxFailPos.offset { - return +func (p *Parser) parseTermArith(lhs *Term, offset int) *Term { + if lhs == nil { + lhs = p.parseTermFactor(nil, offset) + } + if lhs != nil { + if op := p.parseTermOp(tokens.Add, tokens.Sub); op != nil { + if rhs := p.parseTermFactor(nil, p.s.loc.Offset); rhs != nil { + call := p.setLoc(CallTerm(op, lhs, rhs), lhs.Location, offset, p.s.lastEnd) + switch p.s.tok { + case tokens.Add, tokens.Sub: + return p.parseTermArith(call, offset) + default: + return call + } + } } + } + return lhs +} - if pos.offset > p.maxFailPos.offset { - p.maxFailPos = pos - p.maxFailExpected = p.maxFailExpected[:0] +func (p *Parser) parseTermFactor(lhs *Term, offset int) *Term { + if lhs == nil { + lhs = p.parseTerm() + } + if lhs != nil { + if op := p.parseTermOp(tokens.Mul, tokens.Quo, tokens.Rem); op != nil { + if rhs := p.parseTerm(); rhs != nil { + call := p.setLoc(CallTerm(op, lhs, rhs), lhs.Location, offset, p.s.lastEnd) + switch p.s.tok { + case tokens.Mul, tokens.Quo, tokens.Rem: + return p.parseTermFactor(call, offset) + default: + return call + } + } } - - if p.maxFailInvertExpected { - want = "!" + want + } + return lhs +} + +func (p *Parser) parseTerm() *Term { + var term *Term + switch p.s.tok { + case tokens.Null: + term = NullTerm().SetLocation(p.s.Loc()) + case tokens.True: + term = BooleanTerm(true).SetLocation(p.s.Loc()) + case tokens.False: + term = BooleanTerm(false).SetLocation(p.s.Loc()) + case tokens.Sub, tokens.Dot, tokens.Number: + term = p.parseNumber() + case tokens.String: + term = p.parseString() + case tokens.Ident: + term = p.parseVar() + case tokens.LBrack: + term = p.parseArray() + case tokens.LBrace: + term = p.parseSetOrObject() + case tokens.LParen: + offset := p.s.loc.Offset + p.scan() + if r := p.parseTermRelation(); r != nil { + if p.s.tok == tokens.RParen { + r.Location.Text = p.s.Text(offset, p.s.tokEnd) + term = r + } else { + p.error(p.s.Loc(), "non-terminated expression") + } } - p.maxFailExpected = append(p.maxFailExpected, want) + default: + p.illegalToken() + return nil } + + return p.parseTermFinish(term) } -// read advances the parser to the next rune. -func (p *parser) read() { - p.pt.offset += p.pt.w - rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) - p.pt.rn = rn - p.pt.w = n - p.pt.col++ - if rn == '\n' { - p.pt.line++ - p.pt.col = 0 +func (p *Parser) parseTermFinish(head *Term) *Term { + if head == nil { + return nil } - - if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune - if !p.allowInvalidUTF8 { - p.addErr(errInvalidEncoding) + offset := p.s.loc.Offset + p.scanWS() + switch p.s.tok { + case tokens.LParen, tokens.Dot, tokens.LBrack: + return p.parseRef(head, offset) + case tokens.Whitespace: + p.scan() + fallthrough + default: + if _, ok := head.Value.(Var); ok && RootDocumentNames.Contains(head) { + return RefTerm(head).SetLocation(head.Location) + } + return head + } +} + +func (p *Parser) parseNumber() *Term { + var prefix string + loc := p.s.Loc() + if p.s.tok == tokens.Sub { + prefix = "-" + p.scan() + switch p.s.tok { + case tokens.Number, tokens.Dot: + break + default: + p.illegal("expected number") + return nil + } + } + if p.s.tok == tokens.Dot { + prefix += "." + p.scan() + if p.s.tok != tokens.Number { + p.illegal("expected number") + return nil } } -} -// restore parser position to the savepoint pt. -func (p *parser) restore(pt savepoint) { - if p.debug { - defer p.out(p.in("restore")) + // Ensure that the number is valid + s := prefix + p.s.lit + f, ok := new(big.Float).SetString(s) + if !ok { + p.illegal("expected number") + return nil } - if pt.offset == p.pt.offset { - return + + // Put limit on size of exponent to prevent non-linear cost of String() + // function on big.Float from causing denial of service: https://github.com/golang/go/issues/11068 + // + // n == sign * mantissa * 2^exp + // 0.5 <= mantissa < 1.0 + // + // The limit is arbitrary. + exp := f.MantExp(nil) + if exp > 1e5 || exp < -1e5 { + p.error(p.s.Loc(), "number too big") + return nil } - p.pt = pt -} -// Cloner is implemented by any value that has a Clone method, which returns a -// copy of the value. This is mainly used for types which are not passed by -// value (e.g map, slice, chan) or structs that contain such types. -// -// This is used in conjunction with the global state feature to create proper -// copies of the state to allow the parser to properly restore the state in -// the case of backtracking. -type Cloner interface { - Clone() interface{} + // Note: Use the original string, do *not* round trip from + // the big.Float as it can cause precision loss. + r := NumberTerm(json.Number(s)).SetLocation(loc) + return r } -// clone and return parser current state. -func (p *parser) cloneState() storeDict { - if p.debug { - defer p.out(p.in("cloneState")) - } - - state := make(storeDict, len(p.cur.state)) - for k, v := range p.cur.state { - if c, ok := v.(Cloner); ok { - state[k] = c.Clone() - } else { - state[k] = v +func (p *Parser) parseString() *Term { + if p.s.lit[0] == '"' { + var s string + err := json.Unmarshal([]byte(p.s.lit), &s) + if err != nil { + p.errorf(p.s.Loc(), "illegal string literal: %s", p.s.lit) + return nil } + term := StringTerm(s).SetLocation(p.s.Loc()) + return term } - return state + return p.parseRawString() } -// restore parser current state to the state storeDict. -// every restoreState should applied only one time for every cloned state -func (p *parser) restoreState(state storeDict) { - if p.debug { - defer p.out(p.in("restoreState")) +func (p *Parser) parseRawString() *Term { + if len(p.s.lit) < 2 { + return nil } - p.cur.state = state + term := StringTerm(p.s.lit[1 : len(p.s.lit)-1]).SetLocation(p.s.Loc()) + return term } -// get the slice of bytes from the savepoint start to the current position. -func (p *parser) sliceFrom(start savepoint) []byte { - return p.data[start.position.offset:p.pt.position.offset] -} +// this is the name to use for instantiating an empty set, e.g., `set()`. +var setConstructor = RefTerm(VarTerm("set")) -func (p *parser) getMemoized(node interface{}) (resultTuple, bool) { - if len(p.memo) == 0 { - return resultTuple{}, false - } - m := p.memo[p.pt.offset] - if len(m) == 0 { - return resultTuple{}, false - } - res, ok := m[node] - return res, ok -} +func (p *Parser) parseCall(operator *Term, offset int) (term *Term) { -func (p *parser) setMemoized(pt savepoint, node interface{}, tuple resultTuple) { - if p.memo == nil { - p.memo = make(map[int]map[interface{}]resultTuple) - } - m := p.memo[pt.offset] - if m == nil { - m = make(map[interface{}]resultTuple) - p.memo[pt.offset] = m + loc := operator.Location + var end int + + defer func() { + p.setLoc(term, loc, offset, end) + }() + + p.scan() + + if p.s.tok == tokens.RParen { + end = p.s.tokEnd + p.scanWS() + if operator.Equal(setConstructor) { + return SetTerm() + } + return CallTerm(operator) } - m[node] = tuple -} -func (p *parser) buildRulesTable(g *grammar) { - p.rules = make(map[string]*rule, len(g.rules)) - for _, r := range g.rules { - p.rules[r.name] = r + if r := p.parseTermList(tokens.RParen, []*Term{operator}); r != nil { + end = p.s.tokEnd + p.scanWS() + return CallTerm(r...) } + + return nil } -func (p *parser) parse(g *grammar) (val interface{}, err error) { - if len(g.rules) == 0 { - p.addErr(errNoRule) - return nil, p.errs.err() - } +func (p *Parser) parseRef(head *Term, offset int) (term *Term) { - // TODO : not super critical but this could be generated - p.buildRulesTable(g) + loc := head.Location + var end int - if p.recover { - // panic can be used in action code to stop parsing immediately - // and return the panic as an error. - defer func() { - if e := recover(); e != nil { - if p.debug { - defer p.out(p.in("panic handler")) - } - val = nil - switch e := e.(type) { - case error: - p.addErr(e) - default: - p.addErr(fmt.Errorf("%v", e)) - } - err = p.errs.err() - } - }() - } + defer func() { + p.setLoc(term, loc, offset, end) + }() - startRule, ok := p.rules[p.entrypoint] - if !ok { - p.addErr(errInvalidEntrypoint) - return nil, p.errs.err() + switch h := head.Value.(type) { + case Var, Array, Object, Set, *ArrayComprehension, *ObjectComprehension, *SetComprehension, Call: + // ok + default: + p.errorf(loc, "illegal ref (head cannot be %v)", TypeName(h)) } - p.read() // advance to first rune - val, ok = p.parseRule(startRule) - if !ok { - if len(*p.errs) == 0 { - // If parsing fails, but no errors have been recorded, the expected values - // for the farthest parser position are returned as error. - maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) - for _, v := range p.maxFailExpected { - maxFailExpectedMap[v] = struct{}{} - } - expected := make([]string, 0, len(maxFailExpectedMap)) - eof := false - if _, ok := maxFailExpectedMap["!."]; ok { - delete(maxFailExpectedMap, "!.") - eof = true + ref := []*Term{head} + + for { + switch p.s.tok { + case tokens.Dot: + p.scanWS() + if p.s.tok != tokens.Ident { + p.illegal("expected %v", tokens.Ident) + return nil } - for k := range maxFailExpectedMap { - expected = append(expected, k) + ref = append(ref, StringTerm(p.s.lit).SetLocation(p.s.Loc())) + p.scanWS() + case tokens.LParen: + term = p.parseCall(p.setLoc(RefTerm(ref...), loc, offset, p.s.loc.Offset), offset) + if term != nil { + switch p.s.tok { + case tokens.Whitespace: + p.scan() + end = p.s.lastEnd + return term + case tokens.Dot, tokens.LBrack: + term = p.parseRef(term, offset) + } } - sort.Strings(expected) - if eof { - expected = append(expected, "EOF") + end = p.s.tokEnd + return term + case tokens.LBrack: + p.scan() + if term := p.parseTermRelation(); term != nil { + if p.s.tok != tokens.RBrack { + p.illegal("expected %v", tokens.LBrack) + return nil + } + ref = append(ref, term) + p.scanWS() + } else { + return nil } - p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + case tokens.Whitespace: + end = p.s.lastEnd + p.scan() + return RefTerm(ref...) + default: + end = p.s.lastEnd + return RefTerm(ref...) } - - return nil, p.errs.err() } - return val, p.errs.err() } -func listJoin(list []string, sep string, lastSep string) string { - switch len(list) { - case 0: - return "" - case 1: - return list[0] - default: - return fmt.Sprintf("%s %s %s", strings.Join(list[:len(list)-1], sep), lastSep, list[len(list)-1]) - } -} +func (p *Parser) parseArray() (term *Term) { -func (p *parser) parseRule(rule *rule) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseRule " + rule.name)) - } + loc := p.s.Loc() + offset := p.s.loc.Offset - if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } - } + defer func() { + p.setLoc(term, loc, offset, p.s.tokEnd) + }() - start := p.pt - p.rstack = append(p.rstack, rule) - p.pushV() - val, ok := p.parseExpr(rule.expr) - p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } + p.scan() - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) + if p.s.tok == tokens.RBrack { + return ArrayTerm() } - return val, ok -} -func (p *parser) parseExpr(expr interface{}) (interface{}, bool) { - var pt savepoint + potentialComprehension := true - if p.memoize { - res, ok := p.getMemoized(expr) - if ok { - p.restore(res.end) - return res.v, res.b - } - pt = p.pt + // Skip leading commas, eg [, x, y] + // Supported for backwards compatibility. In the future + // we should make this a parse error. + if p.s.tok == tokens.Comma { + potentialComprehension = false + p.scan() } - p.ExprCnt++ - if p.ExprCnt > p.maxExprCnt { - panic(errMaxExprCnt) - } + s := p.save() - var val interface{} - var ok bool - switch expr := expr.(type) { - case *actionExpr: - val, ok = p.parseActionExpr(expr) - case *andCodeExpr: - val, ok = p.parseAndCodeExpr(expr) - case *andExpr: - val, ok = p.parseAndExpr(expr) - case *anyMatcher: - val, ok = p.parseAnyMatcher(expr) - case *charClassMatcher: - val, ok = p.parseCharClassMatcher(expr) - case *choiceExpr: - val, ok = p.parseChoiceExpr(expr) - case *labeledExpr: - val, ok = p.parseLabeledExpr(expr) - case *litMatcher: - val, ok = p.parseLitMatcher(expr) - case *notCodeExpr: - val, ok = p.parseNotCodeExpr(expr) - case *notExpr: - val, ok = p.parseNotExpr(expr) - case *oneOrMoreExpr: - val, ok = p.parseOneOrMoreExpr(expr) - case *recoveryExpr: - val, ok = p.parseRecoveryExpr(expr) - case *ruleRefExpr: - val, ok = p.parseRuleRefExpr(expr) - case *seqExpr: - val, ok = p.parseSeqExpr(expr) - case *stateCodeExpr: - val, ok = p.parseStateCodeExpr(expr) - case *throwExpr: - val, ok = p.parseThrowExpr(expr) - case *zeroOrMoreExpr: - val, ok = p.parseZeroOrMoreExpr(expr) - case *zeroOrOneExpr: - val, ok = p.parseZeroOrOneExpr(expr) - default: - panic(fmt.Sprintf("unknown expression type %T", expr)) - } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } - return val, ok -} + // NOTE(tsandall): The parser cannot attempt a relational term here because + // of ambiguity around comprehensions. For example, given: + // + // {1 | 1} + // + // Does this represent a set comprehension or a set containing binary OR + // call? We resolve the ambiguity by prioritizing comprehensions. + head := p.parseTerm() -func (p *parser) parseActionExpr(act *actionExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseActionExpr")) + if head == nil { + return nil } - start := p.pt - val, ok := p.parseExpr(act.expr) - if ok { - p.cur.pos = start.position - p.cur.text = p.sliceFrom(start) - state := p.cloneState() - actVal, err := act.run(p) - if err != nil { - p.addErrAt(err, start.position, []string{}) + switch p.s.tok { + case tokens.RBrack: + return ArrayTerm(head) + case tokens.Comma: + p.scan() + if terms := p.parseTermList(tokens.RBrack, []*Term{head}); terms != nil { + return NewTerm(Array(terms)) } - p.restoreState(state) - - val = actVal - } - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + return nil + case tokens.Or: + if potentialComprehension { + // Try to parse as if it is an array comprehension + p.scan() + if body := p.parseBody(tokens.RBrack); body != nil { + return ArrayComprehensionTerm(head, body) + } + if p.s.tok != tokens.Comma { + return nil + } + } + // fall back to parsing as a normal array definition + fallthrough + default: + p.restore(s) + if terms := p.parseTermList(tokens.RBrack, nil); terms != nil { + return NewTerm(Array(terms)) + } + return nil } - return val, ok } -func (p *parser) parseAndCodeExpr(and *andCodeExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseAndCodeExpr")) - } +func (p *Parser) parseSetOrObject() (term *Term) { - state := p.cloneState() + loc := p.s.Loc() + offset := p.s.loc.Offset - ok, err := and.run(p) - if err != nil { - p.addErr(err) - } - p.restoreState(state) + defer func() { + p.setLoc(term, loc, offset, p.s.tokEnd) + }() - return nil, ok -} + p.scan() -func (p *parser) parseAndExpr(and *andExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseAndExpr")) + if p.s.tok == tokens.RBrace { + return ObjectTerm() } - pt := p.pt - state := p.cloneState() - p.pushV() - _, ok := p.parseExpr(and.expr) - p.popV() - p.restoreState(state) - p.restore(pt) + potentialComprehension := true - return nil, ok -} - -func (p *parser) parseAnyMatcher(any *anyMatcher) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseAnyMatcher")) + // Skip leading commas, eg {, x, y} + // Supported for backwards compatibility. In the future + // we should make this a parse error. + if p.s.tok == tokens.Comma { + potentialComprehension = false + p.scan() } - if p.pt.rn == utf8.RuneError && p.pt.w == 0 { - // EOF - see utf8.DecodeRune - p.failAt(false, p.pt.position, ".") - return nil, false + s := p.save() + + // Try parsing just a single term first to give comprehensions higher + // priority to "or" calls in ambiguous situations. Eg: { a | b } + // will be a set comprehension. + // + // Note: We don't know yet if it is a set or object being defined. + head := p.parseTerm() + + if head == nil { + return nil } - start := p.pt - p.read() - p.failAt(true, start.position, ".") - return p.sliceFrom(start), true -} -func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseCharClassMatcher")) + switch p.s.tok { + case tokens.Or: + if potentialComprehension { + return p.parseSet(s, head, potentialComprehension) + } + case tokens.RBrace, tokens.Comma: + return p.parseSet(s, head, potentialComprehension) + case tokens.Colon: + return p.parseObject(head, potentialComprehension) } - cur := p.pt.rn - start := p.pt + p.restore(s) - // can't match EOF - if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune - p.failAt(false, start.position, chr.val) - return nil, false + if head = p.parseTermRelation(); head == nil { + return nil } - if chr.ignoreCase { - cur = unicode.ToLower(cur) + switch p.s.tok { + case tokens.RBrace, tokens.Comma: + return p.parseSet(s, head, false) + case tokens.Colon: + // It still might be an object comprehension, eg { a+1: b | ... } + return p.parseObject(head, potentialComprehension) + default: + p.illegal("non-terminated set") } - // try to match in the list of available chars - for _, rn := range chr.chars { - if rn == cur { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false + return nil +} + +func (p *Parser) parseSet(s *state, head *Term, potentialComprehension bool) *Term { + switch p.s.tok { + case tokens.RBrace: + return SetTerm(head) + case tokens.Comma: + p.scan() + if terms := p.parseTermList(tokens.RBrace, []*Term{head}); terms != nil { + return SetTerm(terms...) + } + return nil + case tokens.Or: + if potentialComprehension { + // Try to parse as if it is a set comprehension + p.scan() + if body := p.parseBody(tokens.RBrace); body != nil { + return SetComprehensionTerm(head, body) } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true + if p.s.tok != tokens.Comma { + return nil + } + } + // Fall back to parsing as normal set definition + p.restore(s) + if terms := p.parseTermList(tokens.RBrace, nil); terms != nil { + return SetTerm(terms...) } + return nil } + return nil +} - // try to match in the list of ranges - for i := 0; i < len(chr.ranges); i += 2 { - if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } +func (p *Parser) parseObject(k *Term, potentialComprehension bool) *Term { + // NOTE(tsandall): Assumption: this function is called after parsing the key + // of the head element and then receiving a colon token from the scanner. + // Advance beyond the colon and attempt to parse an object. + p.scan() + + s := p.save() + v := p.parseTerm() + + if v == nil { + return nil } - // try to match in the list of Unicode classes - for _, cl := range chr.classes { - if unicode.Is(cl, cur) { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false + switch p.s.tok { + case tokens.RBrace, tokens.Comma, tokens.Or: + if potentialComprehension { + if term := p.parseObjectFinish(k, v, true); term != nil { + return term } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true } } - if chr.inverted { - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - p.failAt(false, start.position, chr.val) - return nil, false -} + p.restore(s) -func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) - m := p.ChoiceAltCnt[choiceIdent] - if m == nil { - m = make(map[string]int) - p.ChoiceAltCnt[choiceIdent] = m - } - // We increment altI by 1, so the keys do not start at 0 - alt := strconv.Itoa(altI + 1) - if altI == choiceNoMatch { - alt = p.choiceNoMatch + if v = p.parseTermRelation(); v == nil { + return nil } - m[alt]++ -} -func (p *parser) parseChoiceExpr(ch *choiceExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseChoiceExpr")) + switch p.s.tok { + case tokens.Comma, tokens.RBrace: + return p.parseObjectFinish(k, v, false) + default: + p.illegal("non-terminated object") } - for altI, alt := range ch.alternatives { - // dummy assignment to prevent compile error if optimized - _ = altI - - state := p.cloneState() + return nil +} - p.pushV() - val, ok := p.parseExpr(alt) - p.popV() - if ok { - p.incChoiceAltCnt(ch, altI) - return val, ok +func (p *Parser) parseObjectFinish(key, val *Term, potentialComprehension bool) *Term { + switch p.s.tok { + case tokens.RBrace: + return ObjectTerm([2]*Term{key, val}) + case tokens.Or: + if potentialComprehension { + p.scan() + if body := p.parseBody(tokens.RBrace); body != nil { + return ObjectComprehensionTerm(key, val, body) + } + } else { + p.illegal("non-terminated object") + } + case tokens.Comma: + p.scan() + if r := p.parseTermPairList(tokens.RBrace, [][2]*Term{{key, val}}); r != nil { + return ObjectTerm(r...) } - p.restoreState(state) } - p.incChoiceAltCnt(ch, choiceNoMatch) - return nil, false + return nil } -func (p *parser) parseLabeledExpr(lab *labeledExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseLabeledExpr")) +func (p *Parser) parseTermList(end tokens.Token, r []*Term) []*Term { + if p.s.tok == end { + return r } - - p.pushV() - val, ok := p.parseExpr(lab.expr) - p.popV() - if ok && lab.label != "" { - m := p.vstack[len(p.vstack)-1] - m[lab.label] = val + for { + term := p.parseTermRelation() + if term != nil { + r = append(r, term) + switch p.s.tok { + case end: + return r + case tokens.Comma: + p.scan() + if p.s.tok == end { + return r + } + continue + default: + p.illegal(fmt.Sprintf("expected %q or %q", tokens.Comma, end)) + return nil + } + } + return nil } - return val, ok } -func (p *parser) parseLitMatcher(lit *litMatcher) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseLitMatcher")) +func (p *Parser) parseTermPairList(end tokens.Token, r [][2]*Term) [][2]*Term { + if p.s.tok == end { + return r } - - ignoreCase := "" - if lit.ignoreCase { - ignoreCase = "i" - } - val := fmt.Sprintf("%q%s", lit.val, ignoreCase) - start := p.pt - for _, want := range lit.val { - cur := p.pt.rn - if lit.ignoreCase { - cur = unicode.ToLower(cur) - } - if cur != want { - p.failAt(false, start.position, val) - p.restore(start) - return nil, false + for { + key := p.parseTermRelation() + if key != nil { + switch p.s.tok { + case tokens.Colon: + p.scan() + if val := p.parseTermRelation(); val != nil { + r = append(r, [2]*Term{key, val}) + switch p.s.tok { + case end: + return r + case tokens.Comma: + p.scan() + if p.s.tok == end { + return r + } + continue + default: + p.illegal(fmt.Sprintf("expected %q or %q", tokens.Comma, end)) + return nil + } + } + default: + p.illegal(fmt.Sprintf("expected %q", tokens.Colon)) + return nil + } } - p.read() + return nil } - p.failAt(true, start.position, val) - return p.sliceFrom(start), true } -func (p *parser) parseNotCodeExpr(not *notCodeExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseNotCodeExpr")) +func (p *Parser) parseTermOp(values ...tokens.Token) *Term { + for i := range values { + if p.s.tok == values[i] { + r := RefTerm(VarTerm(fmt.Sprint(p.s.tok)).SetLocation(p.s.Loc())).SetLocation(p.s.Loc()) + p.scan() + return r + } } + return nil +} - state := p.cloneState() +func (p *Parser) parseVar() *Term { - ok, err := not.run(p) - if err != nil { - p.addErr(err) - } - p.restoreState(state) + s := p.s.lit - return nil, !ok -} + term := VarTerm(s).SetLocation(p.s.Loc()) -func (p *parser) parseNotExpr(not *notExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseNotExpr")) + // Update wildcard values with unique identifiers + if term.Equal(Wildcard) { + term.Value = Var(p.genwildcard()) } - pt := p.pt - state := p.cloneState() - p.pushV() - p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) - p.maxFailInvertExpected = !p.maxFailInvertExpected - p.popV() - p.restoreState(state) - p.restore(pt) - - return nil, !ok + return term } -func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseOneOrMoreExpr")) - } - - var vals []interface{} +func (p *Parser) genwildcard() string { + c := p.s.wildcard + p.s.wildcard++ + return fmt.Sprintf("%v%d", WildcardPrefix, c) +} - for { - p.pushV() - val, ok := p.parseExpr(expr.expr) - p.popV() - if !ok { - if len(vals) == 0 { - // did not match once, no match - return nil, false - } - return vals, true - } - vals = append(vals, val) - } +func (p *Parser) error(loc *location.Location, reason string) { + p.errorf(loc, reason) } -func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) - } +func (p *Parser) errorf(loc *location.Location, f string, a ...interface{}) { + p.s.errors = append(p.s.errors, &Error{ + Code: ParseErr, + Message: fmt.Sprintf(f, a...), + Location: loc, + Details: newParserErrorDetail(p.s.s.Bytes(), loc.Offset), + }) +} - p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) - p.popRecovery() +func (p *Parser) illegal(note string, a ...interface{}) { - return val, ok -} + tok := p.s.tok.String() -func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseRuleRefExpr " + ref.name)) + if p.s.tok == tokens.Illegal { + p.errorf(p.s.Loc(), "illegal token") + return } - if ref.name == "" { - panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + tokType := "token" + if p.s.tok >= tokens.Package && p.s.tok <= tokens.False { + tokType = "keyword" } - rule := p.rules[ref.name] - if rule == nil { - p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) - return nil, false + note = fmt.Sprintf(note, a...) + if len(note) > 0 { + p.errorf(p.s.Loc(), "unexpected %s %s: %v", tok, tokType, note) + } else { + p.errorf(p.s.Loc(), "unexpected %s %s", tok, tokType) } - return p.parseRule(rule) } -func (p *parser) parseSeqExpr(seq *seqExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseSeqExpr")) - } +func (p *Parser) illegalToken() { + p.illegal("") +} - vals := make([]interface{}, 0, len(seq.exprs)) +func (p *Parser) scan() { + p.doScan(true) +} - pt := p.pt - state := p.cloneState() - for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) - if !ok { - p.restoreState(state) - p.restore(pt) - return nil, false - } - vals = append(vals, val) - } - return vals, true +func (p *Parser) scanWS() { + p.doScan(false) } -func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseStateCodeExpr")) - } +func (p *Parser) doScan(skipws bool) { - err := state.run(p) - if err != nil { - p.addErr(err) + // NOTE(tsandall): the last position is used to compute the "text" field for + // complex AST nodes. Whitespace never affects the last position of an AST + // node so do not update it when scanning. + if p.s.tok != tokens.Whitespace { + p.s.lastEnd = p.s.tokEnd + p.s.skippedNL = false } - return nil, true -} -func (p *parser) parseThrowExpr(expr *throwExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseThrowExpr")) - } + var errs []scanner.Error + for { + var pos scanner.Position + p.s.tok, pos, p.s.lit, errs = p.s.s.Scan() + + p.s.tokEnd = pos.End + p.s.loc.Row = pos.Row + p.s.loc.Col = pos.Col + p.s.loc.Offset = pos.Offset + p.s.loc.Text = p.s.Text(pos.Offset, pos.End) + + for _, err := range errs { + p.error(p.s.Loc(), err.Message) + } + + if len(errs) > 0 { + p.s.tok = tokens.Illegal + } - for i := len(p.recoveryStack) - 1; i >= 0; i-- { - if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { - return val, ok + if p.s.tok == tokens.Whitespace { + if p.s.lit == "\n" { + p.s.skippedNL = true + } + if skipws { + continue } } - } - return nil, false -} + if p.s.tok != tokens.Comment { + break + } -func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseZeroOrMoreExpr")) + // For backwards compatibility leave a nil + // Text value if there is no text rather than + // an empty string. + var commentText []byte + if len(p.s.lit) > 1 { + commentText = []byte(p.s.lit[1:]) + } + comment := NewComment(commentText) + comment.SetLoc(p.s.Loc()) + p.s.comments = append(p.s.comments, comment) } +} + +func (p *Parser) save() *state { + cpy := *p.s + s := *cpy.s + cpy.s = &s + return &cpy +} - var vals []interface{} +func (p *Parser) restore(s *state) { + p.s = s +} - for { - p.pushV() - val, ok := p.parseExpr(expr.expr) - p.popV() - if !ok { - return vals, true +func setLocRecursive(x interface{}, loc *location.Location) { + NewGenericVisitor(func(x interface{}) bool { + if node, ok := x.(Node); ok { + node.SetLoc(loc) } - vals = append(vals, val) + return false + }).Walk(x) +} + +func (p *Parser) setLoc(term *Term, loc *location.Location, offset, end int) *Term { + if term != nil { + cpy := *loc + term.Location = &cpy + term.Location.Text = p.s.Text(offset, end) } + return term } -func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseZeroOrOneExpr")) +func (p *Parser) validateDefaultRuleValue(rule *Rule) bool { + if rule.Head.Value == nil { + p.error(rule.Loc(), fmt.Sprintf("illegal default rule (must have a value)")) + return false } - p.pushV() - val, _ := p.parseExpr(expr.expr) - p.popV() - // whether it matched or not, consider it a match - return val, true + valid := true + vis := NewGenericVisitor(func(x interface{}) bool { + switch x.(type) { + case *ArrayComprehension, *ObjectComprehension, *SetComprehension: // skip closures + return true + case Ref, Var, Call: + p.error(rule.Loc(), fmt.Sprintf("illegal default rule (value cannot contain %v)", TypeName(x))) + valid = false + return true + } + return false + }) + + vis.Walk(rule.Head.Value.Value) + return valid } diff --git a/ast/parser_ext.go b/ast/parser_ext.go index 03bd20f4d4..84748486e2 100644 --- a/ast/parser_ext.go +++ b/ast/parser_ext.go @@ -3,16 +3,16 @@ // license that can be found in the LICENSE file. // This file contains extra functions for parsing Rego. -// Most of the parsing is handled by the auto-generated code in -// parser.go, however, there are additional utilities that are +// Most of the parsing is handled by the code in parser.go, +// however, there are additional utilities that are // helpful for dealing with Rego source inputs (e.g., REPL // statements, source files, etc.) package ast import ( + "bytes" "fmt" - "sort" "strings" "unicode" @@ -255,6 +255,10 @@ func ParsePartialObjectDocRuleFromEqExpr(module *Module, lhs, rhs *Term) (*Rule, return nil, fmt.Errorf("%v cannot be used for rule name", TypeName(lhs.Value)) } + if _, ok := ref[0].Value.(Var); !ok { + return nil, fmt.Errorf("%vs cannot be used in rule head", TypeName(ref[0].Value)) + } + name := ref[0].Value.(Var) key := ref[1] @@ -288,11 +292,16 @@ func ParsePartialSetDocRuleFromTerm(module *Module, term *Term) (*Rule, error) { return nil, fmt.Errorf("refs cannot be used for rule") } + name, ok := ref[0].Value.(Var) + if !ok { + return nil, fmt.Errorf("%vs cannot be used in rule head", TypeName(ref[0].Value)) + } + rule := &Rule{ Location: term.Location, Head: &Head{ Location: term.Location, - Name: ref[0].Value.(Var), + Name: name, Key: ref[1], }, Body: NewBody( @@ -313,11 +322,21 @@ func ParseRuleFromCallEqExpr(module *Module, lhs, rhs *Term) (*Rule, error) { return nil, fmt.Errorf("must be call") } + ref, ok := call[0].Value.(Ref) + if !ok { + return nil, fmt.Errorf("%vs cannot be used in function signature", TypeName(call[0].Value)) + } + + name, ok := ref[0].Value.(Var) + if !ok { + return nil, fmt.Errorf("%vs cannot be used in function signature", TypeName(ref[0].Value)) + } + rule := &Rule{ Location: lhs.Location, Head: &Head{ Location: lhs.Location, - Name: call[0].Value.(Ref)[0].Value.(Var), + Name: name, Args: Args(call[1:]), Value: rhs, }, @@ -497,12 +516,6 @@ func ParseStatement(input string) (Statement, error) { return stmts[0], nil } -// CommentsOption returns a parser option to initialize the comments store within -// the parser. -func CommentsOption() Option { - return GlobalStore(commentsKey, map[commentKey]*Comment{}) -} - type commentKey struct { File string Row int @@ -530,73 +543,13 @@ func (a commentKey) Compare(other commentKey) int { // This is the default return value from the parser. func ParseStatements(filename, input string) ([]Statement, []*Comment, error) { - bs := []byte(input) + stmts, comment, errs := NewParser().WithFilename(filename).WithReader(bytes.NewBufferString(input)).Parse() - parsed, err := Parse(filename, bs, GlobalStore(filenameKey, filename), CommentsOption()) - if err != nil { - return nil, nil, formatParserErrors(filename, bs, err) + if len(errs) > 0 { + return nil, nil, errs } - var comments []*Comment - var sl []interface{} - if p, ok := parsed.(program); ok { - sl = p.buf - commentMap := p.comments.(map[commentKey]*Comment) - commentKeys := []commentKey{} - for k := range commentMap { - commentKeys = append(commentKeys, k) - } - sort.Slice(commentKeys, func(i, j int) bool { - return commentKeys[i].Compare(commentKeys[j]) < 0 - }) - for _, k := range commentKeys { - comments = append(comments, commentMap[k]) - } - } else { - sl = parsed.([]interface{}) - } - stmts := make([]Statement, 0, len(sl)) - - for _, x := range sl { - if rules, ok := x.([]*Rule); ok { - for _, rule := range rules { - stmts = append(stmts, rule) - } - } else { - // Unchecked cast should be safe. A panic indicates grammar is - // out-of-sync. - stmts = append(stmts, x.(Statement)) - } - } - - return stmts, comments, postProcess(filename, stmts) -} - -func formatParserErrors(filename string, bs []byte, err error) error { - // Errors returned by the parser are always of type errList and the errList - // always contains *parserError. - // https://godoc.org/github.com/mna/pigeon#hdr-Error_reporting. - errs := err.(errList) - r := make(Errors, len(errs)) - for i, e := range errs { - r[i] = formatParserError(filename, bs, e.(*parserError)) - } - return r -} - -func formatParserError(filename string, bs []byte, e *parserError) *Error { - loc := NewLocation(nil, filename, e.pos.line, e.pos.col) - inner := e.Inner.Error() - idx := strings.Index(inner, "no match found") - if idx >= 0 { - // Match errors end with "no match found, expected: ...". We do not want to - // include ", expected: ..." as it does not provide any value, so truncate the - // string here. - inner = inner[:idx+14] - } - err := NewError(ParseErr, loc, inner) - err.Details = newParserErrorDetail(bs, e.pos) - return err + return stmts, comment, nil } func parseModule(filename string, stmts []Statement, comments []*Comment) (*Module, error) { @@ -779,9 +732,7 @@ type ParserErrorDetail struct { Idx int `json:"idx"` } -func newParserErrorDetail(bs []byte, pos position) *ParserErrorDetail { - - offset := pos.offset +func newParserErrorDetail(bs []byte, offset int) *ParserErrorDetail { // Find first non-space character at or before offset position. if offset >= len(bs) { diff --git a/ast/parser_internal.go b/ast/parser_internal.go deleted file mode 100644 index f4e32688d8..0000000000 --- a/ast/parser_internal.go +++ /dev/null @@ -1,620 +0,0 @@ -// Copyright 2018 The OPA Authors. All rights reserved. -// Use of this source code is governed by an Apache2 -// license that can be found in the LICENSE file.op - -package ast - -import ( - "bytes" - "encoding/json" - "fmt" - "math/big" -) - -const ( - // commentsKey is the global map key for the comments slice. - commentsKey = "comments" - - // filenameKey is the global map key for the filename. - filenameKey = "filename" -) - -type program struct { - buf []interface{} - comments interface{} -} - -type ruleExt struct { - loc *Location - term *Term - body Body -} - -// currentLocation converts the parser context to a Location object. -func currentLocation(c *current) *Location { - return NewLocation(c.text, c.globalStore[filenameKey].(string), c.pos.line, c.pos.col) -} - -func makeProgram(c *current, vals interface{}) (interface{}, error) { - var buf []interface{} - if vals == nil { - return buf, nil - } - ifaceSlice := vals.([]interface{}) - head := ifaceSlice[0] - buf = append(buf, head) - for _, tail := range ifaceSlice[1].([]interface{}) { - stmt := tail.([]interface{})[1] - buf = append(buf, stmt) - } - return program{buf, c.globalStore[commentsKey]}, nil -} - -func makePackage(loc *Location, value interface{}) (interface{}, error) { - // All packages are implicitly declared under the default root document. - term := value.(*Term) - path := Ref{DefaultRootDocument.Copy().SetLocation(term.Location)} - switch v := term.Value.(type) { - case Ref: - // Convert head of package Ref to String because it will be prefixed - // with the root document variable. - head := StringTerm(string(v[0].Value.(Var))).SetLocation(v[0].Location) - tail := v[1:] - if !tail.IsGround() { - return nil, fmt.Errorf("package name cannot contain variables: %v", v) - } - - // We do not allow non-string values in package names. - // Because documents are typically represented as JSON, non-string keys are - // not allowed for now. - // TODO(tsandall): consider special syntax for namespacing under arrays. - for _, p := range tail { - _, ok := p.Value.(String) - if !ok { - return nil, fmt.Errorf("package name cannot contain non-string values: %v", v) - } - } - path = append(path, head) - path = append(path, tail...) - case Var: - s := StringTerm(string(v)).SetLocation(term.Location) - path = append(path, s) - } - pkg := &Package{Location: loc, Path: path} - return pkg, nil -} - -func makeImport(loc *Location, path, alias interface{}) (interface{}, error) { - imp := &Import{} - imp.Location = loc - imp.Path = path.(*Term) - if err := IsValidImportPath(imp.Path.Value); err != nil { - return nil, err - } - if alias == nil { - return imp, nil - } - aliasSlice := alias.([]interface{}) - // Import definition above describes the "alias" slice. We only care about the "Var" element. - imp.Alias = aliasSlice[3].(*Term).Value.(Var) - return imp, nil -} - -func makeDefaultRule(loc *Location, name, operator, value interface{}) (interface{}, error) { - - if string(operator.([]uint8)) == Assign.Infix { - return nil, fmt.Errorf("default rules must use = operator (not := operator)") - } - - term := value.(*Term) - var err error - - vis := NewGenericVisitor(func(x interface{}) bool { - if err != nil { - return true - } - switch x.(type) { - case *ArrayComprehension, *ObjectComprehension, *SetComprehension: // skip closures - return true - case Ref, Var: - err = fmt.Errorf("default rule value cannot contain %v", TypeName(x)) - return true - } - return false - }) - - vis.Walk(term) - - if err != nil { - return nil, err - } - - body := NewBody(NewExpr(BooleanTerm(true).SetLocation(loc))) - - rule := &Rule{ - Location: loc, - Default: true, - Head: &Head{ - Location: loc, - Name: name.(*Term).Value.(Var), - Value: value.(*Term), - }, - Body: body, - } - rule.Body[0].Location = loc - - return []*Rule{rule}, nil -} - -func makeRule(loc *Location, head, rest interface{}) (interface{}, error) { - - if head == nil { - return nil, nil - } - - sl := rest.([]interface{}) - - rules := []*Rule{ - { - Location: loc, - Head: head.(*Head), - Body: sl[0].(Body), - }, - } - - var ordered bool - prev := rules[0] - - for i, elem := range sl[1].([]interface{}) { - - next := elem.([]interface{}) - re := next[1].(ruleExt) - - if rules[0].Head.Assign { - return nil, errElseAssignOperator - } - - if re.term == nil { - if ordered { - return nil, fmt.Errorf("expected 'else' keyword") - } - rules = append(rules, &Rule{ - Location: re.loc, - Head: prev.Head.Copy(), - Body: re.body, - }) - } else { - if (rules[0].Head.DocKind() != CompleteDoc) || (i != 0 && !ordered) { - return nil, fmt.Errorf("unexpected 'else' keyword") - } - ordered = true - curr := &Rule{ - Location: re.loc, - Head: &Head{ - Name: prev.Head.Name, - Args: prev.Head.Args.Copy(), - Value: re.term, - Location: re.term.Location, - }, - Body: re.body, - } - prev.Else = curr - prev = curr - } - } - - return rules, nil -} - -func makeRuleHead(loc *Location, name, args, key, value interface{}) (interface{}, error) { - - head := &Head{} - - head.Location = loc - head.Name = name.(*Term).Value.(Var) - - if args != nil && key != nil { - return nil, fmt.Errorf("partial rules cannot take arguments") - } - - if args != nil { - argSlice := args.([]interface{}) - head.Args = argSlice[3].(Args) - } - - if key != nil { - keySlice := key.([]interface{}) - // Head definition above describes the "key" slice. We care about the "Term" element. - head.Key = keySlice[3].(*Term) - } - - if value != nil { - valueSlice := value.([]interface{}) - operator := string(valueSlice[1].([]uint8)) - - if operator == Assign.Infix { - if head.Key != nil { - return nil, errPartialRuleAssignOperator - } else if len(head.Args) > 0 { - return nil, errFunctionAssignOperator - } - head.Assign = true - } - - // Head definition above describes the "value" slice. We care about the "Term" element. - head.Value = valueSlice[len(valueSlice)-1].(*Term) - } - - if key == nil && value == nil { - head.Value = BooleanTerm(true).SetLocation(head.Location) - } - - if key != nil && value != nil { - switch head.Key.Value.(type) { - case Var, String, Ref: // nop - default: - return nil, fmt.Errorf("object key must be string, var, or ref, not %v", TypeName(head.Key.Value)) - } - } - - return head, nil -} - -func makeArgs(list interface{}) (interface{}, error) { - termSlice := list.([]*Term) - args := make(Args, len(termSlice)) - for i := 0; i < len(args); i++ { - args[i] = termSlice[i] - } - return args, nil -} - -func makeRuleExt(loc *Location, val, b interface{}) (interface{}, error) { - bs := b.([]interface{}) - body := bs[1].(Body) - - if val == nil { - term := BooleanTerm(true) - term.Location = loc - return ruleExt{term.Location, term, body}, nil - } - - vs := val.([]interface{}) - t := vs[3].(*Term) - return ruleExt{loc, t, body}, nil -} - -func makeLiteral(negated, value, with interface{}) (interface{}, error) { - - expr := value.(*Expr) - - expr.Negated = negated.(bool) - - if with != nil { - expr.With = with.([]*With) - } - - return expr, nil -} - -func makeLiteralExpr(loc *Location, lhs, rest interface{}) (interface{}, error) { - - if rest == nil { - if call, ok := lhs.(*Term).Value.(Call); ok { - return NewExpr([]*Term(call)).SetLocation(loc), nil - } - return NewExpr(lhs).SetLocation(loc), nil - } - - termSlice := rest.([]interface{}) - terms := []*Term{ - termSlice[1].(*Term), - lhs.(*Term), - termSlice[3].(*Term), - } - - expr := NewExpr(terms).SetLocation(loc) - - return expr, nil -} - -func makeSomeDeclLiteral(loc *Location, sl interface{}) (interface{}, error) { - symbols := sl.([]*Term) - return NewExpr(&SomeDecl{Location: loc, Symbols: symbols}).SetLocation(loc), nil -} - -func makeSomeDeclSymbols(head interface{}, rest interface{}) (interface{}, error) { - - var symbols []*Term - - symbols = append(symbols, head.(*Term)) - - if sl1, ok := rest.([]interface{}); ok { - for i := range sl1 { - if sl2, ok := sl1[i].([]interface{}); ok { - symbols = append(symbols, sl2[3].(*Term)) - } - } - } - - return symbols, nil -} - -func makeWithKeywordList(head, tail interface{}) (interface{}, error) { - var withs []*With - - if head == nil { - return withs, nil - } - - sl := tail.([]interface{}) - - withs = make([]*With, 0, len(sl)+1) - withs = append(withs, head.(*With)) - - for i := range sl { - withSlice := sl[i].([]interface{}) - withs = append(withs, withSlice[1].(*With)) - } - - return withs, nil -} - -func makeWithKeyword(loc *Location, target, value interface{}) (interface{}, error) { - w := &With{ - Target: target.(*Term), - Value: value.(*Term), - } - return w.SetLocation(loc), nil -} - -func makeExprTerm(loc *Location, lhs, rest interface{}) (interface{}, error) { - - if rest == nil { - return lhs, nil - } - - sl := rest.([]interface{}) - - if len(sl) == 0 { - return lhs, nil - } - - for i := range sl { - termSlice := sl[i].([]interface{}) - call := Call{ - termSlice[1].(*Term), - lhs.(*Term), - termSlice[3].(*Term), - } - lhs = NewTerm(call).SetLocation(loc) - } - - return lhs, nil -} - -func makeCall(loc *Location, operator, args interface{}) (interface{}, error) { - - termSlice := args.([]*Term) - termOperator := operator.(*Term) - - call := make(Call, len(termSlice)+1) - - if _, ok := termOperator.Value.(Var); ok { - termOperator = RefTerm(termOperator).SetLocation(loc) - } - - call[0] = termOperator - - for i := 1; i < len(call); i++ { - call[i] = termSlice[i-1] - } - - return NewTerm(call).SetLocation(loc), nil -} - -func makeBraceEnclosedBody(loc *Location, body interface{}) (interface{}, error) { - if body != nil { - return body, nil - } - return NewBody(NewExpr(ObjectTerm().SetLocation(loc)).SetLocation(loc)), nil -} - -func makeBody(head, tail interface{}, pos int) (interface{}, error) { - - sl := tail.([]interface{}) - body := make(Body, len(sl)+1) - body[0] = head.(*Expr) - - for i := 1; i < len(body); i++ { - body[i] = sl[i-1].([]interface{})[pos].(*Expr) - } - - return body, nil -} - -func makeExprTermList(head, tail interface{}) (interface{}, error) { - - var terms []*Term - - if head == nil { - return terms, nil - } - - sl := tail.([]interface{}) - - terms = make([]*Term, 0, len(sl)+1) - terms = append(terms, head.(*Term)) - - for i := range sl { - termSlice := sl[i].([]interface{}) - terms = append(terms, termSlice[3].(*Term)) - } - - return terms, nil -} - -func makeExprTermPairList(head, tail interface{}) (interface{}, error) { - - var terms [][2]*Term - - if head == nil { - return terms, nil - } - - sl := tail.([]interface{}) - - terms = make([][2]*Term, 0, len(sl)+1) - terms = append(terms, head.([2]*Term)) - - for i := range sl { - termSlice := sl[i].([]interface{}) - terms = append(terms, termSlice[3].([2]*Term)) - } - - return terms, nil -} - -func makeExprTermPair(key, value interface{}) (interface{}, error) { - return [2]*Term{key.(*Term), value.(*Term)}, nil -} - -func makeInfixOperator(loc *Location, text []byte) (interface{}, error) { - op := string(text) - for _, b := range Builtins { - if string(b.Infix) == op { - op = string(b.Name) - } - } - operator := RefTerm(VarTerm(op).SetLocation(loc)).SetLocation(loc) - return operator, nil -} - -func makeArray(loc *Location, list interface{}) (interface{}, error) { - termSlice := list.([]*Term) - return ArrayTerm(termSlice...).SetLocation(loc), nil -} - -func makeObject(loc *Location, list interface{}) (interface{}, error) { - termPairSlice := list.([][2]*Term) - return ObjectTerm(termPairSlice...).SetLocation(loc), nil -} - -func makeSet(loc *Location, list interface{}) (interface{}, error) { - termSlice := list.([]*Term) - return SetTerm(termSlice...).SetLocation(loc), nil -} - -func makeArrayComprehension(loc *Location, head, body interface{}) (interface{}, error) { - return ArrayComprehensionTerm(head.(*Term), body.(Body)).SetLocation(loc), nil -} - -func makeSetComprehension(loc *Location, head, body interface{}) (interface{}, error) { - return SetComprehensionTerm(head.(*Term), body.(Body)).SetLocation(loc), nil -} - -func makeObjectComprehension(loc *Location, head, body interface{}) (interface{}, error) { - pair := head.([2]*Term) - return ObjectComprehensionTerm(pair[0], pair[1], body.(Body)).SetLocation(loc), nil -} - -func makeRef(loc *Location, head, rest interface{}) (interface{}, error) { - - headTerm := head.(*Term) - ifaceSlice := rest.([]interface{}) - - if len(ifaceSlice) == 0 { - return headTerm, nil - } - - ref := make(Ref, len(ifaceSlice)+1) - ref[0] = headTerm - - for i := 1; i < len(ref); i++ { - ref[i] = ifaceSlice[i-1].(*Term) - } - - return NewTerm(ref).SetLocation(loc), nil -} - -func makeRefOperandDot(loc *Location, val interface{}) (interface{}, error) { - return StringTerm(string(val.(*Term).Value.(Var))).SetLocation(loc), nil -} - -func makeVar(loc *Location, text interface{}) (interface{}, error) { - str := string(text.([]byte)) - return VarTerm(str).SetLocation(loc), nil -} - -func makeNumber(loc *Location, text interface{}) (interface{}, error) { - f, ok := new(big.Float).SetString(string(text.([]byte))) - if !ok { - // This indicates the grammar is out-of-sync with what the string - // representation of floating point numbers. This should not be - // possible. - panic("illegal value") - } - - // Put limit on size of exponent to prevent non-linear cost of String() - // function on big.Float from causing denial of service: https://github.com/golang/go/issues/11068 - // - // n == sign * mantissa * 2^exp - // 0.5 <= mantissa < 1.0 - // - // The limit is arbitrary. - exp := f.MantExp(nil) - if exp > 1e5 || exp < -1e5 { - return nil, fmt.Errorf("number too big") - } - - return NumberTerm(json.Number(f.String())).SetLocation(loc), nil -} - -func makeString(loc *Location, text interface{}) (interface{}, error) { - var v string - err := json.Unmarshal(text.([]byte), &v) - return StringTerm(v).SetLocation(loc), err -} - -func makeRawString(loc *Location, text interface{}) (interface{}, error) { - s := string(text.([]byte)) - s = s[1 : len(s)-1] // Trim surrounding quotes. - return StringTerm(s).SetLocation(loc), nil -} - -func makeNonterminatedString(loc *Location, s string) (interface{}, error) { - return StringTerm(s).SetLocation(loc), fmt.Errorf("found non-terminated string literal") -} - -func makeBool(loc *Location, text interface{}) (interface{}, error) { - var term *Term - if string(text.([]byte)) == "true" { - term = BooleanTerm(true) - } else { - term = BooleanTerm(false) - } - return term.SetLocation(loc), nil -} - -func makeNull(loc *Location) (interface{}, error) { - return NullTerm().SetLocation(loc), nil -} - -func makeComments(c *current, text interface{}) (interface{}, error) { - - var buf bytes.Buffer - for _, x := range text.([]interface{}) { - buf.Write(x.([]byte)) - } - - comment := NewComment(buf.Bytes()) - comment.Location = currentLocation(c) - comments := c.globalStore[commentsKey].(map[commentKey]*Comment) - key := commentKey{ - File: comment.Location.File, - Row: comment.Location.Row, - Col: comment.Location.Col, - } - comments[key] = comment - return comment, nil -} diff --git a/ast/parser_test.go b/ast/parser_test.go index 2e55854e50..575702f2a1 100644 --- a/ast/parser_test.go +++ b/ast/parser_test.go @@ -133,6 +133,11 @@ func TestScalarTerms(t *testing.T) { assertParseError(t, "non-number5", "6false") assertParseError(t, "non-number6", "6[null, null]") assertParseError(t, "non-number7", "6{\"foo\": \"bar\"}") + assertParseError(t, "non-number8", ".0.") + assertParseError(t, "non-number9", "{0e}") + assertParseError(t, "non-number9", "{0e.}") + assertParseError(t, "non-number10", "{0F}") + assertParseError(t, "number too big", "{7e3000000000}") } func TestVarTerms(t *testing.T) { @@ -145,9 +150,11 @@ func TestVarTerms(t *testing.T) { assertParseOneTerm(t, `true prefix`, "trueish", VarTerm("trueish")) assertParseOneTerm(t, `false prefix`, "false_flag", VarTerm("false_flag")) assertParseOneTerm(t, `null prefix`, "nullable", VarTerm("nullable")) + assertParseError(t, "illegal token", `墳`) assertParseError(t, "not keyword", "not") assertParseError(t, `package keyword`, "package") assertParseError(t, "import keyword", "import") + assertParseError(t, "import invalid path", "import x.") } func TestRefTerms(t *testing.T) { @@ -169,6 +176,13 @@ func TestRefTerms(t *testing.T) { assertParseError(t, "missing component 1", "foo.") assertParseError(t, "missing component 2", "foo[].bar") + assertParseError(t, "invalid composite operand", "foo[1,2]") + assertParseError(t, "invalid call", "bar(..") + assertParseError(t, "invalid ref", "bar[..") + assertParseError(t, "invalid ref head type number", "0[0]") + assertParseError(t, "invalid ref head type boolean", "true[0]") + assertParseError(t, "invalid ref head type string", `"foo"[0]`) + assertParseError(t, "invalid ref head type null", `null[0]`) } func TestObjectWithScalars(t *testing.T) { @@ -179,12 +193,22 @@ func TestObjectWithScalars(t *testing.T) { assertParseOneTerm(t, "number key", "{8: 7, \"def\": null}", ObjectTerm(Item(IntNumberTerm(8), IntNumberTerm(7)), Item(StringTerm("def"), NullTerm()))) assertParseOneTerm(t, "number key 2", "{8.5: 7, \"def\": null}", ObjectTerm(Item(FloatNumberTerm(8.5), IntNumberTerm(7)), Item(StringTerm("def"), NullTerm()))) assertParseOneTerm(t, "bool key", "{true: false}", ObjectTerm(Item(BooleanTerm(true), BooleanTerm(false)))) + assertParseOneTerm(t, "trailing comma", `{"a": "bar", "b": 64, }`, ObjectTerm(Item(StringTerm("a"), StringTerm("bar")), Item(StringTerm("b"), IntNumberTerm(64)))) + assertParseOneTerm(t, "leading comma", `{, "a": "bar", "b": 64 }`, ObjectTerm(Item(StringTerm("a"), StringTerm("bar")), Item(StringTerm("b"), IntNumberTerm(64)))) + assertParseOneTerm(t, "leading comma not comprehension", `{, 1 | 1: "bar"}`, ObjectTerm(Item(CallTerm(RefTerm(VarTerm("or")), NumberTerm("1"), NumberTerm("1")), StringTerm("bar")))) } func TestObjectWithVars(t *testing.T) { assertParseOneTerm(t, "var keys", "{foo: \"bar\", bar: 64}", ObjectTerm(Item(VarTerm("foo"), StringTerm("bar")), Item(VarTerm("bar"), IntNumberTerm(64)))) assertParseOneTerm(t, "nested var keys", "{baz: {foo: \"bar\", bar: qux}}", ObjectTerm(Item(VarTerm("baz"), ObjectTerm(Item(VarTerm("foo"), StringTerm("bar")), Item(VarTerm("bar"), VarTerm("qux")))))) - assertParseOneTerm(t, "trailing comma", "{foo: \"bar\", bar: 64, }", ObjectTerm(Item(VarTerm("foo"), StringTerm("bar")), Item(VarTerm("bar"), IntNumberTerm(64)))) + assertParseOneTerm(t, "ambiguous or", `{ a: b+c | d }`, ObjectTerm(Item(VarTerm("a"), CallTerm(RefTerm(VarTerm("or")), CallTerm(RefTerm(VarTerm("plus")), VarTerm("b"), VarTerm("c")), VarTerm("d"))))) +} + +func TestObjectWithRelation(t *testing.T) { + assertParseOneTerm(t, "relation term value", `{"x": 1+1}`, ObjectTerm( + Item(StringTerm("x"), CallTerm(RefTerm(VarTerm("plus")), IntNumberTerm(1), IntNumberTerm(1))), + )) + assertParseError(t, "invalid relation term value", `{"x": 0= }`) } func TestObjectFail(t *testing.T) { @@ -194,6 +218,9 @@ func TestObjectFail(t *testing.T) { assertParseError(t, "non-terminated 4", "{foo: bar, baz: [], ") assertParseError(t, "missing separator", "{foo: bar baz: []}") assertParseError(t, "missing start", "foo: bar, baz: [], qux: corge}") + assertParseError(t, "double comma", "{a:1,,b:2}") + assertParseError(t, "leading double comma", "{,,a:1}") + assertParseError(t, "trailing double comma", "{a:1,,}") } func TestArrayWithScalars(t *testing.T) { @@ -201,7 +228,11 @@ func TestArrayWithScalars(t *testing.T) { assertParseOneTerm(t, "bool", "[true, false, true]", ArrayTerm(BooleanTerm(true), BooleanTerm(false), BooleanTerm(true))) assertParseOneTerm(t, "string", "[\"foo\", \"bar\"]", ArrayTerm(StringTerm("foo"), StringTerm("bar"))) assertParseOneTerm(t, "mixed", "[null, true, 42]", ArrayTerm(NullTerm(), BooleanTerm(true), IntNumberTerm(42))) + assertParseOneTerm(t, "trailing comma - one element", "[null, ]", ArrayTerm(NullTerm())) assertParseOneTerm(t, "trailing comma", "[null, true, ]", ArrayTerm(NullTerm(), BooleanTerm(true))) + assertParseOneTerm(t, "leading comma", "[, null, true]", ArrayTerm(NullTerm(), BooleanTerm(true))) + assertParseOneTerm(t, "leading comma not comprehension", "[, 1 | 1]", ArrayTerm(CallTerm(RefTerm(VarTerm("or")), NumberTerm("1"), NumberTerm("1")))) + assertParseOneTerm(t, "ambiguous or", "[ 1 + 2 | 3 ]", ArrayTerm(CallTerm(RefTerm(VarTerm("or")), CallTerm(RefTerm(VarTerm("plus")), NumberTerm("1"), NumberTerm("2")), NumberTerm("3")))) } func TestArrayWithVars(t *testing.T) { @@ -214,6 +245,10 @@ func TestArrayFail(t *testing.T) { assertParseError(t, "non-terminated 2", "[foo, bar, ") assertParseError(t, "missing separator", "[foo bar]") assertParseError(t, "missing start", "foo, bar, baz]") + assertParseError(t, "bad term", "[!!!]") + assertParseError(t, "double comma", "[a,,b]") + assertParseError(t, "leading double comma", "[,,a]") + assertParseError(t, "trailing double comma", "[a,,]") } func TestSetWithScalars(t *testing.T) { @@ -222,6 +257,9 @@ func TestSetWithScalars(t *testing.T) { assertParseOneTerm(t, "string", "{\"foo\", \"bar\"}", SetTerm(StringTerm("foo"), StringTerm("bar"))) assertParseOneTerm(t, "mixed", "{null, true, 42}", SetTerm(NullTerm(), BooleanTerm(true), IntNumberTerm(42))) assertParseOneTerm(t, "trailing comma", "{null, true,}", SetTerm(NullTerm(), BooleanTerm(true))) + assertParseOneTerm(t, "leading comma", "{, null, true}", SetTerm(NullTerm(), BooleanTerm(true))) + assertParseOneTerm(t, "leading comma not comprehension", "{, 1 | 1}", SetTerm(CallTerm(RefTerm(VarTerm("or")), NumberTerm("1"), NumberTerm("1")))) + assertParseOneTerm(t, "ambiguous or", "{ 1 + 2 | 3}", SetTerm(CallTerm(RefTerm(VarTerm("or")), CallTerm(RefTerm(VarTerm("plus")), NumberTerm("1"), NumberTerm("2")), NumberTerm("3")))) } func TestSetWithVars(t *testing.T) { @@ -235,6 +273,10 @@ func TestSetFail(t *testing.T) { assertParseError(t, "non-terminated 3", "{foo, bar, ") assertParseError(t, "missing separator", "{foo bar}") assertParseError(t, "missing start", "foo, bar, baz}") + assertParseError(t, "bad term", "{!!!}") + assertParseError(t, "double comma", "{a,,b}") + assertParseError(t, "leading double comma", "{,,a}") + assertParseError(t, "trailing double comma", "{a,,}") } func TestEmptyComposites(t *testing.T) { @@ -257,9 +299,8 @@ func TestCompositesWithRefs(t *testing.T) { func TestArrayComprehensions(t *testing.T) { - input := `[{"x": [a[i] | xs = [{"a": ["baz", j]} | q[p]; p.a != "bar"; j = "foo"]; xs[j].a[k] = "foo"]}]` - - expected := ArrayTerm( + nestedTerm := `[{"x": [a[i] | xs = [{"a": ["baz", j]} | q[p]; p.a != "bar"; j = "foo"]; xs[j].a[k] = "foo"]}]` + nestedExpected := ArrayTerm( ObjectTerm(Item( StringTerm("x"), ArrayComprehensionTerm( @@ -284,14 +325,16 @@ func TestArrayComprehensions(t *testing.T) { ), )), ) - - assertParseOneTerm(t, "nested", input, expected) + assertParseOneTerm(t, "nested", nestedTerm, nestedExpected) + assertParseOneTerm(t, "ambiguous or", "[ a | b ]", ArrayComprehensionTerm( + VarTerm("a"), + MustParseBody("b"), + )) } func TestObjectComprehensions(t *testing.T) { - input := `[{"x": {a[i]: b[i] | xs = {"foo":{"a": ["baz", j]} | q[p]; p.a != "bar"; j = "foo"}; xs[j].a[k] = "foo"}}]` - - expected := ArrayTerm( + nestedTerm := `[{"x": {a[i]: b[i] | xs = {"foo":{"a": ["baz", j]} | q[p]; p.a != "bar"; j = "foo"}; xs[j].a[k] = "foo"}}]` + nestedExpected := ArrayTerm( ObjectTerm(Item( StringTerm("x"), ObjectComprehensionTerm( @@ -318,14 +361,21 @@ func TestObjectComprehensions(t *testing.T) { ), )), ) + assertParseOneTerm(t, "nested", nestedTerm, nestedExpected) + assertParseOneTerm(t, "ambiguous or", "{ 1+2: 3 | 4}", ObjectComprehensionTerm( + CallTerm(RefTerm(VarTerm("plus")), NumberTerm("1"), NumberTerm("2")), + NumberTerm("3"), + MustParseBody("4"), + )) +} - assertParseOneTerm(t, "nested", input, expected) +func TestObjectComprehensionError(t *testing.T) { + assertParseError(t, "bad body", "{x: y|!!!}") } func TestSetComprehensions(t *testing.T) { - input := `[{"x": {a[i] | xs = {{"a": ["baz", j]} | q[p]; p.a != "bar"; j = "foo"}; xs[j].a[k] = "foo"}}]` - - expected := ArrayTerm( + nestedTerm := `[{"x": {a[i] | xs = {{"a": ["baz", j]} | q[p]; p.a != "bar"; j = "foo"}; xs[j].a[k] = "foo"}}]` + nestedExpected := ArrayTerm( ObjectTerm(Item( StringTerm("x"), SetComprehensionTerm( @@ -351,7 +401,15 @@ func TestSetComprehensions(t *testing.T) { )), ) - assertParseOneTerm(t, "nested", input, expected) + assertParseOneTerm(t, "nested", nestedTerm, nestedExpected) + assertParseOneTerm(t, "ambiguous or", "{ a | b }", SetComprehensionTerm( + VarTerm("a"), + MustParseBody("b"), + )) +} + +func TestSetComprehensionError(t *testing.T) { + assertParseError(t, "bad body", "{x|!!!}") } func TestSetComprehensionsAlone(t *testing.T) { @@ -472,6 +530,16 @@ func TestExprWith(t *testing.T) { }, }, }) + + assertParseOneExpr(t, "variable target", "true with x as 1", &Expr{ + Terms: BooleanTerm(true), + With: []*With{ + { + Target: VarTerm("x"), + Value: IntNumberTerm(1), + }, + }, + }) } func TestSomeDeclExpr(t *testing.T) { @@ -560,6 +628,20 @@ func TestNestedExpressions(t *testing.T) { n2, n6), n3))}, + {"associativity - factors", "x * y / z % w", + Rem.Expr(Divide.Call(Multiply.Call(x, y), z), w)}, + {"associativity - factors", "w % z / x * y", + Multiply.Expr(Divide.Call(Rem.Call(w, z), x), y)}, + {"associativity - arithetic", "x + y - z", + Minus.Expr(Plus.Call(x, y), z)}, + {"associativity - arithmetic", "z - x + y", + Plus.Expr(Minus.Call(z, x), y)}, + {"associativity - and", "z & x & y", + And.Expr(And.Call(z, x), y)}, + {"associativity - or", "z | x | y", + Or.Expr(Or.Call(z, x), y)}, + {"associativity - relations", "x == y != z", + NotEqual.Expr(Equal.Call(x, y), z)}, {"grouping", "(1 + 2 * 6 / 3) > 4", GreaterThan.Expr( Plus.Call( @@ -644,15 +726,34 @@ func TestNestedExpressions(t *testing.T) { } } +func TestChainedCall(t *testing.T) { + result, err := ParseExpr("foo.bar(1)[0](1).baz") + if err != nil { + t.Fatal(err) + } + + exp := NewExpr(RefTerm( + CallTerm( + RefTerm( + CallTerm( + RefTerm(VarTerm("foo"), StringTerm("bar")), + IntNumberTerm(1)), + IntNumberTerm(0)), + IntNumberTerm(1)), + StringTerm("baz"))) + + if !result.Equal(exp) { + t.Fatalf("expected %v but got: %v", exp, result) + } +} + func TestMultiLineBody(t *testing.T) { input1 := ` - { x = 1 y = 2 z = [ i | [x,y] = arr arr[_] = i] - } ` body1, err := ParseBody(input1) @@ -666,7 +767,7 @@ func TestMultiLineBody(t *testing.T) { t.Errorf("Expected enclosed body to equal %v but got: %v", expected1, body1) } - // Check that parser can handle multiple expressions w/o enclsoing braces. + // Check that parser can handle multiple expressions w/o enclosing braces. input2 := ` x = 1 ; # comment after semicolon y = 2 # comment without semicolon @@ -682,6 +783,117 @@ func TestMultiLineBody(t *testing.T) { if !body2.Equal(expected1) { t.Errorf("Expected unenclosed body to equal %v but got: %v", expected1, body1) } + + assertParseOneBody(t, "whitespace following call", "f(x)\t\n [1]", NewBody( + NewExpr( + []*Term{ + RefTerm(VarTerm("f")), + VarTerm("x"), + }, + ), + NewExpr( + ArrayTerm(IntNumberTerm(1)), + ), + )) + + assertParseOneBody(t, "whitespace following array", "[1]\t\n [2]", NewBody( + NewExpr( + ArrayTerm(IntNumberTerm(1)), + ), + NewExpr( + ArrayTerm(IntNumberTerm(2)), + ), + )) + + assertParseOneBody(t, "whitespace following set", "{1}\t\n {2}", NewBody( + NewExpr( + SetTerm(IntNumberTerm(1)), + ), + NewExpr( + SetTerm(IntNumberTerm(2)), + ), + )) +} + +func TestBitwiseOrVsComprehension(t *testing.T) { + + x := VarTerm("x") + y := VarTerm("y") + z := VarTerm("z") + a := VarTerm("a") + b := VarTerm("b") + + tests := []struct { + note string + input string + exp *Term + }{ + { + note: "array containing bitwise or", + input: "[x|y,z]", + exp: ArrayTerm(Or.Call(x, y), z), + }, + { + note: "array containing bitwise or - last element", + input: "[z,x|y]", + exp: ArrayTerm(z, Or.Call(x, y)), + }, + { + note: "array containing bitwise or - middle", + input: "[z,x|y,a]", + exp: ArrayTerm(z, Or.Call(x, y), a), + }, + { + note: "array containing single bitwise or", + input: "[x|y,]", + exp: ArrayTerm(Or.Call(x, y)), + }, + { + note: "set containing bitwise or", + input: "{x|y,z}", + exp: SetTerm(Or.Call(x, y), z), + }, + { + note: "set containing bitwise or - last element", + input: "{z,x|y}", + exp: SetTerm(z, Or.Call(x, y)), + }, + { + note: "set containing bitwise or - middle", + input: "{z,x|y,a}", + exp: SetTerm(z, Or.Call(x, y), a), + }, + { + note: "set containing single bitwise or", + input: "{x|y,}", + exp: SetTerm(Or.Call(x, y)), + }, + { + note: "object containing bitwise or", + input: "{x:y|z,a:b}", + exp: ObjectTerm([2]*Term{x, Or.Call(y, z)}, [2]*Term{a, b}), + }, + { + note: "object containing single bitwise or", + input: "{x:y|z,}", + exp: ObjectTerm([2]*Term{x, Or.Call(y, z)}), + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + + term, err := ParseTerm(tc.input) + if err != nil { + t.Fatal(err) + } + + if !term.Equal(tc.exp) { + t.Fatalf("Expected %v but got %v", tc.exp, term) + } + }) + } + } func TestPackage(t *testing.T) { @@ -693,6 +905,9 @@ func TestPackage(t *testing.T) { assertParsePackage(t, "space", `package foo["bar baz"]`, &Package{Path: ref3.Value.(Ref)}) assertParseError(t, "non-ground ref", "package foo[x]") assertParseError(t, "non-string value", "package foo.bar[42].baz") + assertParseError(t, "invalid term", "package 42") + assertParseError(t, "scanner error", "package foo.") + assertParseError(t, "non-string first value", "package e().s") } func TestImport(t *testing.T) { @@ -705,9 +920,22 @@ func TestImport(t *testing.T) { assertParseImport(t, "single alias", "import input.foo as bar", &Import{Path: foo, Alias: Var("bar")}) assertParseImport(t, "multiple alias", "import input.foo.bar.baz as qux", &Import{Path: foobarbaz, Alias: Var("qux")}) assertParseImport(t, "white space", "import input.foo.bar[\"white space\"]", &Import{Path: whitespace}) - assertParseErrorContains(t, "non-ground ref", "import data.foo[x]", "rego_parse_error: invalid path data.foo[x]: path elements must be strings") - assertParseErrorContains(t, "non-string", "import input.foo[0]", "rego_parse_error: invalid path input.foo[0]: path elements must be strings") - assertParseErrorContains(t, "unknown root", "import foo.bar", "rego_parse_error: invalid path foo.bar: path must begin with input or data") + assertParseErrorContains(t, "non-ground ref", "import data.foo[x]", "rego_parse_error: unexpected var token: expecting string") + assertParseErrorContains(t, "non-string", "import input.foo[0]", "rego_parse_error: unexpected number token: expecting string") + assertParseErrorContains(t, "unknown root", "import foo.bar", "rego_parse_error: unexpected import path, must begin with one of: {data, input}, got: foo") + + _, _, err := ParseStatements("", "package foo\nimport bar.data\ndefault foo=1") + if err == nil { + t.Fatalf("Expected error, but got nil") + } + if len(err.(Errors)) > 1 { + t.Fatalf("Expected a single error, got %s", err) + } + txt := err.(Errors)[0].Details.Lines()[0] + expected := "import bar.data" + if txt != expected { + t.Fatalf("Expected error detail text '%s' but got '%s'", expected, txt) + } } func TestIsValidImportPath(t *testing.T) { @@ -918,14 +1146,13 @@ func TestRule(t *testing.T) { Body: NewBody(NewExpr(BooleanTerm(true))), }) + // TODO: expect expressions instead? assertParseErrorContains(t, "empty body", `f(_) = y {}`, "rego_parse_error: found empty body") - assertParseErrorContains(t, "object composite key", "p[[x,y]] = z { true }", "rego_parse_error: object key must be string, var, or ref, not array") - assertParseErrorContains(t, "default ref value", "default p = [data.foo]", "rego_parse_error: default rule value cannot contain ref") - assertParseErrorContains(t, "default var value", "default p = [x]", "rego_parse_error: default rule value cannot contain var") assertParseErrorContains(t, "empty rule body", "p {}", "rego_parse_error: found empty body") + assertParseErrorContains(t, "unmatched braces", `f(x) = y { trim(x, ".", y) `, `rego_parse_error: unexpected eof token: expected \n or ; or }`) - assertParseErrorContains(t, "no output", `f(_) = { "foo" = "bar" }`, "rego_parse_error: no match found") - assertParseErrorContains(t, "unmatched braces", `f(x) = y { trim(x, ".", y) `, "rego_parse_error: no match found") + // TODO: how to highlight that assignment is incorrect here? + assertParseErrorContains(t, "no output", `f(_) = { "foo" = "bar" }`, "rego_parse_error: unexpected eq token: expected rule value term") // TODO(tsandall): improve error checking here. This is a common mistake // and the current error message is not very good. Need to investigate if the @@ -936,6 +1163,53 @@ func TestRule(t *testing.T) { assertParseErrorContains(t, "partial assignment", `p[x] := y { true }`, "partial rules must use = operator (not := operator)") assertParseErrorContains(t, "function assignment", `f(x) := y { true }`, "functions must use = operator (not := operator)") assertParseErrorContains(t, "else assignment", `p := y { true } else = 2 { true } `, "else keyword cannot be used on rule declared with := operator") + + assertParseErrorContains(t, "default invalid rule name", `default 0[0`, "unexpected default keyword") + assertParseErrorContains(t, "default invalid rule value", `default a[0`, "illegal default rule (must have a value)") + assertParseRule(t, "default missing value", `default a`, &Rule{ + Default: true, + Head: &Head{ + Name: Var("a"), + Value: BooleanTerm(true), + }, + Body: NewBody(NewExpr(BooleanTerm(true))), + }) + assertParseRule(t, "empty arguments", `f() { x := 1 }`, &Rule{ + Head: &Head{ + Name: "f", + Value: BooleanTerm(true), + }, + Body: MustParseBody(`x := 1`), + }) + + assertParseErrorContains(t, "default invalid rule head ref", `default a = b.c.d`, "illegal default rule (value cannot contain ref)") + assertParseErrorContains(t, "default invalid rule head call", `default a = g(x)`, "illegal default rule (value cannot contain call)") + assertParseErrorContains(t, "default invalid rule head builtin call", `default a = upper("foo")`, "illegal default rule (value cannot contain call)") + assertParseErrorContains(t, "default invalid rule head call", `default a = b`, "illegal default rule (value cannot contain var)") + + assertParseError(t, "extra braces", `{ a := 1 }`) + assertParseError(t, "invalid rule name dots", `a.b = x { x := 1 }`) + assertParseError(t, "invalid rule name dots and call", `a.b(x) { x := 1 }`) + assertParseError(t, "invalid rule name hyphen", `a-b = x { x := 1 }`) + + assertParseRule(t, "wildcard name", `_ { x == 1 }`, &Rule{ + Head: &Head{ + Name: "$0", + Value: BooleanTerm(true), + }, + Body: MustParseBody(`x == 1`), + }) + + assertParseRule(t, "partial object array key", `p[[a, 1, 2]] = x { a := 1; x := "foo" }`, &Rule{ + Head: &Head{ + Name: "p", + Key: ArrayTerm(VarTerm("a"), NumberTerm("1"), NumberTerm("2")), + Value: VarTerm("x"), + }, + Body: MustParseBody(`a := 1; x := "foo"`), + }) + assertParseError(t, "invalid rule body no separator", `p { a = "foo"bar }`) + assertParseError(t, "invalid rule body no newline", `p { a b c }`) } func TestRuleElseKeyword(t *testing.T) { @@ -966,6 +1240,14 @@ func TestRuleElseKeyword(t *testing.T) { } else { x != 150 } + + _ { + x > 0 + } else { + x == -1 + } else { + x > -100 + } ` parsed, err := ParseModule("", mod) @@ -1034,6 +1316,28 @@ func TestRuleElseKeyword(t *testing.T) { }, }, }, + + { + Head: &Head{ + Name: Var("$0"), + Value: BooleanTerm(true), + }, + Body: MustParseBody(`x > 0`), + Else: &Rule{ + Head: &Head{ + Name: Var("$0"), + Value: BooleanTerm(true), + }, + Body: MustParseBody(`x == -1`), + Else: &Rule{ + Head: &Head{ + Name: Var("$0"), + Value: BooleanTerm(true), + }, + Body: MustParseBody(`x > -100`), + }, + }, + }, }, } @@ -1086,7 +1390,7 @@ func TestRuleElseKeyword(t *testing.T) { p[1] { false } else { true } `) - if err == nil || !strings.Contains(err.Error(), "unexpected 'else' keyword") { + if err == nil || !strings.Contains(err.Error(), "else keyword cannot be used on partial rules") { t.Fatalf("Expected parse error but got: %v", err) } @@ -1095,7 +1399,7 @@ func TestRuleElseKeyword(t *testing.T) { p { false } { false } else { true } `) - if err == nil || !strings.Contains(err.Error(), "unexpected 'else' keyword") { + if err == nil || !strings.Contains(err.Error(), "unexpected else keyword") { t.Fatalf("Expected parse error but got: %v", err) } @@ -1104,7 +1408,7 @@ func TestRuleElseKeyword(t *testing.T) { p { false } else { false } { true } `) - if err == nil || !strings.Contains(err.Error(), "expected 'else' keyword") { + if err == nil || !strings.Contains(err.Error(), "expected else keyword") { t.Fatalf("Expected parse error but got: %v", err) } @@ -1254,10 +1558,72 @@ func TestComments(t *testing.T) { } if !expc.Equal(module.Comments[i]) { - t.Errorf("Expected %v for %vith comment but got: %v", expc, i, module.Comments[i]) + comment := module.Comments[i] + fmt.Printf("comment: %v %v %v %v\n", comment.Location.File, comment.Location.Text, comment.Location.Col, comment.Location.Row) + fmt.Printf("expcomm: %v %v %v %v\n", expc.Location.File, expc.Location.Text, expc.Location.Col, expc.Location.Row) + t.Errorf("Expected %q but got: %q (want: %d:%d, got: %d:%d)", expc, comment, exp[i].row, exp[i].col, comment.Location.Row, comment.Location.Col) } } +} + +func TestCommentsWhitespace(t *testing.T) { + cases := []struct { + note string + module string + expected []string + }{ + { + note: "trailing spaces", + module: "# a comment \t \n", + expected: []string{" a comment \t "}, + }, + { + note: "trailing carriage return", + module: "# a comment\r\n", + expected: []string{" a comment"}, + }, + { + note: "trailing carriage return double newline", + module: "# a comment\r\n\n", + expected: []string{" a comment"}, + }, + { + note: "double trailing carriage return newline", + module: "#\r\r\n", + expected: []string{"\r"}, + }, + { + note: "double trailing carriage return", + module: "#\r\r", + expected: []string{"\r"}, + }, + { + note: "carriage return", + module: "#\r", + expected: []string{""}, + }, + { + note: "carriage return in comment", + module: "# abc\rdef\r\n", + expected: []string{" abc\rdef"}, + }, + } + + for _, tc := range cases { + t.Run(tc.note, func(t *testing.T) { + _, comments, err := ParseStatements("", tc.module) + if err != nil { + t.Fatalf("Unexpected parse error: %s", err) + } + for i, exp := range tc.expected { + actual := string(comments[i].Text) + if exp != actual { + t.Errorf("Expected comment text (len %d):\n\n\t%q\n\nbut got (len %d):\n\n\t%q\n\n", len(exp), exp, len(actual), actual) + } + } + }) + } } func TestExample(t *testing.T) { @@ -1331,6 +1697,7 @@ foo["9"] = "10" foo.buz = "bar" bar[1] bar[[{"foo":"baz"}]] +bar.qux input = 1 data = 2 f(1) = 2 @@ -1352,6 +1719,7 @@ d1 := 1234 MustParseRule(`foo["buz"] = "bar" { true }`), MustParseRule(`bar[1] { true }`), MustParseRule(`bar[[{"foo":"baz"}]] { true }`), + MustParseRule(`bar["qux"] { true }`), MustParseRule(`input = 1 { true }`), MustParseRule(`data = 2 { true }`), MustParseRule(`f(1) = 2 { true }`), @@ -1432,6 +1800,11 @@ data = {"bar": 2} { true }` some x` + arrayTerm := ` + package a + [][0] + ` + assertParseModuleError(t, "multiple expressions", multipleExprs) assertParseModuleError(t, "non-equality", nonEquality) assertParseModuleError(t, "non-var name", nonVarName) @@ -1443,6 +1816,10 @@ data = {"bar": 2} { true }` assertParseModuleError(t, "zero args", zeroArgs) assertParseModuleError(t, "assign to term", assignToTerm) assertParseModuleError(t, "some decl", someDecl) + assertParseModuleError(t, "array term", arrayTerm) + assertParseModuleError(t, "call in ref partial set", "package test\nf().x {}") + assertParseModuleError(t, "call in ref partial object", "package test\nf().x = y {}") + assertParseModuleError(t, "number in ref", "package a\n12[3]()=4") if _, err := ParseRuleFromExpr(&Module{}, &Expr{ Terms: struct{}{}, @@ -1522,6 +1899,7 @@ func TestRuleModulePtr(t *testing.T) { } } } + func TestNoMatchError(t *testing.T) { mod := `package test @@ -1531,7 +1909,7 @@ func TestNoMatchError(t *testing.T) { _, err := ParseModule("foo.rego", mod) - expected := "1 error occurred: foo.rego:5: rego_parse_error: no match found" + expected := "1 error occurred: foo.rego:5: rego_parse_error: unexpected } token" if !strings.HasPrefix(err.Error(), expected) { t.Fatalf("Bad parse error, expected %v but got: %v", expected, err) @@ -1543,7 +1921,7 @@ func TestNoMatchError(t *testing.T) { _, err = ParseModule("foo.rego", mod) - loc := NewLocation(nil, "foo.rego", 3, 12) + loc := NewLocation([]byte{'/'}, "foo.rego", 3, 12) if !loc.Equal(err.(Errors)[0].Location) { t.Fatalf("Expected %v but got: %v", loc, err) @@ -1599,7 +1977,7 @@ p { true; }`}, note: "empty body", exp: &ParserErrorDetail{ Line: "p { }", - Idx: 2, + Idx: 4, }, input: ` package test @@ -1617,42 +1995,44 @@ p = "foo`}, note: "rule with error begins with one tab", exp: &ParserErrorDetail{ Line: "\tas", - Idx: 2, + Idx: 1, }, input: ` package test as`, - err: `1 error occurred: test.rego:3: rego_parse_error: no match found + err: `1 error occurred: test.rego:3: rego_parse_error: unexpected as keyword as - ^`}, + ^`}, { note: "rule term with error begins with two tabs", exp: &ParserErrorDetail{ Line: "\t\tas", - Idx: 3, + Idx: 2, }, input: ` package test p = true { as }`, - err: `1 error occurred: test.rego:5: rego_parse_error: no match found + err: `1 error occurred: test.rego:4: rego_parse_error: unexpected as keyword as - ^`}, + ^`}, } for _, tc := range tests { - _, err := ParseModule("test.rego", tc.input) - if err == nil { - t.Fatal("Expected error") - } - detail := err.(Errors)[0].Details - if !reflect.DeepEqual(detail, tc.exp) { - t.Fatalf("Expected %v but got: %v", tc.exp, detail) - } - if tc.err != "" && tc.err != err.Error() { - t.Fatalf("Expected error string %q but got: %q", tc.err, err.Error()) - } + t.Run(tc.note, func(t *testing.T) { + _, err := ParseModule("test.rego", tc.input) + if err == nil { + t.Fatal("Expected error") + } + detail := err.(Errors)[0].Details + if !reflect.DeepEqual(detail, tc.exp) { + t.Fatalf("Expected %v but got: %v", tc.exp, detail) + } + if tc.err != "" && tc.err != err.Error() { + t.Fatalf("Expected error string %q but got: %q", tc.err, err.Error()) + } + }) } } @@ -1687,30 +2067,160 @@ func TestNamespacedBuiltins(t *testing.T) { } } +func TestRuleHeadLocationSetRecursively(t *testing.T) { + + const input = `package pkg + +p[x] { + x = "hi" +} { + x = "bye" +} + +f(x) { + false +} else = false { + true +} +` + + module := MustParseModule(input) + + for _, row := range [][2]int{ + {module.Rules[0].Location.Row, 3}, + {module.Rules[1].Location.Row, 5}, + {module.Rules[1].Head.Location.Row, 5}, + {module.Rules[1].Head.Key.Location.Row, 5}, + {module.Rules[2].Head.Location.Row, 9}, + {module.Rules[2].Location.Row, 9}, + {module.Rules[2].Head.Location.Row, 9}, + {module.Rules[2].Head.Args[0].Location.Row, 9}, + {module.Rules[2].Else.Location.Row, 11}, + {module.Rules[2].Else.Head.Location.Row, 11}, + {module.Rules[2].Else.Head.Args[0].Location.Row, 11}, + } { + if row[0] != row[1] { + t.Fatalf("Expected %d but got %d", row[1], row[0]) + } + } +} + +func TestParserText(t *testing.T) { + + tests := []struct { + note string + input string + want string + }{ + { + note: "relational term", + input: `(1 == (2 > 3))`, + }, + { + note: "array - empty", + input: `[ ]`, + }, + { + note: "array - one element", + input: `[ 1 ]`, + }, + { + note: "array - multiple elements", + input: `[1 , 2 , 3]`, + }, + { + note: "object - empty", + input: `{ }`, + }, + { + note: "object - one element", + input: `{ "foo": 1 }`, + }, + { + note: "object - multiple elements", + input: `{"foo": 1, "bar": 2}`, + }, + { + note: "set - one element", + input: `{ 1 }`, + }, + { + note: "set - multiple elements", + input: `{1 , 2 , 3}`, + }, + { + note: "idents", + input: "foo", + }, + { + note: "ref", + input: `data.foo[x].bar`, + }, + { + note: "call", + input: `data.foo.bar(x)`, + }, + { + note: "ref and call", + input: `data.foo[1](x).bar(y)[z]`, + }, + { + note: "infix", + input: "input = 1", + }, + { + note: "negated", + input: "not x = 1", + }, + { + note: "expr with statements", + input: "x = 1 with input as 2 with input as 3", + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + for _, suffix := range []string{"", "\t\n "} { + input := tc.input + suffix + + stmts, _, err := ParseStatements("test.rego", input) + if err != nil { + t.Fatal(err) + } + + if len(stmts) != 1 { + t.Fatal("expected exactly one statement but got:", stmts) + } + + result := string(stmts[0].Loc().Text) + + if result != tc.input { + t.Fatalf("expected %q but got: %q", tc.input, result) + } + } + }) + } +} + func assertParseError(t *testing.T, msg string, input string) { assertParseErrorFunc(t, msg, input, func(string) {}) } func assertParseErrorContains(t *testing.T, msg string, input string, expected string) { + t.Helper() assertParseErrorFunc(t, msg, input, func(result string) { + t.Helper() if !strings.Contains(result, expected) { - t.Errorf("Error on test %s: expected parse error to contain %v but got: %v", msg, expected, result) - } - }) -} - -func assertParseErrorEquals(t *testing.T, msg string, input string, expected string) { - assertParseErrorFunc(t, msg, input, func(result string) { - if result != expected { - t.Errorf("Error on test %s: expected parse error to equal %v but got: %v", msg, expected, result) + t.Errorf("Error on test \"%s\": expected parse error to contain:\n\n%v\n\nbut got:\n\n%v", msg, expected, result) } }) } func assertParseErrorFunc(t *testing.T, msg string, input string, f func(string)) { + t.Helper() p, err := ParseStatement(input) if err == nil { - t.Errorf("Error on test %s: expected parse error: %v (parsed)", msg, p) + t.Errorf("Error on test \"%s\": expected parse error but parsed successfully:\n\n%v\n\n(parsed)", msg, p) return } result := err.Error() @@ -1721,10 +2231,12 @@ func assertParseErrorFunc(t *testing.T, msg string, input string, f func(string) } func assertParseImport(t *testing.T, msg string, input string, correct *Import) { + t.Helper() assertParseOne(t, msg, input, func(parsed interface{}) { + t.Helper() imp := parsed.(*Import) if !imp.Equal(correct) { - t.Errorf("Error on test %s: imports not equal: %v (parsed), %v (correct)", msg, imp, correct) + t.Errorf("Error on test \"%s\": imports not equal: %v (parsed), %v (correct)", msg, imp, correct) } }) } @@ -1733,7 +2245,7 @@ func assertParseModule(t *testing.T, msg string, input string, correct *Module) m, err := ParseModule("", input) if err != nil { - t.Errorf("Error on test %s: parse error on %s: %s", msg, input, err) + t.Errorf("Error on test \"%s\": parse error on %s: %s", msg, input, err) return } @@ -1746,7 +2258,7 @@ func assertParseModule(t *testing.T, msg string, input string, correct *Module) func assertParseModuleError(t *testing.T, msg, input string) { m, err := ParseModule("", input) if err == nil { - t.Errorf("Error on test %v: expected parse error: %v (parsed)", msg, m) + t.Errorf("Error on test \"%s\": expected parse error: %v (parsed)", msg, m) } } @@ -1754,7 +2266,7 @@ func assertParsePackage(t *testing.T, msg string, input string, correct *Package assertParseOne(t, msg, input, func(parsed interface{}) { pkg := parsed.(*Package) if !pkg.Equal(correct) { - t.Errorf("Error on test %s: packages not equal: %v (parsed), %v (correct)", msg, pkg, correct) + t.Errorf("Error on test \"%s\": packages not equal: %v (parsed), %v (correct)", msg, pkg, correct) } }) } @@ -1763,22 +2275,35 @@ func assertParseOne(t *testing.T, msg string, input string, correct func(interfa t.Helper() p, err := ParseStatement(input) if err != nil { - t.Errorf("Error on test %s: parse error on %s: %s", msg, input, err) + t.Errorf("Error on test \"%s\": parse error on %s: %s", msg, input, err) return } correct(p) } +func assertParseOneBody(t *testing.T, msg string, input string, correct Body) { + t.Helper() + body, err := ParseBody(input) + if err != nil { + t.Fatal(err) + } + if !body.Equal(correct) { + t.Fatalf("Error on test \"%s\": bodies not equal:\n%v (parsed)\n%v (correct)", msg, body, correct) + } +} + func assertParseOneExpr(t *testing.T, msg string, input string, correct *Expr) { + t.Helper() assertParseOne(t, msg, input, func(parsed interface{}) { + t.Helper() body := parsed.(Body) if len(body) != 1 { - t.Errorf("Error on test %s: parser returned multiple expressions: %v", msg, body) + t.Errorf("Error on test \"%s\": parser returned multiple expressions: %v", msg, body) return } expr := body[0] if !expr.Equal(correct) { - t.Errorf("Error on test %s: expressions not equal:\n%v (parsed)\n%v (correct)", msg, expr, correct) + t.Errorf("Error on test \"%s\": expressions not equal:\n%v (parsed)\n%v (correct)", msg, expr, correct) } }) } @@ -1789,10 +2314,12 @@ func assertParseOneExprNegated(t *testing.T, msg string, input string, correct * } func assertParseOneTerm(t *testing.T, msg string, input string, correct *Term) { + t.Helper() assertParseOneExpr(t, msg, input, &Expr{Terms: correct}) } func assertParseOneTermNegated(t *testing.T, msg string, input string, correct *Term) { + t.Helper() assertParseOneExprNegated(t, msg, input, &Expr{Terms: correct}) } @@ -1802,7 +2329,7 @@ func assertParseRule(t *testing.T, msg string, input string, correct *Rule) { t.Helper() rule := parsed.(*Rule) if !rule.Equal(correct) { - t.Errorf("Error on test %s: rules not equal: %v (parsed), %v (correct)", msg, rule, correct) + t.Errorf("Error on test \"%s\": rules not equal: %v (parsed), %v (correct)", msg, rule, correct) } }) } diff --git a/ast/rego.peg b/ast/rego.peg deleted file mode 100644 index 71d04e7271..0000000000 --- a/ast/rego.peg +++ /dev/null @@ -1,311 +0,0 @@ -{ -package ast -} - -Program <- _ vals:(Stmt (ws Stmt)*)? _ EOF { - return makeProgram(c, vals) -} - -Stmt <- val:(Package / Import / Rules / Body / Comment) { - return val, nil -} - -Package <- "package" ws val:(Ref / Var) { - return makePackage(currentLocation(c), val) -} - -Import <- "import" ws path:(Ref / Var) alias:(ws "as" ws Var)? { - return makeImport(currentLocation(c), path, alias) -} - -Rules <- DefaultRules / NormalRules - -DefaultRules <- "default" ws name:Var _ operator:( ":=" / "=" ) _ value:Term { - return makeDefaultRule(currentLocation(c), name, operator, value) -} - -NormalRules <- head:(PartialRuleHead / RuleHead) _ rest:(NonEmptyBraceEnclosedBody ( _ RuleExt)* ) { - return makeRule(currentLocation(c), head, rest) -} - -PartialRuleHead <- name:Var args:( _ "(" _ Args _ ")" _ ) value:( _ ( ":=" / "=" ) _ ExprTerm )? { - return makeRuleHead(currentLocation(c), name, args, nil, value) -} - -RuleHead <- name:Var key:( _ "[" _ ExprTerm _ "]" _ )? value:( _ ( ":=" / "=" ) _ ExprTerm )? { - return makeRuleHead(currentLocation(c), name, nil, key, value) -} - -Args <- list:ExprTermList { - return makeArgs(list) -} - -Else <- "else" value:( _ "=" _ Term )? body:( _ NonEmptyBraceEnclosedBody ) { - return makeRuleExt(currentLocation(c), value, body) -} - -RuleDup <- b:NonEmptyBraceEnclosedBody { - return ruleExt{loc: currentLocation(c), body: b.(Body)}, nil -} - -RuleExt <- Else / RuleDup - -Body <- NonWhitespaceBody / BraceEnclosedBody - -NonEmptyBraceEnclosedBody <- "{" _ val:WhitespaceBody? _ "}" { - if val == nil { - return NewBody(), fmt.Errorf("found empty body") - } - return val, nil -} - -BraceEnclosedBody <- "{" _ val:WhitespaceBody? _ "}" { - return makeBraceEnclosedBody(currentLocation(c), val) -} - -WhitespaceBody <- head:Literal tail:(WhitespaceLiteralSeparator _ Literal)* { - return makeBody(head, tail, 2) -} - -NonWhitespaceBody <- head:Literal tail:( _ NonWhitespaceLiteralSeparator _ Literal)* { - return makeBody(head, tail, 3) -} - -WhitespaceLiteralSeparator <- [ \t]* ((NonWhitespaceLiteralSeparator Comment?) / (Comment? [\r\n])) - -NonWhitespaceLiteralSeparator <- ";" - -Literal <- TermExpr / SomeDecl - -SomeDecl <- "some" ws symbols:SomeDeclList { - return makeSomeDeclLiteral(currentLocation(c), symbols) -} - -SomeDeclList <- head:Var rest:( _ ',' _ Var)* { - return makeSomeDeclSymbols(head, rest) -} - -TermExpr <- negated:NotKeyword? value:LiteralExpr with:WithKeywordList? { - return makeLiteral(negated, value, with) -} - -LiteralExpr <- lhs:ExprTerm rest:( _ LiteralExprOperator _ ExprTerm)? { - return makeLiteralExpr(currentLocation(c), lhs, rest) -} - -LiteralExprOperator <- val:( ":=" / "=" ) { - return makeInfixOperator(currentLocation(c), c.text) -} - -NotKeyword <- val:("not" ws)? { - return val != nil, nil -} - -WithKeywordList <- ws head:WithKeyword rest:( ws WithKeyword )* { - return makeWithKeywordList(head, rest) -} - -WithKeyword <- "with" ws target:ExprTerm ws "as" ws value:ExprTerm { - return makeWithKeyword(currentLocation(c), target, value) -} - -ExprTerm <- lhs:RelationExpr rest:( _ RelationOperator _ RelationExpr )* { - return makeExprTerm(currentLocation(c), lhs, rest) -} - -ExprTermPairList <- head:ExprTermPair? tail:( _ ',' _ ExprTermPair )* _ ","? { - return makeExprTermPairList(head, tail) -} - -ExprTermList <- head:ExprTerm? tail:( _ ',' _ ExprTerm )* _ ","? { - return makeExprTermList(head, tail) -} - -ExprTermPair <- key:ExprTerm _ ':' _ value:ExprTerm { - return makeExprTermPair(key, value) -} - -RelationOperator <- val:("==" / "!=" / "<=" / ">=" / ">" / "<") { - return makeInfixOperator(currentLocation(c), c.text) -} - -RelationExpr <- lhs:BitwiseOrExpr rest:( _ BitwiseOrOperator _ BitwiseOrExpr)* { - return makeExprTerm(currentLocation(c), lhs, rest) -} - -BitwiseOrOperator <- val:"|" { - return makeInfixOperator(currentLocation(c), c.text) -} - -BitwiseOrExpr <- lhs:BitwiseAndExpr rest:( _ BitwiseAndOperator _ BitwiseAndExpr)* { - return makeExprTerm(currentLocation(c), lhs, rest) -} - -BitwiseAndOperator <- val:"&" { - return makeInfixOperator(currentLocation(c), c.text) -} - -BitwiseAndExpr <- lhs:ArithExpr rest:( _ ArithOperator _ ArithExpr)* { - return makeExprTerm(currentLocation(c), lhs, rest) -} - -ArithOperator <- val:("+" / "-") { - return makeInfixOperator(currentLocation(c), c.text) -} - -ArithExpr <- lhs:FactorExpr rest:( _ FactorOperator _ FactorExpr )* { - return makeExprTerm(currentLocation(c), lhs, rest) -} - -FactorOperator <- val:("*" / "/" / "%"){ - return makeInfixOperator(currentLocation(c), c.text) -} - -FactorExpr <- ( "(" _ expr:ExprTerm _ ")" ) { - return expr, nil -} / term:Term { - return term, nil -} - -Call <- operator:(Ref / Var) "(" _ args:ExprTermList _ ")" { - return makeCall(currentLocation(c), operator, args) -} - -Term <- val:( Comprehension / Composite / Scalar / Call / Var ) refs:RefOperand* { - return makeRef(currentLocation(c), val, refs) -} - -TermPair <- key:Term _ ":" _ value:Term { - return makeExprTermPair(key, value) -} - -Comprehension <- ArrayComprehension / ObjectComprehension / SetComprehension - -ArrayComprehension <- "[" _ head:Term _ "|" _ body:WhitespaceBody _ "]" { - return makeArrayComprehension(currentLocation(c), head, body) -} - -ObjectComprehension <- "{" _ head:TermPair _ "|" _ body:WhitespaceBody _ "}" { - return makeObjectComprehension(currentLocation(c), head, body) -} - -SetComprehension <- "{" _ head:Term _ "|" _ body:WhitespaceBody _ "}" { - return makeSetComprehension(currentLocation(c), head, body) -} - -Composite <- Object / Array / Set - -Scalar <- Number / String / Bool / Null - -Object <- '{' _ list:ExprTermPairList _ '}' { - return makeObject(currentLocation(c), list) -} - -Array <- '[' _ list:ExprTermList _ ']' { - return makeArray(currentLocation(c), list) -} - -Set <- SetEmpty / SetNonEmpty - -SetEmpty <- "set(" _ ")" { - var empty []*Term - return makeSet(currentLocation(c), empty) -} - -SetNonEmpty <- '{' _ list:ExprTermList _ '}' { - return makeSet(currentLocation(c), list) -} - -Ref <- head:(Composite / Var) rest:RefOperand+ { - return makeRef(currentLocation(c), head, rest) -} - -RefOperand <- RefOperandDot / RefOperandCanonical - -RefOperandDot <- "." val:Var { - return makeRefOperandDot(currentLocation(c), val) -} - -RefOperandCanonical <- "[" val:ExprTerm "]" { - return val, nil -} - -Var <- val:VarChecked { - return val.([]interface{})[0], nil -} - -VarChecked <- val:VarUnchecked !{ - return IsKeyword(string(val.(*Term).Value.(Var))), nil -} - -VarUnchecked <- VarStart VarChar* { - return makeVar(currentLocation(c), c.text) -} - -Number <- '-'? ( Float / Integer ) { - return makeNumber(currentLocation(c), c.text) -} - -Float <- ExponentFloat / PointFloat - -ExponentFloat <- ( PointFloat / Integer ) Exponent - -PointFloat <- Integer? Fraction - -Fraction <- '.' DecimalDigit+ - -Exponent <- 'e'i [+-]? DecimalDigit+ - -Integer <- '0' / ( NonZeroDecimalDigit DecimalDigit* ) - -String <- QuotedString / RawString - -QuotedString <- '"' Char* '"' { - return makeString(currentLocation(c), c.text) -} / '"' Char* !'"' { - return makeNonterminatedString(currentLocation(c), string(c.text)) -} - -RawString <- '`' [^`]* '`' { - return makeRawString(currentLocation(c), c.text) -} - -Bool <- val:("true" / "false") !VarChar { - return makeBool(currentLocation(c), c.text) -} - -Null <- "null" !VarChar { - return makeNull(currentLocation(c)) -} - -VarStart <- AsciiLetter - -VarChar <- AsciiLetter / DecimalDigit - -AsciiLetter <- [A-Za-z_] - -Char <- ( !EscapedChar . ) / ( '\\' EscapeSequence ) - -EscapedChar <- [\x00-\x1f"\\] - -EscapeSequence <- SingleCharEscape / UnicodeEscape - -SingleCharEscape <- [ " \\ / b f n r t ] - -UnicodeEscape <- 'u' HexDigit HexDigit HexDigit HexDigit - -DecimalDigit <- [0-9] - -NonZeroDecimalDigit <- [1-9] - -HexDigit <- [0-9a-fA-F] - -ws "whitespace" <- [ \t\r\n]+ - -_ "whitespace" <- ( [ \t\r\n] / Comment )* - -Comment <- [ \t]* "#" text:[^\r\n]* { - return makeComments(c, text) -} - -EOF <- !. diff --git a/ast/term.go b/ast/term.go index b6137d2c09..b7b3ac9462 100644 --- a/ast/term.go +++ b/ast/term.go @@ -19,89 +19,18 @@ import ( "github.com/OneOfOne/xxhash" "github.com/pkg/errors" + "github.com/open-policy-agent/opa/ast/location" "github.com/open-policy-agent/opa/util" ) var errFindNotFound = fmt.Errorf("find: not found") -// Location records a position in source code -type Location struct { - Text []byte `json:"-"` // The original text fragment from the source. - File string `json:"file"` // The name of the source file (which may be empty). - Row int `json:"row"` // The line in the source. - Col int `json:"col"` // The column in the row. -} +// Location records a position in source code. +type Location = location.Location // NewLocation returns a new Location object. func NewLocation(text []byte, file string, row int, col int) *Location { - return &Location{Text: text, File: file, Row: row, Col: col} -} - -// Equal checks if two locations are equal to each other. -func (loc *Location) Equal(other *Location) bool { - return bytes.Equal(loc.Text, other.Text) && - loc.File == other.File && - loc.Row == other.Row && - loc.Col == other.Col -} - -// Errorf returns a new error value with a message formatted to include the location -// info (e.g., line, column, filename, etc.) -func (loc *Location) Errorf(f string, a ...interface{}) error { - return errors.New(loc.Format(f, a...)) -} - -// Wrapf returns a new error value that wraps an existing error with a message formatted -// to include the location info (e.g., line, column, filename, etc.) -func (loc *Location) Wrapf(err error, f string, a ...interface{}) error { - return errors.Wrap(err, loc.Format(f, a...)) -} - -// Format returns a formatted string prefixed with the location information. -func (loc *Location) Format(f string, a ...interface{}) string { - if len(loc.File) > 0 { - f = fmt.Sprintf("%v:%v: %v", loc.File, loc.Row, f) - } else { - f = fmt.Sprintf("%v:%v: %v", loc.Row, loc.Col, f) - } - return fmt.Sprintf(f, a...) -} - -func (loc *Location) String() string { - if len(loc.File) > 0 { - return fmt.Sprintf("%v:%v", loc.File, loc.Row) - } - if len(loc.Text) > 0 { - return string(loc.Text) - } - return fmt.Sprintf("%v:%v", loc.Row, loc.Col) -} - -// Compare returns -1, 0, or 1 to indicate if this loc is less than, equal to, -// or greater than the other. Comparison is performed on the file, row, and -// column of the Location (but not on the text.) Nil locations are greater than -// non-nil locations. -func (loc *Location) Compare(other *Location) int { - if loc == nil && other == nil { - return 0 - } else if loc == nil { - return 1 - } else if other == nil { - return -1 - } else if loc.File < other.File { - return -1 - } else if loc.File > other.File { - return 1 - } else if loc.Row < other.Row { - return -1 - } else if loc.Row > other.Row { - return 1 - } else if loc.Col < other.Col { - return -1 - } else if loc.Col > other.Col { - return 1 - } - return 0 + return location.NewLocation(text, file, row, col) } // Value declares the common interface for all Term values. Every kind of Term value diff --git a/ast/term_test.go b/ast/term_test.go index 28f175344b..fd7ccd6f1a 100644 --- a/ast/term_test.go +++ b/ast/term_test.go @@ -818,86 +818,6 @@ func TestValueToInterface(t *testing.T) { } } -func TestLocationCompare(t *testing.T) { - - tests := []struct { - a string - b string - exp int - }{ - { - a: "", - b: "", - exp: 0, - }, - { - a: "", - b: `{"file": "a", "row": 1, "col": 1}`, - exp: 1, - }, - { - a: `{"file": "a", "row": 1, "col": 1}`, - b: "", - exp: -1, - }, - { - a: `{"file": "a", "row": 1, "col": 1}`, - b: `{"file": "a", "row": 1, "col": 1}`, - exp: 0, - }, - { - a: `{"file": "a", "row": 1, "col": 1}`, - b: `{"file": "b", "row": 1, "col": 1}`, - exp: -1, - }, - { - a: `{"file": "b", "row": 1, "col": 1}`, - b: `{"file": "a", "row": 1, "col": 1}`, - exp: 1, - }, - { - a: `{"file": "a", "row": 1, "col": 1}`, - b: `{"file": "a", "row": 2, "col": 1}`, - exp: -1, - }, - { - a: `{"file": "a", "row": 2, "col": 1}`, - b: `{"file": "a", "row": 1, "col": 1}`, - exp: 1, - }, - { - a: `{"file": "a", "row": 1, "col": 1}`, - b: `{"file": "a", "row": 1, "col": 2}`, - exp: -1, - }, - { - a: `{"file": "a", "row": 1, "col": 2}`, - b: `{"file": "a", "row": 1, "col": 1}`, - exp: 1, - }, - } - - unmarshal := func(s string) *Location { - if s != "" { - var loc Location - if err := util.Unmarshal([]byte(s), &loc); err != nil { - t.Fatal(err) - } - return &loc - } - return nil - } - - for _, tc := range tests { - locA := unmarshal(tc.a) - locB := unmarshal(tc.b) - result := locA.Compare(locB) - if tc.exp != result { - t.Fatalf("Expected %v but got %v for %v.Compare(%v)", tc.exp, result, locA, locB) - } - } -} - func assertTermEqual(t *testing.T, x *Term, y *Term) { if !x.Equal(y) { t.Errorf("Failure on equality: \n%s and \n%s\n", x, y) diff --git a/cmd/bench_test.go b/cmd/bench_test.go index 67df0639ea..5e18a6dd35 100644 --- a/cmd/bench_test.go +++ b/cmd/bench_test.go @@ -444,7 +444,7 @@ func TestRenderBenchmarkErrorJSONOutput(t *testing.T) { expected := `{ "errors": [ { - "message": "no match found", + "message": "illegal token", "code": "rego_parse_error", "location": { "file": "", @@ -487,7 +487,7 @@ func testPrettyBenchmarkOutput(t *testing.T, params benchmarkCommandParams) { renderBenchmarkError(params, err, &buf) actual := buf.String() - expected := `1 error occurred: 1:1: rego_parse_error: no match found + expected := `1 error occurred: 1:1: rego_parse_error: illegal token ??? ^ ` diff --git a/docs/devel/DEVELOPMENT.md b/docs/devel/DEVELOPMENT.md index 0348913f2a..f14ba931a3 100644 --- a/docs/devel/DEVELOPMENT.md +++ b/docs/devel/DEVELOPMENT.md @@ -9,31 +9,17 @@ Requirements: - Git - GitHub account (if you are contributing) -- Go (version 1.13 is supported though older versions are likely to work) +- Go (version 1.13+ is supported though older versions are likely to work) - GNU Make ## Getting Started After cloning the repository, just run `make`. This will: -- Install required dependencies, e.g., the parser-generator ("pigeon"). - Build the OPA binary. - Run all of the tests. - Run all of the static analysis checks. -If `make` fails with `main.go:20: running "pigeon": exec: "pigeon": -executable file not found in $PATH` make sure that `$GOPATH/bin` is -in `$PATH`. If `$GOPATH` is undefined, it defaults to -`$HOME/go/bin`: - -``` -export PATH=$PATH:$GOPATH/bin - -# OR - -export PATH=$PATH:$HOME/go/bin -``` - If the build was successful, a binary will be produced in the top directory (`opa__`). Verify the build was successful with `./opa__ run`. @@ -125,9 +111,6 @@ the version desired. This should update the [go.mod](../../go.mod) and (potentia [go.sum](../../go.sum) files. After this you *MUST* run `go mod vendor` to ensure that the `vendor` directory is in sync. -After updating dependencies, be sure to check if the parser-generator ("pigeon") -was updated. If it was, re-generate the parser and commit the changes. - Example workflow for updating a dependency: ```bash @@ -141,7 +124,7 @@ If dependencies have been removed ensure to run `go mod tidy` to clean them up. ### Tool Dependencies -We use some tools such as `pigeon`, `goimports`, etc which are versioned and vendored +We use some tools such as `goimports` which are versioned and vendored with OPA as depedencies. See [tools.go](../../tools.go) for a list of tools. More details on the pattern: [https://github.com/go-modules-by-example/index/blob/master/010_tools/README.md](https://github.com/go-modules-by-example/index/blob/master/010_tools/README.md) diff --git a/format/format.go b/format/format.go index f5cddb2433..ef99f6782e 100644 --- a/format/format.go +++ b/format/format.go @@ -13,17 +13,6 @@ import ( "github.com/open-policy-agent/opa/ast" ) -// Bytes formats Rego source code. The bytes provided do not have to be an entire -// source file, but they must be parse-able. If the bytes are not parse-able, Bytes -// will return an error resulting from the attempt to parse them. -func Bytes(src []byte) ([]byte, error) { - astElem, err := ast.Parse("", src, ast.CommentsOption()) - if err != nil { - return nil, err - } - return Ast(astElem) -} - // Source formats a Rego source file. The bytes provided must describe a complete // Rego module. If they don't, Source will return an error resulting from the attempt // to parse the bytes. @@ -443,13 +432,12 @@ func (w *writer) writeTermParens(parens bool, term *ast.Term, comments []*ast.Co case *ast.SetComprehension: comments = w.writeSetComprehension(x, term.Location, comments) case ast.String: - if term.Location.Text[0] == '.' { - // This string was parsed from a ref, so preserve the value. - w.write(`"` + string(x) + `"`) - } else { + if term.Location.Text[0] == '`' { // To preserve raw strings, we need to output the original text, // not what x.String() would give us. w.write(string(term.Location.Text)) + } else { + w.write(x.String()) } case ast.Call: comments = w.writeCall(parens, x, term.Location, comments) diff --git a/format/format_test.go b/format/format_test.go index 97fb147557..bd04954e65 100644 --- a/format/format_test.go +++ b/format/format_test.go @@ -54,7 +54,7 @@ func TestFormatSourceError(t *testing.T) { t.Fatal("Expected parsing error, not nil") } - exp := "1 error occurred: testfiles/test.rego.error:27: rego_parse_error: no match found" + exp := "1 error occurred: testfiles/test.rego.error:27: rego_parse_error: unexpected eof token" if !strings.HasPrefix(err.Error(), exp) { t.Fatalf("Expected error message '%s', got '%s'", exp, err.Error()) diff --git a/format/testfiles/test.rego.formatted b/format/testfiles/test.rego.formatted index a326955408..5af5761ea6 100644 --- a/format/testfiles/test.rego.formatted +++ b/format/testfiles/test.rego.formatted @@ -61,10 +61,12 @@ fn(x) = y { fn_else(x) = 1 { true -} # foo +} -# bar +# foo else = 2 { + # bar + true } @@ -100,14 +102,16 @@ fn2( trim(a, z, d) # function comment 1 split(c[0], d, b) x = sprintf("hello %v", ["world"]) - #function comment 2 -} # function comment 3 +} + +#function comment 2 +# function comment 3 f[x] { x = "hi" -} # Comment on chain +} -f[x] { +f[x] { # Comment on chain x = "bye" } @@ -184,7 +188,9 @@ p[x] = y { y = x[_] # inner comment } -} # Comment on rule closing brace +} + +# Comment on rule closing brace nested_infix { x + 1 diff --git a/go.mod b/go.mod index 2e592bf637..1f6f9d7b37 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.12 require ( github.com/OneOfOne/xxhash v1.2.7 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/fsnotify/fsnotify v1.4.7 // indirect github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4 github.com/gobwas/glob v0.2.3 @@ -16,7 +15,6 @@ require ( github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/mna/pigeon v0.0.0-20180808201053-bb0192cfc2ae github.com/olekukonko/tablewriter v0.0.1 github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d github.com/pkg/errors v0.0.0-20181023235946-059132a15dd0 diff --git a/go.sum b/go.sum index 8ad0329004..fd87d31bf1 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,7 @@ -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.3 h1:wS8NNaIgtzapuArKIAjsyXtEN/IUjQkbw90xszUdS40= -github.com/OneOfOne/xxhash v1.2.3/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.7 h1:fzrmmkskv067ZQbd9wERNGuxckWw67dyzoMG62p7LMo= github.com/OneOfOne/xxhash v1.2.7/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -32,8 +27,6 @@ github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39 h1:0E3wlIAcvD6z github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mna/pigeon v0.0.0-20180808201053-bb0192cfc2ae h1:yIn3M+2nBaa+i9jUVoO+YmFjdczHt/BgReCj4EJOYOo= -github.com/mna/pigeon v0.0.0-20180808201053-bb0192cfc2ae/go.mod h1:Iym28+kJVnC1hfQvv5MUtI6AiFFzvQjHcvI4RFTG/04= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d h1:zapSxdmZYY6vJWXFKLQ+MkI+agc+HQyfrCGowDSHiKs= @@ -54,8 +47,6 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhD github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v0.0.0-20181021141114-fe5e611709b0 h1:BgSbPgT2Zu8hDen1jJDGLWO8voaSRVrwsk18Q/uSh5M= github.com/spf13/cobra v0.0.0-20181021141114-fe5e611709b0/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v0.0.0-20181024212040-082b515c9490 h1:EmIGPbInxgMLEZd2f2MZwv0lCYiAv93kztj4caWSUZA= diff --git a/internal/presentation/presentation_test.go b/internal/presentation/presentation_test.go index eea72e62e2..231c961d85 100644 --- a/internal/presentation/presentation_test.go +++ b/internal/presentation/presentation_test.go @@ -227,7 +227,7 @@ func TestOutputJSONErrorStructuredAstParseErr(t *testing.T) { expected := `{ "errors": [ { - "message": "no match found", + "message": "illegal ! character", "code": "rego_parse_error", "location": { "file": "parse-err.rego", diff --git a/main.go b/main.go index 2a80f58257..799be8e671 100644 --- a/main.go +++ b/main.go @@ -18,8 +18,5 @@ func main() { } } -// Rego parser generation: -// -//go:generate build/run-pigeon.sh -o ast/parser.go ast/rego.peg -//go:generate build/run-goimports.sh -w ast/parser.go +// WASM base binary generation: //go:generate build/gen-opa-wasm.sh internal/cmd/genopawasm/main.go -o internal/compiler/wasm/opa/opa.go internal/compiler/wasm/opa/opa.wasm diff --git a/server/server_test.go b/server/server_test.go index 354cca15e1..a97389fc77 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -3240,7 +3240,7 @@ func TestBadQueryV1(t *testing.T) { "errors": [ { "code": "rego_parse_error", - "message": "no match found", + "message": "illegal token", "location": { "file": "", "row": 1, diff --git a/tools.go b/tools.go index 61a7ef0ba1..81c924ae1b 100644 --- a/tools.go +++ b/tools.go @@ -10,7 +10,6 @@ package tools import ( - _ "github.com/mna/pigeon" _ "golang.org/x/lint/golint" _ "golang.org/x/tools/cmd/goimports" ) diff --git a/topdown/bits_test.go b/topdown/bits_test.go index 809a7e2976..3953cc9bcf 100644 --- a/topdown/bits_test.go +++ b/topdown/bits_test.go @@ -115,9 +115,9 @@ func TestBuiltinBitsShiftLeft(t *testing.T) { `4294967294`, }, { - "shift of max int64 doesn't overflow, but it's lossy do to conversion to exponent type (see discussion in #2160)", + "shift of max int64 doesn't overflow and is not lossy", []string{fmt.Sprintf(`p = x { x := bits.lsh(%d, 1) }`, math.MaxInt64)}, - `18446744074000000000`, + `18446744073709551614`, }, } diff --git a/topdown/parse_test.go b/topdown/parse_test.go index 16a23762bd..e5f69d4bff 100644 --- a/topdown/parse_test.go +++ b/topdown/parse_test.go @@ -23,6 +23,6 @@ func TestRegoParseModule(t *testing.T) { `p = x { rego.parse_module("x.rego", data.ok, module); x = module["package"].path[1].value }`}, `"foo"`) runTopDownTestCase(t, data, "error", []string{ - `p = x { rego.parse_module("x.rego", data.err, x) }`}, &Error{Code: BuiltinErr, Message: "rego_parse_error: no match found"}) + `p = x { rego.parse_module("x.rego", data.err, x) }`}, &Error{Code: BuiltinErr, Message: "rego_parse_error: unexpected eof token: expected ident"}) } diff --git a/topdown/topdown_test.go b/topdown/topdown_test.go index 443b258dbe..80c030ab3f 100644 --- a/topdown/topdown_test.go +++ b/topdown/topdown_test.go @@ -1394,7 +1394,7 @@ func TestTopDownStrings(t *testing.T) { {"sprintf: int", []string{`p = x { sprintf("hi %02d", [5], x) }`}, `"hi 05"`}, {"sprintf: hex", []string{`p = x { sprintf("hi %02X.%02X", [127, 1], x) }`}, `"hi 7F.01"`}, {"sprintf: float", []string{`p = x { sprintf("hi %.2f", [3.1415], x) }`}, `"hi 3.14"`}, - {"sprintf: float too big", []string{`p = x { sprintf("hi %v", [2e308], x) }`}, `"hi 2e+308"`}, + {"sprintf: float too big", []string{`p = x { sprintf("hi %v", [2e308], x) }`}, `"hi 2e308"`}, {"sprintf: bool", []string{`p = x { sprintf("hi %s", [true], x) }`}, `"hi true"`}, {"sprintf: composite", []string{`p = x { sprintf("hi %v", [["there", 5, 3.14]], x) }`}, `"hi [\"there\", 5, 3.14]"`}, } diff --git a/vendor/github.com/mna/pigeon/.editorconfig b/vendor/github.com/mna/pigeon/.editorconfig deleted file mode 100644 index 7d709fb603..0000000000 --- a/vendor/github.com/mna/pigeon/.editorconfig +++ /dev/null @@ -1,6 +0,0 @@ -# See http://editorconfig.org - -# In Go files we indent with tabs but still -# set indent_size to control the GitHub web viewer. -[*.go] -indent_size=4 diff --git a/vendor/github.com/mna/pigeon/.gitattributes b/vendor/github.com/mna/pigeon/.gitattributes deleted file mode 100644 index 58ef8638ad..0000000000 --- a/vendor/github.com/mna/pigeon/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -*.go text eol=lf -*.peg text eol=lf diff --git a/vendor/github.com/mna/pigeon/.gitignore b/vendor/github.com/mna/pigeon/.gitignore deleted file mode 100644 index 080ebe3139..0000000000 --- a/vendor/github.com/mna/pigeon/.gitignore +++ /dev/null @@ -1,22 +0,0 @@ -# Binaries for programs and plugins -*.exe -*.dll -*.so -*.dylib -pigeon -bin/ -bootstrap/cmd/bootstrap-pigeon/bootstrap-pigeon -bootstrap/cmd/bootstrap-build/bootstrap-build -bootstrap/cmd/pegscan/pegscan -bootstrap/cmd/pegparse/pegparse - -# Test binary, build with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Temporary and swap files -*.swp -*.swo -*~ diff --git a/vendor/github.com/mna/pigeon/.travis.yml b/vendor/github.com/mna/pigeon/.travis.yml deleted file mode 100644 index 22a5cb5df2..0000000000 --- a/vendor/github.com/mna/pigeon/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: go - -install: go get golang.org/x/tools/imports/... - -script: go test -v ./... - -go: - - 1.7.x - - 1.8.x - - 1.9.x - - 1.10.x - - tip diff --git a/vendor/github.com/mna/pigeon/CONTRIBUTING.md b/vendor/github.com/mna/pigeon/CONTRIBUTING.md deleted file mode 100644 index 9d39240e52..0000000000 --- a/vendor/github.com/mna/pigeon/CONTRIBUTING.md +++ /dev/null @@ -1,33 +0,0 @@ -# Contributing to pigeon - -There are various ways to help support this open source project: - -* if you use pigeon and find it useful, talk about it - that's probably the most basic way to help any open-source project: getting the word out that it exists and that it can be useful -* if you use pigeon and find bugs, please [file an issue][0] -* if something is poorly documented, or doesn't work as documented, this is also a bug, please [file an issue][0] -* if you can fix the issue (whether it is documentation- or code-related), then [submit a pull-request][1] - but read on to see what should be done to get it merged -* if you would like to see some new feature/behaviour being implemented, please first [open an issue][0] to discuss it because features are less likely to get merged compared to bug fixes - -## Submitting a pull request - -Assuming you already have a copy of the repository (either via `go get`, a github fork, a clone, etc.), you will also need `make` to regenerate all tools and files generated when a dependency changes. I use GNU make version 4.1, other versions of make may work too but haven't been tested. - -Run `make` in the root directory of the repository. That will create the bootstrap builder, the bootstrap parser, and the final parser, along with some generated Go files. Once `make` is run successfully, run `go test ./...` in the root directory to make sure all tests pass. - -Once this is done and tests pass, you can start implementing the bug fix (or the new feature provided **it has already been discussed and agreed in a github issue** first). - -For a bug fix, the best way to proceed is to first write a test that proves the bug, then write the code that fixes the bug and makes the test pass. All other tests should still pass too (unless it relied on the buggy behaviour, in which case existing tests must be fixed). - -For a new feature, it must be thoroughly tested. New code without new test(s) is unlikely to get merged. - -Respect the coding style of the repository, which means essentially to respect the [coding guidelines of the Go community][2]. Use `gofmt` to format your code, and `goimports` to add and format the list of imported packages (or do it manually, but in a `goimports`-style). - -Once all code is done and tests pass, regenerate the whole tree with `make`, run `make lint` to make sure the code is correct, and run tests again. You are now ready to submit the pull request. - -## Licensing - -All pull requests that get merged will be made available under the BSD 3-Clause license (see the LICENSE file for details), as the rest of the pigeon repository. Do not submit pull requests if you do not want your contributions to be made available under those terms. - -[0]: https://github.com/mna/pigeon/issues/new -[1]: https://github.com/mna/pigeon/pulls -[2]: https://github.com/golang/go/wiki/CodeReviewComments diff --git a/vendor/github.com/mna/pigeon/LICENSE b/vendor/github.com/mna/pigeon/LICENSE deleted file mode 100644 index 2c684aaf65..0000000000 --- a/vendor/github.com/mna/pigeon/LICENSE +++ /dev/null @@ -1,12 +0,0 @@ -Copyright (c) 2015, Martin Angers & Contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/mna/pigeon/Makefile b/vendor/github.com/mna/pigeon/Makefile deleted file mode 100644 index dbf047374a..0000000000 --- a/vendor/github.com/mna/pigeon/Makefile +++ /dev/null @@ -1,177 +0,0 @@ -SHELL = /bin/bash - -# directories and source code lists -ROOT = . -ROOT_SRC = $(ROOT)/*.go -BINDIR = ./bin -EXAMPLES_DIR = $(ROOT)/examples -TEST_DIR = $(ROOT)/test - -# builder and ast packages -BUILDER_DIR = $(ROOT)/builder -BUILDER_SRC = $(BUILDER_DIR)/*.go -AST_DIR = $(ROOT)/ast -AST_SRC = $(AST_DIR)/*.go - -# bootstrap tools variables -BOOTSTRAP_DIR = $(ROOT)/bootstrap -BOOTSTRAP_SRC = $(BOOTSTRAP_DIR)/*.go -BOOTSTRAPBUILD_DIR = $(BOOTSTRAP_DIR)/cmd/bootstrap-build -BOOTSTRAPBUILD_SRC = $(BOOTSTRAPBUILD_DIR)/*.go -BOOTSTRAPPIGEON_DIR = $(BOOTSTRAP_DIR)/cmd/bootstrap-pigeon -BOOTSTRAPPIGEON_SRC = $(BOOTSTRAPPIGEON_DIR)/*.go -STATICCODEGENERATOR_DIR = $(BOOTSTRAP_DIR)/cmd/static_code_generator -STATICCODEGENERATOR_SRC = $(STATICCODEGENERATOR_DIR)/*.go - -# grammar variables -GRAMMAR_DIR = $(ROOT)/grammar -BOOTSTRAP_GRAMMAR = $(GRAMMAR_DIR)/bootstrap.peg -PIGEON_GRAMMAR = $(GRAMMAR_DIR)/pigeon.peg - -TEST_GENERATED_SRC = $(patsubst %.peg,%.go,$(shell echo ./{examples,test}/**/*.peg)) - -all: $(BUILDER_DIR)/generated_static_code.go $(BINDIR)/static_code_generator \ - $(BUILDER_DIR)/generated_static_code_range_table.go \ - $(BINDIR)/bootstrap-build $(BOOTSTRAPPIGEON_DIR)/bootstrap_pigeon.go \ - $(BINDIR)/bootstrap-pigeon $(ROOT)/pigeon.go $(BINDIR)/pigeon \ - $(TEST_GENERATED_SRC) - -$(BINDIR)/static_code_generator: $(STATICCODEGENERATOR_SRC) - go build -o $@ $(STATICCODEGENERATOR_DIR) - -$(BINDIR)/bootstrap-build: $(BOOTSTRAPBUILD_SRC) $(BOOTSTRAP_SRC) $(BUILDER_SRC) \ - $(AST_SRC) - go build -o $@ $(BOOTSTRAPBUILD_DIR) - -$(BOOTSTRAPPIGEON_DIR)/bootstrap_pigeon.go: $(BINDIR)/bootstrap-build \ - $(BOOTSTRAP_GRAMMAR) - $(BINDIR)/bootstrap-build $(BOOTSTRAP_GRAMMAR) > $@ - -$(BINDIR)/bootstrap-pigeon: $(BOOTSTRAPPIGEON_SRC) \ - $(BOOTSTRAPPIGEON_DIR)/bootstrap_pigeon.go - go build -o $@ $(BOOTSTRAPPIGEON_DIR) - -$(ROOT)/pigeon.go: $(BINDIR)/bootstrap-pigeon $(PIGEON_GRAMMAR) - $(BINDIR)/bootstrap-pigeon $(PIGEON_GRAMMAR) > $@ - -$(BINDIR)/pigeon: $(ROOT_SRC) $(ROOT)/pigeon.go - go build -o $@ $(ROOT) - -$(BUILDER_DIR)/generated_static_code.go: $(BUILDER_DIR)/static_code.go $(BINDIR)/static_code_generator - $(BINDIR)/static_code_generator $(BUILDER_DIR)/static_code.go $@ staticCode - -$(BUILDER_DIR)/generated_static_code_range_table.go: $(BUILDER_DIR)/static_code_range_table.go $(BINDIR)/static_code_generator - $(BINDIR)/static_code_generator $(BUILDER_DIR)/static_code_range_table.go $@ rangeTable0 - -$(BOOTSTRAP_GRAMMAR): -$(PIGEON_GRAMMAR): - -# surely there's a better way to define the examples and test targets -$(EXAMPLES_DIR)/json/json.go: $(EXAMPLES_DIR)/json/json.peg $(EXAMPLES_DIR)/json/optimized/json.go $(EXAMPLES_DIR)/json/optimized-grammar/json.go $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(EXAMPLES_DIR)/json/optimized/json.go: $(EXAMPLES_DIR)/json/json.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint -optimize-parser -optimize-basic-latin $< > $@ - -$(EXAMPLES_DIR)/json/optimized-grammar/json.go: $(EXAMPLES_DIR)/json/json.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint -optimize-grammar $< > $@ - -$(EXAMPLES_DIR)/calculator/calculator.go: $(EXAMPLES_DIR)/calculator/calculator.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(EXAMPLES_DIR)/indentation/indentation.go: $(EXAMPLES_DIR)/indentation/indentation.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/andnot/andnot.go: $(TEST_DIR)/andnot/andnot.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/predicates/predicates.go: $(TEST_DIR)/predicates/predicates.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/issue_1/issue_1.go: $(TEST_DIR)/issue_1/issue_1.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/linear/linear.go: $(TEST_DIR)/linear/linear.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/issue_18/issue_18.go: $(TEST_DIR)/issue_18/issue_18.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/runeerror/runeerror.go: $(TEST_DIR)/runeerror/runeerror.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/errorpos/errorpos.go: $(TEST_DIR)/errorpos/errorpos.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/global_store/global_store.go: $(TEST_DIR)/global_store/global_store.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/goto/goto.go: $(TEST_DIR)/goto/goto.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/goto_state/goto_state.go: $(TEST_DIR)/goto_state/goto_state.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/max_expr_cnt/maxexpr.go: $(TEST_DIR)/max_expr_cnt/maxexpr.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/labeled_failures/labeled_failures.go: $(TEST_DIR)/labeled_failures/labeled_failures.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/thrownrecover/thrownrecover.go: $(TEST_DIR)/thrownrecover/thrownrecover.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/alternate_entrypoint/altentry.go: $(TEST_DIR)/alternate_entrypoint/altentry.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint -optimize-grammar -alternate-entrypoints Entry2,Entry3,C $< > $@ - -$(TEST_DIR)/state/state.go: $(TEST_DIR)/state/state.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint -optimize-grammar $< > $@ - -$(TEST_DIR)/stateclone/stateclone.go: $(TEST_DIR)/stateclone/stateclone.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/statereadonly/statereadonly.go: $(TEST_DIR)/statereadonly/statereadonly.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/staterestore/staterestore.go: $(TEST_DIR)/staterestore/staterestore.peg $(TEST_DIR)/staterestore/standard/staterestore.go $(TEST_DIR)/staterestore/optimized/staterestore.go $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/staterestore/standard/staterestore.go: $(TEST_DIR)/staterestore/staterestore.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/staterestore/optimized/staterestore.go: $(TEST_DIR)/staterestore/staterestore.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint -optimize-grammar -optimize-parser -alternate-entrypoints TestAnd,TestNot $< > $@ - -$(TEST_DIR)/emptystate/emptystate.go: $(TEST_DIR)/emptystate/emptystate.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/issue_65/issue_65.go: $(TEST_DIR)/issue_65/issue_65.peg $(TEST_DIR)/issue_65/optimized/issue_65.go $(TEST_DIR)/issue_65/optimized-grammar/issue_65.go $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ - -$(TEST_DIR)/issue_65/optimized/issue_65.go: $(TEST_DIR)/issue_65/issue_65.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint -optimize-parser -optimize-basic-latin $< > $@ - -$(TEST_DIR)/issue_65/optimized-grammar/issue_65.go: $(TEST_DIR)/issue_65/issue_65.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint -optimize-grammar $< > $@ - -lint: - golint ./... - go vet ./... - -gometalinter: - gometalinter ./... - -cmp: - @boot=$$(mktemp) && $(BINDIR)/bootstrap-pigeon $(PIGEON_GRAMMAR) > $$boot && \ - official=$$(mktemp) && $(BINDIR)/pigeon $(PIGEON_GRAMMAR) > $$official && \ - cmp $$boot $$official && \ - unlink $$boot && \ - unlink $$official - -clean: - rm -f $(BUILDER_DIR)/generated_static_code.go $(BUILDER_DIR)/generated_static_code_range_table.go - rm -f $(BOOTSTRAPPIGEON_DIR)/bootstrap_pigeon.go $(ROOT)/pigeon.go $(TEST_GENERATED_SRC) $(EXAMPLES_DIR)/json/optimized/json.go $(EXAMPLES_DIR)/json/optimized-grammar/json.go $(TEST_DIR)/staterestore/optimized/staterestore.go $(TEST_DIR)/staterestore/standard/staterestore.go $(TEST_DIR)/issue_65/optimized/issue_65.go $(TEST_DIR)/issue_65/optimized-grammar/issue_65.go - rm -rf $(BINDIR) - -.PHONY: all clean lint gometalinter cmp - diff --git a/vendor/github.com/mna/pigeon/README.md b/vendor/github.com/mna/pigeon/README.md deleted file mode 100644 index ba4a3342a5..0000000000 --- a/vendor/github.com/mna/pigeon/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# pigeon - a PEG parser generator for Go - -[![GoDoc](https://godoc.org/github.com/mna/pigeon?status.png)](https://godoc.org/github.com/mna/pigeon) -[![build status](https://secure.travis-ci.org/mna/pigeon.png?branch=master)](http://travis-ci.org/mna/pigeon) -[![GoReportCard](https://goreportcard.com/badge/github.com/mna/pigeon)](https://goreportcard.com/report/github.com/mna/pigeon) -[![Software License](https://img.shields.io/badge/license-BSD-blue.svg)](LICENSE) - -The pigeon command generates parsers based on a [parsing expression grammar (PEG)][0]. Its grammar and syntax is inspired by the [PEG.js project][1], while the implementation is loosely based on the [parsing expression grammar for C# 3.0][2] article. It parses Unicode text encoded in UTF-8. - -See the [godoc page][3] for detailed usage. Also have a look at the [Pigeon Wiki](https://github.com/mna/pigeon/wiki) for additional information about Pigeon and PEG in general. - -## Releases - -* v1.0.0 is the tagged release of the original implementation. -* Work has started on v2.0.0 with some planned breaking changes. - -Github user [@mna][6] created the package in April 2015, and [@breml][5] is the package's maintainer as of May 2017. - -### Breaking Changes since v1.0.0 - -* Removed support for Go < v1.7 due to the requirement of the package `context` in [golang.org/x/tools/imports](https://godoc.org/golang.org/x/tools/imports), which was added to the Go stdlib in Go v1.7. This is in compliance with the [Go Release Policy](https://golang.org/doc/devel/release.html#policy) respectively the [Go Release Maintenance](https://github.com/golang/go/wiki/Go-Release-Cycle#release-maintenance), which states support for each major release until there are two newer major releases. - -## Installation - -Provided you have Go correctly installed with the $GOPATH and $GOBIN environment variables set, run: - -``` -$ go get -u github.com/mna/pigeon -``` - -This will install or update the package, and the `pigeon` command will be installed in your $GOBIN directory. Neither this package nor the parsers generated by this command require any third-party dependency, unless such a dependency is used in the code blocks of the grammar. - -## Basic usage - -``` -$ pigeon [options] [PEG_GRAMMAR_FILE] -``` - -By default, the input grammar is read from `stdin` and the generated code is printed to `stdout`. You may save it in a file using the `-o` flag. - -## Example - -Given the following grammar: - -``` -{ -// part of the initializer code block omitted for brevity - -var ops = map[string]func(int, int) int { - "+": func(l, r int) int { - return l + r - }, - "-": func(l, r int) int { - return l - r - }, - "*": func(l, r int) int { - return l * r - }, - "/": func(l, r int) int { - return l / r - }, -} - -func toIfaceSlice(v interface{}) []interface{} { - if v == nil { - return nil - } - return v.([]interface{}) -} - -func eval(first, rest interface{}) int { - l := first.(int) - restSl := toIfaceSlice(rest) - for _, v := range restSl { - restExpr := toIfaceSlice(v) - r := restExpr[3].(int) - op := restExpr[1].(string) - l = ops[op](l, r) - } - return l -} -} - - -Input <- expr:Expr EOF { - return expr, nil -} - -Expr <- _ first:Term rest:( _ AddOp _ Term )* _ { - return eval(first, rest), nil -} - -Term <- first:Factor rest:( _ MulOp _ Factor )* { - return eval(first, rest), nil -} - -Factor <- '(' expr:Expr ')' { - return expr, nil -} / integer:Integer { - return integer, nil -} - -AddOp <- ( '+' / '-' ) { - return string(c.text), nil -} - -MulOp <- ( '*' / '/' ) { - return string(c.text), nil -} - -Integer <- '-'? [0-9]+ { - return strconv.Atoi(string(c.text)) -} - -_ "whitespace" <- [ \n\t\r]* - -EOF <- !. -``` - -The generated parser can parse simple arithmetic operations, e.g.: - -``` -18 + 3 - 27 * (-18 / -3) - -=> -141 -``` - -More examples can be found in the `examples/` subdirectory. - -See the [godoc page][3] for detailed usage. - -## Contributing - -See the CONTRIBUTING.md file. - -## License - -The [BSD 3-Clause license][4]. See the LICENSE file. - -[0]: http://en.wikipedia.org/wiki/Parsing_expression_grammar -[1]: http://pegjs.org/ -[2]: http://www.codeproject.com/Articles/29713/Parsing-Expression-Grammar-Support-for-C-Part -[3]: https://godoc.org/github.com/mna/pigeon -[4]: http://opensource.org/licenses/BSD-3-Clause -[5]: https://github.com/breml -[6]: https://github.com/mna diff --git a/vendor/github.com/mna/pigeon/TODO b/vendor/github.com/mna/pigeon/TODO deleted file mode 100644 index 75a1f2145f..0000000000 --- a/vendor/github.com/mna/pigeon/TODO +++ /dev/null @@ -1,3 +0,0 @@ -- refactor implementation as a VM to avoid stack overflow in pathological cases (and maybe better performance): in branch wip-vm -? options like current receiver name read directly from the grammar file -? type annotations for generated code functions diff --git a/vendor/github.com/mna/pigeon/ast/ast.go b/vendor/github.com/mna/pigeon/ast/ast.go deleted file mode 100644 index c34b7d1767..0000000000 --- a/vendor/github.com/mna/pigeon/ast/ast.go +++ /dev/null @@ -1,662 +0,0 @@ -// Package ast defines the abstract syntax tree for the PEG grammar. -// -// The parser generator's PEG grammar generates a tree using this package -// that is then converted by the builder to the simplified AST used in -// the generated parser. -package ast - -import ( - "bytes" - "fmt" - "strconv" - "strings" -) - -// Pos represents a position in a source file. -type Pos struct { - Filename string - Line int - Col int - Off int -} - -// String returns the textual representation of a position. -func (p Pos) String() string { - if p.Filename != "" { - return fmt.Sprintf("%s:%d:%d (%d)", p.Filename, p.Line, p.Col, p.Off) - } - return fmt.Sprintf("%d:%d (%d)", p.Line, p.Col, p.Off) -} - -// Grammar is the top-level node of the AST for the PEG grammar. -type Grammar struct { - p Pos - Init *CodeBlock - Rules []*Rule -} - -// NewGrammar creates a new grammar at the specified position. -func NewGrammar(p Pos) *Grammar { - return &Grammar{p: p} -} - -// Pos returns the starting position of the node. -func (g *Grammar) Pos() Pos { return g.p } - -// String returns the textual representation of a node. -func (g *Grammar) String() string { - var buf bytes.Buffer - - buf.WriteString(fmt.Sprintf("%s: %T{Init: %v, Rules: [\n", - g.p, g, g.Init)) - for _, r := range g.Rules { - buf.WriteString(fmt.Sprintf("%s,\n", r)) - } - buf.WriteString("]}") - return buf.String() -} - -// Rule represents a rule in the PEG grammar. It has a name, an optional -// display name to be used in error messages, and an expression. -type Rule struct { - p Pos - Name *Identifier - DisplayName *StringLit - Expr Expression -} - -// NewRule creates a rule with at the specified position and with the -// specified name as identifier. -func NewRule(p Pos, name *Identifier) *Rule { - return &Rule{p: p, Name: name} -} - -// Pos returns the starting position of the node. -func (r *Rule) Pos() Pos { return r.p } - -// String returns the textual representation of a node. -func (r *Rule) String() string { - return fmt.Sprintf("%s: %T{Name: %v, DisplayName: %v, Expr: %v}", - r.p, r, r.Name, r.DisplayName, r.Expr) -} - -// Expression is the interface implemented by all expression types. -type Expression interface { - Pos() Pos -} - -// ChoiceExpr is an ordered sequence of expressions. The parser tries to -// match any of the alternatives in sequence and stops at the first one -// that matches. -type ChoiceExpr struct { - p Pos - Alternatives []Expression -} - -// NewChoiceExpr creates a choice expression at the specified position. -func NewChoiceExpr(p Pos) *ChoiceExpr { - return &ChoiceExpr{p: p} -} - -// Pos returns the starting position of the node. -func (c *ChoiceExpr) Pos() Pos { return c.p } - -// String returns the textual representation of a node. -func (c *ChoiceExpr) String() string { - var buf bytes.Buffer - - buf.WriteString(fmt.Sprintf("%s: %T{Alternatives: [\n", c.p, c)) - for _, e := range c.Alternatives { - buf.WriteString(fmt.Sprintf("%s,\n", e)) - } - buf.WriteString("]}") - return buf.String() -} - -// FailureLabel is an identifier, which can by thrown and recovered in a grammar -type FailureLabel string - -// RecoveryExpr is an ordered sequence of expressions. The parser tries to -// match any of the alternatives in sequence and stops at the first one -// that matches. -type RecoveryExpr struct { - p Pos - Expr Expression - RecoverExpr Expression - Labels []FailureLabel -} - -// NewRecoveryExpr creates a choice expression at the specified position. -func NewRecoveryExpr(p Pos) *RecoveryExpr { - return &RecoveryExpr{p: p} -} - -// Pos returns the starting position of the node. -func (r *RecoveryExpr) Pos() Pos { return r.p } - -// String returns the textual representation of a node. -func (r *RecoveryExpr) String() string { - var buf bytes.Buffer - - buf.WriteString(fmt.Sprintf("%s: %T{Expr: %v, RecoverExpr: %v", r.p, r, r.Expr, r.RecoverExpr)) - buf.WriteString(fmt.Sprintf(", Labels: [\n")) - for _, e := range r.Labels { - buf.WriteString(fmt.Sprintf("%s,\n", e)) - } - buf.WriteString("]}") - return buf.String() -} - -// ActionExpr is an expression that has an associated block of code to -// execute when the expression matches. -type ActionExpr struct { - p Pos - Expr Expression - Code *CodeBlock - FuncIx int -} - -// NewActionExpr creates a new action expression at the specified position. -func NewActionExpr(p Pos) *ActionExpr { - return &ActionExpr{p: p} -} - -// Pos returns the starting position of the node. -func (a *ActionExpr) Pos() Pos { return a.p } - -// String returns the textual representation of a node. -func (a *ActionExpr) String() string { - return fmt.Sprintf("%s: %T{Expr: %v, Code: %v}", a.p, a, a.Expr, a.Code) -} - -// ThrowExpr is an expression that throws an FailureLabel to be catched by a -// RecoveryChoiceExpr. -type ThrowExpr struct { - p Pos - Label string -} - -// NewThrowExpr creates a new throw expression at the specified position. -func NewThrowExpr(p Pos) *ThrowExpr { - return &ThrowExpr{p: p} -} - -// Pos returns the starting position of the node. -func (t *ThrowExpr) Pos() Pos { return t.p } - -// String returns the textual representation of a node. -func (t *ThrowExpr) String() string { - return fmt.Sprintf("%s: %T{Label: %v}", t.p, t, t.Label) -} - -// SeqExpr is an ordered sequence of expressions, all of which must match -// if the SeqExpr is to be a match itself. -type SeqExpr struct { - p Pos - Exprs []Expression -} - -// NewSeqExpr creates a new sequence expression at the specified position. -func NewSeqExpr(p Pos) *SeqExpr { - return &SeqExpr{p: p} -} - -// Pos returns the starting position of the node. -func (s *SeqExpr) Pos() Pos { return s.p } - -// String returns the textual representation of a node. -func (s *SeqExpr) String() string { - var buf bytes.Buffer - - buf.WriteString(fmt.Sprintf("%s: %T{Exprs: [\n", s.p, s)) - for _, e := range s.Exprs { - buf.WriteString(fmt.Sprintf("%s,\n", e)) - } - buf.WriteString("]}") - return buf.String() -} - -// LabeledExpr is an expression that has an associated label. Code blocks -// can access the value of the expression using that label, that becomes -// a local variable in the code. -type LabeledExpr struct { - p Pos - Label *Identifier - Expr Expression -} - -// NewLabeledExpr creates a new labeled expression at the specified position. -func NewLabeledExpr(p Pos) *LabeledExpr { - return &LabeledExpr{p: p} -} - -// Pos returns the starting position of the node. -func (l *LabeledExpr) Pos() Pos { return l.p } - -// String returns the textual representation of a node. -func (l *LabeledExpr) String() string { - return fmt.Sprintf("%s: %T{Label: %v, Expr: %v}", l.p, l, l.Label, l.Expr) -} - -// AndExpr is a zero-length matcher that is considered a match if the -// expression it contains is a match. -type AndExpr struct { - p Pos - Expr Expression -} - -// NewAndExpr creates a new and (&) expression at the specified position. -func NewAndExpr(p Pos) *AndExpr { - return &AndExpr{p: p} -} - -// Pos returns the starting position of the node. -func (a *AndExpr) Pos() Pos { return a.p } - -// String returns the textual representation of a node. -func (a *AndExpr) String() string { - return fmt.Sprintf("%s: %T{Expr: %v}", a.p, a, a.Expr) -} - -// NotExpr is a zero-length matcher that is considered a match if the -// expression it contains is not a match. -type NotExpr struct { - p Pos - Expr Expression -} - -// NewNotExpr creates a new not (!) expression at the specified position. -func NewNotExpr(p Pos) *NotExpr { - return &NotExpr{p: p} -} - -// Pos returns the starting position of the node. -func (n *NotExpr) Pos() Pos { return n.p } - -// String returns the textual representation of a node. -func (n *NotExpr) String() string { - return fmt.Sprintf("%s: %T{Expr: %v}", n.p, n, n.Expr) -} - -// ZeroOrOneExpr is an expression that can be matched zero or one time. -type ZeroOrOneExpr struct { - p Pos - Expr Expression -} - -// NewZeroOrOneExpr creates a new zero or one expression at the specified -// position. -func NewZeroOrOneExpr(p Pos) *ZeroOrOneExpr { - return &ZeroOrOneExpr{p: p} -} - -// Pos returns the starting position of the node. -func (z *ZeroOrOneExpr) Pos() Pos { return z.p } - -// String returns the textual representation of a node. -func (z *ZeroOrOneExpr) String() string { - return fmt.Sprintf("%s: %T{Expr: %v}", z.p, z, z.Expr) -} - -// ZeroOrMoreExpr is an expression that can be matched zero or more times. -type ZeroOrMoreExpr struct { - p Pos - Expr Expression -} - -// NewZeroOrMoreExpr creates a new zero or more expression at the specified -// position. -func NewZeroOrMoreExpr(p Pos) *ZeroOrMoreExpr { - return &ZeroOrMoreExpr{p: p} -} - -// Pos returns the starting position of the node. -func (z *ZeroOrMoreExpr) Pos() Pos { return z.p } - -// String returns the textual representation of a node. -func (z *ZeroOrMoreExpr) String() string { - return fmt.Sprintf("%s: %T{Expr: %v}", z.p, z, z.Expr) -} - -// OneOrMoreExpr is an expression that can be matched one or more times. -type OneOrMoreExpr struct { - p Pos - Expr Expression -} - -// NewOneOrMoreExpr creates a new one or more expression at the specified -// position. -func NewOneOrMoreExpr(p Pos) *OneOrMoreExpr { - return &OneOrMoreExpr{p: p} -} - -// Pos returns the starting position of the node. -func (o *OneOrMoreExpr) Pos() Pos { return o.p } - -// String returns the textual representation of a node. -func (o *OneOrMoreExpr) String() string { - return fmt.Sprintf("%s: %T{Expr: %v}", o.p, o, o.Expr) -} - -// RuleRefExpr is an expression that references a rule by name. -type RuleRefExpr struct { - p Pos - Name *Identifier -} - -// NewRuleRefExpr creates a new rule reference expression at the specified -// position. -func NewRuleRefExpr(p Pos) *RuleRefExpr { - return &RuleRefExpr{p: p} -} - -// Pos returns the starting position of the node. -func (r *RuleRefExpr) Pos() Pos { return r.p } - -// String returns the textual representation of a node. -func (r *RuleRefExpr) String() string { - return fmt.Sprintf("%s: %T{Name: %v}", r.p, r, r.Name) -} - -// StateCodeExpr is an expression which can modify the internal state of the parser. -type StateCodeExpr struct { - p Pos - Code *CodeBlock - FuncIx int -} - -// NewStateCodeExpr creates a new state (#) code expression at the specified -// position. -func NewStateCodeExpr(p Pos) *StateCodeExpr { - return &StateCodeExpr{p: p} -} - -// Pos returns the starting position of the node. -func (s *StateCodeExpr) Pos() Pos { return s.p } - -// String returns the textual representation of a node. -func (s *StateCodeExpr) String() string { - return fmt.Sprintf("%s: %T{Code: %v}", s.p, s, s.Code) -} - -// AndCodeExpr is a zero-length matcher that is considered a match if the -// code block returns true. -type AndCodeExpr struct { - p Pos - Code *CodeBlock - FuncIx int -} - -// NewAndCodeExpr creates a new and (&) code expression at the specified -// position. -func NewAndCodeExpr(p Pos) *AndCodeExpr { - return &AndCodeExpr{p: p} -} - -// Pos returns the starting position of the node. -func (a *AndCodeExpr) Pos() Pos { return a.p } - -// String returns the textual representation of a node. -func (a *AndCodeExpr) String() string { - return fmt.Sprintf("%s: %T{Code: %v}", a.p, a, a.Code) -} - -// NotCodeExpr is a zero-length matcher that is considered a match if the -// code block returns false. -type NotCodeExpr struct { - p Pos - Code *CodeBlock - FuncIx int -} - -// NewNotCodeExpr creates a new not (!) code expression at the specified -// position. -func NewNotCodeExpr(p Pos) *NotCodeExpr { - return &NotCodeExpr{p: p} -} - -// Pos returns the starting position of the node. -func (n *NotCodeExpr) Pos() Pos { return n.p } - -// String returns the textual representation of a node. -func (n *NotCodeExpr) String() string { - return fmt.Sprintf("%s: %T{Code: %v}", n.p, n, n.Code) -} - -// LitMatcher is a string literal matcher. The value to match may be a -// double-quoted string, a single-quoted single character, or a back-tick -// quoted raw string. -type LitMatcher struct { - posValue // can be str, rstr or char - IgnoreCase bool -} - -// NewLitMatcher creates a new literal matcher at the specified position and -// with the specified value. -func NewLitMatcher(p Pos, v string) *LitMatcher { - return &LitMatcher{posValue: posValue{p: p, Val: v}} -} - -// Pos returns the starting position of the node. -func (l *LitMatcher) Pos() Pos { return l.p } - -// String returns the textual representation of a node. -func (l *LitMatcher) String() string { - return fmt.Sprintf("%s: %T{Val: %q, IgnoreCase: %t}", l.p, l, l.Val, l.IgnoreCase) -} - -// CharClassMatcher is a character class matcher. The value to match must -// be one of the specified characters, in a range of characters, or in the -// Unicode classes of characters. -type CharClassMatcher struct { - posValue - IgnoreCase bool - Inverted bool - Chars []rune - Ranges []rune // pairs of low/high range - UnicodeClasses []string -} - -// NewCharClassMatcher creates a new character class matcher at the specified -// position and with the specified raw value. It parses the raw value into -// the list of characters, ranges and Unicode classes. -func NewCharClassMatcher(p Pos, raw string) *CharClassMatcher { - c := &CharClassMatcher{posValue: posValue{p: p, Val: raw}} - c.parse() - return c -} - -func (c *CharClassMatcher) parse() { - raw := c.Val - c.IgnoreCase = strings.HasSuffix(raw, "i") - if c.IgnoreCase { - raw = raw[:len(raw)-1] - } - - // "unquote" the character classes - raw = raw[1 : len(raw)-1] - if len(raw) == 0 { - return - } - - c.Inverted = raw[0] == '^' - if c.Inverted { - raw = raw[1:] - if len(raw) == 0 { - return - } - } - - // content of char class is necessarily valid, so escapes are correct - r := strings.NewReader(raw) - var chars []rune - var buf bytes.Buffer -outer: - for { - rn, _, err := r.ReadRune() - if err != nil { - break outer - } - - consumeN := 0 - switch rn { - case '\\': - rn, _, _ := r.ReadRune() - switch rn { - case ']': - chars = append(chars, rn) - continue - - case 'p': - rn, _, _ := r.ReadRune() - if rn == '{' { - buf.Reset() - for { - rn, _, _ := r.ReadRune() - if rn == '}' { - break - } - buf.WriteRune(rn) - } - c.UnicodeClasses = append(c.UnicodeClasses, buf.String()) - } else { - c.UnicodeClasses = append(c.UnicodeClasses, string(rn)) - } - continue - - case 'x': - consumeN = 2 - case 'u': - consumeN = 4 - case 'U': - consumeN = 8 - case '0', '1', '2', '3', '4', '5', '6', '7': - consumeN = 2 - } - - buf.Reset() - buf.WriteRune(rn) - for i := 0; i < consumeN; i++ { - rn, _, _ := r.ReadRune() - buf.WriteRune(rn) - } - rn, _, _, _ = strconv.UnquoteChar("\\"+buf.String(), 0) - chars = append(chars, rn) - - default: - chars = append(chars, rn) - } - } - - // extract ranges and chars - inRange, wasRange := false, false - for i, r := range chars { - if inRange { - c.Ranges = append(c.Ranges, r) - inRange = false - wasRange = true - continue - } - - if r == '-' && !wasRange && len(c.Chars) > 0 && i < len(chars)-1 { - inRange = true - wasRange = false - // start of range is the last Char added - c.Ranges = append(c.Ranges, c.Chars[len(c.Chars)-1]) - c.Chars = c.Chars[:len(c.Chars)-1] - continue - } - wasRange = false - c.Chars = append(c.Chars, r) - } -} - -// Pos returns the starting position of the node. -func (c *CharClassMatcher) Pos() Pos { return c.p } - -// String returns the textual representation of a node. -func (c *CharClassMatcher) String() string { - return fmt.Sprintf("%s: %T{Val: %q, IgnoreCase: %t, Inverted: %t}", - c.p, c, c.Val, c.IgnoreCase, c.Inverted) -} - -// AnyMatcher is a matcher that matches any character except end-of-file. -type AnyMatcher struct { - posValue -} - -// NewAnyMatcher creates a new any matcher at the specified position. The -// value is provided for completeness' sake, but it is always the dot. -func NewAnyMatcher(p Pos, v string) *AnyMatcher { - return &AnyMatcher{posValue{p, v}} -} - -// Pos returns the starting position of the node. -func (a *AnyMatcher) Pos() Pos { return a.p } - -// String returns the textual representation of a node. -func (a *AnyMatcher) String() string { - return fmt.Sprintf("%s: %T{Val: %q}", a.p, a, a.Val) -} - -// CodeBlock represents a code block. -type CodeBlock struct { - posValue -} - -// NewCodeBlock creates a new code block at the specified position and with -// the specified value. The value includes the outer braces. -func NewCodeBlock(p Pos, code string) *CodeBlock { - return &CodeBlock{posValue{p, code}} -} - -// Pos returns the starting position of the node. -func (c *CodeBlock) Pos() Pos { return c.p } - -// String returns the textual representation of a node. -func (c *CodeBlock) String() string { - return fmt.Sprintf("%s: %T{Val: %q}", c.p, c, c.Val) -} - -// Identifier represents an identifier. -type Identifier struct { - posValue -} - -// NewIdentifier creates a new identifier at the specified position and -// with the specified name. -func NewIdentifier(p Pos, name string) *Identifier { - return &Identifier{posValue{p: p, Val: name}} -} - -// Pos returns the starting position of the node. -func (i *Identifier) Pos() Pos { return i.p } - -// String returns the textual representation of a node. -func (i *Identifier) String() string { - return fmt.Sprintf("%s: %T{Val: %q}", i.p, i, i.Val) -} - -// StringLit represents a string literal. -type StringLit struct { - posValue -} - -// NewStringLit creates a new string literal at the specified position and -// with the specified value. -func NewStringLit(p Pos, val string) *StringLit { - return &StringLit{posValue{p: p, Val: val}} -} - -// Pos returns the starting position of the node. -func (s *StringLit) Pos() Pos { return s.p } - -// String returns the textual representation of a node. -func (s *StringLit) String() string { - return fmt.Sprintf("%s: %T{Val: %q}", s.p, s, s.Val) -} - -type posValue struct { - p Pos - Val string -} diff --git a/vendor/github.com/mna/pigeon/ast/ast_optimize.go b/vendor/github.com/mna/pigeon/ast/ast_optimize.go deleted file mode 100644 index 13721070be..0000000000 --- a/vendor/github.com/mna/pigeon/ast/ast_optimize.go +++ /dev/null @@ -1,454 +0,0 @@ -package ast - -import ( - "bytes" - "strconv" - "strings" -) - -type grammarOptimizer struct { - rule string - protectedRules map[string]struct{} - rules map[string]*Rule - ruleUsesRules map[string]map[string]struct{} - ruleUsedByRules map[string]map[string]struct{} - visitor func(expr Expression) Visitor - optimized bool -} - -func newGrammarOptimizer(protectedRules []string) *grammarOptimizer { - pr := make(map[string]struct{}, len(protectedRules)) - for _, nm := range protectedRules { - pr[nm] = struct{}{} - } - - r := grammarOptimizer{ - protectedRules: pr, - rules: make(map[string]*Rule), - ruleUsesRules: make(map[string]map[string]struct{}), - ruleUsedByRules: make(map[string]map[string]struct{}), - } - r.visitor = r.init - return &r -} - -// Visit is a generic Visitor to be used with Walk -// The actual function, which should be used during Walk -// is held in ruleRefOptimizer.visitor -func (r *grammarOptimizer) Visit(expr Expression) Visitor { - return r.visitor(expr) -} - -// init is a Visitor, which is used with the Walk function -// The purpose of this function is to initialize the reference -// maps rules, ruleUsesRules and ruleUsedByRules. -func (r *grammarOptimizer) init(expr Expression) Visitor { - switch expr := expr.(type) { - case *Rule: - // Keep track of current rule, which is processed - r.rule = expr.Name.Val - r.rules[expr.Name.Val] = expr - case *RuleRefExpr: - // Fill ruleUsesRules and ruleUsedByRules for every RuleRefExpr - set(r.ruleUsesRules, r.rule, expr.Name.Val) - set(r.ruleUsedByRules, expr.Name.Val, r.rule) - } - return r -} - -// Add element to map of maps, initialize the inner map -// if necessary. -func set(m map[string]map[string]struct{}, src, dst string) { - if _, ok := m[src]; !ok { - m[src] = make(map[string]struct{}) - } - m[src][dst] = struct{}{} -} - -// optimize is a Visitor, which is used with the Walk function -// The purpose of this function is to perform the actual optimizations. -// See Optimize for a detailed list of the performed optimizations. -func (r *grammarOptimizer) optimize(expr0 Expression) Visitor { - switch expr := expr0.(type) { - case *ActionExpr: - expr.Expr = r.optimizeRule(expr.Expr) - case *AndExpr: - expr.Expr = r.optimizeRule(expr.Expr) - case *ChoiceExpr: - expr.Alternatives = r.optimizeRules(expr.Alternatives) - - // Optimize choice nested in choice - for i := 0; i < len(expr.Alternatives); i++ { - if choice, ok := expr.Alternatives[i].(*ChoiceExpr); ok { - r.optimized = true - if i+1 < len(expr.Alternatives) { - expr.Alternatives = append(expr.Alternatives[:i], append(choice.Alternatives, expr.Alternatives[i+1:]...)...) - } else { - expr.Alternatives = append(expr.Alternatives[:i], choice.Alternatives...) - } - } - - // Combine sequence of single char LitMatcher to CharClassMatcher - if i > 0 { - l0, lok0 := expr.Alternatives[i-1].(*LitMatcher) - l1, lok1 := expr.Alternatives[i].(*LitMatcher) - c0, cok0 := expr.Alternatives[i-1].(*CharClassMatcher) - c1, cok1 := expr.Alternatives[i].(*CharClassMatcher) - - combined := false - - switch { - // Combine two LitMatcher to CharClassMatcher - // "a" / "b" => [ab] - case lok0 && lok1 && len([]rune(l0.Val)) == 1 && len([]rune(l1.Val)) == 1 && l0.IgnoreCase == l1.IgnoreCase: - combined = true - cm := CharClassMatcher{ - Chars: append([]rune(l0.Val), []rune(l1.Val)...), - IgnoreCase: l0.IgnoreCase, - posValue: l0.posValue, - } - expr.Alternatives[i-1] = &cm - - // Combine LitMatcher with CharClassMatcher - // "a" / [bc] => [abc] - case lok0 && cok1 && len([]rune(l0.Val)) == 1 && l0.IgnoreCase == c1.IgnoreCase && !c1.Inverted: - combined = true - c1.Chars = append(c1.Chars, []rune(l0.Val)...) - expr.Alternatives[i-1] = c1 - - // Combine CharClassMatcher with LitMatcher - // [ab] / "c" => [abc] - case cok0 && lok1 && len([]rune(l1.Val)) == 1 && c0.IgnoreCase == l1.IgnoreCase && !c0.Inverted: - combined = true - c0.Chars = append(c0.Chars, []rune(l1.Val)...) - - // Combine CharClassMatcher with CharClassMatcher - // [ab] / [cd] => [abcd] - case cok0 && cok1 && c0.IgnoreCase == c1.IgnoreCase && c0.Inverted == c1.Inverted: - combined = true - c0.Chars = append(c0.Chars, c1.Chars...) - c0.Ranges = append(c0.Ranges, c1.Ranges...) - c0.UnicodeClasses = append(c0.UnicodeClasses, c1.UnicodeClasses...) - } - - // If one of the optimizations was applied, remove the second element from Alternatives - if combined { - r.optimized = true - if i+1 < len(expr.Alternatives) { - expr.Alternatives = append(expr.Alternatives[:i], expr.Alternatives[i+1:]...) - } else { - expr.Alternatives = expr.Alternatives[:i] - } - } - } - } - - case *Grammar: - // Reset optimized at the start of each Walk. - r.optimized = false - for i := 0; i < len(expr.Rules); i++ { - rule := expr.Rules[i] - // Remove Rule, if it is no longer used by any other Rule and it is not the first Rule. - _, used := r.ruleUsedByRules[rule.Name.Val] - _, protected := r.protectedRules[rule.Name.Val] - if !used && !protected { - expr.Rules = append(expr.Rules[:i], expr.Rules[i+1:]...) - r.optimized = true - continue - } - } - case *LabeledExpr: - expr.Expr = r.optimizeRule(expr.Expr) - case *NotExpr: - expr.Expr = r.optimizeRule(expr.Expr) - case *OneOrMoreExpr: - expr.Expr = r.optimizeRule(expr.Expr) - case *Rule: - r.rule = expr.Name.Val - case *SeqExpr: - expr.Exprs = r.optimizeRules(expr.Exprs) - - for i := 0; i < len(expr.Exprs); i++ { - // Optimize nested sequences - if seq, ok := expr.Exprs[i].(*SeqExpr); ok { - r.optimized = true - if i+1 < len(expr.Exprs) { - expr.Exprs = append(expr.Exprs[:i], append(seq.Exprs, expr.Exprs[i+1:]...)...) - } else { - expr.Exprs = append(expr.Exprs[:i], seq.Exprs...) - } - } - - // Combine sequence of LitMatcher - if i > 0 { - l0, ok0 := expr.Exprs[i-1].(*LitMatcher) - l1, ok1 := expr.Exprs[i].(*LitMatcher) - if ok0 && ok1 && l0.IgnoreCase == l1.IgnoreCase { - r.optimized = true - l0.Val += l1.Val - expr.Exprs[i-1] = l0 - if i+1 < len(expr.Exprs) { - expr.Exprs = append(expr.Exprs[:i], expr.Exprs[i+1:]...) - } else { - expr.Exprs = expr.Exprs[:i] - } - } - } - } - - case *ZeroOrMoreExpr: - expr.Expr = r.optimizeRule(expr.Expr) - case *ZeroOrOneExpr: - expr.Expr = r.optimizeRule(expr.Expr) - } - return r -} - -func (r *grammarOptimizer) optimizeRules(exprs []Expression) []Expression { - for i := 0; i < len(exprs); i++ { - exprs[i] = r.optimizeRule(exprs[i]) - } - return exprs -} - -func (r *grammarOptimizer) optimizeRule(expr Expression) Expression { - // Optimize RuleRefExpr - if ruleRef, ok := expr.(*RuleRefExpr); ok { - if _, ok := r.ruleUsesRules[ruleRef.Name.Val]; !ok { - r.optimized = true - delete(r.ruleUsedByRules[ruleRef.Name.Val], r.rule) - if len(r.ruleUsedByRules[ruleRef.Name.Val]) == 0 { - delete(r.ruleUsedByRules, ruleRef.Name.Val) - } - delete(r.ruleUsesRules[r.rule], ruleRef.Name.Val) - if len(r.ruleUsesRules[r.rule]) == 0 { - delete(r.ruleUsesRules, r.rule) - } - // TODO: Check if reference exists, otherwise raise an error, which reference is missing! - return cloneExpr(r.rules[ruleRef.Name.Val].Expr) - } - } - - // Remove Choices with only one Alternative left - if choice, ok := expr.(*ChoiceExpr); ok { - if len(choice.Alternatives) == 1 { - r.optimized = true - return choice.Alternatives[0] - } - } - - // Remove Sequence with only one Expression - if seq, ok := expr.(*SeqExpr); ok { - if len(seq.Exprs) == 1 { - r.optimized = true - return seq.Exprs[0] - } - } - - return expr -} - -// cloneExpr takes an Expression and deep clones it (including all children) -// This is necessary because referenced Rules are denormalized and therefore -// have to become independent from their original Expression -func cloneExpr(expr Expression) Expression { - switch expr := expr.(type) { - case *ActionExpr: - return &ActionExpr{ - Code: expr.Code, - Expr: cloneExpr(expr.Expr), - FuncIx: expr.FuncIx, - p: expr.p, - } - case *AndExpr: - return &AndExpr{ - Expr: cloneExpr(expr.Expr), - p: expr.p, - } - case *AndCodeExpr: - return &AndCodeExpr{ - Code: expr.Code, - FuncIx: expr.FuncIx, - p: expr.p, - } - case *CharClassMatcher: - return &CharClassMatcher{ - Chars: append([]rune{}, expr.Chars...), - IgnoreCase: expr.IgnoreCase, - Inverted: expr.Inverted, - posValue: expr.posValue, - Ranges: append([]rune{}, expr.Ranges...), - UnicodeClasses: append([]string{}, expr.UnicodeClasses...), - } - case *ChoiceExpr: - alts := make([]Expression, 0, len(expr.Alternatives)) - for i := 0; i < len(expr.Alternatives); i++ { - alts = append(alts, cloneExpr(expr.Alternatives[i])) - } - return &ChoiceExpr{ - Alternatives: alts, - p: expr.p, - } - case *LabeledExpr: - return &LabeledExpr{ - Expr: cloneExpr(expr.Expr), - Label: expr.Label, - p: expr.p, - } - case *NotExpr: - return &NotExpr{ - Expr: cloneExpr(expr.Expr), - p: expr.p, - } - case *NotCodeExpr: - return &NotCodeExpr{ - Code: expr.Code, - FuncIx: expr.FuncIx, - p: expr.p, - } - case *OneOrMoreExpr: - return &OneOrMoreExpr{ - Expr: cloneExpr(expr.Expr), - p: expr.p, - } - case *SeqExpr: - exprs := make([]Expression, 0, len(expr.Exprs)) - for i := 0; i < len(expr.Exprs); i++ { - exprs = append(exprs, cloneExpr(expr.Exprs[i])) - } - return &SeqExpr{ - Exprs: exprs, - p: expr.p, - } - case *StateCodeExpr: - return &StateCodeExpr{ - p: expr.p, - Code: expr.Code, - FuncIx: expr.FuncIx, - } - case *ZeroOrMoreExpr: - return &ZeroOrMoreExpr{ - Expr: cloneExpr(expr.Expr), - p: expr.p, - } - case *ZeroOrOneExpr: - return &ZeroOrOneExpr{ - Expr: expr.Expr, - p: expr.p, - } - } - return expr -} - -// cleanupCharClassMatcher is a Visitor, which is used with the Walk function -// The purpose of this function is to cleanup the redundancies created by the -// optimize Visitor. This includes to remove redundant entries in Chars, Ranges -// and UnicodeClasses of the given CharClassMatcher as well as regenerating the -// correct content for the Val field (string representation of the CharClassMatcher) -func (r *grammarOptimizer) cleanupCharClassMatcher(expr0 Expression) Visitor { - // We are only interested in nodes of type *CharClassMatcher - if chr, ok := expr0.(*CharClassMatcher); ok { - // Remove redundancies in Chars - chars := make([]rune, 0, len(chr.Chars)) - charsMap := make(map[rune]struct{}) - for _, c := range chr.Chars { - if _, ok := charsMap[c]; !ok { - charsMap[c] = struct{}{} - chars = append(chars, c) - } - } - if len(chars) > 0 { - chr.Chars = chars - } else { - chr.Chars = nil - } - - // Remove redundancies in Ranges - ranges := make([]rune, 0, len(chr.Ranges)) - rangesMap := make(map[string]struct{}) - for i := 0; i < len(chr.Ranges); i += 2 { - rangeKey := string(chr.Ranges[i]) + "-" + string(chr.Ranges[i+1]) - if _, ok := rangesMap[rangeKey]; !ok { - rangesMap[rangeKey] = struct{}{} - ranges = append(ranges, chr.Ranges[i], chr.Ranges[i+1]) - } - } - if len(ranges) > 0 { - chr.Ranges = ranges - } else { - chr.Ranges = nil - } - - // Remove redundancies in UnicodeClasses - unicodeClasses := make([]string, 0, len(chr.UnicodeClasses)) - unicodeClassesMap := make(map[string]struct{}) - for _, u := range chr.UnicodeClasses { - if _, ok := unicodeClassesMap[u]; !ok { - unicodeClassesMap[u] = struct{}{} - unicodeClasses = append(unicodeClasses, u) - } - } - if len(unicodeClasses) > 0 { - chr.UnicodeClasses = unicodeClasses - } else { - chr.UnicodeClasses = nil - } - - // Regenerate the content for Val - var val bytes.Buffer - val.WriteString("[") - if chr.Inverted { - val.WriteString("^") - } - for _, c := range chr.Chars { - val.WriteString(escapeRune(c)) - } - for i := 0; i < len(chr.Ranges); i += 2 { - val.WriteString(escapeRune(chr.Ranges[i])) - val.WriteString("-") - val.WriteString(escapeRune(chr.Ranges[i+1])) - } - for _, u := range chr.UnicodeClasses { - val.WriteString("\\p" + u) - } - val.WriteString("]") - if chr.IgnoreCase { - val.WriteString("i") - } - chr.posValue.Val = val.String() - } - return r -} - -func escapeRune(r rune) string { - return strings.Trim(strconv.QuoteRune(r), `'`) -} - -// Optimize walks a given grammar and optimizes the grammar in regards -// of parsing performance. This is done with several optimizations: -// * removal of unreferenced rules -// * replace rule references with a copy of the referenced Rule, if the -// referenced rule it self has no references. -// * resolve nested choice expressions -// * resolve choice expressions with only one alternative -// * resolve nested sequences expression -// * resolve sequence expressions with only one element -// * combine character class matcher and literal matcher, where possible -func Optimize(g *Grammar, alternateEntrypoints ...string) { - entrypoints := alternateEntrypoints - if len(g.Rules) > 0 { - entrypoints = append(entrypoints, g.Rules[0].Name.Val) - } - - r := newGrammarOptimizer(entrypoints) - Walk(r, g) - - r.visitor = r.optimize - r.optimized = true - for r.optimized { - Walk(r, g) - } - - r.visitor = r.cleanupCharClassMatcher - Walk(r, g) -} diff --git a/vendor/github.com/mna/pigeon/ast/ast_walk.go b/vendor/github.com/mna/pigeon/ast/ast_walk.go deleted file mode 100644 index 862d7d8dd3..0000000000 --- a/vendor/github.com/mna/pigeon/ast/ast_walk.go +++ /dev/null @@ -1,87 +0,0 @@ -package ast - -import "fmt" - -// A Visitor implements a Visit method, which is invoked for each Expression -// encountered by Walk. -// If the result visitor w is not nil, Walk visits each of the children -// of Expression with the visitor w, followed by a call of w.Visit(nil). -type Visitor interface { - Visit(expr Expression) (w Visitor) -} - -// Walk traverses an AST in depth-first order: It starts by calling -// v.Visit(expr); Expression must not be nil. If the visitor w returned by -// v.Visit(expr) is not nil, Walk is invoked recursively with visitor -// w for each of the non-nil children of Expression, followed by a call of -// w.Visit(nil). -// -func Walk(v Visitor, expr Expression) { - if v = v.Visit(expr); v == nil { - return - } - - switch expr := expr.(type) { - case *ActionExpr: - Walk(v, expr.Expr) - case *AndCodeExpr: - // Nothing to do - case *AndExpr: - Walk(v, expr.Expr) - case *AnyMatcher: - // Nothing to do - case *CharClassMatcher: - // Nothing to do - case *ChoiceExpr: - for _, e := range expr.Alternatives { - Walk(v, e) - } - case *Grammar: - for _, e := range expr.Rules { - Walk(v, e) - } - case *LabeledExpr: - Walk(v, expr.Expr) - case *LitMatcher: - // Nothing to do - case *NotCodeExpr: - // Nothing to do - case *NotExpr: - Walk(v, expr.Expr) - case *OneOrMoreExpr: - Walk(v, expr.Expr) - case *Rule: - Walk(v, expr.Expr) - case *RuleRefExpr: - // Nothing to do - case *SeqExpr: - for _, e := range expr.Exprs { - Walk(v, e) - } - case *StateCodeExpr: - // Nothing to do - case *ZeroOrMoreExpr: - Walk(v, expr.Expr) - case *ZeroOrOneExpr: - Walk(v, expr.Expr) - default: - panic(fmt.Sprintf("unknown expression type %T", expr)) - } -} - -type inspector func(Expression) bool - -func (f inspector) Visit(expr Expression) Visitor { - if f(expr) { - return f - } - return nil -} - -// Inspect traverses an AST in depth-first order: It starts by calling -// f(expr); expr must not be nil. If f returns true, Inspect invokes f -// recursively for each of the non-nil children of expr, followed by a -// call of f(nil). -func Inspect(expr Expression, f func(Expression) bool) { - Walk(inspector(f), expr) -} diff --git a/vendor/github.com/mna/pigeon/builder/builder.go b/vendor/github.com/mna/pigeon/builder/builder.go deleted file mode 100644 index 2f5063509a..0000000000 --- a/vendor/github.com/mna/pigeon/builder/builder.go +++ /dev/null @@ -1,812 +0,0 @@ -// Package builder generates the parser code for a given grammar. It makes -// no attempt to verify the correctness of the grammar. -package builder - -import ( - "bytes" - "fmt" - "io" - "strconv" - "strings" - "text/template" - "unicode" - - "regexp" - - "github.com/mna/pigeon/ast" -) - -const codeGeneratedComment = "// Code generated by pigeon; DO NOT EDIT.\n\n" - -// generated function templates -var ( - onFuncTemplate = `func (%s *current) %s(%s) (interface{}, error) { -%s -} -` - onPredFuncTemplate = `func (%s *current) %s(%s) (bool, error) { -%s -} -` - onStateFuncTemplate = `func (%s *current) %s(%s) (error) { -%s -} -` - callFuncTemplate = `func (p *parser) call%s() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.%[1]s(%s) -} -` - callPredFuncTemplate = `func (p *parser) call%s() (bool, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.%[1]s(%s) -} -` - callStateFuncTemplate = `func (p *parser) call%s() error { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.%[1]s(%s) -} -` -) - -// Option is a function that can set an option on the builder. It returns -// the previous setting as an Option. -type Option func(*builder) Option - -// ReceiverName returns an option that specifies the receiver name to -// use for the current struct (which is the struct on which all code blocks -// except the initializer are generated). -func ReceiverName(nm string) Option { - return func(b *builder) Option { - prev := b.recvName - b.recvName = nm - return ReceiverName(prev) - } -} - -// Optimize returns an option that specifies the optimize option -// If optimize is true, the Debug and Memoize code is completely -// removed from the resulting parser -func Optimize(optimize bool) Option { - return func(b *builder) Option { - prev := b.optimize - b.optimize = optimize - return Optimize(prev) - } -} - -// Nolint returns an option that specifies the nolint option -// If nolint is true, special '// nolint: ...' comments are added -// to the generated parser to suppress warnings by gometalinter. -func Nolint(nolint bool) Option { - return func(b *builder) Option { - prev := b.nolint - b.nolint = nolint - return Optimize(prev) - } -} - -// BasicLatinLookupTable returns an option that specifies the basicLatinLookup option -// If basicLatinLookup is true, a lookup slice for the first 128 chars of -// the Unicode table (Basic Latin) is generated for each CharClassMatcher -// to increase the character matching. -func BasicLatinLookupTable(basicLatinLookupTable bool) Option { - return func(b *builder) Option { - prev := b.basicLatinLookupTable - b.basicLatinLookupTable = basicLatinLookupTable - return BasicLatinLookupTable(prev) - } -} - -// BuildParser builds the PEG parser using the provider grammar. The code is -// written to the specified w. -func BuildParser(w io.Writer, g *ast.Grammar, opts ...Option) error { - b := &builder{w: w, recvName: "c"} - b.setOptions(opts) - return b.buildParser(g) -} - -type builder struct { - w io.Writer - err error - - // options - recvName string - optimize bool - basicLatinLookupTable bool - globalState bool - nolint bool - - ruleName string - exprIndex int - argsStack [][]string - - rangeTable bool -} - -func (b *builder) setOptions(opts []Option) { - for _, opt := range opts { - opt(b) - } -} - -func (b *builder) buildParser(g *ast.Grammar) error { - b.writeInit(g.Init) - b.writeGrammar(g) - - for _, rule := range g.Rules { - b.writeRuleCode(rule) - } - b.writeStaticCode() - - return b.err -} - -func (b *builder) writeInit(init *ast.CodeBlock) { - if init == nil { - return - } - - // remove opening and closing braces - val := codeGeneratedComment + init.Val[1:len(init.Val)-1] - b.writelnf("%s", val) -} - -func (b *builder) writeGrammar(g *ast.Grammar) { - // transform the ast grammar to the self-contained, no dependency version - // of the parser-generator grammar. - b.writelnf("var g = &grammar {") - b.writelnf("\trules: []*rule{") - for _, r := range g.Rules { - b.writeRule(r) - } - b.writelnf("\t},") - b.writelnf("}") -} - -func (b *builder) writeRule(r *ast.Rule) { - if r == nil || r.Name == nil { - return - } - - b.exprIndex = 0 - b.ruleName = r.Name.Val - - b.writelnf("{") - b.writelnf("\tname: %q,", r.Name.Val) - if r.DisplayName != nil && r.DisplayName.Val != "" { - b.writelnf("\tdisplayName: %q,", r.DisplayName.Val) - } - pos := r.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writef("\texpr: ") - b.writeExpr(r.Expr) - b.writelnf("},") -} - -func (b *builder) writeExpr(expr ast.Expression) { - b.exprIndex++ - switch expr := expr.(type) { - case *ast.ActionExpr: - b.writeActionExpr(expr) - case *ast.AndCodeExpr: - b.writeAndCodeExpr(expr) - case *ast.AndExpr: - b.writeAndExpr(expr) - case *ast.AnyMatcher: - b.writeAnyMatcher(expr) - case *ast.CharClassMatcher: - b.writeCharClassMatcher(expr) - case *ast.ChoiceExpr: - b.writeChoiceExpr(expr) - case *ast.LabeledExpr: - b.writeLabeledExpr(expr) - case *ast.LitMatcher: - b.writeLitMatcher(expr) - case *ast.NotCodeExpr: - b.writeNotCodeExpr(expr) - case *ast.NotExpr: - b.writeNotExpr(expr) - case *ast.OneOrMoreExpr: - b.writeOneOrMoreExpr(expr) - case *ast.RecoveryExpr: - b.writeRecoveryExpr(expr) - case *ast.RuleRefExpr: - b.writeRuleRefExpr(expr) - case *ast.SeqExpr: - b.writeSeqExpr(expr) - case *ast.StateCodeExpr: - b.writeStateCodeExpr(expr) - case *ast.ThrowExpr: - b.writeThrowExpr(expr) - case *ast.ZeroOrMoreExpr: - b.writeZeroOrMoreExpr(expr) - case *ast.ZeroOrOneExpr: - b.writeZeroOrOneExpr(expr) - default: - b.err = fmt.Errorf("builder: unknown expression type %T", expr) - } -} - -func (b *builder) writeActionExpr(act *ast.ActionExpr) { - if act == nil { - b.writelnf("nil,") - return - } - if act.FuncIx == 0 { - act.FuncIx = b.exprIndex - } - b.writelnf("&actionExpr{") - pos := act.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writelnf("\trun: (*parser).call%s,", b.funcName(act.FuncIx)) - b.writef("\texpr: ") - b.writeExpr(act.Expr) - b.writelnf("},") -} - -func (b *builder) writeAndCodeExpr(and *ast.AndCodeExpr) { - if and == nil { - b.writelnf("nil,") - return - } - b.writelnf("&andCodeExpr{") - pos := and.Pos() - if and.FuncIx == 0 { - and.FuncIx = b.exprIndex - } - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writelnf("\trun: (*parser).call%s,", b.funcName(and.FuncIx)) - b.writelnf("},") -} - -func (b *builder) writeAndExpr(and *ast.AndExpr) { - if and == nil { - b.writelnf("nil,") - return - } - b.writelnf("&andExpr{") - pos := and.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writef("\texpr: ") - b.writeExpr(and.Expr) - b.writelnf("},") -} - -func (b *builder) writeAnyMatcher(any *ast.AnyMatcher) { - if any == nil { - b.writelnf("nil,") - return - } - b.writelnf("&anyMatcher{") - pos := any.Pos() - b.writelnf("\tline: %d, col: %d, offset: %d,", pos.Line, pos.Col, pos.Off) - b.writelnf("},") -} - -func (b *builder) writeCharClassMatcher(ch *ast.CharClassMatcher) { - if ch == nil { - b.writelnf("nil,") - return - } - b.writelnf("&charClassMatcher{") - pos := ch.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writelnf("\tval: %q,", ch.Val) - if len(ch.Chars) > 0 { - b.writef("\tchars: []rune{") - for _, rn := range ch.Chars { - if ch.IgnoreCase { - b.writef("%q,", unicode.ToLower(rn)) - } else { - b.writef("%q,", rn) - } - } - b.writelnf("},") - } - if len(ch.Ranges) > 0 { - b.writef("\tranges: []rune{") - for _, rn := range ch.Ranges { - if ch.IgnoreCase { - b.writef("%q,", unicode.ToLower(rn)) - } else { - b.writef("%q,", rn) - } - } - b.writelnf("},") - } - if len(ch.UnicodeClasses) > 0 { - b.rangeTable = true - b.writef("\tclasses: []*unicode.RangeTable{") - for _, cl := range ch.UnicodeClasses { - b.writef("rangeTable(%q),", cl) - } - b.writelnf("},") - } - if b.basicLatinLookupTable { - b.writelnf("\tbasicLatinChars: %#v,", BasicLatinLookup(ch.Chars, ch.Ranges, ch.UnicodeClasses, ch.IgnoreCase)) - } - b.writelnf("\tignoreCase: %t,", ch.IgnoreCase) - b.writelnf("\tinverted: %t,", ch.Inverted) - b.writelnf("},") -} - -// BasicLatinLookup calculates the decision results for the first 256 characters of the UTF-8 character -// set for a given set of chars, ranges and unicodeClasses to speedup the CharClassMatcher. -func BasicLatinLookup(chars, ranges []rune, unicodeClasses []string, ignoreCase bool) (basicLatinChars [128]bool) { - for _, rn := range chars { - if rn < 128 { - basicLatinChars[rn] = true - if ignoreCase { - if unicode.IsLower(rn) { - basicLatinChars[unicode.ToUpper(rn)] = true - } else { - basicLatinChars[unicode.ToLower(rn)] = true - } - } - } - } - for i := 0; i < len(ranges); i += 2 { - if ranges[i] < 128 { - for j := ranges[i]; j < 128 && j <= ranges[i+1]; j++ { - basicLatinChars[j] = true - if ignoreCase { - if unicode.IsLower(j) { - basicLatinChars[unicode.ToUpper(j)] = true - } else { - basicLatinChars[unicode.ToLower(j)] = true - } - } - } - } - } - for _, cl := range unicodeClasses { - rt := rangeTable(cl) - for r := rune(0); r < 128; r++ { - if unicode.Is(rt, r) { - basicLatinChars[r] = true - } - } - } - return -} - -func (b *builder) writeChoiceExpr(ch *ast.ChoiceExpr) { - if ch == nil { - b.writelnf("nil,") - return - } - b.writelnf("&choiceExpr{") - pos := ch.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - if len(ch.Alternatives) > 0 { - b.writelnf("\talternatives: []interface{}{") - for _, alt := range ch.Alternatives { - b.writeExpr(alt) - } - b.writelnf("\t},") - } - b.writelnf("},") -} - -func (b *builder) writeLabeledExpr(lab *ast.LabeledExpr) { - if lab == nil { - b.writelnf("nil,") - return - } - b.writelnf("&labeledExpr{") - pos := lab.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - if lab.Label != nil && lab.Label.Val != "" { - b.writelnf("\tlabel: %q,", lab.Label.Val) - } - b.writef("\texpr: ") - b.writeExpr(lab.Expr) - b.writelnf("},") -} - -func (b *builder) writeLitMatcher(lit *ast.LitMatcher) { - if lit == nil { - b.writelnf("nil,") - return - } - b.writelnf("&litMatcher{") - pos := lit.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - if lit.IgnoreCase { - b.writelnf("\tval: %q,", strings.ToLower(lit.Val)) - } else { - b.writelnf("\tval: %q,", lit.Val) - } - b.writelnf("\tignoreCase: %t,", lit.IgnoreCase) - b.writelnf("},") -} - -func (b *builder) writeNotCodeExpr(not *ast.NotCodeExpr) { - if not == nil { - b.writelnf("nil,") - return - } - b.writelnf("¬CodeExpr{") - pos := not.Pos() - if not.FuncIx == 0 { - not.FuncIx = b.exprIndex - } - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writelnf("\trun: (*parser).call%s,", b.funcName(not.FuncIx)) - b.writelnf("},") -} - -func (b *builder) writeNotExpr(not *ast.NotExpr) { - if not == nil { - b.writelnf("nil,") - return - } - b.writelnf("¬Expr{") - pos := not.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writef("\texpr: ") - b.writeExpr(not.Expr) - b.writelnf("},") -} - -func (b *builder) writeOneOrMoreExpr(one *ast.OneOrMoreExpr) { - if one == nil { - b.writelnf("nil,") - return - } - b.writelnf("&oneOrMoreExpr{") - pos := one.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writef("\texpr: ") - b.writeExpr(one.Expr) - b.writelnf("},") -} - -func (b *builder) writeRecoveryExpr(recover *ast.RecoveryExpr) { - if recover == nil { - b.writelnf("nil,") - return - } - b.writelnf("&recoveryExpr{") - pos := recover.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - - b.writef("\texpr: ") - b.writeExpr(recover.Expr) - b.writef("\trecoverExpr: ") - b.writeExpr(recover.RecoverExpr) - b.writelnf("\tfailureLabel: []string{") - for _, label := range recover.Labels { - b.writelnf("%q,", label) - } - b.writelnf("\t},") - b.writelnf("},") -} - -func (b *builder) writeRuleRefExpr(ref *ast.RuleRefExpr) { - if ref == nil { - b.writelnf("nil,") - return - } - b.writelnf("&ruleRefExpr{") - pos := ref.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - if ref.Name != nil && ref.Name.Val != "" { - b.writelnf("\tname: %q,", ref.Name.Val) - } - b.writelnf("},") -} - -func (b *builder) writeSeqExpr(seq *ast.SeqExpr) { - if seq == nil { - b.writelnf("nil,") - return - } - b.writelnf("&seqExpr{") - pos := seq.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - if len(seq.Exprs) > 0 { - b.writelnf("\texprs: []interface{}{") - for _, e := range seq.Exprs { - b.writeExpr(e) - } - b.writelnf("\t},") - } - b.writelnf("},") -} - -func (b *builder) writeStateCodeExpr(state *ast.StateCodeExpr) { - if state == nil { - b.writelnf("nil,") - return - } - b.globalState = true - b.writelnf("&stateCodeExpr{") - pos := state.Pos() - if state.FuncIx == 0 { - state.FuncIx = b.exprIndex - } - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writelnf("\trun: (*parser).call%s,", b.funcName(state.FuncIx)) - b.writelnf("},") -} - -func (b *builder) writeThrowExpr(throw *ast.ThrowExpr) { - if throw == nil { - b.writelnf("nil,") - return - } - b.writelnf("&throwExpr{") - pos := throw.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writelnf("\tlabel: %q,", throw.Label) - b.writelnf("},") -} - -func (b *builder) writeZeroOrMoreExpr(zero *ast.ZeroOrMoreExpr) { - if zero == nil { - b.writelnf("nil,") - return - } - b.writelnf("&zeroOrMoreExpr{") - pos := zero.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writef("\texpr: ") - b.writeExpr(zero.Expr) - b.writelnf("},") -} - -func (b *builder) writeZeroOrOneExpr(zero *ast.ZeroOrOneExpr) { - if zero == nil { - b.writelnf("nil,") - return - } - b.writelnf("&zeroOrOneExpr{") - pos := zero.Pos() - b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) - b.writef("\texpr: ") - b.writeExpr(zero.Expr) - b.writelnf("},") -} - -func (b *builder) writeRuleCode(rule *ast.Rule) { - if rule == nil || rule.Name == nil { - return - } - - // keep trace of the current rule, as the code blocks are created - // in functions named "on<#ExprIndex>". - b.ruleName = rule.Name.Val - b.pushArgsSet() - b.writeExprCode(rule.Expr) - b.popArgsSet() -} - -func (b *builder) pushArgsSet() { - b.argsStack = append(b.argsStack, nil) -} - -func (b *builder) popArgsSet() { - b.argsStack = b.argsStack[:len(b.argsStack)-1] -} - -func (b *builder) addArg(arg *ast.Identifier) { - if arg == nil { - return - } - ix := len(b.argsStack) - 1 - b.argsStack[ix] = append(b.argsStack[ix], arg.Val) -} - -func (b *builder) writeExprCode(expr ast.Expression) { - switch expr := expr.(type) { - case *ast.ActionExpr: - b.writeExprCode(expr.Expr) - b.writeActionExprCode(expr) - - case *ast.AndCodeExpr: - b.writeAndCodeExprCode(expr) - - case *ast.LabeledExpr: - b.addArg(expr.Label) - b.pushArgsSet() - b.writeExprCode(expr.Expr) - b.popArgsSet() - - case *ast.NotCodeExpr: - b.writeNotCodeExprCode(expr) - - case *ast.AndExpr: - b.pushArgsSet() - b.writeExprCode(expr.Expr) - b.popArgsSet() - - case *ast.ChoiceExpr: - for _, alt := range expr.Alternatives { - b.pushArgsSet() - b.writeExprCode(alt) - b.popArgsSet() - } - - case *ast.NotExpr: - b.pushArgsSet() - b.writeExprCode(expr.Expr) - b.popArgsSet() - - case *ast.OneOrMoreExpr: - b.pushArgsSet() - b.writeExprCode(expr.Expr) - b.popArgsSet() - - case *ast.RecoveryExpr: - b.pushArgsSet() - b.writeExprCode(expr.Expr) - b.writeExprCode(expr.RecoverExpr) - b.popArgsSet() - - case *ast.SeqExpr: - for _, sub := range expr.Exprs { - b.writeExprCode(sub) - } - - case *ast.StateCodeExpr: - b.writeStateCodeExprCode(expr) - - case *ast.ZeroOrMoreExpr: - b.pushArgsSet() - b.writeExprCode(expr.Expr) - b.popArgsSet() - - case *ast.ZeroOrOneExpr: - b.pushArgsSet() - b.writeExprCode(expr.Expr) - b.popArgsSet() - } -} - -func (b *builder) writeActionExprCode(act *ast.ActionExpr) { - if act == nil { - return - } - if act.FuncIx > 0 { - b.writeFunc(act.FuncIx, act.Code, callFuncTemplate, onFuncTemplate) - act.FuncIx = 0 // already rendered, prevent duplicates - } -} - -func (b *builder) writeAndCodeExprCode(and *ast.AndCodeExpr) { - if and == nil { - return - } - if and.FuncIx > 0 { - b.writeFunc(and.FuncIx, and.Code, callPredFuncTemplate, onPredFuncTemplate) - and.FuncIx = 0 // already rendered, prevent duplicates - } -} - -func (b *builder) writeNotCodeExprCode(not *ast.NotCodeExpr) { - if not == nil { - return - } - if not.FuncIx > 0 { - b.writeFunc(not.FuncIx, not.Code, callPredFuncTemplate, onPredFuncTemplate) - not.FuncIx = 0 // already rendered, prevent duplicates - } -} - -func (b *builder) writeStateCodeExprCode(state *ast.StateCodeExpr) { - if state == nil { - return - } - if state.FuncIx > 0 { - b.writeFunc(state.FuncIx, state.Code, callStateFuncTemplate, onStateFuncTemplate) - state.FuncIx = 0 // already rendered, prevent duplicates - } -} - -func (b *builder) writeFunc(funcIx int, code *ast.CodeBlock, callTpl, funcTpl string) { - if code == nil { - return - } - val := strings.TrimSpace(code.Val)[1 : len(code.Val)-1] - if len(val) > 0 && val[0] == '\n' { - val = val[1:] - } - if len(val) > 0 && val[len(val)-1] == '\n' { - val = val[:len(val)-1] - } - var args bytes.Buffer - ix := len(b.argsStack) - 1 - if ix >= 0 { - for i, arg := range b.argsStack[ix] { - if i > 0 { - args.WriteString(", ") - } - args.WriteString(arg) - } - } - if args.Len() > 0 { - args.WriteString(" interface{}") - } - - fnNm := b.funcName(funcIx) - b.writelnf(funcTpl, b.recvName, fnNm, args.String(), val) - - args.Reset() - if ix >= 0 { - for i, arg := range b.argsStack[ix] { - if i > 0 { - args.WriteString(", ") - } - args.WriteString(fmt.Sprintf(`stack[%q]`, arg)) - } - } - b.writelnf(callTpl, fnNm, args.String()) -} - -func (b *builder) writeStaticCode() { - buffer := bytes.NewBufferString("") - params := struct { - Optimize bool - BasicLatinLookupTable bool - GlobalState bool - Nolint bool - }{ - Optimize: b.optimize, - BasicLatinLookupTable: b.basicLatinLookupTable, - GlobalState: b.globalState, - Nolint: b.nolint, - } - t := template.Must(template.New("static_code").Parse(staticCode)) - - err := t.Execute(buffer, params) - if err != nil { - // This is very unlikely to ever happen - panic("executing template: " + err.Error()) - } - - // Clean the ==template== comments from the generated parser - lines := strings.Split(buffer.String(), "\n") - buffer.Reset() - re := regexp.MustCompile(`^\s*//\s*(==template==\s*)+$`) - reLineEnd := regexp.MustCompile(`//\s*==template==\s*$`) - for _, line := range lines { - if !re.MatchString(line) { - line = reLineEnd.ReplaceAllString(line, "") - _, err := buffer.WriteString(line + "\n") - if err != nil { - // This is very unlikely to ever happen - panic("unable to write to byte buffer: " + err.Error()) - } - } - } - - b.writeln(buffer.String()) - if b.rangeTable { - b.writeln(rangeTable0) - } -} - -func (b *builder) funcName(ix int) string { - return "on" + b.ruleName + strconv.Itoa(ix) -} - -func (b *builder) writef(f string, args ...interface{}) { - if b.err == nil { - _, b.err = fmt.Fprintf(b.w, f, args...) - } -} - -func (b *builder) writelnf(f string, args ...interface{}) { - b.writef(f+"\n", args...) -} - -func (b *builder) writeln(f string) { - if b.err == nil { - _, b.err = fmt.Fprint(b.w, f+"\n") - } -} diff --git a/vendor/github.com/mna/pigeon/builder/generated_static_code.go b/vendor/github.com/mna/pigeon/builder/generated_static_code.go deleted file mode 100644 index e1d8455404..0000000000 --- a/vendor/github.com/mna/pigeon/builder/generated_static_code.go +++ /dev/null @@ -1,1442 +0,0 @@ -// Code generated by static_code_generator with go generate; DO NOT EDIT. - -package builder - -var staticCode = ` -var ( - // errNoRule is returned when the grammar to parse has no rule. - errNoRule = errors.New("grammar has no rule") - - // errInvalidEntrypoint is returned when the specified entrypoint rule - // does not exit. - errInvalidEntrypoint = errors.New("invalid entrypoint") - - // errInvalidEncoding is returned when the source is not properly - // utf8-encoded. - errInvalidEncoding = errors.New("invalid encoding") - - // errMaxExprCnt is used to signal that the maximum number of - // expressions have been parsed. - errMaxExprCnt = errors.New("max number of expresssions parsed") -) - -// Option is a function that can set an option on the parser. It returns -// the previous setting as an Option. -type Option func(*parser) Option - -// MaxExpressions creates an Option to stop parsing after the provided -// number of expressions have been parsed, if the value is 0 then the parser will -// parse for as many steps as needed (possibly an infinite number). -// -// The default for maxExprCnt is 0. -func MaxExpressions(maxExprCnt uint64) Option { - return func(p *parser) Option { - oldMaxExprCnt := p.maxExprCnt - p.maxExprCnt = maxExprCnt - return MaxExpressions(oldMaxExprCnt) - } -} - -// Entrypoint creates an Option to set the rule name to use as entrypoint. -// The rule name must have been specified in the -alternate-entrypoints -// if generating the parser with the -optimize-grammar flag, otherwise -// it may have been optimized out. Passing an empty string sets the -// entrypoint to the first rule in the grammar. -// -// The default is to start parsing at the first rule in the grammar. -func Entrypoint(ruleName string) Option { - return func(p *parser) Option { - oldEntrypoint := p.entrypoint - p.entrypoint = ruleName - if ruleName == "" { - p.entrypoint = g.rules[0].name - } - return Entrypoint(oldEntrypoint) - } -} - -// ==template== {{ if not .Optimize }} -// Statistics adds a user provided Stats struct to the parser to allow -// the user to process the results after the parsing has finished. -// Also the key for the "no match" counter is set. -// -// Example usage: -// -// input := "input" -// stats := Stats{} -// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) -// if err != nil { -// log.Panicln(err) -// } -// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") -// if err != nil { -// log.Panicln(err) -// } -// fmt.Println(string(b)) -// -func Statistics(stats *Stats, choiceNoMatch string) Option { - return func(p *parser) Option { - oldStats := p.Stats - p.Stats = stats - oldChoiceNoMatch := p.choiceNoMatch - p.choiceNoMatch = choiceNoMatch - if p.Stats.ChoiceAltCnt == nil { - p.Stats.ChoiceAltCnt = make(map[string]map[string]int) - } - return Statistics(oldStats, oldChoiceNoMatch) - } -} - -// Debug creates an Option to set the debug flag to b. When set to true, -// debugging information is printed to stdout while parsing. -// -// The default is false. -func Debug(b bool) Option { - return func(p *parser) Option { - old := p.debug - p.debug = b - return Debug(old) - } -} - -// Memoize creates an Option to set the memoize flag to b. When set to true, -// the parser will cache all results so each expression is evaluated only -// once. This guarantees linear parsing time even for pathological cases, -// at the expense of more memory and slower times for typical cases. -// -// The default is false. -func Memoize(b bool) Option { - return func(p *parser) Option { - old := p.memoize - p.memoize = b - return Memoize(old) - } -} - -// {{ end }} ==template== - -// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. -// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) -// by character class matchers and is matched by the any matcher. -// The returned matched value, c.text and c.offset are NOT affected. -// -// The default is false. -func AllowInvalidUTF8(b bool) Option { - return func(p *parser) Option { - old := p.allowInvalidUTF8 - p.allowInvalidUTF8 = b - return AllowInvalidUTF8(old) - } -} - -// Recover creates an Option to set the recover flag to b. When set to -// true, this causes the parser to recover from panics and convert it -// to an error. Setting it to false can be useful while debugging to -// access the full stack trace. -// -// The default is true. -func Recover(b bool) Option { - return func(p *parser) Option { - old := p.recover - p.recover = b - return Recover(old) - } -} - -// GlobalStore creates an Option to set a key to a certain value in -// the globalStore. -func GlobalStore(key string, value interface{}) Option { - return func(p *parser) Option { - old := p.cur.globalStore[key] - p.cur.globalStore[key] = value - return GlobalStore(key, old) - } -} - -// ==template== {{ if or .GlobalState (not .Optimize) }} - -// InitState creates an Option to set a key to a certain value in -// the global "state" store. -func InitState(key string, value interface{}) Option { - return func(p *parser) Option { - old := p.cur.state[key] - p.cur.state[key] = value - return InitState(key, old) - } -} - -// {{ end }} ==template== - -// ParseFile parses the file identified by filename. -func ParseFile(filename string, opts ...Option) (i interface{}, err error) { //{{ if .Nolint }} nolint: deadcode {{else}} ==template== {{ end }} - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer func() { - if closeErr := f.Close(); closeErr != nil { - err = closeErr - } - }() - return ParseReader(filename, f, opts...) -} - -// ParseReader parses the data from r using filename as information in the -// error messages. -func ParseReader(filename string, r io.Reader, opts ...Option) (interface{}, error) { //{{ if .Nolint }} nolint: deadcode {{else}} ==template== {{ end }} - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - return Parse(filename, b, opts...) -} - -// Parse parses the data from b using filename as information in the -// error messages. -func Parse(filename string, b []byte, opts ...Option) (interface{}, error) { - return newParser(filename, b, opts...).parse(g) -} - -// position records a position in the text. -type position struct { - line, col, offset int -} - -func (p position) String() string { - return fmt.Sprintf("%d:%d [%d]", p.line, p.col, p.offset) -} - -// savepoint stores all state required to go back to this point in the -// parser. -type savepoint struct { - position - rn rune - w int -} - -type current struct { - pos position // start position of the match - text []byte // raw text of the match - - // ==template== {{ if or .GlobalState (not .Optimize) }} - - // state is a store for arbitrary key,value pairs that the user wants to be - // tied to the backtracking of the parser. - // This is always rolled back if a parsing rule fails. - state storeDict - - // {{ end }} ==template== - - // globalStore is a general store for the user to store arbitrary key-value - // pairs that they need to manage and that they do not want tied to the - // backtracking of the parser. This is only modified by the user and never - // rolled back by the parser. It is always up to the user to keep this in a - // consistent state. - globalStore storeDict -} - -type storeDict map[string]interface{} - -// the AST types... - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type grammar struct { - pos position - rules []*rule -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type rule struct { - pos position - name string - displayName string - expr interface{} -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type choiceExpr struct { - pos position - alternatives []interface{} -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type actionExpr struct { - pos position - expr interface{} - run func(*parser) (interface{}, error) -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type recoveryExpr struct { - pos position - expr interface{} - recoverExpr interface{} - failureLabel []string -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type seqExpr struct { - pos position - exprs []interface{} -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type throwExpr struct { - pos position - label string -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type labeledExpr struct { - pos position - label string - expr interface{} -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type expr struct { - pos position - expr interface{} -} - -type andExpr expr //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type notExpr expr //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type zeroOrOneExpr expr //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type zeroOrMoreExpr expr //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type oneOrMoreExpr expr //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type ruleRefExpr struct { - pos position - name string -} - -// ==template== {{ if or .GlobalState (not .Optimize) }} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type stateCodeExpr struct { - pos position - run func(*parser) error -} - -// {{ end }} ==template== - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type andCodeExpr struct { - pos position - run func(*parser) (bool, error) -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type notCodeExpr struct { - pos position - run func(*parser) (bool, error) -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type litMatcher struct { - pos position - val string - ignoreCase bool -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type charClassMatcher struct { - pos position - val string - basicLatinChars [128]bool - chars []rune - ranges []rune - classes []*unicode.RangeTable - ignoreCase bool - inverted bool -} - -type anyMatcher position //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} - -// errList cumulates the errors found by the parser. -type errList []error - -func (e *errList) add(err error) { - *e = append(*e, err) -} - -func (e errList) err() error { - if len(e) == 0 { - return nil - } - e.dedupe() - return e -} - -func (e *errList) dedupe() { - var cleaned []error - set := make(map[string]bool) - for _, err := range *e { - if msg := err.Error(); !set[msg] { - set[msg] = true - cleaned = append(cleaned, err) - } - } - *e = cleaned -} - -func (e errList) Error() string { - switch len(e) { - case 0: - return "" - case 1: - return e[0].Error() - default: - var buf bytes.Buffer - - for i, err := range e { - if i > 0 { - buf.WriteRune('\n') - } - buf.WriteString(err.Error()) - } - return buf.String() - } -} - -// parserError wraps an error with a prefix indicating the rule in which -// the error occurred. The original error is stored in the Inner field. -type parserError struct { - Inner error - pos position - prefix string - expected []string -} - -// Error returns the error message. -func (p *parserError) Error() string { - return p.prefix + ": " + p.Inner.Error() -} - -// newParser creates a parser with the specified input source and options. -func newParser(filename string, b []byte, opts ...Option) *parser { - stats := Stats{ - ChoiceAltCnt: make(map[string]map[string]int), - } - - p := &parser{ - filename: filename, - errs: new(errList), - data: b, - pt: savepoint{position: position{line: 1}}, - recover: true, - cur: current{ - // ==template== {{ if or .GlobalState (not .Optimize) }} - state: make(storeDict), - // {{ end }} ==template== - globalStore: make(storeDict), - }, - maxFailPos: position{col: 1, line: 1}, - maxFailExpected: make([]string, 0, 20), - Stats: &stats, - // start rule is rule [0] unless an alternate entrypoint is specified - entrypoint: g.rules[0].name, - } - p.setOptions(opts) - - if p.maxExprCnt == 0 { - p.maxExprCnt = math.MaxUint64 - } - - return p -} - -// setOptions applies the options to the parser. -func (p *parser) setOptions(opts []Option) { - for _, opt := range opts { - opt(p) - } -} - -//{{ if .Nolint }} nolint: structcheck,deadcode {{else}} ==template== {{ end }} -type resultTuple struct { - v interface{} - b bool - end savepoint -} - -//{{ if .Nolint }} nolint: varcheck {{else}} ==template== {{ end }} -const choiceNoMatch = -1 - -// Stats stores some statistics, gathered during parsing -type Stats struct { - // ExprCnt counts the number of expressions processed during parsing - // This value is compared to the maximum number of expressions allowed - // (set by the MaxExpressions option). - ExprCnt uint64 - - // ChoiceAltCnt is used to count for each ordered choice expression, - // which alternative is used how may times. - // These numbers allow to optimize the order of the ordered choice expression - // to increase the performance of the parser - // - // The outer key of ChoiceAltCnt is composed of the name of the rule as well - // as the line and the column of the ordered choice. - // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. - // For each alternative the number of matches are counted. If an ordered choice does not - // match, a special counter is incremented. The name of this counter is set with - // the parser option Statistics. - // For an alternative to be included in ChoiceAltCnt, it has to match at least once. - ChoiceAltCnt map[string]map[string]int -} - -//{{ if .Nolint }} nolint: structcheck,maligned {{else}} ==template== {{ end }} -type parser struct { - filename string - pt savepoint - cur current - - data []byte - errs *errList - - depth int - recover bool - // ==template== {{ if not .Optimize }} - debug bool - - memoize bool - // memoization table for the packrat algorithm: - // map[offset in source] map[expression or rule] {value, match} - memo map[int]map[interface{}]resultTuple - // {{ end }} ==template== - - // rules table, maps the rule identifier to the rule node - rules map[string]*rule - // variables stack, map of label to value - vstack []map[string]interface{} - // rule stack, allows identification of the current rule in errors - rstack []*rule - - // parse fail - maxFailPos position - maxFailExpected []string - maxFailInvertExpected bool - - // max number of expressions to be parsed - maxExprCnt uint64 - // entrypoint for the parser - entrypoint string - - allowInvalidUTF8 bool - - *Stats - - choiceNoMatch string - // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse - recoveryStack []map[string]interface{} -} - -// push a variable set on the vstack. -func (p *parser) pushV() { - if cap(p.vstack) == len(p.vstack) { - // create new empty slot in the stack - p.vstack = append(p.vstack, nil) - } else { - // slice to 1 more - p.vstack = p.vstack[:len(p.vstack)+1] - } - - // get the last args set - m := p.vstack[len(p.vstack)-1] - if m != nil && len(m) == 0 { - // empty map, all good - return - } - - m = make(map[string]interface{}) - p.vstack[len(p.vstack)-1] = m -} - -// pop a variable set from the vstack. -func (p *parser) popV() { - // if the map is not empty, clear it - m := p.vstack[len(p.vstack)-1] - if len(m) > 0 { - // GC that map - p.vstack[len(p.vstack)-1] = nil - } - p.vstack = p.vstack[:len(p.vstack)-1] -} - -// push a recovery expression with its labels to the recoveryStack -func (p *parser) pushRecovery(labels []string, expr interface{}) { - if cap(p.recoveryStack) == len(p.recoveryStack) { - // create new empty slot in the stack - p.recoveryStack = append(p.recoveryStack, nil) - } else { - // slice to 1 more - p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] - } - - m := make(map[string]interface{}, len(labels)) - for _, fl := range labels { - m[fl] = expr - } - p.recoveryStack[len(p.recoveryStack)-1] = m -} - -// pop a recovery expression from the recoveryStack -func (p *parser) popRecovery() { - // GC that map - p.recoveryStack[len(p.recoveryStack)-1] = nil - - p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] -} - -// ==template== {{ if not .Optimize }} -func (p *parser) print(prefix, s string) string { - if !p.debug { - return s - } - - fmt.Printf("%s %d:%d:%d: %s [%#U]\n", - prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) - return s -} - -func (p *parser) in(s string) string { - p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) -} - -func (p *parser) out(s string) string { - p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) -} - -// {{ end }} ==template== - -func (p *parser) addErr(err error) { - p.addErrAt(err, p.pt.position, []string{}) -} - -func (p *parser) addErrAt(err error, pos position, expected []string) { - var buf bytes.Buffer - if p.filename != "" { - buf.WriteString(p.filename) - } - if buf.Len() > 0 { - buf.WriteString(":") - } - buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) - if len(p.rstack) > 0 { - if buf.Len() > 0 { - buf.WriteString(": ") - } - rule := p.rstack[len(p.rstack)-1] - if rule.displayName != "" { - buf.WriteString("rule " + rule.displayName) - } else { - buf.WriteString("rule " + rule.name) - } - } - pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} - p.errs.add(pe) -} - -func (p *parser) failAt(fail bool, pos position, want string) { - // process fail if parsing fails and not inverted or parsing succeeds and invert is set - if fail == p.maxFailInvertExpected { - if pos.offset < p.maxFailPos.offset { - return - } - - if pos.offset > p.maxFailPos.offset { - p.maxFailPos = pos - p.maxFailExpected = p.maxFailExpected[:0] - } - - if p.maxFailInvertExpected { - want = "!" + want - } - p.maxFailExpected = append(p.maxFailExpected, want) - } -} - -// read advances the parser to the next rune. -func (p *parser) read() { - p.pt.offset += p.pt.w - rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) - p.pt.rn = rn - p.pt.w = n - p.pt.col++ - if rn == '\n' { - p.pt.line++ - p.pt.col = 0 - } - - if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune - if !p.allowInvalidUTF8 { - p.addErr(errInvalidEncoding) - } - } -} - -// restore parser position to the savepoint pt. -func (p *parser) restore(pt savepoint) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("restore")) - } - // {{ end }} ==template== - if pt.offset == p.pt.offset { - return - } - p.pt = pt -} - -// ==template== {{ if or .GlobalState (not .Optimize) }} - -// Cloner is implemented by any value that has a Clone method, which returns a -// copy of the value. This is mainly used for types which are not passed by -// value (e.g map, slice, chan) or structs that contain such types. -// -// This is used in conjunction with the global state feature to create proper -// copies of the state to allow the parser to properly restore the state in -// the case of backtracking. -type Cloner interface { - Clone() interface{} -} - -// clone and return parser current state. -func (p *parser) cloneState() storeDict { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("cloneState")) - } - // {{ end }} ==template== - - state := make(storeDict, len(p.cur.state)) - for k, v := range p.cur.state { - if c, ok := v.(Cloner); ok { - state[k] = c.Clone() - } else { - state[k] = v - } - } - return state -} - -// restore parser current state to the state storeDict. -// every restoreState should applied only one time for every cloned state -func (p *parser) restoreState(state storeDict) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("restoreState")) - } - // {{ end }} ==template== - p.cur.state = state -} - -// {{ end }} ==template== - -// get the slice of bytes from the savepoint start to the current position. -func (p *parser) sliceFrom(start savepoint) []byte { - return p.data[start.position.offset:p.pt.position.offset] -} - -// ==template== {{ if not .Optimize }} -func (p *parser) getMemoized(node interface{}) (resultTuple, bool) { - if len(p.memo) == 0 { - return resultTuple{}, false - } - m := p.memo[p.pt.offset] - if len(m) == 0 { - return resultTuple{}, false - } - res, ok := m[node] - return res, ok -} - -func (p *parser) setMemoized(pt savepoint, node interface{}, tuple resultTuple) { - if p.memo == nil { - p.memo = make(map[int]map[interface{}]resultTuple) - } - m := p.memo[pt.offset] - if m == nil { - m = make(map[interface{}]resultTuple) - p.memo[pt.offset] = m - } - m[node] = tuple -} - -// {{ end }} ==template== - -func (p *parser) buildRulesTable(g *grammar) { - p.rules = make(map[string]*rule, len(g.rules)) - for _, r := range g.rules { - p.rules[r.name] = r - } -} - -//{{ if .Nolint }} nolint: gocyclo {{else}} ==template== {{ end }} -func (p *parser) parse(g *grammar) (val interface{}, err error) { - if len(g.rules) == 0 { - p.addErr(errNoRule) - return nil, p.errs.err() - } - - // TODO : not super critical but this could be generated - p.buildRulesTable(g) - - if p.recover { - // panic can be used in action code to stop parsing immediately - // and return the panic as an error. - defer func() { - if e := recover(); e != nil { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("panic handler")) - } - // {{ end }} ==template== - val = nil - switch e := e.(type) { - case error: - p.addErr(e) - default: - p.addErr(fmt.Errorf("%v", e)) - } - err = p.errs.err() - } - }() - } - - startRule, ok := p.rules[p.entrypoint] - if !ok { - p.addErr(errInvalidEntrypoint) - return nil, p.errs.err() - } - - p.read() // advance to first rune - val, ok = p.parseRule(startRule) - if !ok { - if len(*p.errs) == 0 { - // If parsing fails, but no errors have been recorded, the expected values - // for the farthest parser position are returned as error. - maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) - for _, v := range p.maxFailExpected { - maxFailExpectedMap[v] = struct{}{} - } - expected := make([]string, 0, len(maxFailExpectedMap)) - eof := false - if _, ok := maxFailExpectedMap["!."]; ok { - delete(maxFailExpectedMap, "!.") - eof = true - } - for k := range maxFailExpectedMap { - expected = append(expected, k) - } - sort.Strings(expected) - if eof { - expected = append(expected, "EOF") - } - p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) - } - - return nil, p.errs.err() - } - return val, p.errs.err() -} - -func listJoin(list []string, sep string, lastSep string) string { - switch len(list) { - case 0: - return "" - case 1: - return list[0] - default: - return fmt.Sprintf("%s %s %s", strings.Join(list[:len(list)-1], sep), lastSep, list[len(list)-1]) - } -} - -func (p *parser) parseRule(rule *rule) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseRule " + rule.name)) - } - - if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } - } - - start := p.pt - // {{ end }} ==template== - p.rstack = append(p.rstack, rule) - p.pushV() - val, ok := p.parseExpr(rule.expr) - p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] - // ==template== {{ if not .Optimize }} - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } - // {{ end }} ==template== - return val, ok -} - -//{{ if .Nolint }} nolint: gocyclo {{else}} ==template== {{ end }} -func (p *parser) parseExpr(expr interface{}) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - var pt savepoint - - if p.memoize { - res, ok := p.getMemoized(expr) - if ok { - p.restore(res.end) - return res.v, res.b - } - pt = p.pt - } - - // {{ end }} ==template== - - p.ExprCnt++ - if p.ExprCnt > p.maxExprCnt { - panic(errMaxExprCnt) - } - - var val interface{} - var ok bool - switch expr := expr.(type) { - case *actionExpr: - val, ok = p.parseActionExpr(expr) - case *andCodeExpr: - val, ok = p.parseAndCodeExpr(expr) - case *andExpr: - val, ok = p.parseAndExpr(expr) - case *anyMatcher: - val, ok = p.parseAnyMatcher(expr) - case *charClassMatcher: - val, ok = p.parseCharClassMatcher(expr) - case *choiceExpr: - val, ok = p.parseChoiceExpr(expr) - case *labeledExpr: - val, ok = p.parseLabeledExpr(expr) - case *litMatcher: - val, ok = p.parseLitMatcher(expr) - case *notCodeExpr: - val, ok = p.parseNotCodeExpr(expr) - case *notExpr: - val, ok = p.parseNotExpr(expr) - case *oneOrMoreExpr: - val, ok = p.parseOneOrMoreExpr(expr) - case *recoveryExpr: - val, ok = p.parseRecoveryExpr(expr) - case *ruleRefExpr: - val, ok = p.parseRuleRefExpr(expr) - case *seqExpr: - val, ok = p.parseSeqExpr(expr) - // ==template== {{ if or .GlobalState (not .Optimize) }} - case *stateCodeExpr: - val, ok = p.parseStateCodeExpr(expr) - // {{ end }} ==template== - case *throwExpr: - val, ok = p.parseThrowExpr(expr) - case *zeroOrMoreExpr: - val, ok = p.parseZeroOrMoreExpr(expr) - case *zeroOrOneExpr: - val, ok = p.parseZeroOrOneExpr(expr) - default: - panic(fmt.Sprintf("unknown expression type %T", expr)) - } - // ==template== {{ if not .Optimize }} - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } - // {{ end }} ==template== - return val, ok -} - -func (p *parser) parseActionExpr(act *actionExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseActionExpr")) - } - - // {{ end }} ==template== - start := p.pt - val, ok := p.parseExpr(act.expr) - if ok { - p.cur.pos = start.position - p.cur.text = p.sliceFrom(start) - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - actVal, err := act.run(p) - if err != nil { - p.addErrAt(err, start.position, []string{}) - } - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - - val = actVal - } - // ==template== {{ if not .Optimize }} - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - // {{ end }} ==template== - return val, ok -} - -func (p *parser) parseAndCodeExpr(and *andCodeExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseAndCodeExpr")) - } - - // {{ end }} ==template== - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - - ok, err := and.run(p) - if err != nil { - p.addErr(err) - } - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - - return nil, ok -} - -func (p *parser) parseAndExpr(and *andExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseAndExpr")) - } - - // {{ end }} ==template== - pt := p.pt - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - p.pushV() - _, ok := p.parseExpr(and.expr) - p.popV() - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - p.restore(pt) - - return nil, ok -} - -func (p *parser) parseAnyMatcher(any *anyMatcher) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseAnyMatcher")) - } - - // {{ end }} ==template== - if p.pt.rn == utf8.RuneError && p.pt.w == 0 { - // EOF - see utf8.DecodeRune - p.failAt(false, p.pt.position, ".") - return nil, false - } - start := p.pt - p.read() - p.failAt(true, start.position, ".") - return p.sliceFrom(start), true -} - -//{{ if .Nolint }} nolint: gocyclo {{else}} ==template== {{ end }} -func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseCharClassMatcher")) - } - - // {{ end }} ==template== - cur := p.pt.rn - start := p.pt - - // ==template== {{ if .BasicLatinLookupTable }} - if cur < 128 { - if chr.basicLatinChars[cur] != chr.inverted { - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - p.failAt(false, start.position, chr.val) - return nil, false - } - // {{ end }} ==template== - - // can't match EOF - if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune - p.failAt(false, start.position, chr.val) - return nil, false - } - - if chr.ignoreCase { - cur = unicode.ToLower(cur) - } - - // try to match in the list of available chars - for _, rn := range chr.chars { - if rn == cur { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - // try to match in the list of ranges - for i := 0; i < len(chr.ranges); i += 2 { - if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - // try to match in the list of Unicode classes - for _, cl := range chr.classes { - if unicode.Is(cl, cur) { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - if chr.inverted { - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - p.failAt(false, start.position, chr.val) - return nil, false -} - -// ==template== {{ if not .Optimize }} - -func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) - m := p.ChoiceAltCnt[choiceIdent] - if m == nil { - m = make(map[string]int) - p.ChoiceAltCnt[choiceIdent] = m - } - // We increment altI by 1, so the keys do not start at 0 - alt := strconv.Itoa(altI + 1) - if altI == choiceNoMatch { - alt = p.choiceNoMatch - } - m[alt]++ -} - -// {{ end }} ==template== - -func (p *parser) parseChoiceExpr(ch *choiceExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseChoiceExpr")) - } - - // {{ end }} ==template== - for altI, alt := range ch.alternatives { - // dummy assignment to prevent compile error if optimized - _ = altI - - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - - p.pushV() - val, ok := p.parseExpr(alt) - p.popV() - if ok { - // ==template== {{ if not .Optimize }} - p.incChoiceAltCnt(ch, altI) - // {{ end }} ==template== - return val, ok - } - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - } - // ==template== {{ if not .Optimize }} - p.incChoiceAltCnt(ch, choiceNoMatch) - // {{ end }} ==template== - return nil, false -} - -func (p *parser) parseLabeledExpr(lab *labeledExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseLabeledExpr")) - } - - // {{ end }} ==template== - p.pushV() - val, ok := p.parseExpr(lab.expr) - p.popV() - if ok && lab.label != "" { - m := p.vstack[len(p.vstack)-1] - m[lab.label] = val - } - return val, ok -} - -func (p *parser) parseLitMatcher(lit *litMatcher) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseLitMatcher")) - } - - // {{ end }} ==template== - ignoreCase := "" - if lit.ignoreCase { - ignoreCase = "i" - } - val := fmt.Sprintf("%q%s", lit.val, ignoreCase) - start := p.pt - for _, want := range lit.val { - cur := p.pt.rn - if lit.ignoreCase { - cur = unicode.ToLower(cur) - } - if cur != want { - p.failAt(false, start.position, val) - p.restore(start) - return nil, false - } - p.read() - } - p.failAt(true, start.position, val) - return p.sliceFrom(start), true -} - -func (p *parser) parseNotCodeExpr(not *notCodeExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseNotCodeExpr")) - } - - // {{ end }} ==template== - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - - // {{ end }} ==template== - ok, err := not.run(p) - if err != nil { - p.addErr(err) - } - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - - return nil, !ok -} - -func (p *parser) parseNotExpr(not *notExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseNotExpr")) - } - - // {{ end }} ==template== - pt := p.pt - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - p.pushV() - p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) - p.maxFailInvertExpected = !p.maxFailInvertExpected - p.popV() - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - p.restore(pt) - - return nil, !ok -} - -func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseOneOrMoreExpr")) - } - - // {{ end }} ==template== - var vals []interface{} - - for { - p.pushV() - val, ok := p.parseExpr(expr.expr) - p.popV() - if !ok { - if len(vals) == 0 { - // did not match once, no match - return nil, false - } - return vals, true - } - vals = append(vals, val) - } -} - -func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) - } - - // {{ end }} ==template== - - p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) - p.popRecovery() - - return val, ok -} - -func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseRuleRefExpr " + ref.name)) - } - - // {{ end }} ==template== - if ref.name == "" { - panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) - } - - rule := p.rules[ref.name] - if rule == nil { - p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) - return nil, false - } - return p.parseRule(rule) -} - -func (p *parser) parseSeqExpr(seq *seqExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseSeqExpr")) - } - - // {{ end }} ==template== - vals := make([]interface{}, 0, len(seq.exprs)) - - pt := p.pt - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) - if !ok { - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - p.restore(pt) - return nil, false - } - vals = append(vals, val) - } - return vals, true -} - -// ==template== {{ if or .GlobalState (not .Optimize) }} - -func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseStateCodeExpr")) - } - - // {{ end }} ==template== - err := state.run(p) - if err != nil { - p.addErr(err) - } - return nil, true -} - -// {{ end }} ==template== - -func (p *parser) parseThrowExpr(expr *throwExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseThrowExpr")) - } - - // {{ end }} ==template== - - for i := len(p.recoveryStack) - 1; i >= 0; i-- { - if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { - return val, ok - } - } - } - - return nil, false -} - -func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseZeroOrMoreExpr")) - } - - // {{ end }} ==template== - var vals []interface{} - - for { - p.pushV() - val, ok := p.parseExpr(expr.expr) - p.popV() - if !ok { - return vals, true - } - vals = append(vals, val) - } -} - -func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseZeroOrOneExpr")) - } - - // {{ end }} ==template== - p.pushV() - val, _ := p.parseExpr(expr.expr) - p.popV() - // whether it matched or not, consider it a match - return val, true -} - -` diff --git a/vendor/github.com/mna/pigeon/builder/generated_static_code_range_table.go b/vendor/github.com/mna/pigeon/builder/generated_static_code_range_table.go deleted file mode 100644 index a9c0db8561..0000000000 --- a/vendor/github.com/mna/pigeon/builder/generated_static_code_range_table.go +++ /dev/null @@ -1,21 +0,0 @@ -// Code generated by static_code_generator with go generate; DO NOT EDIT. - -package builder - -var rangeTable0 = ` -func rangeTable(class string) *unicode.RangeTable { - if rt, ok := unicode.Categories[class]; ok { - return rt - } - if rt, ok := unicode.Properties[class]; ok { - return rt - } - if rt, ok := unicode.Scripts[class]; ok { - return rt - } - - // cannot happen - panic(fmt.Sprintf("invalid Unicode class: %s", class)) -} - -` diff --git a/vendor/github.com/mna/pigeon/builder/static_code.go b/vendor/github.com/mna/pigeon/builder/static_code.go deleted file mode 100644 index 1347640e5b..0000000000 --- a/vendor/github.com/mna/pigeon/builder/static_code.go +++ /dev/null @@ -1,1457 +0,0 @@ -//go:generate go run ../bootstrap/cmd/static_code_generator/main.go -- $GOFILE generated_$GOFILE staticCode - -// +build static_code - -package builder - -import ( - "bytes" - "errors" - "fmt" - "io" - "io/ioutil" - "math" - "os" - "sort" - "strconv" - "strings" - "unicode" - "unicode/utf8" -) - -// IMPORTANT: All code below this line is added to the parser as static code -var ( - // errNoRule is returned when the grammar to parse has no rule. - errNoRule = errors.New("grammar has no rule") - - // errInvalidEntrypoint is returned when the specified entrypoint rule - // does not exit. - errInvalidEntrypoint = errors.New("invalid entrypoint") - - // errInvalidEncoding is returned when the source is not properly - // utf8-encoded. - errInvalidEncoding = errors.New("invalid encoding") - - // errMaxExprCnt is used to signal that the maximum number of - // expressions have been parsed. - errMaxExprCnt = errors.New("max number of expresssions parsed") -) - -// Option is a function that can set an option on the parser. It returns -// the previous setting as an Option. -type Option func(*parser) Option - -// MaxExpressions creates an Option to stop parsing after the provided -// number of expressions have been parsed, if the value is 0 then the parser will -// parse for as many steps as needed (possibly an infinite number). -// -// The default for maxExprCnt is 0. -func MaxExpressions(maxExprCnt uint64) Option { - return func(p *parser) Option { - oldMaxExprCnt := p.maxExprCnt - p.maxExprCnt = maxExprCnt - return MaxExpressions(oldMaxExprCnt) - } -} - -// Entrypoint creates an Option to set the rule name to use as entrypoint. -// The rule name must have been specified in the -alternate-entrypoints -// if generating the parser with the -optimize-grammar flag, otherwise -// it may have been optimized out. Passing an empty string sets the -// entrypoint to the first rule in the grammar. -// -// The default is to start parsing at the first rule in the grammar. -func Entrypoint(ruleName string) Option { - return func(p *parser) Option { - oldEntrypoint := p.entrypoint - p.entrypoint = ruleName - if ruleName == "" { - p.entrypoint = g.rules[0].name - } - return Entrypoint(oldEntrypoint) - } -} - -// ==template== {{ if not .Optimize }} -// Statistics adds a user provided Stats struct to the parser to allow -// the user to process the results after the parsing has finished. -// Also the key for the "no match" counter is set. -// -// Example usage: -// -// input := "input" -// stats := Stats{} -// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) -// if err != nil { -// log.Panicln(err) -// } -// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") -// if err != nil { -// log.Panicln(err) -// } -// fmt.Println(string(b)) -// -func Statistics(stats *Stats, choiceNoMatch string) Option { - return func(p *parser) Option { - oldStats := p.Stats - p.Stats = stats - oldChoiceNoMatch := p.choiceNoMatch - p.choiceNoMatch = choiceNoMatch - if p.Stats.ChoiceAltCnt == nil { - p.Stats.ChoiceAltCnt = make(map[string]map[string]int) - } - return Statistics(oldStats, oldChoiceNoMatch) - } -} - -// Debug creates an Option to set the debug flag to b. When set to true, -// debugging information is printed to stdout while parsing. -// -// The default is false. -func Debug(b bool) Option { - return func(p *parser) Option { - old := p.debug - p.debug = b - return Debug(old) - } -} - -// Memoize creates an Option to set the memoize flag to b. When set to true, -// the parser will cache all results so each expression is evaluated only -// once. This guarantees linear parsing time even for pathological cases, -// at the expense of more memory and slower times for typical cases. -// -// The default is false. -func Memoize(b bool) Option { - return func(p *parser) Option { - old := p.memoize - p.memoize = b - return Memoize(old) - } -} - -// {{ end }} ==template== - -// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. -// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) -// by character class matchers and is matched by the any matcher. -// The returned matched value, c.text and c.offset are NOT affected. -// -// The default is false. -func AllowInvalidUTF8(b bool) Option { - return func(p *parser) Option { - old := p.allowInvalidUTF8 - p.allowInvalidUTF8 = b - return AllowInvalidUTF8(old) - } -} - -// Recover creates an Option to set the recover flag to b. When set to -// true, this causes the parser to recover from panics and convert it -// to an error. Setting it to false can be useful while debugging to -// access the full stack trace. -// -// The default is true. -func Recover(b bool) Option { - return func(p *parser) Option { - old := p.recover - p.recover = b - return Recover(old) - } -} - -// GlobalStore creates an Option to set a key to a certain value in -// the globalStore. -func GlobalStore(key string, value interface{}) Option { - return func(p *parser) Option { - old := p.cur.globalStore[key] - p.cur.globalStore[key] = value - return GlobalStore(key, old) - } -} - -// ==template== {{ if or .GlobalState (not .Optimize) }} - -// InitState creates an Option to set a key to a certain value in -// the global "state" store. -func InitState(key string, value interface{}) Option { - return func(p *parser) Option { - old := p.cur.state[key] - p.cur.state[key] = value - return InitState(key, old) - } -} - -// {{ end }} ==template== - -// ParseFile parses the file identified by filename. -func ParseFile(filename string, opts ...Option) (i interface{}, err error) { //{{ if .Nolint }} nolint: deadcode {{else}} ==template== {{ end }} - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer func() { - if closeErr := f.Close(); closeErr != nil { - err = closeErr - } - }() - return ParseReader(filename, f, opts...) -} - -// ParseReader parses the data from r using filename as information in the -// error messages. -func ParseReader(filename string, r io.Reader, opts ...Option) (interface{}, error) { //{{ if .Nolint }} nolint: deadcode {{else}} ==template== {{ end }} - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - return Parse(filename, b, opts...) -} - -// Parse parses the data from b using filename as information in the -// error messages. -func Parse(filename string, b []byte, opts ...Option) (interface{}, error) { - return newParser(filename, b, opts...).parse(g) -} - -// position records a position in the text. -type position struct { - line, col, offset int -} - -func (p position) String() string { - return fmt.Sprintf("%d:%d [%d]", p.line, p.col, p.offset) -} - -// savepoint stores all state required to go back to this point in the -// parser. -type savepoint struct { - position - rn rune - w int -} - -type current struct { - pos position // start position of the match - text []byte // raw text of the match - - // ==template== {{ if or .GlobalState (not .Optimize) }} - - // state is a store for arbitrary key,value pairs that the user wants to be - // tied to the backtracking of the parser. - // This is always rolled back if a parsing rule fails. - state storeDict - - // {{ end }} ==template== - - // globalStore is a general store for the user to store arbitrary key-value - // pairs that they need to manage and that they do not want tied to the - // backtracking of the parser. This is only modified by the user and never - // rolled back by the parser. It is always up to the user to keep this in a - // consistent state. - globalStore storeDict -} - -type storeDict map[string]interface{} - -// the AST types... - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type grammar struct { - pos position - rules []*rule -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type rule struct { - pos position - name string - displayName string - expr interface{} -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type choiceExpr struct { - pos position - alternatives []interface{} -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type actionExpr struct { - pos position - expr interface{} - run func(*parser) (interface{}, error) -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type recoveryExpr struct { - pos position - expr interface{} - recoverExpr interface{} - failureLabel []string -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type seqExpr struct { - pos position - exprs []interface{} -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type throwExpr struct { - pos position - label string -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type labeledExpr struct { - pos position - label string - expr interface{} -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type expr struct { - pos position - expr interface{} -} - -type andExpr expr //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type notExpr expr //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type zeroOrOneExpr expr //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type zeroOrMoreExpr expr //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type oneOrMoreExpr expr //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type ruleRefExpr struct { - pos position - name string -} - -// ==template== {{ if or .GlobalState (not .Optimize) }} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type stateCodeExpr struct { - pos position - run func(*parser) error -} - -// {{ end }} ==template== - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type andCodeExpr struct { - pos position - run func(*parser) (bool, error) -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type notCodeExpr struct { - pos position - run func(*parser) (bool, error) -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type litMatcher struct { - pos position - val string - ignoreCase bool -} - -//{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} -type charClassMatcher struct { - pos position - val string - basicLatinChars [128]bool - chars []rune - ranges []rune - classes []*unicode.RangeTable - ignoreCase bool - inverted bool -} - -type anyMatcher position //{{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} - -// errList cumulates the errors found by the parser. -type errList []error - -func (e *errList) add(err error) { - *e = append(*e, err) -} - -func (e errList) err() error { - if len(e) == 0 { - return nil - } - e.dedupe() - return e -} - -func (e *errList) dedupe() { - var cleaned []error - set := make(map[string]bool) - for _, err := range *e { - if msg := err.Error(); !set[msg] { - set[msg] = true - cleaned = append(cleaned, err) - } - } - *e = cleaned -} - -func (e errList) Error() string { - switch len(e) { - case 0: - return "" - case 1: - return e[0].Error() - default: - var buf bytes.Buffer - - for i, err := range e { - if i > 0 { - buf.WriteRune('\n') - } - buf.WriteString(err.Error()) - } - return buf.String() - } -} - -// parserError wraps an error with a prefix indicating the rule in which -// the error occurred. The original error is stored in the Inner field. -type parserError struct { - Inner error - pos position - prefix string - expected []string -} - -// Error returns the error message. -func (p *parserError) Error() string { - return p.prefix + ": " + p.Inner.Error() -} - -// newParser creates a parser with the specified input source and options. -func newParser(filename string, b []byte, opts ...Option) *parser { - stats := Stats{ - ChoiceAltCnt: make(map[string]map[string]int), - } - - p := &parser{ - filename: filename, - errs: new(errList), - data: b, - pt: savepoint{position: position{line: 1}}, - recover: true, - cur: current{ - // ==template== {{ if or .GlobalState (not .Optimize) }} - state: make(storeDict), - // {{ end }} ==template== - globalStore: make(storeDict), - }, - maxFailPos: position{col: 1, line: 1}, - maxFailExpected: make([]string, 0, 20), - Stats: &stats, - // start rule is rule [0] unless an alternate entrypoint is specified - entrypoint: g.rules[0].name, - } - p.setOptions(opts) - - if p.maxExprCnt == 0 { - p.maxExprCnt = math.MaxUint64 - } - - return p -} - -// setOptions applies the options to the parser. -func (p *parser) setOptions(opts []Option) { - for _, opt := range opts { - opt(p) - } -} - -//{{ if .Nolint }} nolint: structcheck,deadcode {{else}} ==template== {{ end }} -type resultTuple struct { - v interface{} - b bool - end savepoint -} - -//{{ if .Nolint }} nolint: varcheck {{else}} ==template== {{ end }} -const choiceNoMatch = -1 - -// Stats stores some statistics, gathered during parsing -type Stats struct { - // ExprCnt counts the number of expressions processed during parsing - // This value is compared to the maximum number of expressions allowed - // (set by the MaxExpressions option). - ExprCnt uint64 - - // ChoiceAltCnt is used to count for each ordered choice expression, - // which alternative is used how may times. - // These numbers allow to optimize the order of the ordered choice expression - // to increase the performance of the parser - // - // The outer key of ChoiceAltCnt is composed of the name of the rule as well - // as the line and the column of the ordered choice. - // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. - // For each alternative the number of matches are counted. If an ordered choice does not - // match, a special counter is incremented. The name of this counter is set with - // the parser option Statistics. - // For an alternative to be included in ChoiceAltCnt, it has to match at least once. - ChoiceAltCnt map[string]map[string]int -} - -//{{ if .Nolint }} nolint: structcheck,maligned {{else}} ==template== {{ end }} -type parser struct { - filename string - pt savepoint - cur current - - data []byte - errs *errList - - depth int - recover bool - // ==template== {{ if not .Optimize }} - debug bool - - memoize bool - // memoization table for the packrat algorithm: - // map[offset in source] map[expression or rule] {value, match} - memo map[int]map[interface{}]resultTuple - // {{ end }} ==template== - - // rules table, maps the rule identifier to the rule node - rules map[string]*rule - // variables stack, map of label to value - vstack []map[string]interface{} - // rule stack, allows identification of the current rule in errors - rstack []*rule - - // parse fail - maxFailPos position - maxFailExpected []string - maxFailInvertExpected bool - - // max number of expressions to be parsed - maxExprCnt uint64 - // entrypoint for the parser - entrypoint string - - allowInvalidUTF8 bool - - *Stats - - choiceNoMatch string - // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse - recoveryStack []map[string]interface{} -} - -// push a variable set on the vstack. -func (p *parser) pushV() { - if cap(p.vstack) == len(p.vstack) { - // create new empty slot in the stack - p.vstack = append(p.vstack, nil) - } else { - // slice to 1 more - p.vstack = p.vstack[:len(p.vstack)+1] - } - - // get the last args set - m := p.vstack[len(p.vstack)-1] - if m != nil && len(m) == 0 { - // empty map, all good - return - } - - m = make(map[string]interface{}) - p.vstack[len(p.vstack)-1] = m -} - -// pop a variable set from the vstack. -func (p *parser) popV() { - // if the map is not empty, clear it - m := p.vstack[len(p.vstack)-1] - if len(m) > 0 { - // GC that map - p.vstack[len(p.vstack)-1] = nil - } - p.vstack = p.vstack[:len(p.vstack)-1] -} - -// push a recovery expression with its labels to the recoveryStack -func (p *parser) pushRecovery(labels []string, expr interface{}) { - if cap(p.recoveryStack) == len(p.recoveryStack) { - // create new empty slot in the stack - p.recoveryStack = append(p.recoveryStack, nil) - } else { - // slice to 1 more - p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] - } - - m := make(map[string]interface{}, len(labels)) - for _, fl := range labels { - m[fl] = expr - } - p.recoveryStack[len(p.recoveryStack)-1] = m -} - -// pop a recovery expression from the recoveryStack -func (p *parser) popRecovery() { - // GC that map - p.recoveryStack[len(p.recoveryStack)-1] = nil - - p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] -} - -// ==template== {{ if not .Optimize }} -func (p *parser) print(prefix, s string) string { - if !p.debug { - return s - } - - fmt.Printf("%s %d:%d:%d: %s [%#U]\n", - prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) - return s -} - -func (p *parser) in(s string) string { - p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) -} - -func (p *parser) out(s string) string { - p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) -} - -// {{ end }} ==template== - -func (p *parser) addErr(err error) { - p.addErrAt(err, p.pt.position, []string{}) -} - -func (p *parser) addErrAt(err error, pos position, expected []string) { - var buf bytes.Buffer - if p.filename != "" { - buf.WriteString(p.filename) - } - if buf.Len() > 0 { - buf.WriteString(":") - } - buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) - if len(p.rstack) > 0 { - if buf.Len() > 0 { - buf.WriteString(": ") - } - rule := p.rstack[len(p.rstack)-1] - if rule.displayName != "" { - buf.WriteString("rule " + rule.displayName) - } else { - buf.WriteString("rule " + rule.name) - } - } - pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} - p.errs.add(pe) -} - -func (p *parser) failAt(fail bool, pos position, want string) { - // process fail if parsing fails and not inverted or parsing succeeds and invert is set - if fail == p.maxFailInvertExpected { - if pos.offset < p.maxFailPos.offset { - return - } - - if pos.offset > p.maxFailPos.offset { - p.maxFailPos = pos - p.maxFailExpected = p.maxFailExpected[:0] - } - - if p.maxFailInvertExpected { - want = "!" + want - } - p.maxFailExpected = append(p.maxFailExpected, want) - } -} - -// read advances the parser to the next rune. -func (p *parser) read() { - p.pt.offset += p.pt.w - rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) - p.pt.rn = rn - p.pt.w = n - p.pt.col++ - if rn == '\n' { - p.pt.line++ - p.pt.col = 0 - } - - if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune - if !p.allowInvalidUTF8 { - p.addErr(errInvalidEncoding) - } - } -} - -// restore parser position to the savepoint pt. -func (p *parser) restore(pt savepoint) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("restore")) - } - // {{ end }} ==template== - if pt.offset == p.pt.offset { - return - } - p.pt = pt -} - -// ==template== {{ if or .GlobalState (not .Optimize) }} - -// Cloner is implemented by any value that has a Clone method, which returns a -// copy of the value. This is mainly used for types which are not passed by -// value (e.g map, slice, chan) or structs that contain such types. -// -// This is used in conjunction with the global state feature to create proper -// copies of the state to allow the parser to properly restore the state in -// the case of backtracking. -type Cloner interface { - Clone() interface{} -} - -// clone and return parser current state. -func (p *parser) cloneState() storeDict { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("cloneState")) - } - // {{ end }} ==template== - - state := make(storeDict, len(p.cur.state)) - for k, v := range p.cur.state { - if c, ok := v.(Cloner); ok { - state[k] = c.Clone() - } else { - state[k] = v - } - } - return state -} - -// restore parser current state to the state storeDict. -// every restoreState should applied only one time for every cloned state -func (p *parser) restoreState(state storeDict) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("restoreState")) - } - // {{ end }} ==template== - p.cur.state = state -} - -// {{ end }} ==template== - -// get the slice of bytes from the savepoint start to the current position. -func (p *parser) sliceFrom(start savepoint) []byte { - return p.data[start.position.offset:p.pt.position.offset] -} - -// ==template== {{ if not .Optimize }} -func (p *parser) getMemoized(node interface{}) (resultTuple, bool) { - if len(p.memo) == 0 { - return resultTuple{}, false - } - m := p.memo[p.pt.offset] - if len(m) == 0 { - return resultTuple{}, false - } - res, ok := m[node] - return res, ok -} - -func (p *parser) setMemoized(pt savepoint, node interface{}, tuple resultTuple) { - if p.memo == nil { - p.memo = make(map[int]map[interface{}]resultTuple) - } - m := p.memo[pt.offset] - if m == nil { - m = make(map[interface{}]resultTuple) - p.memo[pt.offset] = m - } - m[node] = tuple -} - -// {{ end }} ==template== - -func (p *parser) buildRulesTable(g *grammar) { - p.rules = make(map[string]*rule, len(g.rules)) - for _, r := range g.rules { - p.rules[r.name] = r - } -} - -//{{ if .Nolint }} nolint: gocyclo {{else}} ==template== {{ end }} -func (p *parser) parse(g *grammar) (val interface{}, err error) { - if len(g.rules) == 0 { - p.addErr(errNoRule) - return nil, p.errs.err() - } - - // TODO : not super critical but this could be generated - p.buildRulesTable(g) - - if p.recover { - // panic can be used in action code to stop parsing immediately - // and return the panic as an error. - defer func() { - if e := recover(); e != nil { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("panic handler")) - } - // {{ end }} ==template== - val = nil - switch e := e.(type) { - case error: - p.addErr(e) - default: - p.addErr(fmt.Errorf("%v", e)) - } - err = p.errs.err() - } - }() - } - - startRule, ok := p.rules[p.entrypoint] - if !ok { - p.addErr(errInvalidEntrypoint) - return nil, p.errs.err() - } - - p.read() // advance to first rune - val, ok = p.parseRule(startRule) - if !ok { - if len(*p.errs) == 0 { - // If parsing fails, but no errors have been recorded, the expected values - // for the farthest parser position are returned as error. - maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) - for _, v := range p.maxFailExpected { - maxFailExpectedMap[v] = struct{}{} - } - expected := make([]string, 0, len(maxFailExpectedMap)) - eof := false - if _, ok := maxFailExpectedMap["!."]; ok { - delete(maxFailExpectedMap, "!.") - eof = true - } - for k := range maxFailExpectedMap { - expected = append(expected, k) - } - sort.Strings(expected) - if eof { - expected = append(expected, "EOF") - } - p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) - } - - return nil, p.errs.err() - } - return val, p.errs.err() -} - -func listJoin(list []string, sep string, lastSep string) string { - switch len(list) { - case 0: - return "" - case 1: - return list[0] - default: - return fmt.Sprintf("%s %s %s", strings.Join(list[:len(list)-1], sep), lastSep, list[len(list)-1]) - } -} - -func (p *parser) parseRule(rule *rule) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseRule " + rule.name)) - } - - if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } - } - - start := p.pt - // {{ end }} ==template== - p.rstack = append(p.rstack, rule) - p.pushV() - val, ok := p.parseExpr(rule.expr) - p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] - // ==template== {{ if not .Optimize }} - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } - // {{ end }} ==template== - return val, ok -} - -//{{ if .Nolint }} nolint: gocyclo {{else}} ==template== {{ end }} -func (p *parser) parseExpr(expr interface{}) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - var pt savepoint - - if p.memoize { - res, ok := p.getMemoized(expr) - if ok { - p.restore(res.end) - return res.v, res.b - } - pt = p.pt - } - - // {{ end }} ==template== - - p.ExprCnt++ - if p.ExprCnt > p.maxExprCnt { - panic(errMaxExprCnt) - } - - var val interface{} - var ok bool - switch expr := expr.(type) { - case *actionExpr: - val, ok = p.parseActionExpr(expr) - case *andCodeExpr: - val, ok = p.parseAndCodeExpr(expr) - case *andExpr: - val, ok = p.parseAndExpr(expr) - case *anyMatcher: - val, ok = p.parseAnyMatcher(expr) - case *charClassMatcher: - val, ok = p.parseCharClassMatcher(expr) - case *choiceExpr: - val, ok = p.parseChoiceExpr(expr) - case *labeledExpr: - val, ok = p.parseLabeledExpr(expr) - case *litMatcher: - val, ok = p.parseLitMatcher(expr) - case *notCodeExpr: - val, ok = p.parseNotCodeExpr(expr) - case *notExpr: - val, ok = p.parseNotExpr(expr) - case *oneOrMoreExpr: - val, ok = p.parseOneOrMoreExpr(expr) - case *recoveryExpr: - val, ok = p.parseRecoveryExpr(expr) - case *ruleRefExpr: - val, ok = p.parseRuleRefExpr(expr) - case *seqExpr: - val, ok = p.parseSeqExpr(expr) - // ==template== {{ if or .GlobalState (not .Optimize) }} - case *stateCodeExpr: - val, ok = p.parseStateCodeExpr(expr) - // {{ end }} ==template== - case *throwExpr: - val, ok = p.parseThrowExpr(expr) - case *zeroOrMoreExpr: - val, ok = p.parseZeroOrMoreExpr(expr) - case *zeroOrOneExpr: - val, ok = p.parseZeroOrOneExpr(expr) - default: - panic(fmt.Sprintf("unknown expression type %T", expr)) - } - // ==template== {{ if not .Optimize }} - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } - // {{ end }} ==template== - return val, ok -} - -func (p *parser) parseActionExpr(act *actionExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseActionExpr")) - } - - // {{ end }} ==template== - start := p.pt - val, ok := p.parseExpr(act.expr) - if ok { - p.cur.pos = start.position - p.cur.text = p.sliceFrom(start) - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - actVal, err := act.run(p) - if err != nil { - p.addErrAt(err, start.position, []string{}) - } - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - - val = actVal - } - // ==template== {{ if not .Optimize }} - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - // {{ end }} ==template== - return val, ok -} - -func (p *parser) parseAndCodeExpr(and *andCodeExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseAndCodeExpr")) - } - - // {{ end }} ==template== - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - - ok, err := and.run(p) - if err != nil { - p.addErr(err) - } - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - - return nil, ok -} - -func (p *parser) parseAndExpr(and *andExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseAndExpr")) - } - - // {{ end }} ==template== - pt := p.pt - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - p.pushV() - _, ok := p.parseExpr(and.expr) - p.popV() - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - p.restore(pt) - - return nil, ok -} - -func (p *parser) parseAnyMatcher(any *anyMatcher) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseAnyMatcher")) - } - - // {{ end }} ==template== - if p.pt.rn == utf8.RuneError && p.pt.w == 0 { - // EOF - see utf8.DecodeRune - p.failAt(false, p.pt.position, ".") - return nil, false - } - start := p.pt - p.read() - p.failAt(true, start.position, ".") - return p.sliceFrom(start), true -} - -//{{ if .Nolint }} nolint: gocyclo {{else}} ==template== {{ end }} -func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseCharClassMatcher")) - } - - // {{ end }} ==template== - cur := p.pt.rn - start := p.pt - - // ==template== {{ if .BasicLatinLookupTable }} - if cur < 128 { - if chr.basicLatinChars[cur] != chr.inverted { - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - p.failAt(false, start.position, chr.val) - return nil, false - } - // {{ end }} ==template== - - // can't match EOF - if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune - p.failAt(false, start.position, chr.val) - return nil, false - } - - if chr.ignoreCase { - cur = unicode.ToLower(cur) - } - - // try to match in the list of available chars - for _, rn := range chr.chars { - if rn == cur { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - // try to match in the list of ranges - for i := 0; i < len(chr.ranges); i += 2 { - if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - // try to match in the list of Unicode classes - for _, cl := range chr.classes { - if unicode.Is(cl, cur) { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - if chr.inverted { - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - p.failAt(false, start.position, chr.val) - return nil, false -} - -// ==template== {{ if not .Optimize }} - -func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) - m := p.ChoiceAltCnt[choiceIdent] - if m == nil { - m = make(map[string]int) - p.ChoiceAltCnt[choiceIdent] = m - } - // We increment altI by 1, so the keys do not start at 0 - alt := strconv.Itoa(altI + 1) - if altI == choiceNoMatch { - alt = p.choiceNoMatch - } - m[alt]++ -} - -// {{ end }} ==template== - -func (p *parser) parseChoiceExpr(ch *choiceExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseChoiceExpr")) - } - - // {{ end }} ==template== - for altI, alt := range ch.alternatives { - // dummy assignment to prevent compile error if optimized - _ = altI - - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - - p.pushV() - val, ok := p.parseExpr(alt) - p.popV() - if ok { - // ==template== {{ if not .Optimize }} - p.incChoiceAltCnt(ch, altI) - // {{ end }} ==template== - return val, ok - } - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - } - // ==template== {{ if not .Optimize }} - p.incChoiceAltCnt(ch, choiceNoMatch) - // {{ end }} ==template== - return nil, false -} - -func (p *parser) parseLabeledExpr(lab *labeledExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseLabeledExpr")) - } - - // {{ end }} ==template== - p.pushV() - val, ok := p.parseExpr(lab.expr) - p.popV() - if ok && lab.label != "" { - m := p.vstack[len(p.vstack)-1] - m[lab.label] = val - } - return val, ok -} - -func (p *parser) parseLitMatcher(lit *litMatcher) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseLitMatcher")) - } - - // {{ end }} ==template== - ignoreCase := "" - if lit.ignoreCase { - ignoreCase = "i" - } - val := fmt.Sprintf("%q%s", lit.val, ignoreCase) - start := p.pt - for _, want := range lit.val { - cur := p.pt.rn - if lit.ignoreCase { - cur = unicode.ToLower(cur) - } - if cur != want { - p.failAt(false, start.position, val) - p.restore(start) - return nil, false - } - p.read() - } - p.failAt(true, start.position, val) - return p.sliceFrom(start), true -} - -func (p *parser) parseNotCodeExpr(not *notCodeExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseNotCodeExpr")) - } - - // {{ end }} ==template== - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - - // {{ end }} ==template== - ok, err := not.run(p) - if err != nil { - p.addErr(err) - } - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - - return nil, !ok -} - -func (p *parser) parseNotExpr(not *notExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseNotExpr")) - } - - // {{ end }} ==template== - pt := p.pt - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - p.pushV() - p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) - p.maxFailInvertExpected = !p.maxFailInvertExpected - p.popV() - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - p.restore(pt) - - return nil, !ok -} - -func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseOneOrMoreExpr")) - } - - // {{ end }} ==template== - var vals []interface{} - - for { - p.pushV() - val, ok := p.parseExpr(expr.expr) - p.popV() - if !ok { - if len(vals) == 0 { - // did not match once, no match - return nil, false - } - return vals, true - } - vals = append(vals, val) - } -} - -func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) - } - - // {{ end }} ==template== - - p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) - p.popRecovery() - - return val, ok -} - -func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseRuleRefExpr " + ref.name)) - } - - // {{ end }} ==template== - if ref.name == "" { - panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) - } - - rule := p.rules[ref.name] - if rule == nil { - p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) - return nil, false - } - return p.parseRule(rule) -} - -func (p *parser) parseSeqExpr(seq *seqExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseSeqExpr")) - } - - // {{ end }} ==template== - vals := make([]interface{}, 0, len(seq.exprs)) - - pt := p.pt - // ==template== {{ if or .GlobalState (not .Optimize) }} - state := p.cloneState() - // {{ end }} ==template== - for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) - if !ok { - // ==template== {{ if or .GlobalState (not .Optimize) }} - p.restoreState(state) - // {{ end }} ==template== - p.restore(pt) - return nil, false - } - vals = append(vals, val) - } - return vals, true -} - -// ==template== {{ if or .GlobalState (not .Optimize) }} - -func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseStateCodeExpr")) - } - - // {{ end }} ==template== - err := state.run(p) - if err != nil { - p.addErr(err) - } - return nil, true -} - -// {{ end }} ==template== - -func (p *parser) parseThrowExpr(expr *throwExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseThrowExpr")) - } - - // {{ end }} ==template== - - for i := len(p.recoveryStack) - 1; i >= 0; i-- { - if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { - return val, ok - } - } - } - - return nil, false -} - -func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseZeroOrMoreExpr")) - } - - // {{ end }} ==template== - var vals []interface{} - - for { - p.pushV() - val, ok := p.parseExpr(expr.expr) - p.popV() - if !ok { - return vals, true - } - vals = append(vals, val) - } -} - -func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (interface{}, bool) { - // ==template== {{ if not .Optimize }} - if p.debug { - defer p.out(p.in("parseZeroOrOneExpr")) - } - - // {{ end }} ==template== - p.pushV() - val, _ := p.parseExpr(expr.expr) - p.popV() - // whether it matched or not, consider it a match - return val, true -} diff --git a/vendor/github.com/mna/pigeon/builder/static_code_range_table.go b/vendor/github.com/mna/pigeon/builder/static_code_range_table.go deleted file mode 100644 index d410509606..0000000000 --- a/vendor/github.com/mna/pigeon/builder/static_code_range_table.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:generate go run ../bootstrap/cmd/static_code_generator/main.go -- $GOFILE generated_$GOFILE rangeTable0 - -package builder - -import ( - "fmt" - "unicode" -) - -// IMPORTANT: All code below this line is added to the parser as static code -func rangeTable(class string) *unicode.RangeTable { - if rt, ok := unicode.Categories[class]; ok { - return rt - } - if rt, ok := unicode.Properties[class]; ok { - return rt - } - if rt, ok := unicode.Scripts[class]; ok { - return rt - } - - // cannot happen - panic(fmt.Sprintf("invalid Unicode class: %s", class)) -} diff --git a/vendor/github.com/mna/pigeon/doc.go b/vendor/github.com/mna/pigeon/doc.go deleted file mode 100644 index 0b94889fb4..0000000000 --- a/vendor/github.com/mna/pigeon/doc.go +++ /dev/null @@ -1,594 +0,0 @@ -/* -Command pigeon generates parsers in Go from a PEG grammar. - -From Wikipedia [0]: - - A parsing expression grammar is a type of analytic formal grammar, i.e. - it describes a formal language in terms of a set of rules for recognizing - strings in the language. - -Its features and syntax are inspired by the PEG.js project [1], while -the implementation is loosely based on [2]. Formal presentation of the -PEG theory by Bryan Ford is also an important reference [3]. An introductory -blog post can be found at [4]. - - [0]: http://en.wikipedia.org/wiki/Parsing_expression_grammar - [1]: http://pegjs.org/ - [2]: http://www.codeproject.com/Articles/29713/Parsing-Expression-Grammar-Support-for-C-Part - [3]: http://pdos.csail.mit.edu/~baford/packrat/popl04/peg-popl04.pdf - [4]: http://0value.com/A-PEG-parser-generator-for-Go - -Command-line usage - -The pigeon tool must be called with PEG input as defined -by the accepted PEG syntax below. The grammar may be provided by a -file or read from stdin. The generated parser is written to stdout -by default. - - pigeon [options] [GRAMMAR_FILE] - -The following options can be specified: - - -cache : cache parser results to avoid exponential parsing time in - pathological cases. Can make the parsing slower for typical - cases and uses more memory (default: false). - - -debug : boolean, print debugging info to stdout (default: false). - - -nolint: add '// nolint: ...' comments for generated parser to suppress - warnings by gometalinter (https://github.com/alecthomas/gometalinter). - - -no-recover : boolean, if set, do not recover from a panic. Useful - to access the panic stack when debugging, otherwise the panic - is converted to an error (default: false). - - -o=FILE : string, output file where the generated parser will be - written (default: stdout). - - -optimize-basic-latin : boolean, if set, a lookup table for the first 128 - characters of the Unicode table (Basic Latin) is generated for each character - class matcher. This speeds up the parsing, if parsed data mainly consists - of characters from this range (default: false). - - -optimize-grammar : boolean, (EXPERIMENTAL FEATURE) if set, several performance - optimizations on the grammar are performed, with focus to the reduction of the - grammar depth. - Optimization: - * removal of unreferenced rules - * replace rule references with a copy of the referenced Rule, if the - referenced rule it self has no references. - * resolve nested choice expressions - * resolve choice expressions with only one alternative - * resolve nested sequences expression - * resolve sequence expressions with only one element - * combine character class matcher and literal matcher, where possible - The resulting grammar is usually more memory consuming, but faster for parsing. - The optimization of the grammar is done in multiple rounds (optimize until no - more optimizations have applied). This process takes some time, depending on the - optimization potential of the grammar. - - -optimize-parser : boolean, if set, the options Debug, Memoize and Statistics are - removed from the resulting parser. The global "state" is optimized as well by - either removing all related code if no state change expression is present in the - grammar or by removing the restoration of the global "state" store after action - and predicate code blocks. This saves a few cpu cycles, when using the generated - parser (default: false). - - -x : boolean, if set, do not build the parser, just parse the input grammar - (default: false). - - -receiver-name=NAME : string, name of the receiver variable for the generated - code blocks. Non-initializer code blocks in the grammar end up as methods on the - *current type, and this option sets the name of the receiver (default: c). - - -alternate-entrypoints=RULE[,RULE...] : string, comma-separated list of rule names - that may be used as alternate entrypoints for the parser, in addition to the - default entrypoint (the first rule in the grammar) (default: none). - Such entrypoints can be specified in the call to Parse by passing an - Entrypoint option that specifies the alternate rule name to use. This is only - necessary if the -optimize-parser flag is set, as some rules may be optimized - out of the resulting parser. - -If the code blocks in the grammar (see below, section "Code block") are golint- -and go vet-compliant, then the resulting generated code will also be golint- -and go vet-compliant. - -The generated code doesn't use any third-party dependency unless code blocks -in the grammar require such a dependency. - -PEG syntax - -The accepted syntax for the grammar is formally defined in the -grammar/pigeon.peg file, using the PEG syntax. What follows is an informal -description of this syntax. - -Identifiers, whitespace, comments and literals follow the same -notation as the Go language, as defined in the language specification -(http://golang.org/ref/spec#Source_code_representation): - - // single line comment*/ -// /* multi-line comment */ -/* 'x' (single quotes for single char literal) - "double quotes for string literal" - `backtick quotes for raw string literal` - RuleName (a valid identifier) - -The grammar must be Unicode text encoded in UTF-8. New lines are identified -by the \n character (U+000A). Space (U+0020), horizontal tabs (U+0009) and -carriage returns (U+000D) are considered whitespace and are ignored except -to separate tokens. - -Rules - -A PEG grammar consists of a set of rules. A rule is an identifier followed -by a rule definition operator and an expression. An optional display name - -a string literal used in error messages instead of the rule identifier - can -be specified after the rule identifier. E.g.: - RuleA "friendly name" = 'a'+ // RuleA is one or more lowercase 'a's - -The rule definition operator can be any one of those: - =, <-, ← (U+2190), ⟵ (U+27F5) - -Expressions - -A rule is defined by an expression. The following sections describe the -various expression types. Expressions can be grouped by using parentheses, -and a rule can be referenced by its identifier in place of an expression. - -Choice expression - -The choice expression is a list of expressions that will be tested in the -order they are defined. The first one that matches will be used. Expressions -are separated by the forward slash character "/". E.g.: - ChoiceExpr = A / B / C // A, B and C should be rules declared in the grammar - -Because the first match is used, it is important to think about the order -of expressions. For example, in this rule, "<=" would never be used because -the "<" expression comes first: - BadChoiceExpr = "<" / "<=" - -Sequence expression - -The sequence expression is a list of expressions that must all match in -that same order for the sequence expression to be considered a match. -Expressions are separated by whitespace. E.g.: - SeqExpr = "A" "b" "c" // matches "Abc", but not "Acb" - -Labeled expression - -A labeled expression consists of an identifier followed by a colon ":" -and an expression. A labeled expression introduces a variable named with -the label that can be referenced in the code blocks in the same scope. -The variable will have the value of the expression that follows the colon. -E.g.: - LabeledExpr = value:[a-z]+ { - fmt.Println(value) - return value, nil - } - -The variable is typed as an empty interface, and the underlying type depends -on the following: - -For terminals (character and string literals, character classes and -the any matcher), the value is []byte. E.g.: - Rule = label:'a' { // label is []byte } - -For predicates (& and !), the value is always nil. E.g.: - Rule = label:&'a' { // label is nil } - -For a sequence, the value is a slice of empty interfaces, one for each -expression value in the sequence. The underlying types of each value -in the slice follow the same rules described here, recursively. E.g.: - Rule = label:('a' 'b') { // label is []interface{} } - -For a repetition (+ and *), the value is a slice of empty interfaces, one for -each repetition. The underlying types of each value in the slice follow -the same rules described here, recursively. E.g.: - Rule = label:[a-z]+ { // label is []interface{} } - -For a choice expression, the value is that of the matching choice. E.g.: - Rule = label:('a' / 'b') { // label is []byte } - -For the optional expression (?), the value is nil or the value of the -expression. E.g.: - Rule = label:'a'? { // label is nil or []byte } - -Of course, the type of the value can be anything once an action code block -is used. E.g.: - RuleA = label:'3' { - return 3, nil - } - RuleB = label:RuleA { // label is int } - -And and not expressions - -An expression prefixed with the ampersand "&" is the "and" predicate -expression: it is considered a match if the following expression is a match, -but it does not consume any input. - -An expression prefixed with the exclamation point "!" is the "not" predicate -expression: it is considered a match if the following expression is not -a match, but it does not consume any input. E.g.: - AndExpr = "A" &"B" // matches "A" if followed by a "B" (does not consume "B") - NotExpr = "A" !"B" // matches "A" if not followed by a "B" (does not consume "B") - -The expression following the & and ! operators can be a code block. In that -case, the code block must return a bool and an error. The operator's semantic -is the same, & is a match if the code block returns true, ! is a match if the -code block returns false. The code block has access to any labeled value -defined in its scope. E.g.: - CodeAndExpr = value:[a-z] &{ - // can access the value local variable... - return true, nil - } - -Repeating expressions - -An expression followed by "*", "?" or "+" is a match if the expression -occurs zero or more times ("*"), zero or one time "?" or one or more times -("+") respectively. The match is greedy, it will match as many times as -possible. E.g. - ZeroOrMoreAs = "A"* - -Literal matcher - -A literal matcher tries to match the input against a single character or a -string literal. The literal may be a single-quoted single character, a -double-quoted string or a backtick-quoted raw string. The same rules as in Go -apply regarding the allowed characters and escapes. - -The literal may be followed by a lowercase "i" (outside the ending quote) -to indicate that the match is case-insensitive. E.g.: - LiteralMatch = "Awesome\n"i // matches "awesome" followed by a newline - -Character class matcher - -A character class matcher tries to match the input against a class of characters -inside square brackets "[...]". Inside the brackets, characters represent -themselves and the same escapes as in string literals are available, except -that the single- and double-quote escape is not valid, instead the closing -square bracket "]" must be escaped to be used. - -Character ranges can be specified using the "[a-z]" notation. Unicode -classes can be specified using the "[\pL]" notation, where L is a -single-letter Unicode class of characters, or using the "[\p{Class}]" -notation where Class is a valid Unicode class (e.g. "Latin"). - -As for string literals, a lowercase "i" may follow the matcher (outside -the ending square bracket) to indicate that the match is case-insensitive. -A "^" as first character inside the square brackets indicates that the match -is inverted (it is a match if the input does not match the character class -matcher). E.g.: - NotAZ = [^a-z]i - -Any matcher - -The any matcher is represented by the dot ".". It matches any character -except the end of file, thus the "!." expression is used to indicate "match -the end of file". E.g.: - AnyChar = . // match a single character - EOF = !. - -Code block - -Code blocks can be added to generate custom Go code. There are three kinds -of code blocks: the initializer, the action and the predicate. All code blocks -appear inside curly braces "{...}". - -The initializer must appear first in the grammar, before any rule. It is -copied as-is (minus the wrapping curly braces) at the top of the generated -parser. It may contain function declarations, types, variables, etc. just -like any Go file. Every symbol declared here will be available to all other -code blocks. Although the initializer is optional in a valid grammar, it is -usually required to generate a valid Go source code file (for the package -clause). E.g.: - { - package main - - func someHelper() { - // ... - } - } - -Action code blocks are code blocks declared after an expression in a rule. -Those code blocks are turned into a method on the "*current" type in the -generated source code. The method receives any labeled expression's value -as argument (as interface{}) and must return two values, the first being -the value of the expression (an interface{}), and the second an error. -If a non-nil error is returned, it is added to the list of errors that the -parser will return. E.g.: - RuleA = "A"+ { - // return the matched string, "c" is the default name for - // the *current receiver variable. - return string(c.text), nil - } - -Predicate code blocks are code blocks declared immediately after the and "&" -or the not "!" operators. Like action code blocks, predicate code blocks -are turned into a method on the "*current" type in the generated source code. -The method receives any labeled expression's value as argument (as interface{}) -and must return two opt, the first being a bool and the second an error. -If a non-nil error is returned, it is added to the list of errors that the -parser will return. E.g.: - RuleAB = [ab]i+ &{ - return true, nil - } - -State change code blocks are code blocks starting with "#". In contrast to -action and predicate code blocks, state change code blocks are allowed to -modify values in the global "state" store (see below). -State change code blocks are turned into a method on the "*current" type -in the generated source code. -The method is passed any labeled expression's value as an argument (of type -interface{}) and must return a value of type error. -If a non-nil error is returned, it is added to the list of errors that the -parser will return, note that the parser does NOT backtrack if a non-nil -error is returned. -E.g: - Rule = [a] #{ - c.state["a"]++ - if c.state["a"] > 5 { - return fmt.Errorf("we have seen more than 5 a's") // parser will not backtrack - } - return nil - } -The "*current" type is a struct that provides four useful fields that can be -accessed in action, state change, and predicate code blocks: "pos", "text", -"state" and "globalStore". - -The "pos" field indicates the current position of the parser in the source -input. It is itself a struct with three fields: "line", "col" and "offset". -Line is a 1-based line number, col is a 1-based column number that counts -runes from the start of the line, and offset is a 0-based byte offset. - -The "text" field is the slice of bytes of the current match. It is empty -in a predicate code block. - -The "state" field is a global store, with backtrack support, of type -"map[string]interface{}". The values in the store are tied to the parser's -backtracking, in particular if a rule fails to match then all updates to the -state that occurred in the process of matching the rule are rolled back. For a -key-value store that is not tied to the parser's backtracking, see the -"globalStore". -The values in the "state" store are available for read access in action and -predicate code blocks, any changes made to the "state" store will be reverted -once the action or predicate code block is finished running. To update values -in the "state" use state change code blocks ("#{}"). - -IMPORTANT: - - In order to properly roll back the state if a rule fails to match the - parser must clone the state before trying to match a rule. - - The default clone mechanism makes a "shallow" copy of each value in the - "state", this implies that pointers, maps, slices, channels, and structs - containing any of the previous types are not properly copied. - - To support theses cases pigeon offers the "Cloner" interface which - consists of a single method "Clone". If a value stored in the "state" - store implements this interface, the "Clone" method is used to obtain a - proper copy. - - If a general solution is needed, external libraries which provide deep - copy functionality may be used in the "Clone" method - (e.g. https://github.com/mitchellh/copystructure). - -The "globalStore" field is a global store of type "map[string]interface{}", -which allows to store arbitrary values, which are available in action and -predicate code blocks for read as well as write access. -It is important to notice, that the global store is completely independent from -the backtrack mechanism of PEG and is therefore not set back to its old state -during backtrack. -The initialization of the global store may be achieved by using the GlobalStore -function (http://godoc.org/github.com/mna/pigeon/test/predicates#GlobalStore). -Be aware, that all keys starting with "_pigeon" are reserved for internal use -of pigeon and should not be used nor modified. Those keys are treated as -internal implementation details and therefore there are no guarantees given in -regards of API stability. - -Failure labels, throw and recover - -pigeon supports an extension of the classical PEG syntax called failure labels, -proposed by Maidl et al. in their paper "Error Reporting in Parsing Expression Grammars" [7]. -The used syntax for the introduced expressions is borrowed from their lpeglabel [8] -implementation. - -This extension allows to signal different kinds of errors and to specify, which -recovery pattern should handle a given label. - -With labeled failures it is possible to distinguish between an ordinary failure -and an error. Usually, an ordinary failure is produced when the matching of a -character fails, and this failure is caught by ordered choice. An error -(a non-ordinary failure), by its turn, is produced by the throw operator and -may be caught by the recovery operator. - -In pigeon, the recovery expression consists of the regular expression, the recovery -expression and a set of labels to be matched. First, the regular expression is tried. -If this fails with one of the provided labels, the recovery expression is tried. If -this fails as well, the error is propagated. E.g.: - FailureRecoveryExpr = RegularExpr //{FailureLabel1, FailureLabel2} RecoveryExpr - -To signal a failure condition, the throw expression is used. E.g.: - ThrowExpr = %{FailureLabel1} - -For concrete examples, how to use throw and recover, have a look at the examples -"labeled_failures" and "thrownrecover" in the "test" folder. - -The implementation of the throw and recover operators work as follows: -The failure recover expression adds the recover expression for every failure label -to the recovery stack and runs the regular expression. -The throw expression checks the recovery stack in reversed order for the provided -failure label. If the label is found, the respective recovery expression is run. If -this expression is successful, the parser continues the processing of the input. If -the recovery expression is not successful, the parsing fails and the parser starts -to backtrack. - -If throw and recover expressions are used together with global state, it is the -responsibility of the author of the grammar to reset the global state to a valid -state during the recovery operation. - - [7]: https://arxiv.org/pdf/1405.6646v3.pdf - [8]: https://github.com/sqmedeiros/lpeglabel - -Using the generated parser - -The parser generated by pigeon exports a few symbols so that it can be used -as a package with public functions to parse input text. The exported API is: - - Parse(string, []byte, ...Option) (interface{}, error) - - ParseFile(string, ...Option) (interface{}, error) - - ParseReader(string, io.Reader, ...Option) (interface{}, error) - - AllowInvalidUTF8(bool) Option - - Debug(bool) Option - - Entrypoint(string) Option - - GlobalStore(string, interface{}) Option - - MaxExpressions(uint64) Option - - Memoize(bool) Option - - Recover(bool) Option - - Statistics(*Stats) Option - -See the godoc page of the generated parser for the test/predicates grammar -for an example documentation page of the exported API: -http://godoc.org/github.com/mna/pigeon/test/predicates. - -Like the grammar used to generate the parser, the input text must be -UTF-8-encoded Unicode. - -The start rule of the parser is the first rule in the PEG grammar used -to generate the parser. A call to any of the Parse* functions returns -the value generated by executing the grammar on the provided input text, -and an optional error. - -Typically, the grammar should generate some kind of abstract syntax tree (AST), -but for simple grammars it may evaluate the result immediately, such as in -the examples/calculator example. There are no constraints imposed on the -author of the grammar, it can return whatever is needed. - -Error reporting - -When the parser returns a non-nil error, the error is always of type errList, -which is defined as a slice of errors ([]error). Each error in the list is -of type *parserError. This is a struct that has an "Inner" field that can be -used to access the original error. - -So if a code block returns some well-known error like: - { - return nil, io.EOF - } - -The original error can be accessed this way: - _, err := ParseFile("some_file") - if err != nil { - list := err.(errList) - for _, err := range list { - pe := err.(*parserError) - if pe.Inner == io.EOF { - // ... - } - } - } - -By defaut the parser will continue after an error is returned and will -cumulate all errors found during parsing. If the grammar reaches a point -where it shouldn't continue, a panic statement can be used to terminate -parsing. The panic will be caught at the top-level of the Parse* call -and will be converted into a *parserError like any error, and an errList -will still be returned to the caller. - -The divide by zero error in the examples/calculator grammar leverages this -feature (no special code is needed to handle division by zero, if it -happens, the runtime panics and it is recovered and returned as a parsing -error). - -Providing good error reporting in a parser is not a trivial task. Part -of it is provided by the pigeon tool, by offering features such as -filename, position, expected literals and rule name in the error message, -but an important part of good error reporting needs to be done by the grammar -author. - -For example, many programming languages use double-quotes for string literals. -Usually, if the opening quote is found, the closing quote is expected, and if -none is found, there won't be any other rule that will match, there's no need -to backtrack and try other choices, an error should be added to the list -and the match should be consumed. - -In order to do this, the grammar can look something like this: - - StringLiteral = '"' ValidStringChar* '"' { - // this is the valid case, build string literal node - // node = ... - return node, nil - } / '"' ValidStringChar* !'"' { - // invalid case, build a replacement string literal node or build a BadNode - // node = ... - return node, errors.New("string literal not terminated") - } - -This is just one example, but it illustrates the idea that error reporting -needs to be thought out when designing the grammar. - -Because the above mentioned error types (errList and parserError) are not -exported, additional steps have to be taken, ff the generated parser is used as -library package in other packages (e.g. if the same parser is used in multiple -command line tools). -One possible implementation for exported errors (based on interfaces) and -customized error reporting (caret style formatting of the position, where -the parsing failed) is available in the json example and its command line tool: -http://godoc.org/github.com/mna/pigeon/examples/json - -API stability - -Generated parsers have user-provided code mixed with pigeon code -in the same package, so there is no package -boundary in the resulting code to prevent access to unexported symbols. -What is meant to be implementation -details in pigeon is also available to user code - which doesn't mean -it should be used. - -For this reason, it is important to precisely define what is intended to be -the supported API of pigeon, the parts that will be stable -in future versions. - -The "stability" of the version 1.0 API attempts to make a similar guarantee -as the Go 1 compatibility [5]. The following lists what part of the -current pigeon code falls under that guarantee (features may be added in -the future): - - - The pigeon command-line flags and arguments: those will not be removed - and will maintain the same semantics. - - - The explicitly exported API generated by pigeon. See [6] for the - documentation of this API on a generated parser. - - - The PEG syntax, as documented above. - - - The code blocks (except the initializer) will always be generated as - methods on the *current type, and this type is guaranteed to have - the fields pos (type position) and text (type []byte). There are no - guarantees on other fields and methods of this type. - - - The position type will always have the fields line, col and offset, - all defined as int. There are no guarantees on other fields and methods - of this type. - - - The type of the error value returned by the Parse* functions, when - not nil, will always be errList defined as a []error. There are no - guarantees on methods of this type, other than the fact it implements the - error interface. - - - Individual errors in the errList will always be of type *parserError, - and this type is guaranteed to have an Inner field that contains the - original error value. There are no guarantees on other fields and methods - of this type. - -The above guarantee is given to the version 1.0 (https://github.com/mna/pigeon/releases/tag/v1.0.0) -of pigeon, which has entered maintenance mode (bug fixes only). The current -master branch includes the development toward a future version 2.0, which -intends to further improve pigeon. -While the given API stability should be maintained as far as it makes sense, -breaking changes may be necessary to be able to improve pigeon. -The new version 2.0 API has not yet stabilized and therefore changes to the API -may occur at any time. - -References: - - [5]: https://golang.org/doc/go1compat - [6]: http://godoc.org/github.com/mna/pigeon/test/predicates - -*/ -package main diff --git a/vendor/github.com/mna/pigeon/main.go b/vendor/github.com/mna/pigeon/main.go deleted file mode 100644 index a00be22e00..0000000000 --- a/vendor/github.com/mna/pigeon/main.go +++ /dev/null @@ -1,294 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "errors" - "flag" - "fmt" - "io" - "os" - "strconv" - "strings" - - "golang.org/x/tools/imports" - - "github.com/mna/pigeon/ast" - "github.com/mna/pigeon/builder" -) - -// exit function mockable for tests -var exit = os.Exit - -// ruleNamesFlag is a custom flag that parses a comma-separated -// list of rule names. It implements flag.Value. -type ruleNamesFlag []string - -func (r *ruleNamesFlag) String() string { - return fmt.Sprint(*r) -} - -func (r *ruleNamesFlag) Set(value string) error { - names := strings.Split(value, ",") - *r = append(*r, names...) - return nil -} - -func main() { - fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - - // define command-line flags - var ( - cacheFlag = fs.Bool("cache", false, "cache parsing results") - dbgFlag = fs.Bool("debug", false, "set debug mode") - shortHelpFlag = fs.Bool("h", false, "show help page") - longHelpFlag = fs.Bool("help", false, "show help page") - nolint = fs.Bool("nolint", false, "add '// nolint: ...' comments to suppress warnings by gometalinter") - noRecoverFlag = fs.Bool("no-recover", false, "do not recover from panic") - outputFlag = fs.String("o", "", "output file, defaults to stdout") - optimizeBasicLatinFlag = fs.Bool("optimize-basic-latin", false, "generate optimized parser for Unicode Basic Latin character sets") - optimizeGrammar = fs.Bool("optimize-grammar", false, "optimize the given grammar (EXPERIMENTAL FEATURE)") - optimizeParserFlag = fs.Bool("optimize-parser", false, "generate optimized parser without Debug and Memoize options") - recvrNmFlag = fs.String("receiver-name", "c", "receiver name for the generated methods") - noBuildFlag = fs.Bool("x", false, "do not build, only parse") - - altEntrypointsFlag ruleNamesFlag - ) - fs.Var(&altEntrypointsFlag, "alternate-entrypoints", "comma-separated list of rule names that may be used as entrypoints") - - fs.Usage = usage - err := fs.Parse(os.Args[1:]) - if err != nil { - fmt.Fprintln(os.Stderr, "args parse error:\n", err) - exit(6) - } - - if *shortHelpFlag || *longHelpFlag { - fs.Usage() - exit(0) - } - - if fs.NArg() > 1 { - argError(1, "expected one argument, got %q", strings.Join(fs.Args(), " ")) - } - - // get input source - infile := "" - if fs.NArg() == 1 { - infile = fs.Arg(0) - } - nm, rc := input(infile) - defer func() { - err = rc.Close() - if err != nil { - fmt.Fprintln(os.Stderr, "close file error:\n", err) - } - if r := recover(); r != nil { - panic(r) - } - if err != nil { - exit(7) - } - }() - - // parse input - g, err := ParseReader(nm, rc, Debug(*dbgFlag), Memoize(*cacheFlag), Recover(!*noRecoverFlag)) - if err != nil { - fmt.Fprintln(os.Stderr, "parse error(s):\n", err) - exit(3) - } - - // validate alternate entrypoints - grammar := g.(*ast.Grammar) - rules := make(map[string]struct{}, len(grammar.Rules)) - for _, rule := range grammar.Rules { - rules[rule.Name.Val] = struct{}{} - } - for _, entrypoint := range altEntrypointsFlag { - if entrypoint == "" { - continue - } - - if _, ok := rules[entrypoint]; !ok { - fmt.Fprintf(os.Stderr, "argument error:\nunknown rule name %s used as alternate entrypoint\n", entrypoint) - exit(9) - } - } - - if !*noBuildFlag { - if *optimizeGrammar { - ast.Optimize(grammar, altEntrypointsFlag...) - } - - // generate parser - out := output(*outputFlag) - defer func() { - err := out.Close() - if err != nil { - fmt.Fprintln(os.Stderr, "close file error:\n", err) - exit(8) - } - }() - - outBuf := bytes.NewBuffer([]byte{}) - - curNmOpt := builder.ReceiverName(*recvrNmFlag) - optimizeParser := builder.Optimize(*optimizeParserFlag) - basicLatinOptimize := builder.BasicLatinLookupTable(*optimizeBasicLatinFlag) - nolintOpt := builder.Nolint(*nolint) - if err := builder.BuildParser(outBuf, grammar, curNmOpt, optimizeParser, basicLatinOptimize, nolintOpt); err != nil { - fmt.Fprintln(os.Stderr, "build error: ", err) - exit(5) - } - - // Defaults from golang.org/x/tools/cmd/goimports - options := &imports.Options{ - TabWidth: 8, - TabIndent: true, - Comments: true, - Fragment: true, - } - - formattedBuf, err := imports.Process("filename", outBuf.Bytes(), options) - if err != nil { - if _, err := out.Write(outBuf.Bytes()); err != nil { - fmt.Fprintln(os.Stderr, "write error: ", err) - exit(7) - } - fmt.Fprintln(os.Stderr, "format error: ", err) - exit(6) - } - - if _, err := out.Write(formattedBuf); err != nil { - fmt.Fprintln(os.Stderr, "write error: ", err) - exit(7) - } - } -} - -var usagePage = `usage: %s [options] [GRAMMAR_FILE] - -Pigeon generates a parser based on a PEG grammar. - -By default, pigeon reads the grammar from stdin and writes the -generated parser to stdout. If GRAMMAR_FILE is specified, the -grammar is read from this file instead. If the -o flag is set, -the generated code is written to this file instead. - - -cache - cache parser results to avoid exponential parsing time in - pathological cases. Can make the parsing slower for typical - cases and uses more memory. - -debug - output debugging information while parsing the grammar. - -h -help - display this help message. - -nolint - add '// nolint: ...' comments for generated parser to suppress - warnings by gometalinter (https://github.com/alecthomas/gometalinter). - -no-recover - do not recover from a panic. Useful to access the panic stack - when debugging, otherwise the panic is converted to an error. - -o OUTPUT_FILE - write the generated parser to OUTPUT_FILE. Defaults to stdout. - -optimize-basic-latin - generate optimized parser for Unicode Basic Latin character set - -optimize-grammar - performes several performance optimizations on the grammar (EXPERIMENTAL FEATURE) - -optimize-parser - generate optimized parser without Debug and Memoize options and - with some other optimizations applied. - -receiver-name NAME - use NAME as for the receiver name of the generated methods - for the grammar's code blocks. Defaults to "c". - -x - do not generate the parser, only parse the grammar. - -alternate-entrypoints RULE[,RULE...] - comma-separated list of rule names that may be used as alternate - entrypoints for the parser, in addition to the first rule in the - grammar. - -See https://godoc.org/github.com/mna/pigeon for more information. -` - -// usage prints the help page of the command-line tool. -func usage() { - fmt.Printf(usagePage, os.Args[0]) -} - -// argError prints an error message to stderr, prints the command usage -// and exits with the specified exit code. -func argError(exitCode int, msg string, args ...interface{}) { - fmt.Fprintf(os.Stderr, msg, args...) - fmt.Fprintln(os.Stderr) - usage() - exit(exitCode) -} - -// input gets the name and reader to get input text from. -func input(filename string) (nm string, rc io.ReadCloser) { - nm = "stdin" - inf := os.Stdin - if filename != "" { - f, err := os.Open(filename) - if err != nil { - fmt.Fprintln(os.Stderr, err) - exit(2) - } - inf = f - nm = filename - } - r := bufio.NewReader(inf) - return nm, makeReadCloser(r, inf) -} - -// output gets the writer to write the generated parser to. -func output(filename string) io.WriteCloser { - out := os.Stdout - if filename != "" { - f, err := os.Create(filename) - if err != nil { - fmt.Fprintln(os.Stderr, err) - exit(4) - } - out = f - } - return out -} - -// create a ReadCloser that reads from r and closes c. -func makeReadCloser(r io.Reader, c io.Closer) io.ReadCloser { - rc := struct { - io.Reader - io.Closer - }{r, c} - return io.ReadCloser(rc) -} - -// astPos is a helper method for the PEG grammar parser. It returns the -// position of the current match as an ast.Pos. -func (c *current) astPos() ast.Pos { - return ast.Pos{Line: c.pos.line, Col: c.pos.col, Off: c.pos.offset} -} - -// toIfaceSlice is a helper function for the PEG grammar parser. It converts -// v to a slice of empty interfaces. -func toIfaceSlice(v interface{}) []interface{} { - if v == nil { - return nil - } - return v.([]interface{}) -} - -// validateUnicodeEscape checks that the provided escape sequence is a -// valid Unicode escape sequence. -func validateUnicodeEscape(escape, errMsg string) (interface{}, error) { - r, _, _, err := strconv.UnquoteChar("\\"+escape, '"') - if err != nil { - return nil, errors.New(errMsg) - } - if 0xD800 <= r && r <= 0xDFFF { - return nil, errors.New(errMsg) - } - return nil, nil -} diff --git a/vendor/github.com/mna/pigeon/pigeon.go b/vendor/github.com/mna/pigeon/pigeon.go deleted file mode 100644 index a73b85a9d0..0000000000 --- a/vendor/github.com/mna/pigeon/pigeon.go +++ /dev/null @@ -1,4418 +0,0 @@ -// Code generated by pigeon; DO NOT EDIT. - -package main - -import ( - "bytes" - "errors" - "fmt" - "io" - "io/ioutil" - "math" - "os" - "sort" - "strconv" - "strings" - "unicode" - "unicode/utf8" - - "github.com/mna/pigeon/ast" -) - -var g = &grammar{ - rules: []*rule{ - { - name: "Grammar", - pos: position{line: 5, col: 1, offset: 18}, - expr: &actionExpr{ - pos: position{line: 5, col: 11, offset: 30}, - run: (*parser).callonGrammar1, - expr: &seqExpr{ - pos: position{line: 5, col: 11, offset: 30}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 5, col: 11, offset: 30}, - name: "__", - }, - &labeledExpr{ - pos: position{line: 5, col: 14, offset: 33}, - label: "initializer", - expr: &zeroOrOneExpr{ - pos: position{line: 5, col: 26, offset: 45}, - expr: &seqExpr{ - pos: position{line: 5, col: 28, offset: 47}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 5, col: 28, offset: 47}, - name: "Initializer", - }, - &ruleRefExpr{ - pos: position{line: 5, col: 40, offset: 59}, - name: "__", - }, - }, - }, - }, - }, - &labeledExpr{ - pos: position{line: 5, col: 46, offset: 65}, - label: "rules", - expr: &oneOrMoreExpr{ - pos: position{line: 5, col: 52, offset: 71}, - expr: &seqExpr{ - pos: position{line: 5, col: 54, offset: 73}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 5, col: 54, offset: 73}, - name: "Rule", - }, - &ruleRefExpr{ - pos: position{line: 5, col: 59, offset: 78}, - name: "__", - }, - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 5, col: 65, offset: 84}, - name: "EOF", - }, - }, - }, - }, - }, - { - name: "Initializer", - pos: position{line: 24, col: 1, offset: 525}, - expr: &actionExpr{ - pos: position{line: 24, col: 15, offset: 541}, - run: (*parser).callonInitializer1, - expr: &seqExpr{ - pos: position{line: 24, col: 15, offset: 541}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 24, col: 15, offset: 541}, - label: "code", - expr: &ruleRefExpr{ - pos: position{line: 24, col: 20, offset: 546}, - name: "CodeBlock", - }, - }, - &ruleRefExpr{ - pos: position{line: 24, col: 30, offset: 556}, - name: "EOS", - }, - }, - }, - }, - }, - { - name: "Rule", - pos: position{line: 28, col: 1, offset: 586}, - expr: &actionExpr{ - pos: position{line: 28, col: 8, offset: 595}, - run: (*parser).callonRule1, - expr: &seqExpr{ - pos: position{line: 28, col: 8, offset: 595}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 28, col: 8, offset: 595}, - label: "name", - expr: &ruleRefExpr{ - pos: position{line: 28, col: 13, offset: 600}, - name: "IdentifierName", - }, - }, - &ruleRefExpr{ - pos: position{line: 28, col: 28, offset: 615}, - name: "__", - }, - &labeledExpr{ - pos: position{line: 28, col: 31, offset: 618}, - label: "display", - expr: &zeroOrOneExpr{ - pos: position{line: 28, col: 39, offset: 626}, - expr: &seqExpr{ - pos: position{line: 28, col: 41, offset: 628}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 28, col: 41, offset: 628}, - name: "StringLiteral", - }, - &ruleRefExpr{ - pos: position{line: 28, col: 55, offset: 642}, - name: "__", - }, - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 28, col: 61, offset: 648}, - name: "RuleDefOp", - }, - &ruleRefExpr{ - pos: position{line: 28, col: 71, offset: 658}, - name: "__", - }, - &labeledExpr{ - pos: position{line: 28, col: 74, offset: 661}, - label: "expr", - expr: &ruleRefExpr{ - pos: position{line: 28, col: 79, offset: 666}, - name: "Expression", - }, - }, - &ruleRefExpr{ - pos: position{line: 28, col: 90, offset: 677}, - name: "EOS", - }, - }, - }, - }, - }, - { - name: "Expression", - pos: position{line: 41, col: 1, offset: 961}, - expr: &ruleRefExpr{ - pos: position{line: 41, col: 14, offset: 976}, - name: "RecoveryExpr", - }, - }, - { - name: "RecoveryExpr", - pos: position{line: 43, col: 1, offset: 990}, - expr: &actionExpr{ - pos: position{line: 43, col: 16, offset: 1007}, - run: (*parser).callonRecoveryExpr1, - expr: &seqExpr{ - pos: position{line: 43, col: 16, offset: 1007}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 43, col: 16, offset: 1007}, - label: "expr", - expr: &ruleRefExpr{ - pos: position{line: 43, col: 21, offset: 1012}, - name: "ChoiceExpr", - }, - }, - &labeledExpr{ - pos: position{line: 43, col: 32, offset: 1023}, - label: "recoverExprs", - expr: &zeroOrMoreExpr{ - pos: position{line: 43, col: 45, offset: 1036}, - expr: &seqExpr{ - pos: position{line: 43, col: 47, offset: 1038}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 43, col: 47, offset: 1038}, - name: "__", - }, - &litMatcher{ - pos: position{line: 43, col: 50, offset: 1041}, - val: "//{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 43, col: 56, offset: 1047}, - name: "__", - }, - &ruleRefExpr{ - pos: position{line: 43, col: 59, offset: 1050}, - name: "Labels", - }, - &ruleRefExpr{ - pos: position{line: 43, col: 66, offset: 1057}, - name: "__", - }, - &litMatcher{ - pos: position{line: 43, col: 69, offset: 1060}, - val: "}", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 43, col: 73, offset: 1064}, - name: "__", - }, - &ruleRefExpr{ - pos: position{line: 43, col: 76, offset: 1067}, - name: "ChoiceExpr", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "Labels", - pos: position{line: 58, col: 1, offset: 1481}, - expr: &actionExpr{ - pos: position{line: 58, col: 10, offset: 1492}, - run: (*parser).callonLabels1, - expr: &seqExpr{ - pos: position{line: 58, col: 10, offset: 1492}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 58, col: 10, offset: 1492}, - label: "label", - expr: &ruleRefExpr{ - pos: position{line: 58, col: 16, offset: 1498}, - name: "IdentifierName", - }, - }, - &labeledExpr{ - pos: position{line: 58, col: 31, offset: 1513}, - label: "labels", - expr: &zeroOrMoreExpr{ - pos: position{line: 58, col: 38, offset: 1520}, - expr: &seqExpr{ - pos: position{line: 58, col: 40, offset: 1522}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 58, col: 40, offset: 1522}, - name: "__", - }, - &litMatcher{ - pos: position{line: 58, col: 43, offset: 1525}, - val: ",", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 58, col: 47, offset: 1529}, - name: "__", - }, - &ruleRefExpr{ - pos: position{line: 58, col: 50, offset: 1532}, - name: "IdentifierName", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "ChoiceExpr", - pos: position{line: 67, col: 1, offset: 1861}, - expr: &actionExpr{ - pos: position{line: 67, col: 14, offset: 1876}, - run: (*parser).callonChoiceExpr1, - expr: &seqExpr{ - pos: position{line: 67, col: 14, offset: 1876}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 67, col: 14, offset: 1876}, - label: "first", - expr: &ruleRefExpr{ - pos: position{line: 67, col: 20, offset: 1882}, - name: "ActionExpr", - }, - }, - &labeledExpr{ - pos: position{line: 67, col: 31, offset: 1893}, - label: "rest", - expr: &zeroOrMoreExpr{ - pos: position{line: 67, col: 36, offset: 1898}, - expr: &seqExpr{ - pos: position{line: 67, col: 38, offset: 1900}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 67, col: 38, offset: 1900}, - name: "__", - }, - &litMatcher{ - pos: position{line: 67, col: 41, offset: 1903}, - val: "/", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 67, col: 45, offset: 1907}, - name: "__", - }, - &ruleRefExpr{ - pos: position{line: 67, col: 48, offset: 1910}, - name: "ActionExpr", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "ActionExpr", - pos: position{line: 82, col: 1, offset: 2315}, - expr: &actionExpr{ - pos: position{line: 82, col: 14, offset: 2330}, - run: (*parser).callonActionExpr1, - expr: &seqExpr{ - pos: position{line: 82, col: 14, offset: 2330}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 82, col: 14, offset: 2330}, - label: "expr", - expr: &ruleRefExpr{ - pos: position{line: 82, col: 19, offset: 2335}, - name: "SeqExpr", - }, - }, - &labeledExpr{ - pos: position{line: 82, col: 27, offset: 2343}, - label: "code", - expr: &zeroOrOneExpr{ - pos: position{line: 82, col: 32, offset: 2348}, - expr: &seqExpr{ - pos: position{line: 82, col: 34, offset: 2350}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 82, col: 34, offset: 2350}, - name: "__", - }, - &ruleRefExpr{ - pos: position{line: 82, col: 37, offset: 2353}, - name: "CodeBlock", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "SeqExpr", - pos: position{line: 96, col: 1, offset: 2619}, - expr: &actionExpr{ - pos: position{line: 96, col: 11, offset: 2631}, - run: (*parser).callonSeqExpr1, - expr: &seqExpr{ - pos: position{line: 96, col: 11, offset: 2631}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 96, col: 11, offset: 2631}, - label: "first", - expr: &ruleRefExpr{ - pos: position{line: 96, col: 17, offset: 2637}, - name: "LabeledExpr", - }, - }, - &labeledExpr{ - pos: position{line: 96, col: 29, offset: 2649}, - label: "rest", - expr: &zeroOrMoreExpr{ - pos: position{line: 96, col: 34, offset: 2654}, - expr: &seqExpr{ - pos: position{line: 96, col: 36, offset: 2656}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 96, col: 36, offset: 2656}, - name: "__", - }, - &ruleRefExpr{ - pos: position{line: 96, col: 39, offset: 2659}, - name: "LabeledExpr", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "LabeledExpr", - pos: position{line: 109, col: 1, offset: 3010}, - expr: &choiceExpr{ - pos: position{line: 109, col: 15, offset: 3026}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 109, col: 15, offset: 3026}, - run: (*parser).callonLabeledExpr2, - expr: &seqExpr{ - pos: position{line: 109, col: 15, offset: 3026}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 109, col: 15, offset: 3026}, - label: "label", - expr: &ruleRefExpr{ - pos: position{line: 109, col: 21, offset: 3032}, - name: "Identifier", - }, - }, - &ruleRefExpr{ - pos: position{line: 109, col: 32, offset: 3043}, - name: "__", - }, - &litMatcher{ - pos: position{line: 109, col: 35, offset: 3046}, - val: ":", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 109, col: 39, offset: 3050}, - name: "__", - }, - &labeledExpr{ - pos: position{line: 109, col: 42, offset: 3053}, - label: "expr", - expr: &ruleRefExpr{ - pos: position{line: 109, col: 47, offset: 3058}, - name: "PrefixedExpr", - }, - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 115, col: 5, offset: 3231}, - name: "PrefixedExpr", - }, - &ruleRefExpr{ - pos: position{line: 115, col: 20, offset: 3246}, - name: "ThrowExpr", - }, - }, - }, - }, - { - name: "PrefixedExpr", - pos: position{line: 117, col: 1, offset: 3257}, - expr: &choiceExpr{ - pos: position{line: 117, col: 16, offset: 3274}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 117, col: 16, offset: 3274}, - run: (*parser).callonPrefixedExpr2, - expr: &seqExpr{ - pos: position{line: 117, col: 16, offset: 3274}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 117, col: 16, offset: 3274}, - label: "op", - expr: &ruleRefExpr{ - pos: position{line: 117, col: 19, offset: 3277}, - name: "PrefixedOp", - }, - }, - &ruleRefExpr{ - pos: position{line: 117, col: 30, offset: 3288}, - name: "__", - }, - &labeledExpr{ - pos: position{line: 117, col: 33, offset: 3291}, - label: "expr", - expr: &ruleRefExpr{ - pos: position{line: 117, col: 38, offset: 3296}, - name: "SuffixedExpr", - }, - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 128, col: 5, offset: 3578}, - name: "SuffixedExpr", - }, - }, - }, - }, - { - name: "PrefixedOp", - pos: position{line: 130, col: 1, offset: 3592}, - expr: &actionExpr{ - pos: position{line: 130, col: 14, offset: 3607}, - run: (*parser).callonPrefixedOp1, - expr: &choiceExpr{ - pos: position{line: 130, col: 16, offset: 3609}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 130, col: 16, offset: 3609}, - val: "&", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 130, col: 22, offset: 3615}, - val: "!", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "SuffixedExpr", - pos: position{line: 134, col: 1, offset: 3657}, - expr: &choiceExpr{ - pos: position{line: 134, col: 16, offset: 3674}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 134, col: 16, offset: 3674}, - run: (*parser).callonSuffixedExpr2, - expr: &seqExpr{ - pos: position{line: 134, col: 16, offset: 3674}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 134, col: 16, offset: 3674}, - label: "expr", - expr: &ruleRefExpr{ - pos: position{line: 134, col: 21, offset: 3679}, - name: "PrimaryExpr", - }, - }, - &ruleRefExpr{ - pos: position{line: 134, col: 33, offset: 3691}, - name: "__", - }, - &labeledExpr{ - pos: position{line: 134, col: 36, offset: 3694}, - label: "op", - expr: &ruleRefExpr{ - pos: position{line: 134, col: 39, offset: 3697}, - name: "SuffixedOp", - }, - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 153, col: 5, offset: 4227}, - name: "PrimaryExpr", - }, - }, - }, - }, - { - name: "SuffixedOp", - pos: position{line: 155, col: 1, offset: 4241}, - expr: &actionExpr{ - pos: position{line: 155, col: 14, offset: 4256}, - run: (*parser).callonSuffixedOp1, - expr: &choiceExpr{ - pos: position{line: 155, col: 16, offset: 4258}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 155, col: 16, offset: 4258}, - val: "?", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 155, col: 22, offset: 4264}, - val: "*", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 155, col: 28, offset: 4270}, - val: "+", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "PrimaryExpr", - pos: position{line: 159, col: 1, offset: 4312}, - expr: &choiceExpr{ - pos: position{line: 159, col: 15, offset: 4328}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 159, col: 15, offset: 4328}, - name: "LitMatcher", - }, - &ruleRefExpr{ - pos: position{line: 159, col: 28, offset: 4341}, - name: "CharClassMatcher", - }, - &ruleRefExpr{ - pos: position{line: 159, col: 47, offset: 4360}, - name: "AnyMatcher", - }, - &ruleRefExpr{ - pos: position{line: 159, col: 60, offset: 4373}, - name: "RuleRefExpr", - }, - &ruleRefExpr{ - pos: position{line: 159, col: 74, offset: 4387}, - name: "SemanticPredExpr", - }, - &actionExpr{ - pos: position{line: 159, col: 93, offset: 4406}, - run: (*parser).callonPrimaryExpr7, - expr: &seqExpr{ - pos: position{line: 159, col: 93, offset: 4406}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 159, col: 93, offset: 4406}, - val: "(", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 159, col: 97, offset: 4410}, - name: "__", - }, - &labeledExpr{ - pos: position{line: 159, col: 100, offset: 4413}, - label: "expr", - expr: &ruleRefExpr{ - pos: position{line: 159, col: 105, offset: 4418}, - name: "Expression", - }, - }, - &ruleRefExpr{ - pos: position{line: 159, col: 116, offset: 4429}, - name: "__", - }, - &litMatcher{ - pos: position{line: 159, col: 119, offset: 4432}, - val: ")", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - }, - { - name: "RuleRefExpr", - pos: position{line: 162, col: 1, offset: 4461}, - expr: &actionExpr{ - pos: position{line: 162, col: 15, offset: 4477}, - run: (*parser).callonRuleRefExpr1, - expr: &seqExpr{ - pos: position{line: 162, col: 15, offset: 4477}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 162, col: 15, offset: 4477}, - label: "name", - expr: &ruleRefExpr{ - pos: position{line: 162, col: 20, offset: 4482}, - name: "IdentifierName", - }, - }, - ¬Expr{ - pos: position{line: 162, col: 35, offset: 4497}, - expr: &seqExpr{ - pos: position{line: 162, col: 38, offset: 4500}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 162, col: 38, offset: 4500}, - name: "__", - }, - &zeroOrOneExpr{ - pos: position{line: 162, col: 41, offset: 4503}, - expr: &seqExpr{ - pos: position{line: 162, col: 43, offset: 4505}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 162, col: 43, offset: 4505}, - name: "StringLiteral", - }, - &ruleRefExpr{ - pos: position{line: 162, col: 57, offset: 4519}, - name: "__", - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 162, col: 63, offset: 4525}, - name: "RuleDefOp", - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "SemanticPredExpr", - pos: position{line: 167, col: 1, offset: 4641}, - expr: &actionExpr{ - pos: position{line: 167, col: 20, offset: 4662}, - run: (*parser).callonSemanticPredExpr1, - expr: &seqExpr{ - pos: position{line: 167, col: 20, offset: 4662}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 167, col: 20, offset: 4662}, - label: "op", - expr: &ruleRefExpr{ - pos: position{line: 167, col: 23, offset: 4665}, - name: "SemanticPredOp", - }, - }, - &ruleRefExpr{ - pos: position{line: 167, col: 38, offset: 4680}, - name: "__", - }, - &labeledExpr{ - pos: position{line: 167, col: 41, offset: 4683}, - label: "code", - expr: &ruleRefExpr{ - pos: position{line: 167, col: 46, offset: 4688}, - name: "CodeBlock", - }, - }, - }, - }, - }, - }, - { - name: "SemanticPredOp", - pos: position{line: 187, col: 1, offset: 5135}, - expr: &actionExpr{ - pos: position{line: 187, col: 18, offset: 5154}, - run: (*parser).callonSemanticPredOp1, - expr: &choiceExpr{ - pos: position{line: 187, col: 20, offset: 5156}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 187, col: 20, offset: 5156}, - val: "#", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 187, col: 26, offset: 5162}, - val: "&", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 187, col: 32, offset: 5168}, - val: "!", - ignoreCase: false, - }, - }, - }, - }, - }, - { - name: "RuleDefOp", - pos: position{line: 191, col: 1, offset: 5210}, - expr: &choiceExpr{ - pos: position{line: 191, col: 13, offset: 5224}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 191, col: 13, offset: 5224}, - val: "=", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 191, col: 19, offset: 5230}, - val: "<-", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 191, col: 26, offset: 5237}, - val: "←", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 191, col: 37, offset: 5248}, - val: "⟵", - ignoreCase: false, - }, - }, - }, - }, - { - name: "SourceChar", - pos: position{line: 193, col: 1, offset: 5258}, - expr: &anyMatcher{ - line: 193, col: 14, offset: 5273, - }, - }, - { - name: "Comment", - pos: position{line: 194, col: 1, offset: 5275}, - expr: &choiceExpr{ - pos: position{line: 194, col: 11, offset: 5287}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 194, col: 11, offset: 5287}, - name: "MultiLineComment", - }, - &ruleRefExpr{ - pos: position{line: 194, col: 30, offset: 5306}, - name: "SingleLineComment", - }, - }, - }, - }, - { - name: "MultiLineComment", - pos: position{line: 195, col: 1, offset: 5324}, - expr: &seqExpr{ - pos: position{line: 195, col: 20, offset: 5345}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 195, col: 20, offset: 5345}, - val: "/*", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 195, col: 25, offset: 5350}, - expr: &seqExpr{ - pos: position{line: 195, col: 27, offset: 5352}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 195, col: 27, offset: 5352}, - expr: &litMatcher{ - pos: position{line: 195, col: 28, offset: 5353}, - val: "*/", - ignoreCase: false, - }, - }, - &ruleRefExpr{ - pos: position{line: 195, col: 33, offset: 5358}, - name: "SourceChar", - }, - }, - }, - }, - &litMatcher{ - pos: position{line: 195, col: 47, offset: 5372}, - val: "*/", - ignoreCase: false, - }, - }, - }, - }, - { - name: "MultiLineCommentNoLineTerminator", - pos: position{line: 196, col: 1, offset: 5377}, - expr: &seqExpr{ - pos: position{line: 196, col: 36, offset: 5414}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 196, col: 36, offset: 5414}, - val: "/*", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 196, col: 41, offset: 5419}, - expr: &seqExpr{ - pos: position{line: 196, col: 43, offset: 5421}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 196, col: 43, offset: 5421}, - expr: &choiceExpr{ - pos: position{line: 196, col: 46, offset: 5424}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 196, col: 46, offset: 5424}, - val: "*/", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 196, col: 53, offset: 5431}, - name: "EOL", - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 196, col: 59, offset: 5437}, - name: "SourceChar", - }, - }, - }, - }, - &litMatcher{ - pos: position{line: 196, col: 73, offset: 5451}, - val: "*/", - ignoreCase: false, - }, - }, - }, - }, - { - name: "SingleLineComment", - pos: position{line: 197, col: 1, offset: 5456}, - expr: &seqExpr{ - pos: position{line: 197, col: 21, offset: 5478}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 197, col: 21, offset: 5478}, - expr: &litMatcher{ - pos: position{line: 197, col: 23, offset: 5480}, - val: "//{", - ignoreCase: false, - }, - }, - &litMatcher{ - pos: position{line: 197, col: 30, offset: 5487}, - val: "//", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 197, col: 35, offset: 5492}, - expr: &seqExpr{ - pos: position{line: 197, col: 37, offset: 5494}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 197, col: 37, offset: 5494}, - expr: &ruleRefExpr{ - pos: position{line: 197, col: 38, offset: 5495}, - name: "EOL", - }, - }, - &ruleRefExpr{ - pos: position{line: 197, col: 42, offset: 5499}, - name: "SourceChar", - }, - }, - }, - }, - }, - }, - }, - { - name: "Identifier", - pos: position{line: 199, col: 1, offset: 5514}, - expr: &actionExpr{ - pos: position{line: 199, col: 14, offset: 5529}, - run: (*parser).callonIdentifier1, - expr: &labeledExpr{ - pos: position{line: 199, col: 14, offset: 5529}, - label: "ident", - expr: &ruleRefExpr{ - pos: position{line: 199, col: 20, offset: 5535}, - name: "IdentifierName", - }, - }, - }, - }, - { - name: "IdentifierName", - pos: position{line: 207, col: 1, offset: 5754}, - expr: &actionExpr{ - pos: position{line: 207, col: 18, offset: 5773}, - run: (*parser).callonIdentifierName1, - expr: &seqExpr{ - pos: position{line: 207, col: 18, offset: 5773}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 207, col: 18, offset: 5773}, - name: "IdentifierStart", - }, - &zeroOrMoreExpr{ - pos: position{line: 207, col: 34, offset: 5789}, - expr: &ruleRefExpr{ - pos: position{line: 207, col: 34, offset: 5789}, - name: "IdentifierPart", - }, - }, - }, - }, - }, - }, - { - name: "IdentifierStart", - pos: position{line: 210, col: 1, offset: 5871}, - expr: &charClassMatcher{ - pos: position{line: 210, col: 19, offset: 5891}, - val: "[\\pL_]", - chars: []rune{'_'}, - classes: []*unicode.RangeTable{rangeTable("L")}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "IdentifierPart", - pos: position{line: 211, col: 1, offset: 5898}, - expr: &choiceExpr{ - pos: position{line: 211, col: 18, offset: 5917}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 211, col: 18, offset: 5917}, - name: "IdentifierStart", - }, - &charClassMatcher{ - pos: position{line: 211, col: 36, offset: 5935}, - val: "[\\p{Nd}]", - classes: []*unicode.RangeTable{rangeTable("Nd")}, - ignoreCase: false, - inverted: false, - }, - }, - }, - }, - { - name: "LitMatcher", - pos: position{line: 213, col: 1, offset: 5945}, - expr: &actionExpr{ - pos: position{line: 213, col: 14, offset: 5960}, - run: (*parser).callonLitMatcher1, - expr: &seqExpr{ - pos: position{line: 213, col: 14, offset: 5960}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 213, col: 14, offset: 5960}, - label: "lit", - expr: &ruleRefExpr{ - pos: position{line: 213, col: 18, offset: 5964}, - name: "StringLiteral", - }, - }, - &labeledExpr{ - pos: position{line: 213, col: 32, offset: 5978}, - label: "ignore", - expr: &zeroOrOneExpr{ - pos: position{line: 213, col: 39, offset: 5985}, - expr: &litMatcher{ - pos: position{line: 213, col: 39, offset: 5985}, - val: "i", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - }, - { - name: "StringLiteral", - pos: position{line: 226, col: 1, offset: 6384}, - expr: &choiceExpr{ - pos: position{line: 226, col: 17, offset: 6402}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 226, col: 17, offset: 6402}, - run: (*parser).callonStringLiteral2, - expr: &choiceExpr{ - pos: position{line: 226, col: 19, offset: 6404}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 226, col: 19, offset: 6404}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 226, col: 19, offset: 6404}, - val: "\"", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 226, col: 23, offset: 6408}, - expr: &ruleRefExpr{ - pos: position{line: 226, col: 23, offset: 6408}, - name: "DoubleStringChar", - }, - }, - &litMatcher{ - pos: position{line: 226, col: 41, offset: 6426}, - val: "\"", - ignoreCase: false, - }, - }, - }, - &seqExpr{ - pos: position{line: 226, col: 47, offset: 6432}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 226, col: 47, offset: 6432}, - val: "'", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 226, col: 51, offset: 6436}, - name: "SingleStringChar", - }, - &litMatcher{ - pos: position{line: 226, col: 68, offset: 6453}, - val: "'", - ignoreCase: false, - }, - }, - }, - &seqExpr{ - pos: position{line: 226, col: 74, offset: 6459}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 226, col: 74, offset: 6459}, - val: "`", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 226, col: 78, offset: 6463}, - expr: &ruleRefExpr{ - pos: position{line: 226, col: 78, offset: 6463}, - name: "RawStringChar", - }, - }, - &litMatcher{ - pos: position{line: 226, col: 93, offset: 6478}, - val: "`", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - &actionExpr{ - pos: position{line: 228, col: 5, offset: 6551}, - run: (*parser).callonStringLiteral18, - expr: &choiceExpr{ - pos: position{line: 228, col: 7, offset: 6553}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 228, col: 9, offset: 6555}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 228, col: 9, offset: 6555}, - val: "\"", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 228, col: 13, offset: 6559}, - expr: &ruleRefExpr{ - pos: position{line: 228, col: 13, offset: 6559}, - name: "DoubleStringChar", - }, - }, - &choiceExpr{ - pos: position{line: 228, col: 33, offset: 6579}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 228, col: 33, offset: 6579}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 228, col: 39, offset: 6585}, - name: "EOF", - }, - }, - }, - }, - }, - &seqExpr{ - pos: position{line: 228, col: 51, offset: 6597}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 228, col: 51, offset: 6597}, - val: "'", - ignoreCase: false, - }, - &zeroOrOneExpr{ - pos: position{line: 228, col: 55, offset: 6601}, - expr: &ruleRefExpr{ - pos: position{line: 228, col: 55, offset: 6601}, - name: "SingleStringChar", - }, - }, - &choiceExpr{ - pos: position{line: 228, col: 75, offset: 6621}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 228, col: 75, offset: 6621}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 228, col: 81, offset: 6627}, - name: "EOF", - }, - }, - }, - }, - }, - &seqExpr{ - pos: position{line: 228, col: 91, offset: 6637}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 228, col: 91, offset: 6637}, - val: "`", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 228, col: 95, offset: 6641}, - expr: &ruleRefExpr{ - pos: position{line: 228, col: 95, offset: 6641}, - name: "RawStringChar", - }, - }, - &ruleRefExpr{ - pos: position{line: 228, col: 110, offset: 6656}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "DoubleStringChar", - pos: position{line: 232, col: 1, offset: 6758}, - expr: &choiceExpr{ - pos: position{line: 232, col: 20, offset: 6779}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 232, col: 20, offset: 6779}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 232, col: 20, offset: 6779}, - expr: &choiceExpr{ - pos: position{line: 232, col: 23, offset: 6782}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 232, col: 23, offset: 6782}, - val: "\"", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 232, col: 29, offset: 6788}, - val: "\\", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 232, col: 36, offset: 6795}, - name: "EOL", - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 232, col: 42, offset: 6801}, - name: "SourceChar", - }, - }, - }, - &seqExpr{ - pos: position{line: 232, col: 55, offset: 6814}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 232, col: 55, offset: 6814}, - val: "\\", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 232, col: 60, offset: 6819}, - name: "DoubleStringEscape", - }, - }, - }, - }, - }, - }, - { - name: "SingleStringChar", - pos: position{line: 233, col: 1, offset: 6838}, - expr: &choiceExpr{ - pos: position{line: 233, col: 20, offset: 6859}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 233, col: 20, offset: 6859}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 233, col: 20, offset: 6859}, - expr: &choiceExpr{ - pos: position{line: 233, col: 23, offset: 6862}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 233, col: 23, offset: 6862}, - val: "'", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 233, col: 29, offset: 6868}, - val: "\\", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 233, col: 36, offset: 6875}, - name: "EOL", - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 233, col: 42, offset: 6881}, - name: "SourceChar", - }, - }, - }, - &seqExpr{ - pos: position{line: 233, col: 55, offset: 6894}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 233, col: 55, offset: 6894}, - val: "\\", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 233, col: 60, offset: 6899}, - name: "SingleStringEscape", - }, - }, - }, - }, - }, - }, - { - name: "RawStringChar", - pos: position{line: 234, col: 1, offset: 6918}, - expr: &seqExpr{ - pos: position{line: 234, col: 17, offset: 6936}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 234, col: 17, offset: 6936}, - expr: &litMatcher{ - pos: position{line: 234, col: 18, offset: 6937}, - val: "`", - ignoreCase: false, - }, - }, - &ruleRefExpr{ - pos: position{line: 234, col: 22, offset: 6941}, - name: "SourceChar", - }, - }, - }, - }, - { - name: "DoubleStringEscape", - pos: position{line: 236, col: 1, offset: 6953}, - expr: &choiceExpr{ - pos: position{line: 236, col: 22, offset: 6976}, - alternatives: []interface{}{ - &choiceExpr{ - pos: position{line: 236, col: 24, offset: 6978}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 236, col: 24, offset: 6978}, - val: "\"", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 236, col: 30, offset: 6984}, - name: "CommonEscapeSequence", - }, - }, - }, - &actionExpr{ - pos: position{line: 237, col: 7, offset: 7013}, - run: (*parser).callonDoubleStringEscape5, - expr: &choiceExpr{ - pos: position{line: 237, col: 9, offset: 7015}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 237, col: 9, offset: 7015}, - name: "SourceChar", - }, - &ruleRefExpr{ - pos: position{line: 237, col: 22, offset: 7028}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 237, col: 28, offset: 7034}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - { - name: "SingleStringEscape", - pos: position{line: 240, col: 1, offset: 7099}, - expr: &choiceExpr{ - pos: position{line: 240, col: 22, offset: 7122}, - alternatives: []interface{}{ - &choiceExpr{ - pos: position{line: 240, col: 24, offset: 7124}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 240, col: 24, offset: 7124}, - val: "'", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 240, col: 30, offset: 7130}, - name: "CommonEscapeSequence", - }, - }, - }, - &actionExpr{ - pos: position{line: 241, col: 7, offset: 7159}, - run: (*parser).callonSingleStringEscape5, - expr: &choiceExpr{ - pos: position{line: 241, col: 9, offset: 7161}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 241, col: 9, offset: 7161}, - name: "SourceChar", - }, - &ruleRefExpr{ - pos: position{line: 241, col: 22, offset: 7174}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 241, col: 28, offset: 7180}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - { - name: "CommonEscapeSequence", - pos: position{line: 245, col: 1, offset: 7246}, - expr: &choiceExpr{ - pos: position{line: 245, col: 24, offset: 7271}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 245, col: 24, offset: 7271}, - name: "SingleCharEscape", - }, - &ruleRefExpr{ - pos: position{line: 245, col: 43, offset: 7290}, - name: "OctalEscape", - }, - &ruleRefExpr{ - pos: position{line: 245, col: 57, offset: 7304}, - name: "HexEscape", - }, - &ruleRefExpr{ - pos: position{line: 245, col: 69, offset: 7316}, - name: "LongUnicodeEscape", - }, - &ruleRefExpr{ - pos: position{line: 245, col: 89, offset: 7336}, - name: "ShortUnicodeEscape", - }, - }, - }, - }, - { - name: "SingleCharEscape", - pos: position{line: 246, col: 1, offset: 7355}, - expr: &choiceExpr{ - pos: position{line: 246, col: 20, offset: 7376}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 246, col: 20, offset: 7376}, - val: "a", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 246, col: 26, offset: 7382}, - val: "b", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 246, col: 32, offset: 7388}, - val: "n", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 246, col: 38, offset: 7394}, - val: "f", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 246, col: 44, offset: 7400}, - val: "r", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 246, col: 50, offset: 7406}, - val: "t", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 246, col: 56, offset: 7412}, - val: "v", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 246, col: 62, offset: 7418}, - val: "\\", - ignoreCase: false, - }, - }, - }, - }, - { - name: "OctalEscape", - pos: position{line: 247, col: 1, offset: 7423}, - expr: &choiceExpr{ - pos: position{line: 247, col: 15, offset: 7439}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 247, col: 15, offset: 7439}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 247, col: 15, offset: 7439}, - name: "OctalDigit", - }, - &ruleRefExpr{ - pos: position{line: 247, col: 26, offset: 7450}, - name: "OctalDigit", - }, - &ruleRefExpr{ - pos: position{line: 247, col: 37, offset: 7461}, - name: "OctalDigit", - }, - }, - }, - &actionExpr{ - pos: position{line: 248, col: 7, offset: 7478}, - run: (*parser).callonOctalEscape6, - expr: &seqExpr{ - pos: position{line: 248, col: 7, offset: 7478}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 248, col: 7, offset: 7478}, - name: "OctalDigit", - }, - &choiceExpr{ - pos: position{line: 248, col: 20, offset: 7491}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 248, col: 20, offset: 7491}, - name: "SourceChar", - }, - &ruleRefExpr{ - pos: position{line: 248, col: 33, offset: 7504}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 248, col: 39, offset: 7510}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "HexEscape", - pos: position{line: 251, col: 1, offset: 7571}, - expr: &choiceExpr{ - pos: position{line: 251, col: 13, offset: 7585}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 251, col: 13, offset: 7585}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 251, col: 13, offset: 7585}, - val: "x", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 251, col: 17, offset: 7589}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 251, col: 26, offset: 7598}, - name: "HexDigit", - }, - }, - }, - &actionExpr{ - pos: position{line: 252, col: 7, offset: 7613}, - run: (*parser).callonHexEscape6, - expr: &seqExpr{ - pos: position{line: 252, col: 7, offset: 7613}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 252, col: 7, offset: 7613}, - val: "x", - ignoreCase: false, - }, - &choiceExpr{ - pos: position{line: 252, col: 13, offset: 7619}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 252, col: 13, offset: 7619}, - name: "SourceChar", - }, - &ruleRefExpr{ - pos: position{line: 252, col: 26, offset: 7632}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 252, col: 32, offset: 7638}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "LongUnicodeEscape", - pos: position{line: 255, col: 1, offset: 7705}, - expr: &choiceExpr{ - pos: position{line: 256, col: 5, offset: 7732}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 256, col: 5, offset: 7732}, - run: (*parser).callonLongUnicodeEscape2, - expr: &seqExpr{ - pos: position{line: 256, col: 5, offset: 7732}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 256, col: 5, offset: 7732}, - val: "U", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 256, col: 9, offset: 7736}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 256, col: 18, offset: 7745}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 256, col: 27, offset: 7754}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 256, col: 36, offset: 7763}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 256, col: 45, offset: 7772}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 256, col: 54, offset: 7781}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 256, col: 63, offset: 7790}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 256, col: 72, offset: 7799}, - name: "HexDigit", - }, - }, - }, - }, - &actionExpr{ - pos: position{line: 259, col: 7, offset: 7901}, - run: (*parser).callonLongUnicodeEscape13, - expr: &seqExpr{ - pos: position{line: 259, col: 7, offset: 7901}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 259, col: 7, offset: 7901}, - val: "U", - ignoreCase: false, - }, - &choiceExpr{ - pos: position{line: 259, col: 13, offset: 7907}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 259, col: 13, offset: 7907}, - name: "SourceChar", - }, - &ruleRefExpr{ - pos: position{line: 259, col: 26, offset: 7920}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 259, col: 32, offset: 7926}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "ShortUnicodeEscape", - pos: position{line: 262, col: 1, offset: 7989}, - expr: &choiceExpr{ - pos: position{line: 263, col: 5, offset: 8017}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 263, col: 5, offset: 8017}, - run: (*parser).callonShortUnicodeEscape2, - expr: &seqExpr{ - pos: position{line: 263, col: 5, offset: 8017}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 263, col: 5, offset: 8017}, - val: "u", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 263, col: 9, offset: 8021}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 263, col: 18, offset: 8030}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 263, col: 27, offset: 8039}, - name: "HexDigit", - }, - &ruleRefExpr{ - pos: position{line: 263, col: 36, offset: 8048}, - name: "HexDigit", - }, - }, - }, - }, - &actionExpr{ - pos: position{line: 266, col: 7, offset: 8150}, - run: (*parser).callonShortUnicodeEscape9, - expr: &seqExpr{ - pos: position{line: 266, col: 7, offset: 8150}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 266, col: 7, offset: 8150}, - val: "u", - ignoreCase: false, - }, - &choiceExpr{ - pos: position{line: 266, col: 13, offset: 8156}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 266, col: 13, offset: 8156}, - name: "SourceChar", - }, - &ruleRefExpr{ - pos: position{line: 266, col: 26, offset: 8169}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 266, col: 32, offset: 8175}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "OctalDigit", - pos: position{line: 270, col: 1, offset: 8239}, - expr: &charClassMatcher{ - pos: position{line: 270, col: 14, offset: 8254}, - val: "[0-7]", - ranges: []rune{'0', '7'}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "DecimalDigit", - pos: position{line: 271, col: 1, offset: 8260}, - expr: &charClassMatcher{ - pos: position{line: 271, col: 16, offset: 8277}, - val: "[0-9]", - ranges: []rune{'0', '9'}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "HexDigit", - pos: position{line: 272, col: 1, offset: 8283}, - expr: &charClassMatcher{ - pos: position{line: 272, col: 12, offset: 8296}, - val: "[0-9a-f]i", - ranges: []rune{'0', '9', 'a', 'f'}, - ignoreCase: true, - inverted: false, - }, - }, - { - name: "CharClassMatcher", - pos: position{line: 274, col: 1, offset: 8307}, - expr: &choiceExpr{ - pos: position{line: 274, col: 20, offset: 8328}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 274, col: 20, offset: 8328}, - run: (*parser).callonCharClassMatcher2, - expr: &seqExpr{ - pos: position{line: 274, col: 20, offset: 8328}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 274, col: 20, offset: 8328}, - val: "[", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 274, col: 24, offset: 8332}, - expr: &choiceExpr{ - pos: position{line: 274, col: 26, offset: 8334}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 274, col: 26, offset: 8334}, - name: "ClassCharRange", - }, - &ruleRefExpr{ - pos: position{line: 274, col: 43, offset: 8351}, - name: "ClassChar", - }, - &seqExpr{ - pos: position{line: 274, col: 55, offset: 8363}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 274, col: 55, offset: 8363}, - val: "\\", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 274, col: 60, offset: 8368}, - name: "UnicodeClassEscape", - }, - }, - }, - }, - }, - }, - &litMatcher{ - pos: position{line: 274, col: 82, offset: 8390}, - val: "]", - ignoreCase: false, - }, - &zeroOrOneExpr{ - pos: position{line: 274, col: 86, offset: 8394}, - expr: &litMatcher{ - pos: position{line: 274, col: 86, offset: 8394}, - val: "i", - ignoreCase: false, - }, - }, - }, - }, - }, - &actionExpr{ - pos: position{line: 278, col: 5, offset: 8501}, - run: (*parser).callonCharClassMatcher15, - expr: &seqExpr{ - pos: position{line: 278, col: 5, offset: 8501}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 278, col: 5, offset: 8501}, - val: "[", - ignoreCase: false, - }, - &zeroOrMoreExpr{ - pos: position{line: 278, col: 9, offset: 8505}, - expr: &seqExpr{ - pos: position{line: 278, col: 11, offset: 8507}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 278, col: 11, offset: 8507}, - expr: &ruleRefExpr{ - pos: position{line: 278, col: 14, offset: 8510}, - name: "EOL", - }, - }, - &ruleRefExpr{ - pos: position{line: 278, col: 20, offset: 8516}, - name: "SourceChar", - }, - }, - }, - }, - &choiceExpr{ - pos: position{line: 278, col: 36, offset: 8532}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 278, col: 36, offset: 8532}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 278, col: 42, offset: 8538}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "ClassCharRange", - pos: position{line: 282, col: 1, offset: 8648}, - expr: &seqExpr{ - pos: position{line: 282, col: 18, offset: 8667}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 282, col: 18, offset: 8667}, - name: "ClassChar", - }, - &litMatcher{ - pos: position{line: 282, col: 28, offset: 8677}, - val: "-", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 282, col: 32, offset: 8681}, - name: "ClassChar", - }, - }, - }, - }, - { - name: "ClassChar", - pos: position{line: 283, col: 1, offset: 8691}, - expr: &choiceExpr{ - pos: position{line: 283, col: 13, offset: 8705}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 283, col: 13, offset: 8705}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 283, col: 13, offset: 8705}, - expr: &choiceExpr{ - pos: position{line: 283, col: 16, offset: 8708}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 283, col: 16, offset: 8708}, - val: "]", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 283, col: 22, offset: 8714}, - val: "\\", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 283, col: 29, offset: 8721}, - name: "EOL", - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 283, col: 35, offset: 8727}, - name: "SourceChar", - }, - }, - }, - &seqExpr{ - pos: position{line: 283, col: 48, offset: 8740}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 283, col: 48, offset: 8740}, - val: "\\", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 283, col: 53, offset: 8745}, - name: "CharClassEscape", - }, - }, - }, - }, - }, - }, - { - name: "CharClassEscape", - pos: position{line: 284, col: 1, offset: 8761}, - expr: &choiceExpr{ - pos: position{line: 284, col: 19, offset: 8781}, - alternatives: []interface{}{ - &choiceExpr{ - pos: position{line: 284, col: 21, offset: 8783}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 284, col: 21, offset: 8783}, - val: "]", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 284, col: 27, offset: 8789}, - name: "CommonEscapeSequence", - }, - }, - }, - &actionExpr{ - pos: position{line: 285, col: 7, offset: 8818}, - run: (*parser).callonCharClassEscape5, - expr: &seqExpr{ - pos: position{line: 285, col: 7, offset: 8818}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 285, col: 7, offset: 8818}, - expr: &litMatcher{ - pos: position{line: 285, col: 8, offset: 8819}, - val: "p", - ignoreCase: false, - }, - }, - &choiceExpr{ - pos: position{line: 285, col: 14, offset: 8825}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 285, col: 14, offset: 8825}, - name: "SourceChar", - }, - &ruleRefExpr{ - pos: position{line: 285, col: 27, offset: 8838}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 285, col: 33, offset: 8844}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "UnicodeClassEscape", - pos: position{line: 289, col: 1, offset: 8910}, - expr: &seqExpr{ - pos: position{line: 289, col: 22, offset: 8933}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 289, col: 22, offset: 8933}, - val: "p", - ignoreCase: false, - }, - &choiceExpr{ - pos: position{line: 290, col: 7, offset: 8946}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 290, col: 7, offset: 8946}, - name: "SingleCharUnicodeClass", - }, - &actionExpr{ - pos: position{line: 291, col: 7, offset: 8975}, - run: (*parser).callonUnicodeClassEscape5, - expr: &seqExpr{ - pos: position{line: 291, col: 7, offset: 8975}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 291, col: 7, offset: 8975}, - expr: &litMatcher{ - pos: position{line: 291, col: 8, offset: 8976}, - val: "{", - ignoreCase: false, - }, - }, - &choiceExpr{ - pos: position{line: 291, col: 14, offset: 8982}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 291, col: 14, offset: 8982}, - name: "SourceChar", - }, - &ruleRefExpr{ - pos: position{line: 291, col: 27, offset: 8995}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 291, col: 33, offset: 9001}, - name: "EOF", - }, - }, - }, - }, - }, - }, - &actionExpr{ - pos: position{line: 292, col: 7, offset: 9072}, - run: (*parser).callonUnicodeClassEscape13, - expr: &seqExpr{ - pos: position{line: 292, col: 7, offset: 9072}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 292, col: 7, offset: 9072}, - val: "{", - ignoreCase: false, - }, - &labeledExpr{ - pos: position{line: 292, col: 11, offset: 9076}, - label: "ident", - expr: &ruleRefExpr{ - pos: position{line: 292, col: 17, offset: 9082}, - name: "IdentifierName", - }, - }, - &litMatcher{ - pos: position{line: 292, col: 32, offset: 9097}, - val: "}", - ignoreCase: false, - }, - }, - }, - }, - &actionExpr{ - pos: position{line: 298, col: 7, offset: 9274}, - run: (*parser).callonUnicodeClassEscape19, - expr: &seqExpr{ - pos: position{line: 298, col: 7, offset: 9274}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 298, col: 7, offset: 9274}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 298, col: 11, offset: 9278}, - name: "IdentifierName", - }, - &choiceExpr{ - pos: position{line: 298, col: 28, offset: 9295}, - alternatives: []interface{}{ - &litMatcher{ - pos: position{line: 298, col: 28, offset: 9295}, - val: "]", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 298, col: 34, offset: 9301}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 298, col: 40, offset: 9307}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "SingleCharUnicodeClass", - pos: position{line: 302, col: 1, offset: 9390}, - expr: &charClassMatcher{ - pos: position{line: 302, col: 26, offset: 9417}, - val: "[LMNCPZS]", - chars: []rune{'L', 'M', 'N', 'C', 'P', 'Z', 'S'}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "AnyMatcher", - pos: position{line: 304, col: 1, offset: 9428}, - expr: &actionExpr{ - pos: position{line: 304, col: 14, offset: 9443}, - run: (*parser).callonAnyMatcher1, - expr: &litMatcher{ - pos: position{line: 304, col: 14, offset: 9443}, - val: ".", - ignoreCase: false, - }, - }, - }, - { - name: "ThrowExpr", - pos: position{line: 309, col: 1, offset: 9518}, - expr: &choiceExpr{ - pos: position{line: 309, col: 13, offset: 9532}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 309, col: 13, offset: 9532}, - run: (*parser).callonThrowExpr2, - expr: &seqExpr{ - pos: position{line: 309, col: 13, offset: 9532}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 309, col: 13, offset: 9532}, - val: "%", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 309, col: 17, offset: 9536}, - val: "{", - ignoreCase: false, - }, - &labeledExpr{ - pos: position{line: 309, col: 21, offset: 9540}, - label: "label", - expr: &ruleRefExpr{ - pos: position{line: 309, col: 27, offset: 9546}, - name: "IdentifierName", - }, - }, - &litMatcher{ - pos: position{line: 309, col: 42, offset: 9561}, - val: "}", - ignoreCase: false, - }, - }, - }, - }, - &actionExpr{ - pos: position{line: 313, col: 5, offset: 9669}, - run: (*parser).callonThrowExpr9, - expr: &seqExpr{ - pos: position{line: 313, col: 5, offset: 9669}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 313, col: 5, offset: 9669}, - val: "%", - ignoreCase: false, - }, - &litMatcher{ - pos: position{line: 313, col: 9, offset: 9673}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 313, col: 13, offset: 9677}, - name: "IdentifierName", - }, - &ruleRefExpr{ - pos: position{line: 313, col: 28, offset: 9692}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - { - name: "CodeBlock", - pos: position{line: 317, col: 1, offset: 9763}, - expr: &choiceExpr{ - pos: position{line: 317, col: 13, offset: 9777}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 317, col: 13, offset: 9777}, - run: (*parser).callonCodeBlock2, - expr: &seqExpr{ - pos: position{line: 317, col: 13, offset: 9777}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 317, col: 13, offset: 9777}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 317, col: 17, offset: 9781}, - name: "Code", - }, - &litMatcher{ - pos: position{line: 317, col: 22, offset: 9786}, - val: "}", - ignoreCase: false, - }, - }, - }, - }, - &actionExpr{ - pos: position{line: 321, col: 5, offset: 9885}, - run: (*parser).callonCodeBlock7, - expr: &seqExpr{ - pos: position{line: 321, col: 5, offset: 9885}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 321, col: 5, offset: 9885}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 321, col: 9, offset: 9889}, - name: "Code", - }, - &ruleRefExpr{ - pos: position{line: 321, col: 14, offset: 9894}, - name: "EOF", - }, - }, - }, - }, - }, - }, - }, - { - name: "Code", - pos: position{line: 325, col: 1, offset: 9959}, - expr: &zeroOrMoreExpr{ - pos: position{line: 325, col: 8, offset: 9968}, - expr: &choiceExpr{ - pos: position{line: 325, col: 10, offset: 9970}, - alternatives: []interface{}{ - &oneOrMoreExpr{ - pos: position{line: 325, col: 10, offset: 9970}, - expr: &seqExpr{ - pos: position{line: 325, col: 12, offset: 9972}, - exprs: []interface{}{ - ¬Expr{ - pos: position{line: 325, col: 12, offset: 9972}, - expr: &charClassMatcher{ - pos: position{line: 325, col: 13, offset: 9973}, - val: "[{}]", - chars: []rune{'{', '}'}, - ignoreCase: false, - inverted: false, - }, - }, - &ruleRefExpr{ - pos: position{line: 325, col: 18, offset: 9978}, - name: "SourceChar", - }, - }, - }, - }, - &seqExpr{ - pos: position{line: 325, col: 34, offset: 9994}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 325, col: 34, offset: 9994}, - val: "{", - ignoreCase: false, - }, - &ruleRefExpr{ - pos: position{line: 325, col: 38, offset: 9998}, - name: "Code", - }, - &litMatcher{ - pos: position{line: 325, col: 43, offset: 10003}, - val: "}", - ignoreCase: false, - }, - }, - }, - }, - }, - }, - }, - { - name: "__", - pos: position{line: 327, col: 1, offset: 10011}, - expr: &zeroOrMoreExpr{ - pos: position{line: 327, col: 6, offset: 10018}, - expr: &choiceExpr{ - pos: position{line: 327, col: 8, offset: 10020}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 327, col: 8, offset: 10020}, - name: "Whitespace", - }, - &ruleRefExpr{ - pos: position{line: 327, col: 21, offset: 10033}, - name: "EOL", - }, - &ruleRefExpr{ - pos: position{line: 327, col: 27, offset: 10039}, - name: "Comment", - }, - }, - }, - }, - }, - { - name: "_", - pos: position{line: 328, col: 1, offset: 10050}, - expr: &zeroOrMoreExpr{ - pos: position{line: 328, col: 5, offset: 10056}, - expr: &choiceExpr{ - pos: position{line: 328, col: 7, offset: 10058}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 328, col: 7, offset: 10058}, - name: "Whitespace", - }, - &ruleRefExpr{ - pos: position{line: 328, col: 20, offset: 10071}, - name: "MultiLineCommentNoLineTerminator", - }, - }, - }, - }, - }, - { - name: "Whitespace", - pos: position{line: 330, col: 1, offset: 10108}, - expr: &charClassMatcher{ - pos: position{line: 330, col: 14, offset: 10123}, - val: "[ \\t\\r]", - chars: []rune{' ', '\t', '\r'}, - ignoreCase: false, - inverted: false, - }, - }, - { - name: "EOL", - pos: position{line: 331, col: 1, offset: 10131}, - expr: &litMatcher{ - pos: position{line: 331, col: 7, offset: 10139}, - val: "\n", - ignoreCase: false, - }, - }, - { - name: "EOS", - pos: position{line: 332, col: 1, offset: 10144}, - expr: &choiceExpr{ - pos: position{line: 332, col: 7, offset: 10152}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 332, col: 7, offset: 10152}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 332, col: 7, offset: 10152}, - name: "__", - }, - &litMatcher{ - pos: position{line: 332, col: 10, offset: 10155}, - val: ";", - ignoreCase: false, - }, - }, - }, - &seqExpr{ - pos: position{line: 332, col: 16, offset: 10161}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 332, col: 16, offset: 10161}, - name: "_", - }, - &zeroOrOneExpr{ - pos: position{line: 332, col: 18, offset: 10163}, - expr: &ruleRefExpr{ - pos: position{line: 332, col: 18, offset: 10163}, - name: "SingleLineComment", - }, - }, - &ruleRefExpr{ - pos: position{line: 332, col: 37, offset: 10182}, - name: "EOL", - }, - }, - }, - &seqExpr{ - pos: position{line: 332, col: 43, offset: 10188}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 332, col: 43, offset: 10188}, - name: "__", - }, - &ruleRefExpr{ - pos: position{line: 332, col: 46, offset: 10191}, - name: "EOF", - }, - }, - }, - }, - }, - }, - { - name: "EOF", - pos: position{line: 334, col: 1, offset: 10196}, - expr: ¬Expr{ - pos: position{line: 334, col: 7, offset: 10204}, - expr: &anyMatcher{ - line: 334, col: 8, offset: 10205, - }, - }, - }, - }, -} - -func (c *current) onGrammar1(initializer, rules interface{}) (interface{}, error) { - pos := c.astPos() - - // create the grammar, assign its initializer - g := ast.NewGrammar(pos) - initSlice := toIfaceSlice(initializer) - if len(initSlice) > 0 { - g.Init = initSlice[0].(*ast.CodeBlock) - } - - rulesSlice := toIfaceSlice(rules) - g.Rules = make([]*ast.Rule, len(rulesSlice)) - for i, duo := range rulesSlice { - g.Rules[i] = duo.([]interface{})[0].(*ast.Rule) - } - - return g, nil -} - -func (p *parser) callonGrammar1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onGrammar1(stack["initializer"], stack["rules"]) -} - -func (c *current) onInitializer1(code interface{}) (interface{}, error) { - return code, nil -} - -func (p *parser) callonInitializer1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onInitializer1(stack["code"]) -} - -func (c *current) onRule1(name, display, expr interface{}) (interface{}, error) { - pos := c.astPos() - - rule := ast.NewRule(pos, name.(*ast.Identifier)) - displaySlice := toIfaceSlice(display) - if len(displaySlice) > 0 { - rule.DisplayName = displaySlice[0].(*ast.StringLit) - } - rule.Expr = expr.(ast.Expression) - - return rule, nil -} - -func (p *parser) callonRule1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRule1(stack["name"], stack["display"], stack["expr"]) -} - -func (c *current) onRecoveryExpr1(expr, recoverExprs interface{}) (interface{}, error) { - recoverExprSlice := toIfaceSlice(recoverExprs) - recover := expr.(ast.Expression) - for _, sl := range recoverExprSlice { - pos := c.astPos() - r := ast.NewRecoveryExpr(pos) - r.Expr = recover - r.RecoverExpr = sl.([]interface{})[7].(ast.Expression) - r.Labels = sl.([]interface{})[3].([]ast.FailureLabel) - - recover = r - } - return recover, nil -} - -func (p *parser) callonRecoveryExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRecoveryExpr1(stack["expr"], stack["recoverExprs"]) -} - -func (c *current) onLabels1(label, labels interface{}) (interface{}, error) { - failureLabels := []ast.FailureLabel{ast.FailureLabel(label.(*ast.Identifier).Val)} - labelSlice := toIfaceSlice(labels) - for _, fl := range labelSlice { - failureLabels = append(failureLabels, ast.FailureLabel(fl.([]interface{})[3].(*ast.Identifier).Val)) - } - return failureLabels, nil -} - -func (p *parser) callonLabels1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onLabels1(stack["label"], stack["labels"]) -} - -func (c *current) onChoiceExpr1(first, rest interface{}) (interface{}, error) { - restSlice := toIfaceSlice(rest) - if len(restSlice) == 0 { - return first, nil - } - - pos := c.astPos() - choice := ast.NewChoiceExpr(pos) - choice.Alternatives = []ast.Expression{first.(ast.Expression)} - for _, sl := range restSlice { - choice.Alternatives = append(choice.Alternatives, sl.([]interface{})[3].(ast.Expression)) - } - return choice, nil -} - -func (p *parser) callonChoiceExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onChoiceExpr1(stack["first"], stack["rest"]) -} - -func (c *current) onActionExpr1(expr, code interface{}) (interface{}, error) { - if code == nil { - return expr, nil - } - - pos := c.astPos() - act := ast.NewActionExpr(pos) - act.Expr = expr.(ast.Expression) - codeSlice := toIfaceSlice(code) - act.Code = codeSlice[1].(*ast.CodeBlock) - - return act, nil -} - -func (p *parser) callonActionExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onActionExpr1(stack["expr"], stack["code"]) -} - -func (c *current) onSeqExpr1(first, rest interface{}) (interface{}, error) { - restSlice := toIfaceSlice(rest) - if len(restSlice) == 0 { - return first, nil - } - seq := ast.NewSeqExpr(c.astPos()) - seq.Exprs = []ast.Expression{first.(ast.Expression)} - for _, sl := range restSlice { - seq.Exprs = append(seq.Exprs, sl.([]interface{})[1].(ast.Expression)) - } - return seq, nil -} - -func (p *parser) callonSeqExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSeqExpr1(stack["first"], stack["rest"]) -} - -func (c *current) onLabeledExpr2(label, expr interface{}) (interface{}, error) { - pos := c.astPos() - lab := ast.NewLabeledExpr(pos) - lab.Label = label.(*ast.Identifier) - lab.Expr = expr.(ast.Expression) - return lab, nil -} - -func (p *parser) callonLabeledExpr2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onLabeledExpr2(stack["label"], stack["expr"]) -} - -func (c *current) onPrefixedExpr2(op, expr interface{}) (interface{}, error) { - pos := c.astPos() - opStr := op.(string) - if opStr == "&" { - and := ast.NewAndExpr(pos) - and.Expr = expr.(ast.Expression) - return and, nil - } - not := ast.NewNotExpr(pos) - not.Expr = expr.(ast.Expression) - return not, nil -} - -func (p *parser) callonPrefixedExpr2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onPrefixedExpr2(stack["op"], stack["expr"]) -} - -func (c *current) onPrefixedOp1() (interface{}, error) { - return string(c.text), nil -} - -func (p *parser) callonPrefixedOp1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onPrefixedOp1() -} - -func (c *current) onSuffixedExpr2(expr, op interface{}) (interface{}, error) { - pos := c.astPos() - opStr := op.(string) - switch opStr { - case "?": - zero := ast.NewZeroOrOneExpr(pos) - zero.Expr = expr.(ast.Expression) - return zero, nil - case "*": - zero := ast.NewZeroOrMoreExpr(pos) - zero.Expr = expr.(ast.Expression) - return zero, nil - case "+": - one := ast.NewOneOrMoreExpr(pos) - one.Expr = expr.(ast.Expression) - return one, nil - default: - return nil, errors.New("unknown operator: " + opStr) - } -} - -func (p *parser) callonSuffixedExpr2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSuffixedExpr2(stack["expr"], stack["op"]) -} - -func (c *current) onSuffixedOp1() (interface{}, error) { - return string(c.text), nil -} - -func (p *parser) callonSuffixedOp1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSuffixedOp1() -} - -func (c *current) onPrimaryExpr7(expr interface{}) (interface{}, error) { - return expr, nil -} - -func (p *parser) callonPrimaryExpr7() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onPrimaryExpr7(stack["expr"]) -} - -func (c *current) onRuleRefExpr1(name interface{}) (interface{}, error) { - ref := ast.NewRuleRefExpr(c.astPos()) - ref.Name = name.(*ast.Identifier) - return ref, nil -} - -func (p *parser) callonRuleRefExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onRuleRefExpr1(stack["name"]) -} - -func (c *current) onSemanticPredExpr1(op, code interface{}) (interface{}, error) { - switch op.(string) { - case "#": - state := ast.NewStateCodeExpr(c.astPos()) - state.Code = code.(*ast.CodeBlock) - return state, nil - - case "&": - and := ast.NewAndCodeExpr(c.astPos()) - and.Code = code.(*ast.CodeBlock) - return and, nil - - // case "!": - default: - not := ast.NewNotCodeExpr(c.astPos()) - not.Code = code.(*ast.CodeBlock) - return not, nil - - } -} - -func (p *parser) callonSemanticPredExpr1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSemanticPredExpr1(stack["op"], stack["code"]) -} - -func (c *current) onSemanticPredOp1() (interface{}, error) { - return string(c.text), nil -} - -func (p *parser) callonSemanticPredOp1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSemanticPredOp1() -} - -func (c *current) onIdentifier1(ident interface{}) (interface{}, error) { - astIdent := ast.NewIdentifier(c.astPos(), string(c.text)) - if reservedWords[astIdent.Val] { - return astIdent, errors.New("identifier is a reserved word") - } - return astIdent, nil -} - -func (p *parser) callonIdentifier1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onIdentifier1(stack["ident"]) -} - -func (c *current) onIdentifierName1() (interface{}, error) { - return ast.NewIdentifier(c.astPos(), string(c.text)), nil -} - -func (p *parser) callonIdentifierName1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onIdentifierName1() -} - -func (c *current) onLitMatcher1(lit, ignore interface{}) (interface{}, error) { - rawStr := lit.(*ast.StringLit).Val - s, err := strconv.Unquote(rawStr) - if err != nil { - // an invalid string literal raises an error in the escape rules, - // so simply replace the literal with an empty string here to - // avoid a cascade of errors. - s = "" - } - m := ast.NewLitMatcher(c.astPos(), s) - m.IgnoreCase = ignore != nil - return m, nil -} - -func (p *parser) callonLitMatcher1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onLitMatcher1(stack["lit"], stack["ignore"]) -} - -func (c *current) onStringLiteral2() (interface{}, error) { - return ast.NewStringLit(c.astPos(), string(c.text)), nil -} - -func (p *parser) callonStringLiteral2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onStringLiteral2() -} - -func (c *current) onStringLiteral18() (interface{}, error) { - return ast.NewStringLit(c.astPos(), "``"), errors.New("string literal not terminated") -} - -func (p *parser) callonStringLiteral18() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onStringLiteral18() -} - -func (c *current) onDoubleStringEscape5() (interface{}, error) { - return nil, errors.New("invalid escape character") -} - -func (p *parser) callonDoubleStringEscape5() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onDoubleStringEscape5() -} - -func (c *current) onSingleStringEscape5() (interface{}, error) { - return nil, errors.New("invalid escape character") -} - -func (p *parser) callonSingleStringEscape5() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onSingleStringEscape5() -} - -func (c *current) onOctalEscape6() (interface{}, error) { - return nil, errors.New("invalid octal escape") -} - -func (p *parser) callonOctalEscape6() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onOctalEscape6() -} - -func (c *current) onHexEscape6() (interface{}, error) { - return nil, errors.New("invalid hexadecimal escape") -} - -func (p *parser) callonHexEscape6() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onHexEscape6() -} - -func (c *current) onLongUnicodeEscape2() (interface{}, error) { - return validateUnicodeEscape(string(c.text), "invalid Unicode escape") - -} - -func (p *parser) callonLongUnicodeEscape2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onLongUnicodeEscape2() -} - -func (c *current) onLongUnicodeEscape13() (interface{}, error) { - return nil, errors.New("invalid Unicode escape") -} - -func (p *parser) callonLongUnicodeEscape13() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onLongUnicodeEscape13() -} - -func (c *current) onShortUnicodeEscape2() (interface{}, error) { - return validateUnicodeEscape(string(c.text), "invalid Unicode escape") - -} - -func (p *parser) callonShortUnicodeEscape2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onShortUnicodeEscape2() -} - -func (c *current) onShortUnicodeEscape9() (interface{}, error) { - return nil, errors.New("invalid Unicode escape") -} - -func (p *parser) callonShortUnicodeEscape9() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onShortUnicodeEscape9() -} - -func (c *current) onCharClassMatcher2() (interface{}, error) { - pos := c.astPos() - cc := ast.NewCharClassMatcher(pos, string(c.text)) - return cc, nil -} - -func (p *parser) callonCharClassMatcher2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onCharClassMatcher2() -} - -func (c *current) onCharClassMatcher15() (interface{}, error) { - return ast.NewCharClassMatcher(c.astPos(), "[]"), errors.New("character class not terminated") -} - -func (p *parser) callonCharClassMatcher15() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onCharClassMatcher15() -} - -func (c *current) onCharClassEscape5() (interface{}, error) { - return nil, errors.New("invalid escape character") -} - -func (p *parser) callonCharClassEscape5() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onCharClassEscape5() -} - -func (c *current) onUnicodeClassEscape5() (interface{}, error) { - return nil, errors.New("invalid Unicode class escape") -} - -func (p *parser) callonUnicodeClassEscape5() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onUnicodeClassEscape5() -} - -func (c *current) onUnicodeClassEscape13(ident interface{}) (interface{}, error) { - if !unicodeClasses[ident.(*ast.Identifier).Val] { - return nil, errors.New("invalid Unicode class escape") - } - return nil, nil - -} - -func (p *parser) callonUnicodeClassEscape13() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onUnicodeClassEscape13(stack["ident"]) -} - -func (c *current) onUnicodeClassEscape19() (interface{}, error) { - return nil, errors.New("Unicode class not terminated") - -} - -func (p *parser) callonUnicodeClassEscape19() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onUnicodeClassEscape19() -} - -func (c *current) onAnyMatcher1() (interface{}, error) { - any := ast.NewAnyMatcher(c.astPos(), ".") - return any, nil -} - -func (p *parser) callonAnyMatcher1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onAnyMatcher1() -} - -func (c *current) onThrowExpr2(label interface{}) (interface{}, error) { - t := ast.NewThrowExpr(c.astPos()) - t.Label = label.(*ast.Identifier).Val - return t, nil -} - -func (p *parser) callonThrowExpr2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onThrowExpr2(stack["label"]) -} - -func (c *current) onThrowExpr9() (interface{}, error) { - return nil, errors.New("throw expression not terminated") -} - -func (p *parser) callonThrowExpr9() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onThrowExpr9() -} - -func (c *current) onCodeBlock2() (interface{}, error) { - pos := c.astPos() - cb := ast.NewCodeBlock(pos, string(c.text)) - return cb, nil -} - -func (p *parser) callonCodeBlock2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onCodeBlock2() -} - -func (c *current) onCodeBlock7() (interface{}, error) { - return nil, errors.New("code block not terminated") -} - -func (p *parser) callonCodeBlock7() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onCodeBlock7() -} - -var ( - // errNoRule is returned when the grammar to parse has no rule. - errNoRule = errors.New("grammar has no rule") - - // errInvalidEntrypoint is returned when the specified entrypoint rule - // does not exit. - errInvalidEntrypoint = errors.New("invalid entrypoint") - - // errInvalidEncoding is returned when the source is not properly - // utf8-encoded. - errInvalidEncoding = errors.New("invalid encoding") - - // errMaxExprCnt is used to signal that the maximum number of - // expressions have been parsed. - errMaxExprCnt = errors.New("max number of expresssions parsed") -) - -// Option is a function that can set an option on the parser. It returns -// the previous setting as an Option. -type Option func(*parser) Option - -// MaxExpressions creates an Option to stop parsing after the provided -// number of expressions have been parsed, if the value is 0 then the parser will -// parse for as many steps as needed (possibly an infinite number). -// -// The default for maxExprCnt is 0. -func MaxExpressions(maxExprCnt uint64) Option { - return func(p *parser) Option { - oldMaxExprCnt := p.maxExprCnt - p.maxExprCnt = maxExprCnt - return MaxExpressions(oldMaxExprCnt) - } -} - -// Entrypoint creates an Option to set the rule name to use as entrypoint. -// The rule name must have been specified in the -alternate-entrypoints -// if generating the parser with the -optimize-grammar flag, otherwise -// it may have been optimized out. Passing an empty string sets the -// entrypoint to the first rule in the grammar. -// -// The default is to start parsing at the first rule in the grammar. -func Entrypoint(ruleName string) Option { - return func(p *parser) Option { - oldEntrypoint := p.entrypoint - p.entrypoint = ruleName - if ruleName == "" { - p.entrypoint = g.rules[0].name - } - return Entrypoint(oldEntrypoint) - } -} - -// Statistics adds a user provided Stats struct to the parser to allow -// the user to process the results after the parsing has finished. -// Also the key for the "no match" counter is set. -// -// Example usage: -// -// input := "input" -// stats := Stats{} -// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) -// if err != nil { -// log.Panicln(err) -// } -// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") -// if err != nil { -// log.Panicln(err) -// } -// fmt.Println(string(b)) -// -func Statistics(stats *Stats, choiceNoMatch string) Option { - return func(p *parser) Option { - oldStats := p.Stats - p.Stats = stats - oldChoiceNoMatch := p.choiceNoMatch - p.choiceNoMatch = choiceNoMatch - if p.Stats.ChoiceAltCnt == nil { - p.Stats.ChoiceAltCnt = make(map[string]map[string]int) - } - return Statistics(oldStats, oldChoiceNoMatch) - } -} - -// Debug creates an Option to set the debug flag to b. When set to true, -// debugging information is printed to stdout while parsing. -// -// The default is false. -func Debug(b bool) Option { - return func(p *parser) Option { - old := p.debug - p.debug = b - return Debug(old) - } -} - -// Memoize creates an Option to set the memoize flag to b. When set to true, -// the parser will cache all results so each expression is evaluated only -// once. This guarantees linear parsing time even for pathological cases, -// at the expense of more memory and slower times for typical cases. -// -// The default is false. -func Memoize(b bool) Option { - return func(p *parser) Option { - old := p.memoize - p.memoize = b - return Memoize(old) - } -} - -// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. -// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) -// by character class matchers and is matched by the any matcher. -// The returned matched value, c.text and c.offset are NOT affected. -// -// The default is false. -func AllowInvalidUTF8(b bool) Option { - return func(p *parser) Option { - old := p.allowInvalidUTF8 - p.allowInvalidUTF8 = b - return AllowInvalidUTF8(old) - } -} - -// Recover creates an Option to set the recover flag to b. When set to -// true, this causes the parser to recover from panics and convert it -// to an error. Setting it to false can be useful while debugging to -// access the full stack trace. -// -// The default is true. -func Recover(b bool) Option { - return func(p *parser) Option { - old := p.recover - p.recover = b - return Recover(old) - } -} - -// GlobalStore creates an Option to set a key to a certain value in -// the globalStore. -func GlobalStore(key string, value interface{}) Option { - return func(p *parser) Option { - old := p.cur.globalStore[key] - p.cur.globalStore[key] = value - return GlobalStore(key, old) - } -} - -// InitState creates an Option to set a key to a certain value in -// the global "state" store. -func InitState(key string, value interface{}) Option { - return func(p *parser) Option { - old := p.cur.state[key] - p.cur.state[key] = value - return InitState(key, old) - } -} - -// ParseFile parses the file identified by filename. -func ParseFile(filename string, opts ...Option) (i interface{}, err error) { // nolint: deadcode - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer func() { - if closeErr := f.Close(); closeErr != nil { - err = closeErr - } - }() - return ParseReader(filename, f, opts...) -} - -// ParseReader parses the data from r using filename as information in the -// error messages. -func ParseReader(filename string, r io.Reader, opts ...Option) (interface{}, error) { // nolint: deadcode - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - return Parse(filename, b, opts...) -} - -// Parse parses the data from b using filename as information in the -// error messages. -func Parse(filename string, b []byte, opts ...Option) (interface{}, error) { - return newParser(filename, b, opts...).parse(g) -} - -// position records a position in the text. -type position struct { - line, col, offset int -} - -func (p position) String() string { - return fmt.Sprintf("%d:%d [%d]", p.line, p.col, p.offset) -} - -// savepoint stores all state required to go back to this point in the -// parser. -type savepoint struct { - position - rn rune - w int -} - -type current struct { - pos position // start position of the match - text []byte // raw text of the match - - // state is a store for arbitrary key,value pairs that the user wants to be - // tied to the backtracking of the parser. - // This is always rolled back if a parsing rule fails. - state storeDict - - // globalStore is a general store for the user to store arbitrary key-value - // pairs that they need to manage and that they do not want tied to the - // backtracking of the parser. This is only modified by the user and never - // rolled back by the parser. It is always up to the user to keep this in a - // consistent state. - globalStore storeDict -} - -type storeDict map[string]interface{} - -// the AST types... - -// nolint: structcheck -type grammar struct { - pos position - rules []*rule -} - -// nolint: structcheck -type rule struct { - pos position - name string - displayName string - expr interface{} -} - -// nolint: structcheck -type choiceExpr struct { - pos position - alternatives []interface{} -} - -// nolint: structcheck -type actionExpr struct { - pos position - expr interface{} - run func(*parser) (interface{}, error) -} - -// nolint: structcheck -type recoveryExpr struct { - pos position - expr interface{} - recoverExpr interface{} - failureLabel []string -} - -// nolint: structcheck -type seqExpr struct { - pos position - exprs []interface{} -} - -// nolint: structcheck -type throwExpr struct { - pos position - label string -} - -// nolint: structcheck -type labeledExpr struct { - pos position - label string - expr interface{} -} - -// nolint: structcheck -type expr struct { - pos position - expr interface{} -} - -type andExpr expr // nolint: structcheck -type notExpr expr // nolint: structcheck -type zeroOrOneExpr expr // nolint: structcheck -type zeroOrMoreExpr expr // nolint: structcheck -type oneOrMoreExpr expr // nolint: structcheck - -// nolint: structcheck -type ruleRefExpr struct { - pos position - name string -} - -// nolint: structcheck -type stateCodeExpr struct { - pos position - run func(*parser) error -} - -// nolint: structcheck -type andCodeExpr struct { - pos position - run func(*parser) (bool, error) -} - -// nolint: structcheck -type notCodeExpr struct { - pos position - run func(*parser) (bool, error) -} - -// nolint: structcheck -type litMatcher struct { - pos position - val string - ignoreCase bool -} - -// nolint: structcheck -type charClassMatcher struct { - pos position - val string - basicLatinChars [128]bool - chars []rune - ranges []rune - classes []*unicode.RangeTable - ignoreCase bool - inverted bool -} - -type anyMatcher position // nolint: structcheck - -// errList cumulates the errors found by the parser. -type errList []error - -func (e *errList) add(err error) { - *e = append(*e, err) -} - -func (e errList) err() error { - if len(e) == 0 { - return nil - } - e.dedupe() - return e -} - -func (e *errList) dedupe() { - var cleaned []error - set := make(map[string]bool) - for _, err := range *e { - if msg := err.Error(); !set[msg] { - set[msg] = true - cleaned = append(cleaned, err) - } - } - *e = cleaned -} - -func (e errList) Error() string { - switch len(e) { - case 0: - return "" - case 1: - return e[0].Error() - default: - var buf bytes.Buffer - - for i, err := range e { - if i > 0 { - buf.WriteRune('\n') - } - buf.WriteString(err.Error()) - } - return buf.String() - } -} - -// parserError wraps an error with a prefix indicating the rule in which -// the error occurred. The original error is stored in the Inner field. -type parserError struct { - Inner error - pos position - prefix string - expected []string -} - -// Error returns the error message. -func (p *parserError) Error() string { - return p.prefix + ": " + p.Inner.Error() -} - -// newParser creates a parser with the specified input source and options. -func newParser(filename string, b []byte, opts ...Option) *parser { - stats := Stats{ - ChoiceAltCnt: make(map[string]map[string]int), - } - - p := &parser{ - filename: filename, - errs: new(errList), - data: b, - pt: savepoint{position: position{line: 1}}, - recover: true, - cur: current{ - state: make(storeDict), - globalStore: make(storeDict), - }, - maxFailPos: position{col: 1, line: 1}, - maxFailExpected: make([]string, 0, 20), - Stats: &stats, - // start rule is rule [0] unless an alternate entrypoint is specified - entrypoint: g.rules[0].name, - } - p.setOptions(opts) - - if p.maxExprCnt == 0 { - p.maxExprCnt = math.MaxUint64 - } - - return p -} - -// setOptions applies the options to the parser. -func (p *parser) setOptions(opts []Option) { - for _, opt := range opts { - opt(p) - } -} - -// nolint: structcheck,deadcode -type resultTuple struct { - v interface{} - b bool - end savepoint -} - -// nolint: varcheck -const choiceNoMatch = -1 - -// Stats stores some statistics, gathered during parsing -type Stats struct { - // ExprCnt counts the number of expressions processed during parsing - // This value is compared to the maximum number of expressions allowed - // (set by the MaxExpressions option). - ExprCnt uint64 - - // ChoiceAltCnt is used to count for each ordered choice expression, - // which alternative is used how may times. - // These numbers allow to optimize the order of the ordered choice expression - // to increase the performance of the parser - // - // The outer key of ChoiceAltCnt is composed of the name of the rule as well - // as the line and the column of the ordered choice. - // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. - // For each alternative the number of matches are counted. If an ordered choice does not - // match, a special counter is incremented. The name of this counter is set with - // the parser option Statistics. - // For an alternative to be included in ChoiceAltCnt, it has to match at least once. - ChoiceAltCnt map[string]map[string]int -} - -// nolint: structcheck,maligned -type parser struct { - filename string - pt savepoint - cur current - - data []byte - errs *errList - - depth int - recover bool - debug bool - - memoize bool - // memoization table for the packrat algorithm: - // map[offset in source] map[expression or rule] {value, match} - memo map[int]map[interface{}]resultTuple - - // rules table, maps the rule identifier to the rule node - rules map[string]*rule - // variables stack, map of label to value - vstack []map[string]interface{} - // rule stack, allows identification of the current rule in errors - rstack []*rule - - // parse fail - maxFailPos position - maxFailExpected []string - maxFailInvertExpected bool - - // max number of expressions to be parsed - maxExprCnt uint64 - // entrypoint for the parser - entrypoint string - - allowInvalidUTF8 bool - - *Stats - - choiceNoMatch string - // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse - recoveryStack []map[string]interface{} -} - -// push a variable set on the vstack. -func (p *parser) pushV() { - if cap(p.vstack) == len(p.vstack) { - // create new empty slot in the stack - p.vstack = append(p.vstack, nil) - } else { - // slice to 1 more - p.vstack = p.vstack[:len(p.vstack)+1] - } - - // get the last args set - m := p.vstack[len(p.vstack)-1] - if m != nil && len(m) == 0 { - // empty map, all good - return - } - - m = make(map[string]interface{}) - p.vstack[len(p.vstack)-1] = m -} - -// pop a variable set from the vstack. -func (p *parser) popV() { - // if the map is not empty, clear it - m := p.vstack[len(p.vstack)-1] - if len(m) > 0 { - // GC that map - p.vstack[len(p.vstack)-1] = nil - } - p.vstack = p.vstack[:len(p.vstack)-1] -} - -// push a recovery expression with its labels to the recoveryStack -func (p *parser) pushRecovery(labels []string, expr interface{}) { - if cap(p.recoveryStack) == len(p.recoveryStack) { - // create new empty slot in the stack - p.recoveryStack = append(p.recoveryStack, nil) - } else { - // slice to 1 more - p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] - } - - m := make(map[string]interface{}, len(labels)) - for _, fl := range labels { - m[fl] = expr - } - p.recoveryStack[len(p.recoveryStack)-1] = m -} - -// pop a recovery expression from the recoveryStack -func (p *parser) popRecovery() { - // GC that map - p.recoveryStack[len(p.recoveryStack)-1] = nil - - p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] -} - -func (p *parser) print(prefix, s string) string { - if !p.debug { - return s - } - - fmt.Printf("%s %d:%d:%d: %s [%#U]\n", - prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) - return s -} - -func (p *parser) in(s string) string { - p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) -} - -func (p *parser) out(s string) string { - p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) -} - -func (p *parser) addErr(err error) { - p.addErrAt(err, p.pt.position, []string{}) -} - -func (p *parser) addErrAt(err error, pos position, expected []string) { - var buf bytes.Buffer - if p.filename != "" { - buf.WriteString(p.filename) - } - if buf.Len() > 0 { - buf.WriteString(":") - } - buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) - if len(p.rstack) > 0 { - if buf.Len() > 0 { - buf.WriteString(": ") - } - rule := p.rstack[len(p.rstack)-1] - if rule.displayName != "" { - buf.WriteString("rule " + rule.displayName) - } else { - buf.WriteString("rule " + rule.name) - } - } - pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} - p.errs.add(pe) -} - -func (p *parser) failAt(fail bool, pos position, want string) { - // process fail if parsing fails and not inverted or parsing succeeds and invert is set - if fail == p.maxFailInvertExpected { - if pos.offset < p.maxFailPos.offset { - return - } - - if pos.offset > p.maxFailPos.offset { - p.maxFailPos = pos - p.maxFailExpected = p.maxFailExpected[:0] - } - - if p.maxFailInvertExpected { - want = "!" + want - } - p.maxFailExpected = append(p.maxFailExpected, want) - } -} - -// read advances the parser to the next rune. -func (p *parser) read() { - p.pt.offset += p.pt.w - rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) - p.pt.rn = rn - p.pt.w = n - p.pt.col++ - if rn == '\n' { - p.pt.line++ - p.pt.col = 0 - } - - if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune - if !p.allowInvalidUTF8 { - p.addErr(errInvalidEncoding) - } - } -} - -// restore parser position to the savepoint pt. -func (p *parser) restore(pt savepoint) { - if p.debug { - defer p.out(p.in("restore")) - } - if pt.offset == p.pt.offset { - return - } - p.pt = pt -} - -// Cloner is implemented by any value that has a Clone method, which returns a -// copy of the value. This is mainly used for types which are not passed by -// value (e.g map, slice, chan) or structs that contain such types. -// -// This is used in conjunction with the global state feature to create proper -// copies of the state to allow the parser to properly restore the state in -// the case of backtracking. -type Cloner interface { - Clone() interface{} -} - -// clone and return parser current state. -func (p *parser) cloneState() storeDict { - if p.debug { - defer p.out(p.in("cloneState")) - } - - state := make(storeDict, len(p.cur.state)) - for k, v := range p.cur.state { - if c, ok := v.(Cloner); ok { - state[k] = c.Clone() - } else { - state[k] = v - } - } - return state -} - -// restore parser current state to the state storeDict. -// every restoreState should applied only one time for every cloned state -func (p *parser) restoreState(state storeDict) { - if p.debug { - defer p.out(p.in("restoreState")) - } - p.cur.state = state -} - -// get the slice of bytes from the savepoint start to the current position. -func (p *parser) sliceFrom(start savepoint) []byte { - return p.data[start.position.offset:p.pt.position.offset] -} - -func (p *parser) getMemoized(node interface{}) (resultTuple, bool) { - if len(p.memo) == 0 { - return resultTuple{}, false - } - m := p.memo[p.pt.offset] - if len(m) == 0 { - return resultTuple{}, false - } - res, ok := m[node] - return res, ok -} - -func (p *parser) setMemoized(pt savepoint, node interface{}, tuple resultTuple) { - if p.memo == nil { - p.memo = make(map[int]map[interface{}]resultTuple) - } - m := p.memo[pt.offset] - if m == nil { - m = make(map[interface{}]resultTuple) - p.memo[pt.offset] = m - } - m[node] = tuple -} - -func (p *parser) buildRulesTable(g *grammar) { - p.rules = make(map[string]*rule, len(g.rules)) - for _, r := range g.rules { - p.rules[r.name] = r - } -} - -// nolint: gocyclo -func (p *parser) parse(g *grammar) (val interface{}, err error) { - if len(g.rules) == 0 { - p.addErr(errNoRule) - return nil, p.errs.err() - } - - // TODO : not super critical but this could be generated - p.buildRulesTable(g) - - if p.recover { - // panic can be used in action code to stop parsing immediately - // and return the panic as an error. - defer func() { - if e := recover(); e != nil { - if p.debug { - defer p.out(p.in("panic handler")) - } - val = nil - switch e := e.(type) { - case error: - p.addErr(e) - default: - p.addErr(fmt.Errorf("%v", e)) - } - err = p.errs.err() - } - }() - } - - startRule, ok := p.rules[p.entrypoint] - if !ok { - p.addErr(errInvalidEntrypoint) - return nil, p.errs.err() - } - - p.read() // advance to first rune - val, ok = p.parseRule(startRule) - if !ok { - if len(*p.errs) == 0 { - // If parsing fails, but no errors have been recorded, the expected values - // for the farthest parser position are returned as error. - maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) - for _, v := range p.maxFailExpected { - maxFailExpectedMap[v] = struct{}{} - } - expected := make([]string, 0, len(maxFailExpectedMap)) - eof := false - if _, ok := maxFailExpectedMap["!."]; ok { - delete(maxFailExpectedMap, "!.") - eof = true - } - for k := range maxFailExpectedMap { - expected = append(expected, k) - } - sort.Strings(expected) - if eof { - expected = append(expected, "EOF") - } - p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) - } - - return nil, p.errs.err() - } - return val, p.errs.err() -} - -func listJoin(list []string, sep string, lastSep string) string { - switch len(list) { - case 0: - return "" - case 1: - return list[0] - default: - return fmt.Sprintf("%s %s %s", strings.Join(list[:len(list)-1], sep), lastSep, list[len(list)-1]) - } -} - -func (p *parser) parseRule(rule *rule) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseRule " + rule.name)) - } - - if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } - } - - start := p.pt - p.rstack = append(p.rstack, rule) - p.pushV() - val, ok := p.parseExpr(rule.expr) - p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } - return val, ok -} - -// nolint: gocyclo -func (p *parser) parseExpr(expr interface{}) (interface{}, bool) { - var pt savepoint - - if p.memoize { - res, ok := p.getMemoized(expr) - if ok { - p.restore(res.end) - return res.v, res.b - } - pt = p.pt - } - - p.ExprCnt++ - if p.ExprCnt > p.maxExprCnt { - panic(errMaxExprCnt) - } - - var val interface{} - var ok bool - switch expr := expr.(type) { - case *actionExpr: - val, ok = p.parseActionExpr(expr) - case *andCodeExpr: - val, ok = p.parseAndCodeExpr(expr) - case *andExpr: - val, ok = p.parseAndExpr(expr) - case *anyMatcher: - val, ok = p.parseAnyMatcher(expr) - case *charClassMatcher: - val, ok = p.parseCharClassMatcher(expr) - case *choiceExpr: - val, ok = p.parseChoiceExpr(expr) - case *labeledExpr: - val, ok = p.parseLabeledExpr(expr) - case *litMatcher: - val, ok = p.parseLitMatcher(expr) - case *notCodeExpr: - val, ok = p.parseNotCodeExpr(expr) - case *notExpr: - val, ok = p.parseNotExpr(expr) - case *oneOrMoreExpr: - val, ok = p.parseOneOrMoreExpr(expr) - case *recoveryExpr: - val, ok = p.parseRecoveryExpr(expr) - case *ruleRefExpr: - val, ok = p.parseRuleRefExpr(expr) - case *seqExpr: - val, ok = p.parseSeqExpr(expr) - case *stateCodeExpr: - val, ok = p.parseStateCodeExpr(expr) - case *throwExpr: - val, ok = p.parseThrowExpr(expr) - case *zeroOrMoreExpr: - val, ok = p.parseZeroOrMoreExpr(expr) - case *zeroOrOneExpr: - val, ok = p.parseZeroOrOneExpr(expr) - default: - panic(fmt.Sprintf("unknown expression type %T", expr)) - } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } - return val, ok -} - -func (p *parser) parseActionExpr(act *actionExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseActionExpr")) - } - - start := p.pt - val, ok := p.parseExpr(act.expr) - if ok { - p.cur.pos = start.position - p.cur.text = p.sliceFrom(start) - state := p.cloneState() - actVal, err := act.run(p) - if err != nil { - p.addErrAt(err, start.position, []string{}) - } - p.restoreState(state) - - val = actVal - } - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - return val, ok -} - -func (p *parser) parseAndCodeExpr(and *andCodeExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseAndCodeExpr")) - } - - state := p.cloneState() - - ok, err := and.run(p) - if err != nil { - p.addErr(err) - } - p.restoreState(state) - - return nil, ok -} - -func (p *parser) parseAndExpr(and *andExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseAndExpr")) - } - - pt := p.pt - state := p.cloneState() - p.pushV() - _, ok := p.parseExpr(and.expr) - p.popV() - p.restoreState(state) - p.restore(pt) - - return nil, ok -} - -func (p *parser) parseAnyMatcher(any *anyMatcher) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseAnyMatcher")) - } - - if p.pt.rn == utf8.RuneError && p.pt.w == 0 { - // EOF - see utf8.DecodeRune - p.failAt(false, p.pt.position, ".") - return nil, false - } - start := p.pt - p.read() - p.failAt(true, start.position, ".") - return p.sliceFrom(start), true -} - -// nolint: gocyclo -func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseCharClassMatcher")) - } - - cur := p.pt.rn - start := p.pt - - // can't match EOF - if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune - p.failAt(false, start.position, chr.val) - return nil, false - } - - if chr.ignoreCase { - cur = unicode.ToLower(cur) - } - - // try to match in the list of available chars - for _, rn := range chr.chars { - if rn == cur { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - // try to match in the list of ranges - for i := 0; i < len(chr.ranges); i += 2 { - if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - // try to match in the list of Unicode classes - for _, cl := range chr.classes { - if unicode.Is(cl, cur) { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - if chr.inverted { - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - p.failAt(false, start.position, chr.val) - return nil, false -} - -func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) - m := p.ChoiceAltCnt[choiceIdent] - if m == nil { - m = make(map[string]int) - p.ChoiceAltCnt[choiceIdent] = m - } - // We increment altI by 1, so the keys do not start at 0 - alt := strconv.Itoa(altI + 1) - if altI == choiceNoMatch { - alt = p.choiceNoMatch - } - m[alt]++ -} - -func (p *parser) parseChoiceExpr(ch *choiceExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseChoiceExpr")) - } - - for altI, alt := range ch.alternatives { - // dummy assignment to prevent compile error if optimized - _ = altI - - state := p.cloneState() - - p.pushV() - val, ok := p.parseExpr(alt) - p.popV() - if ok { - p.incChoiceAltCnt(ch, altI) - return val, ok - } - p.restoreState(state) - } - p.incChoiceAltCnt(ch, choiceNoMatch) - return nil, false -} - -func (p *parser) parseLabeledExpr(lab *labeledExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseLabeledExpr")) - } - - p.pushV() - val, ok := p.parseExpr(lab.expr) - p.popV() - if ok && lab.label != "" { - m := p.vstack[len(p.vstack)-1] - m[lab.label] = val - } - return val, ok -} - -func (p *parser) parseLitMatcher(lit *litMatcher) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseLitMatcher")) - } - - ignoreCase := "" - if lit.ignoreCase { - ignoreCase = "i" - } - val := fmt.Sprintf("%q%s", lit.val, ignoreCase) - start := p.pt - for _, want := range lit.val { - cur := p.pt.rn - if lit.ignoreCase { - cur = unicode.ToLower(cur) - } - if cur != want { - p.failAt(false, start.position, val) - p.restore(start) - return nil, false - } - p.read() - } - p.failAt(true, start.position, val) - return p.sliceFrom(start), true -} - -func (p *parser) parseNotCodeExpr(not *notCodeExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseNotCodeExpr")) - } - - state := p.cloneState() - - ok, err := not.run(p) - if err != nil { - p.addErr(err) - } - p.restoreState(state) - - return nil, !ok -} - -func (p *parser) parseNotExpr(not *notExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseNotExpr")) - } - - pt := p.pt - state := p.cloneState() - p.pushV() - p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) - p.maxFailInvertExpected = !p.maxFailInvertExpected - p.popV() - p.restoreState(state) - p.restore(pt) - - return nil, !ok -} - -func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseOneOrMoreExpr")) - } - - var vals []interface{} - - for { - p.pushV() - val, ok := p.parseExpr(expr.expr) - p.popV() - if !ok { - if len(vals) == 0 { - // did not match once, no match - return nil, false - } - return vals, true - } - vals = append(vals, val) - } -} - -func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) - } - - p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) - p.popRecovery() - - return val, ok -} - -func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseRuleRefExpr " + ref.name)) - } - - if ref.name == "" { - panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) - } - - rule := p.rules[ref.name] - if rule == nil { - p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) - return nil, false - } - return p.parseRule(rule) -} - -func (p *parser) parseSeqExpr(seq *seqExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseSeqExpr")) - } - - vals := make([]interface{}, 0, len(seq.exprs)) - - pt := p.pt - state := p.cloneState() - for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) - if !ok { - p.restoreState(state) - p.restore(pt) - return nil, false - } - vals = append(vals, val) - } - return vals, true -} - -func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseStateCodeExpr")) - } - - err := state.run(p) - if err != nil { - p.addErr(err) - } - return nil, true -} - -func (p *parser) parseThrowExpr(expr *throwExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseThrowExpr")) - } - - for i := len(p.recoveryStack) - 1; i >= 0; i-- { - if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { - return val, ok - } - } - } - - return nil, false -} - -func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseZeroOrMoreExpr")) - } - - var vals []interface{} - - for { - p.pushV() - val, ok := p.parseExpr(expr.expr) - p.popV() - if !ok { - return vals, true - } - vals = append(vals, val) - } -} - -func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseZeroOrOneExpr")) - } - - p.pushV() - val, _ := p.parseExpr(expr.expr) - p.popV() - // whether it matched or not, consider it a match - return val, true -} - -func rangeTable(class string) *unicode.RangeTable { - if rt, ok := unicode.Categories[class]; ok { - return rt - } - if rt, ok := unicode.Properties[class]; ok { - return rt - } - if rt, ok := unicode.Scripts[class]; ok { - return rt - } - - // cannot happen - panic(fmt.Sprintf("invalid Unicode class: %s", class)) -} diff --git a/vendor/github.com/mna/pigeon/reserved_words.go b/vendor/github.com/mna/pigeon/reserved_words.go deleted file mode 100644 index 127d27387d..0000000000 --- a/vendor/github.com/mna/pigeon/reserved_words.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -var reservedWords = map[string]bool{ - // Go keywords http://golang.org/ref/spec#Keywords - "break": true, - "case": true, - "chan": true, - "const": true, - "continue": true, - "default": true, - "defer": true, - "else": true, - "fallthrough": true, - "for": true, - "func": true, - "goto": true, - "go": true, - "if": true, - "import": true, - "interface": true, - "map": true, - "package": true, - "range": true, - "return": true, - "select": true, - "struct": true, - "switch": true, - "type": true, - "var": true, - - // predeclared identifiers http://golang.org/ref/spec#Predeclared_identifiers - "bool": true, - "byte": true, - "complex64": true, - "complex128": true, - "error": true, - "float32": true, - "float64": true, - "int8": true, - "int16": true, - "int32": true, - "int64": true, - "int": true, - "rune": true, - "string": true, - "uint8": true, - "uint16": true, - "uint32": true, - "uint64": true, - "uintptr": true, - "uint": true, - "true": true, - "false": true, - "iota": true, - "nil": true, - "append": true, - "cap": true, - "close": true, - "complex": true, - "copy": true, - "delete": true, - "imag": true, - "len": true, - "make": true, - "new": true, - "panic": true, - "println": true, - "print": true, - "real": true, - "recover": true, -} diff --git a/vendor/github.com/mna/pigeon/unicode_classes.go b/vendor/github.com/mna/pigeon/unicode_classes.go deleted file mode 100644 index 3600156296..0000000000 --- a/vendor/github.com/mna/pigeon/unicode_classes.go +++ /dev/null @@ -1,200 +0,0 @@ -// This file is generated by the misc/cmd/unicode-classes tool. -// Do not edit. - -package main - -var unicodeClasses = map[string]bool{ - "ASCII_Hex_Digit": true, - "Arabic": true, - "Armenian": true, - "Avestan": true, - "Balinese": true, - "Bamum": true, - "Bassa_Vah": true, - "Batak": true, - "Bengali": true, - "Bidi_Control": true, - "Bopomofo": true, - "Brahmi": true, - "Braille": true, - "Buginese": true, - "Buhid": true, - "C": true, - "Canadian_Aboriginal": true, - "Carian": true, - "Caucasian_Albanian": true, - "Cc": true, - "Cf": true, - "Chakma": true, - "Cham": true, - "Cherokee": true, - "Co": true, - "Common": true, - "Coptic": true, - "Cs": true, - "Cuneiform": true, - "Cypriot": true, - "Cyrillic": true, - "Dash": true, - "Deprecated": true, - "Deseret": true, - "Devanagari": true, - "Diacritic": true, - "Duployan": true, - "Egyptian_Hieroglyphs": true, - "Elbasan": true, - "Ethiopic": true, - "Extender": true, - "Georgian": true, - "Glagolitic": true, - "Gothic": true, - "Grantha": true, - "Greek": true, - "Gujarati": true, - "Gurmukhi": true, - "Han": true, - "Hangul": true, - "Hanunoo": true, - "Hebrew": true, - "Hex_Digit": true, - "Hiragana": true, - "Hyphen": true, - "IDS_Binary_Operator": true, - "IDS_Trinary_Operator": true, - "Ideographic": true, - "Imperial_Aramaic": true, - "Inherited": true, - "Inscriptional_Pahlavi": true, - "Inscriptional_Parthian": true, - "Javanese": true, - "Join_Control": true, - "Kaithi": true, - "Kannada": true, - "Katakana": true, - "Kayah_Li": true, - "Kharoshthi": true, - "Khmer": true, - "Khojki": true, - "Khudawadi": true, - "L": true, - "Lao": true, - "Latin": true, - "Lepcha": true, - "Limbu": true, - "Linear_A": true, - "Linear_B": true, - "Lisu": true, - "Ll": true, - "Lm": true, - "Lo": true, - "Logical_Order_Exception": true, - "Lt": true, - "Lu": true, - "Lycian": true, - "Lydian": true, - "M": true, - "Mahajani": true, - "Malayalam": true, - "Mandaic": true, - "Manichaean": true, - "Mc": true, - "Me": true, - "Meetei_Mayek": true, - "Mende_Kikakui": true, - "Meroitic_Cursive": true, - "Meroitic_Hieroglyphs": true, - "Miao": true, - "Mn": true, - "Modi": true, - "Mongolian": true, - "Mro": true, - "Myanmar": true, - "N": true, - "Nabataean": true, - "Nd": true, - "New_Tai_Lue": true, - "Nko": true, - "Nl": true, - "No": true, - "Noncharacter_Code_Point": true, - "Ogham": true, - "Ol_Chiki": true, - "Old_Italic": true, - "Old_North_Arabian": true, - "Old_Permic": true, - "Old_Persian": true, - "Old_South_Arabian": true, - "Old_Turkic": true, - "Oriya": true, - "Osmanya": true, - "Other_Alphabetic": true, - "Other_Default_Ignorable_Code_Point": true, - "Other_Grapheme_Extend": true, - "Other_ID_Continue": true, - "Other_ID_Start": true, - "Other_Lowercase": true, - "Other_Math": true, - "Other_Uppercase": true, - "P": true, - "Pahawh_Hmong": true, - "Palmyrene": true, - "Pattern_Syntax": true, - "Pattern_White_Space": true, - "Pau_Cin_Hau": true, - "Pc": true, - "Pd": true, - "Pe": true, - "Pf": true, - "Phags_Pa": true, - "Phoenician": true, - "Pi": true, - "Po": true, - "Ps": true, - "Psalter_Pahlavi": true, - "Quotation_Mark": true, - "Radical": true, - "Rejang": true, - "Runic": true, - "S": true, - "STerm": true, - "Samaritan": true, - "Saurashtra": true, - "Sc": true, - "Sharada": true, - "Shavian": true, - "Siddham": true, - "Sinhala": true, - "Sk": true, - "Sm": true, - "So": true, - "Soft_Dotted": true, - "Sora_Sompeng": true, - "Sundanese": true, - "Syloti_Nagri": true, - "Syriac": true, - "Tagalog": true, - "Tagbanwa": true, - "Tai_Le": true, - "Tai_Tham": true, - "Tai_Viet": true, - "Takri": true, - "Tamil": true, - "Telugu": true, - "Terminal_Punctuation": true, - "Thaana": true, - "Thai": true, - "Tibetan": true, - "Tifinagh": true, - "Tirhuta": true, - "Ugaritic": true, - "Unified_Ideograph": true, - "Vai": true, - "Variation_Selector": true, - "Warang_Citi": true, - "White_Space": true, - "Yi": true, - "Z": true, - "Zl": true, - "Zp": true, - "Zs": true, -} diff --git a/vendor/golang.org/x/tools/imports/forward.go b/vendor/golang.org/x/tools/imports/forward.go deleted file mode 100644 index eef25969de..0000000000 --- a/vendor/golang.org/x/tools/imports/forward.go +++ /dev/null @@ -1,62 +0,0 @@ -// Package imports implements a Go pretty-printer (like package "go/format") -// that also adds or removes import statements as necessary. -package imports // import "golang.org/x/tools/imports" - -import ( - "go/build" - - intimp "golang.org/x/tools/internal/imports" -) - -// Options specifies options for processing files. -type Options struct { - Fragment bool // Accept fragment of a source file (no package statement) - AllErrors bool // Report all errors (not just the first 10 on different lines) - - Comments bool // Print comments (true if nil *Options provided) - TabIndent bool // Use tabs for indent (true if nil *Options provided) - TabWidth int // Tab width (8 if nil *Options provided) - - FormatOnly bool // Disable the insertion and deletion of imports -} - -// Debug controls verbose logging. -var Debug = false - -// LocalPrefix is a comma-separated string of import path prefixes, which, if -// set, instructs Process to sort the import paths with the given prefixes -// into another group after 3rd-party packages. -var LocalPrefix string - -// Process formats and adjusts imports for the provided file. -// If opt is nil the defaults are used. -// -// Note that filename's directory influences which imports can be chosen, -// so it is important that filename be accurate. -// To process data ``as if'' it were in filename, pass the data as a non-nil src. -func Process(filename string, src []byte, opt *Options) ([]byte, error) { - if opt == nil { - opt = &Options{Comments: true, TabIndent: true, TabWidth: 8} - } - intopt := &intimp.Options{ - Env: &intimp.ProcessEnv{ - GOPATH: build.Default.GOPATH, - GOROOT: build.Default.GOROOT, - Debug: Debug, - LocalPrefix: LocalPrefix, - }, - AllErrors: opt.AllErrors, - Comments: opt.Comments, - FormatOnly: opt.FormatOnly, - Fragment: opt.Fragment, - TabIndent: opt.TabIndent, - TabWidth: opt.TabWidth, - } - return intimp.Process(filename, src, intopt) -} - -// VendorlessPath returns the devendorized version of the import path ipath. -// For example, VendorlessPath("foo/bar/vendor/a/b") returns "a/b". -func VendorlessPath(ipath string) string { - return intimp.VendorlessPath(ipath) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index ff3a2a28f8..e1906f946c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -25,10 +25,6 @@ github.com/konsorten/go-windows-terminal-sequences github.com/mattn/go-runewidth # github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/matttproud/golang_protobuf_extensions/pbutil -# github.com/mna/pigeon v0.0.0-20180808201053-bb0192cfc2ae -github.com/mna/pigeon -github.com/mna/pigeon/ast -github.com/mna/pigeon/builder # github.com/olekukonko/tablewriter v0.0.1 github.com/olekukonko/tablewriter # github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d @@ -73,7 +69,6 @@ golang.org/x/tools/go/gcexportdata golang.org/x/tools/go/internal/gcimporter golang.org/x/tools/go/internal/packagesdriver golang.org/x/tools/go/packages -golang.org/x/tools/imports golang.org/x/tools/internal/fastwalk golang.org/x/tools/internal/gopathwalk golang.org/x/tools/internal/imports