Implementing line comments (#3126)

* Implementing line comments Adding tests Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Appeasing the linter Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Further appeasement Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Adding tests to ensure that a # within a string is not incorrectly lexed as a comment Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Adding documentation Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
grafana · Jan 7, 2021 · fcabfec · fcabfec
1 parent ca13e79
commit fcabfec
Show file tree

Hide file tree

Showing 4 changed files with 97 additions and 0 deletions.
diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md
@@ -647,6 +647,23 @@ More details can be found in the [Golang language documentation](https://golang.
 
 `2 * 3 % 2` is evaluated as `(2 * 3) % 2`.
 
+### Comments
+
+LogQL queries can be commented using the `#` character:
+
+```logql
+{app="foo"} # anything that comes after will not be interpreted in your query
+```
+
+With multi-line LogQL queries, the query parser can exclude whole or partial lines using `#`:
+
+```logql
+{app="foo"}
+    | json
+    # this line will be ignored
+    | bar="baz" # this checks if bar = "baz"
+```
+
 ### Pipeline Errors
 
 There are multiple reasons which cause pipeline processing errors, such as:

diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go
@@ -100,7 +100,15 @@ type lexer struct {
 
 func (l *lexer) Lex(lval *exprSymType) int {
 	r := l.Scan()
+
 	switch r {
+	case '#':
+		// Scan until a newline or EOF is encountered
+		for next := l.Peek(); !(next == '\n' || next == scanner.EOF); next = l.Next() {
+		}
+
+		return l.Lex(lval)
+
 	case scanner.EOF:
 		return 0
 

diff --git a/pkg/logql/lex_test.go b/pkg/logql/lex_test.go
@@ -49,6 +49,13 @@ func TestLex(t *testing.T) {
 		{`topk(3,count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{TOPK, OPEN_PARENTHESIS, NUMBER, COMMA, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}},
 		{`bottomk(10,sum(count_over_time({foo="bar"}[5m])) by (foo,bar))`, []int{BOTTOMK, OPEN_PARENTHESIS, NUMBER, COMMA, SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS}},
 		{`sum(max(rate({foo="bar"}[5m])) by (foo,bar)) by (foo)`, []int{SUM, OPEN_PARENTHESIS, MAX, OPEN_PARENTHESIS, RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, CLOSE_PARENTHESIS}},
+		{`{foo="bar"} #|~ "\\w+"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
+		{`#{foo="bar"} |~ "\\w+"`, []int{}},
+		{`{foo="#"}`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
+		{`{foo="bar"} | json | baz="#"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON, PIPE, IDENTIFIER, EQ, STRING}},
+		{`{foo="bar"}
+					# |~ "\\w+"
+					| json`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON}},
 	} {
 		t.Run(tc.input, func(t *testing.T) {
 			actual := []int{}

diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go
@@ -2128,6 +2128,71 @@ func TestParse(t *testing.T) {
 			in:  `quantile_over_time(foo,{namespace="tns"} |= "level=error" | json |foo>=5,bar<25ms| unwrap latency [5m])`,
 			err: ParseError{msg: "syntax error: unexpected IDENTIFIER, expecting NUMBER or { or (", line: 1, col: 20},
 		},
+		{
+			in: `{app="foo"}
+					# |= "bar"
+					| json`,
+			exp: &pipelineExpr{
+				left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}),
+				pipeline: MultiStageExpr{
+					newLabelParserExpr(OpParserTypeJSON, ""),
+				},
+			},
+		},
+		{
+			in: `{app="foo"}
+					#
+					|= "bar"
+					| json`,
+			exp: &pipelineExpr{
+				left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}),
+				pipeline: MultiStageExpr{
+					newLineFilterExpr(nil, labels.MatchEqual, "bar"),
+					newLabelParserExpr(OpParserTypeJSON, ""),
+				},
+			},
+		},
+		{
+			in:  `{app="foo"} # |= "bar" | json`,
+			exp: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}),
+		},
+		{
+			in: `{app="foo"} | json #`,
+			exp: &pipelineExpr{
+				left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}),
+				pipeline: MultiStageExpr{
+					newLabelParserExpr(OpParserTypeJSON, ""),
+				},
+			},
+		},
+		{
+			in:  `#{app="foo"} | json`,
+			err: ParseError{msg: "syntax error: unexpected $end", line: 1, col: 20},
+		},
+		{
+			in:  `{app="#"}`,
+			exp: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "#"}}),
+		},
+		{
+			in: `{app="foo"} |= "#"`,
+			exp: &pipelineExpr{
+				left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}),
+				pipeline: MultiStageExpr{
+					newLineFilterExpr(nil, labels.MatchEqual, "#"),
+				},
+			},
+		},
+		{
+			in: `{app="foo"} | bar="#"`,
+			exp: &pipelineExpr{
+				left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}),
+				pipeline: MultiStageExpr{
+					&labelFilterExpr{
+						LabelFilterer: log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "bar", "#")),
+					},
+				},
+			},
+		},
 	} {
 		t.Run(tc.in, func(t *testing.T) {
 			ast, err := ParseExpr(tc.in)