diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index 1e5f022943e8..5feceb1ecc0d 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -145,40 +145,100 @@ If an extracted label key name already exists in the original log stream, the ex We support currently support json, logfmt and regexp parsers. -The **json** parsers take no parameters and can be added using the expression `| json` in your pipeline. It will extract all json properties as labels if the log line is a valid json document. Nested properties are flattened into label keys using the `_` separator. **Arrays are skipped**. - -For example the json parsers will extract from the following document: - -```json -{ - "protocol": "HTTP/2.0", - "servers": ["129.0.1.1","10.2.1.3"], - "request": { - "time": "6.032", - "method": "GET", - "host": "foo.grafana.net", - "size": "55", - }, - "response": { - "status": 401, - "size": "228", - "latency_seconds": "6.031" - } -} -``` - -The following list of labels: - -```kv -"protocol" => "HTTP/2.0" -"request_time" => "6.032" -"request_method" => "GET" -"request_host" => "foo.grafana.net" -"request_size" => "55" -"response_status" => "401" -"response_size" => "228" -"response_size" => "228" -``` +The **json** parser operates in two modes: + +1. **without** parameters: + + Adding `| json` to your pipeline will extract all json properties as labels if the log line is a valid json document. + Nested properties are flattened into label keys using the `_` separator. + + Note: **Arrays are skipped**. + + For example the json parsers will extract from the following document: + + ```json + { + "protocol": "HTTP/2.0", + "servers": ["129.0.1.1","10.2.1.3"], + "request": { + "time": "6.032", + "method": "GET", + "host": "foo.grafana.net", + "size": "55", + "headers": { + "Accept": "*/*", + "User-Agent": "curl/7.68.0" + } + }, + "response": { + "status": 401, + "size": "228", + "latency_seconds": "6.031" + } + } + ``` + + The following list of labels: + + ```kv + "protocol" => "HTTP/2.0" + "request_time" => "6.032" + "request_method" => "GET" + "request_host" => "foo.grafana.net" + "request_size" => "55" + "response_status" => "401" + "response_size" => "228" + "response_size" => "228" + ``` + +2. **with** parameters: + + Using `| json label="expression", another="expression"` in your pipeline will extract only the + specified json fields to labels. You can specify one or more expressions in this way, the same + as [`label_format`](#labels-format-expression); all expressions must be quoted. + + Currently, we only support field access (`my.field`, `my["field"]`) and array access (`list[0]`), and any combination + of these in any level of nesting (`my.list[0]["field"]`). + + For example, `| json first_server="servers[0]", ua="request.headers[\"User-Agent\"]` will extract from the following document: + + ```json + { + "protocol": "HTTP/2.0", + "servers": ["129.0.1.1","10.2.1.3"], + "request": { + "time": "6.032", + "method": "GET", + "host": "foo.grafana.net", + "size": "55", + "headers": { + "Accept": "*/*", + "User-Agent": "curl/7.68.0" + } + }, + "response": { + "status": 401, + "size": "228", + "latency_seconds": "6.031" + } + } + ``` + + The following list of labels: + + ```kv + "first_server" => "129.0.1.1" + "ua" => "curl/7.68.0" + ``` + + If an array or an object returned by an expression, it will be assigned to the label in json format. + + For example, `| json server_list="servers", headers="request.headers` will extract: + + ```kv + "server_list" => `["129.0.1.1","10.2.1.3"]` + "headers" => `{"Accept": "*/*", "User-Agent": "curl/7.68.0"}` + ``` The **logfmt** parser can be added using the `| logfmt` and will extract all keys and values from the [logfmt](https://brandur.org/logfmt) formatted log line. diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 09c86682a77f..2bd46b4f2848 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -416,6 +416,39 @@ func (e *labelFmtExpr) String() string { return sb.String() } +type jsonExpressionParser struct { + expressions []log.JSONExpression + + implicit +} + +func newJSONExpressionParser(expressions []log.JSONExpression) *jsonExpressionParser { + return &jsonExpressionParser{ + expressions: expressions, + } +} + +func (j *jsonExpressionParser) Shardable() bool { return true } + +func (j *jsonExpressionParser) Stage() (log.Stage, error) { + return log.NewJSONExpressionParser(j.expressions) +} + +func (j *jsonExpressionParser) String() string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("%s %s ", OpPipe, OpParserTypeJSON)) + for i, exp := range j.expressions { + sb.WriteString(exp.Identifier) + sb.WriteString("=") + sb.WriteString(strconv.Quote(exp.Expression)) + + if i+1 != len(j.expressions) { + sb.WriteString(",") + } + } + return sb.String() +} + func mustNewMatcher(t labels.MatchType, n, v string) *labels.Matcher { m, err := labels.NewMatcher(t, n, v) if err != nil { diff --git a/pkg/logql/ast_test.go b/pkg/logql/ast_test.go index a1d8c61f820f..cb2b1795164a 100644 --- a/pkg/logql/ast_test.go +++ b/pkg/logql/ast_test.go @@ -127,6 +127,7 @@ func Test_SampleExpr_String(t *testing.T) { `, `10 / (5/2)`, `10 / (count_over_time({job="postgres"}[5m])/2)`, + `{app="foo"} | json response_status="response.status.code", first_param="request.params[0]"`, } { t.Run(tc, func(t *testing.T) { expr, err := ParseExpr(tc) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index c19f8ae80cdb..ec49ec95e467 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -46,6 +46,9 @@ import ( LabelFormatExpr *labelFmtExpr LabelFormat log.LabelFmt LabelsFormat []log.LabelFmt + JSONExpressionParser *jsonExpressionParser + JSONExpression log.JSONExpression + JSONExpressionList []log.JSONExpression UnwrapExpr *unwrapExpr } @@ -82,6 +85,9 @@ import ( %type labelFormatExpr %type labelFormat %type labelsFormat +%type jsonExpressionParser +%type jsonExpression +%type jsonExpressionList %type unwrapExpr %type unitFilter @@ -211,6 +217,7 @@ pipelineExpr: pipelineStage: lineFilters { $$ = $1 } | PIPE labelParser { $$ = $2 } + | PIPE jsonExpressionParser { $$ = $2 } | PIPE labelFilter { $$ = &labelFilterExpr{LabelFilterer: $2 }} | PIPE lineFormatExpr { $$ = $2 } | PIPE labelFormatExpr { $$ = $2 } @@ -226,6 +233,9 @@ labelParser: | REGEXP STRING { $$ = newLabelParserExpr(OpParserTypeRegexp, $2) } ; +jsonExpressionParser: + JSON jsonExpressionList { $$ = newJSONExpressionParser($2) } + lineFormatExpr: LINE_FMT STRING { $$ = newLineFmtExpr($2) }; labelFormat: @@ -252,6 +262,14 @@ labelFilter: | labelFilter OR labelFilter { $$ = log.NewOrLabelFilter($1, $3 ) } ; +jsonExpression: + IDENTIFIER EQ STRING { $$ = log.NewJSONExpr($1, $3) } + +jsonExpressionList: + jsonExpression { $$ = []log.JSONExpression{$1} } + | jsonExpressionList COMMA jsonExpression { $$ = append($1, $3) } + ; + unitFilter: durationFilter { $$ = $1 } | bytesFilter { $$ = $1 } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index e9b7a0b45207..60835839db16 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -6,6 +6,7 @@ package logql import __yyfmt__ "fmt" //line pkg/logql/expr.y:2 + import ( "github.com/grafana/loki/pkg/logql/log" "github.com/prometheus/prometheus/pkg/labels" @@ -51,6 +52,9 @@ type exprSymType struct { LabelFormatExpr *labelFmtExpr LabelFormat log.LabelFmt LabelsFormat []log.LabelFmt + JSONExpressionParser *jsonExpressionParser + JSONExpression log.JSONExpression + JSONExpressionList []log.JSONExpression UnwrapExpr *unwrapExpr } @@ -202,13 +206,14 @@ var exprToknames = [...]string{ "MOD", "POW", } + var exprStatenames = [...]string{} const exprEofCode = 1 const exprErrCode = 2 const exprInitialStackSize = 16 -//line pkg/logql/expr.y:359 +//line pkg/logql/expr.y:377 //line yacctab:1 var exprExca = [...]int{ @@ -219,134 +224,135 @@ var exprExca = [...]int{ const exprPrivate = 57344 -const exprLast = 446 +const exprLast = 454 var exprAct = [...]int{ - - 73, 176, 4, 158, 55, 56, 104, 185, 150, 64, - 48, 5, 226, 124, 66, 2, 223, 263, 80, 69, - 43, 44, 45, 46, 47, 48, 62, 15, 45, 46, - 47, 48, 62, 60, 61, 12, 120, 122, 123, 60, - 61, 235, 72, 6, 74, 75, 237, 19, 20, 31, + 73, 180, 4, 162, 156, 56, 189, 105, 151, 64, + 55, 66, 2, 5, 125, 45, 46, 47, 48, 48, + 69, 43, 44, 45, 46, 47, 48, 15, 230, 232, + 62, 229, 271, 62, 59, 12, 255, 60, 61, 80, + 60, 61, 254, 6, 283, 285, 281, 19, 20, 31, 32, 34, 35, 33, 36, 37, 38, 39, 21, 22, - 59, 95, 178, 74, 75, 172, 277, 98, 23, 24, - 25, 26, 27, 28, 29, 128, 63, 247, 30, 18, - 224, 133, 63, 126, 246, 62, 222, 243, 121, 16, - 17, 273, 60, 61, 134, 254, 135, 136, 137, 138, - 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, - 172, 187, 160, 122, 123, 178, 155, 187, 96, 223, - 110, 223, 246, 268, 261, 167, 256, 249, 250, 251, - 189, 177, 230, 183, 152, 63, 186, 266, 107, 179, - 188, 180, 40, 41, 42, 49, 50, 53, 54, 51, - 52, 43, 44, 45, 46, 47, 48, 223, 222, 233, - 191, 192, 193, 166, 161, 164, 165, 162, 163, 172, - 181, 115, 114, 253, 218, 153, 151, 220, 110, 225, - 221, 228, 231, 95, 229, 232, 126, 98, 219, 235, - 62, 173, 132, 223, 236, 131, 107, 60, 61, 239, - 41, 42, 49, 50, 53, 54, 51, 52, 43, 44, - 45, 46, 47, 48, 101, 103, 102, 224, 108, 109, - 58, 175, 62, 244, 130, 78, 62, 95, 110, 60, - 61, 255, 245, 60, 61, 95, 227, 257, 175, 184, - 63, 12, 152, 62, 260, 71, 107, 12, 196, 127, - 60, 61, 178, 275, 262, 6, 178, 267, 272, 19, - 20, 31, 32, 34, 35, 33, 36, 37, 38, 39, - 21, 22, 63, 178, 110, 259, 63, 234, 195, 190, - 23, 24, 25, 26, 27, 28, 29, 129, 125, 182, - 30, 18, 107, 63, 174, 12, 12, 270, 119, 271, - 265, 16, 17, 6, 127, 264, 110, 19, 20, 31, + 182, 95, 176, 182, 121, 123, 124, 99, 23, 24, + 25, 26, 27, 28, 29, 129, 228, 229, 30, 18, + 63, 134, 276, 63, 251, 127, 257, 258, 259, 16, + 17, 135, 96, 136, 137, 138, 139, 140, 141, 142, + 143, 144, 145, 146, 147, 148, 149, 74, 75, 176, + 72, 229, 74, 75, 111, 241, 122, 159, 241, 264, + 243, 239, 191, 242, 230, 12, 185, 171, 153, 62, + 116, 236, 108, 128, 187, 115, 60, 61, 181, 262, + 192, 193, 183, 184, 40, 41, 42, 49, 50, 53, + 54, 51, 52, 43, 44, 45, 46, 47, 48, 182, + 269, 195, 196, 197, 41, 42, 49, 50, 53, 54, + 51, 52, 43, 44, 45, 46, 47, 48, 224, 63, + 191, 226, 111, 231, 111, 234, 237, 95, 133, 238, + 227, 99, 127, 225, 235, 132, 131, 78, 153, 190, + 108, 71, 108, 245, 176, 247, 49, 50, 53, 54, + 51, 52, 43, 44, 45, 46, 47, 48, 102, 104, + 103, 254, 109, 110, 232, 280, 177, 267, 240, 252, + 164, 123, 124, 95, 201, 198, 274, 263, 253, 154, + 152, 95, 194, 265, 186, 179, 62, 188, 111, 178, + 62, 120, 268, 60, 61, 12, 229, 60, 61, 202, + 233, 199, 270, 6, 279, 275, 108, 19, 20, 31, 32, 34, 35, 33, 36, 37, 38, 39, 21, 22, - 252, 79, 110, 201, 107, 169, 202, 200, 23, 24, - 25, 26, 27, 28, 29, 77, 152, 76, 30, 18, - 107, 194, 101, 103, 102, 276, 108, 109, 226, 16, - 17, 49, 50, 53, 54, 51, 52, 43, 44, 45, - 46, 47, 48, 81, 82, 83, 84, 85, 86, 87, - 88, 89, 90, 91, 92, 93, 94, 153, 151, 110, - 198, 117, 168, 199, 197, 216, 274, 213, 217, 215, - 214, 212, 269, 152, 238, 116, 210, 107, 118, 211, - 209, 207, 171, 204, 208, 206, 205, 203, 241, 242, - 3, 240, 170, 169, 159, 105, 168, 65, 156, 154, - 149, 113, 68, 258, 70, 70, 159, 157, 100, 99, - 57, 111, 106, 112, 97, 151, 11, 10, 9, 14, - 8, 248, 13, 7, 67, 1, + 182, 170, 165, 168, 169, 166, 167, 118, 23, 24, + 25, 26, 27, 28, 29, 130, 63, 126, 30, 18, + 63, 117, 273, 12, 119, 12, 272, 260, 77, 16, + 17, 6, 76, 128, 111, 19, 20, 31, 32, 34, + 35, 33, 36, 37, 38, 39, 21, 22, 153, 249, + 250, 111, 108, 200, 284, 282, 23, 24, 25, 26, + 27, 28, 29, 179, 62, 153, 30, 18, 62, 108, + 228, 60, 61, 277, 3, 60, 61, 16, 17, 79, + 207, 65, 173, 208, 206, 261, 246, 244, 222, 154, + 152, 223, 221, 204, 58, 172, 205, 203, 182, 111, + 219, 175, 278, 220, 218, 229, 216, 152, 213, 217, + 215, 214, 212, 174, 63, 173, 210, 108, 63, 211, + 209, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 102, 104, 103, 248, 109, + 110, 163, 106, 172, 160, 158, 150, 114, 68, 266, + 157, 70, 70, 163, 155, 98, 161, 101, 100, 57, + 112, 107, 113, 97, 11, 10, 9, 14, 8, 256, + 13, 7, 67, 1, } -var exprPact = [...]int{ - 20, -1000, 82, -1000, -1000, 176, 20, -1000, -1000, -1000, - -1000, -1000, 420, 222, 19, -1000, 330, 328, 202, -1000, +var exprPact = [...]int{ + 20, -1000, 84, -1000, -1000, 330, 20, -1000, -1000, -1000, + -1000, -1000, 426, 178, 87, -1000, 305, 301, 174, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -22, -22, -22, -22, -22, -22, -22, -22, -22, -22, - -22, -22, -22, -22, -22, 176, -1000, 12, 173, 415, - -1000, -1000, -1000, -1000, 148, 147, 82, 379, 282, -1000, - 24, 281, 280, 201, 172, 169, -1000, -1000, 20, 20, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 330, -1000, 232, 374, 421, + -1000, -1000, -1000, -1000, 111, 106, 84, 285, 235, -1000, + 52, 290, 288, 173, 172, 165, -1000, -1000, 20, 20, -1000, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, -1000, 414, -1000, 115, -1000, - -1000, -1000, -1000, 413, -1000, -1000, -1000, 269, 412, 421, - 100, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 419, -1000, - 410, 407, 406, 396, 167, 275, 229, 226, 146, 270, - 232, 112, 106, 260, 139, 288, 288, -43, -43, -64, - -64, -64, -64, -49, -49, -49, -49, -49, -49, -1000, - 115, 269, 269, 269, -1000, 317, -1000, 259, -1000, 236, - 376, 319, 399, 397, 392, 383, 381, -1000, -1000, -1000, - -1000, -1000, -1000, 38, 226, 18, 77, 208, 301, 212, - 108, 38, 20, 135, 258, 170, -1000, -1000, 22, -1000, - 388, 223, 115, 374, -1000, 409, 403, -1000, -1000, -1000, + 20, 20, 20, 20, 20, -1000, 420, -1000, -1000, 179, + -1000, -1000, 425, -1000, 419, -1000, -1000, -1000, 243, 418, + 428, 218, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 427, + -1000, 417, 389, 387, 375, 202, 230, 334, 110, 102, + 225, 240, 175, 117, 223, 103, 143, 143, -56, -56, + -55, -55, -55, -55, -48, -48, -48, -48, -48, -48, + -1000, 179, 243, 243, 243, 216, -1000, 249, -1000, 309, + -1000, 215, -1000, 247, 369, 356, 392, 384, 382, 376, + 364, -1000, -1000, -1000, -1000, -1000, -1000, 82, 110, 16, + 67, 19, 177, 236, 107, 82, 20, 97, 209, 99, + -1000, -1000, 96, -1000, 361, 109, 179, 326, 425, 360, + -1000, 416, 324, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 63, - -28, 18, -1000, 269, -1000, 75, 72, 311, 149, 71, - -1000, -1000, 102, -1000, 20, 418, -1000, -1000, 256, -1000, - -1000, -1000, -1000, 38, -28, 115, -1000, -1000, 101, -1000, - -1000, -1000, -27, 296, 291, 113, 38, 99, -1000, 386, - -1000, 292, -28, -35, -1000, -1000, 290, -1000, -1000, 239, - 67, -1000, 380, -1000, 234, 339, 42, -1000, + -1000, -1000, -1000, -1000, -1000, 60, -13, 16, -1000, 243, + -1000, 33, 31, 298, 341, 115, -1000, -1000, 95, -1000, + 20, 424, -1000, -1000, 208, -1000, -1000, -1000, -1000, -1000, + -1000, 82, -13, 179, -1000, -1000, 137, -1000, -1000, -1000, + -12, 297, 293, 212, 82, 58, -1000, 347, -1000, 377, + -13, -18, -1000, -1000, 255, -1000, -1000, 206, 22, -1000, + 329, -1000, 25, 328, 21, -1000, } -var exprPgo = [...]int{ - 0, 445, 14, 60, 0, 7, 410, 2, 13, 6, - 444, 443, 442, 441, 11, 440, 439, 438, 437, 436, - 321, 434, 4, 5, 433, 432, 431, 8, 430, 429, - 428, 3, 427, 1, 415, +var exprPgo = [...]int{ + 0, 453, 11, 34, 0, 6, 354, 2, 14, 7, + 452, 451, 450, 449, 13, 448, 447, 446, 445, 444, + 359, 443, 10, 5, 442, 441, 440, 8, 439, 438, + 437, 3, 436, 435, 4, 434, 1, 422, } -var exprR1 = [...]int{ +var exprR1 = [...]int{ 0, 1, 2, 2, 7, 7, 7, 7, 7, 7, 6, 6, 6, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 33, 33, 33, + 8, 8, 8, 8, 8, 8, 8, 36, 36, 36, 13, 13, 13, 11, 11, 11, 11, 15, 15, 15, 15, 15, 15, 19, 3, 3, 3, 3, 14, 14, 14, 10, 10, 9, 9, 9, 9, 22, 22, 23, - 23, 23, 23, 23, 28, 28, 21, 21, 21, 29, - 31, 31, 32, 32, 32, 30, 27, 27, 27, 27, - 27, 27, 27, 27, 34, 34, 26, 26, 26, 26, - 26, 26, 26, 24, 24, 24, 24, 24, 24, 24, - 25, 25, 25, 25, 25, 25, 25, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 20, 20, 18, 18, 18, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 12, 12, 12, 12, - 12, 12, 12, 12, 12, 12, 12, 12, 5, 5, - 4, 4, 4, 4, + 23, 23, 23, 23, 23, 28, 28, 21, 21, 21, + 33, 29, 31, 31, 32, 32, 32, 30, 27, 27, + 27, 27, 27, 27, 27, 27, 34, 35, 35, 37, + 37, 26, 26, 26, 26, 26, 26, 26, 24, 24, + 24, 24, 24, 24, 24, 25, 25, 25, 25, 25, + 25, 25, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 20, 20, 18, + 18, 18, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 12, 12, 5, 5, 4, 4, 4, 4, } -var exprR2 = [...]int{ +var exprR2 = [...]int{ 0, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 2, 3, 2, 4, 3, 5, 3, 5, 3, 5, 4, 6, 3, 4, 3, 2, 3, 6, 3, 1, 1, 1, 4, 6, 5, 7, 4, 5, 5, 6, 7, 7, 12, 1, 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, 3, 1, 2, 1, - 2, 2, 2, 2, 2, 3, 1, 1, 2, 2, - 3, 3, 1, 3, 3, 2, 1, 1, 1, 3, - 2, 3, 3, 3, 1, 1, 3, 3, 3, 3, + 2, 2, 2, 2, 2, 2, 3, 1, 1, 2, + 2, 2, 3, 3, 1, 3, 3, 2, 1, 1, + 1, 3, 2, 3, 3, 3, 3, 1, 3, 1, + 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 0, 1, 1, 2, 2, 1, 1, 1, + 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 0, 1, 1, + 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, - 4, 4, 3, 3, + 1, 1, 1, 1, 3, 4, 4, 3, 3, } -var exprChk = [...]int{ +var exprChk = [...]int{ -1000, -1, -2, -6, -7, -14, 23, -11, -15, -17, -18, -19, 15, -12, -16, 7, 69, 70, 59, 27, 28, 38, 39, 48, 49, 50, 51, 52, 53, 54, @@ -356,63 +362,65 @@ var exprChk = [...]int{ 21, 22, 14, 64, -7, -6, -2, -10, 2, -9, 5, 23, 23, -4, 25, 26, 7, 7, 23, -20, 40, -20, -20, -20, -20, -20, -20, -20, -20, -20, - -20, -20, -20, -20, -20, -23, -3, -21, -27, -29, - -30, 41, 43, 42, -9, -34, -25, 23, 45, 46, - 5, -26, -24, 6, 24, 24, 16, 2, 19, 16, - 12, 64, 13, 14, -8, 7, -14, 23, -7, 7, - 23, 23, 23, -7, -2, -2, -2, -2, -2, -2, - -2, -2, -2, -2, -2, -2, -2, -2, -2, 6, - -27, 61, 19, 60, 6, -27, 6, -32, -31, 5, - 12, 64, 67, 68, 65, 66, 63, -9, 6, 6, - 6, 6, 2, 24, 19, 9, -33, -22, 44, -14, - -8, 24, 19, -7, 7, -5, 24, 5, -5, 24, - 19, -27, -27, -27, 24, 19, 12, 8, 4, 7, - 8, 4, 7, 8, 4, 7, 8, 4, 7, 8, - 4, 7, 8, 4, 7, 8, 4, 7, -4, -8, - -33, -22, 9, 44, 9, -33, 47, 24, -33, -22, - 24, -4, -7, 24, 19, 19, 24, 24, 6, -31, - 2, 5, 6, 24, -33, -27, 9, 5, -13, 55, - 56, 57, 9, 24, 24, -33, 24, -7, 5, 19, - -4, 23, -33, 44, 9, 9, 24, -4, 24, 6, - 5, 9, 19, 24, 6, 19, 6, 24, + -20, -20, -20, -20, -20, -23, -3, -21, -33, -27, + -29, -30, 41, 43, 42, -9, -37, -25, 23, 45, + 46, 5, -26, -24, 6, 24, 24, 16, 2, 19, + 16, 12, 64, 13, 14, -8, 7, -14, 23, -7, + 7, 23, 23, 23, -7, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + 6, -27, 61, 19, 60, -35, -34, 5, 6, -27, + 6, -32, -31, 5, 12, 64, 67, 68, 65, 66, + 63, -9, 6, 6, 6, 6, 2, 24, 19, 9, + -36, -22, 44, -14, -8, 24, 19, -7, 7, -5, + 24, 5, -5, 24, 19, -27, -27, -27, 19, 12, + 24, 19, 12, 8, 4, 7, 8, 4, 7, 8, + 4, 7, 8, 4, 7, 8, 4, 7, 8, 4, + 7, 8, 4, 7, -4, -8, -36, -22, 9, 44, + 9, -36, 47, 24, -36, -22, 24, -4, -7, 24, + 19, 19, 24, 24, 6, -34, 6, -31, 2, 5, + 6, 24, -36, -27, 9, 5, -13, 55, 56, 57, + 9, 24, 24, -36, 24, -7, 5, 19, -4, 23, + -36, 44, 9, 9, 24, -4, 24, 6, 5, 9, + 19, 24, 6, 19, 6, 24, } -var exprDef = [...]int{ +var exprDef = [...]int{ 0, -2, 1, 2, 3, 10, 0, 4, 5, 6, - 7, 8, 0, 0, 0, 124, 0, 0, 0, 136, - 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, - 147, 127, 128, 129, 130, 131, 132, 133, 134, 135, - 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, - 122, 122, 122, 122, 122, 11, 57, 59, 0, 0, + 7, 8, 0, 0, 0, 129, 0, 0, 0, 141, + 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, + 152, 132, 133, 134, 135, 136, 137, 138, 139, 140, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 11, 57, 59, 0, 0, 44, 45, 46, 47, 3, 2, 0, 0, 0, 51, - 0, 0, 0, 0, 0, 0, 125, 126, 0, 0, - 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 130, 131, 0, 0, + 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 60, 61, 62, - 63, 66, 67, 0, 76, 77, 78, 0, 0, 0, - 0, 84, 85, 64, 9, 12, 48, 49, 0, 50, - 0, 0, 0, 0, 0, 0, 0, 0, 3, 124, - 0, 0, 0, 3, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, 117, 118, 119, 120, 121, 65, - 80, 0, 0, 0, 68, 0, 69, 75, 72, 0, - 0, 0, 0, 0, 0, 0, 0, 52, 53, 54, - 55, 56, 26, 33, 0, 13, 0, 0, 0, 0, - 0, 37, 0, 3, 124, 0, 152, 148, 0, 153, - 0, 81, 82, 83, 79, 0, 0, 91, 98, 105, - 90, 97, 104, 86, 93, 100, 87, 94, 101, 88, - 95, 102, 89, 96, 103, 92, 99, 106, 35, 0, - 15, 23, 17, 0, 19, 0, 0, 0, 0, 0, - 25, 39, 3, 38, 0, 0, 150, 151, 0, 73, - 74, 70, 71, 34, 24, 29, 21, 27, 0, 30, - 31, 32, 14, 0, 0, 0, 40, 3, 149, 0, - 36, 0, 16, 0, 18, 20, 0, 41, 42, 0, - 0, 22, 0, 28, 0, 0, 0, 43, + 63, 64, 67, 68, 0, 78, 79, 80, 0, 0, + 0, 0, 89, 90, 65, 9, 12, 48, 49, 0, + 50, 0, 0, 0, 0, 0, 0, 0, 0, 3, + 129, 0, 0, 0, 3, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, + 66, 82, 0, 0, 0, 70, 87, 0, 69, 0, + 71, 77, 74, 0, 0, 0, 0, 0, 0, 0, + 0, 52, 53, 54, 55, 56, 26, 33, 0, 13, + 0, 0, 0, 0, 0, 37, 0, 3, 129, 0, + 157, 153, 0, 158, 0, 83, 84, 85, 0, 0, + 81, 0, 0, 96, 103, 110, 95, 102, 109, 91, + 98, 105, 92, 99, 106, 93, 100, 107, 94, 101, + 108, 97, 104, 111, 35, 0, 15, 23, 17, 0, + 19, 0, 0, 0, 0, 0, 25, 39, 3, 38, + 0, 0, 155, 156, 0, 88, 86, 75, 76, 72, + 73, 34, 24, 29, 21, 27, 0, 30, 31, 32, + 14, 0, 0, 0, 40, 3, 154, 0, 36, 0, + 16, 0, 18, 20, 0, 41, 42, 0, 0, 22, + 0, 28, 0, 0, 0, 43, } -var exprTok1 = [...]int{ +var exprTok1 = [...]int{ 1, } -var exprTok2 = [...]int{ +var exprTok2 = [...]int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, @@ -422,6 +430,7 @@ var exprTok2 = [...]int{ 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, } + var exprTok3 = [...]int{ 0, } @@ -765,906 +774,936 @@ exprdefault: case 1: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:107 +//line pkg/logql/expr.y:113 { exprlex.(*parser).expr = exprDollar[1].Expr } case 2: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:110 +//line pkg/logql/expr.y:116 { exprVAL.Expr = exprDollar[1].LogExpr } case 3: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:111 +//line pkg/logql/expr.y:117 { exprVAL.Expr = exprDollar[1].MetricExpr } case 4: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:115 +//line pkg/logql/expr.y:121 { exprVAL.MetricExpr = exprDollar[1].RangeAggregationExpr } case 5: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:116 +//line pkg/logql/expr.y:122 { exprVAL.MetricExpr = exprDollar[1].VectorAggregationExpr } case 6: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:117 +//line pkg/logql/expr.y:123 { exprVAL.MetricExpr = exprDollar[1].BinOpExpr } case 7: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:118 +//line pkg/logql/expr.y:124 { exprVAL.MetricExpr = exprDollar[1].LiteralExpr } case 8: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:119 +//line pkg/logql/expr.y:125 { exprVAL.MetricExpr = exprDollar[1].LabelReplaceExpr } case 9: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:120 +//line pkg/logql/expr.y:126 { exprVAL.MetricExpr = exprDollar[2].MetricExpr } case 10: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:124 +//line pkg/logql/expr.y:130 { exprVAL.LogExpr = newMatcherExpr(exprDollar[1].Selector) } case 11: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:125 +//line pkg/logql/expr.y:131 { exprVAL.LogExpr = newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].PipelineExpr) } case 12: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:126 +//line pkg/logql/expr.y:132 { exprVAL.LogExpr = exprDollar[2].LogExpr } case 13: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:130 +//line pkg/logql/expr.y:136 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].duration, nil) } case 14: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:131 +//line pkg/logql/expr.y:137 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[2].Selector), exprDollar[4].duration, nil) } case 15: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:132 +//line pkg/logql/expr.y:138 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].duration, exprDollar[3].UnwrapExpr) } case 16: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:133 +//line pkg/logql/expr.y:139 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[2].Selector), exprDollar[4].duration, exprDollar[5].UnwrapExpr) } case 17: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:134 +//line pkg/logql/expr.y:140 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[1].Selector), exprDollar[3].duration, exprDollar[2].UnwrapExpr) } case 18: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:135 +//line pkg/logql/expr.y:141 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[2].Selector), exprDollar[5].duration, exprDollar[3].UnwrapExpr) } case 19: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:136 +//line pkg/logql/expr.y:142 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].PipelineExpr), exprDollar[3].duration, nil) } case 20: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:137 +//line pkg/logql/expr.y:143 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[2].Selector), exprDollar[3].PipelineExpr), exprDollar[5].duration, nil) } case 21: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:138 +//line pkg/logql/expr.y:144 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].PipelineExpr), exprDollar[4].duration, exprDollar[3].UnwrapExpr) } case 22: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:139 +//line pkg/logql/expr.y:145 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[2].Selector), exprDollar[3].PipelineExpr), exprDollar[6].duration, exprDollar[4].UnwrapExpr) } case 23: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:140 +//line pkg/logql/expr.y:146 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[3].PipelineExpr), exprDollar[2].duration, nil) } case 24: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:141 +//line pkg/logql/expr.y:147 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[3].PipelineExpr), exprDollar[2].duration, exprDollar[4].UnwrapExpr) } case 25: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:142 +//line pkg/logql/expr.y:148 { exprVAL.LogRangeExpr = exprDollar[2].LogRangeExpr } case 27: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:147 +//line pkg/logql/expr.y:153 { exprVAL.UnwrapExpr = newUnwrapExpr(exprDollar[3].str, "") } case 28: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:148 +//line pkg/logql/expr.y:154 { exprVAL.UnwrapExpr = newUnwrapExpr(exprDollar[5].str, exprDollar[3].ConvOp) } case 29: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:149 +//line pkg/logql/expr.y:155 { exprVAL.UnwrapExpr = exprDollar[1].UnwrapExpr.addPostFilter(exprDollar[3].LabelFilter) } case 30: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:153 +//line pkg/logql/expr.y:159 { exprVAL.ConvOp = OpConvBytes } case 31: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:154 +//line pkg/logql/expr.y:160 { exprVAL.ConvOp = OpConvDuration } case 32: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:155 +//line pkg/logql/expr.y:161 { exprVAL.ConvOp = OpConvDurationSeconds } case 33: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:159 +//line pkg/logql/expr.y:165 { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[3].LogRangeExpr, exprDollar[1].RangeOp, nil, nil) } case 34: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:160 +//line pkg/logql/expr.y:166 { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[5].LogRangeExpr, exprDollar[1].RangeOp, nil, &exprDollar[3].str) } case 35: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:161 +//line pkg/logql/expr.y:167 { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[3].LogRangeExpr, exprDollar[1].RangeOp, exprDollar[5].Grouping, nil) } case 36: exprDollar = exprS[exprpt-7 : exprpt+1] -//line pkg/logql/expr.y:162 +//line pkg/logql/expr.y:168 { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[5].LogRangeExpr, exprDollar[1].RangeOp, exprDollar[7].Grouping, &exprDollar[3].str) } case 37: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:167 +//line pkg/logql/expr.y:173 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].MetricExpr, exprDollar[1].VectorOp, nil, nil) } case 38: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:168 +//line pkg/logql/expr.y:174 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].MetricExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } case 39: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:169 +//line pkg/logql/expr.y:175 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].MetricExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } case 40: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:171 +//line pkg/logql/expr.y:177 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].MetricExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } case 41: exprDollar = exprS[exprpt-7 : exprpt+1] -//line pkg/logql/expr.y:172 +//line pkg/logql/expr.y:178 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].MetricExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } case 42: exprDollar = exprS[exprpt-7 : exprpt+1] -//line pkg/logql/expr.y:173 +//line pkg/logql/expr.y:179 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[6].MetricExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, &exprDollar[4].str) } case 43: exprDollar = exprS[exprpt-12 : exprpt+1] -//line pkg/logql/expr.y:178 +//line pkg/logql/expr.y:184 { exprVAL.LabelReplaceExpr = mustNewLabelReplaceExpr(exprDollar[3].MetricExpr, exprDollar[5].str, exprDollar[7].str, exprDollar[9].str, exprDollar[11].str) } case 44: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:182 +//line pkg/logql/expr.y:188 { exprVAL.Filter = labels.MatchRegexp } case 45: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:183 +//line pkg/logql/expr.y:189 { exprVAL.Filter = labels.MatchEqual } case 46: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:184 +//line pkg/logql/expr.y:190 { exprVAL.Filter = labels.MatchNotRegexp } case 47: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:185 +//line pkg/logql/expr.y:191 { exprVAL.Filter = labels.MatchNotEqual } case 48: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:189 +//line pkg/logql/expr.y:195 { exprVAL.Selector = exprDollar[2].Matchers } case 49: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:190 +//line pkg/logql/expr.y:196 { exprVAL.Selector = exprDollar[2].Matchers } case 50: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:191 +//line pkg/logql/expr.y:197 { } case 51: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:195 +//line pkg/logql/expr.y:201 { exprVAL.Matchers = []*labels.Matcher{exprDollar[1].Matcher} } case 52: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:196 +//line pkg/logql/expr.y:202 { exprVAL.Matchers = append(exprDollar[1].Matchers, exprDollar[3].Matcher) } case 53: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:200 +//line pkg/logql/expr.y:206 { exprVAL.Matcher = mustNewMatcher(labels.MatchEqual, exprDollar[1].str, exprDollar[3].str) } case 54: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:201 +//line pkg/logql/expr.y:207 { exprVAL.Matcher = mustNewMatcher(labels.MatchNotEqual, exprDollar[1].str, exprDollar[3].str) } case 55: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:202 +//line pkg/logql/expr.y:208 { exprVAL.Matcher = mustNewMatcher(labels.MatchRegexp, exprDollar[1].str, exprDollar[3].str) } case 56: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:203 +//line pkg/logql/expr.y:209 { exprVAL.Matcher = mustNewMatcher(labels.MatchNotRegexp, exprDollar[1].str, exprDollar[3].str) } case 57: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:207 +//line pkg/logql/expr.y:213 { exprVAL.PipelineExpr = MultiStageExpr{exprDollar[1].PipelineStage} } case 58: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:208 +//line pkg/logql/expr.y:214 { exprVAL.PipelineExpr = append(exprDollar[1].PipelineExpr, exprDollar[2].PipelineStage) } case 59: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:212 +//line pkg/logql/expr.y:218 { exprVAL.PipelineStage = exprDollar[1].LineFilters } case 60: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:213 +//line pkg/logql/expr.y:219 { exprVAL.PipelineStage = exprDollar[2].LabelParser } case 61: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:214 +//line pkg/logql/expr.y:220 { - exprVAL.PipelineStage = &labelFilterExpr{LabelFilterer: exprDollar[2].LabelFilter} + exprVAL.PipelineStage = exprDollar[2].JSONExpressionParser } case 62: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:215 +//line pkg/logql/expr.y:221 { - exprVAL.PipelineStage = exprDollar[2].LineFormatExpr + exprVAL.PipelineStage = &labelFilterExpr{LabelFilterer: exprDollar[2].LabelFilter} } case 63: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:216 +//line pkg/logql/expr.y:222 { - exprVAL.PipelineStage = exprDollar[2].LabelFormatExpr + exprVAL.PipelineStage = exprDollar[2].LineFormatExpr } case 64: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:220 +//line pkg/logql/expr.y:223 { - exprVAL.LineFilters = newLineFilterExpr(nil, exprDollar[1].Filter, exprDollar[2].str) + exprVAL.PipelineStage = exprDollar[2].LabelFormatExpr } case 65: + exprDollar = exprS[exprpt-2 : exprpt+1] +//line pkg/logql/expr.y:227 + { + exprVAL.LineFilters = newLineFilterExpr(nil, exprDollar[1].Filter, exprDollar[2].str) + } + case 66: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:221 +//line pkg/logql/expr.y:228 { exprVAL.LineFilters = newLineFilterExpr(exprDollar[1].LineFilters, exprDollar[2].Filter, exprDollar[3].str) } - case 66: + case 67: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:224 +//line pkg/logql/expr.y:231 { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeJSON, "") } - case 67: + case 68: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:225 +//line pkg/logql/expr.y:232 { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeLogfmt, "") } - case 68: + case 69: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:226 +//line pkg/logql/expr.y:233 { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeRegexp, exprDollar[2].str) } - case 69: + case 70: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:229 +//line pkg/logql/expr.y:237 + { + exprVAL.JSONExpressionParser = newJSONExpressionParser(exprDollar[2].JSONExpressionList) + } + case 71: + exprDollar = exprS[exprpt-2 : exprpt+1] +//line pkg/logql/expr.y:239 { exprVAL.LineFormatExpr = newLineFmtExpr(exprDollar[2].str) } - case 70: + case 72: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:232 +//line pkg/logql/expr.y:242 { exprVAL.LabelFormat = log.NewRenameLabelFmt(exprDollar[1].str, exprDollar[3].str) } - case 71: + case 73: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:233 +//line pkg/logql/expr.y:243 { exprVAL.LabelFormat = log.NewTemplateLabelFmt(exprDollar[1].str, exprDollar[3].str) } - case 72: + case 74: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:237 +//line pkg/logql/expr.y:247 { exprVAL.LabelsFormat = []log.LabelFmt{exprDollar[1].LabelFormat} } - case 73: + case 75: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:238 +//line pkg/logql/expr.y:248 { exprVAL.LabelsFormat = append(exprDollar[1].LabelsFormat, exprDollar[3].LabelFormat) } - case 75: + case 77: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:242 +//line pkg/logql/expr.y:252 { exprVAL.LabelFormatExpr = newLabelFmtExpr(exprDollar[2].LabelsFormat) } - case 76: + case 78: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:245 +//line pkg/logql/expr.y:255 { exprVAL.LabelFilter = log.NewStringLabelFilter(exprDollar[1].Matcher) } - case 77: + case 79: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:246 +//line pkg/logql/expr.y:256 { exprVAL.LabelFilter = exprDollar[1].UnitFilter } - case 78: + case 80: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:247 +//line pkg/logql/expr.y:257 { exprVAL.LabelFilter = exprDollar[1].NumberFilter } - case 79: + case 81: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:248 +//line pkg/logql/expr.y:258 { exprVAL.LabelFilter = exprDollar[2].LabelFilter } - case 80: + case 82: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:249 +//line pkg/logql/expr.y:259 { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[2].LabelFilter) } - case 81: + case 83: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:250 +//line pkg/logql/expr.y:260 { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 82: + case 84: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:251 +//line pkg/logql/expr.y:261 { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 83: + case 85: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:252 +//line pkg/logql/expr.y:262 { exprVAL.LabelFilter = log.NewOrLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 84: + case 86: + exprDollar = exprS[exprpt-3 : exprpt+1] +//line pkg/logql/expr.y:266 + { + exprVAL.JSONExpression = log.NewJSONExpr(exprDollar[1].str, exprDollar[3].str) + } + case 87: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:256 +//line pkg/logql/expr.y:269 + { + exprVAL.JSONExpressionList = []log.JSONExpression{exprDollar[1].JSONExpression} + } + case 88: + exprDollar = exprS[exprpt-3 : exprpt+1] +//line pkg/logql/expr.y:270 + { + exprVAL.JSONExpressionList = append(exprDollar[1].JSONExpressionList, exprDollar[3].JSONExpression) + } + case 89: + exprDollar = exprS[exprpt-1 : exprpt+1] +//line pkg/logql/expr.y:274 { exprVAL.UnitFilter = exprDollar[1].DurationFilter } - case 85: + case 90: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:257 +//line pkg/logql/expr.y:275 { exprVAL.UnitFilter = exprDollar[1].BytesFilter } - case 86: + case 91: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:260 +//line pkg/logql/expr.y:278 { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].duration) } - case 87: + case 92: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:261 +//line pkg/logql/expr.y:279 { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].duration) } - case 88: + case 93: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:262 +//line pkg/logql/expr.y:280 { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].duration) } - case 89: + case 94: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:263 +//line pkg/logql/expr.y:281 { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].duration) } - case 90: + case 95: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:264 +//line pkg/logql/expr.y:282 { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].duration) } - case 91: + case 96: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:265 +//line pkg/logql/expr.y:283 { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) } - case 92: + case 97: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:266 +//line pkg/logql/expr.y:284 { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) } - case 93: + case 98: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:270 +//line pkg/logql/expr.y:288 { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].bytes) } - case 94: + case 99: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:271 +//line pkg/logql/expr.y:289 { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 95: + case 100: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:272 +//line pkg/logql/expr.y:290 { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].bytes) } - case 96: + case 101: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:273 +//line pkg/logql/expr.y:291 { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 97: + case 102: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:274 +//line pkg/logql/expr.y:292 { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 98: + case 103: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:275 +//line pkg/logql/expr.y:293 { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 99: + case 104: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:276 +//line pkg/logql/expr.y:294 { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 100: + case 105: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:280 +//line pkg/logql/expr.y:298 { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 101: + case 106: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:281 +//line pkg/logql/expr.y:299 { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 102: + case 107: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:282 +//line pkg/logql/expr.y:300 { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 103: + case 108: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:283 +//line pkg/logql/expr.y:301 { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 104: + case 109: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:284 +//line pkg/logql/expr.y:302 { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 105: + case 110: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:285 +//line pkg/logql/expr.y:303 { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 106: + case 111: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:286 +//line pkg/logql/expr.y:304 { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 107: + case 112: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:292 +//line pkg/logql/expr.y:310 { exprVAL.BinOpExpr = mustNewBinOpExpr("or", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 108: + case 113: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:293 +//line pkg/logql/expr.y:311 { exprVAL.BinOpExpr = mustNewBinOpExpr("and", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 109: + case 114: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:294 +//line pkg/logql/expr.y:312 { exprVAL.BinOpExpr = mustNewBinOpExpr("unless", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 110: + case 115: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:295 +//line pkg/logql/expr.y:313 { exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 111: + case 116: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:296 +//line pkg/logql/expr.y:314 { exprVAL.BinOpExpr = mustNewBinOpExpr("-", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 112: + case 117: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:297 +//line pkg/logql/expr.y:315 { exprVAL.BinOpExpr = mustNewBinOpExpr("*", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 113: + case 118: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:298 +//line pkg/logql/expr.y:316 { exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 114: + case 119: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:299 +//line pkg/logql/expr.y:317 { exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 115: + case 120: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:300 +//line pkg/logql/expr.y:318 { exprVAL.BinOpExpr = mustNewBinOpExpr("^", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 116: + case 121: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:301 +//line pkg/logql/expr.y:319 { exprVAL.BinOpExpr = mustNewBinOpExpr("==", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 117: + case 122: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:302 +//line pkg/logql/expr.y:320 { exprVAL.BinOpExpr = mustNewBinOpExpr("!=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 118: + case 123: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:303 +//line pkg/logql/expr.y:321 { exprVAL.BinOpExpr = mustNewBinOpExpr(">", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 119: + case 124: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:304 +//line pkg/logql/expr.y:322 { exprVAL.BinOpExpr = mustNewBinOpExpr(">=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 120: + case 125: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:305 +//line pkg/logql/expr.y:323 { exprVAL.BinOpExpr = mustNewBinOpExpr("<", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 121: + case 126: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:306 +//line pkg/logql/expr.y:324 { exprVAL.BinOpExpr = mustNewBinOpExpr("<=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 122: + case 127: exprDollar = exprS[exprpt-0 : exprpt+1] -//line pkg/logql/expr.y:310 +//line pkg/logql/expr.y:328 { exprVAL.BinOpModifier = BinOpOptions{} } - case 123: + case 128: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:311 +//line pkg/logql/expr.y:329 { exprVAL.BinOpModifier = BinOpOptions{ReturnBool: true} } - case 124: + case 129: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:315 +//line pkg/logql/expr.y:333 { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[1].str, false) } - case 125: + case 130: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:316 +//line pkg/logql/expr.y:334 { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, false) } - case 126: + case 131: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:317 +//line pkg/logql/expr.y:335 { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, true) } - case 127: + case 132: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:321 +//line pkg/logql/expr.y:339 { exprVAL.VectorOp = OpTypeSum } - case 128: + case 133: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:322 +//line pkg/logql/expr.y:340 { exprVAL.VectorOp = OpTypeAvg } - case 129: + case 134: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:323 +//line pkg/logql/expr.y:341 { exprVAL.VectorOp = OpTypeCount } - case 130: + case 135: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:324 +//line pkg/logql/expr.y:342 { exprVAL.VectorOp = OpTypeMax } - case 131: + case 136: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:325 +//line pkg/logql/expr.y:343 { exprVAL.VectorOp = OpTypeMin } - case 132: + case 137: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:326 +//line pkg/logql/expr.y:344 { exprVAL.VectorOp = OpTypeStddev } - case 133: + case 138: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:327 +//line pkg/logql/expr.y:345 { exprVAL.VectorOp = OpTypeStdvar } - case 134: + case 139: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:328 +//line pkg/logql/expr.y:346 { exprVAL.VectorOp = OpTypeBottomK } - case 135: + case 140: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:329 +//line pkg/logql/expr.y:347 { exprVAL.VectorOp = OpTypeTopK } - case 136: + case 141: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:333 +//line pkg/logql/expr.y:351 { exprVAL.RangeOp = OpRangeTypeCount } - case 137: + case 142: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:334 +//line pkg/logql/expr.y:352 { exprVAL.RangeOp = OpRangeTypeRate } - case 138: + case 143: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:335 +//line pkg/logql/expr.y:353 { exprVAL.RangeOp = OpRangeTypeBytes } - case 139: + case 144: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:336 +//line pkg/logql/expr.y:354 { exprVAL.RangeOp = OpRangeTypeBytesRate } - case 140: + case 145: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:337 +//line pkg/logql/expr.y:355 { exprVAL.RangeOp = OpRangeTypeAvg } - case 141: + case 146: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:338 +//line pkg/logql/expr.y:356 { exprVAL.RangeOp = OpRangeTypeSum } - case 142: + case 147: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:339 +//line pkg/logql/expr.y:357 { exprVAL.RangeOp = OpRangeTypeMin } - case 143: + case 148: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:340 +//line pkg/logql/expr.y:358 { exprVAL.RangeOp = OpRangeTypeMax } - case 144: + case 149: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:341 +//line pkg/logql/expr.y:359 { exprVAL.RangeOp = OpRangeTypeStdvar } - case 145: + case 150: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:342 +//line pkg/logql/expr.y:360 { exprVAL.RangeOp = OpRangeTypeStddev } - case 146: + case 151: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:343 +//line pkg/logql/expr.y:361 { exprVAL.RangeOp = OpRangeTypeQuantile } - case 147: + case 152: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:344 +//line pkg/logql/expr.y:362 { exprVAL.RangeOp = OpRangeTypeAbsent } - case 148: + case 153: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:349 +//line pkg/logql/expr.y:367 { exprVAL.Labels = []string{exprDollar[1].str} } - case 149: + case 154: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:350 +//line pkg/logql/expr.y:368 { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 150: + case 155: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:354 +//line pkg/logql/expr.y:372 { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 151: + case 156: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:355 +//line pkg/logql/expr.y:373 { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} } - case 152: + case 157: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:356 +//line pkg/logql/expr.y:374 { exprVAL.Grouping = &grouping{without: false, groups: nil} } - case 153: + case 158: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:357 +//line pkg/logql/expr.y:375 { exprVAL.Grouping = &grouping{without: true, groups: nil} } diff --git a/pkg/logql/lex_test.go b/pkg/logql/lex_test.go index d2be2c0db326..450e958676e5 100644 --- a/pkg/logql/lex_test.go +++ b/pkg/logql/lex_test.go @@ -56,6 +56,7 @@ func TestLex(t *testing.T) { {`{foo="bar"} # |~ "\\w+" | json`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON}}, + {`{foo="bar"} | json code="response.code", param="request.params[0]"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON, IDENTIFIER, EQ, STRING, COMMA, IDENTIFIER, EQ, STRING}}, } { t.Run(tc.input, func(t *testing.T) { actual := []int{} diff --git a/pkg/logql/log/json_expr.go b/pkg/logql/log/json_expr.go new file mode 100644 index 000000000000..81207e13fd05 --- /dev/null +++ b/pkg/logql/log/json_expr.go @@ -0,0 +1,13 @@ +package log + +type JSONExpression struct { + Identifier string + Expression string +} + +func NewJSONExpr(identifier, expression string) JSONExpression { + return JSONExpression{ + Identifier: identifier, + Expression: expression, + } +} diff --git a/pkg/logql/log/jsonexpr/jsonexpr.y b/pkg/logql/log/jsonexpr/jsonexpr.y new file mode 100644 index 000000000000..cc279375d79d --- /dev/null +++ b/pkg/logql/log/jsonexpr/jsonexpr.y @@ -0,0 +1,56 @@ +// Inspired by https://github.com/sjjian/yacc-examples + +%{ +package jsonexpr + +func setScannerData(lex interface{}, data []interface{}) { + lex.(*Scanner).data = data +} + +%} + +%union { + empty struct{} + str string + field string + list []interface{} + int int +} + +%token DOT LSB RSB +%token STRING +%token FIELD +%token INDEX + +%type index index_access +%type field key key_access +%type values + +%% + +json: + values { setScannerData(JSONExprlex, $1) } + +values: + field { $$ = []interface{}{$1} } + | key_access { $$ = []interface{}{$1} } + | index_access { $$ = []interface{}{$1} } + | values key_access { $$ = append($1, $2) } + | values index_access { $$ = append($1, $2) } + | values DOT field { $$ = append($1, $3) } + ; + +key_access: + LSB key RSB { $$ = $2 } + +index_access: + LSB index RSB { $$ = $2 } + +field: + FIELD { $$ = $1 } + +key: + STRING { $$ = $1 } + +index: + INDEX { $$ = $1 } \ No newline at end of file diff --git a/pkg/logql/log/jsonexpr/jsonexpr.y.go b/pkg/logql/log/jsonexpr/jsonexpr.y.go new file mode 100644 index 000000000000..8d21681bd8f4 --- /dev/null +++ b/pkg/logql/log/jsonexpr/jsonexpr.y.go @@ -0,0 +1,517 @@ +// Code generated by goyacc -p JSONExpr -o pkg/logql/log/jsonexpr/jsonexpr.y.go pkg/logql/log/jsonexpr/jsonexpr.y. DO NOT EDIT. + +//line pkg/logql/log/jsonexpr/jsonexpr.y:4 +package jsonexpr + +import __yyfmt__ "fmt" + +//line pkg/logql/log/jsonexpr/jsonexpr.y:4 + +func setScannerData(lex interface{}, data []interface{}) { + lex.(*Scanner).data = data +} + +//line pkg/logql/log/jsonexpr/jsonexpr.y:12 +type JSONExprSymType struct { + yys int + empty struct{} + str string + field string + list []interface{} + int int +} + +const DOT = 57346 +const LSB = 57347 +const RSB = 57348 +const STRING = 57349 +const FIELD = 57350 +const INDEX = 57351 + +var JSONExprToknames = [...]string{ + "$end", + "error", + "$unk", + "DOT", + "LSB", + "RSB", + "STRING", + "FIELD", + "INDEX", +} + +var JSONExprStatenames = [...]string{} + +const JSONExprEofCode = 1 +const JSONExprErrCode = 2 +const JSONExprInitialStackSize = 16 + +//line yacctab:1 +var JSONExprExca = [...]int{ + -1, 1, + 1, -1, + -2, 0, +} + +const JSONExprPrivate = 57344 + +const JSONExprLast = 19 + +var JSONExprAct = [...]int{ + 3, 13, 7, 14, 6, 6, 17, 16, 10, 7, + 4, 15, 5, 8, 1, 9, 2, 11, 12, +} + +var JSONExprPact = [...]int{ + -3, -1000, 4, -1000, -1000, -1000, -1000, -6, -1000, -1000, + -4, 1, 0, -1000, -1000, -1000, -1000, -1000, +} + +var JSONExprPgo = [...]int{ + 0, 18, 12, 0, 17, 10, 16, 14, +} + +var JSONExprR1 = [...]int{ + 0, 7, 6, 6, 6, 6, 6, 6, 5, 2, + 3, 4, 1, +} + +var JSONExprR2 = [...]int{ + 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, + 1, 1, 1, +} + +var JSONExprChk = [...]int{ + -1000, -7, -6, -3, -5, -2, 8, 5, -5, -2, + 4, -4, -1, 7, 9, -3, 6, 6, +} + +var JSONExprDef = [...]int{ + 0, -2, 1, 2, 3, 4, 10, 0, 5, 6, + 0, 0, 0, 11, 12, 7, 8, 9, +} + +var JSONExprTok1 = [...]int{ + 1, +} + +var JSONExprTok2 = [...]int{ + 2, 3, 4, 5, 6, 7, 8, 9, +} + +var JSONExprTok3 = [...]int{ + 0, +} + +var JSONExprErrorMessages = [...]struct { + state int + token int + msg string +}{} + +//line yaccpar:1 + +/* parser for yacc output */ + +var ( + JSONExprDebug = 0 + JSONExprErrorVerbose = false +) + +type JSONExprLexer interface { + Lex(lval *JSONExprSymType) int + Error(s string) +} + +type JSONExprParser interface { + Parse(JSONExprLexer) int + Lookahead() int +} + +type JSONExprParserImpl struct { + lval JSONExprSymType + stack [JSONExprInitialStackSize]JSONExprSymType + char int +} + +func (p *JSONExprParserImpl) Lookahead() int { + return p.char +} + +func JSONExprNewParser() JSONExprParser { + return &JSONExprParserImpl{} +} + +const JSONExprFlag = -1000 + +func JSONExprTokname(c int) string { + if c >= 1 && c-1 < len(JSONExprToknames) { + if JSONExprToknames[c-1] != "" { + return JSONExprToknames[c-1] + } + } + return __yyfmt__.Sprintf("tok-%v", c) +} + +func JSONExprStatname(s int) string { + if s >= 0 && s < len(JSONExprStatenames) { + if JSONExprStatenames[s] != "" { + return JSONExprStatenames[s] + } + } + return __yyfmt__.Sprintf("state-%v", s) +} + +func JSONExprErrorMessage(state, lookAhead int) string { + const TOKSTART = 4 + + if !JSONExprErrorVerbose { + return "syntax error" + } + + for _, e := range JSONExprErrorMessages { + if e.state == state && e.token == lookAhead { + return "syntax error: " + e.msg + } + } + + res := "syntax error: unexpected " + JSONExprTokname(lookAhead) + + // To match Bison, suggest at most four expected tokens. + expected := make([]int, 0, 4) + + // Look for shiftable tokens. + base := JSONExprPact[state] + for tok := TOKSTART; tok-1 < len(JSONExprToknames); tok++ { + if n := base + tok; n >= 0 && n < JSONExprLast && JSONExprChk[JSONExprAct[n]] == tok { + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + } + + if JSONExprDef[state] == -2 { + i := 0 + for JSONExprExca[i] != -1 || JSONExprExca[i+1] != state { + i += 2 + } + + // Look for tokens that we accept or reduce. + for i += 2; JSONExprExca[i] >= 0; i += 2 { + tok := JSONExprExca[i] + if tok < TOKSTART || JSONExprExca[i+1] == 0 { + continue + } + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + + // If the default action is to accept or reduce, give up. + if JSONExprExca[i+1] != 0 { + return res + } + } + + for i, tok := range expected { + if i == 0 { + res += ", expecting " + } else { + res += " or " + } + res += JSONExprTokname(tok) + } + return res +} + +func JSONExprlex1(lex JSONExprLexer, lval *JSONExprSymType) (char, token int) { + token = 0 + char = lex.Lex(lval) + if char <= 0 { + token = JSONExprTok1[0] + goto out + } + if char < len(JSONExprTok1) { + token = JSONExprTok1[char] + goto out + } + if char >= JSONExprPrivate { + if char < JSONExprPrivate+len(JSONExprTok2) { + token = JSONExprTok2[char-JSONExprPrivate] + goto out + } + } + for i := 0; i < len(JSONExprTok3); i += 2 { + token = JSONExprTok3[i+0] + if token == char { + token = JSONExprTok3[i+1] + goto out + } + } + +out: + if token == 0 { + token = JSONExprTok2[1] /* unknown char */ + } + if JSONExprDebug >= 3 { + __yyfmt__.Printf("lex %s(%d)\n", JSONExprTokname(token), uint(char)) + } + return char, token +} + +func JSONExprParse(JSONExprlex JSONExprLexer) int { + return JSONExprNewParser().Parse(JSONExprlex) +} + +func (JSONExprrcvr *JSONExprParserImpl) Parse(JSONExprlex JSONExprLexer) int { + var JSONExprn int + var JSONExprVAL JSONExprSymType + var JSONExprDollar []JSONExprSymType + _ = JSONExprDollar // silence set and not used + JSONExprS := JSONExprrcvr.stack[:] + + Nerrs := 0 /* number of errors */ + Errflag := 0 /* error recovery flag */ + JSONExprstate := 0 + JSONExprrcvr.char = -1 + JSONExprtoken := -1 // JSONExprrcvr.char translated into internal numbering + defer func() { + // Make sure we report no lookahead when not parsing. + JSONExprstate = -1 + JSONExprrcvr.char = -1 + JSONExprtoken = -1 + }() + JSONExprp := -1 + goto JSONExprstack + +ret0: + return 0 + +ret1: + return 1 + +JSONExprstack: + /* put a state and value onto the stack */ + if JSONExprDebug >= 4 { + __yyfmt__.Printf("char %v in %v\n", JSONExprTokname(JSONExprtoken), JSONExprStatname(JSONExprstate)) + } + + JSONExprp++ + if JSONExprp >= len(JSONExprS) { + nyys := make([]JSONExprSymType, len(JSONExprS)*2) + copy(nyys, JSONExprS) + JSONExprS = nyys + } + JSONExprS[JSONExprp] = JSONExprVAL + JSONExprS[JSONExprp].yys = JSONExprstate + +JSONExprnewstate: + JSONExprn = JSONExprPact[JSONExprstate] + if JSONExprn <= JSONExprFlag { + goto JSONExprdefault /* simple state */ + } + if JSONExprrcvr.char < 0 { + JSONExprrcvr.char, JSONExprtoken = JSONExprlex1(JSONExprlex, &JSONExprrcvr.lval) + } + JSONExprn += JSONExprtoken + if JSONExprn < 0 || JSONExprn >= JSONExprLast { + goto JSONExprdefault + } + JSONExprn = JSONExprAct[JSONExprn] + if JSONExprChk[JSONExprn] == JSONExprtoken { /* valid shift */ + JSONExprrcvr.char = -1 + JSONExprtoken = -1 + JSONExprVAL = JSONExprrcvr.lval + JSONExprstate = JSONExprn + if Errflag > 0 { + Errflag-- + } + goto JSONExprstack + } + +JSONExprdefault: + /* default state action */ + JSONExprn = JSONExprDef[JSONExprstate] + if JSONExprn == -2 { + if JSONExprrcvr.char < 0 { + JSONExprrcvr.char, JSONExprtoken = JSONExprlex1(JSONExprlex, &JSONExprrcvr.lval) + } + + /* look through exception table */ + xi := 0 + for { + if JSONExprExca[xi+0] == -1 && JSONExprExca[xi+1] == JSONExprstate { + break + } + xi += 2 + } + for xi += 2; ; xi += 2 { + JSONExprn = JSONExprExca[xi+0] + if JSONExprn < 0 || JSONExprn == JSONExprtoken { + break + } + } + JSONExprn = JSONExprExca[xi+1] + if JSONExprn < 0 { + goto ret0 + } + } + if JSONExprn == 0 { + /* error ... attempt to resume parsing */ + switch Errflag { + case 0: /* brand new error */ + JSONExprlex.Error(JSONExprErrorMessage(JSONExprstate, JSONExprtoken)) + Nerrs++ + if JSONExprDebug >= 1 { + __yyfmt__.Printf("%s", JSONExprStatname(JSONExprstate)) + __yyfmt__.Printf(" saw %s\n", JSONExprTokname(JSONExprtoken)) + } + fallthrough + + case 1, 2: /* incompletely recovered error ... try again */ + Errflag = 3 + + /* find a state where "error" is a legal shift action */ + for JSONExprp >= 0 { + JSONExprn = JSONExprPact[JSONExprS[JSONExprp].yys] + JSONExprErrCode + if JSONExprn >= 0 && JSONExprn < JSONExprLast { + JSONExprstate = JSONExprAct[JSONExprn] /* simulate a shift of "error" */ + if JSONExprChk[JSONExprstate] == JSONExprErrCode { + goto JSONExprstack + } + } + + /* the current p has no shift on "error", pop stack */ + if JSONExprDebug >= 2 { + __yyfmt__.Printf("error recovery pops state %d\n", JSONExprS[JSONExprp].yys) + } + JSONExprp-- + } + /* there is no state on the stack with an error shift ... abort */ + goto ret1 + + case 3: /* no shift yet; clobber input char */ + if JSONExprDebug >= 2 { + __yyfmt__.Printf("error recovery discards %s\n", JSONExprTokname(JSONExprtoken)) + } + if JSONExprtoken == JSONExprEofCode { + goto ret1 + } + JSONExprrcvr.char = -1 + JSONExprtoken = -1 + goto JSONExprnewstate /* try again in the same state */ + } + } + + /* reduction by production JSONExprn */ + if JSONExprDebug >= 2 { + __yyfmt__.Printf("reduce %v in:\n\t%v\n", JSONExprn, JSONExprStatname(JSONExprstate)) + } + + JSONExprnt := JSONExprn + JSONExprpt := JSONExprp + _ = JSONExprpt // guard against "declared and not used" + + JSONExprp -= JSONExprR2[JSONExprn] + // JSONExprp is now the index of $0. Perform the default action. Iff the + // reduced production is ε, $1 is possibly out of range. + if JSONExprp+1 >= len(JSONExprS) { + nyys := make([]JSONExprSymType, len(JSONExprS)*2) + copy(nyys, JSONExprS) + JSONExprS = nyys + } + JSONExprVAL = JSONExprS[JSONExprp+1] + + /* consult goto table to find next state */ + JSONExprn = JSONExprR1[JSONExprn] + JSONExprg := JSONExprPgo[JSONExprn] + JSONExprj := JSONExprg + JSONExprS[JSONExprp].yys + 1 + + if JSONExprj >= JSONExprLast { + JSONExprstate = JSONExprAct[JSONExprg] + } else { + JSONExprstate = JSONExprAct[JSONExprj] + if JSONExprChk[JSONExprstate] != -JSONExprn { + JSONExprstate = JSONExprAct[JSONExprg] + } + } + // dummy call; replaced with literal code + switch JSONExprnt { + + case 1: + JSONExprDollar = JSONExprS[JSONExprpt-1 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:32 + { + setScannerData(JSONExprlex, JSONExprDollar[1].list) + } + case 2: + JSONExprDollar = JSONExprS[JSONExprpt-1 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:35 + { + JSONExprVAL.list = []interface{}{JSONExprDollar[1].str} + } + case 3: + JSONExprDollar = JSONExprS[JSONExprpt-1 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:36 + { + JSONExprVAL.list = []interface{}{JSONExprDollar[1].str} + } + case 4: + JSONExprDollar = JSONExprS[JSONExprpt-1 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:37 + { + JSONExprVAL.list = []interface{}{JSONExprDollar[1].int} + } + case 5: + JSONExprDollar = JSONExprS[JSONExprpt-2 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:38 + { + JSONExprVAL.list = append(JSONExprDollar[1].list, JSONExprDollar[2].str) + } + case 6: + JSONExprDollar = JSONExprS[JSONExprpt-2 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:39 + { + JSONExprVAL.list = append(JSONExprDollar[1].list, JSONExprDollar[2].int) + } + case 7: + JSONExprDollar = JSONExprS[JSONExprpt-3 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:40 + { + JSONExprVAL.list = append(JSONExprDollar[1].list, JSONExprDollar[3].str) + } + case 8: + JSONExprDollar = JSONExprS[JSONExprpt-3 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:44 + { + JSONExprVAL.str = JSONExprDollar[2].str + } + case 9: + JSONExprDollar = JSONExprS[JSONExprpt-3 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:47 + { + JSONExprVAL.int = JSONExprDollar[2].int + } + case 10: + JSONExprDollar = JSONExprS[JSONExprpt-1 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:50 + { + JSONExprVAL.str = JSONExprDollar[1].field + } + case 11: + JSONExprDollar = JSONExprS[JSONExprpt-1 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:53 + { + JSONExprVAL.str = JSONExprDollar[1].str + } + case 12: + JSONExprDollar = JSONExprS[JSONExprpt-1 : JSONExprpt+1] +//line pkg/logql/log/jsonexpr/jsonexpr.y:56 + { + JSONExprVAL.int = JSONExprDollar[1].int + } + } + goto JSONExprstack /* stack new state and value */ +} diff --git a/pkg/logql/log/jsonexpr/jsonexpr_test.go b/pkg/logql/log/jsonexpr/jsonexpr_test.go new file mode 100644 index 000000000000..d8240d2b6a47 --- /dev/null +++ b/pkg/logql/log/jsonexpr/jsonexpr_test.go @@ -0,0 +1,131 @@ +package jsonexpr + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestJSONExpressionParser(t *testing.T) { + // {"app":"foo","field with space":"value","field with ÜFT8👌":true,"namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar", "params": [{"param": true},2,3]}}} + + tests := []struct { + name string + expression string + want []interface{} + error error + }{ + { + "single field", + "app", + []interface{}{"app"}, + nil, + }, + { + "top-level field with spaces", + `["field with space"]`, + []interface{}{"field with space"}, + nil, + }, + { + "top-level field with UTF8", + `["field with ÜFT8👌"]`, + []interface{}{"field with ÜFT8👌"}, + nil, + }, + { + "top-level array access", + `[0]`, + []interface{}{0}, + nil, + }, + { + "nested field", + `pod.uuid`, + []interface{}{"pod", "uuid"}, + nil, + }, + { + "nested field alternate syntax", + `pod["uuid"]`, + []interface{}{"pod", "uuid"}, + nil, + }, + { + "nested field alternate syntax 2", + `["pod"]["uuid"]`, + []interface{}{"pod", "uuid"}, + nil, + }, + { + "array access", + `pod.deployment.params[0]`, + []interface{}{"pod", "deployment", "params", 0}, + nil, + }, + { + "multi-level array access", + `pod.deployment.params[0].param`, + []interface{}{"pod", "deployment", "params", 0, "param"}, + nil, + }, + { + "multi-level array access alternate syntax", + `pod.deployment.params[0]["param"]`, + []interface{}{"pod", "deployment", "params", 0, "param"}, + nil, + }, + { + "empty", + ``, + nil, + nil, + }, + + { + "invalid field access", + `field with space`, + nil, + fmt.Errorf("syntax error: unexpected FIELD"), + }, + { + "missing opening square bracket", + `"pod"]`, + nil, + fmt.Errorf("syntax error: unexpected STRING, expecting LSB or FIELD"), + }, + { + "missing closing square bracket", + `["pod"`, + nil, + fmt.Errorf("syntax error: unexpected $end, expecting RSB"), + }, + { + "missing closing square bracket", + `["pod""deployment"]`, + nil, + fmt.Errorf("syntax error: unexpected STRING, expecting RSB"), + }, + { + "invalid nesting", + `pod..uuid`, + nil, + fmt.Errorf("syntax error: unexpected DOT, expecting FIELD"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parsed, err := Parse(tt.expression, false) + + require.Equal(t, tt.want, parsed) + + if tt.error == nil { + return + } + + require.NotNil(t, err) + require.Equal(t, tt.error.Error(), err.Error()) + }) + } +} diff --git a/pkg/logql/log/jsonexpr/lexer.go b/pkg/logql/log/jsonexpr/lexer.go new file mode 100644 index 000000000000..f92a72ef170a --- /dev/null +++ b/pkg/logql/log/jsonexpr/lexer.go @@ -0,0 +1,164 @@ +package jsonexpr + +import ( + "bufio" + "fmt" + "io" + "strconv" + "text/scanner" +) + +type Scanner struct { + buf *bufio.Reader + data []interface{} + err error + debug bool +} + +func NewScanner(r io.Reader, debug bool) *Scanner { + return &Scanner{ + buf: bufio.NewReader(r), + debug: debug, + } +} + +func (sc *Scanner) Error(s string) { + sc.err = fmt.Errorf(s) + fmt.Printf("syntax error: %s\n", s) +} + +func (sc *Scanner) Reduced(rule, state int, lval *JSONExprSymType) bool { + if sc.debug { + fmt.Printf("rule: %v; state %v; lval: %v\n", rule, state, lval) + } + return false +} + +func (sc *Scanner) Lex(lval *JSONExprSymType) int { + return sc.lex(lval) +} + +func (sc *Scanner) lex(lval *JSONExprSymType) int { + for { + r := sc.read() + + if r == 0 { + return 0 + } + if isWhitespace(r) { + continue + } + + if isDigit(r) { + sc.unread() + val, err := sc.scanInt() + if err != nil { + sc.err = fmt.Errorf(err.Error()) + return 0 + } + + lval.int = val + return INDEX + } + + switch true { + case r == '[': + return LSB + case r == ']': + return RSB + case r == '.': + return DOT + case isIdentifier(r): + sc.unread() + lval.field = sc.scanField() + return FIELD + case r == '"': + sc.unread() + lval.str = sc.scanStr() + return STRING + default: + sc.err = fmt.Errorf("unexpected char %c", r) + return 0 + } + } +} + +func isIdentifier(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || r == '_' +} + +func (sc *Scanner) scanField() string { + var str []rune + + for { + r := sc.read() + if !isIdentifier(r) { + sc.unread() + break + } + + if r == '.' || r == scanner.EOF || r == rune(0) { + sc.unread() + break + } + + str = append(str, r) + } + return string(str) +} + +func (sc *Scanner) scanStr() string { + var str []rune + //begin with ", end with " + r := sc.read() + if r != '"' { + sc.err = fmt.Errorf("unexpected char %c", r) + return "" + } + + for { + r := sc.read() + if r == '"' || r == ']' { + break + } + str = append(str, r) + } + return string(str) +} + +func (sc *Scanner) scanInt() (int, error) { + var number []rune + + for { + r := sc.read() + if r == '.' && len(number) > 0 { + return 0, fmt.Errorf("cannot use float as array index") + } + + if isWhitespace(r) || r == '.' || r == ']' { + sc.unread() + break + } + + if !isDigit(r) { + return 0, fmt.Errorf("non-integer value: %c", r) + } + + number = append(number, r) + } + + return strconv.Atoi(string(number)) +} + +func (sc *Scanner) read() rune { + ch, _, _ := sc.buf.ReadRune() + return ch +} + +func (sc *Scanner) unread() { _ = sc.buf.UnreadRune() } + +func isWhitespace(ch rune) bool { return ch == ' ' || ch == '\t' || ch == '\n' } + +func isDigit(r rune) bool { + return r >= '0' && r <= '9' +} diff --git a/pkg/logql/log/jsonexpr/parser.go b/pkg/logql/log/jsonexpr/parser.go new file mode 100644 index 000000000000..9d14004dcddc --- /dev/null +++ b/pkg/logql/log/jsonexpr/parser.go @@ -0,0 +1,19 @@ +package jsonexpr + +import ( + "strings" +) + +func init() { + JSONExprErrorVerbose = true +} + +func Parse(expr string, debug bool) ([]interface{}, error) { + s := NewScanner(strings.NewReader(expr), debug) + JSONExprParse(s) + + if s.err != nil { + return nil, s.err + } + return s.data, nil +} diff --git a/pkg/logql/log/parser.go b/pkg/logql/log/parser.go index 79360b1d25a8..afc70b63400d 100644 --- a/pkg/logql/log/parser.go +++ b/pkg/logql/log/parser.go @@ -6,6 +6,7 @@ import ( "io" "regexp" + "github.com/grafana/loki/pkg/logql/log/jsonexpr" "github.com/grafana/loki/pkg/logql/log/logfmt" jsoniter "github.com/json-iterator/go" @@ -259,3 +260,53 @@ func (l *LogfmtParser) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { } func (l *LogfmtParser) RequiredLabelNames() []string { return []string{} } + +type JSONExpressionParser struct { + expressions map[string][]interface{} +} + +func NewJSONExpressionParser(expressions []JSONExpression) (*JSONExpressionParser, error) { + var paths = make(map[string][]interface{}) + + for _, exp := range expressions { + path, err := jsonexpr.Parse(exp.Expression, false) + if err != nil { + return nil, fmt.Errorf("cannot parse expression [%s]: %w", exp.Expression, err) + } + + if !model.LabelName(exp.Identifier).IsValid() { + return nil, fmt.Errorf("invalid extracted label name '%s'", exp.Identifier) + } + + paths[exp.Identifier] = path + } + + return &JSONExpressionParser{ + expressions: paths, + }, nil +} + +func (j *JSONExpressionParser) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { + if lbs.ParserLabelHints().NoLabels() { + return line, true + } + + if !jsoniter.ConfigFastest.Valid(line) { + lbs.SetErr(errJSON) + return line, true + } + + for identifier, paths := range j.expressions { + result := jsoniter.ConfigFastest.Get(line, paths...).ToString() + + if lbs.BaseHas(identifier) { + identifier = identifier + duplicateSuffix + } + + lbs.Set(identifier, result) + } + + return line, true +} + +func (j *JSONExpressionParser) RequiredLabelNames() []string { return []string{} } diff --git a/pkg/logql/log/parser_test.go b/pkg/logql/log/parser_test.go index 1f7d97eaf4fe..157bd93e6521 100644 --- a/pkg/logql/log/parser_test.go +++ b/pkg/logql/log/parser_test.go @@ -1,6 +1,7 @@ package log import ( + "fmt" "sort" "testing" @@ -87,6 +88,298 @@ func Test_jsonParser_Parse(t *testing.T) { } } +func TestJSONExpressionParser(t *testing.T) { + testLine := []byte(`{"app":"foo","field with space":"value","field with ÜFT8👌":"value","null_field":null,"bool_field":false,"namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar", "params": [1,2,3]}}}`) + + tests := []struct { + name string + line []byte + expressions []JSONExpression + lbs labels.Labels + want labels.Labels + }{ + { + "single field", + testLine, + []JSONExpression{ + NewJSONExpr("app", "app"), + }, + labels.Labels{}, + labels.Labels{ + {Name: "app", Value: "foo"}, + }, + }, + { + "alternate syntax", + testLine, + []JSONExpression{ + NewJSONExpr("test", `["field with space"]`), + }, + labels.Labels{}, + labels.Labels{ + {Name: "test", Value: "value"}, + }, + }, + { + "multiple fields", + testLine, + []JSONExpression{ + NewJSONExpr("app", "app"), + NewJSONExpr("namespace", "namespace"), + }, + labels.Labels{}, + labels.Labels{ + {Name: "app", Value: "foo"}, + {Name: "namespace", Value: "prod"}, + }, + }, + { + "utf8", + testLine, + []JSONExpression{ + NewJSONExpr("utf8", `["field with ÜFT8👌"]`), + }, + labels.Labels{}, + labels.Labels{ + {Name: "utf8", Value: "value"}, + }, + }, + { + "nested field", + testLine, + []JSONExpression{ + NewJSONExpr("uuid", "pod.uuid"), + }, + labels.Labels{}, + labels.Labels{ + {Name: "uuid", Value: "foo"}, + }, + }, + { + "nested field alternate syntax", + testLine, + []JSONExpression{ + NewJSONExpr("uuid", `pod["uuid"]`), + }, + labels.Labels{}, + labels.Labels{ + {Name: "uuid", Value: "foo"}, + }, + }, + { + "nested field alternate syntax 2", + testLine, + []JSONExpression{ + NewJSONExpr("uuid", `["pod"]["uuid"]`), + }, + labels.Labels{}, + labels.Labels{ + {Name: "uuid", Value: "foo"}, + }, + }, + { + "nested field alternate syntax 3", + testLine, + []JSONExpression{ + NewJSONExpr("uuid", `["pod"].uuid`), + }, + labels.Labels{}, + labels.Labels{ + {Name: "uuid", Value: "foo"}, + }, + }, + { + "array element", + testLine, + []JSONExpression{ + NewJSONExpr("param", `pod.deployment.params[0]`), + }, + labels.Labels{}, + labels.Labels{ + {Name: "param", Value: "1"}, + }, + }, + { + "full array", + testLine, + []JSONExpression{ + NewJSONExpr("params", `pod.deployment.params`), + }, + labels.Labels{}, + labels.Labels{ + {Name: "params", Value: "[1,2,3]"}, + }, + }, + { + "full object", + testLine, + []JSONExpression{ + NewJSONExpr("deployment", `pod.deployment`), + }, + labels.Labels{}, + labels.Labels{ + {Name: "deployment", Value: `{"ref":"foobar", "params": [1,2,3]}`}, + }, + }, + { + "expression matching nothing", + testLine, + []JSONExpression{ + NewJSONExpr("nope", `pod.nope`), + }, + labels.Labels{}, + labels.Labels{ + labels.Label{Name: "nope", Value: ""}, + }, + }, + { + "null field", + testLine, + []JSONExpression{ + NewJSONExpr("nf", `null_field`), + }, + labels.Labels{}, + labels.Labels{ + labels.Label{Name: "nf", Value: ""}, // null is coerced to an empty string + }, + }, + { + "boolean field", + testLine, + []JSONExpression{ + NewJSONExpr("bool", `bool_field`), + }, + labels.Labels{}, + labels.Labels{ + {Name: "bool", Value: `false`}, + }, + }, + { + "label override", + testLine, + []JSONExpression{ + NewJSONExpr("uuid", `pod.uuid`), + }, + labels.Labels{ + {Name: "uuid", Value: "bar"}, + }, + labels.Labels{ + {Name: "uuid", Value: "bar"}, + {Name: "uuid_extracted", Value: "foo"}, + }, + }, + { + "non-matching expression", + testLine, + []JSONExpression{ + NewJSONExpr("request_size", `request.size.invalid`), + }, + labels.Labels{ + {Name: "uuid", Value: "bar"}, + }, + labels.Labels{ + {Name: "uuid", Value: "bar"}, + {Name: "request_size", Value: ""}, + }, + }, + { + "empty line", + []byte("{}"), + []JSONExpression{ + NewJSONExpr("uuid", `pod.uuid`), + }, + labels.Labels{}, + labels.Labels{ + labels.Label{Name: "uuid", Value: ""}, + }, + }, + { + "existing labels are not affected", + testLine, + []JSONExpression{ + NewJSONExpr("uuid", `will.not.work`), + }, + labels.Labels{ + {Name: "foo", Value: "bar"}, + }, + labels.Labels{ + {Name: "foo", Value: "bar"}, + {Name: "uuid", Value: ""}, + }, + }, + { + "invalid JSON line", + []byte(`invalid json`), + []JSONExpression{ + NewJSONExpr("uuid", `will.not.work`), + }, + labels.Labels{ + {Name: "foo", Value: "bar"}, + }, + labels.Labels{ + {Name: "foo", Value: "bar"}, + {Name: ErrorLabel, Value: errJSON}, + }, + }, + } + for _, tt := range tests { + j, err := NewJSONExpressionParser(tt.expressions) + if err != nil { + t.Fatalf("cannot create JSON expression parser: %s", err.Error()) + } + + t.Run(tt.name, func(t *testing.T) { + b := NewBaseLabelsBuilder().ForLabels(tt.lbs, tt.lbs.Hash()) + b.Reset() + _, _ = j.Process(tt.line, b) + sort.Sort(tt.want) + require.Equal(t, tt.want, b.Labels()) + }) + } +} + +func TestJSONExpressionParserFailures(t *testing.T) { + tests := []struct { + name string + expression JSONExpression + error string + }{ + { + "invalid field name", + NewJSONExpr("app", `field with space`), + "unexpected FIELD", + }, + { + "missing opening square bracket", + NewJSONExpr("app", `"pod"]`), + "unexpected STRING, expecting LSB or FIELD", + }, + { + "missing closing square bracket", + NewJSONExpr("app", `["pod"`), + "unexpected $end, expecting RSB", + }, + { + "missing closing square bracket", + NewJSONExpr("app", `["pod""uuid"]`), + "unexpected STRING, expecting RSB", + }, + { + "invalid nesting", + NewJSONExpr("app", `pod..uuid`), + "unexpected DOT, expecting FIELD", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := NewJSONExpressionParser([]JSONExpression{tt.expression}) + + require.NotNil(t, err) + require.Equal(t, err.Error(), fmt.Sprintf("cannot parse expression [%s]: syntax error: %s", tt.expression.Expression, tt.error)) + }) + } +} + func Benchmark_Parser(b *testing.B) { lbs := labels.Labels{ {Name: "cluster", Value: "qa-us-central1"}, diff --git a/pkg/logql/log/pipeline_test.go b/pkg/logql/log/pipeline_test.go index 96f5ff837a2a..21c80757df85 100644 --- a/pkg/logql/log/pipeline_test.go +++ b/pkg/logql/log/pipeline_test.go @@ -93,3 +93,89 @@ func mustFilter(f Filterer, err error) Filterer { } return f } + +func jsonBenchmark(b *testing.B, parser Stage) { + b.ReportAllocs() + + p := NewPipeline([]Stage{ + mustFilter(NewFilter("metrics.go", labels.MatchEqual)).ToStage(), + parser, + }) + line := []byte(`{"ts":"2020-12-27T09:15:54.333026285Z","error":"action could not be completed", "context":{"file": "metrics.go"}}`) + lbs := labels.Labels{ + {Name: "cluster", Value: "ops-tool1"}, + {Name: "name", Value: "querier"}, + {Name: "pod", Value: "querier-5896759c79-q7q9h"}, + {Name: "stream", Value: "stderr"}, + {Name: "container", Value: "querier"}, + {Name: "namespace", Value: "loki-dev"}, + {Name: "job", Value: "loki-dev/querier"}, + {Name: "pod_template_hash", Value: "5896759c79"}, + } + b.ResetTimer() + sp := p.ForStream(lbs) + for n := 0; n < b.N; n++ { + resLine, resLbs, resOK = sp.Process(line) + + if !resOK { + b.Fatalf("resulting line not ok: %s\n", line) + } + + if resLbs.Labels().Get("context_file") != "metrics.go" { + b.Fatalf("label was not extracted correctly! %+v\n", resLbs) + } + } +} + +func invalidJSONBenchmark(b *testing.B, parser Stage) { + b.ReportAllocs() + + p := NewPipeline([]Stage{ + mustFilter(NewFilter("invalid json", labels.MatchEqual)).ToStage(), + parser, + }) + line := []byte(`invalid json`) + b.ResetTimer() + sp := p.ForStream(labels.Labels{}) + for n := 0; n < b.N; n++ { + resLine, resLbs, resOK = sp.Process(line) + + if !resOK { + b.Fatalf("resulting line not ok: %s\n", line) + } + + if resLbs.Labels().Get(ErrorLabel) != errJSON { + b.Fatalf("no %s label found: %+v\n", ErrorLabel, resLbs.Labels()) + } + } +} + +func BenchmarkJSONParser(b *testing.B) { + jsonBenchmark(b, NewJSONParser()) +} + +func BenchmarkJSONParserInvalidLine(b *testing.B) { + invalidJSONBenchmark(b, NewJSONParser()) +} + +func BenchmarkJSONExpressionParser(b *testing.B) { + parser, err := NewJSONExpressionParser([]JSONExpression{ + NewJSONExpr("context_file", "context.file"), + }) + if err != nil { + b.Fatal("cannot create new JSON expression parser") + } + + jsonBenchmark(b, parser) +} + +func BenchmarkJSONExpressionParserInvalidLine(b *testing.B) { + parser, err := NewJSONExpressionParser([]JSONExpression{ + NewJSONExpr("context_file", "some.expression"), + }) + if err != nil { + b.Fatal("cannot create new JSON expression parser") + } + + invalidJSONBenchmark(b, parser) +} diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 0ebcb09df581..ef252c66548d 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -2193,6 +2193,40 @@ func TestParse(t *testing.T) { }, }, }, + { + in: `{app="foo"} | json bob="top.sub[\"index\"]"`, + exp: &pipelineExpr{ + left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), + pipeline: MultiStageExpr{ + newJSONExpressionParser([]log.JSONExpression{ + log.NewJSONExpr("bob", `top.sub["index"]`), + }), + }, + }, + }, + { + in: `{app="foo"} | json bob="top.params[0]"`, + exp: &pipelineExpr{ + left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), + pipeline: MultiStageExpr{ + newJSONExpressionParser([]log.JSONExpression{ + log.NewJSONExpr("bob", `top.params[0]`), + }), + }, + }, + }, + { + in: `{app="foo"} | json response_code="response.code", api_key="request.headers[\"X-API-KEY\"]"`, + exp: &pipelineExpr{ + left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), + pipeline: MultiStageExpr{ + newJSONExpressionParser([]log.JSONExpression{ + log.NewJSONExpr("response_code", `response.code`), + log.NewJSONExpr("api_key", `request.headers["X-API-KEY"]`), + }), + }, + }, + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in)