From e2087727e3047e29dc5ac7aacdd03d4468f7c75b Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Thu, 6 Feb 2020 23:45:00 -0500 Subject: [PATCH 01/24] binops in ast --- pkg/logql/ast.go | 31 +++++ pkg/logql/ast_test.go | 8 ++ pkg/logql/expr.y | 32 ++++- pkg/logql/expr.y.go | 252 +++++++++++++++++++++++---------------- pkg/logql/lex.go | 2 + pkg/logql/parser_test.go | 4 +- 6 files changed, 218 insertions(+), 111 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 84cb803410eb..014e5ed46683 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -227,6 +227,10 @@ const ( OpTypeTopK = "topk" OpTypeCountOverTime = "count_over_time" OpTypeRate = "rate" + + // binops + OpTypeAdd = "+" + OpTypeDiv = "/" ) // SampleExpr is a LogQL expression filtering logs and returning metric samples. @@ -370,6 +374,33 @@ func (e *vectorAggregationExpr) String() string { return formatOperation(e.operation, e.grouping, params...) } +type binOpExpr struct { + SampleExpr + RHS SampleExpr + op string +} + +func (e *binOpExpr) String() string { + return fmt.Sprintf("%s %s %s", e.SampleExpr.String(), e.op, e.RHS.String()) +} + +func mustNewBinOpExpr(op string, lhs, rhs Expr) SampleExpr { + left, ok := lhs.(SampleExpr) + if !ok { + panic(fmt.Errorf("unexpected type for binOpExpr (%T): %+v", lhs, lhs)) + } + + right, ok := rhs.(SampleExpr) + if !ok { + panic(fmt.Errorf("unexpected type for binOpExpr (%T): %+v", rhs, rhs)) + } + return &binOpExpr{ + SampleExpr: left, + RHS: right, + op: op, + } +} + // helper used to impl Stringer for vector and range aggregations // nolint:interfacer func formatOperation(op string, grouping *grouping, params ...string) string { diff --git a/pkg/logql/ast_test.go b/pkg/logql/ast_test.go index 303cf52986ea..063fecf1561b 100644 --- a/pkg/logql/ast_test.go +++ b/pkg/logql/ast_test.go @@ -43,6 +43,14 @@ func Test_SampleExpr_String(t *testing.T) { `sum(count_over_time({job="mysql"}[5m]))`, `topk(10,sum(rate({region="us-east1"}[5m])) by (name))`, `avg( rate( ( {job="nginx"} |= "GET" ) [10s] ) ) by (region)`, + `sum by (cluster) (count_over_time({job="mysql"}[5m]))`, + `sum by (cluster) (count_over_time({job="mysql"}[5m])) / sum by (cluster) (count_over_time({job="postgres"}[5m])) `, + ` + sum by (cluster) (count_over_time({job="postgres"}[5m])) / + sum by (cluster) (count_over_time({job="postgres"}[5m])) / + sum by (cluster) (count_over_time({job="postgres"}[5m])) + `, + `sum by (cluster) (count_over_time({job="mysql"}[5m])) / min(count_over_time({job="mysql"}[5m])) `, } { t.Run(tc, func(t *testing.T) { expr, err := ParseExpr(tc) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index a053b3224d36..32c30af2ff57 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -21,6 +21,8 @@ import ( Selector []*labels.Matcher VectorAggregationExpr SampleExpr VectorOp string + BinOpExpr SampleExpr + BinOp string str string duration time.Duration int int64 @@ -41,26 +43,38 @@ import ( %type selector %type vectorAggregationExpr %type vectorOp +%type binOpExpr +%type binOp %token IDENTIFIER STRING %token DURATION %token MATCHERS LABELS EQ NEQ RE NRE OPEN_BRACE CLOSE_BRACE OPEN_BRACKET CLOSE_BRACKET COMMA DOT PIPE_MATCH PIPE_EXACT OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK +// Operators are listed with increasing precedence. +%token +ADD +DIV + +%left ADD +%left DIV + %% root: expr { exprlex.(*lexer).expr = $1 }; expr: - logExpr { $$ = $1 } - | rangeAggregationExpr { $$ = $1 } - | vectorAggregationExpr { $$ = $1 } + logExpr { $$ = $1 } + | rangeAggregationExpr { $$ = $1 } + | vectorAggregationExpr { $$ = $1 } + | binOpExpr { $$ = $1 } + | OPEN_PARENTHESIS expr CLOSE_PARENTHESIS { $$ = $2 } ; logExpr: selector { $$ = newMatcherExpr($1)} | logExpr filter STRING { $$ = NewFilterExpr( $1, $2, $3 ) } - | OPEN_PARENTHESIS logExpr CLOSE_PARENTHESIS { $$ = $2} + | OPEN_PARENTHESIS logExpr CLOSE_PARENTHESIS { $$ = $2 } | logExpr filter error | logExpr error ; @@ -115,6 +129,16 @@ matcher: | IDENTIFIER NRE STRING { $$ = mustNewMatcher(labels.MatchNotRegexp, $1, $3) } ; +// TODO(owen-d): add (on,ignoring) clauses to binOpExpr +// https://prometheus.io/docs/prometheus/latest/querying/operators/ +binOpExpr: + expr binOp expr { $$ = mustNewBinOpExpr($2, $1, $3) } + +binOp: + ADD { $$ = OpTypeAdd } + | DIV { $$ = OpTypeDiv } + + vectorOp: SUM { $$ = OpTypeSum } | AVG { $$ = OpTypeAvg } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 4730258790db..c6b56d3c8826 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -25,6 +25,8 @@ type exprSymType struct { Selector []*labels.Matcher VectorAggregationExpr SampleExpr VectorOp string + BinOpExpr SampleExpr + BinOp string str string duration time.Duration int int64 @@ -62,6 +64,8 @@ const STDDEV = 57374 const STDVAR = 57375 const BOTTOMK = 57376 const TOPK = 57377 +const ADD = 57378 +const DIV = 57379 var exprToknames = [...]string{ "$end", @@ -99,6 +103,8 @@ var exprToknames = [...]string{ "STDVAR", "BOTTOMK", "TOPK", + "ADD", + "DIV", } var exprStatenames = [...]string{} @@ -113,92 +119,103 @@ var exprExca = [...]int{ -2, 0, -1, 3, 1, 2, + 22, 2, + 36, 2, + 37, 2, + -2, 0, + -1, 33, + 36, 2, + 37, 2, -2, 0, } const exprPrivate = 57344 -const exprLast = 149 +const exprLast = 156 var exprAct = [...]int{ - 31, 5, 4, 22, 36, 69, 10, 41, 30, 49, - 32, 33, 86, 46, 7, 32, 33, 88, 11, 12, - 13, 14, 16, 17, 15, 18, 19, 20, 21, 90, - 89, 48, 45, 44, 11, 12, 13, 14, 16, 17, - 15, 18, 19, 20, 21, 58, 85, 86, 3, 68, - 67, 63, 87, 84, 23, 71, 28, 72, 61, 65, - 64, 47, 27, 29, 26, 80, 81, 58, 82, 83, - 66, 24, 25, 53, 40, 76, 75, 74, 42, 11, - 12, 13, 14, 16, 17, 15, 18, 19, 20, 21, - 92, 93, 62, 59, 10, 78, 10, 59, 77, 73, - 91, 27, 43, 26, 7, 27, 37, 26, 23, 51, - 24, 25, 70, 79, 24, 25, 27, 60, 26, 23, - 9, 50, 23, 61, 52, 24, 25, 27, 40, 26, - 27, 39, 26, 35, 38, 37, 24, 25, 6, 24, - 25, 54, 55, 56, 57, 8, 34, 2, 1, + 36, 5, 4, 26, 41, 76, 11, 3, 48, 24, + 25, 37, 38, 97, 7, 33, 46, 96, 12, 13, + 14, 15, 17, 18, 16, 19, 20, 21, 22, 92, + 24, 25, 35, 66, 37, 38, 91, 52, 51, 2, + 93, 31, 49, 30, 93, 95, 72, 32, 71, 94, + 28, 29, 65, 86, 56, 55, 75, 74, 69, 70, + 53, 27, 78, 43, 79, 68, 54, 34, 73, 31, + 83, 30, 87, 88, 65, 89, 90, 60, 28, 29, + 58, 12, 13, 14, 15, 17, 18, 16, 19, 20, + 21, 22, 57, 85, 82, 59, 84, 99, 100, 12, + 13, 14, 15, 17, 18, 16, 19, 20, 21, 22, + 27, 11, 81, 45, 68, 66, 44, 80, 31, 50, + 30, 27, 40, 31, 42, 30, 27, 28, 29, 31, + 47, 30, 28, 29, 31, 67, 30, 98, 28, 29, + 23, 47, 42, 28, 29, 61, 62, 63, 64, 77, + 6, 10, 8, 9, 39, 1, } var exprPact = [...]int{ - -7, -1000, -1000, 120, -1000, -1000, -1000, 83, 42, -13, - 131, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 129, -1000, -1000, -1000, -1000, -1000, 106, 81, - 9, 40, 10, -12, 107, 59, -1000, 132, -1000, -1000, - -1000, 95, 117, 81, 38, 37, 53, 54, 108, 108, - -1000, -1000, 102, -1000, 94, 72, 71, 70, 93, -1000, - -1000, -1000, 52, 91, -8, -8, 54, 31, 24, 30, - -1000, -5, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 8, 7, -1000, -1000, 96, -1000, -1000, -8, - -8, -1000, -1000, -1000, + -7, -1000, -27, 124, -1000, -1000, -1000, -7, -1000, 46, + 11, 120, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -7, -1000, -1000, 111, -1000, -1000, -1000, + -1000, -1000, -6, 119, 98, 56, 45, 34, 33, 78, + 63, -1000, 136, -27, -1000, -1000, -1000, -1000, 113, 59, + 98, 26, 24, 51, 74, 145, 145, -1000, -1000, 138, + -1000, 112, 107, 89, 65, 91, -1000, -1000, -1000, 108, + 31, -12, -12, 74, 14, 7, 27, -1000, 23, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -5, + -9, -1000, -1000, 133, -1000, -1000, -12, -12, -1000, -1000, + -1000, } var exprPgo = [...]int{ - 0, 148, 147, 3, 0, 5, 48, 7, 4, 146, - 2, 145, 138, 1, 120, + 0, 155, 39, 3, 0, 5, 7, 8, 4, 154, + 2, 153, 152, 1, 151, 150, 140, } var exprR1 = [...]int{ - 0, 1, 2, 2, 2, 6, 6, 6, 6, 6, - 7, 7, 7, 7, 7, 10, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 3, 3, 3, 3, - 12, 12, 12, 9, 9, 8, 8, 8, 8, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 11, 11, - 5, 5, 4, 4, + 0, 1, 2, 2, 2, 2, 2, 6, 6, 6, + 6, 6, 7, 7, 7, 7, 7, 10, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, + 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, + 8, 15, 16, 16, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 11, 11, 5, 5, 4, 4, } var exprR2 = [...]int{ - 0, 1, 1, 1, 1, 1, 3, 3, 3, 2, - 2, 3, 3, 3, 2, 4, 4, 4, 5, 5, - 5, 5, 6, 7, 6, 7, 1, 1, 1, 1, - 3, 3, 3, 1, 3, 3, 3, 3, 3, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 3, 4, 4, + 0, 1, 1, 1, 1, 1, 3, 1, 3, 3, + 3, 2, 2, 3, 3, 3, 2, 4, 4, 4, + 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, + 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, + 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 3, 4, 4, } var exprChk = [...]int{ - -1000, -1, -2, -6, -10, -13, -12, 21, -11, -14, - 13, 25, 26, 27, 28, 31, 29, 30, 32, 33, - 34, 35, -3, 2, 19, 20, 12, 10, -6, 21, - 21, -4, 23, 24, -9, 2, -8, 4, 5, 2, - 22, -7, -6, 21, -10, -13, 4, 21, 21, 21, - 14, 2, 17, 14, 9, 10, 11, 12, -3, 2, - 22, 6, -6, -7, 22, 22, 17, -10, -13, -5, - 4, -5, -8, 5, 5, 5, 5, 5, 2, 22, - -4, -4, -13, -10, 22, 22, 17, 22, 22, 22, - 22, 4, -4, -4, + -1000, -1, -2, -6, -10, -13, -15, 21, -12, -11, + -14, 13, 25, 26, 27, 28, 31, 29, 30, 32, + 33, 34, 35, -16, 36, 37, -3, 2, 19, 20, + 12, 10, -2, -6, 21, 21, -4, 23, 24, -9, + 2, -8, 4, -2, 5, 2, 22, 22, -7, -6, + 21, -10, -13, 4, 21, 21, 21, 14, 2, 17, + 14, 9, 10, 11, 12, -3, 2, 22, 6, -6, + -7, 22, 22, 17, -10, -13, -5, 4, -5, -8, + 5, 5, 5, 5, 5, 2, 22, -4, -4, -13, + -10, 22, 22, 17, 22, 22, 22, 22, 4, -4, + -4, } var exprDef = [...]int{ - 0, -2, 1, -2, 3, 4, 5, 0, 0, 0, - 0, 48, 49, 39, 40, 41, 42, 43, 44, 45, - 46, 47, 0, 9, 26, 27, 28, 29, 0, 0, - 0, 0, 0, 0, 0, 0, 33, 0, 6, 8, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 30, 31, 0, 32, 0, 0, 0, 0, 0, 14, - 15, 10, 0, 0, 16, 17, 0, 0, 0, 0, - 50, 0, 34, 35, 36, 37, 38, 11, 13, 12, - 20, 21, 0, 0, 18, 19, 0, 52, 53, 22, - 24, 51, 23, 25, + 0, -2, 1, -2, 3, 4, 5, 0, 7, 0, + 0, 0, 53, 54, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 0, 42, 43, 0, 11, 28, 29, + 30, 31, 0, -2, 0, 0, 0, 0, 0, 0, + 0, 35, 0, 41, 8, 10, 6, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 32, 33, 0, + 34, 0, 0, 0, 0, 0, 16, 17, 12, 0, + 0, 18, 19, 0, 0, 0, 0, 55, 0, 36, + 37, 38, 39, 40, 13, 15, 14, 22, 23, 0, + 0, 20, 21, 0, 57, 58, 24, 26, 56, 25, + 27, } var exprTok1 = [...]int{ @@ -209,7 +226,7 @@ var exprTok2 = [...]int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, + 32, 33, 34, 35, 36, 37, } var exprTok3 = [...]int{ 0, @@ -574,223 +591,248 @@ exprdefault: case 5: exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.LogExpr = newMatcherExpr(exprDollar[1].Selector) + exprVAL.Expr = exprDollar[1].BinOpExpr } case 6: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.LogExpr = NewFilterExpr(exprDollar[1].LogExpr, exprDollar[2].Filter, exprDollar[3].str) + exprVAL.Expr = exprDollar[2].Expr } case 7: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.LogExpr = newMatcherExpr(exprDollar[1].Selector) + } + case 8: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.LogExpr = NewFilterExpr(exprDollar[1].LogExpr, exprDollar[2].Filter, exprDollar[3].str) + } + case 9: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogExpr = exprDollar[2].LogExpr } - case 10: + case 12: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LogRangeExpr = newLogRange(exprDollar[1].LogExpr, exprDollar[2].duration) } - case 11: + case 13: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogRangeExpr = addFilterToLogRangeExpr(exprDollar[1].LogRangeExpr, exprDollar[2].Filter, exprDollar[3].str) } - case 12: + case 14: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogRangeExpr = exprDollar[2].LogRangeExpr } - case 15: + case 17: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[3].LogRangeExpr, exprDollar[1].RangeOp) } - case 16: + case 18: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].RangeAggregationExpr, exprDollar[1].VectorOp, nil, nil) } - case 17: + case 19: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].VectorAggregationExpr, exprDollar[1].VectorOp, nil, nil) } - case 18: + case 20: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } - case 19: + case 21: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } - case 20: + case 22: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } - case 21: + case 23: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } - case 22: + case 24: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].VectorAggregationExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } - case 23: + case 25: exprDollar = exprS[exprpt-7 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } - case 24: + case 26: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].RangeAggregationExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } - case 25: + case 27: exprDollar = exprS[exprpt-7 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } - case 26: + case 28: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchRegexp } - case 27: + case 29: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchEqual } - case 28: + case 30: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchNotRegexp } - case 29: + case 31: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchNotEqual } - case 30: + case 32: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Selector = exprDollar[2].Matchers } - case 31: + case 33: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Selector = exprDollar[2].Matchers } - case 32: + case 34: exprDollar = exprS[exprpt-3 : exprpt+1] { } - case 33: + case 35: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Matchers = []*labels.Matcher{exprDollar[1].Matcher} } - case 34: + case 36: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matchers = append(exprDollar[1].Matchers, exprDollar[3].Matcher) } - case 35: + case 37: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchEqual, exprDollar[1].str, exprDollar[3].str) } - case 36: + case 38: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchNotEqual, exprDollar[1].str, exprDollar[3].str) } - case 37: + case 39: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchRegexp, exprDollar[1].str, exprDollar[3].str) } - case 38: + case 40: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchNotRegexp, exprDollar[1].str, exprDollar[3].str) } - case 39: + case 41: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr(exprDollar[2].BinOp, exprDollar[1].Expr, exprDollar[3].Expr) + } + case 42: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.BinOp = OpTypeAdd + } + case 43: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.BinOp = OpTypeDiv + } + case 44: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 40: + case 45: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 41: + case 46: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 42: + case 47: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 43: + case 48: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 44: + case 49: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 45: + case 50: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 46: + case 51: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 47: + case 52: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 48: + case 53: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 49: + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 50: + case 55: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 51: + case 56: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 52: + case 57: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 53: + case 58: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index d368e7b69e9a..c3812876872a 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -36,6 +36,8 @@ var tokens = map[string]int{ OpTypeStdvar: STDVAR, OpTypeBottomK: BOTTOMK, OpTypeTopK: TOPK, + "/": DIV, + "+": ADD, } type lexer struct { diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 97b5b3742783..6eab9b23568d 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -534,7 +534,7 @@ func TestParse(t *testing.T) { { in: `{foo="bar"} "foo"`, err: ParseError{ - msg: "syntax error: unexpected STRING, expecting != or !~ or |~ or |=", + msg: "syntax error: unexpected STRING", line: 1, col: 13, }, @@ -542,7 +542,7 @@ func TestParse(t *testing.T) { { in: `{foo="bar"} foo`, err: ParseError{ - msg: "syntax error: unexpected IDENTIFIER, expecting != or !~ or |~ or |=", + msg: "syntax error: unexpected IDENTIFIER", line: 1, col: 13, }, From d55c587cd3a9734c67f1e82a6935e260d98de2cd Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 09:24:16 -0500 Subject: [PATCH 02/24] bin op associativity & precedence --- pkg/logql/expr.y | 19 ++---- pkg/logql/expr.y.go | 123 +++++++++++++++++++------------------- pkg/logql/parser_test.go | 124 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 78 deletions(-) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 32c30af2ff57..76f22dffc174 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -22,7 +22,7 @@ import ( VectorAggregationExpr SampleExpr VectorOp string BinOpExpr SampleExpr - BinOp string + binOp string str string duration time.Duration int int64 @@ -44,7 +44,6 @@ import ( %type vectorAggregationExpr %type vectorOp %type binOpExpr -%type binOp %token IDENTIFIER STRING %token DURATION @@ -52,12 +51,8 @@ import ( OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK // Operators are listed with increasing precedence. -%token -ADD -DIV - -%left ADD -%left DIV +%left ADD +%left DIV %% @@ -132,12 +127,8 @@ matcher: // TODO(owen-d): add (on,ignoring) clauses to binOpExpr // https://prometheus.io/docs/prometheus/latest/querying/operators/ binOpExpr: - expr binOp expr { $$ = mustNewBinOpExpr($2, $1, $3) } - -binOp: - ADD { $$ = OpTypeAdd } - | DIV { $$ = OpTypeDiv } - + expr ADD expr { $$ = mustNewBinOpExpr("+", $1, $3) } + | expr DIV expr { $$ = mustNewBinOpExpr("/", $1, $3) } vectorOp: SUM { $$ = OpTypeSum } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index c6b56d3c8826..ace0d61aab98 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -26,7 +26,7 @@ type exprSymType struct { VectorAggregationExpr SampleExpr VectorOp string BinOpExpr SampleExpr - BinOp string + binOp string str string duration time.Duration int int64 @@ -123,7 +123,7 @@ var exprExca = [...]int{ 36, 2, 37, 2, -2, 0, - -1, 33, + -1, 32, 36, 2, 37, 2, -2, 0, @@ -131,45 +131,45 @@ var exprExca = [...]int{ const exprPrivate = 57344 -const exprLast = 156 +const exprLast = 157 var exprAct = [...]int{ - 36, 5, 4, 26, 41, 76, 11, 3, 48, 24, - 25, 37, 38, 97, 7, 33, 46, 96, 12, 13, - 14, 15, 17, 18, 16, 19, 20, 21, 22, 92, - 24, 25, 35, 66, 37, 38, 91, 52, 51, 2, - 93, 31, 49, 30, 93, 95, 72, 32, 71, 94, - 28, 29, 65, 86, 56, 55, 75, 74, 69, 70, - 53, 27, 78, 43, 79, 68, 54, 34, 73, 31, - 83, 30, 87, 88, 65, 89, 90, 60, 28, 29, + 35, 5, 4, 25, 40, 76, 11, 3, 48, 23, + 24, 24, 36, 37, 7, 32, 46, 97, 12, 13, + 14, 15, 17, 18, 16, 19, 20, 21, 22, 96, + 23, 24, 34, 93, 36, 37, 52, 51, 95, 93, + 92, 49, 2, 91, 94, 72, 71, 11, 56, 55, + 31, 54, 65, 33, 73, 50, 75, 74, 69, 70, + 53, 26, 78, 60, 79, 68, 42, 43, 83, 30, + 82, 29, 87, 88, 65, 89, 90, 81, 27, 28, 58, 12, 13, 14, 15, 17, 18, 16, 19, 20, - 21, 22, 57, 85, 82, 59, 84, 99, 100, 12, + 21, 22, 57, 85, 80, 59, 84, 99, 100, 12, 13, 14, 15, 17, 18, 16, 19, 20, 21, 22, - 27, 11, 81, 45, 68, 66, 44, 80, 31, 50, - 30, 27, 40, 31, 42, 30, 27, 28, 29, 31, - 47, 30, 28, 29, 31, 67, 30, 98, 28, 29, - 23, 47, 42, 28, 29, 61, 62, 63, 64, 77, - 6, 10, 8, 9, 39, 1, + 26, 98, 6, 45, 68, 66, 44, 39, 30, 41, + 29, 66, 41, 30, 77, 29, 26, 27, 28, 30, + 47, 29, 27, 28, 30, 86, 29, 26, 27, 28, + 10, 67, 8, 27, 28, 30, 47, 29, 61, 62, + 63, 64, 9, 38, 27, 28, 1, } var exprPact = [...]int{ - -7, -1000, -27, 124, -1000, -1000, -1000, -7, -1000, 46, - 11, 120, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -7, -1000, -1000, 111, -1000, -1000, -1000, - -1000, -1000, -6, 119, 98, 56, 45, 34, 33, 78, - 63, -1000, 136, -27, -1000, -1000, -1000, -1000, 113, 59, - 98, 26, 24, 51, 74, 145, 145, -1000, -1000, 138, - -1000, 112, 107, 89, 65, 91, -1000, -1000, -1000, 108, - 31, -12, -12, 74, 14, 7, 27, -1000, 23, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -5, - -9, -1000, -1000, 133, -1000, -1000, -12, -12, -1000, -1000, + -7, -1000, -27, 135, -1000, -1000, -1000, -7, -1000, 32, + 11, 115, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -7, -7, 111, -1000, -1000, -1000, -1000, + -1000, -6, 124, 34, 56, 30, 28, 27, 78, 49, + -1000, 139, -26, -1000, -1000, -1000, -1000, -1000, 119, 59, + 34, 24, 23, 37, 74, 120, 120, -1000, -1000, 118, + -1000, 89, 72, 65, 63, 91, -1000, -1000, -1000, 108, + 113, -11, -11, 74, 21, 18, 22, -1000, 16, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 7, + -5, -1000, -1000, 107, -1000, -1000, -11, -11, -1000, -1000, -1000, } var exprPgo = [...]int{ - 0, 155, 39, 3, 0, 5, 7, 8, 4, 154, - 2, 153, 152, 1, 151, 150, 140, + 0, 156, 42, 3, 0, 5, 7, 8, 4, 153, + 2, 152, 142, 1, 140, 112, } var exprR1 = [...]int{ @@ -177,8 +177,8 @@ var exprR1 = [...]int{ 6, 6, 7, 7, 7, 7, 7, 10, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, - 8, 15, 16, 16, 14, 14, 14, 14, 14, 14, - 14, 14, 14, 11, 11, 5, 5, 4, 4, + 8, 15, 15, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 11, 11, 5, 5, 4, 4, } var exprR2 = [...]int{ @@ -186,16 +186,16 @@ var exprR2 = [...]int{ 3, 2, 2, 3, 3, 3, 2, 4, 4, 4, 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, - 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 3, 4, 4, + 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 3, 4, 4, } var exprChk = [...]int{ -1000, -1, -2, -6, -10, -13, -15, 21, -12, -11, -14, 13, 25, 26, 27, 28, 31, 29, 30, 32, - 33, 34, 35, -16, 36, 37, -3, 2, 19, 20, - 12, 10, -2, -6, 21, 21, -4, 23, 24, -9, - 2, -8, 4, -2, 5, 2, 22, 22, -7, -6, + 33, 34, 35, 36, 37, -3, 2, 19, 20, 12, + 10, -2, -6, 21, 21, -4, 23, 24, -9, 2, + -8, 4, -2, -2, 5, 2, 22, 22, -7, -6, 21, -10, -13, 4, 21, 21, 21, 14, 2, 17, 14, 9, 10, 11, 12, -3, 2, 22, 6, -6, -7, 22, 22, 17, -10, -13, -5, 4, -5, -8, @@ -206,15 +206,15 @@ var exprChk = [...]int{ var exprDef = [...]int{ 0, -2, 1, -2, 3, 4, 5, 0, 7, 0, - 0, 0, 53, 54, 44, 45, 46, 47, 48, 49, - 50, 51, 52, 0, 42, 43, 0, 11, 28, 29, - 30, 31, 0, -2, 0, 0, 0, 0, 0, 0, - 0, 35, 0, 41, 8, 10, 6, 9, 0, 0, + 0, 0, 52, 53, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 0, 0, 0, 11, 28, 29, 30, + 31, 0, -2, 0, 0, 0, 0, 0, 0, 0, + 35, 0, 41, 42, 8, 10, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 33, 0, 34, 0, 0, 0, 0, 0, 16, 17, 12, 0, - 0, 18, 19, 0, 0, 0, 0, 55, 0, 36, + 0, 18, 19, 0, 0, 0, 0, 54, 0, 36, 37, 38, 39, 40, 13, 15, 14, 22, 23, 0, - 0, 20, 21, 0, 57, 58, 24, 26, 56, 25, + 0, 20, 21, 0, 56, 57, 24, 26, 55, 25, 27, } var exprTok1 = [...]int{ @@ -750,89 +750,84 @@ exprdefault: case 41: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BinOpExpr = mustNewBinOpExpr(exprDollar[2].BinOp, exprDollar[1].Expr, exprDollar[3].Expr) + exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[1].Expr, exprDollar[3].Expr) } case 42: - exprDollar = exprS[exprpt-1 : exprpt+1] + exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BinOp = OpTypeAdd + exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[1].Expr, exprDollar[3].Expr) } case 43: - exprDollar = exprS[exprpt-1 : exprpt+1] - { - exprVAL.BinOp = OpTypeDiv - } - case 44: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 45: + case 44: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 46: + case 45: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 47: + case 46: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 48: + case 47: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 49: + case 48: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 50: + case 49: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 51: + case 50: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 52: + case 51: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 53: + case 52: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 54: + case 53: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 55: + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 56: + case 55: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 57: + case 56: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 58: + case 57: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 6eab9b23568d..11c92081c55e 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -547,6 +547,130 @@ func TestParse(t *testing.T) { col: 13, }, }, + { + // require left associativity + in: ` +sum(count_over_time({foo="bar"}[5m])) by (foo) / +sum(count_over_time({foo="bar"}[5m])) by (foo) / +sum(count_over_time({foo="bar"}[5m])) by (foo) +`, + exp: mustNewBinOpExpr( + OpTypeDiv, + mustNewBinOpExpr( + OpTypeDiv, + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + }, + { + // operator precedence before left associativity + in: ` +sum(count_over_time({foo="bar"}[5m])) by (foo) + +sum(count_over_time({foo="bar"}[5m])) by (foo) / +sum(count_over_time({foo="bar"}[5m])) by (foo) +`, + exp: mustNewBinOpExpr( + OpTypeAdd, + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewBinOpExpr( + OpTypeDiv, + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + ), + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) From 519d226d3cfcf0c76eb3927befd0e1310dd12f82 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 11:50:34 -0500 Subject: [PATCH 03/24] binOpEvaluator work --- pkg/logql/evaluator.go | 100 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 589fdf802ebd..444cc2a65970 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -94,6 +94,8 @@ func (ev *defaultEvaluator) Evaluator(ctx context.Context, expr SampleExpr, q Pa return ev.vectorAggEvaluator(ctx, e, q) case *rangeAggregationExpr: return ev.rangeAggEvaluator(ctx, e, q) + case *binOpExpr: + return ev.binOpEvaluator(ctx, e, q) default: return nil, errors.Errorf("unexpected type (%T): %v", e, e) @@ -325,3 +327,101 @@ func (ev *defaultEvaluator) rangeAggEvaluator(ctx context.Context, expr *rangeAg }, vecIter.Close) } + +func (ev *defaultEvaluator) binOpEvaluator( + ctx context.Context, + expr *binOpExpr, + q Params, +) (StepEvaluator, error) { + lhs, err := ev.Evaluator(ctx, expr.SampleExpr, q) + if err != nil { + return nil, err + } + rhs, err := ev.Evaluator(ctx, expr.RHS, q) + if err != nil { + return nil, err + } + + return newStepEvaluator(func() (bool, int64, promql.Vector) { + pairs := map[uint64]promql.Vector{} + var ts int64 + + // populate pairs + for _, eval := range []StepEvaluator{lhs, rhs} { + next, timestamp, vec := eval.Next() + ts = timestamp + + // These should _always_ happen at the same step on each evaluator. + if !next { + return next, ts, nil + } + + for _, sample := range vec { + // TODO(owen-d): this seems wildly inefficient: we're calculating + // the hash on each sample & step per evaluator. + // We seem limited to this approach due to using the StepEvaluator ifc. + hash := sample.Metric.Hash() + pair, ok := pairs[hash] + if !ok { + pair = make(promql.Vector, 0, 2) + } + pairs[hash] = append(pair, sample) + + } + } + + results := make(promql.Vector, 0, len(pairs)) + for _, pair := range pairs { + // merge + results = append(results, ev.mergeBinOp(expr.op, pair[0], pair[1:]...)) + } + + return true, ts, results + }, func() (lastError error) { + for _, ev := range []StepEvaluator{lhs, rhs} { + if err := ev.Close(); err != nil { + lastError = err + } + } + return lastError + }) +} + +func (ev *defaultEvaluator) mergeBinOp(op string, first promql.Sample, rest ...promql.Sample) promql.Sample { + // mergers should be able to handle nil rests (think of merging vectors where a series is only present in one), + // but will only ever be passed two samples for merging. We use the (...promql.Sample) type + // in the function signature as it handles both empty and 1-length vectors. + var merger func(first promql.Sample, rest ...promql.Sample) promql.Sample + + switch op { + case OpTypeAdd: + merger = func(first promql.Sample, rest ...promql.Sample) promql.Sample { + res := promql.Sample{ + Metric: first.Metric.Copy(), + Point: first.Point, + } + for _, sample := range rest { + res.Point.V += sample.Point.V + } + return res + } + case OpTypeDiv: + merger = func(first promql.Sample, rest ...promql.Sample) promql.Sample { + res := promql.Sample{ + Metric: first.Metric.Copy(), + Point: first.Point, + } + + for _, sample := range rest { + res.Point.V /= sample.Point.V + } + return res + } + + default: + panic(errors.Errorf("should never happen: unexpected operation: (%s)", op)) + } + + return merger(first, rest...) + +} From 05809bd5eda2de6b14f0338df8e346e8d6f9ab30 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 14:05:59 -0500 Subject: [PATCH 04/24] defers close only if constructed without error --- pkg/logql/engine.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index fab60ff2fcd2..a479df76e8c5 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -214,10 +214,11 @@ func (ng *engine) exec(ctx context.Context, q *query) (promql.Value, error) { func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (promql.Value, error) { stepEvaluator, err := ng.evaluator.Evaluator(ctx, expr, q) - defer helpers.LogError("closing SampleExpr", stepEvaluator.Close) if err != nil { return nil, err } + defer helpers.LogError("closing SampleExpr", stepEvaluator.Close) + seriesIndex := map[uint64]*promql.Series{} next, ts, vec := stepEvaluator.Next() From d6c2217c9e8d3756fcd649040d0d555b39852e07 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 14:14:42 -0500 Subject: [PATCH 05/24] tests binary ops --- pkg/logql/engine_test.go | 87 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index d8717ad485e1..321597707d23 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -653,13 +653,17 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, { - `bottomk(3,rate(({app=~"foo|bar"} |~".+bar")[1m])) without (app)`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + `bottomk(3,rate(({app=~"foo|bar|fuzz|buzz"} |~".+bar")[1m])) without (app)`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ - {newStream(testSize, factor(10, identity), `{app="foo"}`), newStream(testSize, factor(20, identity), `{app="bar"}`), - newStream(testSize, factor(5, identity), `{app="fuzz"}`), newStream(testSize, identity, `{app="buzz"}`)}, + { + newStream(testSize, factor(10, identity), `{app="foo"}`), + newStream(testSize, factor(20, identity), `{app="bar"}`), + newStream(testSize, factor(5, identity), `{app="fuzz"}`), + newStream(testSize, identity, `{app="buzz"}`), + }, }, []SelectParams{ - {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar|fuzz|buzz"}|~".+bar"`}}, }, promql.Matrix{ promql.Series{ @@ -676,6 +680,81 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, }, + // test binary ops + { + `sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m]))`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + }, + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + }, + }, + }, + { + `sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m]))`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}}, + }, + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}}, + }, + }, + }, + // tests precedence: should be x + (x/x) + { + ` +sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + +sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / +sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) +`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1.2}, {T: 90 * 1000, V: 1.2}, {T: 120 * 1000, V: 1.2}, {T: 150 * 1000, V: 1.2}, {T: 180 * 1000, V: 1.2}}, + }, + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1.2}, {T: 90 * 1000, V: 1.2}, {T: 120 * 1000, V: 1.2}, {T: 150 * 1000, V: 1.2}, {T: 180 * 1000, V: 1.2}}, + }, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { From 7ea39b9dfe00c7ff72f9f7124bf804e8e76a7a22 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 17:32:11 -0500 Subject: [PATCH 06/24] more binops --- pkg/logql/ast.go | 10 +- pkg/logql/engine_test.go | 215 ++++++++++++++++++++++++++++++++++-- pkg/logql/evaluator.go | 131 +++++++++++++++++----- pkg/logql/evaluator_test.go | 31 ++++++ pkg/logql/expr.y | 21 +++- pkg/logql/expr.y.go | 210 +++++++++++++++++++++++------------ pkg/logql/lex.go | 12 +- 7 files changed, 510 insertions(+), 120 deletions(-) create mode 100644 pkg/logql/evaluator_test.go diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 014e5ed46683..c1716922ab91 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -229,8 +229,14 @@ const ( OpTypeRate = "rate" // binops - OpTypeAdd = "+" - OpTypeDiv = "/" + OpTypeOr = "or" + OpTypeAnd = "and" + OpTypeUnless = "unless" + OpTypeAdd = "+" + OpTypeSub = "-" + OpTypeMul = "*" + OpTypeDiv = "/" + OpTypeMod = "%" ) // SampleExpr is a LogQL expression filtering logs and returning metric samples. diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 321597707d23..44c6b7805d2a 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -680,60 +680,251 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, }, - // test binary ops + // binops { - `sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m]))`, + ` +rate({app="foo"}[1m]) or +rate({app="bar"}[1m]) +`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ { newStream(testSize, factor(5, identity), `{app="foo"}`), + }, + { newStream(testSize, factor(5, identity), `{app="bar"}`), }, }, []SelectParams{ - {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="foo"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, }, promql.Matrix{ promql.Series{ Metric: labels.Labels{{Name: "app", Value: "bar"}}, - Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, }, promql.Series{ Metric: labels.Labels{{Name: "app", Value: "foo"}}, - Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, }, }, }, { - `sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m]))`, + ` +rate({app=~"foo|bar"}[1m]) and +rate({app="bar"}[1m]) +`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ { newStream(testSize, factor(5, identity), `{app="foo"}`), newStream(testSize, factor(5, identity), `{app="bar"}`), }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, }, []SelectParams{ - {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, }, promql.Matrix{ promql.Series{ Metric: labels.Labels{{Name: "app", Value: "bar"}}, - Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, + }, + }, + }, + { + ` + rate({app=~"foo|bar"}[1m]) unless + rate({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ promql.Series{ Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, + }, + }, + }, + { + ` + rate({app=~"foo|bar"}[1m]) + + rate({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}}, }, }, }, + { + ` + rate({app=~"foo|bar"}[1m]) - + rate({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0}, {T: 90 * 1000, V: 0}, {T: 120 * 1000, V: 0}, {T: 150 * 1000, V: 0}, {T: 180 * 1000, V: 0}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) * + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 144}, {T: 90 * 1000, V: 144}, {T: 120 * 1000, V: 144}, {T: 150 * 1000, V: 144}, {T: 180 * 1000, V: 144}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) * + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 144}, {T: 90 * 1000, V: 144}, {T: 120 * 1000, V: 144}, {T: 150 * 1000, V: 144}, {T: 180 * 1000, V: 144}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) / + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) % + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0}, {T: 90 * 1000, V: 0}, {T: 120 * 1000, V: 0}, {T: 150 * 1000, V: 0}, {T: 180 * 1000, V: 0}}, + }, + }, + }, // tests precedence: should be x + (x/x) { ` -sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + -sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / -sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) -`, + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + `, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ { diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 444cc2a65970..e740f6990176 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -343,11 +343,11 @@ func (ev *defaultEvaluator) binOpEvaluator( } return newStepEvaluator(func() (bool, int64, promql.Vector) { - pairs := map[uint64]promql.Vector{} + pairs := map[uint64][2]*promql.Sample{} var ts int64 // populate pairs - for _, eval := range []StepEvaluator{lhs, rhs} { + for i, eval := range []StepEvaluator{lhs, rhs} { next, timestamp, vec := eval.Next() ts = timestamp @@ -361,19 +361,22 @@ func (ev *defaultEvaluator) binOpEvaluator( // the hash on each sample & step per evaluator. // We seem limited to this approach due to using the StepEvaluator ifc. hash := sample.Metric.Hash() - pair, ok := pairs[hash] - if !ok { - pair = make(promql.Vector, 0, 2) + pair := pairs[hash] + pair[i] = &promql.Sample{ + Metric: sample.Metric, + Point: sample.Point, } - pairs[hash] = append(pair, sample) - + pairs[hash] = pair } } results := make(promql.Vector, 0, len(pairs)) for _, pair := range pairs { + // merge - results = append(results, ev.mergeBinOp(expr.op, pair[0], pair[1:]...)) + if merged := ev.mergeBinOp(expr.op, pair[0], pair[1]); merged != nil { + results = append(results, *merged) + } } return true, ts, results @@ -387,41 +390,117 @@ func (ev *defaultEvaluator) binOpEvaluator( }) } -func (ev *defaultEvaluator) mergeBinOp(op string, first promql.Sample, rest ...promql.Sample) promql.Sample { - // mergers should be able to handle nil rests (think of merging vectors where a series is only present in one), - // but will only ever be passed two samples for merging. We use the (...promql.Sample) type - // in the function signature as it handles both empty and 1-length vectors. - var merger func(first promql.Sample, rest ...promql.Sample) promql.Sample +func (ev *defaultEvaluator) mergeBinOp(op string, left, right *promql.Sample) *promql.Sample { + var merger func(left, right *promql.Sample) *promql.Sample switch op { + case OpTypeOr: + merger = func(left, right *promql.Sample) *promql.Sample { + // return the left entry found (prefers left hand side) + if left != nil { + return left + } + return right + } + + case OpTypeAnd: + merger = func(left, right *promql.Sample) *promql.Sample { + // return left sample if there's a second sample for that label set + if left != nil && right != nil { + return left + } + return nil + } + + case OpTypeUnless: + merger = func(left, right *promql.Sample) *promql.Sample { + // return left sample if there's not a second sample for that label set + if right == nil { + return left + } + return nil + } + case OpTypeAdd: - merger = func(first promql.Sample, rest ...promql.Sample) promql.Sample { + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, + } + res.Point.V += right.Point.V + return &res + } + + case OpTypeSub: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } res := promql.Sample{ - Metric: first.Metric.Copy(), - Point: first.Point, + Metric: left.Metric, + Point: left.Point, + } + res.Point.V -= right.Point.V + return &res + } + + case OpTypeMul: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil } - for _, sample := range rest { - res.Point.V += sample.Point.V + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, } - return res + res.Point.V *= right.Point.V + return &res } + case OpTypeDiv: - merger = func(first promql.Sample, rest ...promql.Sample) promql.Sample { + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } res := promql.Sample{ - Metric: first.Metric.Copy(), - Point: first.Point, + Metric: left.Metric.Copy(), + Point: left.Point, } - for _, sample := range rest { - res.Point.V /= sample.Point.V + // guard against divide by zero + if right.Point.V == 0 { + res.Point.V = math.NaN() + } else { + res.Point.V /= right.Point.V + } + return &res + } + + case OpTypeMod: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, + } + // guard against divide by zero + if right.Point.V == 0 { + res.Point.V = math.NaN() + } else { + res.Point.V = math.Mod(res.Point.V, right.Point.V) } - return res + return &res } default: panic(errors.Errorf("should never happen: unexpected operation: (%s)", op)) } - return merger(first, rest...) + return merger(left, right) } diff --git a/pkg/logql/evaluator_test.go b/pkg/logql/evaluator_test.go new file mode 100644 index 000000000000..23ef3d2f71c9 --- /dev/null +++ b/pkg/logql/evaluator_test.go @@ -0,0 +1,31 @@ +package logql + +import ( + "math" + "testing" + + "github.com/prometheus/prometheus/promql" + "github.com/stretchr/testify/require" +) + +func TestDefaultEvaluator_DivideByZero(t *testing.T) { + ev := &defaultEvaluator{} + + require.Equal(t, true, math.IsNaN(ev.mergeBinOp(OpTypeDiv, + &promql.Sample{ + Point: promql.Point{T: 1, V: 1}, + }, + &promql.Sample{ + Point: promql.Point{T: 1, V: 0}, + }, + ).Point.V)) + + require.Equal(t, true, math.IsNaN(ev.mergeBinOp(OpTypeMod, + &promql.Sample{ + Point: promql.Point{T: 1, V: 1}, + }, + &promql.Sample{ + Point: promql.Point{T: 1, V: 0}, + }, + ).Point.V)) +} diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 76f22dffc174..2e8e68d725c8 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -51,8 +51,10 @@ import ( OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK // Operators are listed with increasing precedence. -%left ADD -%left DIV +%left OR +%left AND UNLESS +%left ADD SUB +%left MUL DIV MOD %% @@ -125,10 +127,19 @@ matcher: ; // TODO(owen-d): add (on,ignoring) clauses to binOpExpr -// https://prometheus.io/docs/prometheus/latest/querying/operators/ +// Comparison operators are currently avoided due to symbol collisions in our grammar: "!=" means not equal in prometheus, +// but is part of our filter grammar. +// reference: https://prometheus.io/docs/prometheus/latest/querying/operators/ +// Operator precedence only works if each of these is listed separately. binOpExpr: - expr ADD expr { $$ = mustNewBinOpExpr("+", $1, $3) } - | expr DIV expr { $$ = mustNewBinOpExpr("/", $1, $3) } + expr OR expr { $$ = mustNewBinOpExpr("or", $1, $3) } + | expr AND expr { $$ = mustNewBinOpExpr("and", $1, $3) } + | expr UNLESS expr { $$ = mustNewBinOpExpr("unless", $1, $3) } + | expr ADD expr { $$ = mustNewBinOpExpr("+", $1, $3) } + | expr SUB expr { $$ = mustNewBinOpExpr("-", $1, $3) } + | expr MUL expr { $$ = mustNewBinOpExpr("*", $1, $3) } + | expr DIV expr { $$ = mustNewBinOpExpr("/", $1, $3) } + | expr MOD expr { $$ = mustNewBinOpExpr("%", $1, $3) } vectorOp: SUM { $$ = OpTypeSum } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index ace0d61aab98..72c1a935ecaf 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -64,8 +64,14 @@ const STDDEV = 57374 const STDVAR = 57375 const BOTTOMK = 57376 const TOPK = 57377 -const ADD = 57378 -const DIV = 57379 +const OR = 57378 +const AND = 57379 +const UNLESS = 57380 +const ADD = 57381 +const SUB = 57382 +const MUL = 57383 +const DIV = 57384 +const MOD = 57385 var exprToknames = [...]string{ "$end", @@ -103,8 +109,14 @@ var exprToknames = [...]string{ "STDVAR", "BOTTOMK", "TOPK", + "OR", + "AND", + "UNLESS", "ADD", + "SUB", + "MUL", "DIV", + "MOD", } var exprStatenames = [...]string{} @@ -122,54 +134,71 @@ var exprExca = [...]int{ 22, 2, 36, 2, 37, 2, + 38, 2, + 39, 2, + 40, 2, + 41, 2, + 42, 2, + 43, 2, -2, 0, - -1, 32, + -1, 38, 36, 2, 37, 2, + 38, 2, + 39, 2, + 40, 2, + 41, 2, + 42, 2, + 43, 2, -2, 0, } const exprPrivate = 57344 -const exprLast = 157 +const exprLast = 199 var exprAct = [...]int{ - 35, 5, 4, 25, 40, 76, 11, 3, 48, 23, - 24, 24, 36, 37, 7, 32, 46, 97, 12, 13, - 14, 15, 17, 18, 16, 19, 20, 21, 22, 96, - 23, 24, 34, 93, 36, 37, 52, 51, 95, 93, - 92, 49, 2, 91, 94, 72, 71, 11, 56, 55, - 31, 54, 65, 33, 73, 50, 75, 74, 69, 70, - 53, 26, 78, 60, 79, 68, 42, 43, 83, 30, - 82, 29, 87, 88, 65, 89, 90, 81, 27, 28, - 58, 12, 13, 14, 15, 17, 18, 16, 19, 20, - 21, 22, 57, 85, 80, 59, 84, 99, 100, 12, - 13, 14, 15, 17, 18, 16, 19, 20, 21, 22, - 26, 98, 6, 45, 68, 66, 44, 39, 30, 41, - 29, 66, 41, 30, 77, 29, 26, 27, 28, 30, - 47, 29, 27, 28, 30, 86, 29, 26, 27, 28, - 10, 67, 8, 27, 28, 30, 47, 29, 61, 62, - 63, 64, 9, 38, 27, 28, 1, + 41, 5, 4, 31, 46, 88, 58, 3, 60, 26, + 27, 28, 29, 30, 105, 38, 28, 29, 30, 107, + 23, 24, 25, 26, 27, 28, 29, 30, 23, 24, + 25, 26, 27, 28, 29, 30, 40, 68, 42, 43, + 42, 43, 64, 63, 109, 108, 104, 61, 24, 25, + 26, 27, 28, 29, 30, 32, 105, 103, 84, 80, + 83, 106, 67, 36, 77, 35, 66, 39, 87, 86, + 81, 82, 33, 34, 90, 59, 91, 85, 72, 97, + 95, 11, 96, 11, 99, 100, 77, 101, 102, 7, + 65, 62, 70, 12, 13, 14, 15, 17, 18, 16, + 19, 20, 21, 22, 69, 57, 94, 71, 56, 111, + 112, 12, 13, 14, 15, 17, 18, 16, 19, 20, + 21, 22, 12, 13, 14, 15, 17, 18, 16, 19, + 20, 21, 22, 2, 78, 93, 92, 45, 32, 47, + 6, 37, 36, 110, 35, 47, 36, 89, 35, 10, + 8, 33, 34, 9, 98, 33, 34, 48, 49, 50, + 51, 52, 53, 54, 55, 78, 44, 1, 0, 32, + 0, 0, 0, 36, 0, 35, 0, 36, 0, 35, + 32, 0, 33, 34, 80, 79, 33, 34, 36, 59, + 35, 73, 74, 75, 76, 0, 0, 33, 34, } var exprPact = [...]int{ - -7, -1000, -27, 135, -1000, -1000, -1000, -7, -1000, 32, - 11, 115, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -7, -7, 111, -1000, -1000, -1000, -1000, - -1000, -6, 124, 34, 56, 30, 28, 27, 78, 49, - -1000, 139, -26, -1000, -1000, -1000, -1000, -1000, 119, 59, - 34, 24, 23, 37, 74, 120, 120, -1000, -1000, 118, - -1000, 89, 72, 65, 63, 91, -1000, -1000, -1000, 108, - 113, -11, -11, 74, 21, 18, 22, -1000, 16, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 7, - -5, -1000, -1000, 107, -1000, -1000, -11, -11, -1000, -1000, - -1000, + 68, -1000, -8, 136, -1000, -1000, -1000, 68, -1000, 46, + 15, 135, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 68, 68, 68, 68, 68, 68, 68, + 68, 103, -1000, -1000, -1000, -1000, -1000, -16, 167, 70, + 86, 45, 41, 16, 90, 64, -1000, 182, 11, -30, + -30, -25, -25, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 163, 178, 70, 38, 36, 60, 97, 143, 143, -1000, + -1000, 141, -1000, 131, 130, 101, 75, 77, -1000, -1000, + -1000, 53, 132, 17, 17, 97, 35, 24, 39, -1000, + -3, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 23, 22, -1000, -1000, 139, -1000, -1000, 17, 17, + -1000, -1000, -1000, } var exprPgo = [...]int{ - 0, 156, 42, 3, 0, 5, 7, 8, 4, 153, - 2, 152, 142, 1, 140, 112, + 0, 167, 133, 3, 0, 5, 7, 8, 4, 166, + 2, 153, 150, 1, 149, 140, } var exprR1 = [...]int{ @@ -177,8 +206,9 @@ var exprR1 = [...]int{ 6, 6, 7, 7, 7, 7, 7, 10, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, - 8, 15, 15, 14, 14, 14, 14, 14, 14, 14, - 14, 14, 11, 11, 5, 5, 4, 4, + 8, 15, 15, 15, 15, 15, 15, 15, 15, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 11, 11, + 5, 5, 4, 4, } var exprR2 = [...]int{ @@ -186,36 +216,39 @@ var exprR2 = [...]int{ 3, 2, 2, 3, 3, 3, 2, 4, 4, 4, 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, - 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 3, 4, 4, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 3, 4, 4, } var exprChk = [...]int{ -1000, -1, -2, -6, -10, -13, -15, 21, -12, -11, -14, 13, 25, 26, 27, 28, 31, 29, 30, 32, - 33, 34, 35, 36, 37, -3, 2, 19, 20, 12, - 10, -2, -6, 21, 21, -4, 23, 24, -9, 2, - -8, 4, -2, -2, 5, 2, 22, 22, -7, -6, - 21, -10, -13, 4, 21, 21, 21, 14, 2, 17, - 14, 9, 10, 11, 12, -3, 2, 22, 6, -6, - -7, 22, 22, 17, -10, -13, -5, 4, -5, -8, - 5, 5, 5, 5, 5, 2, 22, -4, -4, -13, - -10, 22, 22, 17, 22, 22, 22, 22, 4, -4, - -4, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, -3, 2, 19, 20, 12, 10, -2, -6, 21, + 21, -4, 23, 24, -9, 2, -8, 4, -2, -2, + -2, -2, -2, -2, -2, -2, 5, 2, 22, 22, + -7, -6, 21, -10, -13, 4, 21, 21, 21, 14, + 2, 17, 14, 9, 10, 11, 12, -3, 2, 22, + 6, -6, -7, 22, 22, 17, -10, -13, -5, 4, + -5, -8, 5, 5, 5, 5, 5, 2, 22, -4, + -4, -13, -10, 22, 22, 17, 22, 22, 22, 22, + 4, -4, -4, } var exprDef = [...]int{ 0, -2, 1, -2, 3, 4, 5, 0, 7, 0, - 0, 0, 52, 53, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 0, 0, 0, 11, 28, 29, 30, - 31, 0, -2, 0, 0, 0, 0, 0, 0, 0, - 35, 0, 41, 42, 8, 10, 6, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 32, 33, 0, - 34, 0, 0, 0, 0, 0, 16, 17, 12, 0, - 0, 18, 19, 0, 0, 0, 0, 54, 0, 36, - 37, 38, 39, 40, 13, 15, 14, 22, 23, 0, - 0, 20, 21, 0, 56, 57, 24, 26, 55, 25, - 27, + 0, 0, 58, 59, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 11, 28, 29, 30, 31, 0, -2, 0, + 0, 0, 0, 0, 0, 0, 35, 0, 41, 42, + 43, 44, 45, 46, 47, 48, 8, 10, 6, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, + 33, 0, 34, 0, 0, 0, 0, 0, 16, 17, + 12, 0, 0, 18, 19, 0, 0, 0, 0, 60, + 0, 36, 37, 38, 39, 40, 13, 15, 14, 22, + 23, 0, 0, 20, 21, 0, 62, 63, 24, 26, + 61, 25, 27, } var exprTok1 = [...]int{ @@ -226,7 +259,8 @@ var exprTok2 = [...]int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, } var exprTok3 = [...]int{ 0, @@ -750,84 +784,114 @@ exprdefault: case 41: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[1].Expr, exprDollar[3].Expr) + exprVAL.BinOpExpr = mustNewBinOpExpr("or", exprDollar[1].Expr, exprDollar[3].Expr) } case 42: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[1].Expr, exprDollar[3].Expr) + exprVAL.BinOpExpr = mustNewBinOpExpr("and", exprDollar[1].Expr, exprDollar[3].Expr) } case 43: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("unless", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 44: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 45: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("-", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 46: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("*", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 47: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 48: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 49: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 44: + case 50: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 45: + case 51: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 46: + case 52: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 47: + case 53: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 48: + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 49: + case 55: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 50: + case 56: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 51: + case 57: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 52: + case 58: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 53: + case 59: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 54: + case 60: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 55: + case 61: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 56: + case 62: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 57: + case 63: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index c3812876872a..0d70fd551529 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -36,8 +36,16 @@ var tokens = map[string]int{ OpTypeStdvar: STDVAR, OpTypeBottomK: BOTTOMK, OpTypeTopK: TOPK, - "/": DIV, - "+": ADD, + + // binops + OpTypeOr: OR, + OpTypeAnd: AND, + OpTypeUnless: UNLESS, + OpTypeAdd: ADD, + OpTypeSub: SUB, + OpTypeMul: MUL, + OpTypeDiv: DIV, + OpTypeMod: MOD, } type lexer struct { From 3f193851aaf1b034bad4d67be862aa5c2783c986 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 17:58:31 -0500 Subject: [PATCH 07/24] updates docs --- docs/logql.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/logql.md b/docs/logql.md index ba5d2b2980e7..a100ed5b3530 100644 --- a/docs/logql.md +++ b/docs/logql.md @@ -151,3 +151,49 @@ by level: Get the rate of HTTP GET requests from NGINX logs: > `avg(rate(({job="nginx"} |= "GET")[10s])) by (region)` + +### Binary Operators + +#### Arithmetic Binary Operators + +Arithmetic binary operators +The following binary arithmetic operators exist in Loki: + +- + (addition) +- - (subtraction) +- * (multiplication) +- / (division) +- % (modulo) + +Binary arithmetic operators are defined only between two vectors. + +Between two instant vectors, a binary arithmetic operator is applied to each entry in the left-hand side vector and its matching element in the right-hand vector. The result is propagated into the result vector with the grouping labels becoming the output label set. Entries for which no matching entry in the right-hand vector can be found are not part of the result. + +##### Examples + +Get proportion of warning logs to error logs for the `foo` app + +> `sum(rate({app="foo", level="warn"}[1m])) / sum(rate({app="foo", level="error"}[1m]))` + +Operators on the same precedence level are left-associative (queries substituted with numbers here for simplicity). For example, 2 * 3 % 2 is equivalent to (2 * 3) % 2. However, some operators have different priorities: 1 + 2 / 3 will still be 1 + ( 2 / 3 ). These function identically to mathematical conventions. + + +#### Logical/set binary operators + +These logical/set binary operators are only defined between two vectors: + +- and (intersection) +- or (union) +- unless (complement) + +`vector1 and vector2` results in a vector consisting of the elements of vector1 for which there are elements in vector2 with exactly matching label sets. Other elements are dropped. + +`vector1 or vector2` results in a vector that contains all original elements (label sets + values) of vector1 and additionally all elements of vector2 which do not have matching label sets in vector1. + +`vector1 unless vector2` results in a vector consisting of the elements of vector1 for which there are no elements in vector2 with exactly matching label sets. All matching elements in both vectors are dropped. + +##### Examples + +This contrived query will return the intersection of these queries, effectively `rate({app="bar"})` + +> `rate({app=~"foo|bar"}[1m]) and rate({app="bar"}[1m])` From 0ec42b09ec16568777914d3bc10ebf3a89145d22 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 18:40:59 -0500 Subject: [PATCH 08/24] changelog --- CHANGELOG.md | 1 + docs/logql.md | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c910b53664e..52833472abde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features +* [1662](https://github.com/grafana/loki/pull/1662) **owen-d**: Introduces binary operators in LogQL * [1572](https://github.com/grafana/loki/pull/1572) **owen-d**: Introduces the `querier.query-ingesters-within` flag and associated yaml config. When enabled, queries for a time range that do not overlap this lookback interval will not be sent to the ingesters. * [1558](https://github.com/grafana/loki/pull/1558) **owen-d**: Introduces `ingester.max-chunk-age` which specifies the maximum chunk age before it's cut. * [1565](https://github.com/grafana/loki/pull/1565) **owen-d**: The query frontend's `split_queries_by_interval` can now be specified as an override diff --git a/docs/logql.md b/docs/logql.md index a100ed5b3530..5628cee63842 100644 --- a/docs/logql.md +++ b/docs/logql.md @@ -159,11 +159,11 @@ Get the rate of HTTP GET requests from NGINX logs: Arithmetic binary operators The following binary arithmetic operators exist in Loki: -- + (addition) -- - (subtraction) -- * (multiplication) -- / (division) -- % (modulo) +- `+` (addition) +- `-` (subtraction) +- `*` (multiplication) +- `/` (division) +- `%` (modulo) Binary arithmetic operators are defined only between two vectors. @@ -182,9 +182,9 @@ Operators on the same precedence level are left-associative (queries substituted These logical/set binary operators are only defined between two vectors: -- and (intersection) -- or (union) -- unless (complement) +- `and` (intersection) +- `or` (union) +- `unless` (complement) `vector1 and vector2` results in a vector consisting of the elements of vector1 for which there are elements in vector2 with exactly matching label sets. Other elements are dropped. From cee6860979325126edd479945cfdcad8cfffabbf Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Sat, 8 Feb 2020 08:49:53 -0500 Subject: [PATCH 09/24] number literals in ast --- pkg/logql/ast.go | 17 ++++++++++ pkg/logql/evaluator.go | 68 ++++++++++++++++++++++++++++++++++++++++ pkg/logql/parser_test.go | 27 ++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index c1716922ab91..41c25abc994f 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -407,6 +407,23 @@ func mustNewBinOpExpr(op string, lhs, rhs Expr) SampleExpr { } } +type literalExpr struct { + value float64 +} + +func (e *literalExpr) logQLExpr() {} + +func (e *literalExpr) String() string { + return fmt.Sprintf("%f", e.value) +} + +// literlExpr impls SampleExpr & LogSelectorExpr mainly to reduce the need for more complicated typings +// to facilitate sum types. We'll be type switching when evaluating them anyways +// and they will only be present in binary operation legs. +func (e *literalExpr) Selector() LogSelectorExpr { return e } +func (e *literalExpr) Filter() (Filter, error) { return nil, nil } +func (e *literalExpr) Matchers() []*labels.Matcher { return nil } + // helper used to impl Stringer for vector and range aggregations // nolint:interfacer func formatOperation(op string, grouping *grouping, params ...string) string { diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index e740f6990176..070e9d212998 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -333,6 +333,36 @@ func (ev *defaultEvaluator) binOpEvaluator( expr *binOpExpr, q Params, ) (StepEvaluator, error) { + // first check if either side is a literal + leftLit, lOk := expr.SampleExpr.(*literalExpr) + rightLit, rOk := expr.RHS.(*literalExpr) + + // turn expressions like 1+2 -> 3 + if lOk && rOk { + return ev.Evaluator( + ctx, + ev.reduceBinOp(expr.op, leftLit, rightLit), + q, + ) + } + + // match a literal expr with all labels in the other leg + if lOk { + rhs, err := ev.Evaluator(ctx, expr.RHS, q) + if err != nil { + return nil, err + } + return ev.literalEvaluator(expr.op, leftLit, rhs) + } + if rOk { + lhs, err := ev.Evaluator(ctx, expr.SampleExpr, q) + if err != nil { + return nil, err + } + return ev.literalEvaluator(expr.op, rightLit, lhs) + } + + // we have two non literal legs lhs, err := ev.Evaluator(ctx, expr.SampleExpr, q) if err != nil { return nil, err @@ -504,3 +534,41 @@ func (ev *defaultEvaluator) mergeBinOp(op string, left, right *promql.Sample) *p return merger(left, right) } + +// Reduces a binary operation expression. A binop is reducable if both of its legs are literal expressions. +// This is because literals need match all labels, which is currently difficult to encode into StepEvaluators. +// Therefore, we ensure a binop can be reduced/simplified, maintaining the invariant that it does not have two literal legs. +func (ev *defaultEvaluator) reduceBinOp(op string, left, right *literalExpr) SampleExpr { + merged := ev.mergeBinOp( + op, + &promql.Sample{Point: promql.Point{V: left.value}}, + &promql.Sample{Point: promql.Point{V: right.value}}, + ) + return &literalExpr{value: merged.V} +} + +// literalEvaluator merges a literal with a StepEvaluator +func (ev *defaultEvaluator) literalEvaluator(op string, lit *literalExpr, eval StepEvaluator) (StepEvaluator, error) { + return newStepEvaluator( + func() (bool, int64, promql.Vector) { + done, ts, vec := eval.Next() + + results := make(promql.Vector, 0, len(vec)) + for _, sample := range vec { + if merged := ev.mergeBinOp( + op, + &sample, + &promql.Sample{ + Metric: sample.Metric, + Point: promql.Point{T: ts, V: lit.value}, + }, + ); merged != nil { + results = append(results, *merged) + } + } + + return done, ts, results + }, + nil, + ) +} diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 11c92081c55e..de40c7a7029e 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -671,6 +671,33 @@ sum(count_over_time({foo="bar"}[5m])) by (foo) ), ), }, + { + // ensure literal binops are reduced: the 1+2/expr should reduce to 3/expr + in: ` +sum(count_over_time({foo="bar"}[5m])) by (foo) + 2 / 1 +`, + exp: mustNewBinOpExpr( + OpTypeAdd, + mustNewVectorAggregationExpr( + newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewBinOpExpr(OpTypeDiv, &literalExpr{value: 2}, &literalExpr{value: 1}), + ), + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) From ce6ce4e8888a4fe282f2ee212e0fbf07401706a0 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Mon, 10 Feb 2020 12:09:27 -0500 Subject: [PATCH 10/24] [wip] literalExpr parsing --- pkg/logql/ast.go | 11 ++ pkg/logql/expr.y | 9 +- pkg/logql/expr.y.go | 351 +++++++++++++++++++++++--------------------- pkg/logql/lex.go | 4 + 4 files changed, 204 insertions(+), 171 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 41c25abc994f..1d1ab69f91b2 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -411,6 +411,17 @@ type literalExpr struct { value float64 } +func mustNewLiteralExpr(s string) *literalExpr { + fmt.Println("parsing", s) + n, err := strconv.ParseFloat(s, 64) + if err != nil { + panic(err) + } + return &literalExpr{ + value: n, + } +} + func (e *literalExpr) logQLExpr() {} func (e *literalExpr) String() string { diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 2e8e68d725c8..1b16d5fc619d 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -25,7 +25,7 @@ import ( binOp string str string duration time.Duration - int int64 + LiteralExpr *literalExpr } %start root @@ -44,8 +44,9 @@ import ( %type vectorAggregationExpr %type vectorOp %type binOpExpr +%type literalExpr -%token IDENTIFIER STRING +%token IDENTIFIER STRING NUMBER %token DURATION %token MATCHERS LABELS EQ NEQ RE NRE OPEN_BRACE CLOSE_BRACE OPEN_BRACKET CLOSE_BRACKET COMMA DOT PIPE_MATCH PIPE_EXACT OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK @@ -65,6 +66,7 @@ expr: | rangeAggregationExpr { $$ = $1 } | vectorAggregationExpr { $$ = $1 } | binOpExpr { $$ = $1 } + | literalExpr { $$ = $1 } | OPEN_PARENTHESIS expr CLOSE_PARENTHESIS { $$ = $2 } ; @@ -141,6 +143,9 @@ binOpExpr: | expr DIV expr { $$ = mustNewBinOpExpr("/", $1, $3) } | expr MOD expr { $$ = mustNewBinOpExpr("%", $1, $3) } +literalExpr: + NUMBER { $$ = mustNewLiteralExpr( $1 ) } + vectorOp: SUM { $$ = OpTypeSum } | AVG { $$ = OpTypeAvg } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 72c1a935ecaf..3149ec9ca1e7 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -29,49 +29,50 @@ type exprSymType struct { binOp string str string duration time.Duration - int int64 + LiteralExpr *literalExpr } const IDENTIFIER = 57346 const STRING = 57347 -const DURATION = 57348 -const MATCHERS = 57349 -const LABELS = 57350 -const EQ = 57351 -const NEQ = 57352 -const RE = 57353 -const NRE = 57354 -const OPEN_BRACE = 57355 -const CLOSE_BRACE = 57356 -const OPEN_BRACKET = 57357 -const CLOSE_BRACKET = 57358 -const COMMA = 57359 -const DOT = 57360 -const PIPE_MATCH = 57361 -const PIPE_EXACT = 57362 -const OPEN_PARENTHESIS = 57363 -const CLOSE_PARENTHESIS = 57364 -const BY = 57365 -const WITHOUT = 57366 -const COUNT_OVER_TIME = 57367 -const RATE = 57368 -const SUM = 57369 -const AVG = 57370 -const MAX = 57371 -const MIN = 57372 -const COUNT = 57373 -const STDDEV = 57374 -const STDVAR = 57375 -const BOTTOMK = 57376 -const TOPK = 57377 -const OR = 57378 -const AND = 57379 -const UNLESS = 57380 -const ADD = 57381 -const SUB = 57382 -const MUL = 57383 -const DIV = 57384 -const MOD = 57385 +const NUMBER = 57348 +const DURATION = 57349 +const MATCHERS = 57350 +const LABELS = 57351 +const EQ = 57352 +const NEQ = 57353 +const RE = 57354 +const NRE = 57355 +const OPEN_BRACE = 57356 +const CLOSE_BRACE = 57357 +const OPEN_BRACKET = 57358 +const CLOSE_BRACKET = 57359 +const COMMA = 57360 +const DOT = 57361 +const PIPE_MATCH = 57362 +const PIPE_EXACT = 57363 +const OPEN_PARENTHESIS = 57364 +const CLOSE_PARENTHESIS = 57365 +const BY = 57366 +const WITHOUT = 57367 +const COUNT_OVER_TIME = 57368 +const RATE = 57369 +const SUM = 57370 +const AVG = 57371 +const MAX = 57372 +const MIN = 57373 +const COUNT = 57374 +const STDDEV = 57375 +const STDVAR = 57376 +const BOTTOMK = 57377 +const TOPK = 57378 +const OR = 57379 +const AND = 57380 +const UNLESS = 57381 +const ADD = 57382 +const SUB = 57383 +const MUL = 57384 +const DIV = 57385 +const MOD = 57386 var exprToknames = [...]string{ "$end", @@ -79,6 +80,7 @@ var exprToknames = [...]string{ "$unk", "IDENTIFIER", "STRING", + "NUMBER", "DURATION", "MATCHERS", "LABELS", @@ -131,8 +133,7 @@ var exprExca = [...]int{ -2, 0, -1, 3, 1, 2, - 22, 2, - 36, 2, + 23, 2, 37, 2, 38, 2, 39, 2, @@ -140,9 +141,9 @@ var exprExca = [...]int{ 41, 2, 42, 2, 43, 2, + 44, 2, -2, 0, - -1, 38, - 36, 2, + -1, 40, 37, 2, 38, 2, 39, 2, @@ -150,105 +151,107 @@ var exprExca = [...]int{ 41, 2, 42, 2, 43, 2, + 44, 2, -2, 0, } const exprPrivate = 57344 -const exprLast = 199 +const exprLast = 207 var exprAct = [...]int{ - 41, 5, 4, 31, 46, 88, 58, 3, 60, 26, - 27, 28, 29, 30, 105, 38, 28, 29, 30, 107, - 23, 24, 25, 26, 27, 28, 29, 30, 23, 24, - 25, 26, 27, 28, 29, 30, 40, 68, 42, 43, - 42, 43, 64, 63, 109, 108, 104, 61, 24, 25, - 26, 27, 28, 29, 30, 32, 105, 103, 84, 80, - 83, 106, 67, 36, 77, 35, 66, 39, 87, 86, - 81, 82, 33, 34, 90, 59, 91, 85, 72, 97, - 95, 11, 96, 11, 99, 100, 77, 101, 102, 7, - 65, 62, 70, 12, 13, 14, 15, 17, 18, 16, - 19, 20, 21, 22, 69, 57, 94, 71, 56, 111, - 112, 12, 13, 14, 15, 17, 18, 16, 19, 20, - 21, 22, 12, 13, 14, 15, 17, 18, 16, 19, - 20, 21, 22, 2, 78, 93, 92, 45, 32, 47, - 6, 37, 36, 110, 35, 47, 36, 89, 35, 10, - 8, 33, 34, 9, 98, 33, 34, 48, 49, 50, - 51, 52, 53, 54, 55, 78, 44, 1, 0, 32, - 0, 0, 0, 36, 0, 35, 0, 36, 0, 35, - 32, 0, 33, 34, 80, 79, 33, 34, 36, 59, - 35, 73, 74, 75, 76, 0, 0, 33, 34, + 43, 5, 4, 33, 48, 90, 60, 3, 62, 26, + 27, 28, 29, 30, 31, 32, 40, 30, 31, 32, + 25, 26, 27, 28, 29, 30, 31, 32, 25, 26, + 27, 28, 29, 30, 31, 32, 28, 29, 30, 31, + 32, 44, 45, 107, 66, 65, 111, 87, 109, 63, + 14, 15, 16, 17, 19, 20, 18, 21, 22, 23, + 24, 42, 107, 44, 45, 110, 79, 108, 106, 105, + 89, 88, 83, 84, 86, 12, 92, 85, 93, 70, + 69, 68, 41, 13, 97, 13, 101, 102, 79, 103, + 104, 8, 67, 64, 72, 14, 15, 16, 17, 19, + 20, 18, 21, 22, 23, 24, 74, 71, 96, 95, + 73, 113, 114, 94, 14, 15, 16, 17, 19, 20, + 18, 21, 22, 23, 24, 2, 34, 112, 7, 99, + 34, 82, 98, 59, 39, 38, 58, 37, 47, 38, + 49, 37, 49, 91, 35, 36, 6, 61, 35, 36, + 80, 50, 51, 52, 53, 54, 55, 56, 57, 38, + 11, 37, 9, 80, 10, 46, 1, 34, 35, 36, + 0, 100, 38, 0, 37, 0, 38, 0, 37, 0, + 0, 35, 36, 0, 81, 35, 36, 34, 61, 0, + 0, 0, 82, 0, 0, 0, 38, 0, 37, 75, + 76, 77, 78, 0, 0, 35, 36, } var exprPact = [...]int{ - 68, -1000, -8, 136, -1000, -1000, -1000, 68, -1000, 46, - 15, 135, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 68, 68, 68, 68, 68, 68, 68, - 68, 103, -1000, -1000, -1000, -1000, -1000, -16, 167, 70, - 86, 45, 41, 16, 90, 64, -1000, 182, 11, -30, - -30, -25, -25, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 163, 178, 70, 38, 36, 60, 97, 143, 143, -1000, - -1000, 141, -1000, 131, 130, 101, 75, 77, -1000, -1000, - -1000, 53, 132, 17, 17, 97, 35, 24, 39, -1000, - -3, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 23, 22, -1000, -1000, 139, -1000, -1000, 17, 17, - -1000, -1000, -1000, + 69, -1000, -9, 128, -1000, -1000, -1000, -1000, 69, -1000, + 60, 39, -1000, 136, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, 69, 69, 69, 69, 69, + 69, 69, 69, 131, -1000, -1000, -1000, -1000, -1000, -17, + 165, 71, 88, 59, 58, 57, 92, 91, -1000, 189, + -29, -4, -4, -25, -25, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 161, 185, 71, 54, 51, 29, 24, 139, + 139, -1000, -1000, 138, -1000, 108, 104, 103, 79, 127, + -1000, -1000, -1000, 124, 148, 17, 17, 24, 46, 45, + 44, -1000, 25, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 42, 23, -1000, -1000, 123, -1000, -1000, + 17, 17, -1000, -1000, -1000, } var exprPgo = [...]int{ - 0, 167, 133, 3, 0, 5, 7, 8, 4, 166, - 2, 153, 150, 1, 149, 140, + 0, 166, 125, 3, 0, 5, 7, 8, 4, 165, + 2, 164, 162, 1, 160, 146, 128, } var exprR1 = [...]int{ - 0, 1, 2, 2, 2, 2, 2, 6, 6, 6, - 6, 6, 7, 7, 7, 7, 7, 10, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, - 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, - 8, 15, 15, 15, 15, 15, 15, 15, 15, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 11, 11, - 5, 5, 4, 4, + 0, 1, 2, 2, 2, 2, 2, 2, 6, 6, + 6, 6, 6, 7, 7, 7, 7, 7, 10, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 3, + 3, 3, 3, 12, 12, 12, 9, 9, 8, 8, + 8, 8, 15, 15, 15, 15, 15, 15, 15, 15, + 16, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 11, 11, 5, 5, 4, 4, } var exprR2 = [...]int{ - 0, 1, 1, 1, 1, 1, 3, 1, 3, 3, - 3, 2, 2, 3, 3, 3, 2, 4, 4, 4, - 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, - 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, + 0, 1, 1, 1, 1, 1, 1, 3, 1, 3, + 3, 3, 2, 2, 3, 3, 3, 2, 4, 4, + 4, 5, 5, 5, 5, 6, 7, 6, 7, 1, + 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 3, 4, 4, + 1, 1, 1, 3, 4, 4, } var exprChk = [...]int{ - -1000, -1, -2, -6, -10, -13, -15, 21, -12, -11, - -14, 13, 25, 26, 27, 28, 31, 29, 30, 32, - 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, - 43, -3, 2, 19, 20, 12, 10, -2, -6, 21, - 21, -4, 23, 24, -9, 2, -8, 4, -2, -2, - -2, -2, -2, -2, -2, -2, 5, 2, 22, 22, - -7, -6, 21, -10, -13, 4, 21, 21, 21, 14, - 2, 17, 14, 9, 10, 11, 12, -3, 2, 22, - 6, -6, -7, 22, 22, 17, -10, -13, -5, 4, - -5, -8, 5, 5, 5, 5, 5, 2, 22, -4, - -4, -13, -10, 22, 22, 17, 22, 22, 22, 22, - 4, -4, -4, + -1000, -1, -2, -6, -10, -13, -15, -16, 22, -12, + -11, -14, 6, 14, 26, 27, 28, 29, 32, 30, + 31, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, 44, -3, 2, 20, 21, 13, 11, -2, + -6, 22, 22, -4, 24, 25, -9, 2, -8, 4, + -2, -2, -2, -2, -2, -2, -2, -2, 5, 2, + 23, 23, -7, -6, 22, -10, -13, 4, 22, 22, + 22, 15, 2, 18, 15, 10, 11, 12, 13, -3, + 2, 23, 7, -6, -7, 23, 23, 18, -10, -13, + -5, 4, -5, -8, 5, 5, 5, 5, 5, 2, + 23, -4, -4, -13, -10, 23, 23, 18, 23, 23, + 23, 23, 4, -4, -4, } var exprDef = [...]int{ - 0, -2, 1, -2, 3, 4, 5, 0, 7, 0, - 0, 0, 58, 59, 49, 50, 51, 52, 53, 54, - 55, 56, 57, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 11, 28, 29, 30, 31, 0, -2, 0, - 0, 0, 0, 0, 0, 0, 35, 0, 41, 42, - 43, 44, 45, 46, 47, 48, 8, 10, 6, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, - 33, 0, 34, 0, 0, 0, 0, 0, 16, 17, - 12, 0, 0, 18, 19, 0, 0, 0, 0, 60, - 0, 36, 37, 38, 39, 40, 13, 15, 14, 22, - 23, 0, 0, 20, 21, 0, 62, 63, 24, 26, - 61, 25, 27, + 0, -2, 1, -2, 3, 4, 5, 6, 0, 8, + 0, 0, 50, 0, 60, 61, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 12, 29, 30, 31, 32, 0, + -2, 0, 0, 0, 0, 0, 0, 0, 36, 0, + 42, 43, 44, 45, 46, 47, 48, 49, 9, 11, + 7, 10, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 33, 34, 0, 35, 0, 0, 0, 0, 0, + 17, 18, 13, 0, 0, 19, 20, 0, 0, 0, + 0, 62, 0, 37, 38, 39, 40, 41, 14, 16, + 15, 23, 24, 0, 0, 21, 22, 0, 64, 65, + 25, 27, 63, 26, 28, } var exprTok1 = [...]int{ @@ -260,7 +263,7 @@ var exprTok2 = [...]int{ 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 42, 43, + 42, 43, 44, } var exprTok3 = [...]int{ 0, @@ -628,270 +631,280 @@ exprdefault: exprVAL.Expr = exprDollar[1].BinOpExpr } case 6: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.Expr = exprDollar[1].LiteralExpr + } + case 7: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Expr = exprDollar[2].Expr } - case 7: + case 8: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LogExpr = newMatcherExpr(exprDollar[1].Selector) } - case 8: + case 9: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogExpr = NewFilterExpr(exprDollar[1].LogExpr, exprDollar[2].Filter, exprDollar[3].str) } - case 9: + case 10: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogExpr = exprDollar[2].LogExpr } - case 12: + case 13: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LogRangeExpr = newLogRange(exprDollar[1].LogExpr, exprDollar[2].duration) } - case 13: + case 14: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogRangeExpr = addFilterToLogRangeExpr(exprDollar[1].LogRangeExpr, exprDollar[2].Filter, exprDollar[3].str) } - case 14: + case 15: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogRangeExpr = exprDollar[2].LogRangeExpr } - case 17: + case 18: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[3].LogRangeExpr, exprDollar[1].RangeOp) } - case 18: + case 19: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].RangeAggregationExpr, exprDollar[1].VectorOp, nil, nil) } - case 19: + case 20: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].VectorAggregationExpr, exprDollar[1].VectorOp, nil, nil) } - case 20: + case 21: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } - case 21: + case 22: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } - case 22: + case 23: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } - case 23: + case 24: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } - case 24: + case 25: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].VectorAggregationExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } - case 25: + case 26: exprDollar = exprS[exprpt-7 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } - case 26: + case 27: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].RangeAggregationExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } - case 27: + case 28: exprDollar = exprS[exprpt-7 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } - case 28: + case 29: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchRegexp } - case 29: + case 30: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchEqual } - case 30: + case 31: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchNotRegexp } - case 31: + case 32: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchNotEqual } - case 32: + case 33: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Selector = exprDollar[2].Matchers } - case 33: + case 34: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Selector = exprDollar[2].Matchers } - case 34: + case 35: exprDollar = exprS[exprpt-3 : exprpt+1] { } - case 35: + case 36: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Matchers = []*labels.Matcher{exprDollar[1].Matcher} } - case 36: + case 37: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matchers = append(exprDollar[1].Matchers, exprDollar[3].Matcher) } - case 37: + case 38: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchEqual, exprDollar[1].str, exprDollar[3].str) } - case 38: + case 39: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchNotEqual, exprDollar[1].str, exprDollar[3].str) } - case 39: + case 40: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchRegexp, exprDollar[1].str, exprDollar[3].str) } - case 40: + case 41: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchNotRegexp, exprDollar[1].str, exprDollar[3].str) } - case 41: + case 42: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("or", exprDollar[1].Expr, exprDollar[3].Expr) } - case 42: + case 43: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("and", exprDollar[1].Expr, exprDollar[3].Expr) } - case 43: + case 44: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("unless", exprDollar[1].Expr, exprDollar[3].Expr) } - case 44: + case 45: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[1].Expr, exprDollar[3].Expr) } - case 45: + case 46: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("-", exprDollar[1].Expr, exprDollar[3].Expr) } - case 46: + case 47: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("*", exprDollar[1].Expr, exprDollar[3].Expr) } - case 47: + case 48: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[1].Expr, exprDollar[3].Expr) } - case 48: + case 49: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[1].Expr, exprDollar[3].Expr) } - case 49: + case 50: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[1].str) + } + case 51: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 50: + case 52: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 51: + case 53: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 52: + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 53: + case 55: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 54: + case 56: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 55: + case 57: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 56: + case 58: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 57: + case 59: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 58: + case 60: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 59: + case 61: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 60: + case 62: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 61: + case 63: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 62: + case 64: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 63: + case 65: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index 0d70fd551529..796b388646c4 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -61,6 +61,10 @@ func (l *lexer) Lex(lval *exprSymType) int { case scanner.EOF: return 0 + case scanner.Int, scanner.Float: + lval.str = l.TokenText() + return NUMBER + case scanner.String: var err error lval.str, err = strconv.Unquote(l.TokenText()) From bfc70b4eb8cfd95717637f7cb596b35037f5f6a3 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Mon, 10 Feb 2020 12:18:10 -0500 Subject: [PATCH 11/24] number parsing --- pkg/logql/expr.y | 8 +++---- pkg/logql/expr.y.go | 48 ++++++++++++++++++++-------------------- pkg/logql/lex_test.go | 4 ++-- pkg/logql/parser_test.go | 10 ++++++++- 4 files changed, 39 insertions(+), 31 deletions(-) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 1b16d5fc619d..6e702d48eed2 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -97,10 +97,10 @@ vectorAggregationExpr: | vectorOp OPEN_PARENTHESIS rangeAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($3, $1, $5, nil) } | vectorOp OPEN_PARENTHESIS vectorAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($3, $1, $5, nil) } // Aggregations with 2 arguments. - | vectorOp OPEN_PARENTHESIS IDENTIFIER COMMA vectorAggregationExpr CLOSE_PARENTHESIS { $$ = mustNewVectorAggregationExpr($5, $1, nil, &$3) } - | vectorOp OPEN_PARENTHESIS IDENTIFIER COMMA vectorAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($5, $1, $7, &$3) } - | vectorOp OPEN_PARENTHESIS IDENTIFIER COMMA rangeAggregationExpr CLOSE_PARENTHESIS { $$ = mustNewVectorAggregationExpr($5, $1, nil, &$3) } - | vectorOp OPEN_PARENTHESIS IDENTIFIER COMMA rangeAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($5, $1, $7, &$3) } + | vectorOp OPEN_PARENTHESIS NUMBER COMMA vectorAggregationExpr CLOSE_PARENTHESIS { $$ = mustNewVectorAggregationExpr($5, $1, nil, &$3) } + | vectorOp OPEN_PARENTHESIS NUMBER COMMA vectorAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($5, $1, $7, &$3) } + | vectorOp OPEN_PARENTHESIS NUMBER COMMA rangeAggregationExpr CLOSE_PARENTHESIS { $$ = mustNewVectorAggregationExpr($5, $1, nil, &$3) } + | vectorOp OPEN_PARENTHESIS NUMBER COMMA rangeAggregationExpr CLOSE_PARENTHESIS grouping { $$ = mustNewVectorAggregationExpr($5, $1, $7, &$3) } ; filter: diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 3149ec9ca1e7..37d1b07077b0 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -157,7 +157,7 @@ var exprExca = [...]int{ const exprPrivate = 57344 -const exprLast = 207 +const exprLast = 206 var exprAct = [...]int{ @@ -169,39 +169,39 @@ var exprAct = [...]int{ 14, 15, 16, 17, 19, 20, 18, 21, 22, 23, 24, 42, 107, 44, 45, 110, 79, 108, 106, 105, 89, 88, 83, 84, 86, 12, 92, 85, 93, 70, - 69, 68, 41, 13, 97, 13, 101, 102, 79, 103, - 104, 8, 67, 64, 72, 14, 15, 16, 17, 19, + 69, 68, 41, 13, 13, 97, 101, 102, 79, 103, + 104, 8, 64, 67, 72, 14, 15, 16, 17, 19, 20, 18, 21, 22, 23, 24, 74, 71, 96, 95, - 73, 113, 114, 94, 14, 15, 16, 17, 19, 20, - 18, 21, 22, 23, 24, 2, 34, 112, 7, 99, - 34, 82, 98, 59, 39, 38, 58, 37, 47, 38, - 49, 37, 49, 91, 35, 36, 6, 61, 35, 36, - 80, 50, 51, 52, 53, 54, 55, 56, 57, 38, - 11, 37, 9, 80, 10, 46, 1, 34, 35, 36, - 0, 100, 38, 0, 37, 0, 38, 0, 37, 0, - 0, 35, 36, 0, 81, 35, 36, 34, 61, 0, - 0, 0, 82, 0, 0, 0, 38, 0, 37, 75, - 76, 77, 78, 0, 0, 35, 36, + 73, 113, 114, 14, 15, 16, 17, 19, 20, 18, + 21, 22, 23, 24, 2, 34, 94, 7, 99, 34, + 82, 98, 59, 39, 38, 58, 37, 47, 38, 49, + 37, 112, 49, 35, 36, 91, 61, 35, 36, 80, + 50, 51, 52, 53, 54, 55, 56, 57, 38, 6, + 37, 11, 80, 9, 10, 46, 34, 35, 36, 1, + 100, 38, 0, 37, 0, 38, 0, 37, 0, 0, + 35, 36, 0, 81, 35, 36, 34, 61, 0, 0, + 0, 82, 0, 0, 0, 38, 0, 37, 75, 76, + 77, 78, 0, 0, 35, 36, } var exprPact = [...]int{ - 69, -1000, -9, 128, -1000, -1000, -1000, -1000, 69, -1000, - 60, 39, -1000, 136, -1000, -1000, -1000, -1000, -1000, -1000, + 69, -1000, -9, 127, -1000, -1000, -1000, -1000, 69, -1000, + 60, 39, -1000, 135, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 69, 69, 69, 69, 69, - 69, 69, 69, 131, -1000, -1000, -1000, -1000, -1000, -17, - 165, 71, 88, 59, 58, 57, 92, 91, -1000, 189, + 69, 69, 69, 130, -1000, -1000, -1000, -1000, -1000, -17, + 164, 70, 87, 59, 58, 57, 92, 91, -1000, 188, -29, -4, -4, -25, -25, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 161, 185, 71, 54, 51, 29, 24, 139, - 139, -1000, -1000, 138, -1000, 108, 104, 103, 79, 127, - -1000, -1000, -1000, 124, 148, 17, 17, 24, 46, 45, + -1000, -1000, 160, 184, 70, 54, 51, 29, 24, 141, + 141, -1000, -1000, 138, -1000, 121, 104, 103, 80, 126, + -1000, -1000, -1000, 123, 147, 17, 17, 24, 46, 45, 44, -1000, 25, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 42, 23, -1000, -1000, 123, -1000, -1000, + -1000, -1000, -1000, 42, 23, -1000, -1000, 137, -1000, -1000, 17, 17, -1000, -1000, -1000, } var exprPgo = [...]int{ - 0, 166, 125, 3, 0, 5, 7, 8, 4, 165, - 2, 164, 162, 1, 160, 146, 128, + 0, 169, 124, 3, 0, 5, 7, 8, 4, 165, + 2, 164, 163, 1, 161, 159, 127, } var exprR1 = [...]int{ @@ -231,7 +231,7 @@ var exprChk = [...]int{ 42, 43, 44, -3, 2, 20, 21, 13, 11, -2, -6, 22, 22, -4, 24, 25, -9, 2, -8, 4, -2, -2, -2, -2, -2, -2, -2, -2, 5, 2, - 23, 23, -7, -6, 22, -10, -13, 4, 22, 22, + 23, 23, -7, -6, 22, -10, -13, 6, 22, 22, 22, 15, 2, 18, 15, 10, 11, 12, 13, -3, 2, 23, 7, -6, -7, 23, 23, 18, -10, -13, -5, 4, -5, -8, 5, 5, 5, 5, 5, 2, diff --git a/pkg/logql/lex_test.go b/pkg/logql/lex_test.go index 9505a9fc1450..0a433caa2c3e 100644 --- a/pkg/logql/lex_test.go +++ b/pkg/logql/lex_test.go @@ -24,8 +24,8 @@ func TestLex(t *testing.T) { {`rate({foo="bar"}[10s])`, []int{RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS}}, {`count_over_time({foo="bar"}[5m])`, []int{COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS}}, {`sum(count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}}, - {`topk(3,count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{TOPK, OPEN_PARENTHESIS, IDENTIFIER, COMMA, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}}, - {`bottomk(10,sum(count_over_time({foo="bar"}[5m])) by (foo,bar))`, []int{BOTTOMK, OPEN_PARENTHESIS, IDENTIFIER, COMMA, SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS}}, + {`topk(3,count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{TOPK, OPEN_PARENTHESIS, NUMBER, COMMA, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}}, + {`bottomk(10,sum(count_over_time({foo="bar"}[5m])) by (foo,bar))`, []int{BOTTOMK, OPEN_PARENTHESIS, NUMBER, COMMA, SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS}}, {`sum(max(rate({foo="bar"}[5m])) by (foo,bar)) by (foo)`, []int{SUM, OPEN_PARENTHESIS, MAX, OPEN_PARENTHESIS, RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, CLOSE_PARENTHESIS}}, } { t.Run(tc.input, func(t *testing.T) { diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index de40c7a7029e..9b190397f6af 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -219,7 +219,15 @@ func TestParse(t *testing.T) { { in: `bottomk(he,count_over_time({ foo !~ "bar" }[5h]))`, err: ParseError{ - msg: "invalid parameter bottomk(he,", + msg: "syntax error: unexpected IDENTIFIER", + line: 1, + col: 9, + }, + }, + { + in: `bottomk(1.2,count_over_time({ foo !~ "bar" }[5h]))`, + err: ParseError{ + msg: "invalid parameter bottomk(1.2,", line: 0, col: 0, }, From 1db3bd4f2a95dbdf2e8575b0fc54941619ae9fb0 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Mon, 10 Feb 2020 14:55:36 -0500 Subject: [PATCH 12/24] signed literals --- pkg/logql/ast.go | 9 ++- pkg/logql/evaluator.go | 24 ++++++ pkg/logql/expr.y | 4 +- pkg/logql/expr.y.go | 167 +++++++++++++++++++++------------------ pkg/logql/parser_test.go | 9 +++ 5 files changed, 132 insertions(+), 81 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 1d1ab69f91b2..09fdd226276b 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -347,6 +347,7 @@ func mustNewVectorAggregationExpr(left SampleExpr, operation string, gr *groupin if p, err = strconv.Atoi(*params); err != nil { panic(newParseError(fmt.Sprintf("invalid parameter %s(%s,", operation, *params), 0, 0)) } + default: if params != nil { panic(newParseError(fmt.Sprintf("unsupported parameter for operation %s(%s,", operation, *params), 0, 0)) @@ -411,12 +412,16 @@ type literalExpr struct { value float64 } -func mustNewLiteralExpr(s string) *literalExpr { - fmt.Println("parsing", s) +func mustNewLiteralExpr(s string, invert bool) *literalExpr { n, err := strconv.ParseFloat(s, 64) if err != nil { panic(err) } + + if invert { + n = -n + } + return &literalExpr{ value: n, } diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 070e9d212998..9312820eebd0 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -96,6 +96,8 @@ func (ev *defaultEvaluator) Evaluator(ctx context.Context, expr SampleExpr, q Pa return ev.rangeAggEvaluator(ctx, e, q) case *binOpExpr: return ev.binOpEvaluator(ctx, e, q) + case *literalExpr: // this is only called when a singleton is the root node + return singletonEvaluator(e, q) default: return nil, errors.Errorf("unexpected type (%T): %v", e, e) @@ -572,3 +574,25 @@ func (ev *defaultEvaluator) literalEvaluator(op string, lit *literalExpr, eval S nil, ) } + +// singletonEvaluator is an evaluator adapter for a literal expressions that is a root node +func singletonEvaluator(expr *literalExpr, params Params) (StepEvaluator, error) { + var done bool + return newStepEvaluator( + func() (bool, int64, promql.Vector) { + if done { + return done, 0, nil + } + tmp := done + done = true + ts := params.Start().UnixNano() / int64(time.Millisecond) + + return tmp, ts, promql.Vector{promql.Sample{Point: promql.Point{ + T: ts, + V: expr.value, + }}} + }, + nil, + ) + +} diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 6e702d48eed2..de058e9bea79 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -144,7 +144,9 @@ binOpExpr: | expr MOD expr { $$ = mustNewBinOpExpr("%", $1, $3) } literalExpr: - NUMBER { $$ = mustNewLiteralExpr( $1 ) } + NUMBER { $$ = mustNewLiteralExpr( $1, false ) } + | ADD NUMBER { $$ = mustNewLiteralExpr( $2, false ) } + | SUB NUMBER { $$ = mustNewLiteralExpr( $2, true ) } vectorOp: SUM { $$ = OpTypeSum } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 37d1b07077b0..a49454a7dac9 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -143,7 +143,7 @@ var exprExca = [...]int{ 43, 2, 44, 2, -2, 0, - -1, 40, + -1, 42, 37, 2, 38, 2, 39, 2, @@ -157,51 +157,52 @@ var exprExca = [...]int{ const exprPrivate = 57344 -const exprLast = 206 +const exprLast = 215 var exprAct = [...]int{ - 43, 5, 4, 33, 48, 90, 60, 3, 62, 26, - 27, 28, 29, 30, 31, 32, 40, 30, 31, 32, - 25, 26, 27, 28, 29, 30, 31, 32, 25, 26, - 27, 28, 29, 30, 31, 32, 28, 29, 30, 31, - 32, 44, 45, 107, 66, 65, 111, 87, 109, 63, - 14, 15, 16, 17, 19, 20, 18, 21, 22, 23, - 24, 42, 107, 44, 45, 110, 79, 108, 106, 105, - 89, 88, 83, 84, 86, 12, 92, 85, 93, 70, - 69, 68, 41, 13, 13, 97, 101, 102, 79, 103, - 104, 8, 64, 67, 72, 14, 15, 16, 17, 19, - 20, 18, 21, 22, 23, 24, 74, 71, 96, 95, - 73, 113, 114, 14, 15, 16, 17, 19, 20, 18, - 21, 22, 23, 24, 2, 34, 94, 7, 99, 34, - 82, 98, 59, 39, 38, 58, 37, 47, 38, 49, - 37, 112, 49, 35, 36, 91, 61, 35, 36, 80, - 50, 51, 52, 53, 54, 55, 56, 57, 38, 6, - 37, 11, 80, 9, 10, 46, 34, 35, 36, 1, - 100, 38, 0, 37, 0, 38, 0, 37, 0, 0, - 35, 36, 0, 81, 35, 36, 34, 61, 0, 0, - 0, 82, 0, 0, 0, 38, 0, 37, 75, 76, - 77, 78, 0, 0, 35, 36, + 45, 5, 4, 35, 52, 94, 12, 3, 66, 30, + 31, 32, 33, 34, 15, 111, 42, 32, 33, 34, + 113, 44, 8, 46, 47, 115, 16, 17, 18, 19, + 21, 22, 20, 23, 24, 25, 26, 114, 64, 111, + 13, 14, 46, 47, 112, 110, 70, 69, 109, 90, + 89, 67, 27, 28, 29, 30, 31, 32, 33, 34, + 27, 28, 29, 30, 31, 32, 33, 34, 74, 73, + 83, 15, 72, 43, 93, 92, 87, 88, 71, 68, + 96, 91, 97, 28, 29, 30, 31, 32, 33, 34, + 105, 106, 83, 107, 108, 78, 76, 101, 16, 17, + 18, 19, 21, 22, 20, 23, 24, 25, 26, 75, + 103, 63, 77, 102, 62, 117, 118, 16, 17, 18, + 19, 21, 22, 20, 23, 24, 25, 26, 2, 36, + 49, 48, 116, 84, 86, 100, 53, 41, 40, 99, + 39, 98, 40, 51, 39, 53, 95, 37, 38, 7, + 65, 37, 38, 6, 104, 84, 54, 55, 56, 57, + 58, 59, 60, 61, 40, 11, 39, 9, 36, 79, + 80, 81, 82, 37, 38, 10, 85, 40, 50, 39, + 1, 0, 0, 36, 0, 0, 37, 38, 86, 65, + 0, 0, 40, 0, 39, 36, 0, 0, 0, 0, + 0, 37, 38, 0, 40, 0, 39, 0, 0, 0, + 0, 0, 0, 37, 38, } var exprPact = [...]int{ - 69, -1000, -9, 127, -1000, -1000, -1000, -1000, 69, -1000, - 60, 39, -1000, 135, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, 69, 69, 69, 69, 69, - 69, 69, 69, 130, -1000, -1000, -1000, -1000, -1000, -17, - 164, 70, 87, 59, 58, 57, 92, 91, -1000, 188, - -29, -4, -4, -25, -25, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 160, 184, 70, 54, 51, 29, 24, 141, - 141, -1000, -1000, 138, -1000, 121, 104, 103, 80, 126, - -1000, -1000, -1000, 123, 147, 17, 17, 24, 46, 45, - 44, -1000, 25, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 42, 23, -1000, -1000, 137, -1000, -1000, - 17, 17, -1000, -1000, -1000, + 0, -1000, 23, 193, -1000, -1000, -1000, -1000, 0, -1000, + 51, -1, -1000, 125, 124, 141, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 0, 0, 0, + 0, 0, 0, 0, 0, 109, -1000, -1000, -1000, -1000, + -1000, 15, 166, 57, 72, 50, 47, 46, -1000, -1000, + 94, 80, -1000, 159, 45, -31, -31, -25, -25, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, 153, 181, 57, 27, + 26, 63, 91, 142, 142, -1000, -1000, 132, -1000, 136, + 134, 130, 92, 108, -1000, -1000, -1000, 127, 131, 18, + 18, 91, 25, 22, 21, -1000, -3, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 14, 2, -1000, + -1000, 128, -1000, -1000, 18, 18, -1000, -1000, -1000, } var exprPgo = [...]int{ - 0, 169, 124, 3, 0, 5, 7, 8, 4, 165, - 2, 164, 163, 1, 161, 159, 127, + 0, 180, 128, 3, 0, 5, 7, 8, 4, 178, + 2, 175, 167, 1, 165, 153, 149, } var exprR1 = [...]int{ @@ -210,8 +211,8 @@ var exprR1 = [...]int{ 13, 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, 8, 15, 15, 15, 15, 15, 15, 15, 15, - 16, 14, 14, 14, 14, 14, 14, 14, 14, 14, - 11, 11, 5, 5, 4, 4, + 16, 16, 16, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 11, 11, 5, 5, 4, 4, } var exprR2 = [...]int{ @@ -220,38 +221,38 @@ var exprR2 = [...]int{ 4, 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 3, 4, 4, + 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 3, 4, 4, } var exprChk = [...]int{ -1000, -1, -2, -6, -10, -13, -15, -16, 22, -12, - -11, -14, 6, 14, 26, 27, 28, 29, 32, 30, - 31, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 42, 43, 44, -3, 2, 20, 21, 13, 11, -2, - -6, 22, 22, -4, 24, 25, -9, 2, -8, 4, - -2, -2, -2, -2, -2, -2, -2, -2, 5, 2, - 23, 23, -7, -6, 22, -10, -13, 6, 22, 22, - 22, 15, 2, 18, 15, 10, 11, 12, 13, -3, - 2, 23, 7, -6, -7, 23, 23, 18, -10, -13, - -5, 4, -5, -8, 5, 5, 5, 5, 5, 2, - 23, -4, -4, -13, -10, 23, 23, 18, 23, 23, - 23, 23, 4, -4, -4, + -11, -14, 6, 40, 41, 14, 26, 27, 28, 29, + 32, 30, 31, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, -3, 2, 20, 21, 13, + 11, -2, -6, 22, 22, -4, 24, 25, 6, 6, + -9, 2, -8, 4, -2, -2, -2, -2, -2, -2, + -2, -2, 5, 2, 23, 23, -7, -6, 22, -10, + -13, 6, 22, 22, 22, 15, 2, 18, 15, 10, + 11, 12, 13, -3, 2, 23, 7, -6, -7, 23, + 23, 18, -10, -13, -5, 4, -5, -8, 5, 5, + 5, 5, 5, 2, 23, -4, -4, -13, -10, 23, + 23, 18, 23, 23, 23, 23, 4, -4, -4, } var exprDef = [...]int{ 0, -2, 1, -2, 3, 4, 5, 6, 0, 8, - 0, 0, 50, 0, 60, 61, 51, 52, 53, 54, - 55, 56, 57, 58, 59, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 29, 30, 31, 32, 0, - -2, 0, 0, 0, 0, 0, 0, 0, 36, 0, - 42, 43, 44, 45, 46, 47, 48, 49, 9, 11, - 7, 10, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 33, 34, 0, 35, 0, 0, 0, 0, 0, - 17, 18, 13, 0, 0, 19, 20, 0, 0, 0, - 0, 62, 0, 37, 38, 39, 40, 41, 14, 16, - 15, 23, 24, 0, 0, 21, 22, 0, 64, 65, - 25, 27, 63, 26, 28, + 0, 0, 50, 0, 0, 0, 62, 63, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 12, 29, 30, 31, + 32, 0, -2, 0, 0, 0, 0, 0, 51, 52, + 0, 0, 36, 0, 42, 43, 44, 45, 46, 47, + 48, 49, 9, 11, 7, 10, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 33, 34, 0, 35, 0, + 0, 0, 0, 0, 17, 18, 13, 0, 0, 19, + 20, 0, 0, 0, 0, 64, 0, 37, 38, 39, + 40, 41, 14, 16, 15, 23, 24, 0, 0, 21, + 22, 0, 66, 67, 25, 27, 65, 26, 28, } var exprTok1 = [...]int{ @@ -832,79 +833,89 @@ exprdefault: case 50: exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[1].str) + exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[1].str, false) } case 51: + exprDollar = exprS[exprpt-2 : exprpt+1] + { + exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, false) + } + case 52: + exprDollar = exprS[exprpt-2 : exprpt+1] + { + exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, true) + } + case 53: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 52: + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 53: + case 55: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 54: + case 56: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 55: + case 57: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 56: + case 58: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 57: + case 59: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 58: + case 60: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 59: + case 61: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 60: + case 62: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 61: + case 63: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 62: + case 64: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 63: + case 65: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 64: + case 66: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 65: + case 67: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 9b190397f6af..71b322451af4 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -706,6 +706,15 @@ sum(count_over_time({foo="bar"}[5m])) by (foo) + 2 / 1 mustNewBinOpExpr(OpTypeDiv, &literalExpr{value: 2}, &literalExpr{value: 1}), ), }, + { + // test signs + in: `1 + -2 / 1`, + exp: mustNewBinOpExpr( + OpTypeAdd, + &literalExpr{value: 1}, + mustNewBinOpExpr(OpTypeDiv, &literalExpr{value: -2}, &literalExpr{value: 1}), + ), + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) From 178be5f4aca23469e6fcd50815e225c9f12fc53a Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Mon, 10 Feb 2020 15:58:30 -0500 Subject: [PATCH 13/24] propagates evaluator close, handles healthchecks --- pkg/logql/engine.go | 10 ++-------- pkg/logql/engine_test.go | 20 ++++++++++++++++++++ pkg/logql/evaluator.go | 12 ++++++------ pkg/logql/parser_test.go | 9 +++++++++ 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index a479df76e8c5..c878a01ee074 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -180,13 +180,6 @@ func (ng *engine) exec(ctx context.Context, q *query) (promql.Value, error) { qs := q.String() // This is a legacy query used for health checking. Not the best practice, but it works. - if qs == "1+1" { - if IsInstant(q) { - return promql.Vector{}, nil - } - return promql.Matrix{}, nil - } - expr, err := ParseExpr(qs) if err != nil { return nil, err @@ -212,7 +205,6 @@ func (ng *engine) exec(ctx context.Context, q *query) (promql.Value, error) { // evalSample evaluate a sampleExpr func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (promql.Value, error) { - stepEvaluator, err := ng.evaluator.Evaluator(ctx, expr, q) if err != nil { return nil, err @@ -222,10 +214,12 @@ func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (pr seriesIndex := map[uint64]*promql.Series{} next, ts, vec := stepEvaluator.Next() + if IsInstant(q) { sort.Slice(vec, func(i, j int) bool { return labels.Compare(vec[i].Metric, vec[j].Metric) < 0 }) return vec, nil } + for next { for _, p := range vec { var ( diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 44c6b7805d2a..af146689f57a 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -287,6 +287,15 @@ func TestEngine_NewInstantQuery(t *testing.T) { promql.Sample{Point: promql.Point{T: 60 * 1000, V: 0.2}, Metric: labels.Labels{labels.Label{Name: "app", Value: "fuzz"}}}, }, }, + { + // healthcheck + `1+1`, time.Unix(60, 0), logproto.FORWARD, 100, + [][]*logproto.Stream{}, + []SelectParams{}, + promql.Vector{ + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 2}}, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { @@ -946,6 +955,17 @@ rate({app="bar"}[1m]) }, }, }, + { + `1+1--1`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{}, + []SelectParams{}, + promql.Matrix{ + promql.Series{ + Points: []promql.Point{{T: 60 * 1000, V: 3}}, + }, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 9312820eebd0..884e43201585 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -571,20 +571,20 @@ func (ev *defaultEvaluator) literalEvaluator(op string, lit *literalExpr, eval S return done, ts, results }, - nil, + eval.Close, ) } // singletonEvaluator is an evaluator adapter for a literal expressions that is a root node func singletonEvaluator(expr *literalExpr, params Params) (StepEvaluator, error) { - var done bool + ok := true return newStepEvaluator( func() (bool, int64, promql.Vector) { - if done { - return done, 0, nil + if !ok { + return ok, 0, nil } - tmp := done - done = true + tmp := ok + ok = false ts := params.Start().UnixNano() / int64(time.Millisecond) return tmp, ts, promql.Vector{promql.Sample{Point: promql.Point{ diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 71b322451af4..4b65a4ddfd10 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -715,6 +715,15 @@ sum(count_over_time({foo="bar"}[5m])) by (foo) + 2 / 1 mustNewBinOpExpr(OpTypeDiv, &literalExpr{value: -2}, &literalExpr{value: 1}), ), }, + { + // test signs/ops with equal associativity + in: `1 + 1 - -1`, + exp: mustNewBinOpExpr( + OpTypeSub, + mustNewBinOpExpr(OpTypeAdd, &literalExpr{value: 1}, &literalExpr{value: 1}), + &literalExpr{value: -1}, + ), + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) From 92c951675d8d15dc1be2ca150682895d87f0d53e Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Mon, 10 Feb 2020 18:23:38 -0500 Subject: [PATCH 14/24] literal evaluator works on non commutative operations --- pkg/logql/engine_test.go | 36 ++++++++++++++++++++++++++++++++++++ pkg/logql/evaluator.go | 35 ++++++++++++++++++++++++----------- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index af146689f57a..4861de0279b7 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -966,6 +966,42 @@ rate({app="bar"}[1m]) }, }, }, + { + `rate({app="bar"}[1m]) - 1`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: -0.8}, {T: 90 * 1000, V: -0.8}, {T: 120 * 1000, V: -0.8}, {T: 150 * 1000, V: -0.8}, {T: 180 * 1000, V: -0.8}}, + }, + }, + }, + { + `1 - rate({app="bar"}[1m])`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.8}, {T: 90 * 1000, V: 0.8}, {T: 120 * 1000, V: 0.8}, {T: 150 * 1000, V: 0.8}, {T: 180 * 1000, V: 0.8}}, + }, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 884e43201585..3398f040ce5f 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -354,14 +354,14 @@ func (ev *defaultEvaluator) binOpEvaluator( if err != nil { return nil, err } - return ev.literalEvaluator(expr.op, leftLit, rhs) + return ev.literalEvaluator(expr.op, leftLit, rhs, false) } if rOk { lhs, err := ev.Evaluator(ctx, expr.SampleExpr, q) if err != nil { return nil, err } - return ev.literalEvaluator(expr.op, rightLit, lhs) + return ev.literalEvaluator(expr.op, rightLit, lhs, true) } // we have two non literal legs @@ -549,27 +549,40 @@ func (ev *defaultEvaluator) reduceBinOp(op string, left, right *literalExpr) Sam return &literalExpr{value: merged.V} } -// literalEvaluator merges a literal with a StepEvaluator -func (ev *defaultEvaluator) literalEvaluator(op string, lit *literalExpr, eval StepEvaluator) (StepEvaluator, error) { +// literalEvaluator merges a literal with a StepEvaluator. Since order matters in +// non commutative operations, inverted should be true when the literalExpr is not the left argument. +func (ev *defaultEvaluator) literalEvaluator( + op string, + lit *literalExpr, + eval StepEvaluator, + inverted bool, +) (StepEvaluator, error) { return newStepEvaluator( func() (bool, int64, promql.Vector) { - done, ts, vec := eval.Next() + ok, ts, vec := eval.Next() results := make(promql.Vector, 0, len(vec)) for _, sample := range vec { + literalPoint := promql.Sample{ + Metric: sample.Metric, + Point: promql.Point{T: ts, V: lit.value}, + } + + left, right := &literalPoint, &sample + if inverted { + left, right = right, left + } + if merged := ev.mergeBinOp( op, - &sample, - &promql.Sample{ - Metric: sample.Metric, - Point: promql.Point{T: ts, V: lit.value}, - }, + left, + right, ); merged != nil { results = append(results, *merged) } } - return done, ts, results + return ok, ts, results }, eval.Close, ) From c2ceabbb7ac7f580e68e308c37758607cbd44a4d Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Tue, 11 Feb 2020 17:34:50 -0500 Subject: [PATCH 15/24] literalExprs cannot be used as legs of logical/set binops --- pkg/logql/ast.go | 40 +++++++++++++++++++++++++++++++++------- pkg/logql/parser_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 9850aa7a08cf..59ab3f8592ad 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -228,18 +228,25 @@ const ( OpTypeCountOverTime = "count_over_time" OpTypeRate = "rate" - // binops + // binops - logical/set OpTypeOr = "or" OpTypeAnd = "and" OpTypeUnless = "unless" - OpTypeAdd = "+" - OpTypeSub = "-" - OpTypeMul = "*" - OpTypeDiv = "/" - OpTypeMod = "%" - OpTypePow = "^" + + // binops - operations + OpTypeAdd = "+" + OpTypeSub = "-" + OpTypeMul = "*" + OpTypeDiv = "/" + OpTypeMod = "%" + OpTypePow = "^" ) +// IsLogicalBinOp tests whether an operation is a logical/set binary operation +func IsLogicalBinOp(op string) bool { + return op == OpTypeOr || op == OpTypeAnd || op == OpTypeUnless +} + // SampleExpr is a LogQL expression filtering logs and returning metric samples. type SampleExpr interface { // Selector is the LogQL selector to apply when retrieving logs. @@ -410,6 +417,25 @@ func mustNewBinOpExpr(op string, lhs, rhs Expr) SampleExpr { rhs, ), 0, 0)) } + + if IsLogicalBinOp(op) { + if l, ok := left.(*literalExpr); ok { + panic(newParseError(fmt.Sprintf( + "unexpected literal for left leg of logical/set binary operation (%s): %f", + op, + l.value, + ), 0, 0)) + } + + if r, ok := right.(*literalExpr); ok { + panic(newParseError(fmt.Sprintf( + "unexpected literal for right leg of logical/set binary operation (%s): %f", + op, + r.value, + ), 0, 0)) + } + } + return &binOpExpr{ SampleExpr: left, RHS: right, diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index f0df81c159b9..d95ee1d6d997 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -807,6 +807,30 @@ func TestParse(t *testing.T) { col: 0, }, }, + { + in: `sum(count_over_time({foo="bar"}[5m])) by (foo) or 1`, + err: ParseError{ + msg: `unexpected literal for right leg of logical/set binary operation (or): 1.000000`, + line: 0, + col: 0, + }, + }, + { + in: `1 unless sum(count_over_time({foo="bar"}[5m])) by (foo)`, + err: ParseError{ + msg: `unexpected literal for left leg of logical/set binary operation (unless): 1.000000`, + line: 0, + col: 0, + }, + }, + { + in: `sum(count_over_time({foo="bar"}[5m])) by (foo) + 1 or 1`, + err: ParseError{ + msg: `unexpected literal for right leg of logical/set binary operation (or): 1.000000`, + line: 0, + col: 0, + }, + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) From 64954117057bf8a41e95d4443e23b0156139064c Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Tue, 11 Feb 2020 17:42:08 -0500 Subject: [PATCH 16/24] removes comment --- pkg/logql/engine.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index d4e0cba706bb..911d83fbeec4 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -178,7 +178,6 @@ func (ng *engine) exec(ctx context.Context, q *query) (promql.Value, error) { defer cancel() qs := q.String() - // This is a legacy query used for health checking. Not the best practice, but it works. expr, err := ParseExpr(qs) if err != nil { From acaeaf2a1633f8ae36b4c769d51d6595fd5fc1d9 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Tue, 11 Feb 2020 17:45:17 -0500 Subject: [PATCH 17/24] single literalExpr tests --- pkg/logql/engine_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index d4f00788533e..2a08914d1f72 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -297,6 +297,16 @@ func TestEngine_NewInstantQuery(t *testing.T) { promql.Sample{Point: promql.Point{T: 60 * 1000, V: 2}}, }, }, + { + // single literal + `2`, + time.Unix(60, 0), logproto.FORWARD, 100, + [][]*logproto.Stream{}, + []SelectParams{}, + promql.Vector{ + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 2}}, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { @@ -1018,6 +1028,17 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, }, + { + `2`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{}, + []SelectParams{}, + promql.Matrix{ + promql.Series{ + Points: []promql.Point{{T: 60 * 1000, V: 2}}, + }, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { From 706023db7d9026dfd8d8472a08430100ada0834c Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Tue, 11 Feb 2020 18:24:14 -0500 Subject: [PATCH 18/24] reduces binops with literals in ast construction where possible --- pkg/logql/ast.go | 28 ++++++++++++++++++++++++---- pkg/logql/engine_test.go | 18 ++++++++++++++++++ pkg/logql/evaluator.go | 22 ---------------------- pkg/logql/parser_test.go | 6 +++--- 4 files changed, 45 insertions(+), 29 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 59ab3f8592ad..058f2328a30f 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -418,24 +418,32 @@ func mustNewBinOpExpr(op string, lhs, rhs Expr) SampleExpr { ), 0, 0)) } + leftLit, lOk := left.(*literalExpr) + rightLit, rOk := right.(*literalExpr) + if IsLogicalBinOp(op) { - if l, ok := left.(*literalExpr); ok { + if lOk { panic(newParseError(fmt.Sprintf( "unexpected literal for left leg of logical/set binary operation (%s): %f", op, - l.value, + leftLit.value, ), 0, 0)) } - if r, ok := right.(*literalExpr); ok { + if rOk { panic(newParseError(fmt.Sprintf( "unexpected literal for right leg of logical/set binary operation (%s): %f", op, - r.value, + rightLit.value, ), 0, 0)) } } + // map expr like (1+1) -> 2 + if lOk && rOk { + return reduceBinOp(op, leftLit, rightLit) + } + return &binOpExpr{ SampleExpr: left, RHS: right, @@ -443,6 +451,18 @@ func mustNewBinOpExpr(op string, lhs, rhs Expr) SampleExpr { } } +// Reduces a binary operation expression. A binop is reducable if both of its legs are literal expressions. +// This is because literals need match all labels, which is currently difficult to encode into StepEvaluators. +// Therefore, we ensure a binop can be reduced/simplified, maintaining the invariant that it does not have two literal legs. +func reduceBinOp(op string, left, right *literalExpr) *literalExpr { + merged := (&defaultEvaluator{}).mergeBinOp( + op, + &promql.Sample{Point: promql.Point{V: left.value}}, + &promql.Sample{Point: promql.Point{V: right.value}}, + ) + return &literalExpr{value: merged.V} +} + type literalExpr struct { value float64 } diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 2a08914d1f72..3450642a1566 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -1010,6 +1010,24 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, }, + { + `rate({app="bar"}[1m]) - 1 / 2`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: -0.3}, {T: 90 * 1000, V: -0.3}, {T: 120 * 1000, V: -0.3}, {T: 150 * 1000, V: -0.3}, {T: 180 * 1000, V: -0.3}}, + }, + }, + }, { `count_over_time({app="bar"}[1m]) ^ count_over_time({app="bar"}[1m])`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 28781469167b..54cc0c739cc0 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -108,7 +108,6 @@ func (ev *defaultEvaluator) Evaluator(ctx context.Context, expr SampleExpr, q Pa return ev.binOpEvaluator(ctx, e, q) case *literalExpr: // this is only called when a singleton is the root node return singletonEvaluator(e, q) - default: return nil, errors.Errorf("unexpected type (%T): %v", e, e) } @@ -349,15 +348,6 @@ func (ev *defaultEvaluator) binOpEvaluator( leftLit, lOk := expr.SampleExpr.(*literalExpr) rightLit, rOk := expr.RHS.(*literalExpr) - // turn expressions like 1+2 -> 3 - if lOk && rOk { - return ev.Evaluator( - ctx, - ev.reduceBinOp(expr.op, leftLit, rightLit), - q, - ) - } - // match a literal expr with all labels in the other leg if lOk { rhs, err := ev.Evaluator(ctx, expr.RHS, q) @@ -561,18 +551,6 @@ func (ev *defaultEvaluator) mergeBinOp(op string, left, right *promql.Sample) *p } -// Reduces a binary operation expression. A binop is reducable if both of its legs are literal expressions. -// This is because literals need match all labels, which is currently difficult to encode into StepEvaluators. -// Therefore, we ensure a binop can be reduced/simplified, maintaining the invariant that it does not have two literal legs. -func (ev *defaultEvaluator) reduceBinOp(op string, left, right *literalExpr) SampleExpr { - merged := ev.mergeBinOp( - op, - &promql.Sample{Point: promql.Point{V: left.value}}, - &promql.Sample{Point: promql.Point{V: right.value}}, - ) - return &literalExpr{value: merged.V} -} - // literalEvaluator merges a literal with a StepEvaluator. Since order matters in // non commutative operations, inverted should be true when the literalExpr is not the left argument. func (ev *defaultEvaluator) literalEvaluator( diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index d95ee1d6d997..088a5976b604 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -741,8 +741,8 @@ func TestParse(t *testing.T) { ), }, { - // ensure literal binops are reduced: the 1+2/expr should reduce to 3/expr - in: `sum(count_over_time({foo="bar"}[5m])) by (foo) + 2 / 1`, + // reduces binop with two literalExprs + in: `sum(count_over_time({foo="bar"}[5m])) by (foo) + 1 / 2`, exp: mustNewBinOpExpr( OpTypeAdd, mustNewVectorAggregationExpr( @@ -762,7 +762,7 @@ func TestParse(t *testing.T) { }, nil, ), - mustNewBinOpExpr(OpTypeDiv, &literalExpr{value: 2}, &literalExpr{value: 1}), + &literalExpr{value: 0.5}, ), }, { From e33ec3646daf0f0c887c9cb29576c5df59e1bad2 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Tue, 11 Feb 2020 18:35:32 -0500 Subject: [PATCH 19/24] doc updates --- CHANGELOG.md | 1 + docs/logql.md | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52833472abde..854d6d63d85f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features +* [1677](https://github.com/grafana/loki/pull/1677) **owen-d**: Introduces numeric literals in LogQL * [1662](https://github.com/grafana/loki/pull/1662) **owen-d**: Introduces binary operators in LogQL * [1572](https://github.com/grafana/loki/pull/1572) **owen-d**: Introduces the `querier.query-ingesters-within` flag and associated yaml config. When enabled, queries for a time range that do not overlap this lookback interval will not be sent to the ingesters. * [1558](https://github.com/grafana/loki/pull/1558) **owen-d**: Introduces `ingester.max-chunk-age` which specifies the maximum chunk age before it's cut. diff --git a/docs/logql.md b/docs/logql.md index 284e825af054..935a02830c1f 100644 --- a/docs/logql.md +++ b/docs/logql.md @@ -166,12 +166,24 @@ The following binary arithmetic operators exist in Loki: - `%` (modulo) - `^` (power/exponentiation) -Binary arithmetic operators are defined only between two vectors. +Binary arithmetic operators are defined between two literals (scalars), a literal and a vector, and two vectors. -Between two instant vectors, a binary arithmetic operator is applied to each entry in the left-hand side vector and its matching element in the right-hand vector. The result is propagated into the result vector with the grouping labels becoming the output label set. Entries for which no matching entry in the right-hand vector can be found are not part of the result. +Between two literals, the behavior is obvious: they evaluate to another literal that is the result of the operator applied to both scalar operands (1 + 1 = 2). + +Between a vector and a literal, the operator is applied to the value of every data sample in the vector. E.g. if a time series vector is multiplied by 2, the result is another vector in which every sample value of the original vector is multiplied by 2. + +Between two vectors, a binary arithmetic operator is applied to each entry in the left-hand side vector and its matching element in the right-hand vector. The result is propagated into the result vector with the grouping labels becoming the output label set. Entries for which no matching entry in the right-hand vector can be found are not part of the result. ##### Examples +Implement a health check with a simple query: + +> `1 + 1` + +Double the rate of a a log stream's entries: + +> `sum(rate({app="foo"})) * 2` + Get proportion of warning logs to error logs for the `foo` app > `sum(rate({app="foo", level="warn"}[1m])) / sum(rate({app="foo", level="error"}[1m]))` From 99bf381d5f06a95481d807fe0f71c2df3327cbe6 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Thu, 13 Feb 2020 18:07:40 -0500 Subject: [PATCH 20/24] scalar datatypes in logql --- pkg/logql/ast.go | 9 ++++--- pkg/logql/engine.go | 57 ++++++++++++++++++++++++++++++++++++++-- pkg/logql/engine_test.go | 12 +++------ pkg/logql/evaluator.go | 42 ++++++++++++++++++++++------- 4 files changed, 96 insertions(+), 24 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 058f2328a30f..986ec5e59713 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -256,17 +256,18 @@ type SampleExpr interface { // StepEvaluator evaluate a single step of a query. type StepEvaluator interface { - Next() (bool, int64, promql.Vector) + // while Next returns a promql.Value, the only acceptable types are Scalar and Vector. + Next() (bool, int64, promql.Value) // Close all resources used. Close() error } type stepEvaluator struct { - fn func() (bool, int64, promql.Vector) + fn func() (bool, int64, promql.Value) close func() error } -func newStepEvaluator(fn func() (bool, int64, promql.Vector), close func() error) (StepEvaluator, error) { +func newStepEvaluator(fn func() (bool, int64, promql.Value), close func() error) (StepEvaluator, error) { if fn == nil { return nil, errors.New("nil step evaluator fn") } @@ -281,7 +282,7 @@ func newStepEvaluator(fn func() (bool, int64, promql.Vector), close func() error }, nil } -func (e *stepEvaluator) Next() (bool, int64, promql.Vector) { +func (e *stepEvaluator) Next() (bool, int64, promql.Value) { return e.fn() } diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index 911d83fbeec4..2ea32f395631 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -2,6 +2,7 @@ package logql import ( "context" + "fmt" "sort" "time" @@ -212,13 +213,32 @@ func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (pr seriesIndex := map[uint64]*promql.Series{} - next, ts, vec := stepEvaluator.Next() + next, ts, data := stepEvaluator.Next() if GetRangeType(q) == InstantType { + if data.Type() == promql.ValueTypeScalar { + return data, nil + } + + vec, err := assertVec(data) + if err != nil { + return nil, err + } + sort.Slice(vec, func(i, j int) bool { return labels.Compare(vec[i].Metric, vec[j].Metric) < 0 }) return vec, nil } + // handle Scalar responses to range queries + if data.Type() == promql.ValueTypeScalar { + return PopulateMatrixFromScalar(data.(promql.Scalar), q.LiteralParams), nil + } + for next { + vec, err := assertVec(data) + if err != nil { + return nil, err + } + for _, p := range vec { var ( series *promql.Series @@ -238,7 +258,7 @@ func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (pr V: p.V, }) } - next, ts, vec = stepEvaluator.Next() + next, ts, data = stepEvaluator.Next() } series := make([]promql.Series, 0, len(seriesIndex)) @@ -250,6 +270,39 @@ func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (pr return result, nil } +func assertVec(res promql.Value) (promql.Vector, error) { + vec, ok := res.(promql.Vector) + if !ok { + return nil, fmt.Errorf("unexepected promql.Value implementor (%T)", res.Type()) + } + return vec, nil + +} + +func PopulateMatrixFromScalar(data promql.Scalar, params LiteralParams) promql.Matrix { + var ( + start = params.Start() + end = params.End() + step = params.Step() + series = promql.Series{ + Points: make( + []promql.Point, + 0, + // allocate enough space for all needed entries + int(params.End().Sub(params.Start())/params.Step())+1, + ), + } + ) + + for ts := start; !ts.After(end); ts = ts.Add(step) { + series.Points = append(series.Points, promql.Point{ + T: ts.UnixNano() / int64(time.Millisecond), + V: data.V, + }) + } + return promql.Matrix{series} +} + func readStreams(i iter.EntryIterator, size uint32) (Streams, error) { streams := map[string]*logproto.Stream{} respSize := uint32(0) diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 3450642a1566..d741fd7a6fb4 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -293,9 +293,7 @@ func TestEngine_NewInstantQuery(t *testing.T) { `1+1`, time.Unix(60, 0), logproto.FORWARD, 100, [][]*logproto.Stream{}, []SelectParams{}, - promql.Vector{ - promql.Sample{Point: promql.Point{T: 60 * 1000, V: 2}}, - }, + promql.Scalar{T: 60 * 1000, V: 2}, }, { // single literal @@ -303,9 +301,7 @@ func TestEngine_NewInstantQuery(t *testing.T) { time.Unix(60, 0), logproto.FORWARD, 100, [][]*logproto.Stream{}, []SelectParams{}, - promql.Vector{ - promql.Sample{Point: promql.Point{T: 60 * 1000, V: 2}}, - }, + promql.Scalar{T: 60 * 1000, V: 2}, }, } { test := test @@ -970,7 +966,7 @@ func TestEngine_NewRangeQuery(t *testing.T) { []SelectParams{}, promql.Matrix{ promql.Series{ - Points: []promql.Point{{T: 60 * 1000, V: 3}}, + Points: []promql.Point{{T: 60000, V: 3}, {T: 90000, V: 3}, {T: 120000, V: 3}, {T: 150000, V: 3}, {T: 180000, V: 3}}, }, }, }, @@ -1053,7 +1049,7 @@ func TestEngine_NewRangeQuery(t *testing.T) { []SelectParams{}, promql.Matrix{ promql.Series{ - Points: []promql.Point{{T: 60 * 1000, V: 2}}, + Points: []promql.Point{{T: 60 * 1000, V: 2}, {T: 90 * 1000, V: 2}, {T: 120 * 1000, V: 2}, {T: 150 * 1000, V: 2}, {T: 180 * 1000, V: 2}}, }, }, }, diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 54cc0c739cc0..ad1aa42cc42f 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -119,8 +119,15 @@ func (ev *defaultEvaluator) vectorAggEvaluator(ctx context.Context, expr *vector return nil, err } - return newStepEvaluator(func() (bool, int64, promql.Vector) { - next, ts, vec := nextEvaluator.Next() + return newStepEvaluator(func() (bool, int64, promql.Value) { + next, ts, data := nextEvaluator.Next() + + // vector aggregations can only consume vectors and this should never not be a vector. + vec, err := assertVec(data) + if err != nil { + panic(err) + } + if !next { return false, 0, promql.Vector{} } @@ -328,7 +335,7 @@ func (ev *defaultEvaluator) rangeAggEvaluator(ctx context.Context, expr *rangeAg fn = count } - return newStepEvaluator(func() (bool, int64, promql.Vector) { + return newStepEvaluator(func() (bool, int64, promql.Value) { next := vecIter.Next() if !next { return false, 0, promql.Vector{} @@ -374,13 +381,20 @@ func (ev *defaultEvaluator) binOpEvaluator( return nil, err } - return newStepEvaluator(func() (bool, int64, promql.Vector) { + return newStepEvaluator(func() (bool, int64, promql.Value) { pairs := map[uint64][2]*promql.Sample{} var ts int64 // populate pairs for i, eval := range []StepEvaluator{lhs, rhs} { - next, timestamp, vec := eval.Next() + next, timestamp, data := eval.Next() + // since we've already handled literalExprs, + // the step evaluators are guaranteed to produce vectors + vec, err := assertVec(data) + if err != nil { + panic(err) + } + ts = timestamp // These should _always_ happen at the same step on each evaluator. @@ -560,8 +574,16 @@ func (ev *defaultEvaluator) literalEvaluator( inverted bool, ) (StepEvaluator, error) { return newStepEvaluator( - func() (bool, int64, promql.Vector) { - ok, ts, vec := eval.Next() + func() (bool, int64, promql.Value) { + ok, ts, data := eval.Next() + // literalEvaluator must never be called with a non vector + // producing StepEvaluator. This would only happen in the case of + // two literal legs in a binaryExpr, but these are automatically reduced + // to a single Scalar during ast construction in mustNewBinOpExpr. + vec, err := assertVec(data) + if err != nil { + panic(err) + } results := make(promql.Vector, 0, len(vec)) for _, sample := range vec { @@ -594,7 +616,7 @@ func (ev *defaultEvaluator) literalEvaluator( func singletonEvaluator(expr *literalExpr, params Params) (StepEvaluator, error) { ok := true return newStepEvaluator( - func() (bool, int64, promql.Vector) { + func() (bool, int64, promql.Value) { if !ok { return ok, 0, nil } @@ -602,10 +624,10 @@ func singletonEvaluator(expr *literalExpr, params Params) (StepEvaluator, error) ok = false ts := params.Start().UnixNano() / int64(time.Millisecond) - return tmp, ts, promql.Vector{promql.Sample{Point: promql.Point{ + return tmp, ts, promql.Scalar{ T: ts, V: expr.value, - }}} + } }, nil, ) From cb4626fdf8ed81196282bf3e0ab8f4b487bd690d Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Thu, 13 Feb 2020 18:33:09 -0500 Subject: [PATCH 21/24] scalar serialization type --- pkg/loghttp/query.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/loghttp/query.go b/pkg/loghttp/query.go index c6d17c4b3d3f..c6180474ecd8 100644 --- a/pkg/loghttp/query.go +++ b/pkg/loghttp/query.go @@ -46,6 +46,7 @@ type ResultType string // ResultType values const ( ResultTypeStream = "streams" + ResultTypeScalar = "scalar" ResultTypeVector = "vector" ResultTypeMatrix = "matrix" ) @@ -65,6 +66,9 @@ type QueryResponseData struct { // Type implements the promql.Value interface func (Streams) Type() ResultType { return ResultTypeStream } +// Type implements the promql.Value interface +func (Scalar) Type() ResultType { return ResultTypeScalar } + // Type implements the promql.Value interface func (Vector) Type() ResultType { return ResultTypeVector } @@ -127,6 +131,10 @@ func (q *QueryResponseData) UnmarshalJSON(data []byte) error { var v Vector err = json.Unmarshal(unmarshal.Result, &v) value = v + case ResultTypeScalar: + var v Scalar + err = json.Unmarshal(unmarshal.Result, &v) + value = v default: return fmt.Errorf("unknown type: %s", unmarshal.Type) } @@ -171,6 +179,9 @@ func (e *Entry) UnmarshalJSON(data []byte) error { return nil } +// Scalar is a single timestamp/float with no labels +type Scalar model.Scalar + // Vector is a slice of Samples type Vector []model.Sample From e689e8845f4debfadfaaa14502c02c5a2d5b2b55 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Thu, 13 Feb 2020 19:35:37 -0500 Subject: [PATCH 22/24] increases safety and reduces complexity in ast evaluator --- pkg/logql/ast.go | 37 ----------------------- pkg/logql/engine.go | 42 ++++++++++---------------- pkg/logql/evaluator.go | 60 ++++++------------------------------- pkg/logql/step_evaluator.go | 43 ++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 114 deletions(-) create mode 100644 pkg/logql/step_evaluator.go diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 986ec5e59713..1acbdbd82dd9 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -3,7 +3,6 @@ package logql import ( "bytes" "context" - "errors" "fmt" "regexp" "strconv" @@ -254,42 +253,6 @@ type SampleExpr interface { Expr } -// StepEvaluator evaluate a single step of a query. -type StepEvaluator interface { - // while Next returns a promql.Value, the only acceptable types are Scalar and Vector. - Next() (bool, int64, promql.Value) - // Close all resources used. - Close() error -} - -type stepEvaluator struct { - fn func() (bool, int64, promql.Value) - close func() error -} - -func newStepEvaluator(fn func() (bool, int64, promql.Value), close func() error) (StepEvaluator, error) { - if fn == nil { - return nil, errors.New("nil step evaluator fn") - } - - if close == nil { - close = func() error { return nil } - } - - return &stepEvaluator{ - fn: fn, - close: close, - }, nil -} - -func (e *stepEvaluator) Next() (bool, int64, promql.Value) { - return e.fn() -} - -func (e *stepEvaluator) Close() error { - return e.close() -} - type rangeAggregationExpr struct { left *logRange operation string diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index 2ea32f395631..cb94254458ae 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -2,7 +2,6 @@ package logql import ( "context" - "fmt" "sort" "time" @@ -205,6 +204,10 @@ func (ng *engine) exec(ctx context.Context, q *query) (promql.Value, error) { // evalSample evaluate a sampleExpr func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (promql.Value, error) { + if lit, ok := expr.(*literalExpr); ok { + return ng.evalLiteral(ctx, lit, q) + } + stepEvaluator, err := ng.evaluator.Evaluator(ctx, expr, q) if err != nil { return nil, err @@ -213,31 +216,13 @@ func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (pr seriesIndex := map[uint64]*promql.Series{} - next, ts, data := stepEvaluator.Next() + next, ts, vec := stepEvaluator.Next() if GetRangeType(q) == InstantType { - if data.Type() == promql.ValueTypeScalar { - return data, nil - } - - vec, err := assertVec(data) - if err != nil { - return nil, err - } - sort.Slice(vec, func(i, j int) bool { return labels.Compare(vec[i].Metric, vec[j].Metric) < 0 }) return vec, nil } - // handle Scalar responses to range queries - if data.Type() == promql.ValueTypeScalar { - return PopulateMatrixFromScalar(data.(promql.Scalar), q.LiteralParams), nil - } - for next { - vec, err := assertVec(data) - if err != nil { - return nil, err - } for _, p := range vec { var ( @@ -258,7 +243,7 @@ func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (pr V: p.V, }) } - next, ts, data = stepEvaluator.Next() + next, ts, vec = stepEvaluator.Next() } series := make([]promql.Series, 0, len(seriesIndex)) @@ -270,12 +255,17 @@ func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (pr return result, nil } -func assertVec(res promql.Value) (promql.Vector, error) { - vec, ok := res.(promql.Vector) - if !ok { - return nil, fmt.Errorf("unexepected promql.Value implementor (%T)", res.Type()) +func (ng *engine) evalLiteral(_ context.Context, expr *literalExpr, q *query) (promql.Value, error) { + s := promql.Scalar{ + T: q.Start().UnixNano() / int64(time.Millisecond), + V: expr.value, } - return vec, nil + + if GetRangeType(q) == InstantType { + return s, nil + } + + return PopulateMatrixFromScalar(s, q.LiteralParams), nil } diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index ad1aa42cc42f..2140eda55822 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -106,8 +106,6 @@ func (ev *defaultEvaluator) Evaluator(ctx context.Context, expr SampleExpr, q Pa return ev.rangeAggEvaluator(ctx, e, q) case *binOpExpr: return ev.binOpEvaluator(ctx, e, q) - case *literalExpr: // this is only called when a singleton is the root node - return singletonEvaluator(e, q) default: return nil, errors.Errorf("unexpected type (%T): %v", e, e) } @@ -119,14 +117,8 @@ func (ev *defaultEvaluator) vectorAggEvaluator(ctx context.Context, expr *vector return nil, err } - return newStepEvaluator(func() (bool, int64, promql.Value) { - next, ts, data := nextEvaluator.Next() - - // vector aggregations can only consume vectors and this should never not be a vector. - vec, err := assertVec(data) - if err != nil { - panic(err) - } + return newStepEvaluator(func() (bool, int64, promql.Vector) { + next, ts, vec := nextEvaluator.Next() if !next { return false, 0, promql.Vector{} @@ -335,7 +327,7 @@ func (ev *defaultEvaluator) rangeAggEvaluator(ctx context.Context, expr *rangeAg fn = count } - return newStepEvaluator(func() (bool, int64, promql.Value) { + return newStepEvaluator(func() (bool, int64, promql.Vector) { next := vecIter.Next() if !next { return false, 0, promql.Vector{} @@ -346,6 +338,8 @@ func (ev *defaultEvaluator) rangeAggEvaluator(ctx context.Context, expr *rangeAg }, vecIter.Close) } +// binOpExpr explicly does not handle when both legs are literals as +// it makes the type system simpler and these are reduced in mustNewBinOpExpr func (ev *defaultEvaluator) binOpEvaluator( ctx context.Context, expr *binOpExpr, @@ -381,19 +375,13 @@ func (ev *defaultEvaluator) binOpEvaluator( return nil, err } - return newStepEvaluator(func() (bool, int64, promql.Value) { + return newStepEvaluator(func() (bool, int64, promql.Vector) { pairs := map[uint64][2]*promql.Sample{} var ts int64 // populate pairs for i, eval := range []StepEvaluator{lhs, rhs} { - next, timestamp, data := eval.Next() - // since we've already handled literalExprs, - // the step evaluators are guaranteed to produce vectors - vec, err := assertVec(data) - if err != nil { - panic(err) - } + next, timestamp, vec := eval.Next() ts = timestamp @@ -574,16 +562,8 @@ func (ev *defaultEvaluator) literalEvaluator( inverted bool, ) (StepEvaluator, error) { return newStepEvaluator( - func() (bool, int64, promql.Value) { - ok, ts, data := eval.Next() - // literalEvaluator must never be called with a non vector - // producing StepEvaluator. This would only happen in the case of - // two literal legs in a binaryExpr, but these are automatically reduced - // to a single Scalar during ast construction in mustNewBinOpExpr. - vec, err := assertVec(data) - if err != nil { - panic(err) - } + func() (bool, int64, promql.Vector) { + ok, ts, vec := eval.Next() results := make(promql.Vector, 0, len(vec)) for _, sample := range vec { @@ -611,25 +591,3 @@ func (ev *defaultEvaluator) literalEvaluator( eval.Close, ) } - -// singletonEvaluator is an evaluator adapter for a literal expressions that is a root node -func singletonEvaluator(expr *literalExpr, params Params) (StepEvaluator, error) { - ok := true - return newStepEvaluator( - func() (bool, int64, promql.Value) { - if !ok { - return ok, 0, nil - } - tmp := ok - ok = false - ts := params.Start().UnixNano() / int64(time.Millisecond) - - return tmp, ts, promql.Scalar{ - T: ts, - V: expr.value, - } - }, - nil, - ) - -} diff --git a/pkg/logql/step_evaluator.go b/pkg/logql/step_evaluator.go new file mode 100644 index 000000000000..571b2453a350 --- /dev/null +++ b/pkg/logql/step_evaluator.go @@ -0,0 +1,43 @@ +package logql + +import ( + "errors" + + "github.com/prometheus/prometheus/promql" +) + +// StepEvaluator evaluate a single step of a query. +type StepEvaluator interface { + // while Next returns a promql.Value, the only acceptable types are Scalar and Vector. + Next() (bool, int64, promql.Vector) + // Close all resources used. + Close() error +} + +type stepEvaluator struct { + fn func() (bool, int64, promql.Vector) + close func() error +} + +func newStepEvaluator(fn func() (bool, int64, promql.Vector), close func() error) (StepEvaluator, error) { + if fn == nil { + return nil, errors.New("nil step evaluator fn") + } + + if close == nil { + close = func() error { return nil } + } + + return &stepEvaluator{ + fn: fn, + close: close, + }, nil +} + +func (e *stepEvaluator) Next() (bool, int64, promql.Vector) { + return e.fn() +} + +func (e *stepEvaluator) Close() error { + return e.close() +} From d4780b1c51a90ec37d38062ca6140b7fc0593112 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Thu, 13 Feb 2020 19:50:15 -0500 Subject: [PATCH 23/24] recursive literal binop reduction parse test, improves parse errors on literal construction --- pkg/logql/ast.go | 2 +- pkg/logql/parser_test.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 1acbdbd82dd9..162cc068c129 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -434,7 +434,7 @@ type literalExpr struct { func mustNewLiteralExpr(s string, invert bool) *literalExpr { n, err := strconv.ParseFloat(s, 64) if err != nil { - panic(err) + panic(newParseError(fmt.Sprintf("unable to parse literal as a float: %s", err.Error()), 0, 0)) } if invert { diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 088a5976b604..3a35fb4f7626 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -783,6 +783,11 @@ func TestParse(t *testing.T) { &literalExpr{value: -1}, ), }, + { + // ensure binary ops with two literals are reduced recursively + in: `1 + 1 + 1`, + exp: &literalExpr{value: 3}, + }, { in: `{foo="bar"} + {foo="bar"}`, err: ParseError{ From 3fd28bb8004b214f6e750cab8f1f7a8af0bbfc65 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 14 Feb 2020 14:13:15 -0500 Subject: [PATCH 24/24] vector + literal test --- pkg/logql/engine_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index d741fd7a6fb4..b226d116c65d 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -288,6 +288,21 @@ func TestEngine_NewInstantQuery(t *testing.T) { promql.Sample{Point: promql.Point{T: 60 * 1000, V: 0.2}, Metric: labels.Labels{labels.Label{Name: "app", Value: "fuzz"}}}, }, }, + { + `bottomk(3,rate(({app=~"foo|bar"} |~".+bar")[1m])) without (app) + 1`, time.Unix(60, 0), logproto.FORWARD, 100, + [][]*logproto.Stream{ + {newStream(testSize, factor(10, identity), `{app="foo"}`), newStream(testSize, offset(46, identity), `{app="bar"}`), + newStream(testSize, factor(5, identity), `{app="fuzz"}`), newStream(testSize, identity, `{app="buzz"}`)}, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(60, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + }, + promql.Vector{ + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 1.25}, Metric: labels.Labels{labels.Label{Name: "app", Value: "bar"}}}, + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 1.1}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}, + promql.Sample{Point: promql.Point{T: 60 * 1000, V: 1.2}, Metric: labels.Labels{labels.Label{Name: "app", Value: "fuzz"}}}, + }, + }, { // healthcheck `1+1`, time.Unix(60, 0), logproto.FORWARD, 100,