From 4fc7cbd08cfff1f896d1e90cf815b2f059f8f5a2 Mon Sep 17 00:00:00 2001 From: Andrey Karpov Date: Fri, 10 Jan 2025 00:37:29 +0000 Subject: [PATCH 1/2] TraceQL: support mixed-type attribute querying (int/float) --- pkg/parquetquery/iters.go | 10 +- tempodb/encoding/vparquet2/block_traceql.go | 81 +++++++ .../encoding/vparquet2/block_traceql_test.go | 74 +++++- tempodb/encoding/vparquet3/block_traceql.go | 81 +++++++ .../encoding/vparquet3/block_traceql_test.go | 74 +++++- tempodb/encoding/vparquet4/block_traceql.go | 209 +++++++++++++++++ .../encoding/vparquet4/block_traceql_test.go | 216 ++++++++++++++++++ tempodb/tempodb_search_test.go | 108 ++++++++- 8 files changed, 846 insertions(+), 7 deletions(-) diff --git a/pkg/parquetquery/iters.go b/pkg/parquetquery/iters.go index 7d8aea60bc4..1544efca9bb 100644 --- a/pkg/parquetquery/iters.go +++ b/pkg/parquetquery/iters.go @@ -7,6 +7,8 @@ import ( "fmt" "io" "math" + "slices" + "strings" "sync" "sync/atomic" @@ -1805,10 +1807,12 @@ func (j *LeftJoinIterator) String() string { for _, r := range j.required { srequired += "\n\t" + util.TabOut(r) } - soptional := "optional: " - for _, o := range j.optional { - soptional += "\n\t" + util.TabOut(o) + optional := make([]string, len(j.optional)) + for i, o := range j.optional { + optional[i] = "\n\t" + util.TabOut(o) } + slices.Sort(optional) + soptional := "optional: " + strings.Join(optional, "") return fmt.Sprintf("LeftJoinIterator: %d: %s\n%s\n%s", j.definitionLevel, j.pred, srequired, soptional) } diff --git a/tempodb/encoding/vparquet2/block_traceql.go b/tempodb/encoding/vparquet2/block_traceql.go index afbafa11e9d..72044ecd081 100644 --- a/tempodb/encoding/vparquet2/block_traceql.go +++ b/tempodb/encoding/vparquet2/block_traceql.go @@ -1759,6 +1759,71 @@ func createIntPredicate(op traceql.Operator, operands traceql.Operands) (parquet } } +// createIntPredicateFromFloat adapts float-based queries to integer columns. +// If the float operand has no fractional part, it's treated as an integer directly. +// Otherwise, specific shifts are applied (e.g., floor or ceil) depending on the operator, +// or conclude that equality is impossible. +// +// Example: { spanAttr > 3.5 } but 'spanAttr' is stored as int. We'll look for rows where 'spanAttr' >= 4. +func createIntPredicateFromFloat(op traceql.Operator, operands traceql.Operands) (parquetquery.Predicate, error) { + if op == traceql.OpNone { + return nil, nil + } + + f := operands[0].Float() + // Check if f has a fractional part + if _, frac := math.Modf(f); frac == 0 { + // If it's an integer float, treat it purely as int + intOperands := traceql.Operands{traceql.NewStaticInt(int(f))} + return createIntPredicate(op, intOperands) + } + + switch op { + case traceql.OpEqual: + // No integer can be strictly equal to a float with a fractional part + return nil, nil + case traceql.OpNotEqual: + // An integer will always differ from a float that has a fractional part + return parquetquery.NewCallbackPredicate(func() bool { return true }), nil + case traceql.OpGreater, traceql.OpGreaterEqual: + // For > 3.5 or >= 3.5, effectively we do >= 4 + // For > -3.5 or >= -3.5, effectively we do >= -3 + i := int(f) + if i > 0 { + i++ + } + return createIntPredicate(traceql.OpGreaterEqual, traceql.Operands{traceql.NewStaticInt(i)}) + case traceql.OpLess, traceql.OpLessEqual: + // For < 3.5 or <= 3.5, effectively we do <= 3 + // For < -3.5 or <= -3.5, effectively we do <= -4 + i := int(f) + if i < 0 { + i-- + } + return createIntPredicate(traceql.OpLessEqual, traceql.Operands{traceql.NewStaticInt(i)}) + default: + return nil, fmt.Errorf("unsupported operator for float to int conversion: %v", op) + } +} + +// createFloatPredicateFromInt adapts integer-based queries to float columns. +// If the operand can be interpreted as an integer, it's converted to float +// and we delegate further processing to createFloatPredicate. +// +// Example: { spanAttr = 5 } but 'spanAttr' is stored as float. We'll look for rows where 'spanAttr' = 5.0. +func createFloatPredicateFromInt(op traceql.Operator, operands traceql.Operands) (parquetquery.Predicate, error) { + if op == traceql.OpNone { + return nil, nil + } + + if i, ok := operands[0].Int(); ok { + floatOperands := traceql.Operands{traceql.NewStaticFloat(float64(i))} + return createFloatPredicate(op, floatOperands) + } + + return nil, nil +} + func createFloatPredicate(op traceql.Operator, operands traceql.Operands) (parquetquery.Predicate, error) { if op == traceql.OpNone { return nil, nil @@ -1846,13 +1911,29 @@ func createAttributeIterator(makeIter makeIterFn, conditions []traceql.Condition attrStringPreds = append(attrStringPreds, pred) case traceql.TypeInt: + // Create a predicate specifically for integer comparisons pred, err := createIntPredicate(cond.Op, cond.Operands) if err != nil { return nil, fmt.Errorf("creating attribute predicate: %w", err) } attrIntPreds = append(attrIntPreds, pred) + // If the operand can be interpreted as a float, create an additional predicate + if pred, err := createFloatPredicateFromInt(cond.Op, cond.Operands); err != nil { + return nil, fmt.Errorf("creating float attribute predicate from int: %w", err) + } else if pred != nil { + attrFltPreds = append(attrFltPreds, pred) + } + case traceql.TypeFloat: + // Attempt to create a predicate for integer comparisons, if applicable + if pred, err := createIntPredicateFromFloat(cond.Op, cond.Operands); err != nil { + return nil, fmt.Errorf("creating int attribute predicate from float: %w", err) + } else if pred != nil { + attrIntPreds = append(attrIntPreds, pred) + } + + // Create a predicate specifically for float comparisons pred, err := createFloatPredicate(cond.Op, cond.Operands) if err != nil { return nil, fmt.Errorf("creating attribute predicate: %w", err) diff --git a/tempodb/encoding/vparquet2/block_traceql_test.go b/tempodb/encoding/vparquet2/block_traceql_test.go index 92fc146c929..9e148f1b54d 100644 --- a/tempodb/encoding/vparquet2/block_traceql_test.go +++ b/tempodb/encoding/vparquet2/block_traceql_test.go @@ -209,6 +209,39 @@ func TestBackendBlockSearchTraceQL(t *testing.T) { parse(t, `{resource.`+LabelServiceName+` <= 124}`), }, }, + // Cross-type comparisons + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint > 122.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint >= 122.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint <= 123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint = 123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint != 123.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint >= 123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint < 123.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint <= 123.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint > -123.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint >= -123.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint <= -123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint = -123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint != -123.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint >= -123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint < -122.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint <= -122.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag > 455}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag >= 455}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag <= 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag = 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag != 457}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag >= 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag <= 457}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag < 457}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag != 455}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag > 455}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag >= 455}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag > 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag >= 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag <= 457}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag < 457}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag != 457}`), } for _, req := range searchesThatMatch { @@ -316,6 +349,41 @@ func TestBackendBlockSearchTraceQL(t *testing.T) { parse(t, `{`+LabelDuration+` = 100s }`), // Match }, }, + // Cross-type comparisons + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint < 122.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint = 122.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint <= 122.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint < 123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint != 123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint > 123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint >= 123.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint = 123.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint > 123.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint < -123.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint = -122.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint = -123.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint <= -123.9}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint < -123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint != -123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint > -123.0}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint >= -122.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint = -122.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint > -122.1}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag < 455}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag = 455}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag <= 455}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag < 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag != 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag > 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag >= 457}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag = 457}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag > 457}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag < 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag = 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag <= 456}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag >= 457}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag = 457}`), + traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag > 457}`), } for _, req := range searchesThatDontMatch { @@ -431,7 +499,11 @@ func fullyPopulatedTestTrace(id common.ID) *Trace { {Key: "bar", ValueInt: intPtr(123)}, {Key: "float", ValueDouble: fltPtr(456.78)}, {Key: "bool", ValueBool: boolPtr(false)}, - + // For cross-type comparisons + {Key: "crossint", ValueInt: intPtr(123)}, + {Key: "crossnint", ValueInt: intPtr(-123)}, + {Key: "crossfloat_nofrag", ValueDouble: fltPtr(456.0)}, + {Key: "crossfloat_frag", ValueDouble: fltPtr(456.78)}, // Edge-cases {Key: LabelName, Value: strPtr("Bob")}, // Conflicts with intrinsic but still looked up by .name {Key: LabelServiceName, Value: strPtr("spanservicename")}, // Overrides resource-level dedicated column diff --git a/tempodb/encoding/vparquet3/block_traceql.go b/tempodb/encoding/vparquet3/block_traceql.go index 209c757c644..3881ce5c8ed 100644 --- a/tempodb/encoding/vparquet3/block_traceql.go +++ b/tempodb/encoding/vparquet3/block_traceql.go @@ -2063,6 +2063,71 @@ func createIntPredicate(op traceql.Operator, operands traceql.Operands) (parquet } } +// createIntPredicateFromFloat adapts float-based queries to integer columns. +// If the float operand has no fractional part, it's treated as an integer directly. +// Otherwise, specific shifts are applied (e.g., floor or ceil) depending on the operator, +// or conclude that equality is impossible. +// +// Example: { spanAttr > 3.5 } but 'spanAttr' is stored as int. We'll look for rows where 'spanAttr' >= 4. +func createIntPredicateFromFloat(op traceql.Operator, operands traceql.Operands) (parquetquery.Predicate, error) { + if op == traceql.OpNone { + return nil, nil + } + + f := operands[0].Float() + // Check if f has a fractional part + if _, frac := math.Modf(f); frac == 0 { + // If it's an integer float, treat it purely as int + intOperands := traceql.Operands{traceql.NewStaticInt(int(f))} + return createIntPredicate(op, intOperands) + } + + switch op { + case traceql.OpEqual: + // No integer can be strictly equal to a float with a fractional part + return nil, nil + case traceql.OpNotEqual: + // An integer will always differ from a float that has a fractional part + return parquetquery.NewCallbackPredicate(func() bool { return true }), nil + case traceql.OpGreater, traceql.OpGreaterEqual: + // For > 3.5 or >= 3.5, effectively we do >= 4 + // For > -3.5 or >= -3.5, effectively we do >= -3 + i := int(f) + if i > 0 { + i++ + } + return createIntPredicate(traceql.OpGreaterEqual, traceql.Operands{traceql.NewStaticInt(i)}) + case traceql.OpLess, traceql.OpLessEqual: + // For < 3.5 or <= 3.5, effectively we do <= 3 + // For < -3.5 or <= -3.5, effectively we do <= -4 + i := int(f) + if i < 0 { + i-- + } + return createIntPredicate(traceql.OpLessEqual, traceql.Operands{traceql.NewStaticInt(i)}) + default: + return nil, fmt.Errorf("unsupported operator for float to int conversion: %v", op) + } +} + +// createFloatPredicateFromInt adapts integer-based queries to float columns. +// If the operand can be interpreted as an integer, it's converted to float +// and we delegate further processing to createFloatPredicate. +// +// Example: { spanAttr = 5 } but 'spanAttr' is stored as float. We'll look for rows where 'spanAttr' = 5.0. +func createFloatPredicateFromInt(op traceql.Operator, operands traceql.Operands) (parquetquery.Predicate, error) { + if op == traceql.OpNone { + return nil, nil + } + + if i, ok := operands[0].Int(); ok { + floatOperands := traceql.Operands{traceql.NewStaticFloat(float64(i))} + return createFloatPredicate(op, floatOperands) + } + + return nil, nil +} + func createFloatPredicate(op traceql.Operator, operands traceql.Operands) (parquetquery.Predicate, error) { if op == traceql.OpNone { return nil, nil @@ -2164,13 +2229,29 @@ func createAttributeIterator(makeIter makeIterFn, conditions []traceql.Condition attrStringPreds = append(attrStringPreds, pred) case traceql.TypeInt: + // Create a predicate specifically for integer comparisons pred, err := createIntPredicate(cond.Op, cond.Operands) if err != nil { return nil, fmt.Errorf("creating attribute predicate: %w", err) } attrIntPreds = append(attrIntPreds, pred) + // If the operand can be interpreted as a float, create an additional predicate + if pred, err := createFloatPredicateFromInt(cond.Op, cond.Operands); err != nil { + return nil, fmt.Errorf("creating float attribute predicate from int: %w", err) + } else if pred != nil { + attrFltPreds = append(attrFltPreds, pred) + } + case traceql.TypeFloat: + // Attempt to create a predicate for integer comparisons, if applicable + if pred, err := createIntPredicateFromFloat(cond.Op, cond.Operands); err != nil { + return nil, fmt.Errorf("creating int attribute predicate from float: %w", err) + } else if pred != nil { + attrIntPreds = append(attrIntPreds, pred) + } + + // Create a predicate specifically for float comparisons pred, err := createFloatPredicate(cond.Op, cond.Operands) if err != nil { return nil, fmt.Errorf("creating attribute predicate: %w", err) diff --git a/tempodb/encoding/vparquet3/block_traceql_test.go b/tempodb/encoding/vparquet3/block_traceql_test.go index 8ab94aff874..c8199b6bdcf 100644 --- a/tempodb/encoding/vparquet3/block_traceql_test.go +++ b/tempodb/encoding/vparquet3/block_traceql_test.go @@ -222,6 +222,39 @@ func TestBackendBlockSearchTraceQL(t *testing.T) { }, }, }, + // Cross-type comparisons + {".crossint > 122.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint > 122.9}`)}, + {".crossint >= 122.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint >= 122.9}`)}, + {".crossint <= 123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint <= 123.0}`)}, + {".crossint = 123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint = 123.0}`)}, + {".crossint != 123.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint != 123.1}`)}, + {".crossint >= 123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint >= 123.0}`)}, + {".crossint < 123.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint < 123.1}`)}, + {".crossint <= 123.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint <= 123.1}`)}, + {".crossnint > -123.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint > -123.9}`)}, + {".crossnint >= -123.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint >= -123.9}`)}, + {".crossnint <= -123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint <= -123.0}`)}, + {".crossnint = -123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint = -123.0}`)}, + {".crossnint != -123.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint != -123.1}`)}, + {".crossnint >= -123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint >= -123.0}`)}, + {".crossnint < -122.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint < -122.1}`)}, + {".crossnint <= -122.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint <= -122.1}`)}, + {".crossfloat_nofrag > 455", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag > 455}`)}, + {".crossfloat_nofrag >= 455", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag >= 455}`)}, + {".crossfloat_nofrag <= 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag <= 456}`)}, + {".crossfloat_nofrag = 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag = 456}`)}, + {".crossfloat_nofrag != 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag != 457}`)}, + {".crossfloat_nofrag >= 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag >= 456}`)}, + {".crossfloat_nofrag <= 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag <= 457}`)}, + {".crossfloat_nofrag < 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag < 457}`)}, + {".crossfloat_frag != 455", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag != 455}`)}, + {".crossfloat_frag > 455", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag > 455}`)}, + {".crossfloat_frag >= 455", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag >= 455}`)}, + {".crossfloat_frag > 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag > 456}`)}, + {".crossfloat_frag >= 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag >= 456}`)}, + {".crossfloat_frag <= 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag <= 457}`)}, + {".crossfloat_frag < 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag < 457}`)}, + {".crossfloat_frag != 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag != 457}`)}, } for _, tc := range searchesThatMatch { @@ -348,6 +381,41 @@ func TestBackendBlockSearchTraceQL(t *testing.T) { }, }, }, + // Cross-type comparisons + {".crossint < 122.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint < 122.9}`)}, + {".crossint = 122.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint = 122.9}`)}, + {".crossint <= 122.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint <= 122.9}`)}, + {".crossint < 123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint < 123.0}`)}, + {".crossint != 123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint != 123.0}`)}, + {".crossint > 123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint > 123.0}`)}, + {".crossint >= 123.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint >= 123.1}`)}, + {".crossint = 123.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint = 123.1}`)}, + {".crossint > 123.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossint > 123.1}`)}, + {".crossnint < -123.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint < -123.9}`)}, + {".crossnint = -122.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint = -122.9}`)}, + {".crossnint = -123.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint = -123.9}`)}, + {".crossnint <= -123.9", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint <= -123.9}`)}, + {".crossnint < -123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint < -123.0}`)}, + {".crossnint != -123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint != -123.0}`)}, + {".crossnint > -123.0", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint > -123.0}`)}, + {".crossnint >= -122.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint >= -122.1}`)}, + {".crossnint = -122.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint = -122.1}`)}, + {".crossnint > -122.1", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossnint > -122.1}`)}, + {".crossfloat_nofrag < 455", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag < 455}`)}, + {".crossfloat_nofrag = 455", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag = 455}`)}, + {".crossfloat_nofrag <= 455", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag <= 455}`)}, + {".crossfloat_nofrag < 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag < 456}`)}, + {".crossfloat_nofrag != 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag != 456}`)}, + {".crossfloat_nofrag > 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag > 456}`)}, + {".crossfloat_nofrag >= 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag >= 457}`)}, + {".crossfloat_nofrag = 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag = 457}`)}, + {".crossfloat_nofrag > 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_nofrag > 457}`)}, + {".crossfloat_frag < 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag < 456}`)}, + {".crossfloat_frag = 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag = 456}`)}, + {".crossfloat_frag <= 456", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag <= 456}`)}, + {".crossfloat_frag >= 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag >= 457}`)}, + {".crossfloat_frag = 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag = 457}`)}, + {".crossfloat_frag > 457", traceql.MustExtractFetchSpansRequestWithMetadata(`{.crossfloat_frag > 457}`)}, } for _, tc := range searchesThatDontMatch { @@ -475,7 +543,11 @@ func fullyPopulatedTestTrace(id common.ID) *Trace { {Key: "bar", ValueInt: intPtr(123)}, {Key: "float", ValueDouble: fltPtr(456.78)}, {Key: "bool", ValueBool: boolPtr(false)}, - + // For cross-type comparisons + {Key: "crossint", ValueInt: intPtr(123)}, + {Key: "crossnint", ValueInt: intPtr(-123)}, + {Key: "crossfloat_nofrag", ValueDouble: fltPtr(456.0)}, + {Key: "crossfloat_frag", ValueDouble: fltPtr(456.78)}, // Edge-cases {Key: LabelName, Value: strPtr("Bob")}, // Conflicts with intrinsic but still looked up by .name {Key: LabelServiceName, Value: strPtr("spanservicename")}, // Overrides resource-level dedicated column diff --git a/tempodb/encoding/vparquet4/block_traceql.go b/tempodb/encoding/vparquet4/block_traceql.go index f67fc272e36..f4766518f68 100644 --- a/tempodb/encoding/vparquet4/block_traceql.go +++ b/tempodb/encoding/vparquet4/block_traceql.go @@ -1973,6 +1973,51 @@ func createSpanIterator(makeIter makeIterFn, innerIterators []parquetquery.Itera continue } + // If the attribute is integer (most likely `span.http.status_code`), + // we also scan column float64 besides the dedicated column + // in case the attribute somehow was stored as a float. + if entry.typ == traceql.TypeInt && + (operandType(cond.Operands) == traceql.TypeInt || + operandType(cond.Operands) == traceql.TypeFloat) { + + subIters := make([]parquetquery.Iterator, 0, 2) + { + var pred parquetquery.Predicate + var err error + if operandType(cond.Operands) == traceql.TypeInt { + pred, err = createIntPredicate(cond.Op, cond.Operands) + } else { + pred, err = createIntPredicateFromFloat(cond.Op, cond.Operands) + } + if err != nil { + return nil, fmt.Errorf("creating predicate: %w", err) + } + if pred != nil { + subIters = append(subIters, makeIter(entry.columnPath, pred, cond.Attribute.Name)) + } + } + { + var pred parquetquery.Predicate + var err error + if operandType(cond.Operands) == traceql.TypeFloat { + pred, err = createFloatPredicate(cond.Op, cond.Operands) + } else if operandType(cond.Operands) == traceql.TypeInt { + pred, err = createFloatPredicateFromInt(cond.Op, cond.Operands) + } + if err != nil { + return nil, fmt.Errorf("creating predicate: %w", err) + } + if pred != nil { + subIters = append(subIters, makeIter(columnPathResourceAttrDouble, pred, cond.Attribute.Name)) + } + } + if unionItr := unionIfNeeded(DefinitionLevelResourceSpansILSSpan, subIters, nil); unionItr != nil { + iters = append(iters, unionItr) + } + + continue + } + // Compatible type? if entry.typ == operandType(cond.Operands) { pred, err := createPredicate(cond.Op, cond.Operands) @@ -2537,6 +2582,154 @@ func createIntPredicate(op traceql.Operator, operands traceql.Operands) (parquet } } +// createIntPredicateFromFloat adapts a float-based query operand to an int column. +// If the float is exactly representable as an int64 (e.g. 42.0), we compare the +// column to that integer. Otherwise, if the float is non-integer or out of the +// int64 range, we return a "trivial" outcome: +// +// - "=" on a non-integer float returns nil, meaning "no filter" +// - "!=" on a non-integer float always matches, implemented as a predicate +// that returns true for every row. +// - For "<", "<=", ">", ">=", we shift the boundary to the nearest integer. +// For example, "x < 10.3" becomes "x <= 10" for the int column. +// +// Note: If returning nil, no column-level filtering is applied for this condition. +func createIntPredicateFromFloat(op traceql.Operator, operands traceql.Operands) (parquetquery.Predicate, error) { + if op == traceql.OpNone { + return nil, nil + } + + if operands[0].Type != traceql.TypeFloat { + return nil, fmt.Errorf("operand is not float: %s", operands[0].EncodeToString(false)) + } + f := operands[0].Float() + + if math.IsNaN(f) { + return nil, nil + } + + // Check if it's in [MinInt64, MaxInt64) range, and if so, see if it's an integer. + if float64(math.MinInt64) <= f && f < float64(math.MaxInt64) { + if intPart, frac := math.Modf(f); frac == 0 { + intOperands := traceql.Operands{traceql.NewStaticInt(int(intPart))} + return createIntPredicate(op, intOperands) + } + } + + switch op { + case traceql.OpEqual: + return nil, nil + case traceql.OpNotEqual: + return parquetquery.NewCallbackPredicate(func() bool { return true }), nil + case traceql.OpGreater, traceql.OpGreaterEqual: + if f < float64(math.MinInt64) { + return parquetquery.NewCallbackPredicate(func() bool { return true }), nil + } else if float64(math.MaxInt64) <= f { + return nil, nil + } else if 0 < f { + // "x > 10.3" -> "x >= 11" + return parquetquery.NewIntGreaterEqualPredicate(int64(f) + 1), nil + } + // "x > -2.7" -> "x >= -2" + return parquetquery.NewIntGreaterEqualPredicate(int64(f)), nil + case traceql.OpLess, traceql.OpLessEqual: + if f < float64(math.MinInt64) { + return nil, nil + } else if float64(math.MaxInt64) <= f { + return parquetquery.NewCallbackPredicate(func() bool { return true }), nil + } else if f < 0 { + // "x < -2.7" -> "x <= -3" + return parquetquery.NewIntLessEqualPredicate(int64(f) - 1), nil + } + // "x < 10.3" -> "x <= 10" + return parquetquery.NewIntLessEqualPredicate(int64(f)), nil + } + + return nil, fmt.Errorf("operator not supported for integers: %v", op) +} + +// createFloatPredicateFromInt adapts an integer-based query operand to a float column. +// If the integer can be exactly represented as a float64, the float column is compared +// to that exact float value. If the integer cannot be represented exactly, +// the function uses math.Nextafter to determine two adjacent float64 values +// (lowerBound and upperBound) around the nearest representable value. +// +// Behavior for non-representable integers: +// - = always false (returned predicate rejects all rows) +// - != always true (nil is returned, meaning "no filter") +// - < or > boundaries are shifted to the adjacent float64 value. +// +// Note on IEEE 754 binary64 ("double precision"): +// - Up to ±2^53, all integers are exactly representable. +// - Beyond that, the distance ("step") between adjacent float64 values is >= 2, +// so not every 64-bit integer can be represented exactly. +// - Converting int -> float64 in Go uses "round to nearest, ties to even." +func createFloatPredicateFromInt(op traceql.Operator, operands traceql.Operands) (parquetquery.Predicate, error) { + if op == traceql.OpNone { + return nil, nil + } + + i, ok := operands[0].Int() + if !ok { + return nil, fmt.Errorf("operand is not int: %s", operands[0].EncodeToString(false)) + } + + // Convert int64 -> float64 -> int64 to check if it is exactly representable. + i64 := int64(i) + f := float64(i64) + roundTrip := int64(f) + isExact := roundTrip == i64 + + // Identify two adjacent float64 values around i64. + lowerBound, upperBound := f, f + switch { + case roundTrip < 0 && 0 < i64: + // If the sign flipped during cast, it effectively crossed 0. + lowerBound = math.Nextafter(f, math.Inf(-1)) + case i64 < 0 && 0 < roundTrip: + // An impossible case becase math.MinInt exactly maps to float64, hence it cannot jump over 0. But just in case. + upperBound = math.Nextafter(f, math.Inf(+1)) + case i64 < roundTrip: + // Integer became larger when cast to float, so move lowerBound downward. + lowerBound = math.Nextafter(f, math.Inf(-1)) + case roundTrip < i64: + // Integer became smaller when cast to float, so move upperBound upward. + upperBound = math.Nextafter(f, math.Inf(+1)) + } + + switch op { + case traceql.OpEqual: + if isExact { + return parquetquery.NewFloatEqualPredicate(f), nil + } + // There's no float64 that exactly equals i, so filter always false. + return parquetquery.NewCallbackPredicate(func() bool { return false }), nil + case traceql.OpNotEqual: + if isExact { + return parquetquery.NewFloatNotEqualPredicate(f), nil + } + // If it's not exactly representable, it's always "!=", so no filter is needed. + return nil, nil + case traceql.OpGreater: + if isExact { + return parquetquery.NewFloatGreaterPredicate(f), nil + } + // Shift to upperBound for non-exact + return parquetquery.NewFloatGreaterEqualPredicate(upperBound), nil + case traceql.OpGreaterEqual: + return parquetquery.NewFloatGreaterEqualPredicate(upperBound), nil + case traceql.OpLess: + if isExact { + return parquetquery.NewFloatLessPredicate(f), nil + } + return parquetquery.NewFloatLessEqualPredicate(lowerBound), nil + case traceql.OpLessEqual: + return parquetquery.NewFloatLessEqualPredicate(lowerBound), nil + } + + return nil, fmt.Errorf("operator not supported for ints: %+v", op) +} + func createFloatPredicate(op traceql.Operator, operands traceql.Operands) (parquetquery.Predicate, error) { if op == traceql.OpNone { return nil, nil @@ -2637,13 +2830,29 @@ func createAttributeIterator(makeIter makeIterFn, conditions []traceql.Condition attrStringPreds = append(attrStringPreds, pred) case traceql.TypeInt: + // Create a predicate specifically for integer comparisons pred, err := createIntPredicate(cond.Op, cond.Operands) if err != nil { return nil, fmt.Errorf("creating attribute predicate: %w", err) } attrIntPreds = append(attrIntPreds, pred) + // If the operand can be interpreted as a float, create an additional predicate + if pred, err := createFloatPredicateFromInt(cond.Op, cond.Operands); err != nil { + return nil, fmt.Errorf("creating float attribute predicate from int: %w", err) + } else if pred != nil { + attrFltPreds = append(attrFltPreds, pred) + } + case traceql.TypeFloat: + // Attempt to create a predicate for integer comparisons, if applicable + if pred, err := createIntPredicateFromFloat(cond.Op, cond.Operands); err != nil { + return nil, fmt.Errorf("creating int attribute predicate from float: %w", err) + } else if pred != nil { + attrIntPreds = append(attrIntPreds, pred) + } + + // Create a predicate specifically for float comparisons pred, err := createFloatPredicate(cond.Op, cond.Operands) if err != nil { return nil, fmt.Errorf("creating attribute predicate: %w", err) diff --git a/tempodb/encoding/vparquet4/block_traceql_test.go b/tempodb/encoding/vparquet4/block_traceql_test.go index c4ac06b6c25..8cbc5efa90e 100644 --- a/tempodb/encoding/vparquet4/block_traceql_test.go +++ b/tempodb/encoding/vparquet4/block_traceql_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "math" "math/rand" "os" "path" @@ -14,6 +15,7 @@ import ( "time" "github.com/google/uuid" + "github.com/parquet-go/parquet-go" "github.com/stretchr/testify/require" "github.com/grafana/tempo/pkg/parquetquery" @@ -52,6 +54,220 @@ func TestOne(t *testing.T) { fmt.Println(spanSet) } +func TestCreateIntPredicateFromFloat(t *testing.T) { + f := func(query string, predicate string) { + req := traceql.MustExtractFetchSpansRequestWithMetadata(query) + require.Len(t, req.Conditions, 1) + p, err := createIntPredicateFromFloat(req.Conditions[0].Op, req.Conditions[0].Operands) + require.NoError(t, err) + np := "nil" + if p != nil { + r := strings.NewReplacer( + "IntEqualPredicate", "=", + "IntNotEqualPredicate", "!=", + "IntLessPredicate", "<", + "IntLessEqualPredicate", "<=", + "IntGreaterEqualPredicate", ">=", + "IntGreaterPredicate", ">", + ) + np = r.Replace(p.String()) + if np == "CallbackPredicate{}" { + if p.KeepValue(parquet.Value{}) { + np = "callback:true" + } else { + np = "callback:false" + } + } + } + require.Equal(t, predicate, np, "query:%s", query) + } + fe := func(query string, errorMessage string) { + req := traceql.MustExtractFetchSpansRequestWithMetadata(query) + require.Len(t, req.Conditions, 1) + _, err := createIntPredicateFromFloat(req.Conditions[0].Op, req.Conditions[0].Operands) + require.EqualError(t, err, errorMessage, "query:%s", query) + } + + { + p, err := createIntPredicateFromFloat(traceql.OpNone, nil) + require.NoError(t, err) + require.Nil(t, p) + } + { + p, err := createIntPredicateFromFloat(traceql.OpEqual, traceql.Operands{traceql.NewStaticFloat(math.NaN())}) + require.NoError(t, err) + require.Nil(t, p) + } + + // Every float64 in range [math.MinInt64, math.MaxInt64) can be converted to int64 exactly, + // but you need to consider a fractional part of the float64 to adjust an operator. + // It's worth noting that float64 can have a fractional part + // in the range (-float64(1<<52)-0.5, float64(1<<52)+0.5) + f(`{.attr = -123.1}`, `nil`) + f(`{.attr != -123.1}`, `callback:true`) + f(`{.attr < -123.1}`, `<={-124}`) + f(`{.attr <= -123.1}`, `<={-124}`) + f(`{.attr > -123.1}`, `>={-123}`) + f(`{.attr >= -123.1}`, `>={-123}`) + + f(`{.attr = -123.0}`, `={-123}`) + f(`{.attr != -123.0}`, `!={-123}`) + f(`{.attr < -123.0}`, `<{-123}`) + f(`{.attr <= -123.0}`, `<={-123}`) + f(`{.attr > -123.0}`, `>{-123}`) + f(`{.attr >= -123.0}`, `>={-123}`) + + f(`{.attr = 123.0}`, `={123}`) + f(`{.attr != 123.0}`, `!={123}`) + f(`{.attr < 123.0}`, `<{123}`) + f(`{.attr <= 123.0}`, `<={123}`) + f(`{.attr > 123.0}`, `>{123}`) + f(`{.attr >= 123.0}`, `>={123}`) + + f(`{.attr = 123.1}`, `nil`) + f(`{.attr != 123.1}`, `callback:true`) + f(`{.attr < 123.1}`, `<={123}`) + f(`{.attr <= 123.1}`, `<={123}`) + f(`{.attr > 123.1}`, `>={124}`) + f(`{.attr >= 123.1}`, `>={124}`) + + // [MaxInt64, +Inf) + f(`{.attr = 9.223372036854776e+18}`, `nil`) + f(`{.attr != 9.223372036854776e+18}`, `callback:true`) + f(`{.attr > 9.223372036854776e+18}`, `nil`) + f(`{.attr >= 9.223372036854776e+18}`, `nil`) + f(`{.attr < 9.223372036854776e+18}`, `callback:true`) + f(`{.attr <= 9.223372036854776e+18}`, `callback:true`) + + // (-Inf, MinInt64) + f(`{.attr = -9.223372036854777e+18}`, `nil`) + f(`{.attr != -9.223372036854777e+18}`, `callback:true`) + f(`{.attr > -9.223372036854777e+18}`, `callback:true`) + f(`{.attr >= -9.223372036854777e+18}`, `callback:true`) + f(`{.attr < -9.223372036854777e+18}`, `nil`) + f(`{.attr <= -9.223372036854777e+18}`, `nil`) + + fe(`{.attr = 1}`, `operand is not float: 1`) + fe(`{.attr =~ -1.2}`, `operator not supported for integers: =~`) +} + +func TestCreateFloatPredicateFromInt(t *testing.T) { + f := func(query string, predicate string) { + req := traceql.MustExtractFetchSpansRequestWithMetadata(query) + require.Len(t, req.Conditions, 1) + p, err := createFloatPredicateFromInt(req.Conditions[0].Op, req.Conditions[0].Operands) + require.NoError(t, err) + np := "nil" + if p != nil { + r := strings.NewReplacer( + "FloatEqualPredicate", "=", + "FloatNotEqualPredicate", "!=", + "FloatLessPredicate", "<", + "FloatLessEqualPredicate", "<=", + "FloatGreaterEqualPredicate", ">=", + "FloatGreaterPredicate", ">", + ) + np = r.Replace(p.String()) + if np == "CallbackPredicate{}" { + if p.KeepValue(parquet.Value{}) { + np = "callback:true" + } else { + np = "callback:false" + } + } + } + require.Equal(t, predicate, np, "query:%s", query) + } + + // Small integers and basic checks + f(`{.attr = 0}`, "={0.000000}") // exactly representable + f(`{.attr != 0}`, "!={0.000000}") // also exactly + f(`{.attr = 1}`, "={1.000000}") + f(`{.attr = -1}`, "={-1.000000}") + + // Typical checks around 455..457 + f(`{.attr > 455}`, ">{455.000000}") + f(`{.attr >= 455}`, ">={455.000000}") + f(`{.attr <= 456}`, "<={456.000000}") + f(`{.attr = 456}`, "={456.000000}") + f(`{.attr != 457}`, "!={457.000000}") + f(`{.attr >= 456}`, ">={456.000000}") + f(`{.attr <= 457}`, "<={457.000000}") + f(`{.attr < 457}`, "<{457.000000}") + + // Around 2^53, 2^53 = 9007199254740992 + f(`{.attr > 9007199254740991}`, ">{9007199254740991.000000}") // exactly representable + f(`{.attr = 9007199254740992}`, "={9007199254740992.000000}") // also exactly representable + f(`{.attr = 9007199254740993}`, "callback:false") // not representable, hence no float matches + f(`{.attr != 9007199254740993}`, "nil") // not representable, hence always true + f(`{.attr = 9007199254740994}`, "={9007199254740994.000000}") // exactly representable + f(`{.attr != 9007199254740994}`, "!={9007199254740994.000000}") // exactly representable + + // Around -2^53, 2^53 = -9007199254740992 + f(`{.attr = -9007199254740991}`, "={-9007199254740991.000000}") // exactly representable + f(`{.attr = -9007199254740992}`, "={-9007199254740992.000000}") // also exactly representable + f(`{.attr = -9007199254740993}`, "callback:false") // not representable, hence no float matches + f(`{.attr != -9007199254740993}`, "nil") // not representable, hence always true + f(`{.attr = -9007199254740994}`, "={-9007199254740994.000000}") // exactly representable + f(`{.attr != -9007199254740994}`, "!={-9007199254740994.000000}") // exactly representable + + // Very large int boundaries (MinInt = -9223372036854775808, MaxInt = 9223372036854775807) + // math.MaxInt-1023 = 9223372036854774784 + // math.MaxInt-1023 is the largest int that can be exactly represented as a float64 + f(`{.attr != 9223372036854774784}`, "!={9223372036854774784.000000}") + f(`{.attr < 9223372036854774784}`, "<{9223372036854774784.000000}") + f(`{.attr <= 9223372036854774784}`, "<={9223372036854774784.000000}") + f(`{.attr = 9223372036854774784}`, "={9223372036854774784.000000}") + f(`{.attr >= 9223372036854774784}`, ">={9223372036854774784.000000}") + f(`{.attr > 9223372036854774784}`, ">{9223372036854774784.000000}") + // The numbers between `math.MaxInt-1023` and `math.MaxInt` aren't representable as float64 + for i := math.MaxInt - 1023; i < math.MaxInt; i++ { + num := strconv.Itoa(i + 1) + f(`{.attr = `+num+`}`, "callback:false") + f(`{.attr != `+num+`}`, "nil") + f(`{.attr < `+num+`}`, "<={9223372036854774784.000000}") + f(`{.attr > `+num+`}`, ">={9223372036854775808.000000}") + } + // math.MinInt is the smallest int that can be exactly represented as a float64 + // math.MinInt+1024 = -9223372036854774784 + // The next number after math.MinInt representable as a float64 is math.MinInt+1024 + // Parsing math.MinInt isn't supported yet due to https://github.com/grafana/tempo/issues/4623 + // f(`{.attr != -9223372036854775808}`, "!={-9223372036854775808.000000}") + // f(`{.attr < -9223372036854775808}`, "<{-9223372036854775808.000000}") + // f(`{.attr <= -9223372036854775808}`, "<={-9223372036854775808.000000}") + // f(`{.attr = -9223372036854775808}`, "={-9223372036854775808.000000}") + // f(`{.attr >= -9223372036854775808}`, ">={-9223372036854775808.000000}") + // f(`{.attr > -9223372036854775808}`, ">{-9223372036854775808.000000}") + f(`{.attr = -9223372036854775807}`, "callback:false") + f(`{.attr != -9223372036854775807}`, "nil") + f(`{.attr < -9223372036854775807}`, "<={-9223372036854775808.000000}") + f(`{.attr > -9223372036854775807}`, ">={-9223372036854774784.000000}") + f(`{.attr != -9223372036854774784}`, "!={-9223372036854774784.000000}") + f(`{.attr < -9223372036854774784}`, "<{-9223372036854774784.000000}") + f(`{.attr <= -9223372036854774784}`, "<={-9223372036854774784.000000}") + f(`{.attr = -9223372036854774784}`, "={-9223372036854774784.000000}") + f(`{.attr >= -9223372036854774784}`, ">={-9223372036854774784.000000}") + f(`{.attr > -9223372036854774784}`, ">{-9223372036854774784.000000}") + + { + p, err := createFloatPredicateFromInt(traceql.OpNone, nil) + require.NoError(t, err) + require.Nil(t, p) + } + { + req := traceql.MustExtractFetchSpansRequestWithMetadata(`{.attr =~ 1.0}`) + require.Len(t, req.Conditions, 1) + _, err := createFloatPredicateFromInt(req.Conditions[0].Op, req.Conditions[0].Operands) + require.EqualError(t, err, `operand is not int: 1.0`) + } + { + req := traceql.MustExtractFetchSpansRequestWithMetadata(`{.attr =~ -9223372036854774784}`) + require.Len(t, req.Conditions, 1) + _, err := createFloatPredicateFromInt(req.Conditions[0].Op, req.Conditions[0].Operands) + require.EqualError(t, err, `operator not supported for ints: =~`) + } +} + func TestBackendBlockSearchTraceQL(t *testing.T) { numTraces := 250 traces := make([]*Trace, 0, numTraces) diff --git a/tempodb/tempodb_search_test.go b/tempodb/tempodb_search_test.go index 33568c15f27..3c559b2e2ba 100644 --- a/tempodb/tempodb_search_test.go +++ b/tempodb/tempodb_search_test.go @@ -56,6 +56,7 @@ func TestSearchCompleteBlock(t *testing.T) { nestedSet, tagValuesRunner, tagNamesRunner, + traceQLCrossType, ) }) if vers == vparquet4.VersionString { @@ -134,6 +135,100 @@ func traceQLRunner(t *testing.T, _ *tempopb.Trace, wantMeta *tempopb.TraceSearch } } +func traceQLCrossType(t *testing.T, _ *tempopb.Trace, wantMeta *tempopb.TraceSearchMetadata, searchesThatMatch, searchesThatDontMatch []*tempopb.SearchRequest, meta *backend.BlockMeta, r Reader, _ common.BackendBlock) { + ctx := context.Background() + e := traceql.NewEngine() + + quotedAttributesThatMatch := []*tempopb.SearchRequest{ + {Query: `{ .floatAttr > 123.0 }`}, + {Query: `{ .floatAttr >= 123.0 }`}, + {Query: `{ .floatAttr <= 123.4 }`}, + {Query: `{ .floatAttr = 123.4 }`}, + {Query: `{ .floatAttr >= 123.4 }`}, + {Query: `{ .floatAttr <= 123.9 }`}, + {Query: `{ .floatAttr < 123.9 }`}, + {Query: `{ .intAttr > 122 }`}, + {Query: `{ .intAttr >= 122 }`}, + {Query: `{ .intAttr <= 123 }`}, + {Query: `{ .intAttr = 123 }`}, + {Query: `{ .intAttr >= 123 }`}, + {Query: `{ .intAttr <= 124 }`}, + {Query: `{ .intAttr < 124 }`}, + {Query: `{ .floatAttr > 123 }`}, + {Query: `{ .floatAttr >= 123 }`}, + {Query: `{ .floatAttr <= 124 }`}, + {Query: `{ .floatAttr < 124 }`}, + {Query: `{ .intAttr > 122.9 }`}, + {Query: `{ .intAttr >= 122.9 }`}, + {Query: `{ .intAttr <= 123.0 }`}, + {Query: `{ .intAttr = 123.0 }`}, + {Query: `{ .intAttr >= 123.0 }`}, + {Query: `{ .intAttr <= 123.1 }`}, + {Query: `{ .intAttr < 123.1 }`}, + {Query: `{ .intAttr != 123.1 }`}, + } + + searchesThatMatch = append(searchesThatMatch, quotedAttributesThatMatch...) + for _, req := range searchesThatMatch { + fetcher := traceql.NewSpansetFetcherWrapper(func(ctx context.Context, req traceql.FetchSpansRequest) (traceql.FetchSpansResponse, error) { + return r.Fetch(ctx, meta, req, common.DefaultSearchOptions()) + }) + + res, err := e.ExecuteSearch(ctx, req, fetcher) + if errors.Is(err, common.ErrUnsupported) { + continue + } + + require.NoError(t, err, "search request: %+v", req) + actual := actualForExpectedMeta(wantMeta, res) + require.NotNil(t, actual, "search request: %v", req) + actual.SpanSet = nil // todo: add the matching spansets to wantmeta + actual.SpanSets = nil + actual.ServiceStats = nil + require.Equal(t, wantMeta, actual, "search request: %v", req) + } + + quotedAttributesThaDonttMatch := []*tempopb.SearchRequest{ + {Query: `{ .floatAttr < 123.0 }`}, + {Query: `{ .floatAttr <= 123.0 }`}, + {Query: `{ .floatAttr < 123.4 }`}, + {Query: `{ .floatAttr != 123.4 }`}, + {Query: `{ .floatAttr > 123.4 }`}, + {Query: `{ .floatAttr >= 123.9 }`}, + {Query: `{ .floatAttr > 123.9 }`}, + {Query: `{ .intAttr < 122 }`}, + {Query: `{ .intAttr <= 122 }`}, + {Query: `{ .intAttr < 123 }`}, + {Query: `{ .intAttr != 123 }`}, + {Query: `{ .intAttr > 123 }`}, + {Query: `{ .intAttr >= 124 }`}, + {Query: `{ .intAttr > 124 }`}, + {Query: `{ .floatAttr < 123 }`}, + {Query: `{ .floatAttr <= 123 }`}, + {Query: `{ .floatAttr = 123 }`}, + {Query: `{ .floatAttr >= 124 }`}, + {Query: `{ .floatAttr > 124 }`}, + {Query: `{ .intAttr < 122.9 }`}, + {Query: `{ .intAttr <= 122.9 }`}, + {Query: `{ .intAttr < 123.0 }`}, + {Query: `{ .intAttr != 123.0 }`}, + {Query: `{ .intAttr > 123.0 }`}, + {Query: `{ .intAttr >= 123.1 }`}, + {Query: `{ .intAttr > 123.1 }`}, + } + + searchesThatDontMatch = append(searchesThatDontMatch, quotedAttributesThaDonttMatch...) + for _, req := range searchesThatDontMatch { + fetcher := traceql.NewSpansetFetcherWrapper(func(ctx context.Context, req traceql.FetchSpansRequest) (traceql.FetchSpansResponse, error) { + return r.Fetch(ctx, meta, req, common.DefaultSearchOptions()) + }) + + res, err := e.ExecuteSearch(ctx, req, fetcher) + require.NoError(t, err, "search request: %+v", req) + require.Nil(t, actualForExpectedMeta(wantMeta, res), "search request: %v", req) + } +} + func advancedTraceQLRunner(t *testing.T, wantTr *tempopb.Trace, wantMeta *tempopb.TraceSearchMetadata, _, _ []*tempopb.SearchRequest, meta *backend.BlockMeta, r Reader, _ common.BackendBlock) { ctx := context.Background() e := traceql.NewEngine() @@ -1442,7 +1537,7 @@ func tagNamesRunner(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMetada query: "{ resource.cluster = `MyCluster` }", expected: map[string][]string{ "span": {"child", "foo", "http.method", "http.status_code", "http.url", "span-dedicated.01", "span-dedicated.02"}, - "resource": {"bat", "{ } ( ) = ~ ! < > & | ^", "cluster", "container", "k8s.cluster.name", "k8s.container.name", "k8s.namespace.name", "k8s.pod.name", "namespace", "pod", "res-dedicated.01", "res-dedicated.02", "service.name"}, + "resource": {"bat", "{ } ( ) = ~ ! < > & | ^", "intAttr", "floatAttr", "cluster", "container", "k8s.cluster.name", "k8s.container.name", "k8s.namespace.name", "k8s.pod.name", "namespace", "pod", "res-dedicated.01", "res-dedicated.02", "service.name"}, }, }, { @@ -1451,7 +1546,7 @@ func tagNamesRunner(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMetada query: "{ span.foo = `Bar` }", expected: map[string][]string{ "span": {"child", "parent", "{ } ( ) = ~ ! < > & | ^", "foo", "http.method", "http.status_code", "http.url", "span-dedicated.01", "span-dedicated.02"}, - "resource": {"bat", "{ } ( ) = ~ ! < > & | ^", "cluster", "container", "k8s.cluster.name", "k8s.container.name", "k8s.namespace.name", "k8s.pod.name", "namespace", "pod", "res-dedicated.01", "res-dedicated.02", "service.name"}, + "resource": {"bat", "{ } ( ) = ~ ! < > & | ^", "intAttr", "floatAttr", "cluster", "container", "k8s.cluster.name", "k8s.container.name", "k8s.namespace.name", "k8s.pod.name", "namespace", "pod", "res-dedicated.01", "res-dedicated.02", "service.name"}, }, }, } @@ -1811,6 +1906,13 @@ func intKV(k string, v int) *v1_common.KeyValue { } } +func float64KV(k string, v float64) *v1_common.KeyValue { + return &v1_common.KeyValue{ + Key: k, + Value: &v1_common.AnyValue{Value: &v1_common.AnyValue_DoubleValue{DoubleValue: v}}, + } +} + func boolKV(k string) *v1_common.KeyValue { return &v1_common.KeyValue{ Key: k, @@ -1906,6 +2008,8 @@ func makeExpectedTrace() ( stringKV("res-dedicated.01", "res-1a"), stringKV("res-dedicated.02", "res-2a"), stringKV(attributeWithTerminalChars, "foobar"), + intKV("intAttr", 123), + float64KV("floatAttr", 123.4), }, }, ScopeSpans: []*v1.ScopeSpans{ From ff2bc1511aefa137949af6d693870e25085f22fc Mon Sep 17 00:00:00 2001 From: Andrey Karpov Date: Mon, 10 Feb 2025 20:17:43 +0000 Subject: [PATCH 2/2] wip --- tempodb/encoding/vparquet4/block_traceql.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tempodb/encoding/vparquet4/block_traceql.go b/tempodb/encoding/vparquet4/block_traceql.go index f4766518f68..da955c73a21 100644 --- a/tempodb/encoding/vparquet4/block_traceql.go +++ b/tempodb/encoding/vparquet4/block_traceql.go @@ -2008,7 +2008,17 @@ func createSpanIterator(makeIter makeIterFn, innerIterators []parquetquery.Itera return nil, fmt.Errorf("creating predicate: %w", err) } if pred != nil { - subIters = append(subIters, makeIter(columnPathResourceAttrDouble, pred, cond.Attribute.Name)) + subIters = append(subIters, + parquetquery.NewJoinIterator( + DefinitionLevelResourceSpansILSSpanAttrs, + []parquetquery.Iterator{ + makeIter(columnPathSpanAttrKey, parquetquery.NewStringInPredicate([]string{cond.Attribute.Name}), "key"), + makeIter(columnPathResourceAttrDouble, pred, "float"), + }, + &attributeCollector{}, + parquetquery.WithPool(pqAttrPool), + ), + ) } } if unionItr := unionIfNeeded(DefinitionLevelResourceSpansILSSpan, subIters, nil); unionItr != nil {