From c4eb1366d96037379be33a07f45058717e96e8d0 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 7 Dec 2020 15:55:26 -0800 Subject: [PATCH 1/4] xform: add rule expectations for GenerateLookupJoins tests Release note: None --- pkg/sql/opt/xform/testdata/rules/join | 74 +++++++++++++-------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index 5710b1a3dd2d..3250a877d1f6 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -2344,7 +2344,7 @@ CREATE TABLE abcde (a INT, b INT, c INT, d INT, e INT, INDEX (a,b,c)) ---- # Covering case. -opt +opt expect=GenerateLookupJoins SELECT a,b,n,m FROM small JOIN abcd ON a=m ---- inner-join (lookup abcd@secondary) @@ -2356,7 +2356,7 @@ inner-join (lookup abcd@secondary) └── filters (true) # Covering case, left-join. -opt +opt expect=GenerateLookupJoins SELECT a,b,n,m FROM small LEFT JOIN abcd ON a=m ---- left-join (lookup abcd@secondary) @@ -2367,7 +2367,7 @@ left-join (lookup abcd@secondary) └── filters (true) # Covering case, left-join, extra filter bound by index. -opt +opt expect=GenerateLookupJoins SELECT a,b,n,m FROM small LEFT JOIN abcd ON a=m AND b>n ---- left-join (lookup abcd@secondary) @@ -2379,7 +2379,7 @@ left-join (lookup abcd@secondary) └── b:6 > n:2 [outer=(2,6), constraints=(/2: (/NULL - ]; /6: (/NULL - ])] # Non-covering case. -opt +opt expect=GenerateLookupJoins SELECT * FROM small JOIN abcd ON a=m ---- inner-join (lookup abcd) @@ -2397,7 +2397,7 @@ inner-join (lookup abcd) └── filters (true) # Non-covering case, left join. -opt +opt expect=GenerateLookupJoins SELECT * FROM small LEFT JOIN abcd ON a=m ---- left-join (lookup abcd) @@ -2414,7 +2414,7 @@ left-join (lookup abcd) └── filters (true) # Non-covering case, extra filter bound by index. -opt +opt expect=GenerateLookupJoins SELECT * FROM small JOIN abcd ON a=m AND b>n ---- inner-join (lookup abcd) @@ -2433,7 +2433,7 @@ inner-join (lookup abcd) └── filters (true) # Non-covering case, extra filter bound by index, left join. -opt +opt expect=GenerateLookupJoins SELECT * FROM small LEFT JOIN abcd ON a=m AND b>n ---- left-join (lookup abcd) @@ -2451,7 +2451,7 @@ left-join (lookup abcd) └── filters (true) # Non-covering case, extra filter not bound by index. -opt +opt expect=GenerateLookupJoins SELECT * FROM small JOIN abcd ON a=m AND c>n ---- inner-join (lookup abcd) @@ -2472,7 +2472,7 @@ inner-join (lookup abcd) # Non-covering case, extra filter not bound by index, left join. # In this case, we can't yet convert to a lookup join (see # the GenerateLookupJoins custom func). -opt +opt expect-not=GenerateLookupJoins SELECT * FROM small LEFT JOIN abcd ON a=m AND c>n ---- right-join (hash) @@ -2697,7 +2697,7 @@ No new expressions. ---- # Verify we don't generate lookup joins if there is a hint that says otherwise. -memo +memo expect-not=GenerateLookupJoins SELECT a,b,n,m FROM small INNER HASH JOIN abcd ON a=m ---- memo (optimized, ~8KB, required=[presentation: a:5,b:6,n:2,m:1]) @@ -2719,7 +2719,7 @@ memo (optimized, ~8KB, required=[presentation: a:5,b:6,n:2,m:1]) └── G7: (variable m) # Lookup semi-join with index that contains all columns in the join condition. -opt +opt expect=GenerateLookupJoins SELECT m, n FROM small WHERE EXISTS(SELECT 1 FROM abcd WHERE m = a) ---- semi-join (lookup abcd@secondary) @@ -2731,7 +2731,7 @@ semi-join (lookup abcd@secondary) # We should not generate a lookup semi-join when the index doesn't contain all # columns in the join condition. -opt +opt expect-not=GenerateLookupJoins SELECT m, n FROM small WHERE EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c) ---- semi-join (hash) @@ -2745,7 +2745,7 @@ semi-join (hash) └── n:2 = c:7 [outer=(2,7), constraints=(/2: (/NULL - ]; /7: (/NULL - ]), fd=(2)==(7), (7)==(2)] # Lookup anti-join with index that contains all columns in the join condition. -opt +opt expect=GenerateLookupJoins SELECT m, n FROM small WHERE NOT EXISTS(SELECT 1 FROM abcd WHERE m = a) ---- anti-join (lookup abcd@secondary) @@ -2757,7 +2757,7 @@ anti-join (lookup abcd@secondary) # We should not generate a lookup anti-join when the index doesn't contain all # columns in the join condition. -opt +opt expect-not=GenerateLookupJoins SELECT m, n FROM small WHERE NOT EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c) ---- anti-join (hash) @@ -2779,7 +2779,7 @@ anti-join (hash) # into the ON condition). # Covering case. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT a,b,n,m FROM small JOIN abcd ON a=m AND b>1 ---- inner-join (lookup abcd@secondary) @@ -2792,7 +2792,7 @@ inner-join (lookup abcd@secondary) └── b:6 > 1 [outer=(6), constraints=(/6: [/2 - ]; tight)] # Covering case, left-join. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT a,b,n,m FROM small LEFT JOIN abcd ON a=m AND b>1 ---- left-join (lookup abcd@secondary) @@ -2804,7 +2804,7 @@ left-join (lookup abcd@secondary) └── b:6 > 1 [outer=(6), constraints=(/6: [/2 - ]; tight)] # Non-covering case. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small JOIN abcd ON a=m AND b>1 ---- inner-join (lookup abcd) @@ -2823,7 +2823,7 @@ inner-join (lookup abcd) └── filters (true) # Non-covering case, left join. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small LEFT JOIN abcd ON a=m AND b>1 ---- left-join (lookup abcd) @@ -2841,7 +2841,7 @@ left-join (lookup abcd) └── filters (true) # Non-covering case, extra filter bound by index. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small JOIN abcd ON a=m AND b>n AND b>1 ---- inner-join (lookup abcd) @@ -2861,7 +2861,7 @@ inner-join (lookup abcd) └── filters (true) # Non-covering case, extra filter bound by index, left join. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small LEFT JOIN abcd ON a=m AND b>n AND b>1 ---- left-join (lookup abcd) @@ -2880,7 +2880,7 @@ left-join (lookup abcd) └── filters (true) # Non-covering case, extra filter not bound by index. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small JOIN abcd ON a=m AND c>n AND b>1 ---- inner-join (lookup abcd) @@ -2902,7 +2902,7 @@ inner-join (lookup abcd) # Non-covering case, extra filter not bound by index, left join. # In this case, we can't yet convert to a lookup join (see # the GenerateLookupJoins custom func). -opt +opt expect-not=GenerateLookupJoinsWithFilter SELECT * FROM small LEFT JOIN abcd ON a=m AND c>n AND b>1 ---- right-join (hash) @@ -2920,7 +2920,7 @@ right-join (hash) └── c:7 > n:2 [outer=(2,7), constraints=(/2: (/NULL - ]; /7: (/NULL - ])] # Constant columns are projected and used by lookup joiner. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small INNER JOIN abcde ON a=m AND b=10 ---- inner-join (lookup abcde) @@ -2943,7 +2943,7 @@ inner-join (lookup abcde) └── filters (true) # Constant columns not projected if not prefix of an index. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small INNER JOIN abcde ON a=m AND c=10 ---- inner-join (lookup abcde) @@ -2962,7 +2962,7 @@ inner-join (lookup abcde) └── filters (true) # Multiple constant columns projected and used by lookup joiner. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small INNER JOIN abcde ON a=m AND b=10 AND c=10 ---- inner-join (lookup abcde) @@ -2986,7 +2986,7 @@ inner-join (lookup abcde) └── filters (true) # Filters are reduced properly as constant filters are extracted. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small INNER JOIN abcde ON a=m AND b=10 AND c=10 AND d=10 ---- inner-join (lookup abcde) @@ -3011,7 +3011,7 @@ inner-join (lookup abcde) └── d:8 = 10 [outer=(8), constraints=(/8: [/10 - /10]; tight), fd=()-->(8)] # Non equality filters don't trigger constant projection. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small INNER JOIN abcde ON a=m AND b<10 ---- inner-join (lookup abcde) @@ -3031,7 +3031,7 @@ inner-join (lookup abcde) # Lookup Joiner uses the constant equality columns at the same time as the explicit # column equalities. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT a, b, c FROM small INNER LOOKUP JOIN abcde ON m=b AND a=10 AND c=10 ---- project @@ -3058,7 +3058,7 @@ CREATE TABLE bool_col (a INT, b INT, c bool, d bool, e bool, INDEX (a,b,c)) ---- # Projection of constant columns work on boolean expressions. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small INNER JOIN bool_col ON a=m AND b=10 AND c=true ---- inner-join (lookup bool_col) @@ -3081,7 +3081,7 @@ inner-join (lookup bool_col) │ └── filters (true) └── filters (true) -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small INNER JOIN bool_col ON a=m AND b=10 AND c ---- inner-join (lookup bool_col) @@ -3104,7 +3104,7 @@ inner-join (lookup bool_col) │ └── filters (true) └── filters (true) -opt +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small INNER JOIN bool_col ON a=m AND b=10 AND NOT c ---- inner-join (lookup bool_col) @@ -3132,7 +3132,7 @@ CREATE TABLE t(pk INT PRIMARY KEY, col0 INT, col1 INT, col2 INT, col4 INT, UNIQU ---- # Make sure we don't generate a lookup join with no key columns (#41676). -opt +opt expect-not=GenerateLookupJoinsWithFilter SELECT pk FROM t WHERE col4 = 1 AND col0 = 1 AND col2 IN (SELECT col0 FROM t WHERE col0 = 1 AND col2 IS NULL); ---- project @@ -3182,7 +3182,7 @@ project └── filters (true) # Lookup semi-join with covering index. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT m, n FROM small WHERE EXISTS(SELECT 1 FROM abcd WHERE m = a AND a > b) ---- semi-join (lookup abcd@secondary) @@ -3194,7 +3194,7 @@ semi-join (lookup abcd@secondary) └── a:5 > b:6 [outer=(5,6), constraints=(/5: (/NULL - ]; /6: (/NULL - ])] # We should not generate a lookup semi-join when the index is non-covering. -opt +opt expect-not=GenerateLookupJoinsWithFilter SELECT m, n FROM small WHERE EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c AND a > b) ---- semi-join (hash) @@ -3212,7 +3212,7 @@ semi-join (hash) └── n:2 = c:7 [outer=(2,7), constraints=(/2: (/NULL - ]; /7: (/NULL - ]), fd=(2)==(7), (7)==(2)] # Lookup anti-join with covering index. -opt +opt expect=GenerateLookupJoinsWithFilter SELECT m, n FROM small WHERE NOT EXISTS(SELECT 1 FROM abcd WHERE m = a AND a > b) ---- anti-join (lookup abcd@secondary) @@ -3224,7 +3224,7 @@ anti-join (lookup abcd@secondary) └── a:5 > b:6 [outer=(5,6), constraints=(/5: (/NULL - ]; /6: (/NULL - ])] # We should not generate a lookup semi-join when the index is non-covering. -opt +opt expect-not=GenerateLookupJoinsWithFilter SELECT m, n FROM small WHERE NOT EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c AND a > b) ---- anti-join (hash) @@ -3334,7 +3334,7 @@ project └── s:7 = 'foo' [outer=(7), constraints=(/7: [/'foo' - /'foo']; tight), fd=()-->(7)] # Do not generate a lookup join when the predicate is not implied by the filter. -opt +opt expect-not=GenerateLookupJoinsWithFilter SELECT m FROM small JOIN partial_tab ON n = i WHERE s = 'not_implied' ---- project From f00dacaec0c64edc596a675eab097061c7798863 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 7 Dec 2020 16:05:02 -0800 Subject: [PATCH 2/4] opt: generate lookup join for column constrained to multiple constants Previously, the optimizer could create lookup join keys from filters that constrain a column to a single constant value. This was done by wrapping the join input in a Project that projected the constant value, and using this new column as a key column. This commit generalizes this behavior so that lookup join keys can also be created from filters that constrain a column to multiple, non-ranging constant values. The constant values are cross-joined with the input, and the joined column is used as a key column. If a column is constrained to a single constant value, the cross join normalizes to a Project identical to the Projects constructed prior to this commit. Release note (performance improvement): The query optimizer can use filters that constrained columns to multiple constant values to generate lookup joins. For example, a join filter `x.a = y.a AND y.b IN (1, 2)` can be used to generate a lookup join on table `y` assuming that it has an index on `(a, b)` or `(b, a)`. --- pkg/sql/opt/constraint/constraint_set.go | 20 ++- pkg/sql/opt/constraint/constraint_set_test.go | 35 +++-- .../opt/exec/execbuilder/testdata/lookup_join | 6 +- pkg/sql/opt/xform/join_funcs.go | 80 ++++++---- pkg/sql/opt/xform/testdata/rules/join | 138 +++++++++++++++--- pkg/sql/opt/xform/testdata/rules/join_order | 4 +- 6 files changed, 203 insertions(+), 80 deletions(-) diff --git a/pkg/sql/opt/constraint/constraint_set.go b/pkg/sql/opt/constraint/constraint_set.go index 400fd8f4727d..991cd44e8ad4 100644 --- a/pkg/sql/opt/constraint/constraint_set.go +++ b/pkg/sql/opt/constraint/constraint_set.go @@ -312,20 +312,26 @@ func (s *Set) ExtractValueForConstCol(evalCtx *tree.EvalContext, col opt.ColumnI return nil } -// IsSingleColumnConstValue returns true if the Set contains a single constraint -// on a single column which allows for a single constant value. On success, -// returns the column and the constant value. -func (s *Set) IsSingleColumnConstValue( +// HasSingleColumnConstValues returns true if the Set contains a single +// constraint on a single column which allows for one or more non-ranging +// constant values. On success, returns the column and the constant value. +func (s *Set) HasSingleColumnConstValues( evalCtx *tree.EvalContext, -) (col opt.ColumnID, constValue tree.Datum, ok bool) { +) (col opt.ColumnID, constValues tree.Datums, ok bool) { if s.Length() != 1 { return 0, nil, false } c := s.Constraint(0) - if c.Columns.Count() != 1 || c.ExactPrefix(evalCtx) != 1 { + if c.Columns.Count() != 1 || c.Prefix(evalCtx) != 1 { return 0, nil, false } - return c.Columns.Get(0).ID(), c.Spans.Get(0).StartKey().Value(0), true + numSpans := c.Spans.Count() + constValues = make(tree.Datums, numSpans) + for i := range constValues { + val := c.Spans.Get(i).StartKey().Value(0) + constValues[i] = val + } + return c.Columns.Get(0).ID(), constValues, true } // allocConstraint allocates space for a new constraint in the set and returns diff --git a/pkg/sql/opt/constraint/constraint_set_test.go b/pkg/sql/opt/constraint/constraint_set_test.go index 8b1bba9d14ad..7a06ccc58eb6 100644 --- a/pkg/sql/opt/constraint/constraint_set_test.go +++ b/pkg/sql/opt/constraint/constraint_set_test.go @@ -11,6 +11,7 @@ package constraint import ( + "reflect" "testing" "github.com/cockroachdb/cockroach/pkg/sql/opt" @@ -347,32 +348,34 @@ func TestExtractConstCols(t *testing.T) { } } -func TestIsSingleColumnConstValue(t *testing.T) { +func TestHasSingleColumnConstValues(t *testing.T) { type testCase struct { constraints []string col opt.ColumnID - val int + vals []int } cases := []testCase{ - {[]string{`/1: [/10 - /10]`}, 1, 10}, - {[]string{`/-1: [/10 - /10]`}, 1, 10}, - {[]string{`/1: [/10 - /11]`}, 0, 0}, - {[]string{`/1: [/10 - /10] [/11 - /11]`}, 0, 0}, - {[]string{`/1/2: [/10/2 - /10/4]`}, 0, 0}, - {[]string{`/1/2: [/10/2 - /10/2]`}, 0, 0}, + {[]string{`/1: [/10 - /10]`}, 1, []int{10}}, + {[]string{`/-1: [/10 - /10]`}, 1, []int{10}}, + {[]string{`/1: [/10 - /11]`}, 0, nil}, + {[]string{`/1: [/10 - /10] [/11 - /11]`}, 1, []int{10, 11}}, + {[]string{`/1: [/10 - /10] [/11 - /11] [/12 - /12]`}, 1, []int{10, 11, 12}}, + {[]string{`/1: [/10 - /10] [/11 - /11] [/12 - /13]`}, 0, nil}, + {[]string{`/1/2: [/10/2 - /10/4]`}, 0, nil}, + {[]string{`/1/2: [/10/2 - /10/2]`}, 0, nil}, { []string{ `/1: [/10 - /10]`, `/2: [/8 - /8]`, }, - 0, 0, + 0, nil, }, { []string{ `/1: [/10 - /10]`, `/1/2: [/10/8 - /10/8]`, }, - 0, 0, + 0, nil, }, } evalCtx := tree.NewTestingEvalContext(nil) @@ -382,13 +385,13 @@ func TestIsSingleColumnConstValue(t *testing.T) { constraint := ParseConstraint(evalCtx, constraint) cs = cs.Intersect(evalCtx, SingleConstraint(&constraint)) } - col, val, ok := cs.IsSingleColumnConstValue(evalCtx) - intVal := 0 - if ok { - intVal = int(*val.(*tree.DInt)) + col, vals, _ := cs.HasSingleColumnConstValues(evalCtx) + var intVals []int + for _, val := range vals { + intVals = append(intVals, int(*val.(*tree.DInt))) } - if tc.col != col || tc.val != intVal { - t.Errorf("%s: expected %d,%d got %d,%d", cs, tc.col, tc.val, col, intVal) + if tc.col != col || !reflect.DeepEqual(tc.vals, intVals) { + t.Errorf("%s: expected %d,%d got %d,%d", cs, tc.col, tc.vals, col, intVals) } } } diff --git a/pkg/sql/opt/exec/execbuilder/testdata/lookup_join b/pkg/sql/opt/exec/execbuilder/testdata/lookup_join index 9faf22c0bc88..2afd6e8721e5 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/lookup_join +++ b/pkg/sql/opt/exec/execbuilder/testdata/lookup_join @@ -1736,13 +1736,13 @@ vectorized: true │ estimated row count: 10 (missing stats) │ └── • lookup join (semi) - │ columns: ("project_const_col_@15", pk, col0, col3) + │ columns: ("lookup_join_const_col_@15", pk, col0, col3) │ table: tab4@tab4_col3_col4_key - │ equality: (col0, project_const_col_@15) = (col3,col4) + │ equality: (col0, lookup_join_const_col_@15) = (col3,col4) │ equality cols are key │ └── • render - │ columns: ("project_const_col_@15", pk, col0, col3) + │ columns: ("lookup_join_const_col_@15", pk, col0, col3) │ estimated row count: 10 (missing stats) │ render 0: 495.6 │ render 1: pk diff --git a/pkg/sql/opt/xform/join_funcs.go b/pkg/sql/opt/xform/join_funcs.go index c2bdbe43a315..055ea230f771 100644 --- a/pkg/sql/opt/xform/join_funcs.go +++ b/pkg/sql/opt/xform/join_funcs.go @@ -19,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/opt/props/physical" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/errors" ) @@ -189,7 +190,6 @@ func (c *CustomFuncs) GenerateLookupJoins( // an equality with another column or a constant. numIndexKeyCols := index.LaxKeyColumnCount() - var projections memo.ProjectionsExpr var constFilters memo.FiltersExpr // Check if the first column in the index has an equality constraint, or if @@ -198,7 +198,7 @@ func (c *CustomFuncs) GenerateLookupJoins( // in most cases. firstIdxCol := scanPrivate.Table.IndexColumnID(index, 0) if _, ok := rightEq.Find(firstIdxCol); !ok { - if _, _, ok := c.findConstantFilter(onFilters, firstIdxCol); !ok { + if _, _, ok := c.findJoinFilterConstants(onFilters, firstIdxCol); !ok { return } } @@ -211,7 +211,6 @@ func (c *CustomFuncs) GenerateLookupJoins( lookupJoin.KeyCols = make(opt.ColList, 0, numIndexKeyCols) rightSideCols := make(opt.ColList, 0, numIndexKeyCols) - needProjection := false // All the lookup conditions must apply to the prefix of the index and so // the projected columns created must be created in order. @@ -223,33 +222,43 @@ func (c *CustomFuncs) GenerateLookupJoins( continue } - // Try to find a filter that constrains this column to a non-NULL constant - // value. We cannot use a NULL value because the lookup join implements - // logic equivalent to simple equality between columns (where NULL never - // equals anything). - foundVal, onIdx, ok := c.findConstantFilter(onFilters, idxCol) - if !ok || foundVal == tree.DNull { + // Try to find a filter that constrains this column to non-NULL + // constant values. We cannot use a NULL value because the lookup + // join implements logic equivalent to simple equality between + // columns (where NULL never equals anything). + foundVals, onIdx, ok := c.findJoinFilterConstants(onFilters, idxCol) + if !ok { break } - // We will project this constant value in the input to make it an equality - // column. - if projections == nil { - projections = make(memo.ProjectionsExpr, 0, numIndexKeyCols-j) + // We will join these constant values with the input to make + // equality columns for the lookup join. + if constFilters == nil { constFilters = make(memo.FiltersExpr, 0, numIndexKeyCols-j) } idxColType := c.e.f.Metadata().ColumnMeta(idxCol).Type + tupleType := types.MakeTuple([]*types.T{idxColType}) + constRows := make(memo.ScalarListExpr, len(foundVals)) constColID := c.e.f.Metadata().AddColumn( - fmt.Sprintf("project_const_col_@%d", idxCol), + fmt.Sprintf("lookup_join_const_col_@%d", idxCol), idxColType, ) - projections = append(projections, c.e.f.ConstructProjectionsItem( - c.e.f.ConstructConst(foundVal, idxColType), - constColID, - )) + for i := range constRows { + constRows[i] = c.e.f.ConstructTuple( + memo.ScalarListExpr{c.e.f.ConstructConst(foundVals[i], idxColType)}, + tupleType, + ) + } + values := c.e.f.ConstructValues( + constRows, + &memo.ValuesPrivate{ + Cols: opt.ColList{constColID}, + ID: md.NextUniqueID(), + }, + ) + lookupJoin.Input = c.e.f.ConstructInnerJoin(lookupJoin.Input, values, nil /* on */, joinPrivate) - needProjection = true lookupJoin.KeyCols = append(lookupJoin.KeyCols, constColID) rightSideCols = append(rightSideCols, idxCol) constFilters = append(constFilters, onFilters[onIdx]) @@ -265,11 +274,6 @@ func (c *CustomFuncs) GenerateLookupJoins( // is sufficient. lookupJoin.LookupColsAreTableKey = tableFDs.ColsAreLaxKey(rightSideCols.ToSet()) - // Construct the projections for the constant columns. - if needProjection { - lookupJoin.Input = c.e.f.ConstructProject(input, projections, input.Relational().OutputCols) - } - // Remove the redundant filters and update the lookup condition. lookupJoin.On = memo.ExtractRemainingJoinFilters(onFilters, lookupJoin.KeyCols, rightSideCols) lookupJoin.On.RemoveCommonFilters(constFilters) @@ -548,18 +552,30 @@ func (c *CustomFuncs) GenerateInvertedJoins( }) } -// findConstantFilter tries to find a filter that is exactly equivalent to -// constraining the given column to a constant value. Note that the constant -// value can be NULL (for an `x IS NULL` filter). -func (c *CustomFuncs) findConstantFilter( +// findJoinFilterConstants tries to find a filter that is exactly equivalent to +// constraining the given column to a constant value or a set of constant +// values. If successful, the constant values and the index of the constraining +// FiltersItem are returned. Note that the returned constant values do not +// contain NULL. +func (c *CustomFuncs) findJoinFilterConstants( filters memo.FiltersExpr, col opt.ColumnID, -) (value tree.Datum, filterIdx int, ok bool) { +) (values tree.Datums, filterIdx int, ok bool) { for filterIdx := range filters { props := filters[filterIdx].ScalarProps() if props.TightConstraints { - constCol, constVal, ok := props.Constraints.IsSingleColumnConstValue(c.e.evalCtx) - if ok && constCol == col { - return constVal, filterIdx, true + constCol, constVals, ok := props.Constraints.HasSingleColumnConstValues(c.e.evalCtx) + if !ok || constCol != col { + continue + } + hasNull := false + for i := range constVals { + if constVals[i] == tree.DNull { + hasNull = true + break + } + } + if !hasNull { + return constVals, filterIdx, true } } } diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index 3250a877d1f6..cd078eda57d0 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -2933,12 +2933,12 @@ inner-join (lookup abcde) │ ├── key columns: [1 12] = [5 6] │ ├── fd: ()-->(6), (10)-->(5,7), (1)==(5), (5)==(1) │ ├── project - │ │ ├── columns: "project_const_col_@6":12!null m:1 n:2 + │ │ ├── columns: "lookup_join_const_col_@6":12!null m:1 n:2 │ │ ├── fd: ()-->(12) │ │ ├── scan small │ │ │ └── columns: m:1 n:2 │ │ └── projections - │ │ └── 10 [as="project_const_col_@6":12] + │ │ └── 10 [as="lookup_join_const_col_@6":12] │ └── filters (true) └── filters (true) @@ -2975,13 +2975,111 @@ inner-join (lookup abcde) │ ├── key columns: [1 12 13] = [5 6 7] │ ├── fd: ()-->(6,7), (10)-->(5), (1)==(5), (5)==(1) │ ├── project - │ │ ├── columns: "project_const_col_@6":12!null "project_const_col_@7":13!null m:1 n:2 + │ │ ├── columns: "lookup_join_const_col_@7":13!null "lookup_join_const_col_@6":12!null m:1 n:2 │ │ ├── fd: ()-->(12,13) │ │ ├── scan small │ │ │ └── columns: m:1 n:2 │ │ └── projections - │ │ ├── 10 [as="project_const_col_@6":12] - │ │ └── 10 [as="project_const_col_@7":13] + │ │ ├── 10 [as="lookup_join_const_col_@7":13] + │ │ └── 10 [as="lookup_join_const_col_@6":12] + │ └── filters (true) + └── filters (true) + +# Column constrained to multiple constants used by lookup joiner. +opt expect=GenerateLookupJoinsWithFilter +SELECT * FROM small INNER JOIN abcde ON a=m AND b IN (10, 20, 30) +---- +inner-join (lookup abcde) + ├── columns: m:1!null n:2 a:5!null b:6!null c:7 d:8 e:9 + ├── key columns: [10] = [10] + ├── lookup columns are key + ├── fd: (1)==(5), (5)==(1) + ├── inner-join (lookup abcde@secondary) + │ ├── columns: m:1!null n:2 a:5!null b:6!null c:7 abcde.rowid:10!null + │ ├── key columns: [1 12] = [5 6] + │ ├── fd: (10)-->(5-7), (1)==(5), (5)==(1) + │ ├── inner-join (cross) + │ │ ├── columns: m:1 n:2 "lookup_join_const_col_@6":12!null + │ │ ├── multiplicity: left-rows(one-or-more), right-rows(zero-or-more) + │ │ ├── scan small + │ │ │ └── columns: m:1 n:2 + │ │ ├── values + │ │ │ ├── columns: "lookup_join_const_col_@6":12!null + │ │ │ ├── cardinality: [3 - 3] + │ │ │ ├── (10,) + │ │ │ ├── (20,) + │ │ │ └── (30,) + │ │ └── filters (true) + │ └── filters (true) + └── filters (true) + +# One column constrained to multiple constants and another constrained to a +# single constant used by lookup joiner. +opt expect=GenerateLookupJoinsWithFilter +SELECT * FROM small INNER JOIN abcde ON a=m AND b IN (10, 20, 30) AND c=10 +---- +inner-join (lookup abcde) + ├── columns: m:1!null n:2 a:5!null b:6!null c:7!null d:8 e:9 + ├── key columns: [10] = [10] + ├── lookup columns are key + ├── fd: ()-->(7), (1)==(5), (5)==(1) + ├── inner-join (lookup abcde@secondary) + │ ├── columns: m:1!null n:2 a:5!null b:6!null c:7!null abcde.rowid:10!null + │ ├── key columns: [1 12 13] = [5 6 7] + │ ├── fd: ()-->(7), (10)-->(5,6), (1)==(5), (5)==(1) + │ ├── project + │ │ ├── columns: "lookup_join_const_col_@7":13!null m:1 n:2 "lookup_join_const_col_@6":12!null + │ │ ├── fd: ()-->(13) + │ │ ├── inner-join (cross) + │ │ │ ├── columns: m:1 n:2 "lookup_join_const_col_@6":12!null + │ │ │ ├── multiplicity: left-rows(one-or-more), right-rows(zero-or-more) + │ │ │ ├── scan small + │ │ │ │ └── columns: m:1 n:2 + │ │ │ ├── values + │ │ │ │ ├── columns: "lookup_join_const_col_@6":12!null + │ │ │ │ ├── cardinality: [3 - 3] + │ │ │ │ ├── (10,) + │ │ │ │ ├── (20,) + │ │ │ │ └── (30,) + │ │ │ └── filters (true) + │ │ └── projections + │ │ └── 10 [as="lookup_join_const_col_@7":13] + │ └── filters (true) + └── filters (true) + +# Multiple columns constrained to multiple constants used by lookup joiner. +opt expect=GenerateLookupJoinsWithFilter +SELECT * FROM small INNER JOIN abcde ON a=m AND b IN (10, 20) AND c IN (30, 40) +---- +inner-join (lookup abcde) + ├── columns: m:1!null n:2 a:5!null b:6!null c:7!null d:8 e:9 + ├── key columns: [10] = [10] + ├── lookup columns are key + ├── fd: (1)==(5), (5)==(1) + ├── inner-join (lookup abcde@secondary) + │ ├── columns: m:1!null n:2 a:5!null b:6!null c:7!null abcde.rowid:10!null + │ ├── key columns: [1 12 13] = [5 6 7] + │ ├── fd: (10)-->(5-7), (1)==(5), (5)==(1) + │ ├── inner-join (cross) + │ │ ├── columns: m:1 n:2 "lookup_join_const_col_@6":12!null "lookup_join_const_col_@7":13!null + │ │ ├── multiplicity: left-rows(one-or-more), right-rows(zero-or-more) + │ │ ├── inner-join (cross) + │ │ │ ├── columns: m:1 n:2 "lookup_join_const_col_@6":12!null + │ │ │ ├── multiplicity: left-rows(one-or-more), right-rows(zero-or-more) + │ │ │ ├── scan small + │ │ │ │ └── columns: m:1 n:2 + │ │ │ ├── values + │ │ │ │ ├── columns: "lookup_join_const_col_@6":12!null + │ │ │ │ ├── cardinality: [2 - 2] + │ │ │ │ ├── (10,) + │ │ │ │ └── (20,) + │ │ │ └── filters (true) + │ │ ├── values + │ │ │ ├── columns: "lookup_join_const_col_@7":13!null + │ │ │ ├── cardinality: [2 - 2] + │ │ │ ├── (30,) + │ │ │ └── (40,) + │ │ └── filters (true) │ └── filters (true) └── filters (true) @@ -2999,13 +3097,13 @@ inner-join (lookup abcde) │ ├── key columns: [1 12 13] = [5 6 7] │ ├── fd: ()-->(6,7), (10)-->(5), (1)==(5), (5)==(1) │ ├── project - │ │ ├── columns: "project_const_col_@6":12!null "project_const_col_@7":13!null m:1 n:2 + │ │ ├── columns: "lookup_join_const_col_@7":13!null "lookup_join_const_col_@6":12!null m:1 n:2 │ │ ├── fd: ()-->(12,13) │ │ ├── scan small │ │ │ └── columns: m:1 n:2 │ │ └── projections - │ │ ├── 10 [as="project_const_col_@6":12] - │ │ └── 10 [as="project_const_col_@7":13] + │ │ ├── 10 [as="lookup_join_const_col_@7":13] + │ │ └── 10 [as="lookup_join_const_col_@6":12] │ └── filters (true) └── filters └── d:8 = 10 [outer=(8), constraints=(/8: [/10 - /10]; tight), fd=()-->(8)] @@ -3043,13 +3141,13 @@ project ├── key columns: [12 1 13] = [5 6 7] ├── fd: ()-->(5,7), (1)==(6), (6)==(1) ├── project - │ ├── columns: "project_const_col_@5":12!null "project_const_col_@7":13!null m:1 + │ ├── columns: "lookup_join_const_col_@7":13!null "lookup_join_const_col_@5":12!null m:1 │ ├── fd: ()-->(12,13) │ ├── scan small │ │ └── columns: m:1 │ └── projections - │ ├── 10 [as="project_const_col_@5":12] - │ └── 10 [as="project_const_col_@7":13] + │ ├── 10 [as="lookup_join_const_col_@7":13] + │ └── 10 [as="lookup_join_const_col_@5":12] └── filters (true) # Projection of constant columns work with non const expressions as well. @@ -3071,13 +3169,13 @@ inner-join (lookup bool_col) │ ├── key columns: [1 12 13] = [5 6 7] │ ├── fd: ()-->(6,7), (10)-->(5), (1)==(5), (5)==(1) │ ├── project - │ │ ├── columns: "project_const_col_@6":12!null "project_const_col_@7":13!null m:1 n:2 + │ │ ├── columns: "lookup_join_const_col_@7":13!null "lookup_join_const_col_@6":12!null m:1 n:2 │ │ ├── fd: ()-->(12,13) │ │ ├── scan small │ │ │ └── columns: m:1 n:2 │ │ └── projections - │ │ ├── 10 [as="project_const_col_@6":12] - │ │ └── true [as="project_const_col_@7":13] + │ │ ├── true [as="lookup_join_const_col_@7":13] + │ │ └── 10 [as="lookup_join_const_col_@6":12] │ └── filters (true) └── filters (true) @@ -3094,13 +3192,13 @@ inner-join (lookup bool_col) │ ├── key columns: [1 12 13] = [5 6 7] │ ├── fd: ()-->(6,7), (10)-->(5), (1)==(5), (5)==(1) │ ├── project - │ │ ├── columns: "project_const_col_@6":12!null "project_const_col_@7":13!null m:1 n:2 + │ │ ├── columns: "lookup_join_const_col_@7":13!null "lookup_join_const_col_@6":12!null m:1 n:2 │ │ ├── fd: ()-->(12,13) │ │ ├── scan small │ │ │ └── columns: m:1 n:2 │ │ └── projections - │ │ ├── 10 [as="project_const_col_@6":12] - │ │ └── true [as="project_const_col_@7":13] + │ │ ├── true [as="lookup_join_const_col_@7":13] + │ │ └── 10 [as="lookup_join_const_col_@6":12] │ └── filters (true) └── filters (true) @@ -3117,13 +3215,13 @@ inner-join (lookup bool_col) │ ├── key columns: [1 12 13] = [5 6 7] │ ├── fd: ()-->(6,7), (10)-->(5), (1)==(5), (5)==(1) │ ├── project - │ │ ├── columns: "project_const_col_@6":12!null "project_const_col_@7":13!null m:1 n:2 + │ │ ├── columns: "lookup_join_const_col_@7":13!null "lookup_join_const_col_@6":12!null m:1 n:2 │ │ ├── fd: ()-->(12,13) │ │ ├── scan small │ │ │ └── columns: m:1 n:2 │ │ └── projections - │ │ ├── 10 [as="project_const_col_@6":12] - │ │ └── false [as="project_const_col_@7":13] + │ │ ├── false [as="lookup_join_const_col_@7":13] + │ │ └── 10 [as="lookup_join_const_col_@6":12] │ └── filters (true) └── filters (true) diff --git a/pkg/sql/opt/xform/testdata/rules/join_order b/pkg/sql/opt/xform/testdata/rules/join_order index 3ff3d46ca653..5905d1c03fd4 100644 --- a/pkg/sql/opt/xform/testdata/rules/join_order +++ b/pkg/sql/opt/xform/testdata/rules/join_order @@ -280,7 +280,7 @@ New expression 3 of 3: memo join-limit=0 expect-not=ReorderJoins SELECT * FROM bx, cy, abc WHERE a = 1 AND abc.b = bx.b AND abc.c = cy.c ---- -memo (optimized, ~23KB, required=[presentation: b:1,x:2,c:4,y:5,a:7,b:8,c:9,d:10]) +memo (optimized, ~24KB, required=[presentation: b:1,x:2,c:4,y:5,a:7,b:8,c:9,d:10]) ├── G1: (inner-join G2 G3 G4) (merge-join G2 G3 G5 inner-join,+1,+8) │ └── [presentation: b:1,x:2,c:4,y:5,a:7,b:8,c:9,d:10] │ ├── best: (merge-join G2="[ordering: +1]" G3 G5 inner-join,+1,+8) @@ -333,7 +333,7 @@ memo (optimized, ~23KB, required=[presentation: b:1,x:2,c:4,y:5,a:7,b:8,c:9,d:10 memo join-limit=2 SELECT * FROM bx, cy, abc WHERE a = 1 AND abc.b = bx.b AND abc.c = cy.c ---- -memo (optimized, ~35KB, required=[presentation: b:1,x:2,c:4,y:5,a:7,b:8,c:9,d:10]) +memo (optimized, ~37KB, required=[presentation: b:1,x:2,c:4,y:5,a:7,b:8,c:9,d:10]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) (inner-join G5 G6 G7) (inner-join G6 G5 G7) (merge-join G2 G3 G8 inner-join,+1,+8) (lookup-join G3 G8 bx,keyCols=[8],outCols=(1,2,4,5,7-10)) (merge-join G5 G6 G8 inner-join,+4,+9) (lookup-join G6 G8 cy,keyCols=[9],outCols=(1,2,4,5,7-10)) │ └── [presentation: b:1,x:2,c:4,y:5,a:7,b:8,c:9,d:10] │ ├── best: (lookup-join G3 G8 bx,keyCols=[8],outCols=(1,2,4,5,7-10)) From 1a3fd7215c63ad823db01f89b38b0566caf532c3 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 7 Dec 2020 16:45:01 -0800 Subject: [PATCH 3/4] opt: generate lookup joins with CHECK constraints and computed columns Previously, only explicit filters were used to generated lookup join key columns. Now lookup join keys can be generated from CHECK constraints and computed column expressions. With this commit and the previous commit, lookup joins on partitioned indexes are explored by the optimizer. Release note (performance improvement): The query optimizer now explores plans with lookup joins on partitioned indexes, resulting in more efficient query plans in some cases. --- pkg/sql/opt/memo/testdata/stats/lookup-join | 4 +- pkg/sql/opt/xform/general_funcs.go | 74 ++++++++++++ pkg/sql/opt/xform/join_funcs.go | 52 +++++++- pkg/sql/opt/xform/select_funcs.go | 74 ------------ pkg/sql/opt/xform/testdata/external/tpce | 12 +- .../opt/xform/testdata/external/tpce-no-stats | 4 +- pkg/sql/opt/xform/testdata/external/trading | 72 ++++++----- .../xform/testdata/external/trading-mutation | 72 ++++++----- pkg/sql/opt/xform/testdata/physprops/ordering | 4 +- pkg/sql/opt/xform/testdata/rules/groupby | 2 +- pkg/sql/opt/xform/testdata/rules/join | 114 ++++++++++++++++++ 11 files changed, 336 insertions(+), 148 deletions(-) diff --git a/pkg/sql/opt/memo/testdata/stats/lookup-join b/pkg/sql/opt/memo/testdata/stats/lookup-join index cf7eaa3718a6..4d8f20b16e50 100644 --- a/pkg/sql/opt/memo/testdata/stats/lookup-join +++ b/pkg/sql/opt/memo/testdata/stats/lookup-join @@ -406,14 +406,14 @@ inner-join (lookup wxyz) │ ├── stats: [rows=19.8, distinct(1)=19.8, null(1)=0, distinct(6)=19.8, null(6)=0, distinct(7)=1, null(7)=0, distinct(11)=1, null(11)=0] │ ├── fd: ()-->(7), (9)-->(6,8), (1)==(6), (6)==(1) │ ├── project - │ │ ├── columns: "project_const_col_@7":11(int!null) m:1(int) n:2(int) + │ │ ├── columns: "lookup_join_const_col_@7":11(int!null) m:1(int) n:2(int) │ │ ├── stats: [rows=40, distinct(1)=40, null(1)=0, distinct(11)=1, null(11)=0] │ │ ├── fd: ()-->(11) │ │ ├── scan medium │ │ │ ├── columns: m:1(int) n:2(int) │ │ │ └── stats: [rows=40, distinct(1)=40, null(1)=0] │ │ └── projections - │ │ └── 10 [as="project_const_col_@7":11, type=int] + │ │ └── 10 [as="lookup_join_const_col_@7":11, type=int] │ └── filters (true) └── filters (true) diff --git a/pkg/sql/opt/xform/general_funcs.go b/pkg/sql/opt/xform/general_funcs.go index e7dc62c43521..8496785851d6 100644 --- a/pkg/sql/opt/xform/general_funcs.go +++ b/pkg/sql/opt/xform/general_funcs.go @@ -154,3 +154,77 @@ func (c *CustomFuncs) initIdxConstraintForIndex( ) return ic } + +// computedColFilters generates all filters that can be derived from the list of +// computed column expressions from the given table. A computed column can be +// used as a filter when it has a constant value. That is true when: +// +// 1. All other columns it references are constant, because other filters in +// the query constrain them to be so. +// 2. All functions in the computed column expression can be folded into +// constants (i.e. they do not have problematic side effects). +// +// Note that computed columns can depend on other computed columns; in general +// the dependencies form an acyclic directed graph. computedColFilters will +// return filters for all constant computed columns, regardless of the order of +// their dependencies. +// +// As with checkConstraintFilters, computedColFilters do not really filter any +// rows, they are rather facts or guarantees about the data. Treating them as +// filters may allow some indexes to be constrained and used. Consider the +// following example: +// +// CREATE TABLE t ( +// k INT NOT NULL, +// hash INT AS (k % 4) STORED, +// PRIMARY KEY (hash, k) +// ) +// +// SELECT * FROM t WHERE k = 5 +// +// Notice that the filter provided explicitly wouldn't allow the optimizer to +// seek using the primary index (it would have to fall back to a table scan). +// However, column "hash" can be proven to have the constant value of 1, since +// it's dependent on column "k", which has the constant value of 5. This enables +// usage of the primary index: +// +// scan t +// ├── columns: k:1(int!null) hash:2(int!null) +// ├── constraint: /2/1: [/1/5 - /1/5] +// ├── key: (2) +// └── fd: ()-->(1) +// +// The values of both columns in that index are known, enabling a single value +// constraint to be generated. +func (c *CustomFuncs) computedColFilters( + tabID opt.TableID, requiredFilters, optionalFilters memo.FiltersExpr, +) memo.FiltersExpr { + tabMeta := c.e.mem.Metadata().TableMeta(tabID) + if len(tabMeta.ComputedCols) == 0 { + return nil + } + + // Start with set of constant columns, as derived from the list of filter + // conditions. + constCols := make(map[opt.ColumnID]opt.ScalarExpr) + c.findConstantFilterCols(constCols, tabID, requiredFilters) + c.findConstantFilterCols(constCols, tabID, optionalFilters) + if len(constCols) == 0 { + // No constant values could be derived from filters, so assume that there + // are also no constant computed columns. + return nil + } + + // Construct a new filter condition for each computed column that is + // constant (i.e. all of its variables are in the constCols set). + var computedColFilters memo.FiltersExpr + for colID := range tabMeta.ComputedCols { + if c.tryFoldComputedCol(tabMeta, colID, constCols) { + constVal := constCols[colID] + // Note: Eq is not correct here because of NULLs. + eqOp := c.e.f.ConstructIs(c.e.f.ConstructVariable(colID), constVal) + computedColFilters = append(computedColFilters, c.e.f.ConstructFiltersItem(eqOp)) + } + } + return computedColFilters +} diff --git a/pkg/sql/opt/xform/join_funcs.go b/pkg/sql/opt/xform/join_funcs.go index 055ea230f771..89618d607b96 100644 --- a/pkg/sql/opt/xform/join_funcs.go +++ b/pkg/sql/opt/xform/join_funcs.go @@ -162,6 +162,45 @@ func (c *CustomFuncs) GenerateMergeJoins( // "sides" (in this example x,y on the left and z on the right) but there is // no overlap. // +// A lookup join can be created when the ON condition or implicit filters from +// CHECK constraints and computed columns constrain a prefix of the index +// columns to non-ranging constant values. To support this, the constant values +// are cross-joined with the input and used as key columns for the parent lookup +// join. +// +// For example, consider the tables and query below. +// +// CREATE TABLE abc (a INT PRIMARY KEY, b INT, c INT) +// CREATE TABLE xyz ( +// x INT PRIMARY KEY, +// y INT, +// z INT NOT NULL, +// CHECK z IN (1, 2, 3), +// INDEX (z, y) +// ) +// SELECT a, x FROM abc JOIN xyz ON a=y +// +// GenerateLookupJoins will perform the following transformation. +// +// Join LookupJoin(t@idx) +// / \ | +// / \ -> | +// Input Scan(t) Join +// / \ +// / \ +// Input Values(1, 2, 3) +// +// If a column is constrained to a single constant value, inlining normalization +// rules will reduce the cross join into a project. +// +// Join LookupJoin(t@idx) +// / \ | +// / \ -> | +// Input Scan(t) Project +// | +// | +// Input +// func (c *CustomFuncs) GenerateLookupJoins( grp memo.RelExpr, joinType opt.Operator, @@ -182,6 +221,12 @@ func (c *CustomFuncs) GenerateLookupJoins( return } + // Generate implicit filters from CHECK constraints and computed columns as + // optional filters to help generate lookup join keys. + optionalFilters := c.checkConstraintFilters(scanPrivate.Table) + computedColFilters := c.computedColFilters(scanPrivate.Table, on, optionalFilters) + optionalFilters = append(optionalFilters, computedColFilters...) + var pkCols opt.ColList var iter scanIndexIter iter.Init(c.e.mem, &c.im, scanPrivate, on, rejectInvertedIndexes) @@ -191,6 +236,7 @@ func (c *CustomFuncs) GenerateLookupJoins( numIndexKeyCols := index.LaxKeyColumnCount() var constFilters memo.FiltersExpr + allFilters := append(onFilters, optionalFilters...) // Check if the first column in the index has an equality constraint, or if // it is constrained to a constant value. This check doesn't guarantee that @@ -198,7 +244,7 @@ func (c *CustomFuncs) GenerateLookupJoins( // in most cases. firstIdxCol := scanPrivate.Table.IndexColumnID(index, 0) if _, ok := rightEq.Find(firstIdxCol); !ok { - if _, _, ok := c.findJoinFilterConstants(onFilters, firstIdxCol); !ok { + if _, _, ok := c.findJoinFilterConstants(allFilters, firstIdxCol); !ok { return } } @@ -226,7 +272,7 @@ func (c *CustomFuncs) GenerateLookupJoins( // constant values. We cannot use a NULL value because the lookup // join implements logic equivalent to simple equality between // columns (where NULL never equals anything). - foundVals, onIdx, ok := c.findJoinFilterConstants(onFilters, idxCol) + foundVals, allIdx, ok := c.findJoinFilterConstants(allFilters, idxCol) if !ok { break } @@ -261,7 +307,7 @@ func (c *CustomFuncs) GenerateLookupJoins( lookupJoin.KeyCols = append(lookupJoin.KeyCols, constColID) rightSideCols = append(rightSideCols, idxCol) - constFilters = append(constFilters, onFilters[onIdx]) + constFilters = append(constFilters, allFilters[allIdx]) } if len(lookupJoin.KeyCols) == 0 { diff --git a/pkg/sql/opt/xform/select_funcs.go b/pkg/sql/opt/xform/select_funcs.go index 3d9a2c07b3f9..328f0b4b1847 100644 --- a/pkg/sql/opt/xform/select_funcs.go +++ b/pkg/sql/opt/xform/select_funcs.go @@ -372,80 +372,6 @@ func (c *CustomFuncs) GenerateConstrainedScans( }) } -// computedColFilters generates all filters that can be derived from the list of -// computed column expressions from the given table. A computed column can be -// used as a filter when it has a constant value. That is true when: -// -// 1. All other columns it references are constant, because other filters in -// the query constrain them to be so. -// 2. All functions in the computed column expression can be folded into -// constants (i.e. they do not have problematic side effects). -// -// Note that computed columns can depend on other computed columns; in general -// the dependencies form an acyclic directed graph. computedColFilters will -// return filters for all constant computed columns, regardless of the order of -// their dependencies. -// -// As with checkConstraintFilters, computedColFilters do not really filter any -// rows, they are rather facts or guarantees about the data. Treating them as -// filters may allow some indexes to be constrained and used. Consider the -// following example: -// -// CREATE TABLE t ( -// k INT NOT NULL, -// hash INT AS (k % 4) STORED, -// PRIMARY KEY (hash, k) -// ) -// -// SELECT * FROM t WHERE k = 5 -// -// Notice that the filter provided explicitly wouldn't allow the optimizer to -// seek using the primary index (it would have to fall back to a table scan). -// However, column "hash" can be proven to have the constant value of 1, since -// it's dependent on column "k", which has the constant value of 5. This enables -// usage of the primary index: -// -// scan t -// ├── columns: k:1(int!null) hash:2(int!null) -// ├── constraint: /2/1: [/1/5 - /1/5] -// ├── key: (2) -// └── fd: ()-->(1) -// -// The values of both columns in that index are known, enabling a single value -// constraint to be generated. -func (c *CustomFuncs) computedColFilters( - tabID opt.TableID, requiredFilters, optionalFilters memo.FiltersExpr, -) memo.FiltersExpr { - tabMeta := c.e.mem.Metadata().TableMeta(tabID) - if len(tabMeta.ComputedCols) == 0 { - return nil - } - - // Start with set of constant columns, as derived from the list of filter - // conditions. - constCols := make(map[opt.ColumnID]opt.ScalarExpr) - c.findConstantFilterCols(constCols, tabID, requiredFilters) - c.findConstantFilterCols(constCols, tabID, optionalFilters) - if len(constCols) == 0 { - // No constant values could be derived from filters, so assume that there - // are also no constant computed columns. - return nil - } - - // Construct a new filter condition for each computed column that is - // constant (i.e. all of its variables are in the constCols set). - var computedColFilters memo.FiltersExpr - for colID := range tabMeta.ComputedCols { - if c.tryFoldComputedCol(tabMeta, colID, constCols) { - constVal := constCols[colID] - // Note: Eq is not correct here because of NULLs. - eqOp := c.e.f.ConstructIs(c.e.f.ConstructVariable(colID), constVal) - computedColFilters = append(computedColFilters, c.e.f.ConstructFiltersItem(eqOp)) - } - } - return computedColFilters -} - // findConstantFilterCols adds to constFilterCols mappings from table column ID // to the constant value of that column. It does this by iterating over the // given lists of filters and finding expressions that constrain columns to a diff --git a/pkg/sql/opt/xform/testdata/external/tpce b/pkg/sql/opt/xform/testdata/external/tpce index f59b07dc2777..1c7254f4ab7f 100644 --- a/pkg/sql/opt/xform/testdata/external/tpce +++ b/pkg/sql/opt/xform/testdata/external/tpce @@ -1550,7 +1550,7 @@ project │ │ ├── key: (1) │ │ ├── fd: (1)-->(3) │ │ ├── project - │ │ │ ├── columns: "project_const_col_@6":10!null tx_id:1!null tx_rate:3!null + │ │ │ ├── columns: "lookup_join_const_col_@6":10!null tx_id:1!null tx_rate:3!null │ │ │ ├── key: (1) │ │ │ ├── fd: ()-->(10), (1)-->(3) │ │ │ ├── scan taxrate @@ -1560,7 +1560,7 @@ project │ │ │ │ ├── key: (1) │ │ │ │ └── fd: (1)-->(3) │ │ │ └── projections - │ │ │ └── 0 [as="project_const_col_@6":10] + │ │ │ └── 0 [as="lookup_join_const_col_@6":10] │ │ └── filters (true) │ └── aggregations │ └── sum [as=sum:8, outer=(3)] @@ -2526,7 +2526,7 @@ project │ │ ├── key: (1) │ │ ├── fd: (1)-->(3) │ │ ├── project - │ │ │ ├── columns: "project_const_col_@6":10!null tx_id:1!null tx_rate:3!null + │ │ │ ├── columns: "lookup_join_const_col_@6":10!null tx_id:1!null tx_rate:3!null │ │ │ ├── key: (1) │ │ │ ├── fd: ()-->(10), (1)-->(3) │ │ │ ├── scan taxrate @@ -2536,7 +2536,7 @@ project │ │ │ │ ├── key: (1) │ │ │ │ └── fd: (1)-->(3) │ │ │ └── projections - │ │ │ └── 0 [as="project_const_col_@6":10] + │ │ │ └── 0 [as="lookup_join_const_col_@6":10] │ │ └── filters (true) │ └── aggregations │ └── sum [as=sum:8, outer=(3)] @@ -4156,7 +4156,7 @@ update watch_item │ │ ├── key: (7) │ │ ├── fd: ()-->(5,8), (7)-->(9), (4)==(7), (7)==(4) │ │ ├── project - │ │ │ ├── columns: "project_const_col_@5":29!null wl_id:7!null wl_c_id:8!null watch_list.crdb_internal_mvcc_timestamp:9 + │ │ │ ├── columns: "lookup_join_const_col_@5":29!null wl_id:7!null wl_c_id:8!null watch_list.crdb_internal_mvcc_timestamp:9 │ │ │ ├── key: (7) │ │ │ ├── fd: ()-->(8,29), (7)-->(9) │ │ │ ├── select @@ -4170,7 +4170,7 @@ update watch_item │ │ │ │ └── filters │ │ │ │ └── wl_c_id:8 = 0 [outer=(8), constraints=(/8: [/0 - /0]; tight), fd=()-->(8)] │ │ │ └── projections - │ │ │ └── 'SYMB' [as="project_const_col_@5":29] + │ │ │ └── 'SYMB' [as="lookup_join_const_col_@5":29] │ │ └── filters (true) │ └── projections │ └── 'SYMB' [as=wi_s_symb_new:10] diff --git a/pkg/sql/opt/xform/testdata/external/tpce-no-stats b/pkg/sql/opt/xform/testdata/external/tpce-no-stats index 9c604a8e0b8d..ec6d64ebde33 100644 --- a/pkg/sql/opt/xform/testdata/external/tpce-no-stats +++ b/pkg/sql/opt/xform/testdata/external/tpce-no-stats @@ -1269,7 +1269,7 @@ limit │ ├── fd: ()-->(7), (1,2)-->(3,4), (1)==(6), (6)==(1) │ ├── limit hint: 20.00 │ ├── project - │ │ ├── columns: "project_const_col_@7":12!null hh_h_t_id:1!null hh_t_id:2!null hh_before_qty:3!null hh_after_qty:4!null + │ │ ├── columns: "lookup_join_const_col_@7":12!null hh_h_t_id:1!null hh_t_id:2!null hh_before_qty:3!null hh_after_qty:4!null │ │ ├── key: (1,2) │ │ ├── fd: ()-->(12), (1,2)-->(3,4) │ │ ├── limit hint: 200.00 @@ -1279,7 +1279,7 @@ limit │ │ │ ├── fd: (1,2)-->(3,4) │ │ │ └── limit hint: 200.00 │ │ └── projections - │ │ └── 0 [as="project_const_col_@7":12] + │ │ └── 0 [as="lookup_join_const_col_@7":12] │ └── filters (true) └── 20 diff --git a/pkg/sql/opt/xform/testdata/external/trading b/pkg/sql/opt/xform/testdata/external/trading index 7032edb4867e..1a7770a32315 100644 --- a/pkg/sql/opt/xform/testdata/external/trading +++ b/pkg/sql/opt/xform/testdata/external/trading @@ -710,7 +710,7 @@ project │ ├── ordering: -(3|12) opt(1,2,4,10,11) [actual: -3] │ ├── limit hint: 20.00 │ ├── project - │ │ ├── columns: "project_const_col_@10":18!null "project_const_col_@11":19!null transactiondetails.dealerid:1!null transactiondetails.isbuy:2!null transactiondate:3!null cardid:4!null quantity:5!null sellprice:6!null buyprice:7!null + │ │ ├── columns: "lookup_join_const_col_@11":19!null "lookup_join_const_col_@10":18!null transactiondetails.dealerid:1!null transactiondetails.isbuy:2!null transactiondate:3!null cardid:4!null quantity:5!null sellprice:6!null buyprice:7!null │ │ ├── stats: [rows=478.646617] │ │ ├── key: (3,5) │ │ ├── fd: ()-->(1,2,4,18,19), (3,5)-->(6,7) @@ -732,8 +732,8 @@ project │ │ │ ├── ordering: -3 opt(1,2,4) [actual: -3] │ │ │ └── limit hint: 100.00 │ │ └── projections - │ │ ├── 1 [as="project_const_col_@10":18] - │ │ └── false [as="project_const_col_@11":19] + │ │ ├── false [as="lookup_join_const_col_@11":19] + │ │ └── 1 [as="lookup_join_const_col_@10":18] │ └── filters (true) └── 20 @@ -843,7 +843,7 @@ project │ │ │ │ ├── fd: (1)-->(2-6), (2,4,5)~~>(1,3,6), (1,20-22)-->(18,19) │ │ │ │ ├── ordering: +1 │ │ │ │ ├── project - │ │ │ │ │ ├── columns: "project_const_col_@18":39!null "project_const_col_@19":40!null id:1!null name:2!null rarity:3 setname:4 number:5!null isfoil:6!null + │ │ │ │ │ ├── columns: "lookup_join_const_col_@19":40!null "lookup_join_const_col_@18":39!null id:1!null name:2!null rarity:3 setname:4 number:5!null isfoil:6!null │ │ │ │ │ ├── immutable │ │ │ │ │ ├── stats: [rows=19000, distinct(1)=19000, null(1)=0, distinct(2)=13000, null(2)=0, distinct(5)=829, null(5)=0, distinct(6)=5601.15328, null(6)=0, distinct(39)=1, null(39)=0, distinct(40)=1, null(40)=0] │ │ │ │ │ ├── key: (1) @@ -865,8 +865,8 @@ project │ │ │ │ │ │ └── filters │ │ │ │ │ │ └── (name:2, setname:4, number:5) > ('Shock', '7E', 248) [outer=(2,4,5), immutable, constraints=(/2/4/5: [/'Shock'/'7E'/249 - ]; tight)] │ │ │ │ │ └── projections - │ │ │ │ │ ├── 1 [as="project_const_col_@18":39] - │ │ │ │ │ └── false [as="project_const_col_@19":40] + │ │ │ │ │ ├── false [as="lookup_join_const_col_@19":40] + │ │ │ │ │ └── 1 [as="lookup_join_const_col_@18":39] │ │ │ │ └── filters │ │ │ │ └── (transactiondate:20 >= '2020-02-28 00:00:00+00:00') AND (transactiondate:20 <= '2020-03-01 00:00:00+00:00') [outer=(20), constraints=(/20: [/'2020-02-28 00:00:00+00:00' - /'2020-03-01 00:00:00+00:00']; tight)] │ │ │ ├── scan cardsinfo @@ -1110,27 +1110,41 @@ limit │ ├── fd: (10)-->(15) │ ├── project │ │ ├── columns: quantity:14!null accountname:10!null - │ │ ├── inner-join (hash) + │ │ ├── inner-join (lookup inventorydetails) │ │ │ ├── columns: id:6!null quantity:7!null dealerid:8!null cardid:9!null accountname:10!null inventorydetails.quantity:11!null + │ │ │ ├── key columns: [16 6 17] = [8 9 10] + │ │ │ ├── lookup columns are key │ │ │ ├── fd: (8-10)-->(11), (6)==(9), (9)==(6) - │ │ │ ├── select - │ │ │ │ ├── columns: dealerid:8!null cardid:9!null accountname:10!null inventorydetails.quantity:11!null - │ │ │ │ ├── key: (8-10) - │ │ │ │ ├── fd: (8-10)-->(11) - │ │ │ │ ├── scan inventorydetails - │ │ │ │ │ ├── columns: dealerid:8!null cardid:9!null accountname:10!null inventorydetails.quantity:11!null - │ │ │ │ │ ├── constraint: /8/9/10: [/1 - /5] - │ │ │ │ │ ├── key: (8-10) - │ │ │ │ │ └── fd: (8-10)-->(11) - │ │ │ │ └── filters - │ │ │ │ └── accountname:10 IN ('account-1', 'account-2', 'account-3') [outer=(10), constraints=(/10: [/'account-1' - /'account-1'] [/'account-2' - /'account-2'] [/'account-3' - /'account-3']; tight)] - │ │ │ ├── values - │ │ │ │ ├── columns: id:6!null quantity:7!null - │ │ │ │ ├── cardinality: [2 - 2] - │ │ │ │ ├── (42948, 3) - │ │ │ │ └── (24924, 4) - │ │ │ └── filters - │ │ │ └── cardid:9 = id:6 [outer=(6,9), constraints=(/6: (/NULL - ]; /9: (/NULL - ]), fd=(6)==(9), (9)==(6)] + │ │ │ ├── inner-join (cross) + │ │ │ │ ├── columns: id:6!null quantity:7!null "lookup_join_const_col_@8":16!null "lookup_join_const_col_@10":17!null + │ │ │ │ ├── cardinality: [30 - 30] + │ │ │ │ ├── multiplicity: left-rows(one-or-more), right-rows(one-or-more) + │ │ │ │ ├── inner-join (cross) + │ │ │ │ │ ├── columns: id:6!null quantity:7!null "lookup_join_const_col_@8":16!null + │ │ │ │ │ ├── cardinality: [10 - 10] + │ │ │ │ │ ├── multiplicity: left-rows(one-or-more), right-rows(one-or-more) + │ │ │ │ │ ├── values + │ │ │ │ │ │ ├── columns: "lookup_join_const_col_@8":16!null + │ │ │ │ │ │ ├── cardinality: [5 - 5] + │ │ │ │ │ │ ├── (1,) + │ │ │ │ │ │ ├── (2,) + │ │ │ │ │ │ ├── (3,) + │ │ │ │ │ │ ├── (4,) + │ │ │ │ │ │ └── (5,) + │ │ │ │ │ ├── values + │ │ │ │ │ │ ├── columns: id:6!null quantity:7!null + │ │ │ │ │ │ ├── cardinality: [2 - 2] + │ │ │ │ │ │ ├── (42948, 3) + │ │ │ │ │ │ └── (24924, 4) + │ │ │ │ │ └── filters (true) + │ │ │ │ ├── values + │ │ │ │ │ ├── columns: "lookup_join_const_col_@10":17!null + │ │ │ │ │ ├── cardinality: [3 - 3] + │ │ │ │ │ ├── ('account-1',) + │ │ │ │ │ ├── ('account-2',) + │ │ │ │ │ └── ('account-3',) + │ │ │ │ └── filters (true) + │ │ │ └── filters (true) │ │ └── projections │ │ └── CASE WHEN quantity:7 < inventorydetails.quantity:11 THEN quantity:7 ELSE inventorydetails.quantity:11 END [as=quantity:14, outer=(7,11)] │ └── aggregations @@ -1475,7 +1489,7 @@ update cardsinfo [as=ci] │ │ ├── key columns: [39 17] = [28 29] │ │ ├── fd: ()-->(16), (17)-->(18-24,26,27), (24)-->(17-23), (17)==(26), (26)==(17) │ │ ├── project - │ │ │ ├── columns: "project_const_col_@28":39!null ci.dealerid:16!null ci.cardid:17!null buyprice:18!null sellprice:19!null discount:20!null desiredinventory:21!null actualinventory:22!null maxinventory:23!null ci.version:24!null c:26!null q:27!null + │ │ │ ├── columns: "lookup_join_const_col_@28":39!null ci.dealerid:16!null ci.cardid:17!null buyprice:18!null sellprice:19!null discount:20!null desiredinventory:21!null actualinventory:22!null maxinventory:23!null ci.version:24!null c:26!null q:27!null │ │ │ ├── cardinality: [0 - 2] │ │ │ ├── key: (17) │ │ │ ├── fd: ()-->(16,39), (17)-->(18-24,26,27), (24)-->(17-23), (17)==(26), (26)==(17) @@ -1492,7 +1506,7 @@ update cardsinfo [as=ci] │ │ │ │ │ ├── cardinality: [0 - 2] │ │ │ │ │ ├── fd: ()-->(16), (17)-->(18-24), (24)-->(17-23), (17)==(26), (26)==(17) │ │ │ │ │ ├── project - │ │ │ │ │ │ ├── columns: "project_const_col_@16":37!null c:26!null q:27!null + │ │ │ │ │ │ ├── columns: "lookup_join_const_col_@16":37!null c:26!null q:27!null │ │ │ │ │ │ ├── cardinality: [2 - 2] │ │ │ │ │ │ ├── fd: ()-->(37) │ │ │ │ │ │ ├── values @@ -1501,7 +1515,7 @@ update cardsinfo [as=ci] │ │ │ │ │ │ │ ├── (42948, 3) │ │ │ │ │ │ │ └── (24924, 4) │ │ │ │ │ │ └── projections - │ │ │ │ │ │ └── 1 [as="project_const_col_@16":37] + │ │ │ │ │ │ └── 1 [as="lookup_join_const_col_@16":37] │ │ │ │ │ └── filters (true) │ │ │ │ └── aggregations │ │ │ │ ├── first-agg [as=buyprice:18, outer=(18)] @@ -1525,7 +1539,7 @@ update cardsinfo [as=ci] │ │ │ │ └── const-agg [as=ci.dealerid:16, outer=(16)] │ │ │ │ └── ci.dealerid:16 │ │ │ └── projections - │ │ │ └── 1 [as="project_const_col_@28":39] + │ │ │ └── 1 [as="lookup_join_const_col_@28":39] │ │ └── filters (true) │ └── aggregations │ ├── sum-int [as=sum_int:34, outer=(31)] diff --git a/pkg/sql/opt/xform/testdata/external/trading-mutation b/pkg/sql/opt/xform/testdata/external/trading-mutation index 34a77d2f4a75..7641f09129d8 100644 --- a/pkg/sql/opt/xform/testdata/external/trading-mutation +++ b/pkg/sql/opt/xform/testdata/external/trading-mutation @@ -714,7 +714,7 @@ project │ ├── ordering: -(3|14) opt(1,2,4,12,13) [actual: -3] │ ├── limit hint: 20.00 │ ├── project - │ │ ├── columns: "project_const_col_@12":22!null "project_const_col_@13":23!null transactiondetails.dealerid:1!null transactiondetails.isbuy:2!null transactiondate:3!null cardid:4!null quantity:5!null sellprice:6!null buyprice:7!null + │ │ ├── columns: "lookup_join_const_col_@13":23!null "lookup_join_const_col_@12":22!null transactiondetails.dealerid:1!null transactiondetails.isbuy:2!null transactiondate:3!null cardid:4!null quantity:5!null sellprice:6!null buyprice:7!null │ │ ├── stats: [rows=478.646617] │ │ ├── key: (3,5) │ │ ├── fd: ()-->(1,2,4,22,23), (3,5)-->(6,7) @@ -736,8 +736,8 @@ project │ │ │ ├── ordering: -3 opt(1,2,4) [actual: -3] │ │ │ └── limit hint: 100.00 │ │ └── projections - │ │ ├── 1 [as="project_const_col_@12":22] - │ │ └── false [as="project_const_col_@13":23] + │ │ ├── false [as="lookup_join_const_col_@13":23] + │ │ └── 1 [as="lookup_join_const_col_@12":22] │ └── filters (true) └── 20 @@ -847,7 +847,7 @@ project │ │ │ │ ├── fd: (1)-->(2-6), (2,4,5)~~>(1,3,6), (1,24-26)-->(22,23) │ │ │ │ ├── ordering: +1 │ │ │ │ ├── project - │ │ │ │ │ ├── columns: "project_const_col_@22":45!null "project_const_col_@23":46!null id:1!null name:2!null rarity:3 setname:4 number:5!null isfoil:6!null + │ │ │ │ │ ├── columns: "lookup_join_const_col_@23":46!null "lookup_join_const_col_@22":45!null id:1!null name:2!null rarity:3 setname:4 number:5!null isfoil:6!null │ │ │ │ │ ├── immutable │ │ │ │ │ ├── stats: [rows=19000, distinct(1)=19000, null(1)=0, distinct(2)=13000, null(2)=0, distinct(5)=829, null(5)=0, distinct(6)=5601.15328, null(6)=0, distinct(45)=1, null(45)=0, distinct(46)=1, null(46)=0] │ │ │ │ │ ├── key: (1) @@ -869,8 +869,8 @@ project │ │ │ │ │ │ └── filters │ │ │ │ │ │ └── (name:2, setname:4, number:5) > ('Shock', '7E', 248) [outer=(2,4,5), immutable, constraints=(/2/4/5: [/'Shock'/'7E'/249 - ]; tight)] │ │ │ │ │ └── projections - │ │ │ │ │ ├── 1 [as="project_const_col_@22":45] - │ │ │ │ │ └── false [as="project_const_col_@23":46] + │ │ │ │ │ ├── false [as="lookup_join_const_col_@23":46] + │ │ │ │ │ └── 1 [as="lookup_join_const_col_@22":45] │ │ │ │ └── filters │ │ │ │ └── (transactiondate:24 >= '2020-02-28 00:00:00+00:00') AND (transactiondate:24 <= '2020-03-01 00:00:00+00:00') [outer=(24), constraints=(/24: [/'2020-02-28 00:00:00+00:00' - /'2020-03-01 00:00:00+00:00']; tight)] │ │ │ ├── scan cardsinfo @@ -1114,27 +1114,41 @@ limit │ ├── fd: (10)-->(17) │ ├── project │ │ ├── columns: quantity:16!null accountname:10!null - │ │ ├── inner-join (hash) + │ │ ├── inner-join (lookup inventorydetails) │ │ │ ├── columns: id:6!null quantity:7!null dealerid:8!null cardid:9!null accountname:10!null inventorydetails.quantity:11!null + │ │ │ ├── key columns: [18 6 19] = [8 9 10] + │ │ │ ├── lookup columns are key │ │ │ ├── fd: (8-10)-->(11), (6)==(9), (9)==(6) - │ │ │ ├── select - │ │ │ │ ├── columns: dealerid:8!null cardid:9!null accountname:10!null inventorydetails.quantity:11!null - │ │ │ │ ├── key: (8-10) - │ │ │ │ ├── fd: (8-10)-->(11) - │ │ │ │ ├── scan inventorydetails - │ │ │ │ │ ├── columns: dealerid:8!null cardid:9!null accountname:10!null inventorydetails.quantity:11!null - │ │ │ │ │ ├── constraint: /8/9/10: [/1 - /5] - │ │ │ │ │ ├── key: (8-10) - │ │ │ │ │ └── fd: (8-10)-->(11) - │ │ │ │ └── filters - │ │ │ │ └── accountname:10 IN ('account-1', 'account-2', 'account-3') [outer=(10), constraints=(/10: [/'account-1' - /'account-1'] [/'account-2' - /'account-2'] [/'account-3' - /'account-3']; tight)] - │ │ │ ├── values - │ │ │ │ ├── columns: id:6!null quantity:7!null - │ │ │ │ ├── cardinality: [2 - 2] - │ │ │ │ ├── (42948, 3) - │ │ │ │ └── (24924, 4) - │ │ │ └── filters - │ │ │ └── cardid:9 = id:6 [outer=(6,9), constraints=(/6: (/NULL - ]; /9: (/NULL - ]), fd=(6)==(9), (9)==(6)] + │ │ │ ├── inner-join (cross) + │ │ │ │ ├── columns: id:6!null quantity:7!null "lookup_join_const_col_@8":18!null "lookup_join_const_col_@10":19!null + │ │ │ │ ├── cardinality: [30 - 30] + │ │ │ │ ├── multiplicity: left-rows(one-or-more), right-rows(one-or-more) + │ │ │ │ ├── inner-join (cross) + │ │ │ │ │ ├── columns: id:6!null quantity:7!null "lookup_join_const_col_@8":18!null + │ │ │ │ │ ├── cardinality: [10 - 10] + │ │ │ │ │ ├── multiplicity: left-rows(one-or-more), right-rows(one-or-more) + │ │ │ │ │ ├── values + │ │ │ │ │ │ ├── columns: "lookup_join_const_col_@8":18!null + │ │ │ │ │ │ ├── cardinality: [5 - 5] + │ │ │ │ │ │ ├── (1,) + │ │ │ │ │ │ ├── (2,) + │ │ │ │ │ │ ├── (3,) + │ │ │ │ │ │ ├── (4,) + │ │ │ │ │ │ └── (5,) + │ │ │ │ │ ├── values + │ │ │ │ │ │ ├── columns: id:6!null quantity:7!null + │ │ │ │ │ │ ├── cardinality: [2 - 2] + │ │ │ │ │ │ ├── (42948, 3) + │ │ │ │ │ │ └── (24924, 4) + │ │ │ │ │ └── filters (true) + │ │ │ │ ├── values + │ │ │ │ │ ├── columns: "lookup_join_const_col_@10":19!null + │ │ │ │ │ ├── cardinality: [3 - 3] + │ │ │ │ │ ├── ('account-1',) + │ │ │ │ │ ├── ('account-2',) + │ │ │ │ │ └── ('account-3',) + │ │ │ │ └── filters (true) + │ │ │ └── filters (true) │ │ └── projections │ │ └── CASE WHEN quantity:7 < inventorydetails.quantity:11 THEN quantity:7 ELSE inventorydetails.quantity:11 END [as=quantity:16, outer=(7,11)] │ └── aggregations @@ -1491,7 +1505,7 @@ update cardsinfo [as=ci] │ │ ├── key columns: [53 21] = [36 37] │ │ ├── fd: ()-->(20), (21)-->(22-32,34,35), (28)-->(21-27,29-32), (21)==(34), (34)==(21) │ │ ├── project - │ │ │ ├── columns: "project_const_col_@36":53!null ci.dealerid:20!null ci.cardid:21!null buyprice:22!null sellprice:23!null discount:24!null desiredinventory:25!null actualinventory:26!null maxinventory:27!null ci.version:28!null ci.discountbuyprice:29 notes:30 oldinventory:31 ci.extra:32 c:34!null q:35!null + │ │ │ ├── columns: "lookup_join_const_col_@36":53!null ci.dealerid:20!null ci.cardid:21!null buyprice:22!null sellprice:23!null discount:24!null desiredinventory:25!null actualinventory:26!null maxinventory:27!null ci.version:28!null ci.discountbuyprice:29 notes:30 oldinventory:31 ci.extra:32 c:34!null q:35!null │ │ │ ├── cardinality: [0 - 2] │ │ │ ├── key: (21) │ │ │ ├── fd: ()-->(20,53), (21)-->(22-32,34,35), (28)-->(21-27,29-32), (21)==(34), (34)==(21) @@ -1508,7 +1522,7 @@ update cardsinfo [as=ci] │ │ │ │ │ ├── cardinality: [0 - 2] │ │ │ │ │ ├── fd: ()-->(20), (21)-->(22-32), (28)-->(21-27,29-32), (21)==(34), (34)==(21) │ │ │ │ │ ├── project - │ │ │ │ │ │ ├── columns: "project_const_col_@20":51!null c:34!null q:35!null + │ │ │ │ │ │ ├── columns: "lookup_join_const_col_@20":51!null c:34!null q:35!null │ │ │ │ │ │ ├── cardinality: [2 - 2] │ │ │ │ │ │ ├── fd: ()-->(51) │ │ │ │ │ │ ├── values @@ -1517,7 +1531,7 @@ update cardsinfo [as=ci] │ │ │ │ │ │ │ ├── (42948, 3) │ │ │ │ │ │ │ └── (24924, 4) │ │ │ │ │ │ └── projections - │ │ │ │ │ │ └── 1 [as="project_const_col_@20":51] + │ │ │ │ │ │ └── 1 [as="lookup_join_const_col_@20":51] │ │ │ │ │ └── filters (true) │ │ │ │ └── aggregations │ │ │ │ ├── first-agg [as=buyprice:22, outer=(22)] @@ -1549,7 +1563,7 @@ update cardsinfo [as=ci] │ │ │ │ └── const-agg [as=ci.dealerid:20, outer=(20)] │ │ │ │ └── ci.dealerid:20 │ │ │ └── projections - │ │ │ └── 1 [as="project_const_col_@36":53] + │ │ │ └── 1 [as="lookup_join_const_col_@36":53] │ │ └── filters (true) │ └── aggregations │ ├── sum-int [as=sum_int:44, outer=(39)] diff --git a/pkg/sql/opt/xform/testdata/physprops/ordering b/pkg/sql/opt/xform/testdata/physprops/ordering index 155fa568e160..2ee8d04d5346 100644 --- a/pkg/sql/opt/xform/testdata/physprops/ordering +++ b/pkg/sql/opt/xform/testdata/physprops/ordering @@ -2047,7 +2047,7 @@ distinct-on ├── fd: ()-->(7), (1)==(4,8), (4)==(1,8), (8)==(1,4) ├── ordering: +(1|4|8) opt(7) [actual: +1] ├── project - │ ├── columns: "project_const_col_@7":11!null a:1!null b:4!null + │ ├── columns: "lookup_join_const_col_@7":11!null a:1!null b:4!null │ ├── fd: ()-->(11), (1)==(4), (4)==(1) │ ├── ordering: +(1|4) [actual: +1] │ ├── inner-join (lookup t44469_b@secondary) @@ -2061,7 +2061,7 @@ distinct-on │ │ │ └── ordering: +1 │ │ └── filters (true) │ └── projections - │ └── 1 [as="project_const_col_@7":11] + │ └── 1 [as="lookup_join_const_col_@7":11] └── filters (true) # Regression test for #47041: factor check constraints into the (canonical) diff --git a/pkg/sql/opt/xform/testdata/rules/groupby b/pkg/sql/opt/xform/testdata/rules/groupby index 5c3641043285..0d7a898453d8 100644 --- a/pkg/sql/opt/xform/testdata/rules/groupby +++ b/pkg/sql/opt/xform/testdata/rules/groupby @@ -1720,7 +1720,7 @@ memo (optimized, ~4KB, required=[presentation: u:2,v:3,w:4] [ordering: +4]) memo SELECT (SELECT w FROM kuvw WHERE v=1 AND x=u) FROM xyz ORDER BY x+1, x ---- -memo (optimized, ~25KB, required=[presentation: w:10] [ordering: +11,+1]) +memo (optimized, ~27KB, required=[presentation: w:10] [ordering: +11,+1]) ├── G1: (project G2 G3 x) │ ├── [presentation: w:10] [ordering: +11,+1] │ │ ├── best: (sort G1) diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index cd078eda57d0..6c65b6d96e65 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -3225,6 +3225,120 @@ inner-join (lookup bool_col) │ └── filters (true) └── filters (true) +# Use constant CHECK constraints to generate lookup join keys. +exec-ddl +CREATE TABLE abcd_check (a INT, b INT NOT NULL, c INT, d INT, CHECK (b IN (10, 20)), INDEX (a, b, c)) +---- + +opt expect=GenerateLookupJoins +SELECT * FROM small INNER JOIN abcd_check ON a=m +---- +inner-join (lookup abcd_check) + ├── columns: m:1!null n:2 a:5!null b:6!null c:7 d:8 + ├── key columns: [9] = [9] + ├── lookup columns are key + ├── fd: (1)==(5), (5)==(1) + ├── inner-join (lookup abcd_check@secondary) + │ ├── columns: m:1!null n:2 a:5!null b:6!null c:7 abcd_check.rowid:9!null + │ ├── key columns: [1 11] = [5 6] + │ ├── fd: (9)-->(5-7), (1)==(5), (5)==(1) + │ ├── inner-join (cross) + │ │ ├── columns: m:1 n:2 "lookup_join_const_col_@6":11!null + │ │ ├── multiplicity: left-rows(one-or-more), right-rows(zero-or-more) + │ │ ├── scan small + │ │ │ └── columns: m:1 n:2 + │ │ ├── values + │ │ │ ├── columns: "lookup_join_const_col_@6":11!null + │ │ │ ├── cardinality: [2 - 2] + │ │ │ ├── (10,) + │ │ │ └── (20,) + │ │ └── filters (true) + │ └── filters (true) + └── filters (true) + +opt expect=GenerateLookupJoinsWithFilter +SELECT * FROM small INNER JOIN abcd_check ON a=m AND c=30 +---- +inner-join (lookup abcd_check) + ├── columns: m:1!null n:2 a:5!null b:6!null c:7!null d:8 + ├── key columns: [9] = [9] + ├── lookup columns are key + ├── fd: ()-->(7), (1)==(5), (5)==(1) + ├── inner-join (lookup abcd_check@secondary) + │ ├── columns: m:1!null n:2 a:5!null b:6!null c:7!null abcd_check.rowid:9!null + │ ├── key columns: [1 11 12] = [5 6 7] + │ ├── fd: ()-->(7), (9)-->(5,6), (1)==(5), (5)==(1) + │ ├── project + │ │ ├── columns: "lookup_join_const_col_@7":12!null m:1 n:2 "lookup_join_const_col_@6":11!null + │ │ ├── fd: ()-->(12) + │ │ ├── inner-join (cross) + │ │ │ ├── columns: m:1 n:2 "lookup_join_const_col_@6":11!null + │ │ │ ├── multiplicity: left-rows(one-or-more), right-rows(zero-or-more) + │ │ │ ├── scan small + │ │ │ │ └── columns: m:1 n:2 + │ │ │ ├── values + │ │ │ │ ├── columns: "lookup_join_const_col_@6":11!null + │ │ │ │ ├── cardinality: [2 - 2] + │ │ │ │ ├── (10,) + │ │ │ │ └── (20,) + │ │ │ └── filters (true) + │ │ └── projections + │ │ └── 30 [as="lookup_join_const_col_@7":12] + │ └── filters (true) + └── filters (true) + +# Use computed column expressions to generate lookup join keys. +exec-ddl +CREATE TABLE abcd_comp (a INT, b INT AS (d % 4) STORED, c INT, d INT, INDEX (a, b, c)) +---- + +opt expect=GenerateLookupJoinsWithFilter +SELECT * FROM small INNER JOIN abcd_comp ON a=m AND d=5 +---- +inner-join (lookup abcd_comp) + ├── columns: m:1!null n:2 a:5!null b:6 c:7 d:8!null + ├── key columns: [9] = [9] + ├── lookup columns are key + ├── fd: ()-->(8), (1)==(5), (5)==(1) + ├── inner-join (lookup abcd_comp@secondary) + │ ├── columns: m:1!null n:2 a:5!null b:6!null c:7 abcd_comp.rowid:9!null + │ ├── key columns: [1 11] = [5 6] + │ ├── fd: ()-->(6), (9)-->(5,7), (1)==(5), (5)==(1) + │ ├── project + │ │ ├── columns: "lookup_join_const_col_@6":11!null m:1 n:2 + │ │ ├── fd: ()-->(11) + │ │ ├── scan small + │ │ │ └── columns: m:1 n:2 + │ │ └── projections + │ │ └── 1 [as="lookup_join_const_col_@6":11] + │ └── filters (true) + └── filters + └── d:8 = 5 [outer=(8), constraints=(/8: [/5 - /5]; tight), fd=()-->(8)] + +opt expect=GenerateLookupJoinsWithFilter +SELECT * FROM small INNER JOIN abcd_comp ON a=m AND d=5 AND c=30 +---- +inner-join (lookup abcd_comp) + ├── columns: m:1!null n:2 a:5!null b:6 c:7!null d:8!null + ├── key columns: [9] = [9] + ├── lookup columns are key + ├── fd: ()-->(7,8), (1)==(5), (5)==(1) + ├── inner-join (lookup abcd_comp@secondary) + │ ├── columns: m:1!null n:2 a:5!null b:6!null c:7!null abcd_comp.rowid:9!null + │ ├── key columns: [1 11 12] = [5 6 7] + │ ├── fd: ()-->(6,7), (9)-->(5), (1)==(5), (5)==(1) + │ ├── project + │ │ ├── columns: "lookup_join_const_col_@7":12!null "lookup_join_const_col_@6":11!null m:1 n:2 + │ │ ├── fd: ()-->(11,12) + │ │ ├── scan small + │ │ │ └── columns: m:1 n:2 + │ │ └── projections + │ │ ├── 30 [as="lookup_join_const_col_@7":12] + │ │ └── 1 [as="lookup_join_const_col_@6":11] + │ └── filters (true) + └── filters + └── d:8 = 5 [outer=(8), constraints=(/8: [/5 - /5]; tight), fd=()-->(8)] + exec-ddl CREATE TABLE t(pk INT PRIMARY KEY, col0 INT, col1 INT, col2 INT, col4 INT, UNIQUE INDEX (col2)) ---- From 4f82ebc1ec81927279582edf41fe13128519274e Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Wed, 9 Dec 2020 13:17:26 -0800 Subject: [PATCH 4/4] xform: do not propagate join hints to GenerateLookupJoins cross joins This commit fixes a bug that prevented a `LOOKUP` join hint from producing a plan with a lookup join. Previously, the hint was propagated to the synthesized cross join created as input to the lookup join. This artificially inflated the cost of the cross join, making the lookup join too costly to be selected as the optimal plan. Release note: None --- pkg/sql/opt/xform/join_funcs.go | 7 +++++- pkg/sql/opt/xform/testdata/rules/join | 31 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pkg/sql/opt/xform/join_funcs.go b/pkg/sql/opt/xform/join_funcs.go index 89618d607b96..d0c712eb5e14 100644 --- a/pkg/sql/opt/xform/join_funcs.go +++ b/pkg/sql/opt/xform/join_funcs.go @@ -303,7 +303,12 @@ func (c *CustomFuncs) GenerateLookupJoins( ID: md.NextUniqueID(), }, ) - lookupJoin.Input = c.e.f.ConstructInnerJoin(lookupJoin.Input, values, nil /* on */, joinPrivate) + + // We purposefully do not propagate the join flags from joinPrivate. + // If a LOOKUP join hint was propagated to this cross join, the cost + // of the cross join would be artificially inflated and the lookup + // join would not be selected as the optimal plan. + lookupJoin.Input = c.e.f.ConstructInnerJoin(lookupJoin.Input, values, nil /* on */, &memo.JoinPrivate{}) lookupJoin.KeyCols = append(lookupJoin.KeyCols, constColID) rightSideCols = append(rightSideCols, idxCol) diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index 6c65b6d96e65..bb55c7a8c4d6 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -3013,6 +3013,37 @@ inner-join (lookup abcde) │ └── filters (true) └── filters (true) +# Test that a LOOKUP join hint does not propagate to the cross join. If it was, +# the cross join would have an artificially high cost and the lookup join would +# not be selected as the optimal plan. +opt expect=GenerateLookupJoinsWithFilter +SELECT * FROM small INNER LOOKUP JOIN abcde ON a=m AND b IN (10, 20, 30) +---- +inner-join (lookup abcde) + ├── columns: m:1!null n:2 a:5!null b:6!null c:7 d:8 e:9 + ├── key columns: [10] = [10] + ├── lookup columns are key + ├── fd: (1)==(5), (5)==(1) + ├── inner-join (lookup abcde@secondary) + │ ├── columns: m:1!null n:2 a:5!null b:6!null c:7 abcde.rowid:10!null + │ ├── flags: force lookup join (into right side) + │ ├── key columns: [1 12] = [5 6] + │ ├── fd: (10)-->(5-7), (1)==(5), (5)==(1) + │ ├── inner-join (cross) + │ │ ├── columns: m:1 n:2 "lookup_join_const_col_@6":12!null + │ │ ├── multiplicity: left-rows(one-or-more), right-rows(zero-or-more) + │ │ ├── scan small + │ │ │ └── columns: m:1 n:2 + │ │ ├── values + │ │ │ ├── columns: "lookup_join_const_col_@6":12!null + │ │ │ ├── cardinality: [3 - 3] + │ │ │ ├── (10,) + │ │ │ ├── (20,) + │ │ │ └── (30,) + │ │ └── filters (true) + │ └── filters (true) + └── filters (true) + # One column constrained to multiple constants and another constrained to a # single constant used by lookup joiner. opt expect=GenerateLookupJoinsWithFilter