From cc07c86ad258431de685e8f47e229c2101630bf6 Mon Sep 17 00:00:00 2001 From: sumeerbhola Date: Wed, 23 Dec 2020 16:01:15 -0500 Subject: [PATCH] opt,sql: use paired-joins with non-covering indexes for left joins This is done when the left outer/semi/anti join can use a lookup join. Prior to this, when the non-covering index could not fully evaluate the filter for left join we could not generate a lookup join. With this change: - Left outer join becomes a pair of two left outer joins. - Left semi join is a pair of inner join followed by left semi join. - Left anti join is a pair of left outer join followed by left anti join. Informs #55452 Release note (performance improvement): The optimizer can now generate lookup joins in certain cases for non-covering indexes, when performing a left outer/semi/anti join. --- pkg/sql/distsql_physical_planner.go | 22 +- pkg/sql/distsql_spec_exec_factory.go | 1 + pkg/sql/execinfrapb/flow_diagram.go | 3 + .../logictest/testdata/logic_test/lookup_join | 22 +- pkg/sql/lookup_join.go | 6 +- pkg/sql/opt/exec/execbuilder/relational.go | 24 +- .../opt/exec/execbuilder/testdata/lookup_join | 99 ++++++-- pkg/sql/opt/exec/explain/result_columns.go | 7 +- pkg/sql/opt/exec/factory.opt | 1 + pkg/sql/opt/memo/check_expr.go | 32 ++- pkg/sql/opt/memo/expr_format.go | 3 + pkg/sql/opt/ops/relational.opt | 16 +- pkg/sql/opt/xform/join_funcs.go | 100 ++++++-- pkg/sql/opt/xform/testdata/external/liquibase | 88 +++---- pkg/sql/opt/xform/testdata/external/navicat | 88 +++---- pkg/sql/opt/xform/testdata/rules/join | 223 ++++++++++-------- pkg/sql/opt_exec_factory.go | 5 + 17 files changed, 484 insertions(+), 256 deletions(-) diff --git a/pkg/sql/distsql_physical_planner.go b/pkg/sql/distsql_physical_planner.go index ba371d6c0700..c5bf3d4d4b5e 100644 --- a/pkg/sql/distsql_physical_planner.go +++ b/pkg/sql/distsql_physical_planner.go @@ -2228,14 +2228,18 @@ func (dsp *DistSQLPlanner) createPlanForLookupJoin( } joinReaderSpec := execinfrapb.JoinReaderSpec{ - Table: *n.table.desc.TableDesc(), - Type: n.joinType, - LockingStrength: n.table.lockingStrength, - LockingWaitPolicy: n.table.lockingWaitPolicy, - MaintainOrdering: len(n.reqOrdering) > 0, - HasSystemColumns: n.table.containsSystemColumns, - LeftJoinWithPairedJoiner: n.isSecondJoinInPairedJoiner, - LookupBatchBytesLimit: dsp.distSQLSrv.TestingKnobs.JoinReaderBatchBytesLimit, + Table: *n.table.desc.TableDesc(), + Type: n.joinType, + LockingStrength: n.table.lockingStrength, + LockingWaitPolicy: n.table.lockingWaitPolicy, + // TODO(sumeer): specifying ordering here using isFirstJoinInPairedJoiner + // is late in the sense that the cost of this has not been taken into + // account. Make this decision earlier in CustomFuncs.GenerateLookupJoins. + MaintainOrdering: len(n.reqOrdering) > 0 || n.isFirstJoinInPairedJoiner, + HasSystemColumns: n.table.containsSystemColumns, + LeftJoinWithPairedJoiner: n.isSecondJoinInPairedJoiner, + OutputGroupContinuationForLeftRow: n.isFirstJoinInPairedJoiner, + LookupBatchBytesLimit: dsp.distSQLSrv.TestingKnobs.JoinReaderBatchBytesLimit, } joinReaderSpec.IndexIdx, err = getIndexIdx(n.table.index, n.table.desc) if err != nil { @@ -2251,7 +2255,7 @@ func (dsp *DistSQLPlanner) createPlanForLookupJoin( joinReaderSpec.LookupColumnsAreKey = n.eqColsAreKey numInputNodeCols, planToStreamColMap, post, types := - mappingHelperForLookupJoins(plan, n.input, n.table, false /* addContinuationCol */) + mappingHelperForLookupJoins(plan, n.input, n.table, n.isFirstJoinInPairedJoiner) // Set the lookup condition. var indexVarMap []int diff --git a/pkg/sql/distsql_spec_exec_factory.go b/pkg/sql/distsql_spec_exec_factory.go index 9e8ff87bebb2..b513083dd2e0 100644 --- a/pkg/sql/distsql_spec_exec_factory.go +++ b/pkg/sql/distsql_spec_exec_factory.go @@ -658,6 +658,7 @@ func (e *distSQLSpecExecFactory) ConstructLookupJoin( remoteLookupExpr tree.TypedExpr, lookupCols exec.TableColumnOrdinalSet, onCond tree.TypedExpr, + isFirstJoinInPairedJoiner bool, isSecondJoinInPairedJoiner bool, reqOrdering exec.OutputOrdering, locking *tree.LockingItem, diff --git a/pkg/sql/execinfrapb/flow_diagram.go b/pkg/sql/execinfrapb/flow_diagram.go index f4bdaf561e11..ef0bd2f998ae 100644 --- a/pkg/sql/execinfrapb/flow_diagram.go +++ b/pkg/sql/execinfrapb/flow_diagram.go @@ -214,6 +214,9 @@ func (jr *JoinReaderSpec) summary() (string, []string) { if jr.LeftJoinWithPairedJoiner { details = append(details, "second join in paired-join") } + if jr.OutputGroupContinuationForLeftRow { + details = append(details, "first join in paired-join") + } return "JoinReader", details } diff --git a/pkg/sql/logictest/testdata/logic_test/lookup_join b/pkg/sql/logictest/testdata/logic_test/lookup_join index 6a620cc99b38..261bffedc27a 100644 --- a/pkg/sql/logictest/testdata/logic_test/lookup_join +++ b/pkg/sql/logictest/testdata/logic_test/lookup_join @@ -378,7 +378,7 @@ SELECT small.c, large.c FROM small LEFT JOIN large ON small.c = large.b AND larg 27 NULL 30 NULL -## Left join with ON filter on non-covering index +## Left join with ON filter on non-covering index. Will execute as paired-joins. query II rowsort SELECT small.c, large.d FROM small LEFT JOIN large ON small.c = large.b AND large.d < 30 ---- @@ -393,6 +393,26 @@ SELECT small.c, large.d FROM small LEFT JOIN large ON small.c = large.b AND larg 27 NULL 30 NULL +## Left semi join with ON filter on non-covering index. Will execute as paired-joins. +query I rowsort +SELECT small.c FROM small WHERE EXISTS(SELECT 1 FROM large WHERE small.c = large.b AND large.d < 30) +---- +6 +12 + +## Left anti join with ON filter on non-covering index. Will execute as paired-joins. +query I rowsort +SELECT small.c FROM small WHERE NOT EXISTS(SELECT 1 FROM large WHERE small.c = large.b AND large.d < 30) +---- +3 +9 +15 +18 +21 +24 +27 +30 + ########################################################### # LOOKUP JOINS ON IMPLICIT INDEX KEY COLUMNS # # https://github.com/cockroachdb/cockroach/issues/31777 # diff --git a/pkg/sql/lookup_join.go b/pkg/sql/lookup_join.go index f9d03e907527..d91dc01af15a 100644 --- a/pkg/sql/lookup_join.go +++ b/pkg/sql/lookup_join.go @@ -56,13 +56,17 @@ type lookupJoinNode struct { remoteLookupExpr tree.TypedExpr // columns are the produced columns, namely the input columns and (unless the - // join type is semi or anti join) the columns in the table scanNode. + // join type is semi or anti join) the columns in the table scanNode. It + // includes an additional continuation column when IsFirstJoinInPairedJoin + // is true. columns colinfo.ResultColumns // onCond is any ON condition to be used in conjunction with the implicit // equality condition on eqCols or the conditions in lookupExpr. onCond tree.TypedExpr + // At most one of is{First,Second}JoinInPairedJoiner can be true. + isFirstJoinInPairedJoiner bool isSecondJoinInPairedJoiner bool reqOrdering ReqOrdering diff --git a/pkg/sql/opt/exec/execbuilder/relational.go b/pkg/sql/opt/exec/execbuilder/relational.go index 5e8b8ae83c4d..f12eb9934d98 100644 --- a/pkg/sql/opt/exec/execbuilder/relational.go +++ b/pkg/sql/opt/exec/execbuilder/relational.go @@ -1752,10 +1752,25 @@ func (b *Builder) buildLookupJoin(join *memo.LookupJoinExpr) (execPlan, error) { inputCols := join.Input.Relational().OutputCols lookupCols := join.Cols.Difference(inputCols) + if join.IsFirstJoinInPairedJoiner { + lookupCols.Remove(join.ContinuationCol) + } lookupOrdinals, lookupColMap := b.getColumns(lookupCols, join.Table) - allCols := joinOutputMap(input.outputCols, lookupColMap) - + // allExprCols are the columns used in expressions evaluated by this join. + allExprCols := joinOutputMap(input.outputCols, lookupColMap) + allCols := allExprCols + if join.IsFirstJoinInPairedJoiner { + // allCols needs to include the continuation column since it will be + // in the result output by this join. + allCols = allExprCols.Copy() + maxValue, ok := allCols.MaxValue() + if !ok { + return execPlan{}, errors.AssertionFailedf("allCols should not be empty") + } + // Assign the continuation column the next unused value in the map. + allCols.Set(int(join.ContinuationCol), maxValue+1) + } res := execPlan{outputCols: allCols} if join.JoinType == opt.SemiJoinOp || join.JoinType == opt.AntiJoinOp { // For semi and anti join, only the left columns are output. @@ -1763,8 +1778,8 @@ func (b *Builder) buildLookupJoin(join *memo.LookupJoinExpr) (execPlan, error) { } ctx := buildScalarCtx{ - ivh: tree.MakeIndexedVarHelper(nil /* container */, allCols.Len()), - ivarMap: allCols, + ivh: tree.MakeIndexedVarHelper(nil /* container */, allExprCols.Len()), + ivarMap: allExprCols, } var lookupExpr, remoteLookupExpr tree.TypedExpr if len(join.LookupExpr) > 0 { @@ -1809,6 +1824,7 @@ func (b *Builder) buildLookupJoin(join *memo.LookupJoinExpr) (execPlan, error) { remoteLookupExpr, lookupOrdinals, onExpr, + join.IsFirstJoinInPairedJoiner, join.IsSecondJoinInPairedJoiner, res.reqOrdering(join), locking, diff --git a/pkg/sql/opt/exec/execbuilder/testdata/lookup_join b/pkg/sql/opt/exec/execbuilder/testdata/lookup_join index 31b233318fc7..dab38db1f8ae 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/lookup_join +++ b/pkg/sql/opt/exec/execbuilder/testdata/lookup_join @@ -706,8 +706,6 @@ vectorized: true spans: FULL SCAN # Left join with ON filter on non-covering index -# TODO(radu): this doesn't use lookup join yet, the current rules don't cover -# left join with ON condition on columns that are not covered by the index. query T EXPLAIN (VERBOSE) SELECT small.c, large.d FROM small LEFT JOIN large ON small.c = large.b AND large.d < 30 ---- @@ -718,27 +716,88 @@ vectorized: true │ columns: (c, d) │ estimated row count: 336 │ -└── • hash join (right outer) - │ columns: (b, d, c) +└── • project + │ columns: (c, b, d) │ estimated row count: 336 - │ equality: (b) = (c) │ - ├── • filter - │ │ columns: (b, d) - │ │ estimated row count: 3,303 - │ │ filter: d < 30 - │ │ - │ └── • scan - │ columns: (b, d) - │ estimated row count: 10,000 (100% of the table; stats collected ago) - │ table: large@large_pkey - │ spans: FULL SCAN + └── • lookup join (left outer) + │ columns: (c, a, b, cont, d) + │ table: large@large_pkey + │ equality: (a, b) = (a,b) + │ equality cols are key + │ pred: d < 30 + │ + └── • lookup join (left outer) + │ columns: (c, a, b, cont) + │ estimated row count: 1,000 + │ table: large@bc + │ equality: (c) = (b) + │ + └── • scan + columns: (c) + estimated row count: 100 (100% of the table; stats collected ago) + table: small@small_pkey + spans: FULL SCAN + +# Left semi-join with ON filter on non-covering index +query T +EXPLAIN (VERBOSE) SELECT small.c FROM small WHERE EXISTS(SELECT 1 FROM large WHERE small.c = large.b AND large.d < 30) +---- +distribution: full +vectorized: true +· +• project +│ columns: (c) +│ estimated row count: 100 +│ +└── • lookup join (semi) + │ columns: (c, a, b, cont) + │ table: large@large_pkey + │ equality: (a, b) = (a,b) + │ equality cols are key + │ pred: d < 30 │ - └── • scan - columns: (c) - estimated row count: 100 (100% of the table; stats collected ago) - table: small@small_pkey - spans: FULL SCAN + └── • lookup join (inner) + │ columns: (c, a, b, cont) + │ estimated row count: 990 + │ table: large@bc + │ equality: (c) = (b) + │ + └── • scan + columns: (c) + estimated row count: 100 (100% of the table; stats collected ago) + table: small@small_pkey + spans: FULL SCAN + +# Left anti-join with ON filter on non-covering index +query T +EXPLAIN (VERBOSE) SELECT small.c FROM small WHERE NOT EXISTS(SELECT 1 FROM large WHERE small.c = large.b AND large.d < 30) +---- +distribution: full +vectorized: true +· +• project +│ columns: (c) +│ estimated row count: 0 +│ +└── • lookup join (anti) + │ columns: (c, a, b, cont) + │ table: large@large_pkey + │ equality: (a, b) = (a,b) + │ equality cols are key + │ pred: d < 30 + │ + └── • lookup join (left outer) + │ columns: (c, a, b, cont) + │ estimated row count: 1,000 + │ table: large@bc + │ equality: (c) = (b) + │ + └── • scan + columns: (c) + estimated row count: 100 (100% of the table; stats collected ago) + table: small@small_pkey + spans: FULL SCAN ########################################################### # LOOKUP JOINS ON IMPLICIT INDEX KEY COLUMNS # diff --git a/pkg/sql/opt/exec/explain/result_columns.go b/pkg/sql/opt/exec/explain/result_columns.go index 5ae050ba4f8d..5c2a15104d3a 100644 --- a/pkg/sql/opt/exec/explain/result_columns.go +++ b/pkg/sql/opt/exec/explain/result_columns.go @@ -81,7 +81,12 @@ func getResultColumns( case lookupJoinOp: a := args.(*lookupJoinArgs) - return joinColumns(a.JoinType, inputs[0], tableColumns(a.Table, a.LookupCols)), nil + cols := joinColumns(a.JoinType, inputs[0], tableColumns(a.Table, a.LookupCols)) + // The following matches the behavior of execFactory.ConstructLookupJoin. + if a.IsFirstJoinInPairedJoiner { + cols = append(cols, colinfo.ResultColumn{Name: "cont", Typ: types.Bool}) + } + return cols, nil case ordinalityOp: return appendColumns(inputs[0], colinfo.ResultColumn{ diff --git a/pkg/sql/opt/exec/factory.opt b/pkg/sql/opt/exec/factory.opt index ac3a7173c3fb..fae22c98da61 100644 --- a/pkg/sql/opt/exec/factory.opt +++ b/pkg/sql/opt/exec/factory.opt @@ -280,6 +280,7 @@ define LookupJoin { RemoteLookupExpr tree.TypedExpr LookupCols exec.TableColumnOrdinalSet OnCond tree.TypedExpr + IsFirstJoinInPairedJoiner bool IsSecondJoinInPairedJoiner bool ReqOrdering exec.OutputOrdering Locking *tree.LockingItem diff --git a/pkg/sql/opt/memo/check_expr.go b/pkg/sql/opt/memo/check_expr.go index b38368c5d135..14d72e9c8a0d 100644 --- a/pkg/sql/opt/memo/check_expr.go +++ b/pkg/sql/opt/memo/check_expr.go @@ -234,15 +234,33 @@ func (m *Memo) CheckExpr(e opt.Expr) { if !t.Cols.SubsetOf(requiredCols) { panic(errors.AssertionFailedf("lookup join with columns that are not required")) } - if t.IsSecondJoinInPairedJoiner { - ij, ok := t.Input.(*InvertedJoinExpr) - if !ok { + if t.IsFirstJoinInPairedJoiner { + switch t.JoinType { + case opt.InnerJoinOp, opt.LeftJoinOp: + default: panic(errors.AssertionFailedf( - "lookup paired-join is paired with %T instead of inverted join", t.Input)) + "first join in paired joiner must be an inner or left join. found %s", + t.JoinType.String(), + )) } - if !ij.IsFirstJoinInPairedJoiner { - panic(errors.AssertionFailedf( - "lookup paired-join is paired with inverted join that thinks it is unpaired")) + if t.ContinuationCol == 0 { + panic(errors.AssertionFailedf("first join in paired joiner must have a continuation column")) + } + } + if t.IsSecondJoinInPairedJoiner { + switch firstJoin := t.Input.(type) { + case *InvertedJoinExpr: + if !firstJoin.IsFirstJoinInPairedJoiner { + panic(errors.AssertionFailedf( + "lookup paired-join is paired with inverted join that thinks it is unpaired")) + } + case *LookupJoinExpr: + if !firstJoin.IsFirstJoinInPairedJoiner { + panic(errors.AssertionFailedf( + "lookup paired-join is paired with lookup join that thinks it is unpaired")) + } + default: + panic(errors.AssertionFailedf("lookup paired-join is paired with %T", t.Input)) } } diff --git a/pkg/sql/opt/memo/expr_format.go b/pkg/sql/opt/memo/expr_format.go index 607954338846..838c4cb7e7b1 100644 --- a/pkg/sql/opt/memo/expr_format.go +++ b/pkg/sql/opt/memo/expr_format.go @@ -525,6 +525,9 @@ func (f *ExprFmtCtx) formatRelational(e RelExpr, tp treeprinter.Node) { if t.LookupColsAreTableKey { tp.Childf("lookup columns are key") } + if t.IsFirstJoinInPairedJoiner { + f.formatColList(e, tp, "first join in paired joiner; continuation column:", opt.ColList{t.ContinuationCol}) + } if t.IsSecondJoinInPairedJoiner { tp.Childf("second join in paired joiner") } diff --git a/pkg/sql/opt/ops/relational.opt b/pkg/sql/opt/ops/relational.opt index 630aa855f3ce..74e7957d7b4c 100644 --- a/pkg/sql/opt/ops/relational.opt +++ b/pkg/sql/opt/ops/relational.opt @@ -402,10 +402,22 @@ define LookupJoinPrivate { # table (and thus each left row matches with at most one table row). LookupColsAreTableKey bool - # IsSecondJoinInPairedJoiner is true if this is the second join of a - # paired-joiner used for left joins. + # At most one of Is{First,Second}JoinInPairedJoiner can be true. + # + # IsFirstJoinInPairedJoiner is true if this is the first (i.e., lower in the + # plan tree) join of a paired-joiner used for left joins. + IsFirstJoinInPairedJoiner bool + + # IsSecondJoinInPairedJoiner is true if this is the second (i.e., higher in + # the plan tree) join of a paired-joiner used for left joins. IsSecondJoinInPairedJoiner bool + # ContinuationCol is the column ID of the continuation column when + # IsFirstJoinInPairedJoiner is true. The continuation column is a boolean + # column that indicates whether an output row is a continuation of a group + # corresponding to a single left input row. + ContinuationCol ColumnID + # LocalityOptimized is true if this lookup join is part of a locality # optimized search strategy. For semi, inner, and left joins, this means # that RemoteLookupExpr will be non-nil. See comments above that field for diff --git a/pkg/sql/opt/xform/join_funcs.go b/pkg/sql/opt/xform/join_funcs.go index 24199fec20af..9961af400a90 100644 --- a/pkg/sql/opt/xform/join_funcs.go +++ b/pkg/sql/opt/xform/join_funcs.go @@ -158,10 +158,12 @@ func (c *CustomFuncs) GenerateMergeJoins( // Input Scan(t) Input // // -// 2. The index is not covering. We have to generate an index join above the -// lookup join. Note that this index join is also implemented as a -// LookupJoin, because an IndexJoin can only output columns from one table, -// whereas we also need to output columns from Input. +// 2. The index is not covering, but we can fully evaluate the ON condition +// using the index, or we are doing an InnerJoin. We have to generate +// an index join above the lookup join. Note that this index join is also +// implemented as a LookupJoin, because an IndexJoin can only output +// columns from one table, whereas we also need to output columns from +// Input. // // Join LookupJoin(t@primary) // / \ | @@ -178,13 +180,31 @@ func (c *CustomFuncs) GenerateMergeJoins( // // We want to first join abc with the index on y (which provides columns y, x) // and then use a lookup join to retrieve column z. The "index join" (top -// LookupJoin) will produce columns a,b,c,x,y; the lookup columns are just z -// (the original index join produced x,y,z). +// LookupJoin) will produce columns a,b,c,x,y,z; the lookup columns are just z +// (the original lookup join produced a,b,c,x,y). // // Note that the top LookupJoin "sees" column IDs from the table on both // "sides" (in this example x,y on the left and z on the right) but there is // no overlap. // +// 3. The index is not covering and we cannot fully evaluate the ON condition +// using the index, and we are doing a LeftJoin/SemiJoin/AntiJoin. This is +// handled using a lower-upper pair of joins that are further specialized +// as paired-joins. The first (lower) join outputs a continuation column +// that is used by the second (upper) join. Like case 2, both are lookup +// joins, but paired-joins explicitly know their role in the pair and +// behave accordingly. +// +// For example, using the same tables in the example for case 2: +// SELECT * FROM abc LEFT JOIN xyz ON a=y AND b=z +// +// The first join will evaluate a=y and produce columns a,b,c,x,y,cont +// where cont is the continuation column used to group together rows that +// correspond to the same original a,b,c. The second join will fetch z from +// the primary index, evaluate b=z, and produce columns a,b,c,x,y,z. A +// similar approach works for anti-joins and semi-joins. +// +// // A lookup join can be created when the ON condition or implicit filters from // CHECK constraints and computed columns constrain a prefix of the index // columns to non-ranging constant values. To support this, the constant values @@ -585,21 +605,27 @@ func (c *CustomFuncs) generateLookupJoinsImpl( } } - // All code that follows is for case 2 (see function comment). + // All code that follows is for cases 2 and 3 (see function comment). + // We need to generate two joins: a lower join followed by an upper join. + // In case 3, this lower-upper pair of joins is further specialized into + // paired-joins where we refer to the lower as first and upper as second. if scanPrivate.Flags.NoIndexJoin { return } - if joinType == opt.SemiJoinOp || joinType == opt.AntiJoinOp { - // We cannot use a non-covering index for semi and anti join. Note that - // since the semi/anti join doesn't pass through any columns, "non - // covering" here means that not all columns in the ON condition are - // available. - // - // TODO(radu): We could create a semi/anti join on top of an inner join if - // the lookup columns form a key (to guarantee that input rows are not - // duplicated by the inner join). - return + pairedJoins := false + continuationCol := opt.ColumnID(0) + lowerJoinType := joinType + if joinType == opt.SemiJoinOp { + // Case 3: Semi joins are converted to a pair consisting of an inner + // lookup join and semi lookup join. + pairedJoins = true + lowerJoinType = opt.InnerJoinOp + } else if joinType == opt.AntiJoinOp { + // Case 3: Anti joins are converted to a pair consisting of a left + // lookup join and anti lookup join. + pairedJoins = true + lowerJoinType = opt.LeftJoinOp } if pkCols == nil { @@ -619,6 +645,7 @@ func (c *CustomFuncs) generateLookupJoinsImpl( // can refer to: input columns, or columns available in the index. onCols := indexCols.Union(inputProps.OutputCols) if c.FiltersBoundBy(lookupJoin.On, onCols) { + // Case 2. // The ON condition refers only to the columns available in the index. // // For LeftJoin, both LookupJoins perform a LeftJoin. A null-extended row @@ -632,21 +659,33 @@ func (c *CustomFuncs) generateLookupJoinsImpl( // conditions that refer to other columns. We can put the former in the // lower LookupJoin and the latter in the index join. // - // This works for InnerJoin but not for LeftJoin because of a - // technicality: if an input (left) row has matches in the lower - // LookupJoin but has no matches in the index join, only the columns - // looked up by the top index join get NULL-extended. + // This works in a straightforward manner for InnerJoin but not for + // LeftJoin because of a technicality: if an input (left) row has + // matches in the lower LookupJoin but has no matches in the index join, + // only the columns looked up by the top index join get NULL-extended. + // Additionally if none of the lower matches are matches in the index + // join, we want to output only one NULL-extended row. To accomplish + // this, we need to use paired-joins. if joinType == opt.LeftJoinOp { - // TODO(radu): support LeftJoin, perhaps by looking up all columns and - // discarding columns that are already available from the lower - // LookupJoin. This requires a projection to avoid having the same - // ColumnIDs on both sides of the index join. - return + // Case 3. + pairedJoins = true + // The lowerJoinType continues to be LeftJoinOp. } + // We have already set pairedJoins=true for SemiJoin, AntiJoin earlier, + // and we don't need to do that for InnerJoin. The following sets up the + // ON conditions for both Case 2 and Case 3, when doing 2 joins that + // will each evaluate part of the ON condition. conditions := lookupJoin.On lookupJoin.On = c.ExtractBoundConditions(conditions, onCols) indexJoin.On = c.ExtractUnboundConditions(conditions, onCols) } + if pairedJoins { + lookupJoin.JoinType = lowerJoinType + continuationCol = c.constructContinuationColumnForPairedJoin() + lookupJoin.IsFirstJoinInPairedJoiner = true + lookupJoin.ContinuationCol = continuationCol + lookupJoin.Cols.Add(continuationCol) + } indexJoin.Input = c.e.f.ConstructLookupJoin( lookupJoin.Input, @@ -659,6 +698,15 @@ func (c *CustomFuncs) generateLookupJoinsImpl( indexJoin.KeyCols = pkCols indexJoin.Cols = rightCols.Union(inputProps.OutputCols) indexJoin.LookupColsAreTableKey = true + if pairedJoins { + indexJoin.IsSecondJoinInPairedJoiner = true + } + + // If this is a semi- or anti-join, ensure the columns do not include any + // unneeded right-side columns. + if joinType == opt.SemiJoinOp || joinType == opt.AntiJoinOp { + indexJoin.Cols = inputProps.OutputCols.Union(indexJoin.On.OuterCols()) + } // Create the LookupJoin for the index join in the same group. c.e.mem.AddLookupJoinToGroup(&indexJoin, grp) diff --git a/pkg/sql/opt/xform/testdata/external/liquibase b/pkg/sql/opt/xform/testdata/external/liquibase index a9881318772b..0b6e4bb287c5 100644 --- a/pkg/sql/opt/xform/testdata/external/liquibase +++ b/pkg/sql/opt/xform/testdata/external/liquibase @@ -204,66 +204,66 @@ project │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ ├── key: (1,84) │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (84)-->(85), (1,84)-->(36,37,91,105,106), (105)-->(106), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) - │ │ │ │ │ ├── right-join (hash) + │ │ │ │ │ ├── left-join (lookup pg_index [as=ind]) │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 indexrelid:84 indrelid:85 indisclustered:91 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 + │ │ │ │ │ │ ├── key columns: [84] = [84] + │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ ├── second join in paired joiner │ │ │ │ │ │ ├── key: (1,84) │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (84)-->(85), (1,84)-->(36,37,91), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) - │ │ │ │ │ │ ├── select - │ │ │ │ │ │ │ ├── columns: indexrelid:84!null indrelid:85!null indisclustered:91!null - │ │ │ │ │ │ │ ├── key: (84) - │ │ │ │ │ │ │ ├── fd: ()-->(91), (84)-->(85) - │ │ │ │ │ │ │ ├── scan pg_index [as=ind] - │ │ │ │ │ │ │ │ ├── columns: indexrelid:84!null indrelid:85!null indisclustered:91!null - │ │ │ │ │ │ │ │ ├── key: (84) - │ │ │ │ │ │ │ │ └── fd: (84)-->(85,91) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ └── indisclustered:91 [outer=(91), constraints=(/91: [/true - /true]; tight), fd=()-->(91)] - │ │ │ │ │ │ ├── left-join (lookup pg_tablespace [as=t]) - │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 - │ │ │ │ │ │ │ ├── key columns: [8] = [36] - │ │ │ │ │ │ │ ├── lookup columns are key - │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,36,37,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) - │ │ │ │ │ │ │ ├── left-join (lookup pg_foreign_server [as=fs]) - │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 - │ │ │ │ │ │ │ │ ├── key columns: [135] = [139] + │ │ │ │ │ │ ├── left-join (lookup pg_index@pg_index_indrelid_index [as=ind]) + │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 indexrelid:84 indrelid:85 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 continuation:239 + │ │ │ │ │ │ │ ├── key columns: [1] = [85] + │ │ │ │ │ │ │ ├── first join in paired joiner; continuation column: continuation:239 + │ │ │ │ │ │ │ ├── key: (1,84) + │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,36,37,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3), (84)-->(85,239) + │ │ │ │ │ │ │ ├── left-join (lookup pg_tablespace [as=t]) + │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 + │ │ │ │ │ │ │ │ ├── key columns: [8] = [36] │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (3)==(30), (30)==(3) - │ │ │ │ │ │ │ │ ├── left-join (lookup pg_foreign_table [as=ft]) - │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null ftrelid:134 ftserver:135 ftoptions:136 - │ │ │ │ │ │ │ │ │ ├── key columns: [1] = [134] + │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,36,37,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) + │ │ │ │ │ │ │ │ ├── left-join (lookup pg_foreign_server [as=fs]) + │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 + │ │ │ │ │ │ │ │ │ ├── key columns: [135] = [139] │ │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136), (3)==(30), (30)==(3) - │ │ │ │ │ │ │ │ │ ├── inner-join (hash) - │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null - │ │ │ │ │ │ │ │ │ │ ├── multiplicity: left-rows(zero-or-one), right-rows(zero-or-more) + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (3)==(30), (30)==(3) + │ │ │ │ │ │ │ │ │ ├── left-join (lookup pg_foreign_table [as=ft]) + │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null ftrelid:134 ftserver:135 ftoptions:136 + │ │ │ │ │ │ │ │ │ │ ├── key columns: [1] = [134] + │ │ │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (3)==(30), (30)==(3) - │ │ │ │ │ │ │ │ │ │ ├── select - │ │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 + │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136), (3)==(30), (30)==(3) + │ │ │ │ │ │ │ │ │ │ ├── inner-join (hash) + │ │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null + │ │ │ │ │ │ │ │ │ │ │ ├── multiplicity: left-rows(zero-or-one), right-rows(zero-or-more) │ │ │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ │ │ ├── fd: (1)-->(2,3,5,8,10,13,15,17,20,22,23,26,27), (2,3)-->(1,5,8,10,13,15,17,20,22,23,26,27) - │ │ │ │ │ │ │ │ │ │ │ ├── scan pg_class [as=c] + │ │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (3)==(30), (30)==(3) + │ │ │ │ │ │ │ │ │ │ │ ├── select │ │ │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 │ │ │ │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ │ │ │ └── fd: (1)-->(2,3,5,8,10,13,15,17,20,22,23,26,27), (2,3)-->(1,5,8,10,13,15,17,20,22,23,26,27) + │ │ │ │ │ │ │ │ │ │ │ │ ├── fd: (1)-->(2,3,5,8,10,13,15,17,20,22,23,26,27), (2,3)-->(1,5,8,10,13,15,17,20,22,23,26,27) + │ │ │ │ │ │ │ │ │ │ │ │ ├── scan pg_class [as=c] + │ │ │ │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 + │ │ │ │ │ │ │ │ │ │ │ │ │ ├── key: (1) + │ │ │ │ │ │ │ │ │ │ │ │ │ └── fd: (1)-->(2,3,5,8,10,13,15,17,20,22,23,26,27), (2,3)-->(1,5,8,10,13,15,17,20,22,23,26,27) + │ │ │ │ │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ │ │ │ │ └── (c.relkind:17 = 'r') OR (c.relkind:17 = 'f') [outer=(17), constraints=(/17: [/'f' - /'f'] [/'r' - /'r']; tight)] + │ │ │ │ │ │ │ │ │ │ │ ├── scan pg_namespace@pg_namespace_nspname_index [as=n] + │ │ │ │ │ │ │ │ │ │ │ │ ├── columns: n.oid:30!null n.nspname:31!null + │ │ │ │ │ │ │ │ │ │ │ │ ├── constraint: /31: [/'public' - /'public'] + │ │ │ │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ │ │ │ └── fd: ()-->(30,31) │ │ │ │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ │ │ │ └── (c.relkind:17 = 'r') OR (c.relkind:17 = 'f') [outer=(17), constraints=(/17: [/'f' - /'f'] [/'r' - /'r']; tight)] - │ │ │ │ │ │ │ │ │ │ ├── scan pg_namespace@pg_namespace_nspname_index [as=n] - │ │ │ │ │ │ │ │ │ │ │ ├── columns: n.oid:30!null n.nspname:31!null - │ │ │ │ │ │ │ │ │ │ │ ├── constraint: /31: [/'public' - /'public'] - │ │ │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] - │ │ │ │ │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ │ │ │ │ └── fd: ()-->(30,31) - │ │ │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ │ │ └── n.oid:30 = c.relnamespace:3 [outer=(3,30), constraints=(/3: (/NULL - ]; /30: (/NULL - ]), fd=(3)==(30), (30)==(3)] + │ │ │ │ │ │ │ │ │ │ │ └── n.oid:30 = c.relnamespace:3 [outer=(3,30), constraints=(/3: (/NULL - ]; /30: (/NULL - ]), fd=(3)==(30), (30)==(3)] + │ │ │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ └── filters - │ │ │ │ │ │ └── indrelid:85 = c.oid:1 [outer=(1,85), constraints=(/1: (/NULL - ]; /85: (/NULL - ]), fd=(1)==(85), (85)==(1)] + │ │ │ │ │ │ └── indisclustered:91 [outer=(91), constraints=(/91: [/true - /true]; tight), fd=()-->(91)] │ │ │ │ │ └── filters (true) │ │ │ │ └── filters │ │ │ │ └── i.inhrelid:44 = c.oid:1 [outer=(1,44), constraints=(/1: (/NULL - ]; /44: (/NULL - ]), fd=(1)==(44), (44)==(1)] diff --git a/pkg/sql/opt/xform/testdata/external/navicat b/pkg/sql/opt/xform/testdata/external/navicat index 2fdef7864097..16d52516f27a 100644 --- a/pkg/sql/opt/xform/testdata/external/navicat +++ b/pkg/sql/opt/xform/testdata/external/navicat @@ -208,66 +208,66 @@ sort │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ ├── key: (1,84) │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (84)-->(85), (1,84)-->(36,37,91,105,106), (105)-->(106), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) - │ │ │ │ │ ├── right-join (hash) + │ │ │ │ │ ├── left-join (lookup pg_index [as=ind]) │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 indexrelid:84 indrelid:85 indisclustered:91 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 + │ │ │ │ │ │ ├── key columns: [84] = [84] + │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ ├── second join in paired joiner │ │ │ │ │ │ ├── key: (1,84) │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (84)-->(85), (1,84)-->(36,37,91), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) - │ │ │ │ │ │ ├── select - │ │ │ │ │ │ │ ├── columns: indexrelid:84!null indrelid:85!null indisclustered:91!null - │ │ │ │ │ │ │ ├── key: (84) - │ │ │ │ │ │ │ ├── fd: ()-->(91), (84)-->(85) - │ │ │ │ │ │ │ ├── scan pg_index [as=ind] - │ │ │ │ │ │ │ │ ├── columns: indexrelid:84!null indrelid:85!null indisclustered:91!null - │ │ │ │ │ │ │ │ ├── key: (84) - │ │ │ │ │ │ │ │ └── fd: (84)-->(85,91) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ └── indisclustered:91 [outer=(91), constraints=(/91: [/true - /true]; tight), fd=()-->(91)] - │ │ │ │ │ │ ├── left-join (lookup pg_tablespace [as=t]) - │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 - │ │ │ │ │ │ │ ├── key columns: [8] = [36] - │ │ │ │ │ │ │ ├── lookup columns are key - │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,36,37,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) - │ │ │ │ │ │ │ ├── left-join (lookup pg_foreign_server [as=fs]) - │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 - │ │ │ │ │ │ │ │ ├── key columns: [135] = [139] + │ │ │ │ │ │ ├── left-join (lookup pg_index@pg_index_indrelid_index [as=ind]) + │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 indexrelid:84 indrelid:85 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 continuation:239 + │ │ │ │ │ │ │ ├── key columns: [1] = [85] + │ │ │ │ │ │ │ ├── first join in paired joiner; continuation column: continuation:239 + │ │ │ │ │ │ │ ├── key: (1,84) + │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,36,37,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3), (84)-->(85,239) + │ │ │ │ │ │ │ ├── left-join (lookup pg_tablespace [as=t]) + │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 + │ │ │ │ │ │ │ │ ├── key columns: [8] = [36] │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (3)==(30), (30)==(3) - │ │ │ │ │ │ │ │ ├── left-join (lookup pg_foreign_table [as=ft]) - │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null ftrelid:134 ftserver:135 ftoptions:136 - │ │ │ │ │ │ │ │ │ ├── key columns: [1] = [134] + │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,36,37,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) + │ │ │ │ │ │ │ │ ├── left-join (lookup pg_foreign_server [as=fs]) + │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 + │ │ │ │ │ │ │ │ │ ├── key columns: [135] = [139] │ │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136), (3)==(30), (30)==(3) - │ │ │ │ │ │ │ │ │ ├── inner-join (hash) - │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null - │ │ │ │ │ │ │ │ │ │ ├── multiplicity: left-rows(zero-or-one), right-rows(zero-or-more) + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (3)==(30), (30)==(3) + │ │ │ │ │ │ │ │ │ ├── left-join (lookup pg_foreign_table [as=ft]) + │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null ftrelid:134 ftserver:135 ftoptions:136 + │ │ │ │ │ │ │ │ │ │ ├── key columns: [1] = [134] + │ │ │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (3)==(30), (30)==(3) - │ │ │ │ │ │ │ │ │ │ ├── select - │ │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 + │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136), (3)==(30), (30)==(3) + │ │ │ │ │ │ │ │ │ │ ├── inner-join (hash) + │ │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null + │ │ │ │ │ │ │ │ │ │ │ ├── multiplicity: left-rows(zero-or-one), right-rows(zero-or-more) │ │ │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ │ │ ├── fd: (1)-->(2,3,5,8,10,13,15,17,20,22,23,26,27), (2,3)-->(1,5,8,10,13,15,17,20,22,23,26,27) - │ │ │ │ │ │ │ │ │ │ │ ├── scan pg_class [as=c] + │ │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (3)==(30), (30)==(3) + │ │ │ │ │ │ │ │ │ │ │ ├── select │ │ │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 │ │ │ │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ │ │ │ └── fd: (1)-->(2,3,5,8,10,13,15,17,20,22,23,26,27), (2,3)-->(1,5,8,10,13,15,17,20,22,23,26,27) + │ │ │ │ │ │ │ │ │ │ │ │ ├── fd: (1)-->(2,3,5,8,10,13,15,17,20,22,23,26,27), (2,3)-->(1,5,8,10,13,15,17,20,22,23,26,27) + │ │ │ │ │ │ │ │ │ │ │ │ ├── scan pg_class [as=c] + │ │ │ │ │ │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 + │ │ │ │ │ │ │ │ │ │ │ │ │ ├── key: (1) + │ │ │ │ │ │ │ │ │ │ │ │ │ └── fd: (1)-->(2,3,5,8,10,13,15,17,20,22,23,26,27), (2,3)-->(1,5,8,10,13,15,17,20,22,23,26,27) + │ │ │ │ │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ │ │ │ │ └── (c.relkind:17 = 'r') OR (c.relkind:17 = 'f') [outer=(17), constraints=(/17: [/'f' - /'f'] [/'r' - /'r']; tight)] + │ │ │ │ │ │ │ │ │ │ │ ├── scan pg_namespace@pg_namespace_nspname_index [as=n] + │ │ │ │ │ │ │ │ │ │ │ │ ├── columns: n.oid:30!null n.nspname:31!null + │ │ │ │ │ │ │ │ │ │ │ │ ├── constraint: /31: [/'public' - /'public'] + │ │ │ │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ │ │ │ └── fd: ()-->(30,31) │ │ │ │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ │ │ │ └── (c.relkind:17 = 'r') OR (c.relkind:17 = 'f') [outer=(17), constraints=(/17: [/'f' - /'f'] [/'r' - /'r']; tight)] - │ │ │ │ │ │ │ │ │ │ ├── scan pg_namespace@pg_namespace_nspname_index [as=n] - │ │ │ │ │ │ │ │ │ │ │ ├── columns: n.oid:30!null n.nspname:31!null - │ │ │ │ │ │ │ │ │ │ │ ├── constraint: /31: [/'public' - /'public'] - │ │ │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] - │ │ │ │ │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ │ │ │ │ └── fd: ()-->(30,31) - │ │ │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ │ │ └── n.oid:30 = c.relnamespace:3 [outer=(3,30), constraints=(/3: (/NULL - ]; /30: (/NULL - ]), fd=(3)==(30), (30)==(3)] + │ │ │ │ │ │ │ │ │ │ │ └── n.oid:30 = c.relnamespace:3 [outer=(3,30), constraints=(/3: (/NULL - ]; /30: (/NULL - ]), fd=(3)==(30), (30)==(3)] + │ │ │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ └── filters - │ │ │ │ │ │ └── indrelid:85 = c.oid:1 [outer=(1,85), constraints=(/1: (/NULL - ]; /85: (/NULL - ]), fd=(1)==(85), (85)==(1)] + │ │ │ │ │ │ └── indisclustered:91 [outer=(91), constraints=(/91: [/true - /true]; tight), fd=()-->(91)] │ │ │ │ │ └── filters (true) │ │ │ │ └── filters │ │ │ │ └── i.inhrelid:44 = c.oid:1 [outer=(1,44), constraints=(/1: (/NULL - ]; /44: (/NULL - ]), fd=(1)==(44), (44)==(1)] diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index 4c1e6bb55266..85d1b6c6e5f2 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -2324,19 +2324,24 @@ inner-join (lookup abcd) └── c:8 > n:2 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ])] # Non-covering case, extra filter not bound by index, left join. -# In this case, we can't yet convert to a lookup join (see -# the GenerateLookupJoins custom func). -opt expect-not=GenerateLookupJoins +# In this case, we can generate lookup joins as paired-joins. +opt expect=GenerateLookupJoins SELECT * FROM small LEFT JOIN abcd ON a=m AND c>n ---- -right-join (hash) +left-join (lookup abcd) ├── columns: m:1 n:2 a:6 b:7 c:8 - ├── scan abcd - │ └── columns: a:6 b:7 c:8 - ├── scan small - │ └── columns: m:1 n:2 + ├── key columns: [9] = [9] + ├── lookup columns are key + ├── second join in paired joiner + ├── left-join (lookup abcd@abcd_a_b_idx) + │ ├── columns: m:1 n:2 a:6 b:7 abcd.rowid:9 continuation:12 + │ ├── key columns: [1] = [6] + │ ├── first join in paired joiner; continuation column: continuation:12 + │ ├── fd: (9)-->(6,7,12) + │ ├── scan small + │ │ └── columns: m:1 n:2 + │ └── filters (true) └── filters - ├── a:6 = m:1 [outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)] └── c:8 > n:2 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ])] @@ -2583,19 +2588,25 @@ semi-join (lookup abcd@abcd_a_b_idx) │ └── columns: m:1 n:2 └── filters (true) -# We should not generate a lookup semi-join when the index doesn't contain all -# columns in the join condition. -opt expect-not=GenerateLookupJoins +# We can generate a lookup semi-join when the index doesn't contain all +# columns in the join condition, using paired-joins. +opt expect=GenerateLookupJoins SELECT m, n FROM small WHERE EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c) ---- -semi-join (hash) +semi-join (lookup abcd) ├── columns: m:1 n:2 - ├── scan small - │ └── columns: m:1 n:2 - ├── scan abcd - │ └── columns: a:6 c:8 + ├── key columns: [9] = [9] + ├── lookup columns are key + ├── second join in paired joiner + ├── inner-join (lookup abcd@abcd_a_b_idx) + │ ├── columns: m:1!null n:2 a:6!null abcd.rowid:9!null continuation:13 + │ ├── key columns: [1] = [6] + │ ├── first join in paired joiner; continuation column: continuation:13 + │ ├── fd: (9)-->(6,13), (1)==(6), (6)==(1) + │ ├── scan small + │ │ └── columns: m:1 n:2 + │ └── filters (true) └── filters - ├── m:1 = a:6 [outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)] └── n:2 = c:8 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ]), fd=(2)==(8), (8)==(2)] # Lookup anti-join with index that contains all columns in the join condition. @@ -2609,19 +2620,25 @@ anti-join (lookup abcd@abcd_a_b_idx) │ └── columns: m:1 n:2 └── filters (true) -# We should not generate a lookup anti-join when the index doesn't contain all -# columns in the join condition. -opt expect-not=GenerateLookupJoins +# We can generate a lookup anti-join when the index doesn't contain all +# columns in the join condition, using paired-joins. +opt expect=GenerateLookupJoins SELECT m, n FROM small WHERE NOT EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c) ---- -anti-join (hash) +anti-join (lookup abcd) ├── columns: m:1 n:2 - ├── scan small - │ └── columns: m:1 n:2 - ├── scan abcd - │ └── columns: a:6 c:8 + ├── key columns: [9] = [9] + ├── lookup columns are key + ├── second join in paired joiner + ├── left-join (lookup abcd@abcd_a_b_idx) + │ ├── columns: m:1 n:2 a:6 abcd.rowid:9 continuation:13 + │ ├── key columns: [1] = [6] + │ ├── first join in paired joiner; continuation column: continuation:13 + │ ├── fd: (9)-->(6,13) + │ ├── scan small + │ │ └── columns: m:1 n:2 + │ └── filters (true) └── filters - ├── m:1 = a:6 [outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)] └── n:2 = c:8 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ]), fd=(2)==(8), (8)==(2)] # Regression test for #59615. Ensure that invalid lookup joins are not created @@ -2830,6 +2847,10 @@ left-join (lookup lookup_expr [as=t]) │ └── filters (true) └── filters (true) +exec-ddl +DROP INDEX idx_vrw +---- + # The OR filter gets converted to an IN expression in the lookup expression # filters. # TODO(rytaft): The OR filter shouldn't get re-applied as an additional filter @@ -3066,23 +3087,27 @@ inner-join (lookup abcd) └── c:8 > n:2 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ])] # Non-covering case, extra filter not bound by index, left join. -# In this case, we can't yet convert to a lookup join (see -# the GenerateLookupJoins custom func). -opt expect-not=GenerateLookupJoinsWithFilter +# In this case, we can generate lookup joins as paired-joins. +opt expect=GenerateLookupJoinsWithFilter SELECT * FROM small LEFT JOIN abcd ON a=m AND c>n AND b>1 ---- -right-join (hash) +left-join (lookup abcd) ├── columns: m:1 n:2 a:6 b:7 c:8 - ├── select - │ ├── columns: a:6 b:7!null c:8 - │ ├── scan abcd - │ │ └── columns: a:6 b:7 c:8 - │ └── filters - │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] - ├── scan small - │ └── columns: m:1 n:2 + ├── key columns: [9] = [9] + ├── lookup columns are key + ├── second join in paired joiner + ├── left-join (lookup abcd@abcd_a_b_idx) + │ ├── columns: m:1 n:2 a:6 b:7 abcd.rowid:9 continuation:12 + │ ├── lookup expression + │ │ └── filters + │ │ ├── a:6 = m:1 [outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)] + │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] + │ ├── first join in paired joiner; continuation column: continuation:12 + │ ├── fd: (9)-->(6,7,12) + │ ├── scan small + │ │ └── columns: m:1 n:2 + │ └── filters (true) └── filters - ├── a:6 = m:1 [outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)] └── c:8 > n:2 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ])] # Constant columns are projected and used by lookup joiner. @@ -3604,22 +3629,26 @@ semi-join (lookup abcd@abcd_a_b_idx) └── filters └── a:6 > b:7 [outer=(6,7), constraints=(/6: (/NULL - ]; /7: (/NULL - ])] -# We should not generate a lookup semi-join when the index is non-covering. -opt expect-not=GenerateLookupJoinsWithFilter +# We can generate a lookup semi-join when the index is non-covering using +# paired-joins. +opt expect=GenerateLookupJoinsWithFilter SELECT m, n FROM small WHERE EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c AND a > b) ---- -semi-join (hash) +semi-join (lookup abcd) ├── columns: m:1 n:2 - ├── scan small - │ └── columns: m:1 n:2 - ├── select - │ ├── columns: a:6!null b:7!null c:8 - │ ├── scan abcd - │ │ └── columns: a:6 b:7 c:8 + ├── key columns: [9] = [9] + ├── lookup columns are key + ├── second join in paired joiner + ├── inner-join (lookup abcd@abcd_a_b_idx) + │ ├── columns: m:1!null n:2 a:6!null b:7!null abcd.rowid:9!null continuation:13 + │ ├── key columns: [1] = [6] + │ ├── first join in paired joiner; continuation column: continuation:13 + │ ├── fd: (9)-->(6,7,13), (1)==(6), (6)==(1) + │ ├── scan small + │ │ └── columns: m:1 n:2 │ └── filters │ └── a:6 > b:7 [outer=(6,7), constraints=(/6: (/NULL - ]; /7: (/NULL - ])] └── filters - ├── m:1 = a:6 [outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)] └── n:2 = c:8 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ]), fd=(2)==(8), (8)==(2)] # Lookup anti-join with covering index. @@ -3634,22 +3663,26 @@ anti-join (lookup abcd@abcd_a_b_idx) └── filters └── a:6 > b:7 [outer=(6,7), constraints=(/6: (/NULL - ]; /7: (/NULL - ])] -# We should not generate a lookup semi-join when the index is non-covering. -opt expect-not=GenerateLookupJoinsWithFilter +# We can generate a lookup semi-join when the index is non-covering using +# paired-joins. +opt expect=GenerateLookupJoinsWithFilter SELECT m, n FROM small WHERE NOT EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c AND a > b) ---- -anti-join (hash) +anti-join (lookup abcd) ├── columns: m:1 n:2 - ├── scan small - │ └── columns: m:1 n:2 - ├── select - │ ├── columns: a:6!null b:7!null c:8 - │ ├── scan abcd - │ │ └── columns: a:6 b:7 c:8 + ├── key columns: [9] = [9] + ├── lookup columns are key + ├── second join in paired joiner + ├── left-join (lookup abcd@abcd_a_b_idx) + │ ├── columns: m:1 n:2 a:6 b:7 abcd.rowid:9 continuation:13 + │ ├── key columns: [1] = [6] + │ ├── first join in paired joiner; continuation column: continuation:13 + │ ├── fd: (9)-->(6,7,13) + │ ├── scan small + │ │ └── columns: m:1 n:2 │ └── filters │ └── a:6 > b:7 [outer=(6,7), constraints=(/6: (/NULL - ]; /7: (/NULL - ])] └── filters - ├── m:1 = a:6 [outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)] └── n:2 = c:8 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ]), fd=(2)==(8), (8)==(2)] # -------------------------------------------------- @@ -3838,30 +3871,28 @@ project │ └── columns: m:1 n:2 └── filters (true) -# We should not generate a lookup semi-join when the index does not cover "s" -# which is referenced in the remaining filter. -opt expect-not=GenerateLookupJoinsWithFilter +# We can generate a lookup semi-join when the index does not cover "s", +# which is referenced in the remaining filter, by using paired-joins. +opt expect=GenerateLookupJoinsWithFilter SELECT m FROM small WHERE EXISTS (SELECT 1 FROM partial_tab WHERE s = 'foo' AND n = i) ---- project ├── columns: m:1 - └── semi-join (hash) + └── semi-join (lookup partial_tab) ├── columns: m:1 n:2 - ├── scan small - │ └── columns: m:1 n:2 - ├── select - │ ├── columns: i:7 s:8!null - │ ├── fd: ()-->(8) - │ ├── index-join partial_tab - │ │ ├── columns: i:7 s:8 - │ │ └── scan partial_tab@partial_idx,partial - │ │ ├── columns: k:6!null i:7 - │ │ ├── key: (6) - │ │ └── fd: (6)-->(7) - │ └── filters - │ └── s:8 = 'foo' [outer=(8), constraints=(/8: [/'foo' - /'foo']; tight), fd=()-->(8)] + ├── key columns: [6] = [6] + ├── lookup columns are key + ├── second join in paired joiner + ├── inner-join (lookup partial_tab@partial_idx,partial) + │ ├── columns: m:1 n:2!null k:6!null i:7!null continuation:12 + │ ├── key columns: [2] = [7] + │ ├── first join in paired joiner; continuation column: continuation:12 + │ ├── fd: (6)-->(7,12), (2)==(7), (7)==(2) + │ ├── scan small + │ │ └── columns: m:1 n:2 + │ └── filters (true) └── filters - └── n:2 = i:7 [outer=(2,7), constraints=(/2: (/NULL - ]; /7: (/NULL - ]), fd=(2)==(7), (7)==(2)] + └── s:8 = 'foo' [outer=(8), constraints=(/8: [/'foo' - /'foo']; tight), fd=()-->(8)] # Generate a lookup anti-join when the index does not cover "s", but the # reference to "s" no longer exists in the filters. @@ -3877,30 +3908,28 @@ project │ └── columns: m:1 n:2 └── filters (true) -# We should not generate a lookup anti-join when the index does not cover "s" -# which is referenced in the remaining filter. -opt expect-not=GenerateLookupJoinsWithFilter +# We can generate a lookup anti-join when the index does not cover "s", +# which is referenced in the remaining filter, by using paired-joins. +opt expect=GenerateLookupJoinsWithFilter SELECT m FROM small WHERE NOT EXISTS (SELECT 1 FROM partial_tab WHERE s = 'foo' AND n = i) ---- project ├── columns: m:1 - └── anti-join (hash) + └── anti-join (lookup partial_tab) ├── columns: m:1 n:2 - ├── scan small - │ └── columns: m:1 n:2 - ├── select - │ ├── columns: i:7 s:8!null - │ ├── fd: ()-->(8) - │ ├── index-join partial_tab - │ │ ├── columns: i:7 s:8 - │ │ └── scan partial_tab@partial_idx,partial - │ │ ├── columns: k:6!null i:7 - │ │ ├── key: (6) - │ │ └── fd: (6)-->(7) - │ └── filters - │ └── s:8 = 'foo' [outer=(8), constraints=(/8: [/'foo' - /'foo']; tight), fd=()-->(8)] + ├── key columns: [6] = [6] + ├── lookup columns are key + ├── second join in paired joiner + ├── left-join (lookup partial_tab@partial_idx,partial) + │ ├── columns: m:1 n:2 k:6 i:7 continuation:12 + │ ├── key columns: [2] = [7] + │ ├── first join in paired joiner; continuation column: continuation:12 + │ ├── fd: (6)-->(7,12) + │ ├── scan small + │ │ └── columns: m:1 n:2 + │ └── filters (true) └── filters - └── n:2 = i:7 [outer=(2,7), constraints=(/2: (/NULL - ]; /7: (/NULL - ]), fd=(2)==(7), (7)==(2)] + └── s:8 = 'foo' [outer=(8), constraints=(/8: [/'foo' - /'foo']; tight), fd=()-->(8)] # A lookup semi-join on a partial index should have the same cost as a lookup # semi-join on a non-partial index. @@ -5346,7 +5375,7 @@ WHERE n.name = 'Upper West Side' OR n.name = 'Upper East Side' GROUP BY n.name, n.geom ---- -memo (optimized, ~33KB, required=[presentation: name:16,popn_per_sqkm:22]) +memo (optimized, ~34KB, required=[presentation: name:16,popn_per_sqkm:22]) ├── G1: (project G2 G3 name) │ └── [presentation: name:16,popn_per_sqkm:22] │ ├── best: (project G2 G3 name) diff --git a/pkg/sql/opt_exec_factory.go b/pkg/sql/opt_exec_factory.go index 701c4fb03cbe..edf4b4f87222 100644 --- a/pkg/sql/opt_exec_factory.go +++ b/pkg/sql/opt_exec_factory.go @@ -638,6 +638,7 @@ func (ef *execFactory) ConstructLookupJoin( remoteLookupExpr tree.TypedExpr, lookupCols exec.TableColumnOrdinalSet, onCond tree.TypedExpr, + isFirstJoinInPairedJoiner bool, isSecondJoinInPairedJoiner bool, reqOrdering exec.OutputOrdering, locking *tree.LockingItem, @@ -674,6 +675,7 @@ func (ef *execFactory) ConstructLookupJoin( table: tableScan, joinType: joinType, eqColsAreKey: eqColsAreKey, + isFirstJoinInPairedJoiner: isFirstJoinInPairedJoiner, isSecondJoinInPairedJoiner: isSecondJoinInPairedJoiner, reqOrdering: ReqOrdering(reqOrdering), } @@ -692,6 +694,9 @@ func (ef *execFactory) ConstructLookupJoin( n.onCond = pred.iVarHelper.Rebind(onCond) } n.columns = pred.cols + if isFirstJoinInPairedJoiner { + n.columns = append(n.columns, colinfo.ResultColumn{Name: "cont", Typ: types.Bool}) + } return n, nil }