From 67893af5836f3a76ecd829e475b8660e1a22cde5 Mon Sep 17 00:00:00 2001 From: Rebecca Taft Date: Sun, 29 May 2022 20:48:23 -0500 Subject: [PATCH] opt: fix bug with incorrect results produced by paired left lookup join Prior to this patch, it was possible for a paired join to produce incorrect results for a left lookup join. In particular, some output rows had non-NULL values for right-side columns when the right-side columns should have been NULL. This commit fixes the issue by updating the optimizer to ensure that only columns from the second join in the paired join (the index join) are projected, not columns from the first (the lookup join). Fixes #81968 Release note (bug fix): Fixed an issue where a left lookup join could have incorrect results. In particular, some output rows could have non-NULL values for right-side columns when the right-side columns should have been NULL. This issue only exists in 22.1.0 and prior development releases of 22.1. --- .../logictest/testdata/logic_test/lookup_join | 39 ++++ .../opt/exec/execbuilder/testdata/lookup_join | 12 +- pkg/sql/opt/xform/join_funcs.go | 65 ++++++- pkg/sql/opt/xform/testdata/external/liquibase | 12 +- pkg/sql/opt/xform/testdata/external/navicat | 12 +- pkg/sql/opt/xform/testdata/rules/join | 181 ++++++++++++------ 6 files changed, 244 insertions(+), 77 deletions(-) diff --git a/pkg/sql/logictest/testdata/logic_test/lookup_join b/pkg/sql/logictest/testdata/logic_test/lookup_join index 89b634101e3f..f8f45a6e56d2 100644 --- a/pkg/sql/logictest/testdata/logic_test/lookup_join +++ b/pkg/sql/logictest/testdata/logic_test/lookup_join @@ -693,3 +693,42 @@ query I SELECT k FROM t79384a INNER LOOKUP JOIN t79384b ON k = a AND b IN (1, 2, 3) AND c > 0 ---- 1 + +# Regression test for #81968. Paired left joins should set all right columns +# for unmatched rows to NULL. +statement ok +CREATE TABLE items ( + id INT NOT NULL PRIMARY KEY, + chat_id INT NOT NULL, + author_id INT NOT NULL, + INDEX chat_id_idx (chat_id) +); +CREATE TABLE views ( + chat_id INT NOT NULL, + user_id INT NOT NULL, + PRIMARY KEY (chat_id, user_id) +); +INSERT INTO views(chat_id, user_id) VALUES (1, 1); +INSERT INTO items(id, chat_id, author_id) VALUES +(1, 1, 1), +(2, 1, 1), +(3, 1, 1); + +query I +SELECT (SELECT count(items.id) + FROM items + WHERE items.chat_id = views.chat_id + AND items.author_id != views.user_id) +FROM views +WHERE chat_id = 1 + AND user_id = 1; +---- +0 + +query IIIII +SELECT * FROM views LEFT LOOKUP JOIN items +ON items.chat_id = views.chat_id +AND items.author_id != views.user_id +WHERE views.chat_id = 1 and views.user_id = 1; +---- +1 1 NULL NULL NULL diff --git a/pkg/sql/opt/exec/execbuilder/testdata/lookup_join b/pkg/sql/opt/exec/execbuilder/testdata/lookup_join index 9eee624d5f36..d25334f7e868 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/lookup_join +++ b/pkg/sql/opt/exec/execbuilder/testdata/lookup_join @@ -721,14 +721,14 @@ vectorized: true │ estimated row count: 336 │ └── • lookup join (left outer) - │ columns: (c, a, b, cont, d) + │ columns: (c, a, b, c, cont, b, d) │ table: large@large_pkey │ equality: (a, b) = (a,b) │ equality cols are key │ pred: d < 30 │ └── • lookup join (left outer) - │ columns: (c, a, b, cont) + │ columns: (c, a, b, c, cont) │ estimated row count: 1,000 │ table: large@bc │ equality: (c) = (b) @@ -751,14 +751,14 @@ vectorized: true │ estimated row count: 100 │ └── • lookup join (semi) - │ columns: (c, a, b, cont) + │ columns: (c, a, b, c, cont) │ table: large@large_pkey │ equality: (a, b) = (a,b) │ equality cols are key │ pred: d < 30 │ └── • lookup join (inner) - │ columns: (c, a, b, cont) + │ columns: (c, a, b, c, cont) │ estimated row count: 990 │ table: large@bc │ equality: (c) = (b) @@ -781,14 +781,14 @@ vectorized: true │ estimated row count: 0 │ └── • lookup join (anti) - │ columns: (c, a, b, cont) + │ columns: (c, a, b, c, cont) │ table: large@large_pkey │ equality: (a, b) = (a,b) │ equality cols are key │ pred: d < 30 │ └── • lookup join (left outer) - │ columns: (c, a, b, cont) + │ columns: (c, a, b, c, cont) │ estimated row count: 1,000 │ table: large@bc │ equality: (c) = (b) diff --git a/pkg/sql/opt/xform/join_funcs.go b/pkg/sql/opt/xform/join_funcs.go index a78f88f0a72b..cf1a52b7ab3c 100644 --- a/pkg/sql/opt/xform/join_funcs.go +++ b/pkg/sql/opt/xform/join_funcs.go @@ -354,6 +354,7 @@ func (c *CustomFuncs) generateLookupJoinsImpl( var pkCols opt.ColList var eqColMap opt.ColMap + var newScanPrivate *memo.ScanPrivate var iter scanIndexIter iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, on, rejectInvertedIndexes) iter.ForEach(func(index cat.Index, onFilters memo.FiltersExpr, indexCols opt.ColSet, _ bool, _ memo.ProjectionsExpr) { @@ -730,11 +731,38 @@ func (c *CustomFuncs) generateLookupJoinsImpl( indexJoin.On = c.ExtractUnboundConditions(conditions, onCols) } if pairedJoins { + // Create a new ScanPrivate, which will be used below for the first lookup + // join in the pair. Note: this must happen before the continuation column + // is created to ensure that the continuation column will have the highest + // column ID. + // + // See the comment where this newScanPrivate is used below in mapLookupJoin + // for details about why it's needed. + if newScanPrivate == nil { + newScanPrivate = c.DuplicateScanPrivate(scanPrivate) + } + lookupJoin.JoinType = lowerJoinType continuationCol = c.constructContinuationColumnForPairedJoin() lookupJoin.IsFirstJoinInPairedJoiner = true lookupJoin.ContinuationCol = continuationCol lookupJoin.Cols.Add(continuationCol) + + // Map the lookup join to use the new table and column IDs from the + // newScanPrivate created above. We want to make sure that the column IDs + // returned by the lookup join are different from the IDs that will be + // returned by the top level index join. + // + // In addition to avoiding subtle bugs in the optimizer when the same + // column ID is reused, this mapping is also essential for correct behavior + // at execution time in the case of a left paired join. This is because a + // row that matches in the first left join (the lookup join) might be a + // false positive and fail to match in the second left join (the index + // join). If an original left row has no matches after the second left join, + // it must appear as a null-extended row with all right-hand columns null. + // If one of the right-hand columns comes from the lookup join, however, + // it might incorrectly show up as non-null (see #58892 and #81968). + c.mapLookupJoin(&lookupJoin, indexCols, newScanPrivate) } indexJoin.Input = c.e.f.ConstructLookupJoin( @@ -745,7 +773,7 @@ func (c *CustomFuncs) generateLookupJoinsImpl( indexJoin.JoinType = joinType indexJoin.Table = scanPrivate.Table indexJoin.Index = cat.PrimaryIndex - indexJoin.KeyCols = pkCols + indexJoin.KeyCols = c.getPkCols(lookupJoin.Table) indexJoin.Cols = rightCols.Union(inputProps.OutputCols) indexJoin.LookupColsAreTableKey = true if pairedJoins { @@ -951,6 +979,41 @@ func (c *CustomFuncs) constructContinuationColumnForPairedJoin() opt.ColumnID { return c.e.f.Metadata().AddColumn("continuation", c.BoolType()) } +// mapLookupJoin maps the given lookup join to use the table and columns +// provided in newScanPrivate. The lookup join is modified in place. indexCols +// contains the pre-calculated index columns used by the given lookupJoin. +// +// Note that columns from the input are not mapped. For example, KeyCols +// does not need to be mapped below since it only contains input columns. +func (c *CustomFuncs) mapLookupJoin( + lookupJoin *memo.LookupJoinExpr, indexCols opt.ColSet, newScanPrivate *memo.ScanPrivate, +) { + tabID := lookupJoin.Table + newTabID := newScanPrivate.Table + + // Get the new index columns. + newIndexCols := c.e.mem.Metadata().TableMeta(newTabID).IndexColumns(lookupJoin.Index) + + // Create a map from the source columns to the destination columns. + var srcColsToDstCols opt.ColMap + for srcCol, ok := indexCols.Next(0); ok; srcCol, ok = indexCols.Next(srcCol + 1) { + ord := tabID.ColumnOrdinal(srcCol) + dstCol := newTabID.ColumnID(ord) + srcColsToDstCols.Set(int(srcCol), int(dstCol)) + } + + lookupJoin.Table = newTabID + lookupExpr := c.RemapCols(&lookupJoin.LookupExpr, srcColsToDstCols).(*memo.FiltersExpr) + lookupJoin.LookupExpr = *lookupExpr + remoteLookupExpr := c.RemapCols(&lookupJoin.RemoteLookupExpr, srcColsToDstCols).(*memo.FiltersExpr) + lookupJoin.RemoteLookupExpr = *remoteLookupExpr + lookupJoin.Cols = lookupJoin.Cols.Difference(indexCols).Union(newIndexCols) + constFilters := c.RemapCols(&lookupJoin.ConstFilters, srcColsToDstCols).(*memo.FiltersExpr) + lookupJoin.ConstFilters = *constFilters + on := c.RemapCols(&lookupJoin.On, srcColsToDstCols).(*memo.FiltersExpr) + lookupJoin.On = *on +} + // GenerateInvertedJoins is similar to GenerateLookupJoins, but instead // of generating lookup joins with regular indexes, it generates lookup joins // with inverted indexes. Similar to GenerateLookupJoins, there are two cases diff --git a/pkg/sql/opt/xform/testdata/external/liquibase b/pkg/sql/opt/xform/testdata/external/liquibase index 0b6e4bb287c5..5e4f5ac25981 100644 --- a/pkg/sql/opt/xform/testdata/external/liquibase +++ b/pkg/sql/opt/xform/testdata/external/liquibase @@ -206,17 +206,17 @@ project │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (84)-->(85), (1,84)-->(36,37,91,105,106), (105)-->(106), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) │ │ │ │ │ ├── left-join (lookup pg_index [as=ind]) │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 indexrelid:84 indrelid:85 indisclustered:91 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 - │ │ │ │ │ │ ├── key columns: [84] = [84] + │ │ │ │ │ │ ├── key columns: [932] = [84] │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ ├── second join in paired joiner │ │ │ │ │ │ ├── key: (1,84) │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (84)-->(85), (1,84)-->(36,37,91), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) │ │ │ │ │ │ ├── left-join (lookup pg_index@pg_index_indrelid_index [as=ind]) - │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 indexrelid:84 indrelid:85 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 continuation:239 - │ │ │ │ │ │ │ ├── key columns: [1] = [85] - │ │ │ │ │ │ │ ├── first join in paired joiner; continuation column: continuation:239 - │ │ │ │ │ │ │ ├── key: (1,84) - │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,36,37,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3), (84)-->(85,239) + │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 indexrelid:932 indrelid:933 continuation:953 + │ │ │ │ │ │ │ ├── key columns: [1] = [933] + │ │ │ │ │ │ │ ├── first join in paired joiner; continuation column: continuation:953 + │ │ │ │ │ │ │ ├── key: (1,932) + │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,36,37,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3), (932)-->(933,953) │ │ │ │ │ │ │ ├── left-join (lookup pg_tablespace [as=t]) │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 │ │ │ │ │ │ │ │ ├── key columns: [8] = [36] diff --git a/pkg/sql/opt/xform/testdata/external/navicat b/pkg/sql/opt/xform/testdata/external/navicat index 16d52516f27a..04905afaed9f 100644 --- a/pkg/sql/opt/xform/testdata/external/navicat +++ b/pkg/sql/opt/xform/testdata/external/navicat @@ -210,17 +210,17 @@ sort │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (84)-->(85), (1,84)-->(36,37,91,105,106), (105)-->(106), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) │ │ │ │ │ ├── left-join (lookup pg_index [as=ind]) │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 indexrelid:84 indrelid:85 indisclustered:91 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 - │ │ │ │ │ │ ├── key columns: [84] = [84] + │ │ │ │ │ │ ├── key columns: [932] = [84] │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ ├── second join in paired joiner │ │ │ │ │ │ ├── key: (1,84) │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (84)-->(85), (1,84)-->(36,37,91), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3) │ │ │ │ │ │ ├── left-join (lookup pg_index@pg_index_indrelid_index [as=ind]) - │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 indexrelid:84 indrelid:85 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 continuation:239 - │ │ │ │ │ │ │ ├── key columns: [1] = [85] - │ │ │ │ │ │ │ ├── first join in paired joiner; continuation column: continuation:239 - │ │ │ │ │ │ │ ├── key: (1,84) - │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,36,37,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3), (84)-->(85,239) + │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 indexrelid:932 indrelid:933 continuation:953 + │ │ │ │ │ │ │ ├── key columns: [1] = [933] + │ │ │ │ │ │ │ ├── first join in paired joiner; continuation column: continuation:953 + │ │ │ │ │ │ │ ├── key: (1,932) + │ │ │ │ │ │ │ ├── fd: ()-->(3,30,31), (1)-->(2,5,8,10,13,15,17,20,22,23,26,27,36,37,134-136,139,140), (2)-->(1,5,8,10,13,15,17,20,22,23,26,27), (134)-->(135,136,139,140), (139)~~>(140), (140)~~>(139), (36)-->(37), (37)-->(36), (3)==(30), (30)==(3), (932)-->(933,953) │ │ │ │ │ │ │ ├── left-join (lookup pg_tablespace [as=t]) │ │ │ │ │ │ │ │ ├── columns: c.oid:1!null c.relname:2!null c.relnamespace:3!null c.relowner:5!null c.reltablespace:8!null c.reltuples:10!null c.relhasindex:13!null c.relpersistence:15!null c.relkind:17!null c.relhasoids:20!null c.relhasrules:22!null c.relhastriggers:23!null c.relacl:26 c.reloptions:27 n.oid:30!null n.nspname:31!null t.oid:36 spcname:37 ftrelid:134 ftserver:135 ftoptions:136 fs.oid:139 srvname:140 │ │ │ │ │ │ │ │ ├── key columns: [8] = [36] diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index a33e40858126..33ac80ae94a5 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -2375,14 +2375,14 @@ SELECT * FROM small LEFT JOIN abcd ON a=m AND c>n ---- left-join (lookup abcd) ├── columns: m:1 n:2 a:6 b:7 c:8 - ├── key columns: [9] = [9] + ├── key columns: [15] = [9] ├── lookup columns are key ├── second join in paired joiner ├── left-join (lookup abcd@abcd_a_b_idx) - │ ├── columns: m:1 n:2 a:6 b:7 abcd.rowid:9 continuation:12 - │ ├── key columns: [1] = [6] - │ ├── first join in paired joiner; continuation column: continuation:12 - │ ├── fd: (9)-->(6,7,12) + │ ├── columns: m:1 n:2 a:12 b:13 abcd.rowid:15 continuation:18 + │ ├── key columns: [1] = [12] + │ ├── first join in paired joiner; continuation column: continuation:18 + │ ├── fd: (15)-->(12,13,18) │ ├── scan small │ │ └── columns: m:1 n:2 │ └── filters (true) @@ -2640,14 +2640,14 @@ SELECT m, n FROM small WHERE EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c) ---- semi-join (lookup abcd) ├── columns: m:1 n:2 - ├── key columns: [9] = [9] + ├── key columns: [16] = [9] ├── lookup columns are key ├── second join in paired joiner ├── inner-join (lookup abcd@abcd_a_b_idx) - │ ├── columns: m:1!null n:2 a:6!null abcd.rowid:9!null continuation:13 - │ ├── key columns: [1] = [6] - │ ├── first join in paired joiner; continuation column: continuation:13 - │ ├── fd: (9)-->(6,13), (1)==(6), (6)==(1) + │ ├── columns: m:1!null n:2 a:13!null b:14 abcd.rowid:16!null continuation:19 + │ ├── key columns: [1] = [13] + │ ├── first join in paired joiner; continuation column: continuation:19 + │ ├── fd: (16)-->(13,14,19), (1)==(13), (13)==(1) │ ├── scan small │ │ └── columns: m:1 n:2 │ └── filters (true) @@ -2672,14 +2672,14 @@ SELECT m, n FROM small WHERE NOT EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c ---- anti-join (lookup abcd) ├── columns: m:1 n:2 - ├── key columns: [9] = [9] + ├── key columns: [16] = [9] ├── lookup columns are key ├── second join in paired joiner ├── left-join (lookup abcd@abcd_a_b_idx) - │ ├── columns: m:1 n:2 a:6 abcd.rowid:9 continuation:13 - │ ├── key columns: [1] = [6] - │ ├── first join in paired joiner; continuation column: continuation:13 - │ ├── fd: (9)-->(6,13) + │ ├── columns: m:1 n:2 a:13 b:14 abcd.rowid:16 continuation:19 + │ ├── key columns: [1] = [13] + │ ├── first join in paired joiner; continuation column: continuation:19 + │ ├── fd: (16)-->(13,14,19) │ ├── scan small │ │ └── columns: m:1 n:2 │ └── filters (true) @@ -3156,6 +3156,71 @@ left-join (lookup t59738_ab) │ └── d:2 > 5 [outer=(2), constraints=(/2: [/6 - ]; tight)] └── filters (true) +# Regression test for #81968. The index used for the first join in a paired left +# join should have different column IDs than the index used for the second join. +exec-ddl +CREATE TABLE items ( + id INT NOT NULL PRIMARY KEY, + chat_id INT NOT NULL, + author_id INT NOT NULL, + INDEX chat_id_idx (chat_id) +) +---- + +exec-ddl +CREATE TABLE views ( + chat_id INT NOT NULL, + user_id INT NOT NULL, + PRIMARY KEY (chat_id, user_id) +) +---- + +opt +SELECT (SELECT count(items.id) + FROM items + WHERE items.chat_id = views.chat_id + AND items.author_id != views.user_id) +FROM views +WHERE chat_id = 1 + AND user_id = 1; +---- +project + ├── columns: count:11!null + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(11) + ├── group-by (streaming) + │ ├── columns: count:10!null + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(10) + │ ├── left-join (lookup items) + │ │ ├── columns: views.chat_id:1!null user_id:2!null items.chat_id:6 author_id:7 + │ │ ├── key columns: [12] = [5] + │ │ ├── lookup columns are key + │ │ ├── second join in paired joiner + │ │ ├── fd: ()-->(1,2,6) + │ │ ├── left-join (lookup items@chat_id_idx) + │ │ │ ├── columns: views.chat_id:1!null user_id:2!null id:12 items.chat_id:13 continuation:17 + │ │ │ ├── key columns: [1] = [13] + │ │ │ ├── first join in paired joiner; continuation column: continuation:17 + │ │ │ ├── key: (12) + │ │ │ ├── fd: ()-->(1,2,13), (12)-->(17) + │ │ │ ├── scan views + │ │ │ │ ├── columns: views.chat_id:1!null user_id:2!null + │ │ │ │ ├── constraint: /1/2: [/1/1 - /1/1] + │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ ├── key: () + │ │ │ │ └── fd: ()-->(1,2) + │ │ │ └── filters (true) + │ │ └── filters + │ │ └── author_id:7 != user_id:2 [outer=(2,7), constraints=(/2: (/NULL - ]; /7: (/NULL - ])] + │ └── aggregations + │ └── count [as=count:10, outer=(6)] + │ └── items.chat_id:6 + └── projections + └── count:10 [as=count:11, outer=(10)] + # -------------------------------------------------- # GenerateLookupJoinsWithFilter # -------------------------------------------------- @@ -3306,17 +3371,17 @@ SELECT * FROM small LEFT JOIN abcd ON a=m AND c>n AND b>1 ---- left-join (lookup abcd) ├── columns: m:1 n:2 a:6 b:7 c:8 - ├── key columns: [9] = [9] + ├── key columns: [15] = [9] ├── lookup columns are key ├── second join in paired joiner ├── left-join (lookup abcd@abcd_a_b_idx) - │ ├── columns: m:1 n:2 a:6 b:7 abcd.rowid:9 continuation:12 + │ ├── columns: m:1 n:2 a:12 b:13 abcd.rowid:15 continuation:18 │ ├── lookup expression │ │ └── filters - │ │ ├── a:6 = m:1 [outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)] - │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] - │ ├── first join in paired joiner; continuation column: continuation:12 - │ ├── fd: (9)-->(6,7,12) + │ │ ├── a:12 = m:1 [outer=(1,12), constraints=(/1: (/NULL - ]; /12: (/NULL - ]), fd=(1)==(12), (12)==(1)] + │ │ └── b:13 > 1 [outer=(13), constraints=(/13: [/2 - ]; tight)] + │ ├── first join in paired joiner; continuation column: continuation:18 + │ ├── fd: (15)-->(12,13,18) │ ├── scan small │ │ └── columns: m:1 n:2 │ └── filters (true) @@ -3860,18 +3925,18 @@ SELECT m, n FROM small WHERE EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c AND ---- semi-join (lookup abcd) ├── columns: m:1 n:2 - ├── key columns: [9] = [9] + ├── key columns: [16] = [9] ├── lookup columns are key ├── second join in paired joiner ├── inner-join (lookup abcd@abcd_a_b_idx) - │ ├── columns: m:1!null n:2 a:6!null b:7!null abcd.rowid:9!null continuation:13 - │ ├── key columns: [1] = [6] - │ ├── first join in paired joiner; continuation column: continuation:13 - │ ├── fd: (9)-->(6,7,13), (1)==(6), (6)==(1) + │ ├── columns: m:1!null n:2 a:13!null b:14!null abcd.rowid:16!null continuation:19 + │ ├── key columns: [1] = [13] + │ ├── first join in paired joiner; continuation column: continuation:19 + │ ├── fd: (16)-->(13,14,19), (1)==(13), (13)==(1) │ ├── scan small │ │ └── columns: m:1 n:2 │ └── filters - │ └── a:6 > b:7 [outer=(6,7), constraints=(/6: (/NULL - ]; /7: (/NULL - ])] + │ └── a:13 > b:14 [outer=(13,14), constraints=(/13: (/NULL - ]; /14: (/NULL - ])] └── filters └── n:2 = c:8 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ]), fd=(2)==(8), (8)==(2)] @@ -3894,18 +3959,18 @@ SELECT m, n FROM small WHERE NOT EXISTS(SELECT 1 FROM abcd WHERE m = a AND n = c ---- anti-join (lookup abcd) ├── columns: m:1 n:2 - ├── key columns: [9] = [9] + ├── key columns: [16] = [9] ├── lookup columns are key ├── second join in paired joiner ├── left-join (lookup abcd@abcd_a_b_idx) - │ ├── columns: m:1 n:2 a:6 b:7 abcd.rowid:9 continuation:13 - │ ├── key columns: [1] = [6] - │ ├── first join in paired joiner; continuation column: continuation:13 - │ ├── fd: (9)-->(6,7,13) + │ ├── columns: m:1 n:2 a:13 b:14 abcd.rowid:16 continuation:19 + │ ├── key columns: [1] = [13] + │ ├── first join in paired joiner; continuation column: continuation:19 + │ ├── fd: (16)-->(13,14,19) │ ├── scan small │ │ └── columns: m:1 n:2 │ └── filters - │ └── a:6 > b:7 [outer=(6,7), constraints=(/6: (/NULL - ]; /7: (/NULL - ])] + │ └── a:13 > b:14 [outer=(13,14), constraints=(/13: (/NULL - ]; /14: (/NULL - ])] └── filters └── n:2 = c:8 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ]), fd=(2)==(8), (8)==(2)] @@ -4160,14 +4225,14 @@ project ├── columns: m:1 └── semi-join (lookup partial_tab) ├── columns: m:1 n:2 - ├── key columns: [6] = [6] + ├── key columns: [12] = [6] ├── lookup columns are key ├── second join in paired joiner ├── inner-join (lookup partial_tab@partial_idx,partial) - │ ├── columns: m:1 n:2!null k:6!null i:7!null continuation:12 - │ ├── key columns: [2] = [7] - │ ├── first join in paired joiner; continuation column: continuation:12 - │ ├── fd: (6)-->(7,12), (2)==(7), (7)==(2) + │ ├── columns: m:1 n:2!null k:12!null i:13!null continuation:17 + │ ├── key columns: [2] = [13] + │ ├── first join in paired joiner; continuation column: continuation:17 + │ ├── fd: (12)-->(13,17), (2)==(13), (13)==(2) │ ├── scan small │ │ └── columns: m:1 n:2 │ └── filters (true) @@ -4197,14 +4262,14 @@ project ├── columns: m:1 └── anti-join (lookup partial_tab) ├── columns: m:1 n:2 - ├── key columns: [6] = [6] + ├── key columns: [12] = [6] ├── lookup columns are key ├── second join in paired joiner ├── left-join (lookup partial_tab@partial_idx,partial) - │ ├── columns: m:1 n:2 k:6 i:7 continuation:12 - │ ├── key columns: [2] = [7] - │ ├── first join in paired joiner; continuation column: continuation:12 - │ ├── fd: (6)-->(7,12) + │ ├── columns: m:1 n:2 k:12 i:13 continuation:17 + │ ├── key columns: [2] = [13] + │ ├── first join in paired joiner; continuation column: continuation:17 + │ ├── fd: (12)-->(13,17) │ ├── scan small │ │ └── columns: m:1 n:2 │ └── filters (true) @@ -5455,16 +5520,16 @@ project ├── immutable └── left-join (lookup virt) ├── columns: m:1 j:8 v1:9 - ├── key columns: [6] = [6] + ├── key columns: [16] = [6] ├── lookup columns are key ├── second join in paired joiner ├── immutable ├── left-join (lookup virt@v1_storing_i) - │ ├── columns: m:1 k:6 v1:9 continuation:16 + │ ├── columns: m:1 k:16 i:17 v1:19 continuation:26 │ ├── flags: force lookup join (into right side) - │ ├── key columns: [1] = [9] - │ ├── first join in paired joiner; continuation column: continuation:16 - │ ├── fd: (6)-->(9,16) + │ ├── key columns: [1] = [19] + │ ├── first join in paired joiner; continuation column: continuation:26 + │ ├── fd: (16)-->(17,19,26), (17)~~>(19) │ ├── scan small │ │ └── columns: m:1 │ └── filters (true) @@ -6446,17 +6511,17 @@ project ├── fd: (6)-->(8) └── left-join (lookup virt) ├── columns: m:1 k:6 j:8 v3:11 - ├── key columns: [6] = [6] + ├── key columns: [16] = [6] ├── lookup columns are key ├── second join in paired joiner ├── immutable ├── fd: (6)-->(8,11) ├── left-join (lookup virt@v3_partial,partial) - │ ├── columns: m:1 k:6 v3:11 continuation:16 + │ ├── columns: m:1 k:16 i:17 v3:21 continuation:26 │ ├── flags: force lookup join (into right side) - │ ├── key columns: [1] = [11] - │ ├── first join in paired joiner; continuation column: continuation:16 - │ ├── fd: (6)-->(11,16) + │ ├── key columns: [1] = [21] + │ ├── first join in paired joiner; continuation column: continuation:26 + │ ├── fd: (16)-->(17,21,26) │ ├── scan small │ │ └── columns: m:1 │ └── filters (true) @@ -6582,17 +6647,17 @@ project ├── fd: (6)-->(7,8) └── left-join (lookup virt) ├── columns: m:1 k:6 i:7 j:8 v3:11 - ├── key columns: [6] = [6] + ├── key columns: [16] = [6] ├── lookup columns are key ├── second join in paired joiner ├── immutable ├── fd: (6)-->(7,8), (7,8)-->(11) ├── left-join (lookup virt@v3_partial,partial) - │ ├── columns: m:1 k:6 i:7 v3:11 continuation:16 + │ ├── columns: m:1 k:16 i:17 v3:21 continuation:26 │ ├── flags: force lookup join (into right side) - │ ├── key columns: [1] = [11] - │ ├── first join in paired joiner; continuation column: continuation:16 - │ ├── fd: (6)-->(7,11,16) + │ ├── key columns: [1] = [21] + │ ├── first join in paired joiner; continuation column: continuation:26 + │ ├── fd: (16)-->(17,21,26) │ ├── scan small │ │ └── columns: m:1 │ └── filters (true)