From 6a18630c53f6f79f4c3c62244f30e1a5e7a815bf Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 31 Jan 2023 20:57:55 +0800 Subject: [PATCH] planner: forbid using IndexFullScan on multi-valued indexes (#40907) ref pingcap/tidb#40191 --- planner/core/indexmerge_path.go | 3 +- planner/core/indexmerge_path_test.go | 48 +++++++++++--- .../core/testdata/index_merge_suite_in.json | 6 +- .../core/testdata/index_merge_suite_out.json | 66 ++++++++++++------- 4 files changed, 90 insertions(+), 33 deletions(-) diff --git a/planner/core/indexmerge_path.go b/planner/core/indexmerge_path.go index aad7f412441d3..27500e1d8d816 100644 --- a/planner/core/indexmerge_path.go +++ b/planner/core/indexmerge_path.go @@ -587,8 +587,7 @@ func (ds *DataSource) generateIndexMerge4MVIndex(normalPathCnt int, filters []ex } accessFilters, remainingFilters := ds.collectFilters4MVIndex(filters, idxCols) - if len(accessFilters) == 0 && // cannot use any filter on this MVIndex - !ds.possibleAccessPaths[idx].Forced { // whether this index is forced by use-index hint + if len(accessFilters) == 0 { // cannot use any filter on this MVIndex continue } diff --git a/planner/core/indexmerge_path_test.go b/planner/core/indexmerge_path_test.go index 6c2ab7a8ce087..b1487249b0902 100644 --- a/planner/core/indexmerge_path_test.go +++ b/planner/core/indexmerge_path_test.go @@ -15,6 +15,7 @@ package core_test import ( + "context" "fmt" "math/rand" "strings" @@ -224,6 +225,7 @@ func TestEnforceMVIndex(t *testing.T) { var output []struct { SQL string Plan []string + Err string } planSuiteData := core.GetIndexMergeSuiteData() planSuiteData.LoadTestCases(t, &input, &output) @@ -232,11 +234,21 @@ func TestEnforceMVIndex(t *testing.T) { testdata.OnRecord(func() { output[i].SQL = query }) - result := tk.MustQuery("explain format = 'brief' " + query) - testdata.OnRecord(func() { - output[i].Plan = testdata.ConvertRowsToStrings(result.Rows()) - }) - result.Check(testkit.Rows(output[i].Plan...)) + rs, err := tk.Exec("explain format = 'brief' " + query) + if err != nil { + testdata.OnRecord(func() { + output[i].Err = err.Error() + output[i].Plan = nil + }) + require.Equal(t, output[i].Err, err.Error()) + } else { + result := tk.ResultSetToResultWithCtx(context.Background(), rs, "") + testdata.OnRecord(func() { + output[i].Err = "" + output[i].Plan = testdata.ConvertRowsToStrings(result.Rows()) + }) + result.Check(testkit.Rows(output[i].Plan...)) + } } } @@ -270,6 +282,26 @@ func TestMVIndexInvisible(t *testing.T) { ` └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo`)) } +func TestMVIndexFullScan(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + tk.MustExec(`create table t(j json, index kj((cast(j as signed array))))`) + tk.MustExec(`insert into t values ('[1]')`) + tk.MustExec(`insert into t values ('[1, 2]')`) + tk.MustExec(`insert into t values ('[]')`) + tk.MustExec(`insert into t values (NULL)`) + + tk.MustQuery(`select /*+ use_index_merge(t, kj) */ count(*) from t`).Check(testkit.Rows("4")) + tk.MustQuery(`select /*+ use_index_merge(t, kj) */ count(*) from t where (1 member of (j))`).Check(testkit.Rows("2")) + tk.MustQuery(`select /*+ use_index_merge(t, kj) */ count(*) from t where json_contains((j), '[1]')`).Check(testkit.Rows("2")) + tk.MustQuery(`select /*+ use_index_merge(t, kj) */ count(*) from t where json_overlaps((j), '[1]')`).Check(testkit.Rows("2")) + + // Forbid IndexMerge+IndexFullScan since IndexFullScan on MVIndex cannot read all rows some cases. + tk.MustGetErrMsg(`select /*+ use_index(t, kj) */ count(*) from t`, "[planner:1815]Internal : Can't find a proper physical plan for this query") +} + func TestMVIndexRandom(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -284,8 +316,8 @@ func TestMVIndexRandom(t *testing.T) { {"unsigned", randMVIndexValOpts{"unsigned", 0, 3}, randMVIndexValOpts{"unsigned", 0, 3}}, // unsigned-index + unsigned-values {"char(3)", randMVIndexValOpts{"string", 3, 3}, randMVIndexValOpts{"string", 3, 3}}, {"char(3)", randMVIndexValOpts{"string", 3, 3}, randMVIndexValOpts{"string", 1, 3}}, - //{"char(3)", randMVIndexValOpts{"string", 3, 3}, randMVIndexValOpts{"string", 5, 3}}, - //{"date", randMVIndexValOpts{"date", 0, 3}, randMVIndexValOpts{"date", 0, 3}}, + {"char(3)", randMVIndexValOpts{"string", 3, 3}, randMVIndexValOpts{"string", 5, 3}}, + {"date", randMVIndexValOpts{"date", 0, 3}, randMVIndexValOpts{"date", 0, 3}}, } { tk.MustExec("drop table if exists t") tk.MustExec(fmt.Sprintf(`create table t(a int, j json, index kj((cast(j as %v array))))`, testCase.indexType)) @@ -306,7 +338,7 @@ func TestMVIndexRandom(t *testing.T) { for i := 0; i < nQueries; i++ { conds := randMVIndexConds(rand.Intn(3)+1, testCase.queryValsOpts) r1 := tk.MustQuery("select /*+ ignore_index(t, kj) */ * from t where " + conds).Sort() - tk.MustQuery("select /*+ use_index(t, kj) */ * from t where " + conds).Sort().Check(r1.Rows()) + tk.MustQuery("select /*+ use_index_merge(t, kj) */ * from t where " + conds).Sort().Check(r1.Rows()) } } } diff --git a/planner/core/testdata/index_merge_suite_in.json b/planner/core/testdata/index_merge_suite_in.json index c64b00b983b78..d660364305397 100644 --- a/planner/core/testdata/index_merge_suite_in.json +++ b/planner/core/testdata/index_merge_suite_in.json @@ -7,7 +7,11 @@ "select /*+ use_index(t, kj) */ * from t where a<10", "select /*+ use_index(t, kj) */ * from t where (1 member of (j))", "select /*+ use_index(t, kj) */ * from t where (1 member of (j)) and a=10", - "select /*+ use_index(t, kj) */ * from t where (1 member of (j)) or a=10" + "select /*+ use_index(t, kj) */ * from t where (1 member of (j)) or a=10", + "select /*+ use_index_merge(t, kj) */ * from t", + "select /*+ use_index_merge(t, kj) */ a from t", + "select /*+ use_index_merge(t, kj) */ * from t where a<10", + "select /*+ use_index_merge(t, kj) */ * from t where (1 member of (j)) or a=10" ] }, { diff --git a/planner/core/testdata/index_merge_suite_out.json b/planner/core/testdata/index_merge_suite_out.json index a28164f5787fe..2c66948aca057 100644 --- a/planner/core/testdata/index_merge_suite_out.json +++ b/planner/core/testdata/index_merge_suite_out.json @@ -4,28 +4,18 @@ "Cases": [ { "SQL": "select /*+ use_index(t, kj) */ * from t", - "Plan": [ - "IndexMerge 10000.00 root type: union", - "├─IndexFullScan(Build) 10000.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) keep order:false, stats:pseudo", - "└─TableRowIDScan(Probe) 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" - ] + "Plan": null, + "Err": "[planner:1815]Internal : Can't find a proper physical plan for this query" }, { "SQL": "select /*+ use_index(t, kj) */ a from t", - "Plan": [ - "IndexMerge 10000.00 root type: union", - "├─IndexFullScan(Build) 10000.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) keep order:false, stats:pseudo", - "└─TableRowIDScan(Probe) 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" - ] + "Plan": null, + "Err": "[planner:1815]Internal : Can't find a proper physical plan for this query" }, { "SQL": "select /*+ use_index(t, kj) */ * from t where a<10", - "Plan": [ - "IndexMerge 3323.33 root type: union", - "├─IndexFullScan(Build) 10000.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) keep order:false, stats:pseudo", - "└─Selection(Probe) 3323.33 cop[tikv] lt(test.t.a, 10)", - " └─TableRowIDScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" - ] + "Plan": null, + "Err": "[planner:1815]Internal : Can't find a proper physical plan for this query" }, { "SQL": "select /*+ use_index(t, kj) */ * from t where (1 member of (j))", @@ -34,7 +24,8 @@ "└─IndexMerge 10.00 root type: union", " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo", " └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo" - ] + ], + "Err": "" }, { "SQL": "select /*+ use_index(t, kj) */ * from t where (1 member of (j)) and a=10", @@ -44,16 +35,47 @@ " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo", " └─Selection(Probe) 0.01 cop[tikv] eq(test.t.a, 10)", " └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo" - ] + ], + "Err": "" }, { "SQL": "select /*+ use_index(t, kj) */ * from t where (1 member of (j)) or a=10", + "Plan": null, + "Err": "[planner:1815]Internal : Can't find a proper physical plan for this query" + }, + { + "SQL": "select /*+ use_index_merge(t, kj) */ * from t", + "Plan": [ + "TableReader 10000.00 root data:TableFullScan", + "└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Err": "" + }, + { + "SQL": "select /*+ use_index_merge(t, kj) */ a from t", + "Plan": [ + "TableReader 10000.00 root data:TableFullScan", + "└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Err": "" + }, + { + "SQL": "select /*+ use_index_merge(t, kj) */ * from t where a<10", + "Plan": [ + "TableReader 3323.33 root data:Selection", + "└─Selection 3323.33 cop[tikv] lt(test.t.a, 10)", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Err": "" + }, + { + "SQL": "select /*+ use_index_merge(t, kj) */ * from t where (1 member of (j)) or a=10", "Plan": [ "Selection 8000.00 root or(json_memberof(cast(1, json BINARY), test.t.j), eq(test.t.a, 10))", - "└─IndexMerge 10000.00 root type: union", - " ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) keep order:false, stats:pseudo", - " └─TableRowIDScan(Probe) 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" - ] + "└─TableReader 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Err": "" } ] },