From 721feb13980097af9c9f15b11f6e0501476405d7 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 26 Jul 2023 16:21:05 +0800 Subject: [PATCH 1/2] This is an automated cherry-pick of #45565 Signed-off-by: ti-chi-bot --- planner/core/main_test.go | 9 +++ planner/core/rule_join_reorder.go | 7 ++- planner/core/rule_join_reorder_test.go | 41 +++++++++----- .../core/testdata/join_reorder_suite_in.json | 13 +++++ .../core/testdata/join_reorder_suite_out.json | 56 +++++++++++++++++++ 5 files changed, 110 insertions(+), 16 deletions(-) diff --git a/planner/core/main_test.go b/planner/core/main_test.go index 230cc3f4bdd2d..2e7e91de70be6 100644 --- a/planner/core/main_test.go +++ b/planner/core/main_test.go @@ -35,6 +35,7 @@ func TestMain(m *testing.M) { testDataMap.LoadTestSuiteData("testdata", "integration_partition_suite") testDataMap.LoadTestSuiteData("testdata", "index_merge_suite") +<<<<<<< HEAD testDataMap.LoadTestSuiteData("testdata", "plan_normalized_suite") testDataMap.LoadTestSuiteData("testdata", "stats_suite") testDataMap.LoadTestSuiteData("testdata", "ordered_result_mode_suite") @@ -51,6 +52,10 @@ func TestMain(m *testing.M) { testDataMap.LoadTestSuiteData("testdata", "flat_plan_suite") testDataMap.LoadTestSuiteData("testdata", "binary_plan_suite") testDataMap.LoadTestSuiteData("testdata", "json_plan_suite") +======= + testDataMap.LoadTestSuiteData("testdata", "runtime_filter_generator_suite") + testDataMap.LoadTestSuiteData("testdata", "join_reorder_suite") +>>>>>>> dc04e2ca077 (planner: clean the one side filters for outer join reorder which is not done in #44409 (#45565)) indexMergeSuiteData = testDataMap["index_merge_suite"] planSuiteUnexportedData = testDataMap["plan_suite_unexported"] @@ -139,3 +144,7 @@ func GetIndexMergeSuiteData() testdata.TestData { func GetJSONPlanSuiteData() testdata.TestData { return testDataMap["json_plan_suite"] } + +func GetJoinReorderData() testdata.TestData { + return testDataMap["join_reorder_suite"] +} diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index ac37494a6302c..c5a721f4659d0 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -426,12 +426,17 @@ func (s *baseSingleGroupJoinOrderSolver) makeJoin(leftPlan, rightPlan LogicalPla remainOtherConds, otherConds = expression.FilterOutInPlace(remainOtherConds, func(expr expression.Expression) bool { return expression.ExprFromSchema(expr, mergedSchema) }) - if (joinType.JoinType == LeftOuterJoin || joinType.JoinType == RightOuterJoin || joinType.JoinType == LeftOuterSemiJoin || joinType.JoinType == AntiLeftOuterSemiJoin) && len(otherConds) > 0 { + + if joinType.JoinType == LeftOuterJoin || joinType.JoinType == RightOuterJoin || joinType.JoinType == LeftOuterSemiJoin || joinType.JoinType == AntiLeftOuterSemiJoin { // the original outer join's other conditions has been bound to the outer join Edge, // these remained other condition here shouldn't be appended to it because on-mismatch // logic will produce more append-null rows which is banned in original semantic. remainOtherConds = append(remainOtherConds, otherConds...) // nozero + remainOtherConds = append(remainOtherConds, leftConds...) // nozero + remainOtherConds = append(remainOtherConds, rightConds...) // nozero otherConds = otherConds[:0] + leftConds = leftConds[:0] + rightConds = rightConds[:0] } if len(joinType.outerBindCondition) > 0 { remainOBOtherConds := make([]expression.Expression, len(joinType.outerBindCondition)) diff --git a/planner/core/rule_join_reorder_test.go b/planner/core/rule_join_reorder_test.go index 2e28001a08544..f2f2b81e6f11f 100644 --- a/planner/core/rule_join_reorder_test.go +++ b/planner/core/rule_join_reorder_test.go @@ -17,10 +17,14 @@ package core_test import ( "testing" +<<<<<<< HEAD "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/parser/model" plannercore "github.com/pingcap/tidb/planner/core" +======= + "github.com/pingcap/tidb/planner/core" +>>>>>>> dc04e2ca077 (planner: clean the one side filters for outer join reorder which is not done in #44409 (#45565)) "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/testkit/testdata" "github.com/stretchr/testify/require" @@ -366,21 +370,28 @@ func TestAdditionOtherConditionsRemained4OuterJoin(t *testing.T) { tk.MustExec("INSERT INTO queries_program(`id`, `identifier_id`) values(8, 13), (9, 14);") tk.MustExec("INSERT INTO queries_channel(`id`, `identifier_id`) values(5, 13);") - tk.MustQuery("SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC;").Check(testkit.Rows("" + - "13 i1")) - tk.MustQuery("SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` RIGHT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC;").Check(testkit.Rows("" + - "13 i1")) - tk.MustQuery("explain format = 'brief' SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC;").Check(testkit.Rows(""+ - "Sort 2.50 root test.queries_identifier.id", - "└─Projection 2.50 root test.queries_identifier.id, test.queries_identifier.name", - " └─Selection 2.50 root or(and(eq(test.queries_channel.id, 5), eq(test.queries_program.id, 9)), eq(test.queries_program.id, 8))", - " └─IndexJoin 3.12 root left outer join, inner:IndexReader, outer key:test.queries_identifier.id, inner key:test.queries_channel.identifier_id, equal cond:eq(test.queries_identifier.id, test.queries_channel.identifier_id)", - " ├─IndexHashJoin(Build) 2.50 root inner join, inner:TableReader, outer key:test.queries_program.identifier_id, inner key:test.queries_identifier.id, equal cond:eq(test.queries_program.identifier_id, test.queries_identifier.id)", - " │ ├─Batch_Point_Get(Build) 2.00 root table:queries_program handle:[8 9], keep order:false, desc:false", - " │ └─TableReader(Probe) 2.00 root data:TableRangeScan", - " │ └─TableRangeScan 2.00 cop[tikv] table:queries_identifier range: decided by [test.queries_program.identifier_id], keep order:false, stats:pseudo", - " └─IndexReader(Probe) 2.50 root index:IndexRangeScan", - " └─IndexRangeScan 2.50 cop[tikv] table:queries_channel, index:identifier_id(identifier_id) range: decided by [eq(test.queries_channel.identifier_id, test.queries_identifier.id)], keep order:false, stats:pseudo")) + tk.MustExec("create table t(a int)") + tk.MustExec("create table t1(a int, b int)") + tk.MustExec("create table t2(a int, b int, c int)") + tk.MustExec("create table t3(a int, b int)") + tk.MustExec("create table t4(a int, b int)") + + testData := core.GetJoinReorderData() + var ( + input []string + output []struct { + SQL string + Output []string + } + ) + testData.LoadTestCases(t, &input, &output) + for i, sql := range input { + testdata.OnRecord(func() { + output[i].SQL = sql + output[i].Output = testdata.ConvertRowsToStrings(tk.MustQuery(sql).Rows()) + }) + tk.MustQuery(sql).Check(testkit.Rows(output[i].Output...)) + } } func TestOuterJoinWIthEqCondCrossInnerJoin(t *testing.T) { diff --git a/planner/core/testdata/join_reorder_suite_in.json b/planner/core/testdata/join_reorder_suite_in.json index 5ef0edb64351b..3e2dd82a77d5a 100644 --- a/planner/core/testdata/join_reorder_suite_in.json +++ b/planner/core/testdata/join_reorder_suite_in.json @@ -1,5 +1,6 @@ [ { +<<<<<<< HEAD "name": "TestStraightJoinHint", "cases": [ "select /*+ straight_join() */ * from t, t1, t2, t3 where t.a = t1.a and t1.b=t2.b;", @@ -524,3 +525,15 @@ ] } ] +======= + "name": "TestAdditionOtherConditionsRemained4OuterJoin", + "cases": [ + "SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC", + "SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` RIGHT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC", + "explain format = 'brief' SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC;", + // The where clause should be a Selection out of joins. + "explain format='brief' select * from t left join t1 on t.a=t1.a inner join t2 on t.a=t2.a and t2.c = 100 left join t3 on t2.a=t3.a and t3.b > 1 left join t4 on t2.a = t4.a where (t2.b > 100 or t.a > 10 or t1.b < 10)" + ] + } +] +>>>>>>> dc04e2ca077 (planner: clean the one side filters for outer join reorder which is not done in #44409 (#45565)) diff --git a/planner/core/testdata/join_reorder_suite_out.json b/planner/core/testdata/join_reorder_suite_out.json index e2a6c562726d4..940036343f24c 100644 --- a/planner/core/testdata/join_reorder_suite_out.json +++ b/planner/core/testdata/join_reorder_suite_out.json @@ -1,5 +1,6 @@ [ { +<<<<<<< HEAD "Name": "TestStraightJoinHint", "Cases": [ { @@ -9656,6 +9657,61 @@ "Warning": [ "Warning 1815 There are no matching table names for (t2) in optimizer hint /*+ LEADING(t3, t2) */. Maybe you can use the table alias name", "Warning 1815 leading hint is inapplicable, check if the leading hint table is valid" +======= + "Name": "TestAdditionOtherConditionsRemained4OuterJoin", + "Cases": [ + { + "SQL": "SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC", + "Output": [ + "13 i1" + ] + }, + { + "SQL": "SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` RIGHT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC", + "Output": [ + "13 i1" + ] + }, + { + "SQL": "explain format = 'brief' SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC;", + "Output": [ + "Sort 2.50 root test.queries_identifier.id", + "└─Projection 2.50 root test.queries_identifier.id, test.queries_identifier.name", + " └─Selection 2.50 root or(and(eq(test.queries_channel.id, 5), eq(test.queries_program.id, 9)), eq(test.queries_program.id, 8))", + " └─IndexJoin 3.12 root left outer join, inner:IndexReader, outer key:test.queries_identifier.id, inner key:test.queries_channel.identifier_id, equal cond:eq(test.queries_identifier.id, test.queries_channel.identifier_id)", + " ├─IndexHashJoin(Build) 2.50 root inner join, inner:TableReader, outer key:test.queries_program.identifier_id, inner key:test.queries_identifier.id, equal cond:eq(test.queries_program.identifier_id, test.queries_identifier.id)", + " │ ├─Batch_Point_Get(Build) 2.00 root table:queries_program handle:[8 9], keep order:false, desc:false", + " │ └─TableReader(Probe) 2.00 root data:TableRangeScan", + " │ └─TableRangeScan 2.00 cop[tikv] table:queries_identifier range: decided by [test.queries_program.identifier_id], keep order:false, stats:pseudo", + " └─IndexReader(Probe) 2.50 root index:IndexRangeScan", + " └─IndexRangeScan 2.50 cop[tikv] table:queries_channel, index:identifier_id(identifier_id) range: decided by [eq(test.queries_channel.identifier_id, test.queries_identifier.id)], keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format='brief' select * from t left join t1 on t.a=t1.a inner join t2 on t.a=t2.a and t2.c = 100 left join t3 on t2.a=t3.a and t3.b > 1 left join t4 on t2.a = t4.a where (t2.b > 100 or t.a > 10 or t1.b < 10)", + "Output": [ + "Projection 19.51 root test.t.a, test.t1.a, test.t1.b, test.t2.a, test.t2.b, test.t2.c, test.t3.a, test.t3.b, test.t4.a, test.t4.b", + "└─Selection 19.51 root or(gt(test.t2.b, 100), or(gt(test.t.a, 10), lt(test.t1.b, 10))), or(gt(test.t2.b, 100), or(gt(test.t2.a, 10), lt(test.t1.b, 10)))", + " └─HashJoin 24.39 root left outer join, equal:[eq(test.t2.a, test.t4.a)]", + " ├─HashJoin(Build) 19.51 root left outer join, equal:[eq(test.t.a, test.t1.a)]", + " │ ├─HashJoin(Build) 15.61 root inner join, equal:[eq(test.t2.a, test.t.a)]", + " │ │ ├─HashJoin(Build) 12.49 root left outer join, equal:[eq(test.t2.a, test.t3.a)]", + " │ │ │ ├─TableReader(Build) 9.99 root data:Selection", + " │ │ │ │ └─Selection 9.99 cop[tikv] eq(test.t2.c, 100), not(isnull(test.t2.a))", + " │ │ │ │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + " │ │ │ └─TableReader(Probe) 3330.00 root data:Selection", + " │ │ │ └─Selection 3330.00 cop[tikv] gt(test.t3.b, 1), not(isnull(test.t3.a))", + " │ │ │ └─TableFullScan 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo", + " │ │ └─TableReader(Probe) 9990.00 root data:Selection", + " │ │ └─Selection 9990.00 cop[tikv] not(isnull(test.t.a))", + " │ │ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + " │ └─TableReader(Probe) 9990.00 root data:Selection", + " │ └─Selection 9990.00 cop[tikv] not(isnull(test.t1.a))", + " │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", + " └─TableReader(Probe) 9990.00 root data:Selection", + " └─Selection 9990.00 cop[tikv] not(isnull(test.t4.a))", + " └─TableFullScan 10000.00 cop[tikv] table:t4 keep order:false, stats:pseudo" +>>>>>>> dc04e2ca077 (planner: clean the one side filters for outer join reorder which is not done in #44409 (#45565)) ] } ] From 64b0a942396c7f94375f385d6b45a1b9b8eacef8 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 16 Aug 2023 10:16:00 +0800 Subject: [PATCH 2/2] fix conflict --- planner/core/main_test.go | 4 ---- planner/core/rule_join_reorder_test.go | 6 +----- planner/core/testdata/join_reorder_suite_in.json | 7 ++----- planner/core/testdata/join_reorder_suite_out.json | 8 +++++--- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/planner/core/main_test.go b/planner/core/main_test.go index 2e7e91de70be6..b3ad18514e9f2 100644 --- a/planner/core/main_test.go +++ b/planner/core/main_test.go @@ -35,7 +35,6 @@ func TestMain(m *testing.M) { testDataMap.LoadTestSuiteData("testdata", "integration_partition_suite") testDataMap.LoadTestSuiteData("testdata", "index_merge_suite") -<<<<<<< HEAD testDataMap.LoadTestSuiteData("testdata", "plan_normalized_suite") testDataMap.LoadTestSuiteData("testdata", "stats_suite") testDataMap.LoadTestSuiteData("testdata", "ordered_result_mode_suite") @@ -52,10 +51,7 @@ func TestMain(m *testing.M) { testDataMap.LoadTestSuiteData("testdata", "flat_plan_suite") testDataMap.LoadTestSuiteData("testdata", "binary_plan_suite") testDataMap.LoadTestSuiteData("testdata", "json_plan_suite") -======= - testDataMap.LoadTestSuiteData("testdata", "runtime_filter_generator_suite") testDataMap.LoadTestSuiteData("testdata", "join_reorder_suite") ->>>>>>> dc04e2ca077 (planner: clean the one side filters for outer join reorder which is not done in #44409 (#45565)) indexMergeSuiteData = testDataMap["index_merge_suite"] planSuiteUnexportedData = testDataMap["plan_suite_unexported"] diff --git a/planner/core/rule_join_reorder_test.go b/planner/core/rule_join_reorder_test.go index f2f2b81e6f11f..2acce89698255 100644 --- a/planner/core/rule_join_reorder_test.go +++ b/planner/core/rule_join_reorder_test.go @@ -17,14 +17,10 @@ package core_test import ( "testing" -<<<<<<< HEAD "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/parser/model" plannercore "github.com/pingcap/tidb/planner/core" -======= - "github.com/pingcap/tidb/planner/core" ->>>>>>> dc04e2ca077 (planner: clean the one side filters for outer join reorder which is not done in #44409 (#45565)) "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/testkit/testdata" "github.com/stretchr/testify/require" @@ -376,7 +372,7 @@ func TestAdditionOtherConditionsRemained4OuterJoin(t *testing.T) { tk.MustExec("create table t3(a int, b int)") tk.MustExec("create table t4(a int, b int)") - testData := core.GetJoinReorderData() + testData := plannercore.GetJoinReorderData() var ( input []string output []struct { diff --git a/planner/core/testdata/join_reorder_suite_in.json b/planner/core/testdata/join_reorder_suite_in.json index 3e2dd82a77d5a..232b0740d1eaf 100644 --- a/planner/core/testdata/join_reorder_suite_in.json +++ b/planner/core/testdata/join_reorder_suite_in.json @@ -1,6 +1,5 @@ [ { -<<<<<<< HEAD "name": "TestStraightJoinHint", "cases": [ "select /*+ straight_join() */ * from t, t1, t2, t3 where t.a = t1.a and t1.b=t2.b;", @@ -523,9 +522,8 @@ "select /*+ leading(t2@sel_2, t1) */ t1.a, (select min(t2.a) from t2) from t1 left join t3 on t1.a = t3.a;", "select /*+ leading(t3, t2@sel_2) */ t1.a, (select min(t2.a) from t2) from t1 right join t3 on t1.a = t3.a;" ] - } -] -======= + }, + { "name": "TestAdditionOtherConditionsRemained4OuterJoin", "cases": [ "SELECT `queries_identifier`.`id`, `queries_identifier`.`name` FROM `queries_identifier` LEFT OUTER JOIN `queries_channel` ON (`queries_identifier`.`id` = `queries_channel`.`identifier_id`) INNER JOIN `queries_program` ON (`queries_identifier`.`id` = `queries_program`.`identifier_id`) WHERE ((`queries_channel`.`id` = 5 AND `queries_program`.`id` = 9) OR `queries_program`.`id` = 8) ORDER BY `queries_identifier`.`id` ASC", @@ -536,4 +534,3 @@ ] } ] ->>>>>>> dc04e2ca077 (planner: clean the one side filters for outer join reorder which is not done in #44409 (#45565)) diff --git a/planner/core/testdata/join_reorder_suite_out.json b/planner/core/testdata/join_reorder_suite_out.json index 940036343f24c..ab24533395990 100644 --- a/planner/core/testdata/join_reorder_suite_out.json +++ b/planner/core/testdata/join_reorder_suite_out.json @@ -1,6 +1,5 @@ [ { -<<<<<<< HEAD "Name": "TestStraightJoinHint", "Cases": [ { @@ -9657,7 +9656,11 @@ "Warning": [ "Warning 1815 There are no matching table names for (t2) in optimizer hint /*+ LEADING(t3, t2) */. Maybe you can use the table alias name", "Warning 1815 leading hint is inapplicable, check if the leading hint table is valid" -======= + ] + } + ] + }, + { "Name": "TestAdditionOtherConditionsRemained4OuterJoin", "Cases": [ { @@ -9711,7 +9714,6 @@ " └─TableReader(Probe) 9990.00 root data:Selection", " └─Selection 9990.00 cop[tikv] not(isnull(test.t4.a))", " └─TableFullScan 10000.00 cop[tikv] table:t4 keep order:false, stats:pseudo" ->>>>>>> dc04e2ca077 (planner: clean the one side filters for outer join reorder which is not done in #44409 (#45565)) ] } ]