Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

planner/cascades: add transformation rule PushTopNDownUnionAll #14078

Closed
wants to merge 17 commits into from
Closed

planner/cascades: add transformation rule PushTopNDownUnionAll #14078

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
ee9bced
planner/cascades: add transformation rule PushTopNDownUnionAll
Reminiscent Dec 16, 2019
4b07224
fix bugs
Reminiscent Dec 16, 2019
0f635e2
Merge branch 'master' of https://github.com/pingcap/tidb into TopUnio…
Reminiscent Dec 16, 2019
5ab0bbe
Adjustment format
Reminiscent Dec 16, 2019
2c6b50e
fix bug
Reminiscent Dec 16, 2019
83c6ff2
tikv: check lock timeout again after resolving lock (#14066)
sticnarf Dec 16, 2019
ed43031
expression: implement vectorized evaluation for `builtinBenchmarkSig`…
pingyu Dec 16, 2019
e9bc30b
Adjust format
Reminiscent Dec 16, 2019
3c6fd97
address comments
Reminiscent Dec 17, 2019
95126de
Merge remote-tracking branch 'origin/TopNnionAll' into TopUnionAll
Reminiscent Dec 17, 2019
b70c724
fix CI problems
Reminiscent Dec 17, 2019
2f8da86
address comments
Reminiscent Dec 17, 2019
2012004
Merge branch 'master' of https://github.com/pingcap/tidb into TopUnio…
Reminiscent Dec 17, 2019
d0aec0b
address comments
Reminiscent Dec 17, 2019
89b5a04
for test1
Reminiscent Dec 17, 2019
4dc7afd
for test2
Reminiscent Dec 17, 2019
033d15e
for test3
Reminiscent Dec 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion planner/cascades/testdata/transformation_rules_suite_in.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,10 @@
"select c from t order by t.a + t.b limit 1",
"select a, b, c from t t1 where t1.a in (select t2.a as a from t t2 where t2.b > t1.b order by t1.b limit 1)",
"select a, b, c from t t1 where t1.a in (select a from (select t2.a as a, t1.b as b from t t2 where t2.b > t1.b) x order by b limit 1)",
"select a, b from (select @i as a, @i := @i+1 as b from t) t order by a desc limit 1"
"select a, b from (select @i as a, @i := @i+1 as b from t) t order by a desc limit 1",
"(select a from t) union all (select b from t) order by a limit 2;",
"(select a from t) union all (select sum(a) from t where a > 2 group by b) order by a limit 2;",
"(select a from t where a = 1) union all (select b from t where a = 2) union all (select c from t where a = 3) order by a limit 2;"
]
},
{
Expand Down
189 changes: 189 additions & 0 deletions planner/cascades/testdata/transformation_rules_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,195 @@
"Group#10 Schema:[test.t.a]",
" IndexScan_8 table:t, index:c, d, e"
]
},
{
"SQL": "(select a from t) union all (select b from t) order by a limit 2;",
"Result": [
"Group#0 Schema:[Column#25]",
" Union_5 input:[Group#1,Group#2]",
"Group#1 Schema:[Column#25]",
" Projection_6 input:[Group#3], test.t.a",
"Group#3 Schema:[test.t.a]",
" Projection_4 input:[Group#4], test.t.a",
"Group#4 Schema:[test.t.a]",
" TopN_29 input:[Group#5], test.t.a:asc, offset:0, count:2",
"Group#5 Schema:[test.t.a]",
" TiKVSingleGather_11 input:[Group#6], table:t",
" TiKVSingleGather_23 input:[Group#7], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_21 input:[Group#8], table:t, index:c_d_e_str",
" TiKVSingleGather_19 input:[Group#9], table:t, index:f_g",
" TiKVSingleGather_17 input:[Group#10], table:t, index:g",
" TiKVSingleGather_15 input:[Group#11], table:t, index:f",
" TiKVSingleGather_13 input:[Group#12], table:t, index:c_d_e",
"Group#6 Schema:[test.t.a]",
" TableScan_10 table:t, pk col:test.t.a",
"Group#7 Schema:[test.t.a]",
" IndexScan_22 table:t, index:e_str, d_str, c_str",
"Group#8 Schema:[test.t.a]",
" IndexScan_20 table:t, index:c_str, d_str, e_str",
"Group#9 Schema:[test.t.a]",
" IndexScan_18 table:t, index:f, g",
"Group#10 Schema:[test.t.a]",
" IndexScan_16 table:t, index:g",
"Group#11 Schema:[test.t.a]",
" IndexScan_14 table:t, index:f",
"Group#12 Schema:[test.t.a]",
" IndexScan_12 table:t, index:c, d, e",
"Group#2 Schema:[Column#25]",
" Projection_7 input:[Group#13], test.t.b",
"Group#13 Schema:[test.t.b]",
" Projection_2 input:[Group#14], test.t.b",
"Group#14 Schema:[test.t.b]",
" TopN_31 input:[Group#15], test.t.b:asc, offset:0, count:2",
"Group#15 Schema:[test.t.b]",
" TiKVSingleGather_25 input:[Group#16], table:t",
"Group#16 Schema:[test.t.b]",
" TableScan_24 table:t"
]
},
{
"SQL": "(select a from t) union all (select sum(a) from t where a > 2 group by b) limit 2;",
Reminiscent marked this conversation as resolved.
Show resolved Hide resolved
"Result": [
"Group#0 Schema:[Column#26]",
" Limit_10 input:[Group#1], offset:0, count:2",
"Group#1 Schema:[Column#26]",
" Union_7 input:[Group#2,Group#3]",
"Group#2 Schema:[Column#26]",
" Projection_8 input:[Group#4], cast(test.t.a)->Column#26",
"Group#4 Schema:[test.t.a]",
" Projection_6 input:[Group#5], test.t.a",
"Group#5 Schema:[test.t.a]",
" TiKVSingleGather_12 input:[Group#6], table:t",
" TiKVSingleGather_24 input:[Group#7], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_22 input:[Group#8], table:t, index:c_d_e_str",
" TiKVSingleGather_20 input:[Group#9], table:t, index:f_g",
" TiKVSingleGather_18 input:[Group#10], table:t, index:g",
" TiKVSingleGather_16 input:[Group#11], table:t, index:f",
" TiKVSingleGather_14 input:[Group#12], table:t, index:c_d_e",
"Group#6 Schema:[test.t.a]",
" TableScan_11 table:t, pk col:test.t.a",
"Group#7 Schema:[test.t.a]",
" IndexScan_23 table:t, index:e_str, d_str, c_str",
"Group#8 Schema:[test.t.a]",
" IndexScan_21 table:t, index:c_str, d_str, e_str",
"Group#9 Schema:[test.t.a]",
" IndexScan_19 table:t, index:f, g",
"Group#10 Schema:[test.t.a]",
" IndexScan_17 table:t, index:g",
"Group#11 Schema:[test.t.a]",
" IndexScan_15 table:t, index:f",
"Group#12 Schema:[test.t.a]",
" IndexScan_13 table:t, index:c, d, e",
"Group#3 Schema:[Column#26]",
" Projection_9 input:[Group#13], cast(Column#13)->Column#26",
"Group#13 Schema:[Column#13]",
" Projection_4 input:[Group#14], Column#13",
"Group#14 Schema:[Column#13]",
" Aggregation_3 input:[Group#15], group by:test.t.b, funcs:sum(test.t.a)",
"Group#15 Schema:[test.t.a,test.t.b]",
" Selection_2 input:[Group#16], gt(test.t.a, 2)",
"Group#16 Schema:[test.t.a,test.t.b]",
" TiKVSingleGather_26 input:[Group#17], table:t",
"Group#17 Schema:[test.t.a,test.t.b]",
" TableScan_25 table:t, pk col:test.t.a"
]
},
{
"SQL": "(select a from t where a=1) union all (select a from t where a=2) union all (select a from t where a=3) order by a limit 2;",
"Result": [
"Group#0 Schema:[Column#37]",
" Union_10 input:[Group#1,Group#2,Group#3]",
"Group#1 Schema:[Column#37]",
" Projection_11 input:[Group#4], test.t.a",
"Group#4 Schema:[test.t.a]",
" Projection_9 input:[Group#5], test.t.a",
"Group#5 Schema:[test.t.a]",
" TopN_61 input:[Group#6], test.t.a:asc, offset:0, count:2",
"Group#6 Schema:[test.t.a]",
" Selection_8 input:[Group#7], eq(test.t.a, 1)",
"Group#7 Schema:[test.t.a]",
" TiKVSingleGather_17 input:[Group#8], table:t",
" TiKVSingleGather_29 input:[Group#9], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_27 input:[Group#10], table:t, index:c_d_e_str",
" TiKVSingleGather_25 input:[Group#11], table:t, index:f_g",
" TiKVSingleGather_23 input:[Group#12], table:t, index:g",
" TiKVSingleGather_21 input:[Group#13], table:t, index:f",
" TiKVSingleGather_19 input:[Group#14], table:t, index:c_d_e",
"Group#8 Schema:[test.t.a]",
" TableScan_16 table:t, pk col:test.t.a",
"Group#9 Schema:[test.t.a]",
" IndexScan_28 table:t, index:e_str, d_str, c_str",
"Group#10 Schema:[test.t.a]",
" IndexScan_26 table:t, index:c_str, d_str, e_str",
"Group#11 Schema:[test.t.a]",
" IndexScan_24 table:t, index:f, g",
"Group#12 Schema:[test.t.a]",
" IndexScan_22 table:t, index:g",
"Group#13 Schema:[test.t.a]",
" IndexScan_20 table:t, index:f",
"Group#14 Schema:[test.t.a]",
" IndexScan_18 table:t, index:c, d, e",
"Group#2 Schema:[Column#37]",
" Projection_12 input:[Group#15], test.t.a",
"Group#15 Schema:[test.t.a]",
" Projection_6 input:[Group#16], test.t.a",
"Group#16 Schema:[test.t.a]",
" TopN_63 input:[Group#17], test.t.a:asc, offset:0, count:2",
"Group#17 Schema:[test.t.a]",
" Selection_5 input:[Group#18], eq(test.t.a, 2)",
"Group#18 Schema:[test.t.a]",
" TiKVSingleGather_31 input:[Group#19], table:t",
" TiKVSingleGather_43 input:[Group#20], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_41 input:[Group#21], table:t, index:c_d_e_str",
" TiKVSingleGather_39 input:[Group#22], table:t, index:f_g",
" TiKVSingleGather_37 input:[Group#23], table:t, index:g",
" TiKVSingleGather_35 input:[Group#24], table:t, index:f",
" TiKVSingleGather_33 input:[Group#25], table:t, index:c_d_e",
"Group#19 Schema:[test.t.a]",
" TableScan_30 table:t, pk col:test.t.a",
"Group#20 Schema:[test.t.a]",
" IndexScan_42 table:t, index:e_str, d_str, c_str",
"Group#21 Schema:[test.t.a]",
" IndexScan_40 table:t, index:c_str, d_str, e_str",
"Group#22 Schema:[test.t.a]",
" IndexScan_38 table:t, index:f, g",
"Group#23 Schema:[test.t.a]",
" IndexScan_36 table:t, index:g",
"Group#24 Schema:[test.t.a]",
" IndexScan_34 table:t, index:f",
"Group#25 Schema:[test.t.a]",
" IndexScan_32 table:t, index:c, d, e",
"Group#3 Schema:[Column#37]",
" Projection_13 input:[Group#26], test.t.a",
"Group#26 Schema:[test.t.a]",
" Projection_3 input:[Group#27], test.t.a",
"Group#27 Schema:[test.t.a]",
" TopN_65 input:[Group#28], test.t.a:asc, offset:0, count:2",
"Group#28 Schema:[test.t.a]",
" Selection_2 input:[Group#29], eq(test.t.a, 3)",
"Group#29 Schema:[test.t.a]",
" TiKVSingleGather_45 input:[Group#30], table:t",
" TiKVSingleGather_57 input:[Group#31], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_55 input:[Group#32], table:t, index:c_d_e_str",
" TiKVSingleGather_53 input:[Group#33], table:t, index:f_g",
" TiKVSingleGather_51 input:[Group#34], table:t, index:g",
" TiKVSingleGather_49 input:[Group#35], table:t, index:f",
" TiKVSingleGather_47 input:[Group#36], table:t, index:c_d_e",
"Group#30 Schema:[test.t.a]",
" TableScan_44 table:t, pk col:test.t.a",
"Group#31 Schema:[test.t.a]",
" IndexScan_56 table:t, index:e_str, d_str, c_str",
"Group#32 Schema:[test.t.a]",
" IndexScan_54 table:t, index:c_str, d_str, e_str",
"Group#33 Schema:[test.t.a]",
" IndexScan_52 table:t, index:f, g",
"Group#34 Schema:[test.t.a]",
" IndexScan_50 table:t, index:g",
"Group#35 Schema:[test.t.a]",
" IndexScan_48 table:t, index:f",
"Group#36 Schema:[test.t.a]",
" IndexScan_46 table:t, index:c, d, e"
]
}
]
},
Expand Down
49 changes: 49 additions & 0 deletions planner/cascades/transformation_rules.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ var defaultTransformationMap = map[memo.Operand][]Transformation{
},
memo.OperandTopN: {
NewRulePushTopNDownProjection(),
NewRulePushTopNDownUnionAll(),
},
}

Expand Down Expand Up @@ -869,6 +870,54 @@ func (r *PushTopNDownProjection) OnTransform(old *memo.ExprIter) (newExprs []*me
return []*memo.GroupExpr{projExpr}, true, false, nil
}

// PushTopNDownUnionAll pushes topN to union all.
type PushTopNDownUnionAll struct {
baseRule
}

// NewRulePushTopNDownUnionAll creates a new Transformation PushTopNDownUnionAll.
// The pattern of this rule is `TopN->UnionAll->X` to `TopN->UnionAll->TopN->X`.
Reminiscent marked this conversation as resolved.
Show resolved Hide resolved
func NewRulePushTopNDownUnionAll() Transformation {
rule := &PushTopNDownUnionAll{}
rule.pattern = memo.BuildPattern(
memo.OperandTopN,
memo.EngineTiDBOnly,
memo.NewPattern(memo.OperandUnionAll, memo.EngineTiDBOnly),
)
return rule
}

Reminiscent marked this conversation as resolved.
Show resolved Hide resolved
// OnTransform implements Transformation interface.
// It will transform `TopN->UnionAll->X` to `TopN->UnionAll->TopN->X`.
func (r *PushTopNDownUnionAll) OnTransform(old *memo.ExprIter) (newExprs []*memo.GroupExpr, eraseOld bool, eraseAll bool, err error) {
topN := old.GetExpr().ExprNode.(*plannercore.LogicalTopN)
unionAll := old.Children[0].GetExpr().ExprNode.(*plannercore.LogicalUnionAll)

newTopN := plannercore.LogicalTopN{
Count: topN.Count + topN.Offset,
}.Init(topN.SCtx(), topN.SelectBlockOffset())
newTopN.ByItems = make([]*plannercore.ByItems, 0, len(topN.ByItems))
for _, by := range topN.ByItems {
newTopN.ByItems = append(newTopN.ByItems, &plannercore.ByItems{
Expr: by.Expr,
Desc: by.Desc})
}
Reminiscent marked this conversation as resolved.
Show resolved Hide resolved

newUnionAllExpr := memo.NewGroupExpr(unionAll)
for _, childGroup := range old.Children[0].GetExpr().Children {
newTopNExpr := memo.NewGroupExpr(newTopN)
newTopNExpr.Children = append(newTopNExpr.Children, childGroup)
newTopNGroup := memo.NewGroupWithSchema(newTopNExpr, childGroup.Prop.Schema)

newUnionAllExpr.Children = append(newUnionAllExpr.Children, newTopNGroup)
}

newTopNExpr := memo.NewGroupExpr(newTopN)
Reminiscent marked this conversation as resolved.
Show resolved Hide resolved
newUnionAllGroup := memo.NewGroupWithSchema(newUnionAllExpr, unionAll.Schema())
newTopNExpr.SetChildren(newUnionAllGroup)
return []*memo.GroupExpr{newTopNExpr}, true, false, nil
francis0407 marked this conversation as resolved.
Show resolved Hide resolved
}

// MergeAggregationProjection merges the Projection below an Aggregation as a new Aggregation.
// The Projection may be regenerated in the ImplementationPhase. But this rule allows the
// Aggregation to match other rules, such as MergeAdjacentAggregation.
Expand Down
1 change: 1 addition & 0 deletions planner/cascades/transformation_rules_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ func (s *testTransformationRuleSuite) TestTopNRules(c *C) {
},
memo.OperandTopN: {
NewRulePushTopNDownProjection(),
NewRulePushTopNDownUnionAll(),
},
})
var input []string
Expand Down