Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

planner/cascades: add transformation rule PushTopNDownUnionAll #14078

Closed
wants to merge 17 commits into from
Closed

planner/cascades: add transformation rule PushTopNDownUnionAll #14078

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
ee9bced
planner/cascades: add transformation rule PushTopNDownUnionAll
Reminiscent Dec 16, 2019
4b07224
fix bugs
Reminiscent Dec 16, 2019
0f635e2
Merge branch 'master' of https://github.com/pingcap/tidb into TopUnio…
Reminiscent Dec 16, 2019
5ab0bbe
Adjustment format
Reminiscent Dec 16, 2019
2c6b50e
fix bug
Reminiscent Dec 16, 2019
83c6ff2
tikv: check lock timeout again after resolving lock (#14066)
sticnarf Dec 16, 2019
ed43031
expression: implement vectorized evaluation for `builtinBenchmarkSig`…
pingyu Dec 16, 2019
e9bc30b
Adjust format
Reminiscent Dec 16, 2019
3c6fd97
address comments
Reminiscent Dec 17, 2019
95126de
Merge remote-tracking branch 'origin/TopNnionAll' into TopUnionAll
Reminiscent Dec 17, 2019
b70c724
fix CI problems
Reminiscent Dec 17, 2019
2f8da86
address comments
Reminiscent Dec 17, 2019
2012004
Merge branch 'master' of https://github.com/pingcap/tidb into TopUnio…
Reminiscent Dec 17, 2019
d0aec0b
address comments
Reminiscent Dec 17, 2019
89b5a04
for test1
Reminiscent Dec 17, 2019
4dc7afd
for test2
Reminiscent Dec 17, 2019
033d15e
for test3
Reminiscent Dec 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion planner/cascades/testdata/transformation_rules_suite_in.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@
"select c from t order by t.a + t.b limit 1",
"select a, b, c from t t1 where t1.a in (select t2.a as a from t t2 where t2.b > t1.b order by t1.b limit 1)",
"select a, b, c from t t1 where t1.a in (select a from (select t2.a as a, t1.b as b from t t2 where t2.b > t1.b) x order by b limit 1)",
"select a, b from (select @i as a, @i := @i+1 as b from t) t order by a desc limit 1"
"select a, b from (select @i as a, @i := @i+1 as b from t) t order by a desc limit 1",
"(select a from t) union all (select b from t) order by a limit 2;",
"(select a from t where a=1) union all (select a from t where a=2) union all (select a from t where a=3) order by a limit 2;"
]
},
{
Expand Down
142 changes: 142 additions & 0 deletions planner/cascades/testdata/transformation_rules_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,148 @@
"Group#10 Schema:[test.t.a]",
" IndexScan_8 table:t, index:c, d, e"
]
},
{
"SQL": "(select a from t) union all (select b from t) order by a limit 2;",
"Result": [
"Group#0 Schema:[Column#25]",
" Union_5 input:[Group#1,Group#2]",
"Group#1 Schema:[Column#25]",
" Projection_6 input:[Group#3], test.t.a",
"Group#3 Schema:[test.t.a]",
" Projection_4 input:[Group#4], test.t.a",
"Group#4 Schema:[test.t.a]",
" TopN_29 input:[Group#5], test.t.a:asc, offset:0, count:2",
"Group#5 Schema:[test.t.a]",
" TiKVSingleGather_11 input:[Group#6], table:t",
" TiKVSingleGather_23 input:[Group#7], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_21 input:[Group#8], table:t, index:c_d_e_str",
" TiKVSingleGather_19 input:[Group#9], table:t, index:f_g",
" TiKVSingleGather_17 input:[Group#10], table:t, index:g",
" TiKVSingleGather_15 input:[Group#11], table:t, index:f",
" TiKVSingleGather_13 input:[Group#12], table:t, index:c_d_e",
"Group#6 Schema:[test.t.a]",
" TableScan_10 table:t, pk col:test.t.a",
"Group#7 Schema:[test.t.a]",
" IndexScan_22 table:t, index:e_str, d_str, c_str",
"Group#8 Schema:[test.t.a]",
" IndexScan_20 table:t, index:c_str, d_str, e_str",
"Group#9 Schema:[test.t.a]",
" IndexScan_18 table:t, index:f, g",
"Group#10 Schema:[test.t.a]",
" IndexScan_16 table:t, index:g",
"Group#11 Schema:[test.t.a]",
" IndexScan_14 table:t, index:f",
"Group#12 Schema:[test.t.a]",
" IndexScan_12 table:t, index:c, d, e",
"Group#2 Schema:[Column#25]",
" Projection_7 input:[Group#13], test.t.b",
"Group#13 Schema:[test.t.b]",
" Projection_2 input:[Group#14], test.t.b",
"Group#14 Schema:[test.t.b]",
" TopN_31 input:[Group#15], test.t.b:asc, offset:0, count:2",
"Group#15 Schema:[test.t.b]",
" TiKVSingleGather_25 input:[Group#16], table:t",
"Group#16 Schema:[test.t.b]",
" TableScan_24 table:t"
]
},
{
"SQL": "(select a from t where a=1) union all (select a from t where a=2) union all (select a from t where a=3) order by a limit 2;",
"Result": [
"Group#0 Schema:[Column#37]",
" Union_10 input:[Group#1,Group#2,Group#3]",
"Group#1 Schema:[Column#37]",
" Projection_11 input:[Group#4], test.t.a",
"Group#4 Schema:[test.t.a]",
" Projection_9 input:[Group#5], test.t.a",
"Group#5 Schema:[test.t.a]",
" TopN_61 input:[Group#6], test.t.a:asc, offset:0, count:2",
"Group#6 Schema:[test.t.a]",
" Selection_8 input:[Group#7], eq(test.t.a, 1)",
"Group#7 Schema:[test.t.a]",
" TiKVSingleGather_17 input:[Group#8], table:t",
" TiKVSingleGather_29 input:[Group#9], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_27 input:[Group#10], table:t, index:c_d_e_str",
" TiKVSingleGather_25 input:[Group#11], table:t, index:f_g",
" TiKVSingleGather_23 input:[Group#12], table:t, index:g",
" TiKVSingleGather_21 input:[Group#13], table:t, index:f",
" TiKVSingleGather_19 input:[Group#14], table:t, index:c_d_e",
"Group#8 Schema:[test.t.a]",
" TableScan_16 table:t, pk col:test.t.a",
"Group#9 Schema:[test.t.a]",
" IndexScan_28 table:t, index:e_str, d_str, c_str",
"Group#10 Schema:[test.t.a]",
" IndexScan_26 table:t, index:c_str, d_str, e_str",
"Group#11 Schema:[test.t.a]",
" IndexScan_24 table:t, index:f, g",
"Group#12 Schema:[test.t.a]",
" IndexScan_22 table:t, index:g",
"Group#13 Schema:[test.t.a]",
" IndexScan_20 table:t, index:f",
"Group#14 Schema:[test.t.a]",
" IndexScan_18 table:t, index:c, d, e",
"Group#2 Schema:[Column#37]",
" Projection_12 input:[Group#15], test.t.a",
"Group#15 Schema:[test.t.a]",
" Projection_6 input:[Group#16], test.t.a",
"Group#16 Schema:[test.t.a]",
" TopN_63 input:[Group#17], test.t.a:asc, offset:0, count:2",
"Group#17 Schema:[test.t.a]",
" Selection_5 input:[Group#18], eq(test.t.a, 2)",
"Group#18 Schema:[test.t.a]",
" TiKVSingleGather_31 input:[Group#19], table:t",
" TiKVSingleGather_43 input:[Group#20], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_41 input:[Group#21], table:t, index:c_d_e_str",
" TiKVSingleGather_39 input:[Group#22], table:t, index:f_g",
" TiKVSingleGather_37 input:[Group#23], table:t, index:g",
" TiKVSingleGather_35 input:[Group#24], table:t, index:f",
" TiKVSingleGather_33 input:[Group#25], table:t, index:c_d_e",
"Group#19 Schema:[test.t.a]",
" TableScan_30 table:t, pk col:test.t.a",
"Group#20 Schema:[test.t.a]",
" IndexScan_42 table:t, index:e_str, d_str, c_str",
"Group#21 Schema:[test.t.a]",
" IndexScan_40 table:t, index:c_str, d_str, e_str",
"Group#22 Schema:[test.t.a]",
" IndexScan_38 table:t, index:f, g",
"Group#23 Schema:[test.t.a]",
" IndexScan_36 table:t, index:g",
"Group#24 Schema:[test.t.a]",
" IndexScan_34 table:t, index:f",
"Group#25 Schema:[test.t.a]",
" IndexScan_32 table:t, index:c, d, e",
"Group#3 Schema:[Column#37]",
" Projection_13 input:[Group#26], test.t.a",
"Group#26 Schema:[test.t.a]",
" Projection_3 input:[Group#27], test.t.a",
"Group#27 Schema:[test.t.a]",
" TopN_65 input:[Group#28], test.t.a:asc, offset:0, count:2",
"Group#28 Schema:[test.t.a]",
" Selection_2 input:[Group#29], eq(test.t.a, 3)",
"Group#29 Schema:[test.t.a]",
" TiKVSingleGather_45 input:[Group#30], table:t",
" TiKVSingleGather_57 input:[Group#31], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_55 input:[Group#32], table:t, index:c_d_e_str",
" TiKVSingleGather_53 input:[Group#33], table:t, index:f_g",
" TiKVSingleGather_51 input:[Group#34], table:t, index:g",
" TiKVSingleGather_49 input:[Group#35], table:t, index:f",
" TiKVSingleGather_47 input:[Group#36], table:t, index:c_d_e",
"Group#30 Schema:[test.t.a]",
" TableScan_44 table:t, pk col:test.t.a",
"Group#31 Schema:[test.t.a]",
" IndexScan_56 table:t, index:e_str, d_str, c_str",
"Group#32 Schema:[test.t.a]",
" IndexScan_54 table:t, index:c_str, d_str, e_str",
"Group#33 Schema:[test.t.a]",
" IndexScan_52 table:t, index:f, g",
"Group#34 Schema:[test.t.a]",
" IndexScan_50 table:t, index:g",
"Group#35 Schema:[test.t.a]",
" IndexScan_48 table:t, index:f",
"Group#36 Schema:[test.t.a]",
" IndexScan_46 table:t, index:c, d, e"
]
}
]
},
Expand Down
45 changes: 45 additions & 0 deletions planner/cascades/transformation_rules.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ var defaultTransformationMap = map[memo.Operand][]Transformation{
},
memo.OperandTopN: {
NewRulePushTopNDownProjection(),
NewRulePushTopNDownUnionAll(),
},
}

Expand Down Expand Up @@ -869,6 +870,50 @@ func (r *PushTopNDownProjection) OnTransform(old *memo.ExprIter) (newExprs []*me
return []*memo.GroupExpr{projExpr}, true, false, nil
}

// PushTopNDownUnionAll pushes topN to union all.
type PushTopNDownUnionAll struct {
baseRule
}

// NewRulePushTopNDownUnionAll creates a new Transformation PushTopNDownUnionAll.
// The pattern of this rule is `TopN->UnionAll->X` to `UnionAll->TopN->X`.
func NewRulePushTopNDownUnionAll() Transformation {
rule := &PushTopNDownUnionAll{}
rule.pattern = memo.BuildPattern(
memo.OperandTopN,
memo.EngineTiDBOnly,
memo.NewPattern(memo.OperandUnionAll, memo.EngineTiDBOnly),
)
return rule
}

Reminiscent marked this conversation as resolved.
Show resolved Hide resolved
// OnTransform implements Transformation interface.
// This rule tries to pushes the TopN through UnionAll.
func (r *PushTopNDownUnionAll) OnTransform(old *memo.ExprIter) (newExprs []*memo.GroupExpr, eraseOld bool, eraseAll bool, err error) {
topN := old.GetExpr().ExprNode.(*plannercore.LogicalTopN)
unionAll := old.Children[0].GetExpr().ExprNode.(*plannercore.LogicalUnionAll)

newTopN := plannercore.LogicalTopN{
Offset: topN.Offset,
Count: topN.Count,
Reminiscent marked this conversation as resolved.
Show resolved Hide resolved
}.Init(topN.SCtx(), topN.SelectBlockOffset())

newTopN.ByItems = make([]*plannercore.ByItems, 0, len(topN.ByItems))
for _, by := range topN.ByItems {
newTopN.ByItems = append(newTopN.ByItems, &plannercore.ByItems{by.Expr, by.Desc})
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have you ever test TopN by items from different union children? I guess you should divide the TopN by each child's schema.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PlanBuilder ensures that all of the UnionAll's children have the same schema, by adding Projections below the UnionAll. So we have no need to deal with the schema here. It will be done by PushTopNDownProjection.

}

newUnionAllExpr := memo.NewGroupExpr(unionAll)
for _, childGroup := range old.Children[0].GetExpr().Children {
newTopNExpr := memo.NewGroupExpr(topN)
Reminiscent marked this conversation as resolved.
Show resolved Hide resolved
newTopNExpr.Children = append(newTopNExpr.Children, childGroup)
newTopNGroup := memo.NewGroupWithSchema(newTopNExpr, childGroup.Prop.Schema)

newUnionAllExpr.Children = append(newUnionAllExpr.Children, newTopNGroup)
}
return []*memo.GroupExpr{newUnionAllExpr}, true, false, nil
}

// MergeAggregationProjection merges the Projection below an Aggregation as a new Aggregation.
// The Projection may be regenerated in the ImplementationPhase. But this rule allows the
// Aggregation to match other rules, such as MergeAdjacentAggregation.
Expand Down
1 change: 1 addition & 0 deletions planner/cascades/transformation_rules_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ func (s *testTransformationRuleSuite) TestTopNRules(c *C) {
},
memo.OperandTopN: {
NewRulePushTopNDownProjection(),
NewRulePushTopNDownUnionAll(),
},
})
var input []string
Expand Down