Skip to content

Commit

Permalink
planner/core: implement skyline pruning (#9337) (#9356)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alivxxx @jackysp
alivxxx authored and jackysp committed Feb 19, 2019
1 parent 813a4e6 commit b9bca50
Show file tree
Hide file tree
Showing 16 changed files with 417 additions and 155 deletions.
5 changes: 0 additions & 5 deletions cmd/explaintest/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -566,11 +566,6 @@ func loadAllTests() ([]string, error) {
if strings.HasSuffix(name, ".test") {
name = strings.TrimSuffix(name, ".test")

// if we use record and the result file exists, skip generating
if record && resultExists(name) {
continue
}

if create && !strings.HasSuffix(name, "_stats") {
continue
}
Expand Down
21 changes: 21 additions & 0 deletions cmd/explaintest/r/access_path_selection.result
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
CREATE TABLE `outdated_statistics` (
`a` int,
`b` int,
`c` int,
INDEX idx_a(a),
INDEX idx_ab(a,b)
);
insert into outdated_statistics values (2, 2, 2);
insert into outdated_statistics values (3, 3, 3);
insert into outdated_statistics values (4, 4, 4);
analyze table outdated_statistics;
insert into outdated_statistics values (1, 1, 1);
insert into outdated_statistics values (1, 2, 2);
insert into outdated_statistics values (1, 3, 3);
analyze table outdated_statistics index idx_ab;
explain select * from outdated_statistics where a=1 and b=1 and c=1;
id count task operator info
IndexLookUp_11 0.00 root
├─IndexScan_8 1.00 cop table:outdated_statistics, index:a, b, range:[1 1,1 1], keep order:false
└─Selection_10 0.00 cop eq(test.outdated_statistics.c, 1)
└─TableScan_9 1.00 cop table:outdated_statistics, keep order:false
17 changes: 7 additions & 10 deletions cmd/explaintest/r/explain_complex.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,9 +153,9 @@ id count task operator info
Projection_9 0.00 root dt.id, dt.aid, dt.pt, dt.dic, dt.cm, rr.gid, rr.acd, rr.t, dt.p1, dt.p2, dt.p3, dt.p4, dt.p5, dt.p6_md5, dt.p7_md5
└─Limit_12 0.00 root offset:0, count:2000
└─IndexJoin_18 0.00 root inner join, inner:IndexLookUp_17, outer key:dt.aid, dt.dic, inner key:rr.aid, rr.dic
├─TableReader_42 0.00 root data:Selection_41
│ └─Selection_41 0.00 cop eq(dt.bm, 0), eq(dt.pt, "ios"), gt(dt.t, 1478185592)
│ └─TableScan_40 10000.00 cop table:dt, range:[0,+inf], keep order:false, stats:pseudo
├─TableReader_38 0.00 root data:Selection_37
│ └─Selection_37 0.00 cop eq(dt.bm, 0), eq(dt.pt, "ios"), gt(dt.t, 1478185592)
│ └─TableScan_36 10000.00 cop table:dt, range:[0,+inf], keep order:false, stats:pseudo
└─IndexLookUp_17 3.33 root
├─IndexScan_14 10.00 cop table:rr, index:aid, dic, range: decided by [dt.aid dt.dic], keep order:false, stats:pseudo
└─Selection_16 3.33 cop eq(rr.pt, "ios"), gt(rr.t, 1478185592)
Expand All @@ -164,10 +164,10 @@ explain select pc,cr,count(DISTINCT uid) as pay_users,count(oid) as pay_times,su
id count task operator info
Projection_5 1.00 root test.pp.pc, test.pp.cr, 3_col_0, 3_col_1, 3_col_2
└─HashAgg_7 1.00 root group by:test.pp.cr, test.pp.pc, funcs:count(distinct test.pp.uid), count(test.pp.oid), sum(test.pp.am), firstrow(test.pp.pc), firstrow(test.pp.cr)
└─IndexLookUp_28 0.00 root
├─IndexScan_25 0.40 cop table:pp, index:uid, pi, range:[18089709 510017,18089709 510017], [18089709 520017,18089709 520017], [18090780 510017,18090780 510017], [18090780 520017,18090780 520017], keep order:false, stats:pseudo
└─Selection_27 0.00 cop eq(test.pp.ps, 2), ge(test.pp.ppt, 1478188800), lt(test.pp.ppt, 1478275200)
└─TableScan_26 0.40 cop table:pp, keep order:false, stats:pseudo
└─IndexLookUp_24 0.00 root
├─IndexScan_21 0.40 cop table:pp, index:uid, pi, range:[18089709 510017,18089709 510017], [18089709 520017,18089709 520017], [18090780 510017,18090780 510017], [18090780 520017,18090780 520017], keep order:false, stats:pseudo
└─Selection_23 0.00 cop eq(test.pp.ps, 2), ge(test.pp.ppt, 1478188800), lt(test.pp.ppt, 1478275200)
└─TableScan_22 0.40 cop table:pp, keep order:false, stats:pseudo
CREATE TABLE `tbl_001` (`a` int, `b` int);
CREATE TABLE `tbl_002` (`a` int, `b` int);
CREATE TABLE `tbl_003` (`a` int, `b` int);
Expand Down Expand Up @@ -200,6 +200,3 @@ HashAgg_34 72000.00 root group by:col_1, funcs:sum(col_0)
│ └─TableScan_58 10000.00 cop table:tbl_008, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_62 10000.00 root data:TableScan_61
└─TableScan_61 10000.00 cop table:tbl_009, range:[-inf,+inf], keep order:false, stats:pseudo



17 changes: 7 additions & 10 deletions cmd/explaintest/r/explain_complex_stats.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,9 @@ id count task operator info
Projection_9 428.32 root dt.id, dt.aid, dt.pt, dt.dic, dt.cm, rr.gid, rr.acd, rr.t, dt.p1, dt.p2, dt.p3, dt.p4, dt.p5, dt.p6_md5, dt.p7_md5
└─Limit_12 428.32 root offset:0, count:2000
└─IndexJoin_18 428.32 root inner join, inner:IndexLookUp_17, outer key:dt.aid, dt.dic, inner key:rr.aid, rr.dic
├─TableReader_42 428.32 root data:Selection_41
│ └─Selection_41 428.32 cop eq(dt.bm, 0), eq(dt.pt, "ios"), gt(dt.t, 1478185592)
│ └─TableScan_40 2000.00 cop table:dt, range:[0,+inf], keep order:false
├─TableReader_38 428.32 root data:Selection_37
│ └─Selection_37 428.32 cop eq(dt.bm, 0), eq(dt.pt, "ios"), gt(dt.t, 1478185592)
│ └─TableScan_36 2000.00 cop table:dt, range:[0,+inf], keep order:false
└─IndexLookUp_17 970.00 root
├─IndexScan_14 1.00 cop table:rr, index:aid, dic, range: decided by [dt.aid dt.dic], keep order:false
└─Selection_16 970.00 cop eq(rr.pt, "ios"), gt(rr.t, 1478185592)
Expand All @@ -172,10 +172,10 @@ explain select pc,cr,count(DISTINCT uid) as pay_users,count(oid) as pay_times,su
id count task operator info
Projection_5 207.86 root test.pp.pc, test.pp.cr, 3_col_0, 3_col_1, 3_col_2
└─HashAgg_7 207.86 root group by:test.pp.cr, test.pp.pc, funcs:count(distinct test.pp.uid), count(test.pp.oid), sum(test.pp.am), firstrow(test.pp.pc), firstrow(test.pp.cr)
└─IndexLookUp_28 207.86 root
├─IndexScan_22 627.00 cop table:pp, index:ps, range:[2,2], keep order:false
└─Selection_24 207.86 cop ge(test.pp.ppt, 1478188800), in(test.pp.pi, 510017, 520017), in(test.pp.uid, 18089709, 18090780), lt(test.pp.ppt, 1478275200)
└─TableScan_23 627.00 cop table:pp, keep order:false
└─IndexLookUp_24 207.86 root
├─IndexScan_18 627.00 cop table:pp, index:ps, range:[2,2], keep order:false
└─Selection_20 207.86 cop ge(test.pp.ppt, 1478188800), in(test.pp.pi, 510017, 520017), in(test.pp.uid, 18089709, 18090780), lt(test.pp.ppt, 1478275200)
└─TableScan_19 627.00 cop table:pp, keep order:false
drop table if exists tbl_001;
CREATE TABLE tbl_001 (a int, b int);
load stats 's/explain_complex_stats_tbl_001.json';
Expand Down Expand Up @@ -226,6 +226,3 @@ HashAgg_34 18000.00 root group by:col_1, funcs:sum(col_0)
│ └─TableScan_58 2000.00 cop table:tbl_008, range:[-inf,+inf], keep order:false
└─TableReader_62 2000.00 root data:TableScan_61
└─TableScan_61 2000.00 cop table:tbl_009, range:[-inf,+inf], keep order:false



90 changes: 45 additions & 45 deletions cmd/explaintest/r/explain_easy.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ TableReader_6 3333.33 root data:TableScan_5
└─TableScan_5 3333.33 cop table:t1, range:(0,+inf], keep order:false, stats:pseudo
explain select t1.c1, t1.c2 from t1 where t1.c2 = 1;
id count task operator info
IndexReader_9 10.00 root index:IndexScan_8
└─IndexScan_8 10.00 cop table:t1, index:c2, range:[1,1], keep order:false, stats:pseudo
IndexReader_6 10.00 root index:IndexScan_5
└─IndexScan_5 10.00 cop table:t1, index:c2, range:[1,1], keep order:false, stats:pseudo
explain select * from t1 left join t2 on t1.c2 = t2.c1 where t1.c1 > 1;
id count task operator info
IndexJoin_11 4166.67 root left outer join, inner:IndexLookUp_10, outer key:test.t1.c2, inner key:test.t2.c1
Expand Down Expand Up @@ -101,10 +101,10 @@ Projection_13 10000.00 root k
└─MergeJoin_15 10000.00 root left outer join, left key:test.t1.c1, right key:s.c1
├─TableReader_18 10000.00 root data:TableScan_17
│ └─TableScan_17 10000.00 cop table:t1, range:[-inf,+inf], keep order:true, stats:pseudo
└─Selection_20 8000.00 root ne(k, 0)
└─Projection_21 10000.00 root 1, s.c1
└─TableReader_23 10000.00 root data:TableScan_22
└─TableScan_22 10000.00 cop table:s, range:[-inf,+inf], keep order:true, stats:pseudo
└─Selection_19 8000.00 root ne(k, 0)
└─Projection_20 10000.00 root 1, s.c1
└─TableReader_22 10000.00 root data:TableScan_21
└─TableScan_21 10000.00 cop table:s, range:[-inf,+inf], keep order:true, stats:pseudo
explain select * from information_schema.columns;
id count task operator info
MemTableScan_4 10000.00 root
Expand All @@ -122,8 +122,8 @@ Projection_12 10000.00 root eq(test.t1.c2, test.t2.c2)
explain select * from t1 order by c1 desc limit 1;
id count task operator info
Limit_10 1.00 root offset:0, count:1
└─TableReader_21 1.00 root data:Limit_20
└─Limit_20 1.00 cop offset:0, count:1
└─TableReader_20 1.00 root data:Limit_19
└─Limit_19 1.00 cop offset:0, count:1
└─TableScan_18 1.00 cop table:t1, range:[-inf,+inf], keep order:true, desc, stats:pseudo
explain select * from t4 use index(idx) where a > 1 and b > 1 and c > 1 limit 1;
id count task operator info
Expand All @@ -137,8 +137,8 @@ Limit_9 1.00 root offset:0, count:1
explain select * from t4 where a > 1 and c > 1 limit 1;
id count task operator info
Limit_8 1.00 root offset:0, count:1
└─TableReader_15 1.00 root data:Limit_14
└─Limit_14 1.00 cop offset:0, count:1
└─TableReader_14 1.00 root data:Limit_13
└─Limit_13 1.00 cop offset:0, count:1
└─Selection_12 1.00 cop gt(test.t4.c, 1)
└─TableScan_11 3.00 cop table:t4, range:(1,+inf], keep order:false, stats:pseudo
explain select ifnull(null, t1.c1) from t1;
Expand All @@ -154,42 +154,42 @@ id count task operator info
Union_17 26000.00 root
├─HashAgg_21 16000.00 root group by:c1, funcs:firstrow(join_agg_0)
│ └─Union_22 16000.00 root
│ ├─StreamAgg_35 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1)
│ │ └─IndexReader_36 8000.00 root index:StreamAgg_26
│ ├─StreamAgg_34 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1)
│ │ └─IndexReader_35 8000.00 root index:StreamAgg_26
│ │ └─StreamAgg_26 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1)
│ │ └─IndexScan_34 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
│ └─StreamAgg_52 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1)
│ └─IndexReader_53 8000.00 root index:StreamAgg_43
│ └─StreamAgg_43 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1)
│ └─IndexScan_51 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
└─TableReader_59 10000.00 root data:TableScan_58
└─TableScan_58 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo
│ │ └─IndexScan_33 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
│ └─StreamAgg_49 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1)
│ └─IndexReader_50 8000.00 root index:StreamAgg_41
│ └─StreamAgg_41 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1)
│ └─IndexScan_48 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
└─TableReader_55 10000.00 root data:TableScan_54
└─TableScan_54 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select c1 from t2 union all select c1 from t2 union select c1 from t2;
id count task operator info
HashAgg_18 24000.00 root group by:c1, funcs:firstrow(join_agg_0)
└─Union_19 24000.00 root
├─StreamAgg_32 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1)
│ └─IndexReader_33 8000.00 root index:StreamAgg_23
├─StreamAgg_31 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1)
│ └─IndexReader_32 8000.00 root index:StreamAgg_23
│ └─StreamAgg_23 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1)
│ └─IndexScan_31 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
├─StreamAgg_49 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1)
│ └─IndexReader_50 8000.00 root index:StreamAgg_40
│ └─StreamAgg_40 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1)
│ └─IndexScan_48 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
└─StreamAgg_66 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1)
└─IndexReader_67 8000.00 root index:StreamAgg_57
└─StreamAgg_57 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1)
└─IndexScan_65 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
│ └─IndexScan_30 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
├─StreamAgg_46 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1)
│ └─IndexReader_47 8000.00 root index:StreamAgg_38
│ └─StreamAgg_38 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1)
│ └─IndexScan_45 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
└─StreamAgg_61 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1)
└─IndexReader_62 8000.00 root index:StreamAgg_53
└─StreamAgg_53 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1)
└─IndexScan_60 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
set @@session.tidb_opt_insubquery_unfold = 0;
explain select sum(t1.c1 in (select c1 from t2)) from t1;
id count task operator info
StreamAgg_12 1.00 root funcs:sum(col_0)
└─Projection_35 10000.00 root cast(5_aux_0)
└─MergeJoin_28 10000.00 root left outer semi join, left key:test.t1.c1, right key:test.t2.c1
└─Projection_33 10000.00 root cast(5_aux_0)
└─MergeJoin_26 10000.00 root left outer semi join, left key:test.t1.c1, right key:test.t2.c1
├─TableReader_19 10000.00 root data:TableScan_18
│ └─TableScan_18 10000.00 cop table:t1, range:[-inf,+inf], keep order:true, stats:pseudo
└─IndexReader_23 10000.00 root index:IndexScan_22
└─IndexScan_22 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
└─IndexReader_21 10000.00 root index:IndexScan_20
└─IndexScan_20 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo
explain select 1 in (select c2 from t2) from t1;
id count task operator info
Projection_6 10000.00 root 5_aux_0
Expand Down Expand Up @@ -217,25 +217,25 @@ subgraph cluster12{
node [style=filled, color=lightgrey]
color=black
label = "root"
"StreamAgg_12" -> "Projection_35"
"Projection_35" -> "MergeJoin_28"
"MergeJoin_28" -> "TableReader_19"
"MergeJoin_28" -> "IndexReader_23"
"StreamAgg_12" -> "Projection_33"
"Projection_33" -> "MergeJoin_26"
"MergeJoin_26" -> "TableReader_19"
"MergeJoin_26" -> "IndexReader_21"
}
subgraph cluster18{
node [style=filled, color=lightgrey]
color=black
label = "cop"
"TableScan_18"
}
subgraph cluster22{
subgraph cluster20{
node [style=filled, color=lightgrey]
color=black
label = "cop"
"IndexScan_22"
"IndexScan_20"
}
"TableReader_19" -> "TableScan_18"
"IndexReader_23" -> "IndexScan_22"
"IndexReader_21" -> "IndexScan_20"
}

explain format="dot" select 1 in (select c2 from t2) from t1;
Expand Down Expand Up @@ -333,17 +333,17 @@ drop table if exists t;
create table t(a bigint, b bigint, index idx(a, b));
explain select * from t where a in (1, 2) and a in (1, 3);
id count task operator info
IndexReader_9 10.00 root index:IndexScan_8
└─IndexScan_8 10.00 cop table:t, index:a, b, range:[1,1], keep order:false, stats:pseudo
IndexReader_6 10.00 root index:IndexScan_5
└─IndexScan_5 10.00 cop table:t, index:a, b, range:[1,1], keep order:false, stats:pseudo
explain select * from t where b in (1, 2) and b in (1, 3);
id count task operator info
TableReader_7 10.00 root data:Selection_6
└─Selection_6 10.00 cop in(test.t.b, 1, 2), in(test.t.b, 1, 3)
└─TableScan_5 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where a = 1 and a = 1;
id count task operator info
IndexReader_9 10.00 root index:IndexScan_8
└─IndexScan_8 10.00 cop table:t, index:a, b, range:[1,1], keep order:false, stats:pseudo
IndexReader_6 10.00 root index:IndexScan_5
└─IndexScan_5 10.00 cop table:t, index:a, b, range:[1,1], keep order:false, stats:pseudo
explain select * from t where a = 1 and a = 2;
id count task operator info
TableDual_5 0.00 root rows:0
Expand Down
8 changes: 4 additions & 4 deletions cmd/explaintest/r/explain_easy_stats.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ TableReader_6 1999.00 root data:TableScan_5
└─TableScan_5 1999.00 cop table:t1, range:(0,+inf], keep order:false
explain select t1.c1, t1.c2 from t1 where t1.c2 = 1;
id count task operator info
IndexReader_9 0.00 root index:IndexScan_8
└─IndexScan_8 0.00 cop table:t1, index:c2, range:[1,1], keep order:false
IndexReader_6 0.00 root index:IndexScan_5
└─IndexScan_5 0.00 cop table:t1, index:c2, range:[1,1], keep order:false
explain select * from t1 left join t2 on t1.c2 = t2.c1 where t1.c1 > 1;
id count task operator info
Projection_6 2481.25 root test.t1.c1, test.t1.c2, test.t1.c3, test.t2.c1, test.t2.c2
Expand Down Expand Up @@ -110,8 +110,8 @@ Projection_12 1999.00 root eq(test.t1.c2, test.t2.c2)
explain select * from t1 order by c1 desc limit 1;
id count task operator info
Limit_10 1.00 root offset:0, count:1
└─TableReader_21 1.00 root data:Limit_20
└─Limit_20 1.00 cop offset:0, count:1
└─TableReader_20 1.00 root data:Limit_19
└─Limit_19 1.00 cop offset:0, count:1
└─TableScan_18 1.00 cop table:t1, range:[-inf,+inf], keep order:true, desc
set @@session.tidb_opt_insubquery_unfold = 0;
explain select 1 in (select c2 from t2) from t1;
Expand Down
Loading

0 comments on commit b9bca50

Please sign in to comment.