Skip to content

Commit

Permalink
planner/core: support partition pruning for partition expression floo…
Browse files Browse the repository at this point in the history 
…r(unix_timestamp()) (#16402)
  • Loading branch information
@tiancaiamao
tiancaiamao authored Apr 17, 2020
1 parent 10b1181 commit d8e6cf8
Show file tree
Hide file tree
Showing 4 changed files with 264 additions and 4 deletions.
174 changes: 174 additions & 0 deletions cmd/explaintest/r/partition_pruning.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4198,3 +4198,177 @@ id estRows task access object operator info
TableReader_8 8000.00 root data:Selection_7
└─Selection_7 8000.00 cop[tikv] or(and(eq(test.t.a, 11), eq(test.t.b, 1)), and(eq(test.t.a, 12), eq(test.t.b, 1)))
└─TableFullScan_6 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
drop table if exists t;
create table t (ts timestamp(3) not null default current_timestamp(3))
partition by range (floor(unix_timestamp(ts))) (
partition p0 values less than (unix_timestamp('2020-04-05 00:00:00')),
partition p1 values less than (unix_timestamp('2020-04-15 00:00:00')),
partition p2 values less than (unix_timestamp('2020-04-25 00:00:00'))
);
explain select * from t where ts = '2020-04-06 00:00:00' -- p1;
id estRows task access object operator info
TableReader_8 10.00 root data:Selection_7
└─Selection_7 10.00 cop[tikv] eq(test.t.ts, 2020-04-06 00:00:00.000000)
└─TableFullScan_6 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
explain select * from t where ts = '2020-04-05 00:00:00.001' -- p1;
id estRows task access object operator info
TableReader_8 10.00 root data:Selection_7
└─Selection_7 10.00 cop[tikv] eq(test.t.ts, 2020-04-05 00:00:00.001000)
└─TableFullScan_6 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-15 00:00:00' -- p2;
id estRows task access object operator info
TableReader_8 3333.33 root data:Selection_7
└─Selection_7 3333.33 cop[tikv] gt(test.t.ts, 2020-04-15 00:00:00.000000)
└─TableFullScan_6 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-14 23:59:59.999' -- p1,p2;
id estRows task access object operator info
Union_8 6666.67 root
├─TableReader_11 3333.33 root data:Selection_10
│ └─Selection_10 3333.33 cop[tikv] gt(test.t.ts, 2020-04-14 23:59:59.999000)
│ └─TableFullScan_9 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
└─TableReader_14 3333.33 root data:Selection_13
└─Selection_13 3333.33 cop[tikv] gt(test.t.ts, 2020-04-14 23:59:59.999000)
└─TableFullScan_12 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-15 00:00:00.001' -- p2;
id estRows task access object operator info
TableReader_8 3333.33 root data:Selection_7
└─Selection_7 3333.33 cop[tikv] gt(test.t.ts, 2020-04-15 00:00:00.001000)
└─TableFullScan_6 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-26 00:00:00.001' -- dual;
id estRows task access object operator info
TableDual_6 0.00 root rows:0
explain select * from t where ts >= '2020-04-04 12:22:32' -- p0,p1,p2;
id estRows task access object operator info
Union_9 10000.00 root
├─TableReader_12 3333.33 root data:Selection_11
│ └─Selection_11 3333.33 cop[tikv] ge(test.t.ts, 2020-04-04 12:22:32.000000)
│ └─TableFullScan_10 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
├─TableReader_15 3333.33 root data:Selection_14
│ └─Selection_14 3333.33 cop[tikv] ge(test.t.ts, 2020-04-04 12:22:32.000000)
│ └─TableFullScan_13 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
└─TableReader_18 3333.33 root data:Selection_17
└─Selection_17 3333.33 cop[tikv] ge(test.t.ts, 2020-04-04 12:22:32.000000)
└─TableFullScan_16 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
explain select * from t where ts >= '2020-04-05 00:00:00' -- p1,p2;
id estRows task access object operator info
Union_8 6666.67 root
├─TableReader_11 3333.33 root data:Selection_10
│ └─Selection_10 3333.33 cop[tikv] ge(test.t.ts, 2020-04-05 00:00:00.000000)
│ └─TableFullScan_9 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
└─TableReader_14 3333.33 root data:Selection_13
└─Selection_13 3333.33 cop[tikv] ge(test.t.ts, 2020-04-05 00:00:00.000000)
└─TableFullScan_12 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
explain select * from t where ts >= '2020-04-25 00:00:00' -- dual;
id estRows task access object operator info
TableDual_6 0.00 root rows:0
explain select * from t where ts < '2020-04-25 00:00:00' -- p0,p1,p2;
id estRows task access object operator info
Union_9 9970.00 root
├─TableReader_12 3323.33 root data:Selection_11
│ └─Selection_11 3323.33 cop[tikv] lt(test.t.ts, 2020-04-25 00:00:00.000000)
│ └─TableFullScan_10 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
├─TableReader_15 3323.33 root data:Selection_14
│ └─Selection_14 3323.33 cop[tikv] lt(test.t.ts, 2020-04-25 00:00:00.000000)
│ └─TableFullScan_13 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
└─TableReader_18 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop[tikv] lt(test.t.ts, 2020-04-25 00:00:00.000000)
└─TableFullScan_16 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
explain select * from t where ts < '2020-04-15 00:00:00.001' -- p0,p1,p2;
id estRows task access object operator info
Union_9 9970.00 root
├─TableReader_12 3323.33 root data:Selection_11
│ └─Selection_11 3323.33 cop[tikv] lt(test.t.ts, 2020-04-15 00:00:00.001000)
│ └─TableFullScan_10 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
├─TableReader_15 3323.33 root data:Selection_14
│ └─Selection_14 3323.33 cop[tikv] lt(test.t.ts, 2020-04-15 00:00:00.001000)
│ └─TableFullScan_13 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
└─TableReader_18 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop[tikv] lt(test.t.ts, 2020-04-15 00:00:00.001000)
└─TableFullScan_16 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
explain select * from t where ts < '2020-04-15 00:00:00' -- expect perfect : p0,p1, obtain: p0,p1,p2;
id estRows task access object operator info
Union_9 9970.00 root
├─TableReader_12 3323.33 root data:Selection_11
│ └─Selection_11 3323.33 cop[tikv] lt(test.t.ts, 2020-04-15 00:00:00.000000)
│ └─TableFullScan_10 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
├─TableReader_15 3323.33 root data:Selection_14
│ └─Selection_14 3323.33 cop[tikv] lt(test.t.ts, 2020-04-15 00:00:00.000000)
│ └─TableFullScan_13 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
└─TableReader_18 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop[tikv] lt(test.t.ts, 2020-04-15 00:00:00.000000)
└─TableFullScan_16 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
explain select * from t where ts < '2020-04-14 23:59:59.999' -- p0,p1;
id estRows task access object operator info
Union_8 6646.67 root
├─TableReader_11 3323.33 root data:Selection_10
│ └─Selection_10 3323.33 cop[tikv] lt(test.t.ts, 2020-04-14 23:59:59.999000)
│ └─TableFullScan_9 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
└─TableReader_14 3323.33 root data:Selection_13
└─Selection_13 3323.33 cop[tikv] lt(test.t.ts, 2020-04-14 23:59:59.999000)
└─TableFullScan_12 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
explain select * from t where ts < '2020-04-03 00:00:00' -- p0;
id estRows task access object operator info
TableReader_8 3323.33 root data:Selection_7
└─Selection_7 3323.33 cop[tikv] lt(test.t.ts, 2020-04-03 00:00:00.000000)
└─TableFullScan_6 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
explain select * from t where ts < '2021-05-03 00:00:00' -- p0,p1,p2;
id estRows task access object operator info
Union_9 9970.00 root
├─TableReader_12 3323.33 root data:Selection_11
│ └─Selection_11 3323.33 cop[tikv] lt(test.t.ts, 2021-05-03 00:00:00.000000)
│ └─TableFullScan_10 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
├─TableReader_15 3323.33 root data:Selection_14
│ └─Selection_14 3323.33 cop[tikv] lt(test.t.ts, 2021-05-03 00:00:00.000000)
│ └─TableFullScan_13 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
└─TableReader_18 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop[tikv] lt(test.t.ts, 2021-05-03 00:00:00.000000)
└─TableFullScan_16 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
explain select * from t where ts <= '2020-04-05 00:00:00' -- p0,p1;
id estRows task access object operator info
Union_8 6646.67 root
├─TableReader_11 3323.33 root data:Selection_10
│ └─Selection_10 3323.33 cop[tikv] le(test.t.ts, 2020-04-05 00:00:00.000000)
│ └─TableFullScan_9 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
└─TableReader_14 3323.33 root data:Selection_13
└─Selection_13 3323.33 cop[tikv] le(test.t.ts, 2020-04-05 00:00:00.000000)
└─TableFullScan_12 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
explain select * from t where ts <= '2020-04-03 00:00:00' -- p0;
id estRows task access object operator info
TableReader_8 3323.33 root data:Selection_7
└─Selection_7 3323.33 cop[tikv] le(test.t.ts, 2020-04-03 00:00:00.000000)
└─TableFullScan_6 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
explain select * from t where ts <= '2020-04-14 23:59:59.123' -- p0,p1;
id estRows task access object operator info
Union_8 6646.67 root
├─TableReader_11 3323.33 root data:Selection_10
│ └─Selection_10 3323.33 cop[tikv] le(test.t.ts, 2020-04-14 23:59:59.123000)
│ └─TableFullScan_9 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
└─TableReader_14 3323.33 root data:Selection_13
└─Selection_13 3323.33 cop[tikv] le(test.t.ts, 2020-04-14 23:59:59.123000)
└─TableFullScan_12 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
explain select * from t where ts <= '2020-04-25 00:00:00' -- p0,p1,p2;
id estRows task access object operator info
Union_9 9970.00 root
├─TableReader_12 3323.33 root data:Selection_11
│ └─Selection_11 3323.33 cop[tikv] le(test.t.ts, 2020-04-25 00:00:00.000000)
│ └─TableFullScan_10 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
├─TableReader_15 3323.33 root data:Selection_14
│ └─Selection_14 3323.33 cop[tikv] le(test.t.ts, 2020-04-25 00:00:00.000000)
│ └─TableFullScan_13 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
└─TableReader_18 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop[tikv] le(test.t.ts, 2020-04-25 00:00:00.000000)
└─TableFullScan_16 10000.00 cop[tikv] table:t, partition:p2 keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-25 00:00:00' or ts < '2020-01-02 00:00:00' -- p0;
id estRows task access object operator info
TableReader_8 6656.67 root data:Selection_7
└─Selection_7 6656.67 cop[tikv] or(gt(test.t.ts, 2020-04-25 00:00:00.000000), lt(test.t.ts, 2020-01-02 00:00:00.000000))
└─TableFullScan_6 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-02 00:00:00' and ts < '2020-04-07 00:00:00' -- p0,p1;
id estRows task access object operator info
Union_8 500.00 root
├─TableReader_11 250.00 root data:Selection_10
│ └─Selection_10 250.00 cop[tikv] gt(test.t.ts, 2020-04-02 00:00:00.000000), lt(test.t.ts, 2020-04-07 00:00:00.000000)
│ └─TableFullScan_9 10000.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo
└─TableReader_14 250.00 root data:Selection_13
└─Selection_13 250.00 cop[tikv] gt(test.t.ts, 2020-04-02 00:00:00.000000), lt(test.t.ts, 2020-04-07 00:00:00.000000)
└─TableFullScan_12 10000.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo
35 changes: 35 additions & 0 deletions cmd/explaintest/t/partition_pruning.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -976,3 +976,38 @@ PARTITION BY RANGE COLUMNS(a) (
);

desc select * from t where a = 11 and b = 1 or a = 12 and b = 1;

#
# MySQL doesn't support partition pruning for 'floor(unix_timestamp(ts))' but it works on TiDB
# https://github.com/pingcap/tidb/issues/16354
#
drop table if exists t;
create table t (ts timestamp(3) not null default current_timestamp(3))
partition by range (floor(unix_timestamp(ts))) (
partition p0 values less than (unix_timestamp('2020-04-05 00:00:00')),
partition p1 values less than (unix_timestamp('2020-04-15 00:00:00')),
partition p2 values less than (unix_timestamp('2020-04-25 00:00:00'))
);

explain select * from t where ts = '2020-04-06 00:00:00' -- p1;
explain select * from t where ts = '2020-04-05 00:00:00.001' -- p1;
explain select * from t where ts > '2020-04-15 00:00:00' -- p2;
explain select * from t where ts > '2020-04-14 23:59:59.999' -- p1,p2;
explain select * from t where ts > '2020-04-15 00:00:00.001' -- p2;
explain select * from t where ts > '2020-04-26 00:00:00.001' -- dual;
explain select * from t where ts >= '2020-04-04 12:22:32' -- p0,p1,p2;
explain select * from t where ts >= '2020-04-05 00:00:00' -- p1,p2;
explain select * from t where ts >= '2020-04-25 00:00:00' -- dual;
explain select * from t where ts < '2020-04-25 00:00:00' -- p0,p1,p2;
explain select * from t where ts < '2020-04-15 00:00:00.001' -- p0,p1,p2;
explain select * from t where ts < '2020-04-15 00:00:00' -- expect perfect : p0,p1, obtain: p0,p1,p2;
explain select * from t where ts < '2020-04-14 23:59:59.999' -- p0,p1;
explain select * from t where ts < '2020-04-03 00:00:00' -- p0;
explain select * from t where ts < '2021-05-03 00:00:00' -- p0,p1,p2;
explain select * from t where ts <= '2020-04-05 00:00:00' -- p0,p1;
explain select * from t where ts <= '2020-04-03 00:00:00' -- p0;
explain select * from t where ts <= '2020-04-14 23:59:59.123' -- p0,p1;
explain select * from t where ts <= '2020-04-25 00:00:00' -- p0,p1,p2;
explain select * from t where ts > '2020-04-25 00:00:00' or ts < '2020-01-02 00:00:00' -- p0;
explain select * from t where ts > '2020-04-02 00:00:00' and ts < '2020-04-07 00:00:00' -- p0,p1;

21 changes: 21 additions & 0 deletions planner/core/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -826,6 +826,27 @@ func (s *testIntegrationSuite) TestIssue15846(c *C) {
tk.MustQuery("SELECT t1.c0 FROM t1 LEFT JOIN t0 ON 1;").Check(testkit.Rows("0", "0"))
}

func (s *testIntegrationSuite) TestFloorUnixTimestampPruning(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists floor_unix_timestamp")
tk.MustExec(`create table floor_unix_timestamp (ts timestamp(3))
partition by range (floor(unix_timestamp(ts))) (
partition p0 values less than (unix_timestamp('2020-04-05 00:00:00')),
partition p1 values less than (unix_timestamp('2020-04-12 00:00:00')),
partition p2 values less than (unix_timestamp('2020-04-15 00:00:00')))`)
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-04 00:00:00')")
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-04 23:59:59.999')")
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-05 00:00:00')")
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-05 00:00:00.001')")
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-12 01:02:03.456')")
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-14 00:00:42')")
tk.MustQuery("select count(*) from floor_unix_timestamp where '2020-04-05 00:00:00.001' = ts").Check(testkit.Rows("1"))
tk.MustQuery("select * from floor_unix_timestamp where ts > '2020-04-05 00:00:00' order by ts").Check(testkit.Rows("2020-04-05 00:00:00.001", "2020-04-12 01:02:03.456", "2020-04-14 00:00:42.000"))
tk.MustQuery("select count(*) from floor_unix_timestamp where ts <= '2020-04-05 23:00:00'").Check(testkit.Rows("4"))
tk.MustQuery("select * from floor_unix_timestamp partition(p1, p2) where ts > '2020-04-14 00:00:00'").Check(testkit.Rows("2020-04-14 00:00:42.000"))
}

func (s *testIntegrationSuite) TestIssue16290And16292(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
Expand Down
38 changes: 34 additions & 4 deletions planner/core/rule_partition_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,19 @@ func makePartitionByFnCol(sctx sessionctx.Context, columns []*expression.Column,
var fn *expression.ScalarFunction
switch raw := partExpr.(type) {
case *expression.ScalarFunction:
// Special handle for floor(unix_timestamp(ts)) as partition expression.
// This pattern is so common for timestamp(3) column as partition expression that it deserve an optimization.
if raw.FuncName.L == ast.Floor {
if ut, ok := raw.GetArgs()[0].(*expression.ScalarFunction); ok && ut.FuncName.L == ast.UnixTimestamp {
args := ut.GetArgs()
if len(args) == 1 {
if c, ok1 := args[0].(*expression.Column); ok1 {
return c, raw, nil
}
}
}
}

if _, ok := monotoneIncFuncs[raw.FuncName.L]; ok {
fn = raw
args := fn.GetArgs()
Expand Down Expand Up @@ -507,10 +520,8 @@ func (p *rangePruner) extractDataForPrune(sctx sessionctx.Context, expr expressi
var constExpr expression.Expression
if p.partFn != nil {
// If the partition expression is fn(col), change constExpr to fn(constExpr).
// No 'copy on write' for the expression here, this is a dangerous operation.
args := p.partFn.GetArgs()
args[0] = con
constExpr = p.partFn
constExpr = replaceColumnWithConst(p.partFn, con)

// Sometimes we need to relax the condition, < to <=, > to >=.
// For example, the following case doesn't hold:
// col < '2020-02-11 17:34:11' => to_days(col) < to_days(2020-02-11 17:34:11)
Expand All @@ -529,6 +540,25 @@ func (p *rangePruner) extractDataForPrune(sctx sessionctx.Context, expr expressi
return ret, false
}

// replaceColumnWithConst change fn(col) to fn(const)
func replaceColumnWithConst(partFn *expression.ScalarFunction, con *expression.Constant) *expression.ScalarFunction {
args := partFn.GetArgs()
// The partition function may be floor(unix_timestamp(ts)) instead of a simple fn(col).
if partFn.FuncName.L == ast.Floor {
ut := args[0].(*expression.ScalarFunction)
if ut.FuncName.L == ast.UnixTimestamp {
args = ut.GetArgs()
args[0] = con
return partFn
}
}

// No 'copy on write' for the expression here, this is a dangerous operation.
args[0] = con
return partFn

}

// opposite turns > to <, >= to <= and so on.
func opposite(op string) string {
switch op {
Expand Down

0 comments on commit d8e6cf8

Please sign in to comment.