Skip to content

Commit

Permalink
planner/core: support partition pruning for partition expression floo…
Browse files Browse the repository at this point in the history 
…r(unix_timestamp()) (pingcap#16402)
  • Loading branch information
@tiancaiamao
tiancaiamao committed Apr 19, 2020
1 parent 0c67b23 commit bd0ed21
Show file tree
Hide file tree
Showing 4 changed files with 263 additions and 4 deletions.
174 changes: 174 additions & 0 deletions cmd/explaintest/r/partition_pruning.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4218,3 +4218,177 @@ id count task operator info
TableReader_8 10.00 root data:Selection_7
└─Selection_7 10.00 cop eq(test.t.name, "1")
└─TableScan_6 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
drop table if exists t;
create table t (ts timestamp(3) not null default current_timestamp(3))
partition by range (floor(unix_timestamp(ts))) (
partition p0 values less than (unix_timestamp('2020-04-05 00:00:00')),
partition p1 values less than (unix_timestamp('2020-04-15 00:00:00')),
partition p2 values less than (unix_timestamp('2020-04-25 00:00:00'))
);
explain select * from t where ts = '2020-04-06 00:00:00' -- p1;
id count task operator info
TableReader_8 10.00 root data:Selection_7
└─Selection_7 10.00 cop eq(test.t.ts, 2020-04-06 00:00:00.000000)
└─TableScan_6 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts = '2020-04-05 00:00:00.001' -- p1;
id count task operator info
TableReader_8 10.00 root data:Selection_7
└─Selection_7 10.00 cop eq(test.t.ts, 2020-04-05 00:00:00.001000)
└─TableScan_6 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-15 00:00:00' -- p2;
id count task operator info
TableReader_8 3333.33 root data:Selection_7
└─Selection_7 3333.33 cop gt(test.t.ts, 2020-04-15 00:00:00.000000)
└─TableScan_6 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-14 23:59:59.999' -- p1,p2;
id count task operator info
Union_8 6666.67 root
├─TableReader_11 3333.33 root data:Selection_10
│ └─Selection_10 3333.33 cop gt(test.t.ts, 2020-04-14 23:59:59.999000)
│ └─TableScan_9 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_14 3333.33 root data:Selection_13
└─Selection_13 3333.33 cop gt(test.t.ts, 2020-04-14 23:59:59.999000)
└─TableScan_12 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-15 00:00:00.001' -- p2;
id count task operator info
TableReader_8 3333.33 root data:Selection_7
└─Selection_7 3333.33 cop gt(test.t.ts, 2020-04-15 00:00:00.001000)
└─TableScan_6 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-26 00:00:00.001' -- dual;
id count task operator info
TableDual_6 0.00 root rows:0
explain select * from t where ts >= '2020-04-04 12:22:32' -- p0,p1,p2;
id count task operator info
Union_9 10000.00 root
├─TableReader_12 3333.33 root data:Selection_11
│ └─Selection_11 3333.33 cop ge(test.t.ts, 2020-04-04 12:22:32.000000)
│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
├─TableReader_15 3333.33 root data:Selection_14
│ └─Selection_14 3333.33 cop ge(test.t.ts, 2020-04-04 12:22:32.000000)
│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_18 3333.33 root data:Selection_17
└─Selection_17 3333.33 cop ge(test.t.ts, 2020-04-04 12:22:32.000000)
└─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts >= '2020-04-05 00:00:00' -- p1,p2;
id count task operator info
Union_8 6666.67 root
├─TableReader_11 3333.33 root data:Selection_10
│ └─Selection_10 3333.33 cop ge(test.t.ts, 2020-04-05 00:00:00.000000)
│ └─TableScan_9 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_14 3333.33 root data:Selection_13
└─Selection_13 3333.33 cop ge(test.t.ts, 2020-04-05 00:00:00.000000)
└─TableScan_12 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts >= '2020-04-25 00:00:00' -- dual;
id count task operator info
TableDual_6 0.00 root rows:0
explain select * from t where ts < '2020-04-25 00:00:00' -- p0,p1,p2;
id count task operator info
Union_9 9970.00 root
├─TableReader_12 3323.33 root data:Selection_11
│ └─Selection_11 3323.33 cop lt(test.t.ts, 2020-04-25 00:00:00.000000)
│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
├─TableReader_15 3323.33 root data:Selection_14
│ └─Selection_14 3323.33 cop lt(test.t.ts, 2020-04-25 00:00:00.000000)
│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_18 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop lt(test.t.ts, 2020-04-25 00:00:00.000000)
└─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts < '2020-04-15 00:00:00.001' -- p0,p1,p2;
id count task operator info
Union_9 9970.00 root
├─TableReader_12 3323.33 root data:Selection_11
│ └─Selection_11 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.001000)
│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
├─TableReader_15 3323.33 root data:Selection_14
│ └─Selection_14 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.001000)
│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_18 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.001000)
└─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts < '2020-04-15 00:00:00' -- expect perfect : p0,p1, obtain: p0,p1,p2;
id count task operator info
Union_9 9970.00 root
├─TableReader_12 3323.33 root data:Selection_11
│ └─Selection_11 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.000000)
│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
├─TableReader_15 3323.33 root data:Selection_14
│ └─Selection_14 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.000000)
│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_18 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.000000)
└─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts < '2020-04-14 23:59:59.999' -- p0,p1;
id count task operator info
Union_8 6646.67 root
├─TableReader_11 3323.33 root data:Selection_10
│ └─Selection_10 3323.33 cop lt(test.t.ts, 2020-04-14 23:59:59.999000)
│ └─TableScan_9 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_14 3323.33 root data:Selection_13
└─Selection_13 3323.33 cop lt(test.t.ts, 2020-04-14 23:59:59.999000)
└─TableScan_12 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts < '2020-04-03 00:00:00' -- p0;
id count task operator info
TableReader_8 3323.33 root data:Selection_7
└─Selection_7 3323.33 cop lt(test.t.ts, 2020-04-03 00:00:00.000000)
└─TableScan_6 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts < '2021-05-03 00:00:00' -- p0,p1,p2;
id count task operator info
Union_9 9970.00 root
├─TableReader_12 3323.33 root data:Selection_11
│ └─Selection_11 3323.33 cop lt(test.t.ts, 2021-05-03 00:00:00.000000)
│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
├─TableReader_15 3323.33 root data:Selection_14
│ └─Selection_14 3323.33 cop lt(test.t.ts, 2021-05-03 00:00:00.000000)
│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_18 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop lt(test.t.ts, 2021-05-03 00:00:00.000000)
└─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts <= '2020-04-05 00:00:00' -- p0,p1;
id count task operator info
Union_8 6646.67 root
├─TableReader_11 3323.33 root data:Selection_10
│ └─Selection_10 3323.33 cop le(test.t.ts, 2020-04-05 00:00:00.000000)
│ └─TableScan_9 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_14 3323.33 root data:Selection_13
└─Selection_13 3323.33 cop le(test.t.ts, 2020-04-05 00:00:00.000000)
└─TableScan_12 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts <= '2020-04-03 00:00:00' -- p0;
id count task operator info
TableReader_8 3323.33 root data:Selection_7
└─Selection_7 3323.33 cop le(test.t.ts, 2020-04-03 00:00:00.000000)
└─TableScan_6 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts <= '2020-04-14 23:59:59.123' -- p0,p1;
id count task operator info
Union_8 6646.67 root
├─TableReader_11 3323.33 root data:Selection_10
│ └─Selection_10 3323.33 cop le(test.t.ts, 2020-04-14 23:59:59.123000)
│ └─TableScan_9 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_14 3323.33 root data:Selection_13
└─Selection_13 3323.33 cop le(test.t.ts, 2020-04-14 23:59:59.123000)
└─TableScan_12 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts <= '2020-04-25 00:00:00' -- p0,p1,p2;
id count task operator info
Union_9 9970.00 root
├─TableReader_12 3323.33 root data:Selection_11
│ └─Selection_11 3323.33 cop le(test.t.ts, 2020-04-25 00:00:00.000000)
│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
├─TableReader_15 3323.33 root data:Selection_14
│ └─Selection_14 3323.33 cop le(test.t.ts, 2020-04-25 00:00:00.000000)
│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_18 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop le(test.t.ts, 2020-04-25 00:00:00.000000)
└─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-25 00:00:00' or ts < '2020-01-02 00:00:00' -- p0;
id count task operator info
TableReader_8 6656.67 root data:Selection_7
└─Selection_7 6656.67 cop or(gt(test.t.ts, 2020-04-25 00:00:00.000000), lt(test.t.ts, 2020-01-02 00:00:00.000000))
└─TableScan_6 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
explain select * from t where ts > '2020-04-02 00:00:00' and ts < '2020-04-07 00:00:00' -- p0,p1;
id count task operator info
Union_8 500.00 root
├─TableReader_11 250.00 root data:Selection_10
│ └─Selection_10 250.00 cop gt(test.t.ts, 2020-04-02 00:00:00.000000), lt(test.t.ts, 2020-04-07 00:00:00.000000)
│ └─TableScan_9 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_14 250.00 root data:Selection_13
└─Selection_13 250.00 cop gt(test.t.ts, 2020-04-02 00:00:00.000000), lt(test.t.ts, 2020-04-07 00:00:00.000000)
└─TableScan_12 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo
34 changes: 34 additions & 0 deletions cmd/explaintest/t/partition_pruning.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -980,3 +980,37 @@ desc select * from t where a = 11 and b = 1 or a = 12 and b = 1;
drop table if exists t;
create table t (id int, name varchar(20)) partition by hash(id) partitions 128;
explain SELECT * FROM t partition (p1) where name = '1';

#
# MySQL doesn't support partition pruning for 'floor(unix_timestamp(ts))' but it works on TiDB
# https://github.com/pingcap/tidb/issues/16354
#
drop table if exists t;
create table t (ts timestamp(3) not null default current_timestamp(3))
partition by range (floor(unix_timestamp(ts))) (
partition p0 values less than (unix_timestamp('2020-04-05 00:00:00')),
partition p1 values less than (unix_timestamp('2020-04-15 00:00:00')),
partition p2 values less than (unix_timestamp('2020-04-25 00:00:00'))
);

explain select * from t where ts = '2020-04-06 00:00:00' -- p1;
explain select * from t where ts = '2020-04-05 00:00:00.001' -- p1;
explain select * from t where ts > '2020-04-15 00:00:00' -- p2;
explain select * from t where ts > '2020-04-14 23:59:59.999' -- p1,p2;
explain select * from t where ts > '2020-04-15 00:00:00.001' -- p2;
explain select * from t where ts > '2020-04-26 00:00:00.001' -- dual;
explain select * from t where ts >= '2020-04-04 12:22:32' -- p0,p1,p2;
explain select * from t where ts >= '2020-04-05 00:00:00' -- p1,p2;
explain select * from t where ts >= '2020-04-25 00:00:00' -- dual;
explain select * from t where ts < '2020-04-25 00:00:00' -- p0,p1,p2;
explain select * from t where ts < '2020-04-15 00:00:00.001' -- p0,p1,p2;
explain select * from t where ts < '2020-04-15 00:00:00' -- expect perfect : p0,p1, obtain: p0,p1,p2;
explain select * from t where ts < '2020-04-14 23:59:59.999' -- p0,p1;
explain select * from t where ts < '2020-04-03 00:00:00' -- p0;
explain select * from t where ts < '2021-05-03 00:00:00' -- p0,p1,p2;
explain select * from t where ts <= '2020-04-05 00:00:00' -- p0,p1;
explain select * from t where ts <= '2020-04-03 00:00:00' -- p0;
explain select * from t where ts <= '2020-04-14 23:59:59.123' -- p0,p1;
explain select * from t where ts <= '2020-04-25 00:00:00' -- p0,p1,p2;
explain select * from t where ts > '2020-04-25 00:00:00' or ts < '2020-01-02 00:00:00' -- p0;
explain select * from t where ts > '2020-04-02 00:00:00' and ts < '2020-04-07 00:00:00' -- p0,p1;
21 changes: 21 additions & 0 deletions planner/core/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,3 +331,24 @@ func (s *testIntegrationSuite) TestIssue16440(c *C) {
tk.MustQuery("SELECT t1.c0 FROM t1 WHERE NOT t1.c0;").Check(testkit.Rows())
tk.MustExec("drop table t1")
}

func (s *testIntegrationSuite) TestFloorUnixTimestampPruning(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists floor_unix_timestamp")
tk.MustExec(`create table floor_unix_timestamp (ts timestamp(3))
partition by range (floor(unix_timestamp(ts))) (
partition p0 values less than (unix_timestamp('2020-04-05 00:00:00')),
partition p1 values less than (unix_timestamp('2020-04-12 00:00:00')),
partition p2 values less than (unix_timestamp('2020-04-15 00:00:00')))`)
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-04 00:00:00')")
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-04 23:59:59.999')")
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-05 00:00:00')")
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-05 00:00:00.001')")
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-12 01:02:03.456')")
tk.MustExec("insert into floor_unix_timestamp values ('2020-04-14 00:00:42')")
tk.MustQuery("select count(*) from floor_unix_timestamp where '2020-04-05 00:00:00.001' = ts").Check(testkit.Rows("1"))
tk.MustQuery("select * from floor_unix_timestamp where ts > '2020-04-05 00:00:00' order by ts").Check(testkit.Rows("2020-04-05 00:00:00.001", "2020-04-12 01:02:03.456", "2020-04-14 00:00:42.000"))
tk.MustQuery("select count(*) from floor_unix_timestamp where ts <= '2020-04-05 23:00:00'").Check(testkit.Rows("4"))
tk.MustQuery("select * from floor_unix_timestamp partition(p1, p2) where ts > '2020-04-14 00:00:00'").Check(testkit.Rows("2020-04-14 00:00:42.000"))
}
38 changes: 34 additions & 4 deletions planner/core/rule_partition_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -517,6 +517,19 @@ func makePartitionByFnCol(ds *DataSource, partitionExpr string) (*expression.Col
var fn *expression.ScalarFunction
switch raw := partExpr.(type) {
case *expression.ScalarFunction:
// Special handle for floor(unix_timestamp(ts)) as partition expression.
// This pattern is so common for timestamp(3) column as partition expression that it deserve an optimization.
if raw.FuncName.L == ast.Floor {
if ut, ok := raw.GetArgs()[0].(*expression.ScalarFunction); ok && ut.FuncName.L == ast.UnixTimestamp {
args := ut.GetArgs()
if len(args) == 1 {
if c, ok1 := args[0].(*expression.Column); ok1 {
return c, raw, nil
}
}
}
}

if _, ok := monotoneIncFuncs[raw.FuncName.L]; ok {
fn = raw
col = fn.GetArgs()[0].(*expression.Column)
Expand Down Expand Up @@ -622,10 +635,8 @@ func extractDataForPrune(sctx sessionctx.Context, expr expression.Expression, pa
var constExpr expression.Expression
if partFn != nil {
// If the partition expression is fn(col), change constExpr to fn(constExpr).
// No 'copy on write' for the expression here, this is a dangerous operation.
args := partFn.GetArgs()
args[0] = con
constExpr = partFn
constExpr = replaceColumnWithConst(partFn, con)

// Sometimes we need to relax the condition, < to <=, > to >=.
// For example, the following case doesn't hold:
// col < '2020-02-11 17:34:11' => to_days(col) < to_days(2020-02-11 17:34:11)
Expand All @@ -644,6 +655,25 @@ func extractDataForPrune(sctx sessionctx.Context, expr expression.Expression, pa
return ret, false
}

// replaceColumnWithConst change fn(col) to fn(const)
func replaceColumnWithConst(partFn *expression.ScalarFunction, con *expression.Constant) *expression.ScalarFunction {
args := partFn.GetArgs()
// The partition function may be floor(unix_timestamp(ts)) instead of a simple fn(col).
if partFn.FuncName.L == ast.Floor {
ut := args[0].(*expression.ScalarFunction)
if ut.FuncName.L == ast.UnixTimestamp {
args = ut.GetArgs()
args[0] = con
return partFn
}
}

// No 'copy on write' for the expression here, this is a dangerous operation.
args[0] = con
return partFn

}

// opposite turns > to <, >= to <= and so on.
func opposite(op string) string {
switch op {
Expand Down

0 comments on commit bd0ed21

Please sign in to comment.