diff --git a/cmd/explaintest/r/partition_pruning.result b/cmd/explaintest/r/partition_pruning.result index 3387febe57d42..651e590accb8d 100644 --- a/cmd/explaintest/r/partition_pruning.result +++ b/cmd/explaintest/r/partition_pruning.result @@ -4218,3 +4218,177 @@ id count task operator info TableReader_8 10.00 root data:Selection_7 └─Selection_7 10.00 cop eq(test.t.name, "1") └─TableScan_6 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +drop table if exists t; +create table t (ts timestamp(3) not null default current_timestamp(3)) +partition by range (floor(unix_timestamp(ts))) ( +partition p0 values less than (unix_timestamp('2020-04-05 00:00:00')), +partition p1 values less than (unix_timestamp('2020-04-15 00:00:00')), +partition p2 values less than (unix_timestamp('2020-04-25 00:00:00')) +); +explain select * from t where ts = '2020-04-06 00:00:00' -- p1; +id count task operator info +TableReader_8 10.00 root data:Selection_7 +└─Selection_7 10.00 cop eq(test.t.ts, 2020-04-06 00:00:00.000000) + └─TableScan_6 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts = '2020-04-05 00:00:00.001' -- p1; +id count task operator info +TableReader_8 10.00 root data:Selection_7 +└─Selection_7 10.00 cop eq(test.t.ts, 2020-04-05 00:00:00.001000) + └─TableScan_6 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts > '2020-04-15 00:00:00' -- p2; +id count task operator info +TableReader_8 3333.33 root data:Selection_7 +└─Selection_7 3333.33 cop gt(test.t.ts, 2020-04-15 00:00:00.000000) + └─TableScan_6 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts > '2020-04-14 23:59:59.999' -- p1,p2; +id count task operator info +Union_8 6666.67 root +├─TableReader_11 3333.33 root data:Selection_10 +│ └─Selection_10 3333.33 cop gt(test.t.ts, 2020-04-14 23:59:59.999000) +│ └─TableScan_9 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_14 3333.33 root data:Selection_13 + └─Selection_13 3333.33 cop gt(test.t.ts, 2020-04-14 23:59:59.999000) + └─TableScan_12 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts > '2020-04-15 00:00:00.001' -- p2; +id count task operator info +TableReader_8 3333.33 root data:Selection_7 +└─Selection_7 3333.33 cop gt(test.t.ts, 2020-04-15 00:00:00.001000) + └─TableScan_6 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts > '2020-04-26 00:00:00.001' -- dual; +id count task operator info +TableDual_6 0.00 root rows:0 +explain select * from t where ts >= '2020-04-04 12:22:32' -- p0,p1,p2; +id count task operator info +Union_9 10000.00 root +├─TableReader_12 3333.33 root data:Selection_11 +│ └─Selection_11 3333.33 cop ge(test.t.ts, 2020-04-04 12:22:32.000000) +│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +├─TableReader_15 3333.33 root data:Selection_14 +│ └─Selection_14 3333.33 cop ge(test.t.ts, 2020-04-04 12:22:32.000000) +│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_18 3333.33 root data:Selection_17 + └─Selection_17 3333.33 cop ge(test.t.ts, 2020-04-04 12:22:32.000000) + └─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts >= '2020-04-05 00:00:00' -- p1,p2; +id count task operator info +Union_8 6666.67 root +├─TableReader_11 3333.33 root data:Selection_10 +│ └─Selection_10 3333.33 cop ge(test.t.ts, 2020-04-05 00:00:00.000000) +│ └─TableScan_9 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_14 3333.33 root data:Selection_13 + └─Selection_13 3333.33 cop ge(test.t.ts, 2020-04-05 00:00:00.000000) + └─TableScan_12 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts >= '2020-04-25 00:00:00' -- dual; +id count task operator info +TableDual_6 0.00 root rows:0 +explain select * from t where ts < '2020-04-25 00:00:00' -- p0,p1,p2; +id count task operator info +Union_9 9970.00 root +├─TableReader_12 3323.33 root data:Selection_11 +│ └─Selection_11 3323.33 cop lt(test.t.ts, 2020-04-25 00:00:00.000000) +│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +├─TableReader_15 3323.33 root data:Selection_14 +│ └─Selection_14 3323.33 cop lt(test.t.ts, 2020-04-25 00:00:00.000000) +│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_18 3323.33 root data:Selection_17 + └─Selection_17 3323.33 cop lt(test.t.ts, 2020-04-25 00:00:00.000000) + └─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts < '2020-04-15 00:00:00.001' -- p0,p1,p2; +id count task operator info +Union_9 9970.00 root +├─TableReader_12 3323.33 root data:Selection_11 +│ └─Selection_11 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.001000) +│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +├─TableReader_15 3323.33 root data:Selection_14 +│ └─Selection_14 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.001000) +│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_18 3323.33 root data:Selection_17 + └─Selection_17 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.001000) + └─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts < '2020-04-15 00:00:00' -- expect perfect : p0,p1, obtain: p0,p1,p2; +id count task operator info +Union_9 9970.00 root +├─TableReader_12 3323.33 root data:Selection_11 +│ └─Selection_11 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.000000) +│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +├─TableReader_15 3323.33 root data:Selection_14 +│ └─Selection_14 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.000000) +│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_18 3323.33 root data:Selection_17 + └─Selection_17 3323.33 cop lt(test.t.ts, 2020-04-15 00:00:00.000000) + └─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts < '2020-04-14 23:59:59.999' -- p0,p1; +id count task operator info +Union_8 6646.67 root +├─TableReader_11 3323.33 root data:Selection_10 +│ └─Selection_10 3323.33 cop lt(test.t.ts, 2020-04-14 23:59:59.999000) +│ └─TableScan_9 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_14 3323.33 root data:Selection_13 + └─Selection_13 3323.33 cop lt(test.t.ts, 2020-04-14 23:59:59.999000) + └─TableScan_12 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts < '2020-04-03 00:00:00' -- p0; +id count task operator info +TableReader_8 3323.33 root data:Selection_7 +└─Selection_7 3323.33 cop lt(test.t.ts, 2020-04-03 00:00:00.000000) + └─TableScan_6 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts < '2021-05-03 00:00:00' -- p0,p1,p2; +id count task operator info +Union_9 9970.00 root +├─TableReader_12 3323.33 root data:Selection_11 +│ └─Selection_11 3323.33 cop lt(test.t.ts, 2021-05-03 00:00:00.000000) +│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +├─TableReader_15 3323.33 root data:Selection_14 +│ └─Selection_14 3323.33 cop lt(test.t.ts, 2021-05-03 00:00:00.000000) +│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_18 3323.33 root data:Selection_17 + └─Selection_17 3323.33 cop lt(test.t.ts, 2021-05-03 00:00:00.000000) + └─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts <= '2020-04-05 00:00:00' -- p0,p1; +id count task operator info +Union_8 6646.67 root +├─TableReader_11 3323.33 root data:Selection_10 +│ └─Selection_10 3323.33 cop le(test.t.ts, 2020-04-05 00:00:00.000000) +│ └─TableScan_9 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_14 3323.33 root data:Selection_13 + └─Selection_13 3323.33 cop le(test.t.ts, 2020-04-05 00:00:00.000000) + └─TableScan_12 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts <= '2020-04-03 00:00:00' -- p0; +id count task operator info +TableReader_8 3323.33 root data:Selection_7 +└─Selection_7 3323.33 cop le(test.t.ts, 2020-04-03 00:00:00.000000) + └─TableScan_6 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts <= '2020-04-14 23:59:59.123' -- p0,p1; +id count task operator info +Union_8 6646.67 root +├─TableReader_11 3323.33 root data:Selection_10 +│ └─Selection_10 3323.33 cop le(test.t.ts, 2020-04-14 23:59:59.123000) +│ └─TableScan_9 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_14 3323.33 root data:Selection_13 + └─Selection_13 3323.33 cop le(test.t.ts, 2020-04-14 23:59:59.123000) + └─TableScan_12 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts <= '2020-04-25 00:00:00' -- p0,p1,p2; +id count task operator info +Union_9 9970.00 root +├─TableReader_12 3323.33 root data:Selection_11 +│ └─Selection_11 3323.33 cop le(test.t.ts, 2020-04-25 00:00:00.000000) +│ └─TableScan_10 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +├─TableReader_15 3323.33 root data:Selection_14 +│ └─Selection_14 3323.33 cop le(test.t.ts, 2020-04-25 00:00:00.000000) +│ └─TableScan_13 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_18 3323.33 root data:Selection_17 + └─Selection_17 3323.33 cop le(test.t.ts, 2020-04-25 00:00:00.000000) + └─TableScan_16 10000.00 cop table:t, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts > '2020-04-25 00:00:00' or ts < '2020-01-02 00:00:00' -- p0; +id count task operator info +TableReader_8 6656.67 root data:Selection_7 +└─Selection_7 6656.67 cop or(gt(test.t.ts, 2020-04-25 00:00:00.000000), lt(test.t.ts, 2020-01-02 00:00:00.000000)) + └─TableScan_6 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +explain select * from t where ts > '2020-04-02 00:00:00' and ts < '2020-04-07 00:00:00' -- p0,p1; +id count task operator info +Union_8 500.00 root +├─TableReader_11 250.00 root data:Selection_10 +│ └─Selection_10 250.00 cop gt(test.t.ts, 2020-04-02 00:00:00.000000), lt(test.t.ts, 2020-04-07 00:00:00.000000) +│ └─TableScan_9 10000.00 cop table:t, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_14 250.00 root data:Selection_13 + └─Selection_13 250.00 cop gt(test.t.ts, 2020-04-02 00:00:00.000000), lt(test.t.ts, 2020-04-07 00:00:00.000000) + └─TableScan_12 10000.00 cop table:t, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo diff --git a/cmd/explaintest/t/partition_pruning.test b/cmd/explaintest/t/partition_pruning.test index ce57050338df0..f24b8358d7d4b 100644 --- a/cmd/explaintest/t/partition_pruning.test +++ b/cmd/explaintest/t/partition_pruning.test @@ -980,3 +980,37 @@ desc select * from t where a = 11 and b = 1 or a = 12 and b = 1; drop table if exists t; create table t (id int, name varchar(20)) partition by hash(id) partitions 128; explain SELECT * FROM t partition (p1) where name = '1'; + +# +# MySQL doesn't support partition pruning for 'floor(unix_timestamp(ts))' but it works on TiDB +# https://github.com/pingcap/tidb/issues/16354 +# +drop table if exists t; +create table t (ts timestamp(3) not null default current_timestamp(3)) +partition by range (floor(unix_timestamp(ts))) ( + partition p0 values less than (unix_timestamp('2020-04-05 00:00:00')), + partition p1 values less than (unix_timestamp('2020-04-15 00:00:00')), + partition p2 values less than (unix_timestamp('2020-04-25 00:00:00')) +); + +explain select * from t where ts = '2020-04-06 00:00:00' -- p1; +explain select * from t where ts = '2020-04-05 00:00:00.001' -- p1; +explain select * from t where ts > '2020-04-15 00:00:00' -- p2; +explain select * from t where ts > '2020-04-14 23:59:59.999' -- p1,p2; +explain select * from t where ts > '2020-04-15 00:00:00.001' -- p2; +explain select * from t where ts > '2020-04-26 00:00:00.001' -- dual; +explain select * from t where ts >= '2020-04-04 12:22:32' -- p0,p1,p2; +explain select * from t where ts >= '2020-04-05 00:00:00' -- p1,p2; +explain select * from t where ts >= '2020-04-25 00:00:00' -- dual; +explain select * from t where ts < '2020-04-25 00:00:00' -- p0,p1,p2; +explain select * from t where ts < '2020-04-15 00:00:00.001' -- p0,p1,p2; +explain select * from t where ts < '2020-04-15 00:00:00' -- expect perfect : p0,p1, obtain: p0,p1,p2; +explain select * from t where ts < '2020-04-14 23:59:59.999' -- p0,p1; +explain select * from t where ts < '2020-04-03 00:00:00' -- p0; +explain select * from t where ts < '2021-05-03 00:00:00' -- p0,p1,p2; +explain select * from t where ts <= '2020-04-05 00:00:00' -- p0,p1; +explain select * from t where ts <= '2020-04-03 00:00:00' -- p0; +explain select * from t where ts <= '2020-04-14 23:59:59.123' -- p0,p1; +explain select * from t where ts <= '2020-04-25 00:00:00' -- p0,p1,p2; +explain select * from t where ts > '2020-04-25 00:00:00' or ts < '2020-01-02 00:00:00' -- p0; +explain select * from t where ts > '2020-04-02 00:00:00' and ts < '2020-04-07 00:00:00' -- p0,p1; diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index aee00881c5479..e39f35609883d 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -331,3 +331,24 @@ func (s *testIntegrationSuite) TestIssue16440(c *C) { tk.MustQuery("SELECT t1.c0 FROM t1 WHERE NOT t1.c0;").Check(testkit.Rows()) tk.MustExec("drop table t1") } + +func (s *testIntegrationSuite) TestFloorUnixTimestampPruning(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists floor_unix_timestamp") + tk.MustExec(`create table floor_unix_timestamp (ts timestamp(3)) +partition by range (floor(unix_timestamp(ts))) ( +partition p0 values less than (unix_timestamp('2020-04-05 00:00:00')), +partition p1 values less than (unix_timestamp('2020-04-12 00:00:00')), +partition p2 values less than (unix_timestamp('2020-04-15 00:00:00')))`) + tk.MustExec("insert into floor_unix_timestamp values ('2020-04-04 00:00:00')") + tk.MustExec("insert into floor_unix_timestamp values ('2020-04-04 23:59:59.999')") + tk.MustExec("insert into floor_unix_timestamp values ('2020-04-05 00:00:00')") + tk.MustExec("insert into floor_unix_timestamp values ('2020-04-05 00:00:00.001')") + tk.MustExec("insert into floor_unix_timestamp values ('2020-04-12 01:02:03.456')") + tk.MustExec("insert into floor_unix_timestamp values ('2020-04-14 00:00:42')") + tk.MustQuery("select count(*) from floor_unix_timestamp where '2020-04-05 00:00:00.001' = ts").Check(testkit.Rows("1")) + tk.MustQuery("select * from floor_unix_timestamp where ts > '2020-04-05 00:00:00' order by ts").Check(testkit.Rows("2020-04-05 00:00:00.001", "2020-04-12 01:02:03.456", "2020-04-14 00:00:42.000")) + tk.MustQuery("select count(*) from floor_unix_timestamp where ts <= '2020-04-05 23:00:00'").Check(testkit.Rows("4")) + tk.MustQuery("select * from floor_unix_timestamp partition(p1, p2) where ts > '2020-04-14 00:00:00'").Check(testkit.Rows("2020-04-14 00:00:42.000")) +} diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 5402babdd90c1..969773f88477f 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -517,6 +517,19 @@ func makePartitionByFnCol(ds *DataSource, partitionExpr string) (*expression.Col var fn *expression.ScalarFunction switch raw := partExpr.(type) { case *expression.ScalarFunction: + // Special handle for floor(unix_timestamp(ts)) as partition expression. + // This pattern is so common for timestamp(3) column as partition expression that it deserve an optimization. + if raw.FuncName.L == ast.Floor { + if ut, ok := raw.GetArgs()[0].(*expression.ScalarFunction); ok && ut.FuncName.L == ast.UnixTimestamp { + args := ut.GetArgs() + if len(args) == 1 { + if c, ok1 := args[0].(*expression.Column); ok1 { + return c, raw, nil + } + } + } + } + if _, ok := monotoneIncFuncs[raw.FuncName.L]; ok { fn = raw col = fn.GetArgs()[0].(*expression.Column) @@ -622,10 +635,8 @@ func extractDataForPrune(sctx sessionctx.Context, expr expression.Expression, pa var constExpr expression.Expression if partFn != nil { // If the partition expression is fn(col), change constExpr to fn(constExpr). - // No 'copy on write' for the expression here, this is a dangerous operation. - args := partFn.GetArgs() - args[0] = con - constExpr = partFn + constExpr = replaceColumnWithConst(partFn, con) + // Sometimes we need to relax the condition, < to <=, > to >=. // For example, the following case doesn't hold: // col < '2020-02-11 17:34:11' => to_days(col) < to_days(2020-02-11 17:34:11) @@ -644,6 +655,25 @@ func extractDataForPrune(sctx sessionctx.Context, expr expression.Expression, pa return ret, false } +// replaceColumnWithConst change fn(col) to fn(const) +func replaceColumnWithConst(partFn *expression.ScalarFunction, con *expression.Constant) *expression.ScalarFunction { + args := partFn.GetArgs() + // The partition function may be floor(unix_timestamp(ts)) instead of a simple fn(col). + if partFn.FuncName.L == ast.Floor { + ut := args[0].(*expression.ScalarFunction) + if ut.FuncName.L == ast.UnixTimestamp { + args = ut.GetArgs() + args[0] = con + return partFn + } + } + + // No 'copy on write' for the expression here, this is a dangerous operation. + args[0] = con + return partFn + +} + // opposite turns > to <, >= to <= and so on. func opposite(op string) string { switch op {