ranger: merge multiple EQ or In expressions if possible

cmd/explaintest/r/explain_easy.result

@@ -326,3 +326,29 @@ id	count	task	operator info
 Projection_3	10000.00	root	0
 └─TableReader_5	10000.00	root	data:TableScan_4
   └─TableScan_4	10000.00	cop	table:t, range:[-inf,+inf], keep order:false, stats:pseudo
+drop table if exists t;
+create table t(a bigint, b bigint, index idx(a, b));
+explain select * from t where a in (1, 2) and a in (1, 3);
+id	count	task	operator info
+IndexReader_9	10.00	root	index:IndexScan_8
+└─IndexScan_8	10.00	cop	table:t, index:a, b, range:[1,1], keep order:false, stats:pseudo
+explain select * from t where b in (1, 2) and b in (1, 3);
+id	count	task	operator info
+TableReader_7	10.00	root	data:Selection_6
+└─Selection_6	10.00	cop	in(test.t.b, 1, 2), in(test.t.b, 1, 3)
+  └─TableScan_5	10000.00	cop	table:t, range:[-inf,+inf], keep order:false, stats:pseudo
+explain select * from t where a = 1 and a = 1;
+id	count	task	operator info
+IndexReader_9	10.00	root	index:IndexScan_8
+└─IndexScan_8	10.00	cop	table:t, index:a, b, range:[1,1], keep order:false, stats:pseudo
+explain select * from t where a = 1 and a = 2;
+id	count	task	operator info
+TableDual_5	0.00	root	rows:0
+explain select * from t where b = 1 and b = 2;
+id	count	task	operator info
+TableDual_5	0.00	root	rows:0
+drop table if exists t;
+create table t(a bigint primary key);
+explain select * from t where a = 1 and a = 2;
+id	count	task	operator info
+TableDual_5	0.00	root	rows:0

cmd/explaintest/t/explain_easy.test

@@ -58,3 +58,15 @@ explain select t.a = '123455' from t;
 explain select t.a > '123455' from t;
 explain select t.a != '123455' from t;
 explain select t.a = 12345678912345678998789678687678.111 from t;
+
+drop table if exists t;
+create table t(a bigint, b bigint, index idx(a, b));
+explain select * from t where a in (1, 2) and a in (1, 3);
+explain select * from t where b in (1, 2) and b in (1, 3);
+explain select * from t where a = 1 and a = 1;
+explain select * from t where a = 1 and a = 2;
+explain select * from t where b = 1 and b = 2;
+
+drop table if exists t;
+create table t(a bigint primary key);
+explain select * from t where a = 1 and a = 2;

plan/cbo_test.go

@@ -553,7 +553,7 @@ func (s *testAnalyzeSuite) TestPreparedNullParam(c *C) {
 		testKit.MustExec("insert into t values (1), (2), (3)")
 
 		sql := "select * from t where id = ?"
-		best := "IndexReader(Index(t.id)[])"
+		best := "Dual"
 
 		ctx := testKit.Se.(sessionctx.Context)
 		stmts, err := session.Parse(ctx, sql)

plan/find_best_task.go

@@ -252,6 +252,14 @@ func (ds *DataSource) findBestTask(prop *requiredProp) (t task, err error) {
 	t = invalidTask
 
 	for _, path := range ds.possibleAccessPaths {
+		// if we already know the range of the scan is empty, just return a TableDual
+		if len(path.ranges) == 0 && !ds.ctx.GetSessionVars().StmtCtx.UseCache {
+			dual := PhysicalTableDual{}.init(ds.ctx, ds.stats)
+			dual.SetSchema(ds.schema)
+			return &rootTask{
+				p: dual,
+			}, nil
+		}
 		if path.isTablePath {
 			tblTask, err := ds.convertToTableScan(prop, path)
 			if err != nil {

plan/physical_plan_test.go

@@ -72,7 +72,7 @@ func (s *testPlanSuite) TestDAGPlanBuilderSimpleCase(c *C) {
 		},
 		{
 			sql:  "select * from t where (t.c > 0 and t.c < 1) or (t.c > 2 and t.c < 3) or (t.c > 4 and t.c < 5) or (t.c > 6 and t.c < 7) or (t.c > 9 and t.c < 10)",
-			best: "IndexLookUp(Index(t.c_d_e)[], Table(t))",
+			best: "Dual",
 		},
 		// Test TopN to table branch in double read.
 		{
@@ -87,7 +87,7 @@ func (s *testPlanSuite) TestDAGPlanBuilderSimpleCase(c *C) {
 		// Test Null Range but the column has not null flag.
 		{
 			sql:  "select * from t where t.c is null",
-			best: "IndexLookUp(Index(t.c_d_e)[], Table(t))",
+			best: "Dual",
 		},
 		// Test TopN to index branch in double read.
 		{
@@ -1029,7 +1029,7 @@ func (s *testPlanSuite) TestRefine(c *C) {
 		},
 		{
 			sql:  "select a from t where c in (1, 2, 3) and (d > 3 and d < 4 or d > 5 and d < 6)",
-			best: "IndexReader(Index(t.c_d_e)[])->Projection",
+			best: "Dual->Projection",
 		},
 		{
 			sql:  "select a from t where c in (1, 2, 3) and (d > 2 and d < 4 or d > 5 and d < 7)",

plan/stats.go

@@ -147,8 +147,8 @@ func (ds *DataSource) deriveStats() (*statsInfo, error) {
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
-			// If there's only point range. Just remove other possible paths.
-			if noIntervalRanges {
+			// If we have point or empty range, just remove other possible paths.
+			if noIntervalRanges || len(path.ranges) == 0 {
 				ds.possibleAccessPaths[0] = path
 				ds.possibleAccessPaths = ds.possibleAccessPaths[:1]
 				break
@@ -159,8 +159,8 @@ func (ds *DataSource) deriveStats() (*statsInfo, error) {
 		if err != nil {
 			return nil, errors.Trace(err)
 		}
-		// If there's only point range and this index is unique key. Just remove other possible paths.
-		if noIntervalRanges && path.index.Unique {
+		// If we have empty range, or point range on unique index, just remove other possible paths.
+		if (noIntervalRanges && path.index.Unique) || len(path.ranges) == 0 {
 			ds.possibleAccessPaths[0] = path
 			ds.possibleAccessPaths = ds.possibleAccessPaths[:1]
 			break

util/ranger/detacher.go

@@ -146,19 +146,22 @@ func detachCNFCondAndBuildRangeForIndex(sctx sessionctx.Context, conditions []ex
 		err     error
 	)
 
-	accessConds, filterConds := extractEqAndInCondition(conditions, cols, lengths)
+	accessConds, filterConds, newConditions, emptyRange := extractEqAndInCondition(sctx, conditions, cols, lengths)
+	if emptyRange {
+		return ranges, nil, nil, 0, nil
+	}
 
 	for ; eqCount < len(accessConds); eqCount++ {
 		if accessConds[eqCount].(*expression.ScalarFunction).FuncName.L != ast.EQ {
 			break
 		}
 	}
 	// We should remove all accessConds, so that they will not be added to filter conditions.
-	conditions = removeAccessConditions(conditions, accessConds)
+	newConditions = removeAccessConditions(newConditions, accessConds)
 	eqOrInCount := len(accessConds)
 	if eqOrInCount == len(cols) {
 		// If curIndex equals to len of index columns, it means the rest conditions haven't been appended to filter conditions.
-		filterConds = append(filterConds, conditions...)
+		filterConds = append(filterConds, newConditions...)
 		ranges, err = buildCNFIndexRange(sctx.GetSessionVars().StmtCtx, cols, tpSlice, lengths, eqOrInCount, accessConds)
 		if err != nil {
 			return nil, nil, nil, 0, errors.Trace(err)
@@ -171,11 +174,11 @@ func detachCNFCondAndBuildRangeForIndex(sctx sessionctx.Context, conditions []ex
 		shouldReserve: lengths[eqOrInCount] != types.UnspecifiedLength,
 	}
 	if considerDNF {
-		accesses, filters := detachColumnCNFConditions(sctx, conditions, checker)
+		accesses, filters := detachColumnCNFConditions(sctx, newConditions, checker)
 		accessConds = append(accessConds, accesses...)
 		filterConds = append(filterConds, filters...)
 	} else {
-		for _, cond := range conditions {
+		for _, cond := range newConditions {
 			if !checker.check(cond) {
 				filterConds = append(filterConds, cond)
 				continue
@@ -187,14 +190,45 @@ func detachCNFCondAndBuildRangeForIndex(sctx sessionctx.Context, conditions []ex
 	return ranges, accessConds, filterConds, eqCount, errors.Trace(err)
 }
 
-func extractEqAndInCondition(conditions []expression.Expression, cols []*expression.Column,
-	lengths []int) (accesses, filters []expression.Expression) {
-	accesses = make([]expression.Expression, len(cols))
+func extractEqAndInCondition(sctx sessionctx.Context, conditions []expression.Expression,
+	cols []*expression.Column, lengths []int) ([]expression.Expression, []expression.Expression, []expression.Expression, bool) {
+	var filters []expression.Expression
+	rb := builder{sc: sctx.GetSessionVars().StmtCtx}
+	accesses := make([]expression.Expression, len(cols))
+	points := make([][]point, len(cols))
+	mergedAccesses := make([]expression.Expression, len(cols))
+	newConditions := make([]expression.Expression, 0, len(conditions))
 	for _, cond := range conditions {
 		offset := getEqOrInColOffset(cond, cols)
-		if offset != -1 {
+		if offset == -1 {
+			newConditions = append(newConditions, cond)
+			continue
+		}
+		if accesses[offset] == nil {
 			accesses[offset] = cond
+			continue
+		}
+		// Multiple Eq/In conditions for one column in CNF, apply intersection on them
+		// Lazily compute the points for the previously visited Eq/In
+		if mergedAccesses[offset] == nil {
+			mergedAccesses[offset] = accesses[offset]
+			points[offset] = rb.build(accesses[offset])
+		}
+		points[offset] = rb.intersection(points[offset], rb.build(cond))
+		// Early termination if false expression found
+		if len(points[offset]) == 0 {
+			return nil, nil, nil, true
+		}
+	}
+	for i, ma := range mergedAccesses {
+		if ma == nil {
+			if accesses[i] != nil {
+				newConditions = append(newConditions, accesses[i])
+			}
+			continue
 		}
+		accesses[i] = points2EqOrInCond(sctx, points[i], mergedAccesses[i])
+		newConditions = append(newConditions, accesses[i])
 	}
 	for i, cond := range accesses {
 		if cond == nil {
@@ -205,7 +239,7 @@ func extractEqAndInCondition(conditions []expression.Expression, cols []*express
 			filters = append(filters, cond)
 		}
 	}
-	return accesses, filters
+	return accesses, filters, newConditions, false
 }
 
 // detachDNFCondAndBuildRangeForIndex will detach the index filters from table filters when it's a DNF.

util/ranger/ranger.go

@@ -24,6 +24,7 @@ import (
 	"github.com/pingcap/tidb/expression"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/mysql"
+	"github.com/pingcap/tidb/sessionctx"
 	"github.com/pingcap/tidb/sessionctx/stmtctx"
 	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util/charset"
@@ -466,3 +467,38 @@ func newFieldType(tp *types.FieldType) *types.FieldType {
 		return tp
 	}
 }
+
+// points2EqOrInCond constructs a 'EQUAL' or 'IN' scalar function based on the
+// 'points'. The target column is extracted from the 'expr'.
+// NOTE:
+// 1. 'expr' must be either 'EQUAL' or 'IN' function.
+// 2. 'points' should not be empty.
+func points2EqOrInCond(ctx sessionctx.Context, points []point, expr expression.Expression) expression.Expression {
+	// len(points) cannot be 0 here, since we impose early termination in extractEqAndInCondition
+	sf, _ := expr.(*expression.ScalarFunction)
+	// Constant and Column args should have same RetType, simply get from first arg
+	retType := sf.GetArgs()[0].GetType()
+	args := make([]expression.Expression, 0, len(points)/2)
+	if sf.FuncName.L == ast.EQ {
+		if c, ok := sf.GetArgs()[0].(*expression.Column); ok {
+			args = append(args, c)
+		} else if c, ok := sf.GetArgs()[1].(*expression.Column); ok {
+			args = append(args, c)
+		}
+	} else {
+		args = append(args, sf.GetArgs()[0])
+	}
+	for i := 0; i < len(points); i = i + 2 {
+		value := &expression.Constant{
+			Value:   points[i].value,
+			RetType: retType,
+		}
+		args = append(args, value)
+	}
+	funcName := ast.EQ
+	if len(args) > 2 {
+		funcName = ast.In
+	}
+	f := expression.NewFunctionInternal(ctx, funcName, sf.GetType(), args...)
+	return f
+}

util/ranger/ranger_test.go

@@ -446,6 +446,20 @@ func (s *testRangerSuite) TestIndexRange(c *C) {
 			filterConds: "[]",
 			resultStr:   "[(NULL +inf,1 NULL) (1 +inf,2 NULL) (2 +inf,3 NULL) (3 +inf,+inf +inf]]",
 		},
+		{
+			indexPos:    1,
+			exprStr:     "c in (1, 2) and c in (1, 3)",
+			accessConds: "[eq(test.t.c, 1)]",
+			filterConds: "[]",
+			resultStr:   "[[1,1]]",
+		},
+		{
+			indexPos:    1,
+			exprStr:     "c = 1 and c = 2",
+			accessConds: "[]",
+			filterConds: "[]",
+			resultStr:   "[]",
+		},
 		{
 			indexPos:    0,
 			exprStr:     "a in (NULL)",