ranger: simple improvement for range calc for pk. (pingcap#4767)

executor/executor_test.go

@@ -2269,6 +2269,16 @@ func (s *testSuite) TestSubqueryInValues(c *C) {
 	tk.MustQuery("select * from t").Check(testkit.Rows("1 asd"))
 }
 
+func (s *testSuite) TestEnhancedRangeAccess(c *C) {
+	tk := testkit.NewTestKitWithInit(c, s.store)
+
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t (a int primary key, b int)")
+	tk.MustExec("insert into t values(1, 2), (2, 1)")
+	tk.MustQuery("select * from t where (a = 1 and b = 2) or (a = 2 and b = 1)").Check(testkit.Rows("1 2", "2 1"))
+	tk.MustQuery("select * from t where (a = 1 and b = 1) or (a = 2 and b = 2)").Check(nil)
+}
+
 // Issue #4810
 func (s *testSuite) TestMaxInt64Handle(c *C) {
 	tk := testkit.NewTestKitWithInit(c, s.store)

expression/expression.go

@@ -145,6 +145,30 @@ func ComposeDNFCondition(ctx context.Context, conditions ...Expression) Expressi
 	return composeConditionWithBinaryOp(ctx, conditions, ast.LogicOr)
 }
 
+func extractBinaryOpItems(conditions *ScalarFunction, funcName string) []Expression {
+	var ret []Expression
+	for _, arg := range conditions.GetArgs() {
+		if sf, ok := arg.(*ScalarFunction); ok && sf.FuncName.L == funcName {
+			ret = append(ret, extractBinaryOpItems(sf, funcName)...)
+		} else {
+			ret = append(ret, arg)
+		}
+	}
+	return ret
+}
+
+// FlattenDNFConditions extracts DNF expression's leaf item.
+// e.g. or(or(a=1, a=2), or(a=3, a=4)), we'll get [a=1, a=2, a=3, a=4].
+func FlattenDNFConditions(DNFCondition *ScalarFunction) []Expression {
+	return extractBinaryOpItems(DNFCondition, ast.LogicOr)
+}
+
+// FlattenCNFConditions extracts CNF expression's leaf item.
+// e.g. and(and(a>1, a>2), and(a>3, a>4)), we'll get [a>1, a>2, a>3, a>4].
+func FlattenCNFConditions(CNFCondition *ScalarFunction) []Expression {
+	return extractBinaryOpItems(CNFCondition, ast.LogicAnd)
+}
+
 // Assignment represents a set assignment in Update, such as
 // Update t set c1 = hex(12), c2 = c3 where c2 = 1
 type Assignment struct {

plan/new_physical_plan_builder.go

@@ -927,7 +927,7 @@ func (p *DataSource) convertToTableScan(prop *requiredProp) (task task, err erro
 		}
 		if pkCol != nil {
 			var ranges []types.Range
-			ts.AccessCondition, ts.filterCondition = ranger.DetachColumnConditions(conds, pkCol.ColName)
+			ts.AccessCondition, ts.filterCondition = ranger.DetachCondsForTableRange(p.ctx, conds, pkCol)
 			ranges, err = ranger.BuildRange(sc, ts.AccessCondition, ranger.IntRangeType, []*expression.Column{pkCol}, nil)
 			ts.Ranges = ranger.Ranges2IntRanges(ranges)
 			if err != nil {

util/ranger/new_refiner.go

@@ -16,6 +16,7 @@ package ranger
 import (
 	"github.com/juju/errors"
 	"github.com/pingcap/tidb/ast"
+	"github.com/pingcap/tidb/context"
 	"github.com/pingcap/tidb/expression"
 	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/util/types"
@@ -132,9 +133,74 @@ func getEQColOffset(expr expression.Expression, cols []*expression.Column) int {
 	return -1
 }
 
+// detachColumnCNFConditions detaches the condition for calculating range from the other conditions.
+// Please make sure that the top level is CNF form.
+func detachColumnCNFConditions(conditions []expression.Expression, checker *conditionChecker) ([]expression.Expression, []expression.Expression) {
+	var accessConditions, filterConditions []expression.Expression
+	for _, cond := range conditions {
+		if sf, ok := cond.(*expression.ScalarFunction); ok && sf.FuncName.L == ast.LogicOr {
+			dnfItems := expression.FlattenDNFConditions(sf)
+			colulmnDNFItems, hasResidual := detachColumnDNFConditions(dnfItems, checker)
+			// If this CNF has expression that cannot be resolved as access condition, then the total DNF expression
+			// should be also appended into filter condition.
+			if hasResidual {
+				filterConditions = append(filterConditions, cond)
+			}
+			if len(colulmnDNFItems) == 0 {
+				continue
+			}
+			rebuildDNF := expression.ComposeDNFCondition(nil, colulmnDNFItems...)
+			accessConditions = append(accessConditions, rebuildDNF)
+			continue
+		}
+		if !checker.check(cond) {
+			filterConditions = append(filterConditions, cond)
+			continue
+		}
+		accessConditions = append(accessConditions, cond)
+		if checker.shouldReserve {
+			filterConditions = append(filterConditions, cond)
+			checker.shouldReserve = false
+		}
+	}
+	return accessConditions, filterConditions
+}
+
+// detachColumnDNFConditions detaches the condition for calculating range from the other conditions.
+// Please make sure that the top level is DNF form.
+func detachColumnDNFConditions(conditions []expression.Expression, checker *conditionChecker) ([]expression.Expression, bool) {
+	var (
+		hasResidualConditions bool
+		accessConditions      []expression.Expression
+	)
+	for _, cond := range conditions {
+		if sf, ok := cond.(*expression.ScalarFunction); ok && sf.FuncName.L == ast.LogicAnd {
+			cnfItems := expression.FlattenCNFConditions(sf)
+			columnCNFItems, others := detachColumnCNFConditions(cnfItems, checker)
+			if len(others) > 0 {
+				hasResidualConditions = true
+			}
+			if len(columnCNFItems) == 0 {
+				continue
+			}
+			rebuildCNF := expression.ComposeCNFCondition(nil, columnCNFItems...)
+			accessConditions = append(accessConditions, rebuildCNF)
+		} else if checker.check(cond) {
+			accessConditions = append(accessConditions, cond)
+			if checker.shouldReserve {
+				hasResidualConditions = true
+				checker.shouldReserve = false
+			}
+		} else {
+			return nil, true
+		}
+	}
+	return accessConditions, hasResidualConditions
+}
+
 // DetachIndexConditions will detach the index filters from table filters.
-func DetachIndexConditions(conditions []expression.Expression, cols []*expression.Column, lengths []int) (accessConds []expression.Expression,
-	filterConds []expression.Expression) {
+func DetachIndexConditions(conditions []expression.Expression, cols []*expression.Column,
+	lengths []int) (accessConds []expression.Expression, filterConds []expression.Expression) {
 	accessConds = make([]expression.Expression, len(cols))
 	// PushDownNot here can convert query 'not (a != 1)' to 'a = 1'.
 	for i, cond := range conditions {
@@ -222,6 +288,19 @@ func DetachCondsForSelectivity(conds []expression.Expression, rangeType int, col
 	return nil, conds
 }
 
+// DetachCondsForTableRange detaches the conditions used for range calculation form other useless conditions for
+// calculating the table range.
+func DetachCondsForTableRange(ctx context.Context, conds []expression.Expression, col *expression.Column) (accessContditions, otherConditions []expression.Expression) {
+	checker := &conditionChecker{
+		colName: col.ColName,
+		length:  types.UnspecifiedLength,
+	}
+	for i, cond := range conds {
+		conds[i] = expression.PushDownNot(cond, false, ctx)
+	}
+	return detachColumnCNFConditions(conds, checker)
+}
+
 // BuildRange is a method which can calculate IntColumnRange, ColumnRange, IndexRange.
 func BuildRange(sc *variable.StatementContext, conds []expression.Expression, rangeType int, cols []*expression.Column,
 	lengths []int) (retRanges []types.Range, _ error) {