stats: refine row count estimation (pingcap#6746)

doerjiayi · Jun 4, 2018 · c348afe · c348afe
1 parent 30ff743
commit c348afe
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 18 deletions.
diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result
@@ -47,7 +47,7 @@ TableReader_23	IndexJoin_11		root	data:TableScan_22	3333.33
 IndexScan_8			cop	table:t2, index:c1, range:[<nil>,+inf], keep order:false	10000.00
 TableScan_9			cop	table:t2, keep order:false	10000.00
 IndexLookUp_10	IndexJoin_11		root	index:IndexScan_8, table:TableScan_9	10000.00
-IndexJoin_11		TableReader_23,IndexLookUp_10	root	outer:TableReader_23, left outer join, outer key:test.t1.c2, inner key:test.t2.c1	312.50
+IndexJoin_11		TableReader_23,IndexLookUp_10	root	outer:TableReader_23, left outer join, outer key:test.t1.c2, inner key:test.t2.c1	4166.67
 explain update t1 set t1.c2 = 2 where t1.c1 = 1;
 id	parents	children	task	operator info	count
 TableScan_4			cop	table:t1, range:[1,1], keep order:false	1.00
@@ -76,10 +76,10 @@ TopN_7		TableReader_15	root	test.t2.c2:asc, offset:0, count:1	1.00
 explain select * from t1 where c1 > 1 and c2 = 1 and c3 < 1;
 id	parents	children	task	operator info	count
 IndexScan_8	Selection_10		cop	table:t1, index:c2, range:[1,1], keep order:false	10.00
-Selection_10		IndexScan_8	cop	gt(test.t1.c1, 1)	0.25
-TableScan_9	Selection_11		cop	table:t1, keep order:false	0.25
-Selection_11		TableScan_9	cop	lt(test.t1.c3, 1)	0.08
-IndexLookUp_12			root	index:Selection_10, table:Selection_11	0.08
+Selection_10		IndexScan_8	cop	gt(test.t1.c1, 1)	3.33
+TableScan_9	Selection_11		cop	table:t1, keep order:false	3.33
+Selection_11		TableScan_9	cop	lt(test.t1.c3, 1)	1.11
+IndexLookUp_12			root	index:Selection_10, table:Selection_11	1.11
 explain select * from t1 where c1 = 1 and c2 > 1;
 id	parents	children	task	operator info	count
 TableScan_5	Selection_6		cop	table:t1, range:[1,1], keep order:false	1.00
@@ -129,16 +129,16 @@ TableReader_21	Limit_10		root	data:Limit_20	1.00
 Limit_10		TableReader_21	root	offset:0, count:1	1.00
 explain select * from t4 use index(idx) where a > 1 and b > 1 and c > 1 limit 1;
 id	parents	children	task	operator info	count
-IndexScan_12	Selection_14		cop	table:t4, index:a, b, range:(1 +inf,+inf +inf], keep order:false	16.67
-Selection_14		IndexScan_12	cop	gt(test.t4.b, 1)	5.56
-TableScan_13	Selection_15		cop	table:t4, keep order:false	5.56
+IndexScan_12	Selection_14		cop	table:t4, index:a, b, range:(1 +inf,+inf +inf], keep order:false	9.00
+Selection_14		IndexScan_12	cop	gt(test.t4.b, 1)	3.00
+TableScan_13	Selection_15		cop	table:t4, keep order:false	3.00
 Selection_15	Limit_16	TableScan_13	cop	gt(test.t4.c, 1)	1.00
 Limit_16		Selection_15	cop	offset:0, count:1	1.00
 IndexLookUp_17	Limit_9		root	index:Selection_14, table:Limit_16	1.00
 Limit_9		IndexLookUp_17	root	offset:0, count:1	1.00
 explain select * from t4 where a > 1 and c > 1 limit 1;
 id	parents	children	task	operator info	count
-TableScan_11	Selection_12		cop	table:t4, range:(1,+inf], keep order:false	16.67
+TableScan_11	Selection_12		cop	table:t4, range:(1,+inf], keep order:false	3.00
 Selection_12	Limit_14	TableScan_11	cop	gt(test.t4.c, 1)	1.00
 Limit_14		Selection_12	cop	offset:0, count:1	1.00
 TableReader_15	Limit_8		root	data:Limit_14	1.00

diff --git a/cmd/explaintest/r/topn_push_down.result b/cmd/explaintest/r/topn_push_down.result
@@ -168,18 +168,18 @@ ORDER BY te.expect_time asc
 LIMIT 0, 5;
 id	parents	children	task	operator info	count
 IndexScan_100	Selection_102		cop	table:tr, index:shop_identy, trade_status, business_type, trade_pay_status, trade_type, delivery_type, source, biz_date, range:[810094178,810094178], keep order:false	10.00
-Selection_102		IndexScan_100	cop	eq(tr.business_type, 18), in(tr.trade_type, 1)	0.01
-TableScan_101	Selection_103		cop	table:tr, keep order:false	0.01
-Selection_103		TableScan_101	cop	eq(tr.brand_identy, 32314), eq(tr.domain_type, 2)	0.01
-IndexLookUp_104	IndexJoin_34		root	index:Selection_102, table:Selection_103	0.01
+Selection_102		IndexScan_100	cop	eq(tr.business_type, 18), in(tr.trade_type, 1)	0.00
+TableScan_101	Selection_103		cop	table:tr, keep order:false	0.00
+Selection_103		TableScan_101	cop	eq(tr.brand_identy, 32314), eq(tr.domain_type, 2)	0.00
+IndexLookUp_104	IndexJoin_34		root	index:Selection_102, table:Selection_103	0.00
 IndexScan_30			cop	table:te, index:trade_id, range:[<nil>,+inf], keep order:false	10000.00
 TableScan_31	Selection_32		cop	table:te, keep order:false	10000.00
 Selection_32		TableScan_31	cop	ge(te.expect_time, 2018-04-23 00:00:00.000000), le(te.expect_time, 2018-04-23 23:59:59.000000)	250.00
 IndexLookUp_33	IndexJoin_34		root	index:IndexScan_30, table:Selection_32	250.00
-IndexJoin_34	TopN_139	IndexLookUp_104,IndexLookUp_33	root	outer:IndexLookUp_104, inner join, outer key:tr.id, inner key:te.trade_id	0.01
-TopN_139	IndexJoin_136	IndexJoin_34	root	te.expect_time:asc, offset:0, count:5	0.01
+IndexJoin_34	TopN_139	IndexLookUp_104,IndexLookUp_33	root	outer:IndexLookUp_104, inner join, outer key:tr.id, inner key:te.trade_id	0.00
+TopN_139	IndexJoin_136	IndexJoin_34	root	te.expect_time:asc, offset:0, count:5	0.00
 IndexScan_134			cop	table:p, index:relate_id, range:[<nil>,+inf], keep order:false	10000.00
 IndexReader_135	IndexJoin_136		root	index:IndexScan_134	10000.00
-IndexJoin_136	Limit_18	TopN_139,IndexReader_135	root	outer:TopN_139, left outer join, outer key:tr.id, inner key:p.relate_id	0.01
-Limit_18	Projection_12	IndexJoin_136	root	offset:0, count:5	0.01
-Projection_12		Limit_18	root	te.expect_time	0.01
+IndexJoin_136	Limit_18	TopN_139,IndexReader_135	root	outer:TopN_139, left outer join, outer key:tr.id, inner key:p.relate_id	0.00
+Limit_18	Projection_12	IndexJoin_136	root	offset:0, count:5	0.00
+Projection_12		Limit_18	root	te.expect_time	0.00
diff --git a/statistics/selectivity.go b/statistics/selectivity.go
@@ -229,6 +229,10 @@ func getUsableSetsByGreedy(sets []*exprSet) (newBlocks []*exprSet) {
 		for i, set := range sets {
 			set.mask &= mask
 			bits := popCount(set.mask)
+			// This set cannot cover any thing, just skip it.
+			if bits == 0 {
+				continue
+			}
 			if (bestTp == colType && set.tp < colType) || bestCount < bits {
 				bestID, bestCount, bestTp = i, bits, set.tp
 			}

diff --git a/statistics/table.go b/statistics/table.go
@@ -463,7 +463,13 @@ func getPseudoRowCountBySignedIntRanges(intRanges []*ranger.Range, tableRowCount
 	for _, rg := range intRanges {
 		var cnt float64
 		low := rg.LowVal[0].GetInt64()
+		if rg.LowVal[0].Kind() == types.KindNull || rg.LowVal[0].Kind() == types.KindMinNotNull {
+			low = math.MinInt64
+		}
 		high := rg.HighVal[0].GetInt64()
+		if rg.HighVal[0].Kind() == types.KindMaxValue {
+			high = math.MaxInt64
+		}
 		if low == math.MinInt64 && high == math.MaxInt64 {
 			cnt = tableRowCount
 		} else if low == math.MinInt64 {
@@ -493,7 +499,13 @@ func getPseudoRowCountByUnsignedIntRanges(intRanges []*ranger.Range, tableRowCou
 	for _, rg := range intRanges {
 		var cnt float64
 		low := rg.LowVal[0].GetUint64()
+		if rg.LowVal[0].Kind() == types.KindNull || rg.LowVal[0].Kind() == types.KindMinNotNull {
+			low = 0
+		}
 		high := rg.HighVal[0].GetUint64()
+		if rg.HighVal[0].Kind() == types.KindMaxValue {
+			high = math.MaxUint64
+		}
 		if low == 0 && high == math.MaxUint64 {
 			cnt = tableRowCount
 		} else if low == 0 {