From 2009741d57c5591c10f837166264c78cac1261f5 Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Fri, 16 Aug 2019 17:37:16 +0800 Subject: [PATCH] planner: increase default concurrency factor of cost computing (#11752) --- .../r/access_path_selection.result | 21 ---------- cmd/explaintest/r/explain_easy.result | 28 ++++++------- cmd/explaintest/t/access_path_selection.test | 23 ----------- planner/core/cbo_test.go | 40 ++++++++----------- planner/core/find_best_task.go | 2 +- planner/core/logical_plan_test.go | 10 +---- planner/core/physical_plan_test.go | 2 - util/ranger/ranger_test.go | 10 ++--- 8 files changed, 38 insertions(+), 98 deletions(-) diff --git a/cmd/explaintest/r/access_path_selection.result b/cmd/explaintest/r/access_path_selection.result index 58ea41eca95ca..c66fb99a5ea22 100644 --- a/cmd/explaintest/r/access_path_selection.result +++ b/cmd/explaintest/r/access_path_selection.result @@ -23,24 +23,3 @@ id count task operator info IndexReader_11 1104.45 root index:Selection_10 └─Selection_10 1104.45 cop lt(test.access_path_selection.b, 3) └─IndexScan_9 3323.33 cop table:access_path_selection, index:a, b, range:[-inf,3), keep order:false, stats:pseudo -CREATE TABLE `outdated_statistics` ( -`a` int, -`b` int, -`c` int, -INDEX idx_a(a), -INDEX idx_ab(a,b) -); -insert into outdated_statistics values (2, 2, 2); -insert into outdated_statistics values (3, 3, 3); -insert into outdated_statistics values (4, 4, 4); -analyze table outdated_statistics; -insert into outdated_statistics values (1, 1, 1); -insert into outdated_statistics values (1, 2, 2); -insert into outdated_statistics values (1, 3, 3); -analyze table outdated_statistics index idx_ab; -explain select * from outdated_statistics where a=1 and b=1 and c=1; -id count task operator info -IndexLookUp_11 0.00 root -├─IndexScan_8 1.00 cop table:outdated_statistics, index:a, b, range:[1 1,1 1], keep order:false -└─Selection_10 0.00 cop eq(test.outdated_statistics.c, 1) - └─TableScan_9 1.00 cop table:outdated_statistics, keep order:false diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result index 100fa4e0aa067..48b9e49a90947 100644 --- a/cmd/explaintest/r/explain_easy.result +++ b/cmd/explaintest/r/explain_easy.result @@ -365,13 +365,13 @@ Projection_10 5.00 root 9_aux_0 ├─TableReader_14 5.00 root data:TableScan_13 │ └─TableScan_13 5.00 cop table:t, range:[-inf,+inf], keep order:false └─StreamAgg_19 1.00 root funcs:count(1) - └─IndexJoin_44 2.40 root left outer join, inner:TableReader_43, outer key:test.s.a, inner key:test.t1.a - ├─IndexReader_35 2.40 root index:Selection_34 - │ └─Selection_34 2.40 cop eq(3, test.t.a) - │ └─IndexScan_33 3.00 cop table:s, index:b, range:[3,3], keep order:false - └─TableReader_43 0.80 root data:Selection_42 - └─Selection_42 0.80 cop eq(3, test.t.a) - └─TableScan_41 1.00 cop table:t1, range: decided by [test.s.a], keep order:false + └─MergeJoin_39 2.40 root left outer join, left key:test.s.a, right key:test.t1.a + ├─TableReader_29 2.40 root data:Selection_28 + │ └─Selection_28 2.40 cop eq(3, test.t.a), eq(test.s.b, 3) + │ └─TableScan_27 5.00 cop table:s, range:[-inf,+inf], keep order:true + └─TableReader_32 4.00 root data:Selection_31 + └─Selection_31 4.00 cop eq(3, test.t.a) + └─TableScan_30 5.00 cop table:t1, range:[-inf,+inf], keep order:true explain select t.c in (select count(*) from t s right join t t1 on s.a = t1.a where 3 = t.a and t1.b = 3) from t; id count task operator info Projection_10 5.00 root 9_aux_0 @@ -379,13 +379,13 @@ Projection_10 5.00 root 9_aux_0 ├─TableReader_14 5.00 root data:TableScan_13 │ └─TableScan_13 5.00 cop table:t, range:[-inf,+inf], keep order:false └─StreamAgg_19 1.00 root funcs:count(1) - └─IndexJoin_42 2.40 root right outer join, inner:TableReader_41, outer key:test.t1.a, inner key:test.s.a - ├─TableReader_41 0.80 root data:Selection_40 - │ └─Selection_40 0.80 cop eq(3, test.t.a) - │ └─TableScan_39 1.00 cop table:s, range: decided by [test.t1.a], keep order:false - └─IndexReader_34 2.40 root index:Selection_33 - └─Selection_33 2.40 cop eq(3, test.t.a) - └─IndexScan_32 3.00 cop table:t1, index:b, range:[3,3], keep order:false + └─MergeJoin_38 2.40 root right outer join, left key:test.s.a, right key:test.t1.a + ├─TableReader_28 4.00 root data:Selection_27 + │ └─Selection_27 4.00 cop eq(3, test.t.a) + │ └─TableScan_26 5.00 cop table:s, range:[-inf,+inf], keep order:true + └─TableReader_31 2.40 root data:Selection_30 + └─Selection_30 2.40 cop eq(3, test.t.a), eq(test.t1.b, 3) + └─TableScan_29 5.00 cop table:t1, range:[-inf,+inf], keep order:true drop table if exists t; create table t(a int unsigned); explain select t.a = '123455' from t; diff --git a/cmd/explaintest/t/access_path_selection.test b/cmd/explaintest/t/access_path_selection.test index a7085832dda89..fef9118ca606d 100644 --- a/cmd/explaintest/t/access_path_selection.test +++ b/cmd/explaintest/t/access_path_selection.test @@ -12,26 +12,3 @@ explain select a, b from access_path_selection where a < 3; # In this query, IDX_ab can't be used, so IDX_b is the best. explain select a, b from access_path_selection where b < 3; explain select a, b from access_path_selection where a < 3 and b < 3; - -CREATE TABLE `outdated_statistics` ( -`a` int, -`b` int, -`c` int, -INDEX idx_a(a), -INDEX idx_ab(a,b) -); -insert into outdated_statistics values (2, 2, 2); -insert into outdated_statistics values (3, 3, 3); -insert into outdated_statistics values (4, 4, 4); -analyze table outdated_statistics; -insert into outdated_statistics values (1, 1, 1); -insert into outdated_statistics values (1, 2, 2); -insert into outdated_statistics values (1, 3, 3); -# Only update idx_ab and leave idx_a outdated. -# Then the estimated number of rows on idx_ab is 1 while it's 0 on idx_a. -# But for this query, idx_ab is always better than idx_a, -# because idx_a can't take column b into account while idx_ab can. -# This wrong case can be solved by Skyline Pruning, so we may update its -# result after Skyline Pruning is introduced. -analyze table outdated_statistics index idx_ab; -explain select * from outdated_statistics where a=1 and b=1 and c=1; diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 2225347dc2f57..62812263e52d8 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -281,9 +281,6 @@ func (s *testAnalyzeSuite) TestIndexRead(c *C) { } testKit.MustExec("analyze table t1") ctx := testKit.Se.(sessionctx.Context) - sessionVars := ctx.GetSessionVars() - sessionVars.HashAggFinalConcurrency = 1 - sessionVars.HashAggPartialConcurrency = 1 tests := []struct { sql string best string @@ -496,8 +493,8 @@ func (s *testAnalyzeSuite) TestAnalyze(c *C) { best: "TableReader(Table(t)->Sel([le(test.t.a, 2)]))", }, { - sql: "select * from t where t.b < 2", - best: "IndexLookUp(Index(t.b)[[-inf,2)], Table(t))", + sql: "select b from t where t.b < 2", + best: "IndexReader(Index(t.b)[[-inf,2)])", }, { sql: "select * from t where t.a = 1 and t.b <= 2", @@ -528,8 +525,8 @@ func (s *testAnalyzeSuite) TestAnalyze(c *C) { best: "UnionAll{TableReader(Table(t4)->Sel([le(test.t4.a, 2)]))->TableReader(Table(t4)->Sel([le(test.t4.a, 2)]))}", }, { - sql: "select * from t4 where t4.b < 2", - best: "UnionAll{IndexLookUp(Index(t4.b)[[-inf,2)], Table(t4))->IndexLookUp(Index(t4.b)[[-inf,2)], Table(t4))}", + sql: "select b from t4 where t4.b < 2", + best: "UnionAll{IndexReader(Index(t4.b)[[-inf,2)])->IndexReader(Index(t4.b)[[-inf,2)])}", }, { sql: "select * from t4 where t4.a = 1 and t4.b <= 2", @@ -685,13 +682,9 @@ func (s *testAnalyzeSuite) TestCorrelatedEstimation(c *C) { }() tk.MustExec("use test") tk.MustExec("set sql_mode='STRICT_TRANS_TABLES'") // disable only full group by - tk.MustExec("create table t(a int, b int, c int, index idx(c))") + tk.MustExec("create table t(a int, b int, c int, index idx(c,b,a))") tk.MustExec("insert into t values(1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5), (6,6,6), (7,7,7), (8,8,8), (9,9,9),(10,10,10)") tk.MustExec("analyze table t") - ctx := tk.Se.(sessionctx.Context) - sessionVars := ctx.GetSessionVars() - sessionVars.HashAggFinalConcurrency = 1 - sessionVars.HashAggPartialConcurrency = 1 tk.MustQuery("explain select t.c in (select count(*) from t s , t t1 where s.a = t.a and s.a = t1.a) from t;"). Check(testkit.Rows( "Projection_11 10.00 root 9_aux_0", @@ -715,10 +708,9 @@ func (s *testAnalyzeSuite) TestCorrelatedEstimation(c *C) { " │ └─TableScan_11 10.00 cop table:t, range:[-inf,+inf], keep order:false", " └─MaxOneRow_13 1.00 root ", " └─Projection_14 0.10 root concat(cast(test.t1.a), \",\", cast(test.t1.b))", - " └─IndexLookUp_21 0.10 root ", - " ├─IndexScan_18 1.00 cop table:t1, index:c, range: decided by [eq(test.t1.c, test.t.c)], keep order:false", - " └─Selection_20 0.10 cop eq(test.t1.a, test.t.a)", - " └─TableScan_19 1.00 cop table:t, keep order:false", + " └─IndexReader_17 0.10 root index:Selection_16", + " └─Selection_16 0.10 cop eq(test.t1.a, test.t.a)", + " └─IndexScan_15 1.00 cop table:t1, index:c, b, a, range: decided by [eq(test.t1.c, test.t.c)], keep order:false", )) } @@ -1085,16 +1077,16 @@ func (s *testAnalyzeSuite) TestLimitCrossEstimation(c *C) { )) // Multi-column filter. tk.MustExec("drop table t") - tk.MustExec("create table t(a int primary key, b int, c int, d bigint default 2147483648, e bigint default 2147483648, f bigint default 2147483648, index idx_b(b))") + tk.MustExec("create table t(a int primary key, b int, c int, d bigint default 2147483648, e bigint default 2147483648, f bigint default 2147483648, index idx(b,d,a,c))") tk.MustExec("insert into t(a, b, c) values (1, 1, 1),(2, 1, 2),(3, 1, 1),(4, 1, 2),(5, 1, 1),(6, 1, 2),(7, 1, 1),(8, 1, 2),(9, 1, 1),(10, 1, 2),(11, 1, 1),(12, 1, 2),(13, 1, 1),(14, 1, 2),(15, 1, 1),(16, 1, 2),(17, 1, 1),(18, 1, 2),(19, 1, 1),(20, 2, 2),(21, 2, 1),(22, 2, 2),(23, 2, 1),(24, 2, 2),(25, 2, 1)") tk.MustExec("analyze table t") - tk.MustQuery("EXPLAIN SELECT * FROM t WHERE b = 2 and c > 0 ORDER BY a limit 1").Check(testkit.Rows( - "TopN_9 1.00 root test.t.a:asc, offset:0, count:1", - "└─IndexLookUp_22 1.00 root ", - " ├─IndexScan_18 6.00 cop table:t, index:b, range:[2,2], keep order:false", - " └─TopN_21 1.00 cop test.t.a:asc, offset:0, count:1", - " └─Selection_20 6.00 cop gt(test.t.c, 0)", - " └─TableScan_19 6.00 cop table:t, keep order:false", + tk.MustQuery("EXPLAIN SELECT a FROM t WHERE b = 2 and c > 0 ORDER BY a limit 1").Check(testkit.Rows( + "Projection_7 1.00 root test.t.a", + "└─TopN_8 1.00 root test.t.a:asc, offset:0, count:1", + " └─IndexReader_17 1.00 root index:TopN_16", + " └─TopN_16 1.00 cop test.t.a:asc, offset:0, count:1", + " └─Selection_15 6.00 cop gt(test.t.c, 0)", + " └─IndexScan_14 6.00 cop table:t, index:b, d, a, c, range:[2,2], keep order:false", )) } diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index d25415edd9fbf..69631ffa35089 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -37,7 +37,7 @@ const ( scanFactor = 1.5 * netWorkFactor descScanFactor = 2 * scanFactor memoryFactor = 0.001 - concurrencyFactor = 0.001 + concurrencyFactor = 3.0 selectionFactor = 0.8 distinctFactor = 0.8 diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index a2843b98fc6f8..b01c5ebe8fc30 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -2249,7 +2249,7 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { }, { sql: "select sum(avg(a)) over() from t", - result: "TableReader(Table(t)->HashAgg)->HashAgg->Window(sum(sel_agg_2) over())->Projection", + result: "TableReader(Table(t)->StreamAgg)->StreamAgg->Window(sum(sel_agg_2) over())->Projection", }, { sql: "select b from t order by(sum(a) over())", @@ -2261,7 +2261,7 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { }, { sql: "select b from t order by(sum(avg(a)) over())", - result: "TableReader(Table(t)->HashAgg)->HashAgg->Window(sum(sel_agg_2) over())->Sort->Projection", + result: "TableReader(Table(t)->StreamAgg)->StreamAgg->Window(sum(sel_agg_2) over())->Sort->Projection", }, { sql: "select a from t having (select sum(a) over() as w from t tt where a > t.a)", @@ -2480,14 +2480,8 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { } s.Parser.EnableWindowFunc(true) - sessionVars := s.ctx.GetSessionVars() - finalCon, partialCon := sessionVars.HashAggFinalConcurrency, sessionVars.HashAggPartialConcurrency - sessionVars.HashAggFinalConcurrency = 1 - sessionVars.HashAggPartialConcurrency = 1 defer func() { s.Parser.EnableWindowFunc(false) - sessionVars.HashAggFinalConcurrency = finalCon - sessionVars.HashAggPartialConcurrency = partialCon }() ctx := context.TODO() for i, tt := range tests { diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 3127481b4e8ca..3d0af8ae10559 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1232,8 +1232,6 @@ func (s *testPlanSuite) TestAggEliminater(c *C) { _, err = se.Execute(context.Background(), "use test") c.Assert(err, IsNil) se.Execute(context.Background(), "set sql_mode='STRICT_TRANS_TABLES'") // disable only full group by - se.GetSessionVars().HashAggFinalConcurrency = 1 - se.GetSessionVars().HashAggPartialConcurrency = 1 tests := []struct { sql string best string diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index ee9f9c3e396ad..4d4fffd035d30 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -1102,15 +1102,15 @@ func (s *testRangerSuite) TestCompIndexInExprCorrCol(c *C) { testKit.MustExec("analyze table t") testKit.MustQuery("explain select t.e in (select count(*) from t s use index(idx), t t1 where s.b = 1 and s.c in (1, 2) and s.d = t.a and s.a = t1.a) from t").Check(testkit.Rows( "Projection_11 2.00 root 9_aux_0", - "└─Apply_13 2.00 root CARTESIAN left outer semi join, inner:HashAgg_18, other cond:eq(test.t.e, 7_col_0)", + "└─Apply_13 2.00 root CARTESIAN left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.e, 7_col_0)", " ├─TableReader_15 2.00 root data:TableScan_14", " │ └─TableScan_14 2.00 cop table:t, range:[-inf,+inf], keep order:false", - " └─HashAgg_18 1.00 root funcs:count(1)", - " └─HashLeftJoin_24 2.00 root inner join, inner:TableReader_29, equal:[eq(test.s.a, test.t1.a)]", + " └─StreamAgg_20 1.00 root funcs:count(1)", + " └─IndexJoin_32 2.00 root inner join, inner:TableReader_31, outer key:test.s.a, inner key:test.t1.a", " ├─IndexReader_27 2.00 root index:IndexScan_26", " │ └─IndexScan_26 2.00 cop table:s, index:b, c, d, range: decided by [eq(test.s.b, 1) in(test.s.c, 1, 2) eq(test.s.d, test.t.a)], keep order:false", - " └─TableReader_29 2.00 root data:TableScan_28", - " └─TableScan_28 2.00 cop table:t1, range:[-inf,+inf], keep order:false", + " └─TableReader_31 1.00 root data:TableScan_30", + " └─TableScan_30 1.00 cop table:t1, range: decided by [test.s.a], keep order:false", )) testKit.MustQuery("select t.e in (select count(*) from t s use index(idx), t t1 where s.b = 1 and s.c in (1, 2) and s.d = t.a and s.a = t1.a) from t").Check(testkit.Rows( "1",