Add options to analyze tables after tpc-h data loaded (#41)

* Add options to analyze tables after tpc-h data loaded Signed-off-by: JaySon-Huang <[email protected]> * Update README Signed-off-by: JaySon-Huang <[email protected]>
pingcap · May 11, 2020 · 883c1a1 · 883c1a1
1 parent 3b4c573
commit 883c1a1
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 5 deletions.
diff --git a/README.md b/README.md
@@ -63,6 +63,8 @@ If you want to import tpcc data into TiDB, please refer to [import-to-tidb](docs
 ```bash
 # Prepare data with scale factor 1
 ./bin/go-tpc tpch --sf=1 prepare
+# Prepare data with scale factor 1, create tiflash replica, and analyze table after data loaded
+./bin/go-tpc tpch --sf 1 --analyze --tiflash prepare
 # Run TPCH workloads with result checking
 ./bin/go-tpc tpch --sf=1 --check=true run
 # Run TPCH workloads without result checking

diff --git a/cmd/go-tpc/tpch.go b/cmd/go-tpc/tpch.go
@@ -57,6 +57,24 @@ func registerTpch(root *cobra.Command) {
 		false,
 		"Create tiflash replica")
 
+	cmdPrepare.PersistentFlags().BoolVar(&tpchConfig.AnalyzeTable.Enable,
+		"analyze",
+		false,
+		"After data loaded, analyze table to collect column statistics")
+	// https://pingcap.com/docs/stable/reference/performance/statistics/#control-analyze-concurrency
+	cmdPrepare.PersistentFlags().IntVar(&tpchConfig.AnalyzeTable.BuildStatsConcurrency,
+		"tidb_build_stats_concurrency",
+		4,
+		"tidb_build_stats_concurrency param for analyze jobs")
+	cmdPrepare.PersistentFlags().IntVar(&tpchConfig.AnalyzeTable.DistsqlScanConcurrency,
+		"tidb_distsql_scan_concurrency",
+		15,
+		"tidb_distsql_scan_concurrency param for analyze jobs")
+	cmdPrepare.PersistentFlags().IntVar(&tpchConfig.AnalyzeTable.IndexSerialScanConcurrency,
+		"tidb_index_serial_scan_concurrency",
+		1,
+		"tidb_index_serial_scan_concurrency param for analyze jobs")
+
 	var cmdRun = &cobra.Command{
 		Use:   "run",
 		Short: "Run workload",

diff --git a/tpch/ddl.go b/tpch/ddl.go
@@ -5,6 +5,12 @@ import (
 	"fmt"
 )
 
+var allTables []string
+
+func init() {
+	allTables = []string{"lineitem", "partsupp", "supplier", "part", "orders", "customer", "region", "nation"}
+}
+
 func (w *Workloader) createTableDDL(ctx context.Context, query string, tableName string, action string) error {
 	s := w.getState(ctx)
 	fmt.Printf("%s %s\n", action, tableName)
@@ -154,11 +160,8 @@ CREATE TABLE IF NOT EXISTS lineitem (
 
 func (w *Workloader) dropTable(ctx context.Context) error {
 	s := w.getState(ctx)
-	tables := []string{
-		"lineitem", "partsupp", "supplier", "part", "orders", "customer", "region", "nation",
-	}
 
-	for _, tbl := range tables {
+	for _, tbl := range allTables {
 		fmt.Printf("DROP TABLE IF EXISTS %s\n", tbl)
 		if _, err := s.Conn.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", tbl)); err != nil {
 			return err

diff --git a/tpch/workload.go b/tpch/workload.go
@@ -16,6 +16,14 @@ type contextKey string
 
 const stateKey = contextKey("tpch")
 
+// analyzeConfig is the configuration for analyze after data loaded
+type analyzeConfig struct {
+	Enable                     bool
+	BuildStatsConcurrency      int
+	DistsqlScanConcurrency     int
+	IndexSerialScanConcurrency int
+}
+
 // Config is the configuration for tpch workload
 type Config struct {
 	DBName               string
@@ -24,6 +32,7 @@ type Config struct {
 	ScaleFactor          int
 	EnableOutputCheck    bool
 	CreateTiFlashReplica bool
+	AnalyzeTable         analyzeConfig
 }
 
 type tpchState struct {
@@ -98,7 +107,29 @@ func (w Workloader) Prepare(ctx context.Context, threadID int) error {
 		dbgen.TRegion: newRegionLoader(ctx, s.Conn),
 	}
 	dbgen.InitDbGen(int64(w.cfg.ScaleFactor))
-	return dbgen.DbGen(sqlLoader)
+	if err := dbgen.DbGen(sqlLoader); err != nil {
+		return err
+	}
+
+	// After data loaded, analyze tables to speed up queries.
+	if w.cfg.AnalyzeTable.Enable {
+		if err := w.analyzeTables(ctx, w.cfg.AnalyzeTable); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (w Workloader) analyzeTables(ctx context.Context, acfg analyzeConfig) error {
+	s := w.getState(ctx)
+	for _, tbl := range allTables {
+		fmt.Printf("analyzing table %s\n", tbl)
+		if _, err := s.Conn.ExecContext(ctx, fmt.Sprintf("SET @@session.tidb_build_stats_concurrency=%d; SET @@session.tidb_distsql_scan_concurrency=%d; SET @@session.tidb_index_serial_scan_concurrency=%d; ANALYZE TABLE %s", acfg.BuildStatsConcurrency, acfg.DistsqlScanConcurrency, acfg.IndexSerialScanConcurrency, tbl)); err != nil {
+			return err
+		}
+		fmt.Printf("analyze table %s done\n", tbl)
+	}
+	return nil
 }
 
 // CheckPrepare checks prepare