infoschema: fill data length fields for tables (pingcap#7657)

infoschema/tables.go

@@ -16,6 +16,7 @@ package infoschema
 import (
 	"fmt"
 	"sort"
+	"sync"
 	"time"
 
 	"github.com/pingcap/tidb/kv"
@@ -711,6 +712,111 @@ func getRowCountAllTable(ctx sessionctx.Context) (map[int64]uint64, error) {
 	return rowCountMap, nil
 }
 
+type tableHistID struct {
+	tableID int64
+	histID  int64
+}
+
+func getColLengthAllTables(ctx sessionctx.Context) (map[tableHistID]uint64, error) {
+	rows, _, err := ctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(ctx, "select table_id, hist_id, tot_col_size from mysql.stats_histograms where is_index = 0")
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	colLengthMap := make(map[tableHistID]uint64, len(rows))
+	for _, row := range rows {
+		tableID := row.GetInt64(0)
+		histID := row.GetInt64(1)
+		totalSize := row.GetInt64(2)
+		if totalSize < 0 {
+			totalSize = 0
+		}
+		colLengthMap[tableHistID{tableID: tableID, histID: histID}] = uint64(totalSize)
+	}
+	return colLengthMap, nil
+}
+
+func getDataAndIndexLength(info *model.TableInfo, rowCount uint64, columnLengthMap map[tableHistID]uint64) (uint64, uint64) {
+	columnLength := make(map[string]uint64)
+	for _, col := range info.Columns {
+		if col.State != model.StatePublic {
+			continue
+		}
+		length := col.FieldType.StorageLength()
+		if length != types.VarStorageLen {
+			columnLength[col.Name.L] = rowCount * uint64(length)
+		} else {
+			length := columnLengthMap[tableHistID{tableID: info.ID, histID: col.ID}]
+			columnLength[col.Name.L] = length
+		}
+	}
+	dataLength, indexLength := uint64(0), uint64(0)
+	for _, length := range columnLength {
+		dataLength += length
+	}
+	for _, idx := range info.Indices {
+		if idx.State != model.StatePublic {
+			continue
+		}
+		for _, col := range idx.Columns {
+			if col.Length == types.UnspecifiedLength {
+				indexLength += columnLength[col.Name.L]
+			} else {
+				indexLength += rowCount * uint64(col.Length)
+			}
+		}
+	}
+	return dataLength, indexLength
+}
+
+type statsCache struct {
+	mu         sync.Mutex
+	loading    bool
+	modifyTime time.Time
+	tableRows  map[int64]uint64
+	colLength  map[tableHistID]uint64
+}
+
+var tableStatsCache = &statsCache{}
+
+// TableStatsCacheExpiry is the expiry time for table stats cache.
+var TableStatsCacheExpiry = 3 * time.Second
+
+func (c *statsCache) setLoading(loading bool) {
+	c.mu.Lock()
+	c.loading = loading
+	c.mu.Unlock()
+}
+
+func (c *statsCache) get(ctx sessionctx.Context) (map[int64]uint64, map[tableHistID]uint64, error) {
+	c.mu.Lock()
+	if time.Now().Sub(c.modifyTime) < TableStatsCacheExpiry || c.loading {
+		tableRows, colLength := c.tableRows, c.colLength
+		c.mu.Unlock()
+		return tableRows, colLength, nil
+	}
+	c.loading = true
+	c.mu.Unlock()
+
+	tableRows, err := getRowCountAllTable(ctx)
+	if err != nil {
+		c.setLoading(false)
+		return nil, nil, errors.Trace(err)
+	}
+	colLength, err := getColLengthAllTables(ctx)
+	if err != nil {
+		c.setLoading(false)
+		return nil, nil, errors.Trace(err)
+	}
+
+	c.mu.Lock()
+	c.loading = false
+	c.tableRows = tableRows
+	c.colLength = colLength
+	c.modifyTime = time.Now()
+	c.mu.Unlock()
+	return tableRows, colLength, nil
+}
+
 func getAutoIncrementID(ctx sessionctx.Context, schema *model.DBInfo, tblInfo *model.TableInfo) (int64, error) {
 	hasAutoIncID := false
 	for _, col := range tblInfo.Cols() {
@@ -735,7 +841,7 @@ func getAutoIncrementID(ctx sessionctx.Context, schema *model.DBInfo, tblInfo *m
 }
 
 func dataForTables(ctx sessionctx.Context, schemas []*model.DBInfo) ([][]types.Datum, error) {
-	tableRowsMap, err := getRowCountAllTable(ctx)
+	tableRowsMap, colLengthMap, err := tableStatsCache.get(ctx)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
@@ -763,28 +869,34 @@ func dataForTables(ctx sessionctx.Context, schemas []*model.DBInfo) ([][]types.D
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
+			rowCount := tableRowsMap[table.ID]
+			dataLength, indexLength := getDataAndIndexLength(table, rowCount, colLengthMap)
+			avgRowLength := uint64(0)
+			if rowCount != 0 {
+				avgRowLength = dataLength / rowCount
+			}
 			record := types.MakeDatums(
-				catalogVal,             // TABLE_CATALOG
-				schema.Name.O,          // TABLE_SCHEMA
-				table.Name.O,           // TABLE_NAME
-				"BASE TABLE",           // TABLE_TYPE
-				"InnoDB",               // ENGINE
-				uint64(10),             // VERSION
-				"Compact",              // ROW_FORMAT
-				tableRowsMap[table.ID], // TABLE_ROWS
-				uint64(0),              // AVG_ROW_LENGTH
-				uint64(16384),          // DATA_LENGTH
-				uint64(0),              // MAX_DATA_LENGTH
-				uint64(0),              // INDEX_LENGTH
-				uint64(0),              // DATA_FREE
-				autoIncID,              // AUTO_INCREMENT
-				createTime,             // CREATE_TIME
-				nil,                    // UPDATE_TIME
-				nil,                    // CHECK_TIME
-				collation,              // TABLE_COLLATION
-				nil,                    // CHECKSUM
-				"",                     // CREATE_OPTIONS
-				table.Comment,          // TABLE_COMMENT
+				catalogVal,    // TABLE_CATALOG
+				schema.Name.O, // TABLE_SCHEMA
+				table.Name.O,  // TABLE_NAME
+				"BASE TABLE",  // TABLE_TYPE
+				"InnoDB",      // ENGINE
+				uint64(10),    // VERSION
+				"Compact",     // ROW_FORMAT
+				rowCount,      // TABLE_ROWS
+				avgRowLength,  // AVG_ROW_LENGTH
+				dataLength,    // DATA_LENGTH
+				uint64(0),     // MAX_DATA_LENGTH
+				indexLength,   // INDEX_LENGTH
+				uint64(0),     // DATA_FREE
+				autoIncID,     // AUTO_INCREMENT
+				createTime,    // CREATE_TIME
+				nil,           // UPDATE_TIME
+				nil,           // CHECK_TIME
+				collation,     // TABLE_COLLATION
+				nil,           // CHECKSUM
+				"",            // CREATE_OPTIONS
+				table.Comment, // TABLE_COMMENT
 			)
 			rows = append(rows, record)
 		}

infoschema/tables_test.go

@@ -15,6 +15,7 @@ package infoschema_test
 
 import (
 	. "github.com/pingcap/check"
+	"github.com/pingcap/tidb/infoschema"
 	"github.com/pingcap/tidb/session"
 	"github.com/pingcap/tidb/statistics"
 	"github.com/pingcap/tidb/store/mockstore"
@@ -50,52 +51,9 @@ func (s *testSuite) TestInfoschemaFieldValue(c *C) {
 		Check(testkit.Rows("3 3 <nil> <nil> <nil>", "3 3 <nil> <nil> <nil>", "3 3 <nil> <nil> <nil>", "3 3 <nil> <nil> <nil>")) // FIXME: for mysql last two will be "255 255 <nil> <nil> <nil>", "255 255 <nil> <nil> <nil>"
 	tk.MustQuery("select NUMERIC_SCALE from information_schema.COLUMNS where table_name='floatschema'").
 		Check(testkit.Rows("<nil>", "3"))
-}
-
-func (s *testSuite) TestDataForTableRowsCountField(c *C) {
-	testleak.BeforeTest()
-	defer testleak.AfterTest(c)()
-	store, err := mockstore.NewMockTikvStore()
-	c.Assert(err, IsNil)
-	defer store.Close()
-	session.SetStatsLease(0)
-	do, err := session.BootstrapSession(store)
-	c.Assert(err, IsNil)
-	defer do.Close()
-
-	h := do.StatsHandle()
-	is := do.InfoSchema()
-	tk := testkit.NewTestKit(c, store)
-
-	tk.MustExec("use test")
-	tk.MustExec("drop table if exists t")
-	tk.MustExec("create table t (c int, d int)")
-	h.HandleDDLEvent(<-h.DDLEventCh())
-	tk.MustQuery("select table_rows from information_schema.tables where table_name='t'").Check(
-		testkit.Rows("0"))
-	tk.MustExec("insert into t(c, d) values(1, 2), (2, 3), (3, 4)")
-	h.DumpStatsDeltaToKV(statistics.DumpAll)
-	h.Update(is)
-	tk.MustQuery("select table_rows from information_schema.tables where table_name='t'").Check(
-		testkit.Rows("3"))
-	tk.MustExec("insert into t(c, d) values(4, 5)")
-	h.DumpStatsDeltaToKV(statistics.DumpAll)
-	h.Update(is)
-	tk.MustQuery("select table_rows from information_schema.tables where table_name='t'").Check(
-		testkit.Rows("4"))
-	tk.MustExec("delete from t where c >= 3")
-	h.DumpStatsDeltaToKV(statistics.DumpAll)
-	h.Update(is)
-	tk.MustQuery("select table_rows from information_schema.tables where table_name='t'").Check(
-		testkit.Rows("2"))
-	tk.MustExec("delete from t where c=3")
-	h.DumpStatsDeltaToKV(statistics.DumpAll)
-	h.Update(is)
-	tk.MustQuery("select table_rows from information_schema.tables where table_name='t'").Check(
-		testkit.Rows("2"))
 
 	// Test for auto increment ID.
-	tk.MustExec("drop table t")
+	tk.MustExec("drop table if exists t")
 	tk.MustExec("create table t (c int auto_increment primary key, d int)")
 	tk.MustQuery("select auto_increment from information_schema.tables where table_name='t'").Check(
 		testkit.Rows("1"))
@@ -126,6 +84,52 @@ func (s *testSuite) TestDataForTableRowsCountField(c *C) {
 	tk1.MustQuery("select distinct(table_schema) from information_schema.tables").Check(testkit.Rows("INFORMATION_SCHEMA"))
 }
 
+func (s *testSuite) TestDataForTableStatsField(c *C) {
+	testleak.BeforeTest()
+	defer testleak.AfterTest(c)()
+	store, err := mockstore.NewMockTikvStore()
+	c.Assert(err, IsNil)
+	defer store.Close()
+	session.SetStatsLease(0)
+	do, err := session.BootstrapSession(store)
+	c.Assert(err, IsNil)
+	defer do.Close()
+	oldExpiryTime := infoschema.TableStatsCacheExpiry
+	infoschema.TableStatsCacheExpiry = 0
+	defer func() { infoschema.TableStatsCacheExpiry = oldExpiryTime }()
+
+	h := do.StatsHandle()
+	is := do.InfoSchema()
+	tk := testkit.NewTestKit(c, store)
+
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t (c int, d int, e char(5), index idx(e))")
+	h.HandleDDLEvent(<-h.DDLEventCh())
+	tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check(
+		testkit.Rows("0 0 0 0"))
+	tk.MustExec(`insert into t(c, d, e) values(1, 2, "c"), (2, 3, "d"), (3, 4, "e")`)
+	h.DumpStatsDeltaToKV(statistics.DumpAll)
+	h.Update(is)
+	tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check(
+		testkit.Rows("3 17 51 3"))
+	tk.MustExec(`insert into t(c, d, e) values(4, 5, "f")`)
+	h.DumpStatsDeltaToKV(statistics.DumpAll)
+	h.Update(is)
+	tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check(
+		testkit.Rows("4 17 68 4"))
+	tk.MustExec("delete from t where c >= 3")
+	h.DumpStatsDeltaToKV(statistics.DumpAll)
+	h.Update(is)
+	tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check(
+		testkit.Rows("2 17 34 2"))
+	tk.MustExec("delete from t where c=3")
+	h.DumpStatsDeltaToKV(statistics.DumpAll)
+	h.Update(is)
+	tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check(
+		testkit.Rows("2 17 34 2"))
+}
+
 func (s *testSuite) TestCharacterSetCollations(c *C) {
 
 	testleak.BeforeTest()

types/field_type.go

@@ -1414,3 +1414,23 @@ func SetBinChsClnFlag(ft *FieldType) {
 	ft.Collate = charset.CollationBin
 	ft.Flag |= mysql.BinaryFlag
 }
+
+// VarStorageLen indicates this column is a variable length column.
+const VarStorageLen = -1
+
+// StorageLength is the length of stored value for the type.
+func (ft *FieldType) StorageLength() int {
+	switch ft.Tp {
+	case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong,
+		mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeFloat, mysql.TypeYear, mysql.TypeDuration,
+		mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp, mysql.TypeEnum, mysql.TypeSet,
+		mysql.TypeBit:
+		// This may not be the accurate length, because we may encode them as varint.
+		return 8
+	case mysql.TypeNewDecimal:
+		precision, frac := ft.Flen-ft.Decimal, ft.Decimal
+		return precision/digitsPerWord*wordSize + dig2bytes[precision%digitsPerWord] + frac/digitsPerWord*wordSize + dig2bytes[frac%digitsPerWord]
+	default:
+		return VarStorageLen
+	}
+}