diff --git a/infoschema/tables.go b/infoschema/tables.go index 8925393e7a8ae..835c2044e55d5 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -16,6 +16,7 @@ package infoschema import ( "fmt" "sort" + "sync" "time" "github.com/pingcap/tidb/kv" @@ -711,6 +712,111 @@ func getRowCountAllTable(ctx sessionctx.Context) (map[int64]uint64, error) { return rowCountMap, nil } +type tableHistID struct { + tableID int64 + histID int64 +} + +func getColLengthAllTables(ctx sessionctx.Context) (map[tableHistID]uint64, error) { + rows, _, err := ctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(ctx, "select table_id, hist_id, tot_col_size from mysql.stats_histograms where is_index = 0") + if err != nil { + return nil, errors.Trace(err) + } + colLengthMap := make(map[tableHistID]uint64, len(rows)) + for _, row := range rows { + tableID := row.GetInt64(0) + histID := row.GetInt64(1) + totalSize := row.GetInt64(2) + if totalSize < 0 { + totalSize = 0 + } + colLengthMap[tableHistID{tableID: tableID, histID: histID}] = uint64(totalSize) + } + return colLengthMap, nil +} + +func getDataAndIndexLength(info *model.TableInfo, rowCount uint64, columnLengthMap map[tableHistID]uint64) (uint64, uint64) { + columnLength := make(map[string]uint64) + for _, col := range info.Columns { + if col.State != model.StatePublic { + continue + } + length := col.FieldType.StorageLength() + if length != types.VarStorageLen { + columnLength[col.Name.L] = rowCount * uint64(length) + } else { + length := columnLengthMap[tableHistID{tableID: info.ID, histID: col.ID}] + columnLength[col.Name.L] = length + } + } + dataLength, indexLength := uint64(0), uint64(0) + for _, length := range columnLength { + dataLength += length + } + for _, idx := range info.Indices { + if idx.State != model.StatePublic { + continue + } + for _, col := range idx.Columns { + if col.Length == types.UnspecifiedLength { + indexLength += columnLength[col.Name.L] + } else { + indexLength += rowCount * uint64(col.Length) + } + } + } + return dataLength, indexLength +} + +type statsCache struct { + mu sync.Mutex + loading bool + modifyTime time.Time + tableRows map[int64]uint64 + colLength map[tableHistID]uint64 +} + +var tableStatsCache = &statsCache{} + +// TableStatsCacheExpiry is the expiry time for table stats cache. +var TableStatsCacheExpiry = 3 * time.Second + +func (c *statsCache) setLoading(loading bool) { + c.mu.Lock() + c.loading = loading + c.mu.Unlock() +} + +func (c *statsCache) get(ctx sessionctx.Context) (map[int64]uint64, map[tableHistID]uint64, error) { + c.mu.Lock() + if time.Now().Sub(c.modifyTime) < TableStatsCacheExpiry || c.loading { + tableRows, colLength := c.tableRows, c.colLength + c.mu.Unlock() + return tableRows, colLength, nil + } + c.loading = true + c.mu.Unlock() + + tableRows, err := getRowCountAllTable(ctx) + if err != nil { + c.setLoading(false) + return nil, nil, errors.Trace(err) + } + colLength, err := getColLengthAllTables(ctx) + if err != nil { + c.setLoading(false) + return nil, nil, errors.Trace(err) + } + + c.mu.Lock() + c.loading = false + c.tableRows = tableRows + c.colLength = colLength + c.modifyTime = time.Now() + c.mu.Unlock() + return tableRows, colLength, nil +} + func getAutoIncrementID(ctx sessionctx.Context, schema *model.DBInfo, tblInfo *model.TableInfo) (int64, error) { hasAutoIncID := false for _, col := range tblInfo.Cols() { @@ -735,7 +841,7 @@ func getAutoIncrementID(ctx sessionctx.Context, schema *model.DBInfo, tblInfo *m } func dataForTables(ctx sessionctx.Context, schemas []*model.DBInfo) ([][]types.Datum, error) { - tableRowsMap, err := getRowCountAllTable(ctx) + tableRowsMap, colLengthMap, err := tableStatsCache.get(ctx) if err != nil { return nil, errors.Trace(err) } @@ -763,28 +869,34 @@ func dataForTables(ctx sessionctx.Context, schemas []*model.DBInfo) ([][]types.D if err != nil { return nil, errors.Trace(err) } + rowCount := tableRowsMap[table.ID] + dataLength, indexLength := getDataAndIndexLength(table, rowCount, colLengthMap) + avgRowLength := uint64(0) + if rowCount != 0 { + avgRowLength = dataLength / rowCount + } record := types.MakeDatums( - catalogVal, // TABLE_CATALOG - schema.Name.O, // TABLE_SCHEMA - table.Name.O, // TABLE_NAME - "BASE TABLE", // TABLE_TYPE - "InnoDB", // ENGINE - uint64(10), // VERSION - "Compact", // ROW_FORMAT - tableRowsMap[table.ID], // TABLE_ROWS - uint64(0), // AVG_ROW_LENGTH - uint64(16384), // DATA_LENGTH - uint64(0), // MAX_DATA_LENGTH - uint64(0), // INDEX_LENGTH - uint64(0), // DATA_FREE - autoIncID, // AUTO_INCREMENT - createTime, // CREATE_TIME - nil, // UPDATE_TIME - nil, // CHECK_TIME - collation, // TABLE_COLLATION - nil, // CHECKSUM - "", // CREATE_OPTIONS - table.Comment, // TABLE_COMMENT + catalogVal, // TABLE_CATALOG + schema.Name.O, // TABLE_SCHEMA + table.Name.O, // TABLE_NAME + "BASE TABLE", // TABLE_TYPE + "InnoDB", // ENGINE + uint64(10), // VERSION + "Compact", // ROW_FORMAT + rowCount, // TABLE_ROWS + avgRowLength, // AVG_ROW_LENGTH + dataLength, // DATA_LENGTH + uint64(0), // MAX_DATA_LENGTH + indexLength, // INDEX_LENGTH + uint64(0), // DATA_FREE + autoIncID, // AUTO_INCREMENT + createTime, // CREATE_TIME + nil, // UPDATE_TIME + nil, // CHECK_TIME + collation, // TABLE_COLLATION + nil, // CHECKSUM + "", // CREATE_OPTIONS + table.Comment, // TABLE_COMMENT ) rows = append(rows, record) } diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index 249bacf047ea4..fb98618963c89 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -15,6 +15,7 @@ package infoschema_test import ( . "github.com/pingcap/check" + "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/store/mockstore" @@ -50,52 +51,9 @@ func (s *testSuite) TestInfoschemaFieldValue(c *C) { Check(testkit.Rows("3 3 ", "3 3 ", "3 3 ", "3 3 ")) // FIXME: for mysql last two will be "255 255 ", "255 255 " tk.MustQuery("select NUMERIC_SCALE from information_schema.COLUMNS where table_name='floatschema'"). Check(testkit.Rows("", "3")) -} - -func (s *testSuite) TestDataForTableRowsCountField(c *C) { - testleak.BeforeTest() - defer testleak.AfterTest(c)() - store, err := mockstore.NewMockTikvStore() - c.Assert(err, IsNil) - defer store.Close() - session.SetStatsLease(0) - do, err := session.BootstrapSession(store) - c.Assert(err, IsNil) - defer do.Close() - - h := do.StatsHandle() - is := do.InfoSchema() - tk := testkit.NewTestKit(c, store) - - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t (c int, d int)") - h.HandleDDLEvent(<-h.DDLEventCh()) - tk.MustQuery("select table_rows from information_schema.tables where table_name='t'").Check( - testkit.Rows("0")) - tk.MustExec("insert into t(c, d) values(1, 2), (2, 3), (3, 4)") - h.DumpStatsDeltaToKV(statistics.DumpAll) - h.Update(is) - tk.MustQuery("select table_rows from information_schema.tables where table_name='t'").Check( - testkit.Rows("3")) - tk.MustExec("insert into t(c, d) values(4, 5)") - h.DumpStatsDeltaToKV(statistics.DumpAll) - h.Update(is) - tk.MustQuery("select table_rows from information_schema.tables where table_name='t'").Check( - testkit.Rows("4")) - tk.MustExec("delete from t where c >= 3") - h.DumpStatsDeltaToKV(statistics.DumpAll) - h.Update(is) - tk.MustQuery("select table_rows from information_schema.tables where table_name='t'").Check( - testkit.Rows("2")) - tk.MustExec("delete from t where c=3") - h.DumpStatsDeltaToKV(statistics.DumpAll) - h.Update(is) - tk.MustQuery("select table_rows from information_schema.tables where table_name='t'").Check( - testkit.Rows("2")) // Test for auto increment ID. - tk.MustExec("drop table t") + tk.MustExec("drop table if exists t") tk.MustExec("create table t (c int auto_increment primary key, d int)") tk.MustQuery("select auto_increment from information_schema.tables where table_name='t'").Check( testkit.Rows("1")) @@ -126,6 +84,52 @@ func (s *testSuite) TestDataForTableRowsCountField(c *C) { tk1.MustQuery("select distinct(table_schema) from information_schema.tables").Check(testkit.Rows("INFORMATION_SCHEMA")) } +func (s *testSuite) TestDataForTableStatsField(c *C) { + testleak.BeforeTest() + defer testleak.AfterTest(c)() + store, err := mockstore.NewMockTikvStore() + c.Assert(err, IsNil) + defer store.Close() + session.SetStatsLease(0) + do, err := session.BootstrapSession(store) + c.Assert(err, IsNil) + defer do.Close() + oldExpiryTime := infoschema.TableStatsCacheExpiry + infoschema.TableStatsCacheExpiry = 0 + defer func() { infoschema.TableStatsCacheExpiry = oldExpiryTime }() + + h := do.StatsHandle() + is := do.InfoSchema() + tk := testkit.NewTestKit(c, store) + + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (c int, d int, e char(5), index idx(e))") + h.HandleDDLEvent(<-h.DDLEventCh()) + tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check( + testkit.Rows("0 0 0 0")) + tk.MustExec(`insert into t(c, d, e) values(1, 2, "c"), (2, 3, "d"), (3, 4, "e")`) + h.DumpStatsDeltaToKV(statistics.DumpAll) + h.Update(is) + tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check( + testkit.Rows("3 17 51 3")) + tk.MustExec(`insert into t(c, d, e) values(4, 5, "f")`) + h.DumpStatsDeltaToKV(statistics.DumpAll) + h.Update(is) + tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check( + testkit.Rows("4 17 68 4")) + tk.MustExec("delete from t where c >= 3") + h.DumpStatsDeltaToKV(statistics.DumpAll) + h.Update(is) + tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check( + testkit.Rows("2 17 34 2")) + tk.MustExec("delete from t where c=3") + h.DumpStatsDeltaToKV(statistics.DumpAll) + h.Update(is) + tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check( + testkit.Rows("2 17 34 2")) +} + func (s *testSuite) TestCharacterSetCollations(c *C) { testleak.BeforeTest() diff --git a/types/field_type.go b/types/field_type.go index 996fc9afed85a..bb247ba8e7e20 100644 --- a/types/field_type.go +++ b/types/field_type.go @@ -1414,3 +1414,23 @@ func SetBinChsClnFlag(ft *FieldType) { ft.Collate = charset.CollationBin ft.Flag |= mysql.BinaryFlag } + +// VarStorageLen indicates this column is a variable length column. +const VarStorageLen = -1 + +// StorageLength is the length of stored value for the type. +func (ft *FieldType) StorageLength() int { + switch ft.Tp { + case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, + mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeFloat, mysql.TypeYear, mysql.TypeDuration, + mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp, mysql.TypeEnum, mysql.TypeSet, + mysql.TypeBit: + // This may not be the accurate length, because we may encode them as varint. + return 8 + case mysql.TypeNewDecimal: + precision, frac := ft.Flen-ft.Decimal, ft.Decimal + return precision/digitsPerWord*wordSize + dig2bytes[precision%digitsPerWord] + frac/digitsPerWord*wordSize + dig2bytes[frac%digitsPerWord] + default: + return VarStorageLen + } +}