From 961cfb22f81677a10eb3f28151141c71850c721f Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Mon, 22 Oct 2018 15:21:51 +0800 Subject: [PATCH] stats: support show stats for partition table --- executor/show_stats.go | 135 ++++++++++++++++++++---------- executor/show_stats_test.go | 53 +++++++++--- planner/core/planbuilder.go | 16 ++-- server/statistics_handler_test.go | 2 +- 4 files changed, 143 insertions(+), 63 deletions(-) diff --git a/executor/show_stats.go b/executor/show_stats.go index 6c8c4f0ddc258..af947e9e7ad92 100644 --- a/executor/show_stats.go +++ b/executor/show_stats.go @@ -30,34 +30,45 @@ func (e *ShowExec) fetchShowStatsMeta() error { dbs := do.InfoSchema().AllSchemas() for _, db := range dbs { for _, tbl := range db.Tables { - statsTbl := h.GetTableStats(tbl) - if !statsTbl.Pseudo { - e.appendRow([]interface{}{ - db.Name.O, - tbl.Name.O, - e.versionToTime(statsTbl.Version), - statsTbl.ModifyCount, - statsTbl.Count, - }) + pi := tbl.GetPartitionInfo() + if pi == nil { + e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl)) + } else { + for _, def := range pi.Definitions { + e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) + } } } } return nil } +func (e *ShowExec) appendTableForStatsMeta(dbName, tblName, partitionName string, statsTbl *statistics.Table) { + if statsTbl.Pseudo { + return + } + e.appendRow([]interface{}{ + dbName, + tblName, + partitionName, + e.versionToTime(statsTbl.Version), + statsTbl.ModifyCount, + statsTbl.Count, + }) +} + func (e *ShowExec) fetchShowStatsHistogram() error { do := domain.GetDomain(e.ctx) h := do.StatsHandle() dbs := do.InfoSchema().AllSchemas() for _, db := range dbs { for _, tbl := range db.Tables { - statsTbl := h.GetTableStats(tbl) - if !statsTbl.Pseudo { - for _, col := range statsTbl.Columns { - e.histogramToRow(db.Name.O, tbl.Name.O, col.Info.Name.O, 0, col.Histogram, col.AvgColSize(statsTbl.Count)) - } - for _, idx := range statsTbl.Indices { - e.histogramToRow(db.Name.O, tbl.Name.O, idx.Info.Name.O, 1, idx.Histogram, 0) + pi := tbl.GetPartitionInfo() + if pi == nil { + e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl)) + } else { + for _, def := range pi.Definitions { + e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) } } } @@ -65,10 +76,23 @@ func (e *ShowExec) fetchShowStatsHistogram() error { return nil } -func (e *ShowExec) histogramToRow(dbName string, tblName string, colName string, isIndex int, hist statistics.Histogram, avgColSize float64) { +func (e *ShowExec) appendTableForStatsHistograms(dbName, tblName, partitionName string, statsTbl *statistics.Table) { + if statsTbl.Pseudo { + return + } + for _, col := range statsTbl.Columns { + e.histogramToRow(dbName, tblName, partitionName, col.Info.Name.O, 0, col.Histogram, col.AvgColSize(statsTbl.Count)) + } + for _, idx := range statsTbl.Indices { + e.histogramToRow(dbName, tblName, partitionName, idx.Info.Name.O, 1, idx.Histogram, 0) + } +} + +func (e *ShowExec) histogramToRow(dbName, tblName, partitionName, colName string, isIndex int, hist statistics.Histogram, avgColSize float64) { e.appendRow([]interface{}{ dbName, tblName, + partitionName, colName, isIndex, e.versionToTime(hist.LastUpdateVersion), @@ -89,19 +113,12 @@ func (e *ShowExec) fetchShowStatsBuckets() error { dbs := do.InfoSchema().AllSchemas() for _, db := range dbs { for _, tbl := range db.Tables { - statsTbl := h.GetTableStats(tbl) - if !statsTbl.Pseudo { - for _, col := range statsTbl.Columns { - err := e.bucketsToRows(db.Name.O, tbl.Name.O, col.Info.Name.O, 0, col.Histogram) - if err != nil { - return errors.Trace(err) - } - } - for _, idx := range statsTbl.Indices { - err := e.bucketsToRows(db.Name.O, tbl.Name.O, idx.Info.Name.O, len(idx.Info.Columns), idx.Histogram) - if err != nil { - return errors.Trace(err) - } + pi := tbl.GetPartitionInfo() + if pi == nil { + e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl)) + } else { + for _, def := range pi.Definitions { + e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) } } } @@ -109,9 +126,28 @@ func (e *ShowExec) fetchShowStatsBuckets() error { return nil } +func (e *ShowExec) appendTableForStatsBuckets(dbName, tblName, partitionName string, statsTbl *statistics.Table) error { + if statsTbl.Pseudo { + return nil + } + for _, col := range statsTbl.Columns { + err := e.bucketsToRows(dbName, tblName, partitionName, col.Info.Name.O, 0, col.Histogram) + if err != nil { + return errors.Trace(err) + } + } + for _, idx := range statsTbl.Indices { + err := e.bucketsToRows(dbName, tblName, partitionName, idx.Info.Name.O, len(idx.Info.Columns), idx.Histogram) + if err != nil { + return errors.Trace(err) + } + } + return nil +} + // bucketsToRows converts histogram buckets to rows. If the histogram is built from index, then numOfCols equals to number // of index columns, else numOfCols is 0. -func (e *ShowExec) bucketsToRows(dbName, tblName, colName string, numOfCols int, hist statistics.Histogram) error { +func (e *ShowExec) bucketsToRows(dbName, tblName, partitionName, colName string, numOfCols int, hist statistics.Histogram) error { isIndex := 0 if numOfCols > 0 { isIndex = 1 @@ -128,6 +164,7 @@ func (e *ShowExec) bucketsToRows(dbName, tblName, colName string, numOfCols int, e.appendRow([]interface{}{ dbName, tblName, + partitionName, colName, isIndex, i, @@ -146,20 +183,32 @@ func (e *ShowExec) fetchShowStatsHealthy() { dbs := do.InfoSchema().AllSchemas() for _, db := range dbs { for _, tbl := range db.Tables { - statsTbl := h.GetTableStats(tbl) - if !statsTbl.Pseudo { - var healthy int64 - if statsTbl.ModifyCount < statsTbl.Count { - healthy = int64((1.0 - float64(statsTbl.ModifyCount)/float64(statsTbl.Count)) * 100.0) - } else if statsTbl.ModifyCount == 0 { - healthy = 100 + pi := tbl.GetPartitionInfo() + if pi == nil { + e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl)) + } else { + for _, def := range pi.Definitions { + e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) } - e.appendRow([]interface{}{ - db.Name.O, - tbl.Name.O, - healthy, - }) } } } } + +func (e *ShowExec) appendTableForStatsHealthy(dbName, tblName, partitionName string, statsTbl *statistics.Table) { + if statsTbl.Pseudo { + return + } + var healthy int64 + if statsTbl.ModifyCount < statsTbl.Count { + healthy = int64((1.0 - float64(statsTbl.ModifyCount)/float64(statsTbl.Count)) * 100.0) + } else if statsTbl.ModifyCount == 0 { + healthy = 100 + } + e.appendRow([]interface{}{ + dbName, + tblName, + partitionName, + healthy, + }) +} diff --git a/executor/show_stats_test.go b/executor/show_stats_test.go index 03ddf989095f4..c0cc6e70f7cd1 100644 --- a/executor/show_stats_test.go +++ b/executor/show_stats_test.go @@ -44,11 +44,11 @@ func (s *testSuite) TestShowStatsHistograms(c *C) { tk.MustExec("analyze table t") result := tk.MustQuery("show stats_histograms").Sort() c.Assert(len(result.Rows()), Equals, 2) - c.Assert(result.Rows()[0][2], Equals, "a") - c.Assert(result.Rows()[1][2], Equals, "b") + c.Assert(result.Rows()[0][3], Equals, "a") + c.Assert(result.Rows()[1][3], Equals, "b") result = tk.MustQuery("show stats_histograms where column_name = 'a'") c.Assert(len(result.Rows()), Equals, 1) - c.Assert(result.Rows()[0][2], Equals, "a") + c.Assert(result.Rows()[0][3], Equals, "a") } func (s *testSuite) TestShowStatsBuckets(c *C) { @@ -60,9 +60,9 @@ func (s *testSuite) TestShowStatsBuckets(c *C) { tk.MustExec("insert into t values (1,1)") tk.MustExec("analyze table t") result := tk.MustQuery("show stats_buckets").Sort() - result.Sort().Check(testkit.Rows("test t a 0 0 1 1 1 1", "test t b 0 0 1 1 1 1", "test t idx 1 0 1 1 (1, 1) (1, 1)")) + result.Check(testkit.Rows("test t a 0 0 1 1 1 1", "test t b 0 0 1 1 1 1", "test t idx 1 0 1 1 (1, 1) (1, 1)")) result = tk.MustQuery("show stats_buckets where column_name = 'idx'") - result.Check(testkit.Rows("test t idx 1 0 1 1 (1, 1) (1, 1)")) + result.Check(testkit.Rows("test t idx 1 0 1 1 (1, 1) (1, 1)")) } func (s *testSuite) TestShowStatsHealthy(c *C) { @@ -72,22 +72,22 @@ func (s *testSuite) TestShowStatsHealthy(c *C) { tk.MustExec("create table t (a int)") tk.MustExec("create index idx on t(a)") tk.MustExec("analyze table t") - tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100")) + tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100")) tk.MustExec("insert into t values (1), (2)") do, _ := session.GetDomain(s.store) do.StatsHandle().DumpStatsDeltaToKV(statistics.DumpAll) tk.MustExec("analyze table t") - tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100")) + tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100")) tk.MustExec("insert into t values (3), (4), (5), (6), (7), (8), (9), (10)") do.StatsHandle().DumpStatsDeltaToKV(statistics.DumpAll) do.StatsHandle().Update(do.InfoSchema()) - tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 19")) + tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 19")) tk.MustExec("analyze table t") - tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100")) + tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100")) tk.MustExec("delete from t") do.StatsHandle().DumpStatsDeltaToKV(statistics.DumpAll) do.StatsHandle().Update(do.InfoSchema()) - tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 0")) + tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 0")) } func (s *testSuite) TestShowStatsHasNullValue(c *C) { @@ -96,5 +96,36 @@ func (s *testSuite) TestShowStatsHasNullValue(c *C) { tk.MustExec("create table t (a int, index idx(a))") tk.MustExec("insert into t values(NULL)") tk.MustExec("analyze table t") - tk.MustQuery("show stats_buckets").Check(testkit.Rows("test t idx 1 0 1 1 NULL NULL")) + tk.MustQuery("show stats_buckets").Check(testkit.Rows("test t idx 1 0 1 1 NULL NULL")) +} + +func (s *testSuite) TestShowPartitionStats(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("set @@session.tidb_enable_table_partition=1") + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + createTable := `CREATE TABLE t (a int, b int, primary key(a), index idx(b)) + PARTITION BY RANGE ( a ) (PARTITION p0 VALUES LESS THAN (6))` + tk.MustExec(createTable) + tk.MustExec(`insert into t values (1, 1)`) + tk.MustExec("analyze table t") + + result := tk.MustQuery("show stats_meta") + c.Assert(len(result.Rows()), Equals, 1) + c.Assert(result.Rows()[0][0], Equals, "test") + c.Assert(result.Rows()[0][1], Equals, "t") + c.Assert(result.Rows()[0][2], Equals, "p0") + + result = tk.MustQuery("show stats_histograms").Sort() + c.Assert(len(result.Rows()), Equals, 2) + c.Assert(result.Rows()[0][2], Equals, "p0") + c.Assert(result.Rows()[0][3], Equals, "a") + c.Assert(result.Rows()[1][2], Equals, "p0") + c.Assert(result.Rows()[1][3], Equals, "idx") + + result = tk.MustQuery("show stats_buckets").Sort() + result.Check(testkit.Rows("test t p0 a 0 0 1 1 1 1", "test t p0 idx 1 0 1 1 1 1")) + + result = tk.MustQuery("show stats_healthy") + result.Check(testkit.Rows("test t p0 100")) } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 570c67ed87a7c..d3637367fc230 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1634,20 +1634,20 @@ func buildShowSchema(s *ast.ShowStmt) (schema *expression.Schema) { ftypes = []byte{mysql.TypeLonglong, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeLong, mysql.TypeVarchar, mysql.TypeString, mysql.TypeLonglong} case ast.ShowStatsMeta: - names = []string{"Db_name", "Table_name", "Update_time", "Modify_count", "Row_count"} - ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeDatetime, mysql.TypeLonglong, mysql.TypeLonglong} + names = []string{"Db_name", "Table_name", "Partition_name", "Update_time", "Modify_count", "Row_count"} + ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeDatetime, mysql.TypeLonglong, mysql.TypeLonglong} case ast.ShowStatsHistograms: - names = []string{"Db_name", "Table_name", "Column_name", "Is_index", "Update_time", "Distinct_count", "Null_count", "Avg_col_size"} - ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeTiny, mysql.TypeDatetime, + names = []string{"Db_name", "Table_name", "Partition_name", "Column_name", "Is_index", "Update_time", "Distinct_count", "Null_count", "Avg_col_size"} + ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeTiny, mysql.TypeDatetime, mysql.TypeLonglong, mysql.TypeLonglong, mysql.TypeDouble} case ast.ShowStatsBuckets: - names = []string{"Db_name", "Table_name", "Column_name", "Is_index", "Bucket_id", "Count", + names = []string{"Db_name", "Table_name", "Partition_name", "Column_name", "Is_index", "Bucket_id", "Count", "Repeats", "Lower_Bound", "Upper_Bound"} - ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeTiny, mysql.TypeLonglong, + ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeTiny, mysql.TypeLonglong, mysql.TypeLonglong, mysql.TypeLonglong, mysql.TypeVarchar, mysql.TypeVarchar} case ast.ShowStatsHealthy: - names = []string{"Db_name", "Table_name", "Healthy"} - ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeLonglong} + names = []string{"Db_name", "Table_name", "Partition_name", "Healthy"} + ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeLonglong} case ast.ShowProfiles: // ShowProfiles is deprecated. names = []string{"Query_ID", "Duration", "Query"} ftypes = []byte{mysql.TypeLong, mysql.TypeDouble, mysql.TypeVarchar} diff --git a/server/statistics_handler_test.go b/server/statistics_handler_test.go index ce67e3ac6f179..292433e7fd8a4 100644 --- a/server/statistics_handler_test.go +++ b/server/statistics_handler_test.go @@ -153,7 +153,7 @@ func (ds *testDumpStatsSuite) checkData(c *C, path string) { var dbName, tableName string var modifyCount, count int64 var other interface{} - err = rows.Scan(&dbName, &tableName, &other, &modifyCount, &count) + err = rows.Scan(&dbName, &tableName, &other, &other, &modifyCount, &count) dbt.Check(err, IsNil) dbt.Check(dbName, Equals, "tidb") dbt.Check(tableName, Equals, "test")