From 461f4dba4cff29d20eb87671f7d387cdbce4f71d Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Sun, 28 Apr 2019 17:31:20 +0800 Subject: [PATCH 01/11] server, statistics: support dump history stats --- server/http_handler.go | 1 + server/http_status.go | 1 + server/statistics_handler.go | 85 +++++++++++++++++++++++++++++-- server/statistics_handler_test.go | 71 +++++++++++++++++++++++--- statistics/handle/dump.go | 11 ++-- statistics/handle/dump_test.go | 6 +-- statistics/handle/handle.go | 13 +++-- 7 files changed, 166 insertions(+), 22 deletions(-) diff --git a/server/http_handler.go b/server/http_handler.go index 3a2db8174d8e4..0f65c2da63766 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -72,6 +72,7 @@ const ( pColumnFlag = "colFlag" pColumnLen = "colLen" pRowBin = "rowBin" + pSnapshot = "snapshot" ) // For query string diff --git a/server/http_status.go b/server/http_status.go index 62bd51c548c4f..6ea8d519332b8 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -58,6 +58,7 @@ func (s *Server) startHTTPServer() { // HTTP path for dump statistics. router.Handle("/stats/dump/{db}/{table}", s.newStatsHandler()).Name("StatsDump") + router.Handle("/stats/dump/{db}/{table}/{snapshot}", s.newStatsHistoryHandler()).Name("StatsHistoryDump") router.Handle("/settings", settingsHandler{}).Name("Settings") router.Handle("/binlog/recover", binlogRecover{}).Name("BinlogRecover") diff --git a/server/statistics_handler.go b/server/statistics_handler.go index 40761114073f1..8cf278117ddca 100644 --- a/server/statistics_handler.go +++ b/server/statistics_handler.go @@ -14,12 +14,18 @@ package server import ( - "net/http" - "github.com/gorilla/mux" "github.com/pingcap/parser/model" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/gcutil" + "github.com/pingcap/tidb/util/sqlexec" + "github.com/sirupsen/logrus" + "net/http" + "time" ) // StatsHandler is the handler for dumping statistics. @@ -51,7 +57,7 @@ func (sh StatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { if err != nil { writeError(w, err) } else { - js, err := h.DumpStatsToJSON(params[pDBName], tbl.Meta()) + js, err := h.DumpStatsToJSON(params[pDBName], tbl.Meta(), nil) if err != nil { writeError(w, err) } else { @@ -59,3 +65,76 @@ func (sh StatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } } + +// StatsHistoryHandler is the handler for dumping statistics. +type StatsHistoryHandler struct { + do *domain.Domain +} + +func (s *Server) newStatsHistoryHandler() *StatsHistoryHandler { + store, ok := s.driver.(*TiDBDriver) + if !ok { + panic("Illegal driver") + } + + do, err := session.GetDomain(store.store) + if err != nil { + panic("Failed to get domain") + } + return &StatsHistoryHandler{do} +} + +func (sh StatsHistoryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/json") + + params := mux.Vars(req) + se, err := session.CreateSession(sh.do.Store()) + if err != nil { + writeError(w, err) + return + } + se.GetSessionVars().StmtCtx.TimeZone = time.Local + logrus.Warning("time.Local", time.Local) + logrus.Warning("params[pSnapshot] ", params[pSnapshot]) + t, err := types.ParseTime(se.GetSessionVars().StmtCtx, params[pSnapshot], mysql.TypeTimestamp, 6) + if err != nil { + writeError(w, err) + return + } + t1, err := t.Time.GoTime(time.Local) + if err != nil { + writeError(w, err) + return + } + logrus.Warning("t1", t1.String()) + snapshot := variable.GoTimeToTS(t1) + err = gcutil.ValidateSnapshot(se, snapshot) + if err != nil { + writeError(w, err) + return + } + + is, err := sh.do.GetSnapshotInfoSchema(snapshot) + if err != nil { + writeError(w, err) + return + } + if is == nil { + logrus.Warning("is is nil") + } + h := sh.do.StatsHandle() + logrus.Warning("is ", is, " params ", params) + tbl, err := is.TableByName(model.NewCIStr(params[pDBName]), model.NewCIStr(params[pTableName])) + if err != nil { + writeError(w, err) + return + } + se.GetSessionVars().SnapshotInfoschema, se.GetSessionVars().SnapshotTS = is, snapshot + historyStatsExec := se.(sqlexec.RestrictedSQLExecutor) + js, err := h.DumpStatsToJSON(params[pDBName], tbl.Meta(), historyStatsExec) + if err != nil { + writeError(w, err) + } else { + writeData(w, js) + } +} diff --git a/server/statistics_handler_test.go b/server/statistics_handler_test.go index 8a7b5ff47113e..ee5b287778904 100644 --- a/server/statistics_handler_test.go +++ b/server/statistics_handler_test.go @@ -19,6 +19,7 @@ import ( "io/ioutil" "net/http" "os" + "time" "github.com/go-sql-driver/mysql" "github.com/gorilla/mux" @@ -105,6 +106,37 @@ func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) { c.Assert(err, IsNil) fp.Write(js) ds.checkData(c, path) + + // sleep for 1 seconds to ensure the existence of tidb.test + time.Sleep(time.Second) + timeBeforeDropStats := time.Now() + snapshot := timeBeforeDropStats.Format("20060102150405") + ds.prepare4DumpHistoryStats(c) + + // test dump history stats + resp1, err := http.Get("http://127.0.0.1:10090/stats/dump/tidb/test") + c.Assert(err, IsNil) + defer resp1.Body.Close() + js, err = ioutil.ReadAll(resp1.Body) + c.Assert(err, IsNil) + c.Assert(string(js), Equals, "null") + + path1 := "/tmp/stats_history.json" + fp1, err := os.Create(path1) + c.Assert(err, IsNil) + c.Assert(fp1, NotNil) + defer func() { + c.Assert(fp1.Close(), IsNil) + c.Assert(os.Remove(path1), IsNil) + }() + + resp1, err = http.Get("http://127.0.0.1:10090/stats/dump/tidb/test/" + snapshot) + c.Assert(err, IsNil) + + js, err = ioutil.ReadAll(resp1.Body) + c.Assert(err, IsNil) + fp1.Write(js) + ds.checkData(c, path1) } func (ds *testDumpStatsSuite) prepareData(c *C) { @@ -128,6 +160,25 @@ func (ds *testDumpStatsSuite) prepareData(c *C) { c.Assert(h.Update(is), IsNil) } +func (ds *testDumpStatsSuite) prepare4DumpHistoryStats(c *C) { + db, err := sql.Open("mysql", getDSN()) + c.Assert(err, IsNil, Commentf("Error connecting")) + defer db.Close() + + dbt := &DBTest{c, db} + + safePointName := "tikv_gc_safe_point" + safePointValue := "20060102-15:04:05 -0700" + safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" + updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') + ON DUPLICATE KEY + UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) + dbt.mustExec(updateSafePoint) + + dbt.mustExec("drop table tidb.test") + dbt.mustExec("create table tidb.test (a int, b varchar(20))") +} + func (ds *testDumpStatsSuite) checkData(c *C, path string) { db, err := sql.Open("mysql", getDSN(func(config *mysql.Config) { config.AllowAllFiles = true @@ -135,13 +186,7 @@ func (ds *testDumpStatsSuite) checkData(c *C, path string) { })) c.Assert(err, IsNil, Commentf("Error connecting")) dbt := &DBTest{c, db} - defer func() { - dbt.mustExec("drop database tidb") - dbt.mustExec("truncate table mysql.stats_meta") - dbt.mustExec("truncate table mysql.stats_histograms") - dbt.mustExec("truncate table mysql.stats_buckets") - db.Close() - }() + defer db.Close() dbt.mustExec("use tidb") dbt.mustExec("drop stats test") @@ -160,3 +205,15 @@ func (ds *testDumpStatsSuite) checkData(c *C, path string) { dbt.Check(modifyCount, Equals, int64(3)) dbt.Check(count, Equals, int64(4)) } + +func (ds *testDumpStatsSuite) clearData(c *C, path string) { + db, err := sql.Open("mysql", getDSN()) + c.Assert(err, IsNil, Commentf("Error connecting")) + defer db.Close() + + dbt := &DBTest{c, db} + dbt.mustExec("drop database tidb") + dbt.mustExec("truncate table mysql.stats_meta") + dbt.mustExec("truncate table mysql.stats_histograms") + dbt.mustExec("truncate table mysql.stats_buckets") +} diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go index 1a4098bfde8f4..0df43a303df39 100644 --- a/statistics/handle/dump.go +++ b/statistics/handle/dump.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tipb/go-tipb" ) @@ -59,10 +60,10 @@ func dumpJSONCol(hist *statistics.Histogram, CMSketch *statistics.CMSketch) *jso } // DumpStatsToJSON dumps statistic to json. -func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JSONTable, error) { +func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo, historyStatsExec sqlexec.RestrictedSQLExecutor) (*JSONTable, error) { pi := tableInfo.GetPartitionInfo() if pi == nil { - return h.tableStatsToJSON(dbName, tableInfo, tableInfo.ID) + return h.tableStatsToJSON(dbName, tableInfo, tableInfo.ID, historyStatsExec) } jsonTbl := &JSONTable{ DatabaseName: dbName, @@ -70,7 +71,7 @@ func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JS Partitions: make(map[string]*JSONTable, len(pi.Definitions)), } for _, def := range pi.Definitions { - tbl, err := h.tableStatsToJSON(dbName, tableInfo, def.ID) + tbl, err := h.tableStatsToJSON(dbName, tableInfo, def.ID, historyStatsExec) if err != nil { return nil, errors.Trace(err) } @@ -82,8 +83,8 @@ func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JS return jsonTbl, nil } -func (h *Handle) tableStatsToJSON(dbName string, tableInfo *model.TableInfo, physicalID int64) (*JSONTable, error) { - tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, true) +func (h *Handle) tableStatsToJSON(dbName string, tableInfo *model.TableInfo, physicalID int64, historyStatsExec sqlexec.RestrictedSQLExecutor) (*JSONTable, error) { + tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, true, historyStatsExec) if err != nil { return nil, errors.Trace(err) } diff --git a/statistics/handle/dump_test.go b/statistics/handle/dump_test.go index a56e1163de907..ddbc0c3abb734 100644 --- a/statistics/handle/dump_test.go +++ b/statistics/handle/dump_test.go @@ -40,7 +40,7 @@ func (s *testStatsSuite) TestConversion(c *C) { tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) - jsonTbl, err := h.DumpStatsToJSON("test", tableInfo.Meta()) + jsonTbl, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil) c.Assert(err, IsNil) loadTbl, err := handle.TableStatsFromJSON(tableInfo.Meta(), tableInfo.Meta().ID, jsonTbl) c.Assert(err, IsNil) @@ -78,7 +78,7 @@ PARTITION BY RANGE ( a ) ( table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tableInfo := table.Meta() - jsonTbl, err := h.DumpStatsToJSON("test", tableInfo) + jsonTbl, err := h.DumpStatsToJSON("test", tableInfo, nil) c.Assert(err, IsNil) pi := tableInfo.GetPartitionInfo() originTables := make([]*statistics.Table, 0, len(pi.Definitions)) @@ -113,6 +113,6 @@ func (s *testStatsSuite) TestDumpAlteredTable(c *C) { tk.MustExec("alter table t drop column a") table, err := s.do.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) - _, err = h.DumpStatsToJSON("test", table.Meta()) + _, err = h.DumpStatsToJSON("test", table.Meta(), nil) c.Assert(err, IsNil) } diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 7545d22e35cdf..d65ae587bf0da 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -160,7 +160,7 @@ func (h *Handle) Update(is infoschema.InfoSchema) error { continue } tableInfo := table.Meta() - tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, false) + tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, false, nil) // Error is not nil may mean that there are some ddl changes on this table, we will not update it. if err != nil { logutil.Logger(context.Background()).Debug("error occurred when read table stats", zap.String("table", tableInfo.Name.O), zap.Error(err)) @@ -452,7 +452,7 @@ func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table, } // tableStatsFromStorage loads table stats info from storage. -func (h *Handle) tableStatsFromStorage(tableInfo *model.TableInfo, physicalID int64, loadAll bool) (*statistics.Table, error) { +func (h *Handle) tableStatsFromStorage(tableInfo *model.TableInfo, physicalID int64, loadAll bool, historyStatsExec sqlexec.RestrictedSQLExecutor) (_ *statistics.Table, err error) { table, ok := h.StatsCache.Load().(StatsCache)[physicalID] // If table stats is pseudo, we also need to copy it, since we will use the column stats when // the average error rate of it is small. @@ -472,9 +472,14 @@ func (h *Handle) tableStatsFromStorage(tableInfo *model.TableInfo, physicalID in } table.Pseudo = false selSQL := fmt.Sprintf("select table_id, is_index, hist_id, distinct_count, version, null_count, tot_col_size, stats_ver, flag, correlation from mysql.stats_histograms where table_id = %d", physicalID) - rows, _, err := h.restrictedExec.ExecRestrictedSQL(nil, selSQL) + var rows []chunk.Row + if historyStatsExec != nil { + rows, _, err = historyStatsExec.ExecRestrictedSQLWithSnapshot(nil, selSQL) + } else { + rows, _, err = h.restrictedExec.ExecRestrictedSQL(nil, selSQL) + } if err != nil { - return nil, errors.Trace(err) + return nil, err } // Check deleted table. if len(rows) == 0 { From b57698932fcc687854ea7e57fa9dd8d7250c0034 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Sun, 28 Apr 2019 17:32:57 +0800 Subject: [PATCH 02/11] format --- server/statistics_handler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/statistics_handler.go b/server/statistics_handler.go index 8cf278117ddca..a8f35d72e3e09 100644 --- a/server/statistics_handler.go +++ b/server/statistics_handler.go @@ -14,6 +14,8 @@ package server import ( + "time" + "github.com/gorilla/mux" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -25,7 +27,6 @@ import ( "github.com/pingcap/tidb/util/sqlexec" "github.com/sirupsen/logrus" "net/http" - "time" ) // StatsHandler is the handler for dumping statistics. From 09cf2fd1e1a2fb069d9cf11ab92be93122a8f581 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Sun, 28 Apr 2019 17:33:10 +0800 Subject: [PATCH 03/11] format --- server/statistics_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/statistics_handler.go b/server/statistics_handler.go index a8f35d72e3e09..9c5d3df3bc7c3 100644 --- a/server/statistics_handler.go +++ b/server/statistics_handler.go @@ -15,6 +15,7 @@ package server import ( "time" + "net/http" "github.com/gorilla/mux" "github.com/pingcap/parser/model" @@ -26,7 +27,6 @@ import ( "github.com/pingcap/tidb/util/gcutil" "github.com/pingcap/tidb/util/sqlexec" "github.com/sirupsen/logrus" - "net/http" ) // StatsHandler is the handler for dumping statistics. From c5e3d91c7f8f24bb3c290d68ed6fe2972923244b Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Sun, 28 Apr 2019 17:42:40 +0800 Subject: [PATCH 04/11] remove useless code --- server/statistics_handler.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/statistics_handler.go b/server/statistics_handler.go index 9c5d3df3bc7c3..0d5e9a63b5e8f 100644 --- a/server/statistics_handler.go +++ b/server/statistics_handler.go @@ -26,7 +26,6 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/gcutil" "github.com/pingcap/tidb/util/sqlexec" - "github.com/sirupsen/logrus" ) // StatsHandler is the handler for dumping statistics. @@ -95,8 +94,6 @@ func (sh StatsHistoryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request return } se.GetSessionVars().StmtCtx.TimeZone = time.Local - logrus.Warning("time.Local", time.Local) - logrus.Warning("params[pSnapshot] ", params[pSnapshot]) t, err := types.ParseTime(se.GetSessionVars().StmtCtx, params[pSnapshot], mysql.TypeTimestamp, 6) if err != nil { writeError(w, err) @@ -107,7 +104,6 @@ func (sh StatsHistoryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request writeError(w, err) return } - logrus.Warning("t1", t1.String()) snapshot := variable.GoTimeToTS(t1) err = gcutil.ValidateSnapshot(se, snapshot) if err != nil { @@ -120,11 +116,7 @@ func (sh StatsHistoryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request writeError(w, err) return } - if is == nil { - logrus.Warning("is is nil") - } h := sh.do.StatsHandle() - logrus.Warning("is ", is, " params ", params) tbl, err := is.TableByName(model.NewCIStr(params[pDBName]), model.NewCIStr(params[pTableName])) if err != nil { writeError(w, err) From 24630d73d607a8171b954140d10ae5768c2ccf30 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Sun, 28 Apr 2019 18:38:06 +0800 Subject: [PATCH 05/11] go fmt --- server/statistics_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/statistics_handler.go b/server/statistics_handler.go index 0d5e9a63b5e8f..b80674c2ef028 100644 --- a/server/statistics_handler.go +++ b/server/statistics_handler.go @@ -14,8 +14,8 @@ package server import ( - "time" "net/http" + "time" "github.com/gorilla/mux" "github.com/pingcap/parser/model" From 59dbcaaadde961c008fcccd8f24c4808b269c41f Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Mon, 29 Apr 2019 13:10:01 +0800 Subject: [PATCH 06/11] add doc --- docs/tidb_http_api.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/tidb_http_api.md b/docs/tidb_http_api.md index 1db55ec710643..97fb0b8e41c44 100644 --- a/docs/tidb_http_api.md +++ b/docs/tidb_http_api.md @@ -396,3 +396,15 @@ timezone.* Param: - seconds: profile time(s), default is 10s. + +1. Get statistics data of specified table. + + ```shell + curl http://{TiDBIP}:10080/stats/dump/{db}/{table} + ``` + +1. Get statistics data of specific table and timestamp. + + ```shell + curl http://{TiDBIP}:10080/stats/dump/{db}/{table}/{yyyyMMddHHmmss} + ``` \ No newline at end of file From ce5fbd362d865a4c58085c02ee5d45f272b81ac2 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Mon, 29 Apr 2019 13:13:18 +0800 Subject: [PATCH 07/11] fix ci --- statistics/handle/dump_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statistics/handle/dump_test.go b/statistics/handle/dump_test.go index 0aaab1fd3ec51..7f080b92ad32c 100644 --- a/statistics/handle/dump_test.go +++ b/statistics/handle/dump_test.go @@ -150,7 +150,7 @@ func (s *testStatsSuite) TestDumpCMSketchWithTopN(c *C) { c.Assert(cmsFromStore, NotNil) c.Check(cms.Equal(cmsFromStore), IsTrue) - jsonTable, err := h.DumpStatsToJSON("test", tableInfo) + jsonTable, err := h.DumpStatsToJSON("test", tableInfo, nil) c.Check(err, IsNil) err = h.LoadStatsFromJSON(is, jsonTable) c.Check(err, IsNil) From 83e5e8f90711e064c812e205a3448bed39597f0e Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Mon, 29 Apr 2019 15:50:16 +0800 Subject: [PATCH 08/11] address comment --- server/statistics_handler.go | 7 ++++-- statistics/handle/handle.go | 42 ++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/server/statistics_handler.go b/server/statistics_handler.go index b80674c2ef028..4bf7e9c832c4c 100644 --- a/server/statistics_handler.go +++ b/server/statistics_handler.go @@ -17,6 +17,7 @@ import ( "net/http" "time" + "context" "github.com/gorilla/mux" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -25,7 +26,9 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/gcutil" + "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/sqlexec" + "go.uber.org/zap" ) // StatsHandler is the handler for dumping statistics. @@ -36,12 +39,12 @@ type StatsHandler struct { func (s *Server) newStatsHandler() *StatsHandler { store, ok := s.driver.(*TiDBDriver) if !ok { - panic("Illegal driver") + logutil.Logger(context.Background()).Error("Illegal driver") } do, err := session.GetDomain(store.store) if err != nil { - panic("Failed to get domain") + logutil.Logger(context.Background()).Error("Failed to get domain", zap.Error(err)) } return &StatsHandler{do} } diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 88779d9be8729..f1ced3119bc11 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -21,6 +21,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/ddl/util" @@ -264,11 +265,11 @@ func (h *Handle) LoadNeededHistograms() error { statistics.HistogramNeededColumns.Delete(col) continue } - hg, err := h.histogramFromStorage(col.TableID, c.ID, &c.Info.FieldType, c.NDV, 0, c.LastUpdateVersion, c.NullCount, c.TotColSize, c.Correlation) + hg, err := h.histogramFromStorage(col.TableID, c.ID, &c.Info.FieldType, c.NDV, 0, c.LastUpdateVersion, c.NullCount, c.TotColSize, c.Correlation, nil) if err != nil { return errors.Trace(err) } - cms, err := h.cmSketchFromStorage(col.TableID, 0, col.ColumnID) + cms, err := h.cmSketchFromStorage(col.TableID, 0, col.ColumnID, nil) if err != nil { return errors.Trace(err) } @@ -316,9 +317,14 @@ func (h *Handle) FlushStats() { } } -func (h *Handle) cmSketchFromStorage(tblID int64, isIndex, histID int64) (*statistics.CMSketch, error) { +func (h *Handle) cmSketchFromStorage(tblID int64, isIndex, histID int64, historyStatsExec sqlexec.RestrictedSQLExecutor) (_ *statistics.CMSketch, err error) { selSQL := fmt.Sprintf("select cm_sketch from mysql.stats_histograms where table_id = %d and is_index = %d and hist_id = %d", tblID, isIndex, histID) - rows, _, err := h.restrictedExec.ExecRestrictedSQL(nil, selSQL) + var rows []chunk.Row + if historyStatsExec != nil { + rows, _, err = historyStatsExec.ExecRestrictedSQLWithSnapshot(nil, selSQL) + } else { + rows, _, err = h.restrictedExec.ExecRestrictedSQL(nil, selSQL) + } if err != nil { return nil, errors.Trace(err) } @@ -328,7 +334,7 @@ func (h *Handle) cmSketchFromStorage(tblID int64, isIndex, histID int64) (*stati return statistics.LoadCMSketchWithTopN(h.restrictedExec, tblID, isIndex, histID, rows[0].GetBytes(0)) } -func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, tableInfo *model.TableInfo) error { +func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, tableInfo *model.TableInfo, historyStatsExec sqlexec.RestrictedSQLExecutor) error { histID := row.GetInt64(2) distinct := row.GetInt64(3) histVer := row.GetUint64(4) @@ -347,11 +353,11 @@ func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, t continue } if idx == nil || idx.LastUpdateVersion < histVer { - hg, err := h.histogramFromStorage(table.PhysicalID, histID, types.NewFieldType(mysql.TypeBlob), distinct, 1, histVer, nullCount, 0, 0) + hg, err := h.histogramFromStorage(table.PhysicalID, histID, types.NewFieldType(mysql.TypeBlob), distinct, 1, histVer, nullCount, 0, 0, historyStatsExec) if err != nil { return errors.Trace(err) } - cms, err := h.cmSketchFromStorage(table.PhysicalID, 1, idxInfo.ID) + cms, err := h.cmSketchFromStorage(table.PhysicalID, 1, idxInfo.ID, historyStatsExec) if err != nil { return errors.Trace(err) } @@ -367,7 +373,7 @@ func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, t return nil } -func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table, tableInfo *model.TableInfo, loadAll bool) error { +func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table, tableInfo *model.TableInfo, loadAll bool, historyStatsExec sqlexec.RestrictedSQLExecutor) error { histID := row.GetInt64(2) distinct := row.GetInt64(3) histVer := row.GetUint64(4) @@ -414,11 +420,11 @@ func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table, break } if col == nil || col.LastUpdateVersion < histVer || loadAll { - hg, err := h.histogramFromStorage(table.PhysicalID, histID, &colInfo.FieldType, distinct, 0, histVer, nullCount, totColSize, correlation) + hg, err := h.histogramFromStorage(table.PhysicalID, histID, &colInfo.FieldType, distinct, 0, histVer, nullCount, totColSize, correlation, historyStatsExec) if err != nil { return errors.Trace(err) } - cms, err := h.cmSketchFromStorage(table.PhysicalID, 0, colInfo.ID) + cms, err := h.cmSketchFromStorage(table.PhysicalID, 0, colInfo.ID, historyStatsExec) if err != nil { return errors.Trace(err) } @@ -487,11 +493,11 @@ func (h *Handle) tableStatsFromStorage(tableInfo *model.TableInfo, physicalID in } for _, row := range rows { if row.GetInt64(1) > 0 { - if err := h.indexStatsFromStorage(row, table, tableInfo); err != nil { + if err := h.indexStatsFromStorage(row, table, tableInfo, historyStatsExec); err != nil { return nil, errors.Trace(err) } } else { - if err := h.columnStatsFromStorage(row, table, tableInfo, loadAll); err != nil { + if err := h.columnStatsFromStorage(row, table, tableInfo, loadAll, historyStatsExec); err != nil { return nil, errors.Trace(err) } } @@ -610,9 +616,17 @@ func (h *Handle) SaveMetaToStorage(tableID, count, modifyCount int64) (err error return } -func (h *Handle) histogramFromStorage(tableID int64, colID int64, tp *types.FieldType, distinct int64, isIndex int, ver uint64, nullCount int64, totColSize int64, corr float64) (*statistics.Histogram, error) { +func (h *Handle) histogramFromStorage(tableID int64, colID int64, tp *types.FieldType, distinct int64, isIndex int, ver uint64, nullCount int64, totColSize int64, corr float64, historyStatsExec sqlexec.RestrictedSQLExecutor) (_ *statistics.Histogram, err error) { selSQL := fmt.Sprintf("select count, repeats, lower_bound, upper_bound from mysql.stats_buckets where table_id = %d and is_index = %d and hist_id = %d order by bucket_id", tableID, isIndex, colID) - rows, fields, err := h.restrictedExec.ExecRestrictedSQL(nil, selSQL) + var ( + rows []chunk.Row + fields []*ast.ResultField + ) + if historyStatsExec != nil { + rows, fields, err = historyStatsExec.ExecRestrictedSQLWithSnapshot(nil, selSQL) + } else { + rows, fields, err = h.restrictedExec.ExecRestrictedSQL(nil, selSQL) + } if err != nil { return nil, errors.Trace(err) } From 42d045df3aa498e95c1d4007104e88f933a7636b Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Tue, 7 May 2019 11:26:09 +0800 Subject: [PATCH 09/11] address comment --- server/statistics_handler.go | 16 +++++++++++----- server/statistics_handler_test.go | 2 ++ statistics/handle/dump.go | 11 +++++++---- statistics/handle/handle.go | 22 +++++++++++++++++++++- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/server/statistics_handler.go b/server/statistics_handler.go index 4bf7e9c832c4c..24d9e7a202aa7 100644 --- a/server/statistics_handler.go +++ b/server/statistics_handler.go @@ -14,10 +14,10 @@ package server import ( + "errors" "net/http" "time" - "context" "github.com/gorilla/mux" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -26,9 +26,7 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/gcutil" - "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/sqlexec" - "go.uber.org/zap" ) // StatsHandler is the handler for dumping statistics. @@ -39,17 +37,21 @@ type StatsHandler struct { func (s *Server) newStatsHandler() *StatsHandler { store, ok := s.driver.(*TiDBDriver) if !ok { - logutil.Logger(context.Background()).Error("Illegal driver") + panic("Illegal driver") } do, err := session.GetDomain(store.store) if err != nil { - logutil.Logger(context.Background()).Error("Failed to get domain", zap.Error(err)) + panic("Failed to get domain") } return &StatsHandler{do} } func (sh StatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodGet { + writeError(w, errors.New("only http GET method is supported")) + return + } w.Header().Set("Content-Type", "application/json") params := mux.Vars(req) @@ -88,6 +90,10 @@ func (s *Server) newStatsHistoryHandler() *StatsHistoryHandler { } func (sh StatsHistoryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodGet { + writeError(w, errors.New("only http GET method is supported")) + return + } w.Header().Set("Content-Type", "application/json") params := mux.Vars(req) diff --git a/server/statistics_handler_test.go b/server/statistics_handler_test.go index ee5b287778904..965c66332ff45 100644 --- a/server/statistics_handler_test.go +++ b/server/statistics_handler_test.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/tidb/statistics/handle" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/mockstore/mocktikv" + "github.com/sirupsen/logrus" ) type testDumpStatsSuite struct { @@ -130,6 +131,7 @@ func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) { c.Assert(os.Remove(path1), IsNil) }() + logrus.Warning("after snapshot") resp1, err = http.Get("http://127.0.0.1:10090/stats/dump/tidb/test/" + snapshot) c.Assert(err, IsNil) diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go index 0df43a303df39..baae7bf23eeb8 100644 --- a/statistics/handle/dump.go +++ b/statistics/handle/dump.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tipb/go-tipb" + "github.com/sirupsen/logrus" ) // JSONTable is used for dumping statistics. @@ -85,12 +86,14 @@ func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo, hist func (h *Handle) tableStatsToJSON(dbName string, tableInfo *model.TableInfo, physicalID int64, historyStatsExec sqlexec.RestrictedSQLExecutor) (*JSONTable, error) { tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, true, historyStatsExec) - if err != nil { - return nil, errors.Trace(err) + if err != nil || tbl == nil { + return nil, err } - if tbl == nil { - return nil, nil + tbl.Version, tbl.ModifyCount, tbl.Count, err = h.statsMetaByTableIDFromStorage(physicalID, historyStatsExec) + if err != nil { + return nil, err } + logrus.Warning("tbl.ModifyCount", tbl.ModifyCount, "tbl.Indices", len(tbl.Indices)) jsonTbl := &JSONTable{ DatabaseName: dbName, TableName: tableInfo.Name.L, diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 10d2b0c6ed1b1..a014af2a1c943 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/sqlexec" + "github.com/sirupsen/logrus" atomic2 "go.uber.org/atomic" "go.uber.org/zap" ) @@ -354,6 +355,7 @@ func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, t continue } if idx == nil || idx.LastUpdateVersion < histVer { + logrus.Warning("go into this") hg, err := h.histogramFromStorage(table.PhysicalID, histID, types.NewFieldType(mysql.TypeBlob), distinct, 1, histVer, nullCount, 0, 0, historyStatsExec) if err != nil { return errors.Trace(err) @@ -404,6 +406,7 @@ func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table, !isHandle && (col == nil || col.Len() == 0 && col.LastUpdateVersion < histVer) && !loadAll + logrus.Warning("notNeedLoad", notNeedLoad) if notNeedLoad { count, err := h.columnCountFromStorage(table.PhysicalID, histID) if err != nil { @@ -463,7 +466,7 @@ func (h *Handle) tableStatsFromStorage(tableInfo *model.TableInfo, physicalID in table, ok := h.StatsCache.Load().(StatsCache)[physicalID] // If table stats is pseudo, we also need to copy it, since we will use the column stats when // the average error rate of it is small. - if !ok { + if !ok || historyStatsExec != nil { histColl := statistics.HistColl{ PhysicalID: physicalID, HavePhysicalID: true, @@ -673,3 +676,20 @@ func (h *Handle) columnCountFromStorage(tableID, colID int64) (int64, error) { } return rows[0].GetMyDecimal(0).ToInt() } + +func (h *Handle) statsMetaByTableIDFromStorage(tableID int64, historyStatsExec sqlexec.RestrictedSQLExecutor) (version uint64, modifyCount, count int64, err error) { + selSQL := fmt.Sprintf("SELECT version, modify_count, count from mysql.stats_meta where table_id = %d order by version", tableID) + var rows []chunk.Row + if historyStatsExec == nil { + rows, _, err = h.restrictedExec.ExecRestrictedSQL(nil, selSQL) + } else { + rows, _, err = historyStatsExec.ExecRestrictedSQLWithSnapshot(nil, selSQL) + } + if err != nil || len(rows) == 0 { + return + } + version = rows[0].GetUint64(0) + modifyCount = rows[0].GetInt64(1) + count = rows[0].GetInt64(2) + return +} From 0952fd1f644cd50168a6d35702176f31648dfa76 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Tue, 7 May 2019 11:32:21 +0800 Subject: [PATCH 10/11] remove useless code --- docs/tidb_http_api.md | 5 ++++- server/statistics_handler_test.go | 2 -- statistics/handle/dump.go | 2 -- statistics/handle/handle.go | 3 --- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/tidb_http_api.md b/docs/tidb_http_api.md index 97fb0b8e41c44..e2a1bef7fcb7a 100644 --- a/docs/tidb_http_api.md +++ b/docs/tidb_http_api.md @@ -407,4 +407,7 @@ timezone.* ```shell curl http://{TiDBIP}:10080/stats/dump/{db}/{table}/{yyyyMMddHHmmss} - ``` \ No newline at end of file + ``` + ```shell + curl http://{TiDBIP}:10080/stats/dump/{db}/{table}/{yyyy-MM-dd HH:mm:ss} + ``` diff --git a/server/statistics_handler_test.go b/server/statistics_handler_test.go index 965c66332ff45..ee5b287778904 100644 --- a/server/statistics_handler_test.go +++ b/server/statistics_handler_test.go @@ -31,7 +31,6 @@ import ( "github.com/pingcap/tidb/statistics/handle" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/mockstore/mocktikv" - "github.com/sirupsen/logrus" ) type testDumpStatsSuite struct { @@ -131,7 +130,6 @@ func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) { c.Assert(os.Remove(path1), IsNil) }() - logrus.Warning("after snapshot") resp1, err = http.Get("http://127.0.0.1:10090/stats/dump/tidb/test/" + snapshot) c.Assert(err, IsNil) diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go index baae7bf23eeb8..7c6a931f9c501 100644 --- a/statistics/handle/dump.go +++ b/statistics/handle/dump.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tipb/go-tipb" - "github.com/sirupsen/logrus" ) // JSONTable is used for dumping statistics. @@ -93,7 +92,6 @@ func (h *Handle) tableStatsToJSON(dbName string, tableInfo *model.TableInfo, phy if err != nil { return nil, err } - logrus.Warning("tbl.ModifyCount", tbl.ModifyCount, "tbl.Indices", len(tbl.Indices)) jsonTbl := &JSONTable{ DatabaseName: dbName, TableName: tableInfo.Name.L, diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index a014af2a1c943..6cfbf16450bcc 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -35,7 +35,6 @@ import ( "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/sqlexec" - "github.com/sirupsen/logrus" atomic2 "go.uber.org/atomic" "go.uber.org/zap" ) @@ -355,7 +354,6 @@ func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, t continue } if idx == nil || idx.LastUpdateVersion < histVer { - logrus.Warning("go into this") hg, err := h.histogramFromStorage(table.PhysicalID, histID, types.NewFieldType(mysql.TypeBlob), distinct, 1, histVer, nullCount, 0, 0, historyStatsExec) if err != nil { return errors.Trace(err) @@ -406,7 +404,6 @@ func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table, !isHandle && (col == nil || col.Len() == 0 && col.LastUpdateVersion < histVer) && !loadAll - logrus.Warning("notNeedLoad", notNeedLoad) if notNeedLoad { count, err := h.columnCountFromStorage(table.PhysicalID, histID) if err != nil { From a913b808f929a133ee4bcde6401072504be14512 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Tue, 7 May 2019 13:20:13 +0800 Subject: [PATCH 11/11] address comment --- server/statistics_handler.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/server/statistics_handler.go b/server/statistics_handler.go index 24d9e7a202aa7..b80674c2ef028 100644 --- a/server/statistics_handler.go +++ b/server/statistics_handler.go @@ -14,7 +14,6 @@ package server import ( - "errors" "net/http" "time" @@ -48,10 +47,6 @@ func (s *Server) newStatsHandler() *StatsHandler { } func (sh StatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { - writeError(w, errors.New("only http GET method is supported")) - return - } w.Header().Set("Content-Type", "application/json") params := mux.Vars(req) @@ -90,10 +85,6 @@ func (s *Server) newStatsHistoryHandler() *StatsHistoryHandler { } func (sh StatsHistoryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { - writeError(w, errors.New("only http GET method is supported")) - return - } w.Header().Set("Content-Type", "application/json") params := mux.Vars(req)