From d668844d4aeadc8097ab44f72d427495a74aa8a5 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 28 Aug 2019 15:56:30 +0800 Subject: [PATCH 01/26] some implementation for statement summary --- config/config.go | 11 ++++ config/config.toml.example | 7 ++ infoschema/perfschema/const.go | 5 ++ infoschema/perfschema/tables.go | 55 ++++++++++++++++ util/stmtsummary/statement_summary.go | 92 +++++++++++++++++++++++++++ 5 files changed, 170 insertions(+) create mode 100644 util/stmtsummary/statement_summary.go diff --git a/config/config.go b/config/config.go index e9bd29edd393f..b014953223ab9 100644 --- a/config/config.go +++ b/config/config.go @@ -96,6 +96,7 @@ type Config struct { EnableTableLock bool `toml:"enable-table-lock" json:"enable-table-lock"` DelayCleanTableLock uint64 `toml:"delay-clean-table-lock" json:"delay-clean-table-lock"` SplitRegionMaxNum uint64 `toml:"split-region-max-num" json:"split-region-max-num"` + StmtSummary StmtSummary `toml:"stmt-summary" json:"stmt-summary"` } // Log is the log section of config. @@ -316,6 +317,12 @@ type PessimisticTxn struct { TTL string `toml:"ttl" json:"ttl"` } +type StmtSummary struct { + Enable bool `toml:"enable" json:"enable"` + // How many statements can be kept in the table + MaxStmtCount uint16 `toml:"max-stmt-count" json:"max-stmt-count"` +} + var defaultConf = Config{ Host: "0.0.0.0", AdvertiseAddress: "", @@ -410,6 +417,10 @@ var defaultConf = Config{ MaxRetryCount: 256, TTL: "40s", }, + StmtSummary: StmtSummary{ + Enable: false, + MaxStmtCount: 100, + }, } var ( diff --git a/config/config.toml.example b/config/config.toml.example index 2997fcabe746f..ae3920ed7eee6 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -315,3 +315,10 @@ max-retry-count = 256 # default TTL in milliseconds for pessimistic lock. # The value must between "15s" and "120s". ttl = "40s" + +[stmt-summary] +# enable statement summary. +enable = false + +# max count of statement summary kept in memory. +max-stmt-count = 100 diff --git a/infoschema/perfschema/const.go b/infoschema/perfschema/const.go index a5764ce79d7fc..a49f1efa87a19 100644 --- a/infoschema/perfschema/const.go +++ b/infoschema/perfschema/const.go @@ -39,6 +39,7 @@ var perfSchemaTables = []string{ tableStagesCurrent, tableStagesHistory, tableStagesHistoryLong, + tableEventsStatementsSummaryByDigest, } // tableGlobalStatus contains the column name definitions for table global_status, same as MySQL. @@ -374,3 +375,7 @@ const tableStagesHistoryLong = "CREATE TABLE if not exists performance_schema.ev "WORK_ESTIMATED BIGINT(20) UNSIGNED," + "NESTING_EVENT_ID BIGINT(20) UNSIGNED," + "NESTING_EVENT_TYPE ENUM('TRANSACTION','STATEMENT','STAGE'));" + +const tableEventsStatementsSummaryByDigest = "CREATE TABLE if not exists events_statements_summary_by_digest (" + + "SCHEMA_NAME varchar(64) DEFAULT NULL," + + "DIGEST varchar(64) DEFAULT NULL);" diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go index dbe3e68155659..b1f30c3bc9db2 100644 --- a/infoschema/perfschema/tables.go +++ b/infoschema/perfschema/tables.go @@ -16,8 +16,15 @@ package perfschema import ( "github.com/pingcap/parser/model" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/types" +) + +const ( + tableNameEventsStatementsSummaryByDigest = "events_statements_summary_by_digest" ) // perfSchemaTable stands for the fake table all its data is in the memory. @@ -77,3 +84,51 @@ func (vt *perfSchemaTable) GetPhysicalID() int64 { func (vt *perfSchemaTable) Meta() *model.TableInfo { return vt.meta } + +func (vt *perfSchemaTable) getRows(ctx sessionctx.Context, cols []*table.Column) (fullRows [][]types.Datum, err error) { + switch vt.meta.Name.O { + case tableNameEventsStatementsSummaryByDigest: + fullRows = dataForEventsStatementsSummaryByDigest() + } + if err != nil { + return nil, err + } + if len(cols) == len(vt.cols) { + return + } + rows := make([][]types.Datum, len(fullRows)) + for i, fullRow := range fullRows { + row := make([]types.Datum, len(cols)) + for j, col := range cols { + row[j] = fullRow[col.Offset] + } + rows[i] = row + } + return rows, nil +} + +// IterRecords implements table.Table IterRecords interface. +func (vt *perfSchemaTable) IterRecords(ctx sessionctx.Context, startKey kv.Key, cols []*table.Column, + fn table.RecordIterFunc) error { + if len(startKey) != 0 { + return table.ErrUnsupportedOp + } + rows, err := vt.getRows(ctx, cols) + if err != nil { + return err + } + for i, row := range rows { + more, err := fn(int64(i), row, cols) + if err != nil { + return err + } + if !more { + break + } + } + return nil +} + +func dataForEventsStatementsSummaryByDigest() [][]types.Datum { + return nil +} diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go new file mode 100644 index 0000000000000..17d09c67628bc --- /dev/null +++ b/util/stmtsummary/statement_summary.go @@ -0,0 +1,92 @@ +// Copyright 2016 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package stmtsummary + +import ( + "sync" + + "github.com/pingcap/parser" + "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/types" +) + +type StmtSummaryByDigest struct { + sync.RWMutex + summaryMap map[string]*stmtSummary +} + +// Summary of each type of statements. +type stmtSummary struct { + schemaName string + normalizedSql string + sampleSql string + sumTotalLatency uint64 + maxLatency uint64 + minLatency uint64 + sumRowsAffected uint64 + sumRowsSent uint64 +} + +// Used for recording execution status of each statement. +type StmtExecInfo struct { + schemaName string + originalSql string + totalLatency uint64 + rowsAffected uint64 + rowsSent uint64 +} + +// Convert StmtExecInfo to stmtSummary +func convertStmtExecInfoToSummary(sei *StmtExecInfo, normalizedSql string) *stmtSummary { + return nil +} + +// Add a new StmtExecInfo to stmtSummary +func addStmtExecInfoToSummary(sei *StmtExecInfo, ss *stmtSummary) { + return +} + +func (ss *StmtSummaryByDigest) AddStatement(sei *StmtExecInfo) { + normalizeSQL := parser.Normalize(sei.originalSql) + digest := parser.DigestHash(normalizeSQL) + + ss.Lock() + summary, ok := ss.summaryMap[digest] + if ok { + addStmtExecInfoToSummary(sei, summary) + } else { + maxStmtCount := config.GetGlobalConfig().StmtSummary.MaxStmtCount + if len(ss.summaryMap) >= int(maxStmtCount) { + ss.removeLeastUsed() + } + + summary = convertStmtExecInfoToSummary(sei, normalizeSQL) + ss.summaryMap[digest] = summary + } + ss.Unlock() +} + +func (ss *StmtSummaryByDigest) removeLeastUsed() { + +} + +func (ss *StmtSummaryByDigest) ToDatum() [][]types.Datum { + var rows [][]types.Datum + + ss.RLock() + // TODO + ss.RUnlock() + + return rows +} From ecc243c585aa8f79f9c5c9bc6619bf924130e11e Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Fri, 30 Aug 2019 18:14:53 +0800 Subject: [PATCH 02/26] a demo for statement summary --- config/config.go | 7 +- config/config.toml.example | 6 +- executor/adapter.go | 22 ++++ infoschema/perfschema/const.go | 16 ++- infoschema/perfschema/tables.go | 93 +++++++++++++++- session/session.go | 1 + sessionctx/variable/session.go | 6 + sessionctx/variable/sysvar.go | 1 + sessionctx/variable/tidb_vars.go | 4 + sessionctx/variable/varsutil.go | 3 +- util/kvcache/simple_lru.go | 21 ++++ util/stmtsummary/statement_summary.go | 151 ++++++++++++++++++++------ 12 files changed, 282 insertions(+), 49 deletions(-) diff --git a/config/config.go b/config/config.go index b014953223ab9..c421939fb6d47 100644 --- a/config/config.go +++ b/config/config.go @@ -318,9 +318,10 @@ type PessimisticTxn struct { } type StmtSummary struct { - Enable bool `toml:"enable" json:"enable"` // How many statements can be kept in the table - MaxStmtCount uint16 `toml:"max-stmt-count" json:"max-stmt-count"` + MaxStmtCount uint `toml:"max-stmt-count" json:"max-stmt-count"` + // How long can a normalizedSQL / sampleSQL be + MaxSqlLength uint `toml:"max-sql-length" json:"max-sql-length"` } var defaultConf = Config{ @@ -418,8 +419,8 @@ var defaultConf = Config{ TTL: "40s", }, StmtSummary: StmtSummary{ - Enable: false, MaxStmtCount: 100, + MaxSqlLength: 4096, }, } diff --git a/config/config.toml.example b/config/config.toml.example index ae3920ed7eee6..28dab02b95159 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -317,8 +317,8 @@ max-retry-count = 256 ttl = "40s" [stmt-summary] -# enable statement summary. -enable = false - # max count of statement summary kept in memory. max-stmt-count = 100 + +# max length of displayed normalized sql and sample sql. +max-sql-length = 4096 diff --git a/executor/adapter.go b/executor/adapter.go index c6e9c9dbd129b..8fac43552243b 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -46,6 +46,7 @@ import ( "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/sqlexec" + "github.com/pingcap/tidb/util/stmtsummary" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -180,6 +181,7 @@ func (a *recordSet) Close() error { a.stmt.LogSlowQuery(a.txnStartTS, a.lastErr == nil) a.stmt.Ctx.GetSessionVars().PrevStmt = a.stmt.OriginText() a.stmt.logAudit() + a.stmt.summaryStmt() return err } @@ -779,6 +781,26 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool) { } } +func (a *ExecStmt) summaryStmt() { + sessVars := a.Ctx.GetSessionVars() + if !sessVars.EnableStmtSummary || sessVars.InRestrictedSQL { + return + } + stmtCtx := sessVars.StmtCtx + normalizedSQL, digest := stmtCtx.SQLDigest() + costTime := time.Since(a.StartTime) + stmtsummary.StmtSummary.AddStatement(&stmtsummary.StmtExecInfo{ + SchemaName: sessVars.CurrentDB, + OriginalSQL: a.Text, + NormalizedSQL: normalizedSQL, + Digest: digest, + TotalLatency: uint64(costTime.Nanoseconds()), + AffectedRows: stmtCtx.AffectedRows(), + SentRows: 0, + StartTime: a.StartTime, + }) +} + // IsPointGetWithPKOrUniqueKeyByAutoCommit returns true when meets following conditions: // 1. ctx is auto commit tagged // 2. txn is not valid diff --git a/infoschema/perfschema/const.go b/infoschema/perfschema/const.go index a49f1efa87a19..2df0496b494e6 100644 --- a/infoschema/perfschema/const.go +++ b/infoschema/perfschema/const.go @@ -376,6 +376,18 @@ const tableStagesHistoryLong = "CREATE TABLE if not exists performance_schema.ev "NESTING_EVENT_ID BIGINT(20) UNSIGNED," + "NESTING_EVENT_TYPE ENUM('TRANSACTION','STATEMENT','STAGE'));" +// tableEventsStatementsSummaryByDigest contains the column name definitions for table +// events_statements_summary_by_digest, same as MySQL. const tableEventsStatementsSummaryByDigest = "CREATE TABLE if not exists events_statements_summary_by_digest (" + - "SCHEMA_NAME varchar(64) DEFAULT NULL," + - "DIGEST varchar(64) DEFAULT NULL);" + "SCHEMA_NAME VARCHAR(64) DEFAULT NULL," + + "DIGEST VARCHAR(64) DEFAULT NULL," + + "DIGEST_TEXT VARCHAR(4096) DEFAULT NULL," + + "EXEC_COUNT BIGINT(20) UNSIGNED NOT NULL," + + "SUM_LATENCY BIGINT(20) UNSIGNED NOT NULL," + + "MAX_LATENCY BIGINT(20) UNSIGNED NOT NULL," + + "MIN_LATENCY BIGINT(20) UNSIGNED NOT NULL," + + "AVG_LATENCY BIGINT(20) UNSIGNED NOT NULL," + + "SUM_ROWS_AFFECTED BIGINT(20) UNSIGNED NOT NULL," + + "FIRST_SEEN TIMESTAMP(6) NOT NULL," + + "LAST_SEEN TIMESTAMP(6) NOT NULL," + + "QUERY_SAMPLE_TEXT VARCHAR(4096) DEFAULT NULL);" diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go index b1f30c3bc9db2..ffc1e1d8d4b72 100644 --- a/infoschema/perfschema/tables.go +++ b/infoschema/perfschema/tables.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/stmtsummary" ) const ( @@ -88,10 +89,7 @@ func (vt *perfSchemaTable) Meta() *model.TableInfo { func (vt *perfSchemaTable) getRows(ctx sessionctx.Context, cols []*table.Column) (fullRows [][]types.Datum, err error) { switch vt.meta.Name.O { case tableNameEventsStatementsSummaryByDigest: - fullRows = dataForEventsStatementsSummaryByDigest() - } - if err != nil { - return nil, err + fullRows = dataForEventsStatementsSummaryByDigest(ctx) } if len(cols) == len(vt.cols) { return @@ -129,6 +127,91 @@ func (vt *perfSchemaTable) IterRecords(ctx sessionctx.Context, startKey kv.Key, return nil } -func dataForEventsStatementsSummaryByDigest() [][]types.Datum { +// RowWithCols implements table.Table RowWithCols interface. +func (vt *perfSchemaTable) RowWithCols(ctx sessionctx.Context, h int64, cols []*table.Column) ([]types.Datum, error) { + return nil, table.ErrUnsupportedOp +} + +// Row implements table.Table Row interface. +func (vt *perfSchemaTable) Row(ctx sessionctx.Context, h int64) ([]types.Datum, error) { + return nil, table.ErrUnsupportedOp +} + +// Indices implements table.Table Indices interface. +func (vt *perfSchemaTable) Indices() []table.Index { + return nil +} + +// WritableIndices implements table.Table WritableIndices interface. +func (vt *perfSchemaTable) WritableIndices() []table.Index { + return nil +} + +// DeletableIndices implements table.Table DeletableIndices interface. +func (vt *perfSchemaTable) DeletableIndices() []table.Index { + return nil +} + +// RecordPrefix implements table.Table RecordPrefix interface. +func (vt *perfSchemaTable) RecordPrefix() kv.Key { return nil } + +// IndexPrefix implements table.Table IndexPrefix interface. +func (vt *perfSchemaTable) IndexPrefix() kv.Key { + return nil +} + +// FirstKey implements table.Table FirstKey interface. +func (vt *perfSchemaTable) FirstKey() kv.Key { + return nil +} + +// RecordKey implements table.Table RecordKey interface. +func (vt *perfSchemaTable) RecordKey(h int64) kv.Key { + return nil +} + +// AddRecord implements table.Table AddRecord interface. +func (vt *perfSchemaTable) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...table.AddRecordOption) (recordID int64, err error) { + return 0, table.ErrUnsupportedOp +} + +// RemoveRecord implements table.Table RemoveRecord interface. +func (vt *perfSchemaTable) RemoveRecord(ctx sessionctx.Context, h int64, r []types.Datum) error { + return table.ErrUnsupportedOp +} + +// UpdateRecord implements table.Table UpdateRecord interface. +func (vt *perfSchemaTable) UpdateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datum, touched []bool) error { + return table.ErrUnsupportedOp +} + +// AllocHandle implements table.Table AllocHandle interface. +func (vt *perfSchemaTable) AllocHandle(ctx sessionctx.Context) (int64, error) { + return 0, table.ErrUnsupportedOp +} + +// Allocator implements table.Table Allocator interface. +func (vt *perfSchemaTable) Allocator(ctx sessionctx.Context) autoid.Allocator { + return nil +} + +// RebaseAutoID implements table.Table RebaseAutoID interface. +func (vt *perfSchemaTable) RebaseAutoID(ctx sessionctx.Context, newBase int64, isSetStep bool) error { + return table.ErrUnsupportedOp +} + +// Seek implements table.Table Seek interface. +func (vt *perfSchemaTable) Seek(ctx sessionctx.Context, h int64) (int64, bool, error) { + return 0, false, table.ErrUnsupportedOp +} + +// Type implements table.Table Type interface. +func (vt *perfSchemaTable) Type() table.Type { + return table.VirtualTable +} + +func dataForEventsStatementsSummaryByDigest(ctx sessionctx.Context) [][]types.Datum { + return stmtsummary.StmtSummary.ToDatum() +} diff --git a/session/session.go b/session/session.go index cdc3783d2920f..4c25828592d63 100644 --- a/session/session.go +++ b/session/session.go @@ -1720,6 +1720,7 @@ var builtinGlobalVariable = []string{ variable.TiDBEnableNoopFuncs, variable.TiDBEnableIndexMerge, variable.TiDBTxnMode, + variable.TiDBEnableStmtSummary, } var ( diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index e402b060e02cf..6d424070a3d8b 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -420,6 +420,9 @@ type SessionVars struct { // AllowRemoveAutoInc indicates whether a user can drop the auto_increment column attribute or not. AllowRemoveAutoInc bool + + // Enable statement summary + EnableStmtSummary bool } // ConnectionInfo present connection used by audit. @@ -477,6 +480,7 @@ func NewSessionVars() *SessionVars { EnableNoopFuncs: DefTiDBEnableNoopFuncs, ReplicaRead: kv.ReplicaReadLeader, AllowRemoveAutoInc: DefTiDBAllowRemoveAutoInc, + EnableStmtSummary: DefTiDBEnableStmtSummary, } vars.Concurrency = Concurrency{ IndexLookupConcurrency: DefIndexLookupConcurrency, @@ -863,6 +867,8 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { } case TiDBAllowRemoveAutoInc: s.AllowRemoveAutoInc = TiDBOptOn(val) + case TiDBEnableStmtSummary: + s.EnableStmtSummary = TiDBOptOn(val) } s.systems[name] = val return nil diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index e28182eba7faf..283be87f0aafc 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -707,6 +707,7 @@ var defaultSysVars = []*SysVar{ {ScopeGlobal | ScopeSession, TiDBEnableNoopFuncs, BoolToIntStr(DefTiDBEnableNoopFuncs)}, {ScopeSession, TiDBReplicaRead, "leader"}, {ScopeSession, TiDBAllowRemoveAutoInc, BoolToIntStr(DefTiDBAllowRemoveAutoInc)}, + {ScopeGlobal, TiDBEnableStmtSummary, BoolToIntStr(DefTiDBEnableStmtSummary)}, } // SynonymsSysVariables is synonyms of system variables. diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 48a454cac6734..83dfe6b8dd909 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -296,6 +296,9 @@ const ( // TiDBEnableNoopFuncs set true will enable using fake funcs(like get_lock release_lock) TiDBEnableNoopFuncs = "tidb_enable_noop_functions" + + // TiDBEnableStmtSummary indicates to enable statement summary + TiDBEnableStmtSummary = "tidb_enable_stmt_summary" ) // Default TiDB system variable values. @@ -362,6 +365,7 @@ const ( DefWaitSplitRegionTimeout = 300 // 300s DefTiDBEnableNoopFuncs = false DefTiDBAllowRemoveAutoInc = false + DefTiDBEnableStmtSummary = false ) // Process global variables. diff --git a/sessionctx/variable/varsutil.go b/sessionctx/variable/varsutil.go index 563f6ceb031d4..dcbd97ae1198b 100644 --- a/sessionctx/variable/varsutil.go +++ b/sessionctx/variable/varsutil.go @@ -380,7 +380,8 @@ func ValidateSetSystemVar(vars *SessionVars, name string, value string) (string, TiDBBatchInsert, TiDBDisableTxnAutoRetry, TiDBEnableStreaming, TiDBBatchDelete, TiDBBatchCommit, TiDBEnableCascadesPlanner, TiDBEnableWindowFunction, TiDBCheckMb4ValueInUTF8, TiDBLowResolutionTSO, TiDBEnableIndexMerge, TiDBEnableNoopFuncs, - TiDBScatterRegion, TiDBGeneralLog, TiDBConstraintCheckInPlace, TiDBEnableVectorizedExpression: + TiDBScatterRegion, TiDBGeneralLog, TiDBConstraintCheckInPlace, TiDBEnableVectorizedExpression, + TiDBEnableStmtSummary: fallthrough case GeneralLog, AvoidTemporalUpgrade, BigTables, CheckProxyUsers, LogBin, CoreFile, EndMakersInJSON, SQLLogBin, OfflineMode, PseudoSlaveMode, LowPriorityUpdates, diff --git a/util/kvcache/simple_lru.go b/util/kvcache/simple_lru.go index 7120e3a5abb7c..9dd87f9eeaf71 100644 --- a/util/kvcache/simple_lru.go +++ b/util/kvcache/simple_lru.go @@ -38,6 +38,7 @@ type cacheEntry struct { type SimpleLRUCache struct { capacity uint size uint + // 0 indicates no quota quota uint64 guard float64 elements map[string]*list.Element @@ -88,6 +89,16 @@ func (l *SimpleLRUCache) Put(key Key, value Value) { l.elements[hash] = element l.size++ + if l.quota <= 0 { + if l.size > l.capacity { + lru := l.cache.Back() + l.cache.Remove(lru) + delete(l.elements, string(lru.Value.(*cacheEntry).key.Hash())) + l.size-- + } + return + } + memUsed, err := memory.MemUsed() if err != nil { l.DeleteAll() @@ -137,3 +148,13 @@ func (l *SimpleLRUCache) DeleteAll() { func (l *SimpleLRUCache) Size() int { return int(l.size) } + +// Values return all values in cache. +func (l *SimpleLRUCache) Values() []Value { + values := make([]Value, 0, l.cache.Len()) + for ele := l.cache.Front(); ele != nil; ele = ele.Next() { + value := ele.Value.(*cacheEntry).value + values = append(values, value) + } + return values +} diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index 17d09c67628bc..f05c5c2608120 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -14,78 +14,159 @@ package stmtsummary import ( + "strings" "sync" + "time" - "github.com/pingcap/parser" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/hack" + "github.com/pingcap/tidb/util/kvcache" ) -type StmtSummaryByDigest struct { +type stmtSummaryCacheKey struct { + // Same statements may appear in different schema, but they refer to different tables. + schemaName string + digest string + hash []byte +} + +// Hash implements SimpleLRUCache.Key +func (key *stmtSummaryCacheKey) Hash() []byte { + if len(key.hash) == 0 { + key.hash = make([]byte, 0, len(key.schemaName)+len(key.digest)) + key.hash = append(key.hash, hack.Slice(key.digest)...) + key.hash = append(key.hash, hack.Slice(strings.ToLower(key.schemaName))...) + } + return key.hash +} + +// A LRU cache that stores statement summary. +type stmtSummaryByDigest struct { sync.RWMutex - summaryMap map[string]*stmtSummary + summaryMap *kvcache.SimpleLRUCache } +var StmtSummary = NewStmtSummaryByDigest() + // Summary of each type of statements. type stmtSummary struct { schemaName string - normalizedSql string - sampleSql string + digest string + normalizedSQL string + sampleSQL string + execCount uint64 sumTotalLatency uint64 maxLatency uint64 minLatency uint64 - sumRowsAffected uint64 - sumRowsSent uint64 + sumAffectedRows uint64 + // Number of rows sent to client. + sumSentRows uint64 + // The first time this type of SQL executes. + firstSeen time.Time + // The last time this type of SQL executes. + lastSeen time.Time } // Used for recording execution status of each statement. type StmtExecInfo struct { - schemaName string - originalSql string - totalLatency uint64 - rowsAffected uint64 - rowsSent uint64 + SchemaName string + OriginalSQL string + NormalizedSQL string + Digest string + TotalLatency uint64 + AffectedRows uint64 + // Number of rows sent to client. + SentRows uint64 + StartTime time.Time +} + +func NewStmtSummaryByDigest() *stmtSummaryByDigest { + maxStmtCount := config.GetGlobalConfig().StmtSummary.MaxStmtCount + return &stmtSummaryByDigest{ + summaryMap: kvcache.NewSimpleLRUCache(maxStmtCount, 0, 0), + } } // Convert StmtExecInfo to stmtSummary -func convertStmtExecInfoToSummary(sei *StmtExecInfo, normalizedSql string) *stmtSummary { - return nil +func convertStmtExecInfoToSummary(sei *StmtExecInfo) *stmtSummary { + return &stmtSummary{ + schemaName: sei.SchemaName, + digest: sei.Digest, + normalizedSQL: sei.NormalizedSQL, + sampleSQL: sei.OriginalSQL, + execCount: 1, + sumTotalLatency: sei.TotalLatency, + maxLatency: sei.TotalLatency, + minLatency: sei.TotalLatency, + sumAffectedRows: sei.AffectedRows, + sumSentRows: sei.SentRows, + firstSeen: sei.StartTime, + lastSeen: sei.StartTime, + } } // Add a new StmtExecInfo to stmtSummary func addStmtExecInfoToSummary(sei *StmtExecInfo, ss *stmtSummary) { - return + ss.sumTotalLatency += sei.TotalLatency + ss.execCount++ + if sei.TotalLatency > ss.maxLatency { + ss.maxLatency = sei.TotalLatency + } + if sei.TotalLatency < ss.minLatency { + ss.minLatency = sei.TotalLatency + } + // TODO: uint64 overflow + ss.sumAffectedRows += sei.AffectedRows + ss.sumSentRows += sei.SentRows + if ss.lastSeen.Before(sei.StartTime) { + ss.lastSeen = sei.StartTime + } } -func (ss *StmtSummaryByDigest) AddStatement(sei *StmtExecInfo) { - normalizeSQL := parser.Normalize(sei.originalSql) - digest := parser.DigestHash(normalizeSQL) +// Add a statement to statement summary. +func (ss *stmtSummaryByDigest) AddStatement(sei *StmtExecInfo) { + key := &stmtSummaryCacheKey{ + schemaName: sei.SchemaName, + digest: sei.Digest, + } ss.Lock() - summary, ok := ss.summaryMap[digest] + summary, ok := ss.summaryMap.Get(key) if ok { - addStmtExecInfoToSummary(sei, summary) + addStmtExecInfoToSummary(sei, summary.(*stmtSummary)) } else { - maxStmtCount := config.GetGlobalConfig().StmtSummary.MaxStmtCount - if len(ss.summaryMap) >= int(maxStmtCount) { - ss.removeLeastUsed() - } - - summary = convertStmtExecInfoToSummary(sei, normalizeSQL) - ss.summaryMap[digest] = summary + summary = convertStmtExecInfoToSummary(sei) + ss.summaryMap.Put(key, summary) } ss.Unlock() } -func (ss *StmtSummaryByDigest) removeLeastUsed() { - -} - -func (ss *StmtSummaryByDigest) ToDatum() [][]types.Datum { - var rows [][]types.Datum - +// Convert statement summary to Datum +func (ss *stmtSummaryByDigest) ToDatum() [][]types.Datum { ss.RLock() - // TODO + rows := make([][]types.Datum, 0, ss.summaryMap.Size()) + // Summary of each statement may be modified at the same time, + // so surround the whole block with read lock. + for _, value := range ss.summaryMap.Values() { + summary := value.(*stmtSummary) + record := types.MakeDatums( + summary.schemaName, + summary.digest, + summary.normalizedSQL, + summary.execCount, + summary.sumTotalLatency, + summary.maxLatency, + summary.minLatency, + summary.sumTotalLatency/summary.execCount, // AVG_LATENCY + summary.sumAffectedRows, + types.Time{Time: types.FromGoTime(summary.firstSeen), Type: mysql.TypeTimestamp}, + types.Time{Time: types.FromGoTime(summary.lastSeen), Type: mysql.TypeTimestamp}, + summary.sampleSQL, + ) + rows = append(rows, record) + } ss.RUnlock() return rows From 43c91afbba4245a3adf7ca73f3533561bd1726d5 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Mon, 2 Sep 2019 15:03:41 +0800 Subject: [PATCH 03/26] add MaxSqlLength limit for sampleSQL in stmtSummary --- config/config.go | 6 +++--- executor/adapter.go | 4 ++-- tidb-server/main.go | 2 +- util/stmtsummary/statement_summary.go | 29 +++++++++++++++++++-------- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/config/config.go b/config/config.go index c421939fb6d47..657bf31cefe84 100644 --- a/config/config.go +++ b/config/config.go @@ -93,9 +93,9 @@ type Config struct { TreatOldVersionUTF8AsUTF8MB4 bool `toml:"treat-old-version-utf8-as-utf8mb4" json:"treat-old-version-utf8-as-utf8mb4"` // EnableTableLock indicate whether enable table lock. // TODO: remove this after table lock features stable. - EnableTableLock bool `toml:"enable-table-lock" json:"enable-table-lock"` - DelayCleanTableLock uint64 `toml:"delay-clean-table-lock" json:"delay-clean-table-lock"` - SplitRegionMaxNum uint64 `toml:"split-region-max-num" json:"split-region-max-num"` + EnableTableLock bool `toml:"enable-table-lock" json:"enable-table-lock"` + DelayCleanTableLock uint64 `toml:"delay-clean-table-lock" json:"delay-clean-table-lock"` + SplitRegionMaxNum uint64 `toml:"split-region-max-num" json:"split-region-max-num"` StmtSummary StmtSummary `toml:"stmt-summary" json:"stmt-summary"` } diff --git a/executor/adapter.go b/executor/adapter.go index 8fac43552243b..71825c24af0f0 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -788,7 +788,7 @@ func (a *ExecStmt) summaryStmt() { } stmtCtx := sessVars.StmtCtx normalizedSQL, digest := stmtCtx.SQLDigest() - costTime := time.Since(a.StartTime) + costTime := time.Since(sessVars.StartTime) stmtsummary.StmtSummary.AddStatement(&stmtsummary.StmtExecInfo{ SchemaName: sessVars.CurrentDB, OriginalSQL: a.Text, @@ -797,7 +797,7 @@ func (a *ExecStmt) summaryStmt() { TotalLatency: uint64(costTime.Nanoseconds()), AffectedRows: stmtCtx.AffectedRows(), SentRows: 0, - StartTime: a.StartTime, + StartTime: sessVars.StartTime, }) } diff --git a/tidb-server/main.go b/tidb-server/main.go index cdfe515e030d0..8530bff678996 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -362,7 +362,7 @@ func loadConfig() string { // hotReloadConfigItems lists all config items which support hot-reload. var hotReloadConfigItems = []string{"Performance.MaxProcs", "Performance.MaxMemory", "Performance.CrossJoin", "Performance.FeedbackProbability", "Performance.QueryFeedbackLimit", "Performance.PseudoEstimateRatio", - "OOMAction", "MemQuotaQuery"} + "OOMAction", "MemQuotaQuery", "StmtSummary.MaxStmtCount", "StmtSummary.MaxSqlLength"} func reloadConfig(nc, c *config.Config) { // Just a part of config items need to be reload explicitly. diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index f05c5c2608120..005b6b05c4604 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -29,7 +29,9 @@ type stmtSummaryCacheKey struct { // Same statements may appear in different schema, but they refer to different tables. schemaName string digest string - hash []byte + // TODO: add plan digest + // `hash` is the hash value of this object + hash []byte } // Hash implements SimpleLRUCache.Key @@ -57,7 +59,7 @@ type stmtSummary struct { normalizedSQL string sampleSQL string execCount uint64 - sumTotalLatency uint64 + sumLatency uint64 maxLatency uint64 minLatency uint64 sumAffectedRows uint64 @@ -91,13 +93,24 @@ func NewStmtSummaryByDigest() *stmtSummaryByDigest { // Convert StmtExecInfo to stmtSummary func convertStmtExecInfoToSummary(sei *StmtExecInfo) *stmtSummary { + // Trim SQL to size MaxSqlLength + maxSqlLength := config.GetGlobalConfig().StmtSummary.MaxSqlLength + normalizedSQL := sei.NormalizedSQL + if len(normalizedSQL) > int(maxSqlLength) { + normalizedSQL = normalizedSQL[:maxSqlLength] + } + sampleSQL := sei.OriginalSQL + if len(sampleSQL) > int(maxSqlLength) { + sampleSQL = sampleSQL[:maxSqlLength] + } + return &stmtSummary{ schemaName: sei.SchemaName, digest: sei.Digest, - normalizedSQL: sei.NormalizedSQL, - sampleSQL: sei.OriginalSQL, + normalizedSQL: normalizedSQL, + sampleSQL: sampleSQL, execCount: 1, - sumTotalLatency: sei.TotalLatency, + sumLatency: sei.TotalLatency, maxLatency: sei.TotalLatency, minLatency: sei.TotalLatency, sumAffectedRows: sei.AffectedRows, @@ -109,7 +122,7 @@ func convertStmtExecInfoToSummary(sei *StmtExecInfo) *stmtSummary { // Add a new StmtExecInfo to stmtSummary func addStmtExecInfoToSummary(sei *StmtExecInfo, ss *stmtSummary) { - ss.sumTotalLatency += sei.TotalLatency + ss.sumLatency += sei.TotalLatency ss.execCount++ if sei.TotalLatency > ss.maxLatency { ss.maxLatency = sei.TotalLatency @@ -156,10 +169,10 @@ func (ss *stmtSummaryByDigest) ToDatum() [][]types.Datum { summary.digest, summary.normalizedSQL, summary.execCount, - summary.sumTotalLatency, + summary.sumLatency, summary.maxLatency, summary.minLatency, - summary.sumTotalLatency/summary.execCount, // AVG_LATENCY + summary.sumLatency/summary.execCount, // AVG_LATENCY summary.sumAffectedRows, types.Time{Time: types.FromGoTime(summary.firstSeen), Type: mysql.TypeTimestamp}, types.Time{Time: types.FromGoTime(summary.lastSeen), Type: mysql.TypeTimestamp}, From 31d4612ca414fd5ff8c80264d3239b3697b51530 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Tue, 3 Sep 2019 16:11:05 +0800 Subject: [PATCH 04/26] Add test cases for stmtSummary. --- executor/adapter.go | 5 +- infoschema/perfschema/const.go | 4 +- infoschema/perfschema/tables_test.go | 57 +++- session/tidb.go | 1 + util/stmtsummary/statement_summary.go | 10 + util/stmtsummary/statement_summary_test.go | 303 +++++++++++++++++++++ 6 files changed, 367 insertions(+), 13 deletions(-) create mode 100644 util/stmtsummary/statement_summary_test.go diff --git a/executor/adapter.go b/executor/adapter.go index 71825c24af0f0..514d2399e78c2 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -181,7 +181,7 @@ func (a *recordSet) Close() error { a.stmt.LogSlowQuery(a.txnStartTS, a.lastErr == nil) a.stmt.Ctx.GetSessionVars().PrevStmt = a.stmt.OriginText() a.stmt.logAudit() - a.stmt.summaryStmt() + a.stmt.SummaryStmt() return err } @@ -781,7 +781,8 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool) { } } -func (a *ExecStmt) summaryStmt() { +// Collect statements for performance_schema.events_statements_summary_by_digest +func (a *ExecStmt) SummaryStmt() { sessVars := a.Ctx.GetSessionVars() if !sessVars.EnableStmtSummary || sessVars.InRestrictedSQL { return diff --git a/infoschema/perfschema/const.go b/infoschema/perfschema/const.go index 2df0496b494e6..2bb4579f08890 100644 --- a/infoschema/perfschema/const.go +++ b/infoschema/perfschema/const.go @@ -381,7 +381,7 @@ const tableStagesHistoryLong = "CREATE TABLE if not exists performance_schema.ev const tableEventsStatementsSummaryByDigest = "CREATE TABLE if not exists events_statements_summary_by_digest (" + "SCHEMA_NAME VARCHAR(64) DEFAULT NULL," + "DIGEST VARCHAR(64) DEFAULT NULL," + - "DIGEST_TEXT VARCHAR(4096) DEFAULT NULL," + + "DIGEST_TEXT LONGTEXT DEFAULT NULL," + "EXEC_COUNT BIGINT(20) UNSIGNED NOT NULL," + "SUM_LATENCY BIGINT(20) UNSIGNED NOT NULL," + "MAX_LATENCY BIGINT(20) UNSIGNED NOT NULL," + @@ -390,4 +390,4 @@ const tableEventsStatementsSummaryByDigest = "CREATE TABLE if not exists events_ "SUM_ROWS_AFFECTED BIGINT(20) UNSIGNED NOT NULL," + "FIRST_SEEN TIMESTAMP(6) NOT NULL," + "LAST_SEEN TIMESTAMP(6) NOT NULL," + - "QUERY_SAMPLE_TEXT VARCHAR(4096) DEFAULT NULL);" + "QUERY_SAMPLE_TEXT LONGTEXT DEFAULT NULL);" diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index 4349e4e77ca2c..706d9eb8b0908 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -17,6 +17,8 @@ import ( "testing" . "github.com/pingcap/check" + "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/testkit" @@ -28,22 +30,32 @@ func TestT(t *testing.T) { TestingT(t) } -var _ = Suite(&testSuite{}) +var _ = Suite(&testTableSuite{}) -type testSuite struct { +type testTableSuite struct { + store kv.Storage + dom *domain.Domain } -func (s *testSuite) TestPerfSchemaTables(c *C) { +func (s *testTableSuite) SetUpSuite(c *C) { testleak.BeforeTest() - defer testleak.AfterTest(c)() - store, err := mockstore.NewMockTikvStore() + + var err error + s.store, err = mockstore.NewMockTikvStore() c.Assert(err, IsNil) - defer store.Close() - do, err := session.BootstrapSession(store) + session.DisableStats4Test() + s.dom, err = session.BootstrapSession(s.store) c.Assert(err, IsNil) - defer do.Close() +} + +func (s *testTableSuite) TearDownSuite(c *C) { + defer testleak.AfterTest(c)() + s.dom.Close() + s.store.Close() +} - tk := testkit.NewTestKit(c, store) +func (s *testTableSuite) TestPerfSchemaTables(c *C) { + tk := testkit.NewTestKit(c, s.store) tk.MustExec("use performance_schema") tk.MustQuery("select * from global_status where variable_name = 'Ssl_verify_mode'").Check(testkit.Rows()) @@ -51,3 +63,30 @@ func (s *testSuite) TestPerfSchemaTables(c *C) { tk.MustQuery("select * from setup_actors").Check(testkit.Rows()) tk.MustQuery("select * from events_stages_history_long").Check(testkit.Rows()) } + +func (s *testTableSuite) TestStmtSummaryTable(c *C) { + + tk1 := testkit.NewTestKit(c, s.store) + tk1.MustExec("set global tidb_enable_stmt_summary = 1") + defer func() { + tk1.MustExec("set global tidb_enable_stmt_summary = 0") + }() + tk1.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) + + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b varchar(10))") + + tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("0")) + tk.MustExec("insert into t values(1, 'a')") + tk.MustExec("insert into t values(2, 'b')") + tk.MustExec("insert into t VALUES(3, 'c')") + tk.MustExec("/**/insert into t values(4, 'd')") + tk.MustQuery(`select schema_name, exec_count, sum_rows_affected, query_sample_text + from performance_schema.events_statements_summary_by_digest + where digest_text like 'insert into t%'`, + ).Check(testkit.Rows("test 4 4 insert into t values(1, 'a')")) + + tk.MustQuery("select * from t where a=1") +} diff --git a/session/tidb.go b/session/tidb.go index 054fd5c302ab4..5cafad530d09a 100644 --- a/session/tidb.go +++ b/session/tidb.go @@ -218,6 +218,7 @@ func runStmt(ctx context.Context, sctx sessionctx.Context, s sqlexec.Statement) // then it could include the transaction commit time. if rs == nil { s.(*executor.ExecStmt).LogSlowQuery(origTxnCtx.StartTS, err == nil) + s.(*executor.ExecStmt).SummaryStmt() sessVars.PrevStmt = s.OriginText() } }() diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index 005b6b05c4604..1d439e2b6dfb9 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -133,6 +133,9 @@ func addStmtExecInfoToSummary(sei *StmtExecInfo, ss *stmtSummary) { // TODO: uint64 overflow ss.sumAffectedRows += sei.AffectedRows ss.sumSentRows += sei.SentRows + if sei.StartTime.Before(ss.firstSeen) { + ss.firstSeen = sei.StartTime + } if ss.lastSeen.Before(sei.StartTime) { ss.lastSeen = sei.StartTime } @@ -156,6 +159,13 @@ func (ss *stmtSummaryByDigest) AddStatement(sei *StmtExecInfo) { ss.Unlock() } +// Remove all statement summaries. +func (ss *stmtSummaryByDigest) Clear() { + ss.Lock() + ss.summaryMap.DeleteAll() + ss.Unlock() +} + // Convert statement summary to Datum func (ss *stmtSummaryByDigest) ToDatum() [][]types.Datum { ss.RLock() diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go new file mode 100644 index 0000000000000..8a8c63bca82ec --- /dev/null +++ b/util/stmtsummary/statement_summary_test.go @@ -0,0 +1,303 @@ +package stmtsummary + +import ( + "fmt" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/config" + "strings" + "sync" + "testing" + "time" + + . "github.com/pingcap/check" + "github.com/pingcap/tidb/types" +) + +var _ = Suite(&testStmtSummarySuite{}) + +type testStmtSummarySuite struct { + stmtSummaryByDigest *stmtSummaryByDigest +} + +func (s *testStmtSummarySuite) SetUpSuite(c *C) { + s.stmtSummaryByDigest = NewStmtSummaryByDigest() +} + +func TestT(t *testing.T) { + CustomVerboseFlag = true + TestingT(t) +} + +// Test stmtSummaryByDigest.AddStatement +func (s *testStmtSummarySuite) TestAddStatement(c *C) { + s.stmtSummaryByDigest.Clear() + + // First statement + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + key := &stmtSummaryCacheKey{ + schemaName: stmtExecInfo1.SchemaName, + digest: stmtExecInfo1.Digest, + } + expectedSummary := stmtSummary{ + schemaName: stmtExecInfo1.SchemaName, + digest: stmtExecInfo1.Digest, + normalizedSQL: stmtExecInfo1.NormalizedSQL, + sampleSQL: stmtExecInfo1.OriginalSQL, + execCount: 1, + sumLatency: stmtExecInfo1.TotalLatency, + maxLatency: stmtExecInfo1.TotalLatency, + minLatency: stmtExecInfo1.TotalLatency, + sumAffectedRows: stmtExecInfo1.AffectedRows, + sumSentRows: stmtExecInfo1.SentRows, + firstSeen: stmtExecInfo1.StartTime, + lastSeen: stmtExecInfo1.StartTime, + } + + s.stmtSummaryByDigest.AddStatement(stmtExecInfo1) + summary, ok := s.stmtSummaryByDigest.summaryMap.Get(key) + c.Assert(ok, IsTrue) + c.Assert(*summary.(*stmtSummary), Equals, expectedSummary) + + // Second statement + stmtExecInfo2 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql2", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 50000, + AffectedRows: 500, + SentRows: 500, + StartTime: time.Date(2019, 1, 1, 10, 10, 20, 10, time.UTC), + } + expectedSummary.execCount++ + expectedSummary.sumLatency += stmtExecInfo2.TotalLatency + expectedSummary.maxLatency = stmtExecInfo2.TotalLatency + expectedSummary.sumAffectedRows += stmtExecInfo2.AffectedRows + expectedSummary.sumSentRows += stmtExecInfo2.SentRows + expectedSummary.lastSeen = stmtExecInfo2.StartTime + + s.stmtSummaryByDigest.AddStatement(stmtExecInfo2) + summary, ok = s.stmtSummaryByDigest.summaryMap.Get(key) + c.Assert(ok, IsTrue) + c.Assert(*summary.(*stmtSummary), Equals, expectedSummary) + + // Third statement + stmtExecInfo3 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql3", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 1000, + AffectedRows: 10, + SentRows: 10, + StartTime: time.Date(2019, 1, 1, 10, 10, 0, 10, time.UTC), + } + expectedSummary.execCount++ + expectedSummary.sumLatency += stmtExecInfo3.TotalLatency + expectedSummary.minLatency = stmtExecInfo3.TotalLatency + expectedSummary.sumAffectedRows += stmtExecInfo3.AffectedRows + expectedSummary.sumSentRows += stmtExecInfo3.SentRows + expectedSummary.firstSeen = stmtExecInfo3.StartTime + + s.stmtSummaryByDigest.AddStatement(stmtExecInfo3) + summary, ok = s.stmtSummaryByDigest.summaryMap.Get(key) + c.Assert(ok, IsTrue) + c.Assert(*summary.(*stmtSummary), Equals, expectedSummary) + + // Fourth statement that in a different schema + stmtExecInfo4 := &StmtExecInfo{ + SchemaName: "schema_name2", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 1000, + AffectedRows: 10, + SentRows: 10, + StartTime: time.Date(2019, 1, 1, 10, 10, 0, 10, time.UTC), + } + key = &stmtSummaryCacheKey{ + schemaName: stmtExecInfo4.SchemaName, + digest: stmtExecInfo4.Digest, + } + + s.stmtSummaryByDigest.AddStatement(stmtExecInfo4) + c.Assert(s.stmtSummaryByDigest.summaryMap.Size(), Equals, 2) + summary, ok = s.stmtSummaryByDigest.summaryMap.Get(key) + c.Assert(ok, IsTrue) + + // Fifth statement that has a different digest + stmtExecInfo5 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql2", + Digest: "digest2", + TotalLatency: 1000, + AffectedRows: 10, + SentRows: 10, + StartTime: time.Date(2019, 1, 1, 10, 10, 0, 10, time.UTC), + } + key = &stmtSummaryCacheKey{ + schemaName: stmtExecInfo5.SchemaName, + digest: stmtExecInfo5.Digest, + } + + s.stmtSummaryByDigest.AddStatement(stmtExecInfo5) + c.Assert(s.stmtSummaryByDigest.summaryMap.Size(), Equals, 3) + summary, ok = s.stmtSummaryByDigest.summaryMap.Get(key) + c.Assert(ok, IsTrue) +} + +func match(c *C, row []types.Datum, expected ...interface{}) { + c.Assert(len(row), Equals, len(expected)) + for i := range row { + got := fmt.Sprintf("%v", row[i].GetValue()) + need := fmt.Sprintf("%v", expected[i]) + c.Assert(got, Equals, need) + } +} + +// Test stmtSummaryByDigest.ToDatum +func (s *testStmtSummarySuite) TestToDatum(c *C) { + s.stmtSummaryByDigest.Clear() + + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + s.stmtSummaryByDigest.AddStatement(stmtExecInfo1) + datums := s.stmtSummaryByDigest.ToDatum() + c.Assert(len(datums), Equals, 1) + t := types.Time{Time: types.FromGoTime(stmtExecInfo1.StartTime), Type: mysql.TypeTimestamp} + match(c, datums[0], stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, stmtExecInfo1.NormalizedSQL, + 1, stmtExecInfo1.TotalLatency, stmtExecInfo1.TotalLatency, stmtExecInfo1.TotalLatency, stmtExecInfo1.TotalLatency, + stmtExecInfo1.AffectedRows, t, t, stmtExecInfo1.OriginalSQL) +} + +// Test AddStatement and ToDatum parallel +func (s *testStmtSummarySuite) TestAddStatementParallel(c *C) { + s.stmtSummaryByDigest.Clear() + + threads := 8 + loops := 32 + wg := sync.WaitGroup{} + wg.Add(threads) + + addStmtFunc := func() { + defer wg.Done() + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + + // Add 32 times with different digest + for i := 0; i < loops; i++ { + stmtExecInfo1.Digest = fmt.Sprintf("digest%d", i) + s.stmtSummaryByDigest.AddStatement(stmtExecInfo1) + } + + // There would be 32 summaries + datums := s.stmtSummaryByDigest.ToDatum() + c.Assert(len(datums), Equals, loops) + } + + for i := 0; i < threads; i++ { + go addStmtFunc() + } + wg.Wait() + + datums := s.stmtSummaryByDigest.ToDatum() + c.Assert(len(datums), Equals, loops) +} + +// Test max number of statement count. +func (s *testStmtSummarySuite) TestMaxStmtCount(c *C) { + s.stmtSummaryByDigest.Clear() + + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + + maxStmtCount := config.GetGlobalConfig().StmtSummary.MaxStmtCount + + // 1000 digests + loops := int(maxStmtCount) * 10 + for i := 0; i < loops; i++ { + stmtExecInfo1.Digest = fmt.Sprintf("digest%d", i) + s.stmtSummaryByDigest.AddStatement(stmtExecInfo1) + } + + // Summary count should be MaxStmtCount + sm := s.stmtSummaryByDigest.summaryMap + c.Assert(sm.Size(), Equals, int(maxStmtCount)) + + // LRU cache should work + for i := loops - int(maxStmtCount); i < loops; i++ { + key := &stmtSummaryCacheKey{ + schemaName: stmtExecInfo1.SchemaName, + digest: fmt.Sprintf("digest%d", i), + } + _, ok := sm.Get(key) + c.Assert(ok, IsTrue) + } +} + +// Test max length of normalized and sample SQL. +func (s *testStmtSummarySuite) TestMaxSqlLength(c *C) { + s.stmtSummaryByDigest.Clear() + + // Create a long SQL + maxSqlLength := config.GetGlobalConfig().StmtSummary.MaxSqlLength + length := int(maxSqlLength) * 10 + str := strings.Repeat("a", length) + + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: str, + NormalizedSQL: str, + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + + s.stmtSummaryByDigest.AddStatement(stmtExecInfo1) + key := &stmtSummaryCacheKey{ + schemaName: stmtExecInfo1.SchemaName, + digest: stmtExecInfo1.Digest, + } + value, ok := s.stmtSummaryByDigest.summaryMap.Get(key) + c.Assert(ok, IsTrue) + // Length of normalizedSQL and sampleSQL should be maxSqlLength + summary := value.(*stmtSummary) + c.Assert(len(summary.normalizedSQL), Equals, int(maxSqlLength)) + c.Assert(len(summary.sampleSQL), Equals, int(maxSqlLength)) +} From 60c3c76efc6c033286c3196f2495ecb89ce6d987 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Tue, 3 Sep 2019 16:19:40 +0800 Subject: [PATCH 05/26] remove duplicate code for table_test.go --- infoschema/perfschema/tables_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index 706d9eb8b0908..67b75cac94ab8 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -65,7 +65,6 @@ func (s *testTableSuite) TestPerfSchemaTables(c *C) { } func (s *testTableSuite) TestStmtSummaryTable(c *C) { - tk1 := testkit.NewTestKit(c, s.store) tk1.MustExec("set global tidb_enable_stmt_summary = 1") defer func() { @@ -78,7 +77,6 @@ func (s *testTableSuite) TestStmtSummaryTable(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b varchar(10))") - tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("0")) tk.MustExec("insert into t values(1, 'a')") tk.MustExec("insert into t values(2, 'b')") tk.MustExec("insert into t VALUES(3, 'c')") From 250e762ea331eba0344969e62ddcbbc01665ec43 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Tue, 3 Sep 2019 16:26:39 +0800 Subject: [PATCH 06/26] Revert "remove duplicate code for table_test.go" This reverts commit 16f76cb2356c48384e9bc87481c6e9c808bc1bc4. --- infoschema/perfschema/tables_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index 67b75cac94ab8..706d9eb8b0908 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -65,6 +65,7 @@ func (s *testTableSuite) TestPerfSchemaTables(c *C) { } func (s *testTableSuite) TestStmtSummaryTable(c *C) { + tk1 := testkit.NewTestKit(c, s.store) tk1.MustExec("set global tidb_enable_stmt_summary = 1") defer func() { @@ -77,6 +78,7 @@ func (s *testTableSuite) TestStmtSummaryTable(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b varchar(10))") + tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("0")) tk.MustExec("insert into t values(1, 'a')") tk.MustExec("insert into t values(2, 'b')") tk.MustExec("insert into t VALUES(3, 'c')") From a144b7bf5dd03c9d185e7b6cd9c3f04121fe2a54 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 4 Sep 2019 11:12:54 +0800 Subject: [PATCH 07/26] Add a test case for events_statements_summary_by_digest --- infoschema/perfschema/tables_test.go | 37 +++++++++++++++++++--------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index 706d9eb8b0908..bd432e26da0ae 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -14,8 +14,6 @@ package perfschema_test import ( - "testing" - . "github.com/pingcap/check" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" @@ -23,6 +21,7 @@ import ( "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" + "testing" ) func TestT(t *testing.T) { @@ -65,20 +64,24 @@ func (s *testTableSuite) TestPerfSchemaTables(c *C) { } func (s *testTableSuite) TestStmtSummaryTable(c *C) { - - tk1 := testkit.NewTestKit(c, s.store) - tk1.MustExec("set global tidb_enable_stmt_summary = 1") - defer func() { - tk1.MustExec("set global tidb_enable_stmt_summary = 0") - }() - tk1.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) - tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b varchar(10))") - tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("0")) + // statement summary is disabled in default + tk.MustQuery("select @@session.tidb_enable_stmt_summary").Check(testkit.Rows("0")) + tk.MustExec("insert into t values(1, 'a')") + tk.MustQuery("select * from performance_schema.events_statements_summary_by_digest").Check(testkit.Rows()) + + tk.MustExec("set session tidb_enable_stmt_summary = 1") + defer func() { + tk.MustExec("set session tidb_enable_stmt_summary = 0") + }() + tk.MustQuery("select @@session.tidb_enable_stmt_summary").Check(testkit.Rows("1")) + + tk.MustExec("use test") tk.MustExec("insert into t values(1, 'a')") tk.MustExec("insert into t values(2, 'b')") tk.MustExec("insert into t VALUES(3, 'c')") @@ -88,5 +91,15 @@ func (s *testTableSuite) TestStmtSummaryTable(c *C) { where digest_text like 'insert into t%'`, ).Check(testkit.Rows("test 4 4 insert into t values(1, 'a')")) - tk.MustQuery("select * from t where a=1") + tk.MustQuery("select * from t where a=2") + tk.MustQuery(`select schema_name, exec_count, sum_rows_affected, query_sample_text + from performance_schema.events_statements_summary_by_digest + where digest_text like 'select * from t%'`, + ).Check(testkit.Rows("test 1 0 select * from t where a=2")) + + // select ... order by + tk.MustQuery(`select schema_name, exec_count, sum_rows_affected, query_sample_text + from performance_schema.events_statements_summary_by_digest + order by exec_count desc limit 1`, + ).Check(testkit.Rows("test 4 4 insert into t values(1, 'a')")) } From c5d21789ffac9dc8d2df4eb7a3521a510334f90f Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 4 Sep 2019 11:40:13 +0800 Subject: [PATCH 08/26] format the code in tables_test.go --- infoschema/perfschema/tables_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index bd432e26da0ae..b482452d6d546 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -14,6 +14,8 @@ package perfschema_test import ( + "testing" + . "github.com/pingcap/check" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" @@ -21,7 +23,6 @@ import ( "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" - "testing" ) func TestT(t *testing.T) { @@ -63,6 +64,7 @@ func (s *testTableSuite) TestPerfSchemaTables(c *C) { tk.MustQuery("select * from events_stages_history_long").Check(testkit.Rows()) } +// Test events_statements_summary_by_digest func (s *testTableSuite) TestStmtSummaryTable(c *C) { tk := testkit.NewTestKit(c, s.store) From e6d1865dafbd161d23146517da3b511ae7b05e94 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 4 Sep 2019 12:15:24 +0800 Subject: [PATCH 09/26] rename variable names for StmtSummary. --- config/config.go | 5 +++-- executor/adapter.go | 2 +- tidb-server/main.go | 2 +- util/stmtsummary/statement_summary.go | 16 +++++++++------- util/stmtsummary/statement_summary_test.go | 12 ++++++------ 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/config/config.go b/config/config.go index 657bf31cefe84..c37253cd1d8ee 100644 --- a/config/config.go +++ b/config/config.go @@ -317,11 +317,12 @@ type PessimisticTxn struct { TTL string `toml:"ttl" json:"ttl"` } +// StmtSummary is the config for statement summary. type StmtSummary struct { // How many statements can be kept in the table MaxStmtCount uint `toml:"max-stmt-count" json:"max-stmt-count"` // How long can a normalizedSQL / sampleSQL be - MaxSqlLength uint `toml:"max-sql-length" json:"max-sql-length"` + MaxSQLLength uint `toml:"max-sql-length" json:"max-sql-length"` } var defaultConf = Config{ @@ -420,7 +421,7 @@ var defaultConf = Config{ }, StmtSummary: StmtSummary{ MaxStmtCount: 100, - MaxSqlLength: 4096, + MaxSQLLength: 4096, }, } diff --git a/executor/adapter.go b/executor/adapter.go index 514d2399e78c2..e39b378cbf10e 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -781,7 +781,7 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool) { } } -// Collect statements for performance_schema.events_statements_summary_by_digest +// SummaryStmt Collects statements for performance_schema.events_statements_summary_by_digest func (a *ExecStmt) SummaryStmt() { sessVars := a.Ctx.GetSessionVars() if !sessVars.EnableStmtSummary || sessVars.InRestrictedSQL { diff --git a/tidb-server/main.go b/tidb-server/main.go index 8530bff678996..c2e2d80fbe531 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -362,7 +362,7 @@ func loadConfig() string { // hotReloadConfigItems lists all config items which support hot-reload. var hotReloadConfigItems = []string{"Performance.MaxProcs", "Performance.MaxMemory", "Performance.CrossJoin", "Performance.FeedbackProbability", "Performance.QueryFeedbackLimit", "Performance.PseudoEstimateRatio", - "OOMAction", "MemQuotaQuery", "StmtSummary.MaxStmtCount", "StmtSummary.MaxSqlLength"} + "OOMAction", "MemQuotaQuery", "StmtSummary.MaxStmtCount", "StmtSummary.MaxSQLLength"} func reloadConfig(nc, c *config.Config) { // Just a part of config items need to be reload explicitly. diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index 1d439e2b6dfb9..9c26801c15f74 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -50,6 +50,7 @@ type stmtSummaryByDigest struct { summaryMap *kvcache.SimpleLRUCache } +// StmtSummary is the global object for statement summary. var StmtSummary = NewStmtSummaryByDigest() // Summary of each type of statements. @@ -71,7 +72,7 @@ type stmtSummary struct { lastSeen time.Time } -// Used for recording execution status of each statement. +// StmtExecInfo records execution status of each statement. type StmtExecInfo struct { SchemaName string OriginalSQL string @@ -84,6 +85,7 @@ type StmtExecInfo struct { StartTime time.Time } +// NewStmtSummaryByDigest creates a stmtSummaryByDigest. func NewStmtSummaryByDigest() *stmtSummaryByDigest { maxStmtCount := config.GetGlobalConfig().StmtSummary.MaxStmtCount return &stmtSummaryByDigest{ @@ -93,15 +95,15 @@ func NewStmtSummaryByDigest() *stmtSummaryByDigest { // Convert StmtExecInfo to stmtSummary func convertStmtExecInfoToSummary(sei *StmtExecInfo) *stmtSummary { - // Trim SQL to size MaxSqlLength - maxSqlLength := config.GetGlobalConfig().StmtSummary.MaxSqlLength + // Trim SQL to size MaxSQLLength + maxSQLLength := config.GetGlobalConfig().StmtSummary.MaxSQLLength normalizedSQL := sei.NormalizedSQL - if len(normalizedSQL) > int(maxSqlLength) { - normalizedSQL = normalizedSQL[:maxSqlLength] + if len(normalizedSQL) > int(maxSQLLength) { + normalizedSQL = normalizedSQL[:maxSQLLength] } sampleSQL := sei.OriginalSQL - if len(sampleSQL) > int(maxSqlLength) { - sampleSQL = sampleSQL[:maxSqlLength] + if len(sampleSQL) > int(maxSQLLength) { + sampleSQL = sampleSQL[:maxSQLLength] } return &stmtSummary{ diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index 8a8c63bca82ec..364d99c8caf62 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -270,12 +270,12 @@ func (s *testStmtSummarySuite) TestMaxStmtCount(c *C) { } // Test max length of normalized and sample SQL. -func (s *testStmtSummarySuite) TestMaxSqlLength(c *C) { +func (s *testStmtSummarySuite) TestMaxSQLLength(c *C) { s.stmtSummaryByDigest.Clear() // Create a long SQL - maxSqlLength := config.GetGlobalConfig().StmtSummary.MaxSqlLength - length := int(maxSqlLength) * 10 + maxSQLLength := config.GetGlobalConfig().StmtSummary.MaxSQLLength + length := int(maxSQLLength) * 10 str := strings.Repeat("a", length) stmtExecInfo1 := &StmtExecInfo{ @@ -296,8 +296,8 @@ func (s *testStmtSummarySuite) TestMaxSqlLength(c *C) { } value, ok := s.stmtSummaryByDigest.summaryMap.Get(key) c.Assert(ok, IsTrue) - // Length of normalizedSQL and sampleSQL should be maxSqlLength + // Length of normalizedSQL and sampleSQL should be maxSQLLength summary := value.(*stmtSummary) - c.Assert(len(summary.normalizedSQL), Equals, int(maxSqlLength)) - c.Assert(len(summary.sampleSQL), Equals, int(maxSqlLength)) + c.Assert(len(summary.normalizedSQL), Equals, int(maxSQLLength)) + c.Assert(len(summary.sampleSQL), Equals, int(maxSQLLength)) } From 6c8b6e6207cfd95faf0766fea79df392fe0a75bb Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 4 Sep 2019 12:45:55 +0800 Subject: [PATCH 10/26] remove ineffectual assignment in statement_summary_test --- util/stmtsummary/statement_summary_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index 364d99c8caf62..472761c9c6a32 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -131,7 +131,7 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { s.stmtSummaryByDigest.AddStatement(stmtExecInfo4) c.Assert(s.stmtSummaryByDigest.summaryMap.Size(), Equals, 2) - summary, ok = s.stmtSummaryByDigest.summaryMap.Get(key) + _, ok = s.stmtSummaryByDigest.summaryMap.Get(key) c.Assert(ok, IsTrue) // Fifth statement that has a different digest @@ -152,7 +152,7 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { s.stmtSummaryByDigest.AddStatement(stmtExecInfo5) c.Assert(s.stmtSummaryByDigest.summaryMap.Size(), Equals, 3) - summary, ok = s.stmtSummaryByDigest.summaryMap.Get(key) + _, ok = s.stmtSummaryByDigest.summaryMap.Get(key) c.Assert(ok, IsTrue) } From 122af5771dc137b5b2de70a57b40ba2f9848dda0 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 4 Sep 2019 13:37:59 +0800 Subject: [PATCH 11/26] tidb_enable_stmt_summary can only be a global variable --- infoschema/perfschema/tables_test.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index b482452d6d546..41c225fb66db7 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -14,8 +14,6 @@ package perfschema_test import ( - "testing" - . "github.com/pingcap/check" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" @@ -23,6 +21,7 @@ import ( "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" + "testing" ) func TestT(t *testing.T) { @@ -66,24 +65,29 @@ func (s *testTableSuite) TestPerfSchemaTables(c *C) { // Test events_statements_summary_by_digest func (s *testTableSuite) TestStmtSummaryTable(c *C) { - tk := testkit.NewTestKit(c, s.store) + tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b varchar(10))") - // statement summary is disabled in default - tk.MustQuery("select @@session.tidb_enable_stmt_summary").Check(testkit.Rows("0")) + // Statement summary is disabled in default + tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("0")) tk.MustExec("insert into t values(1, 'a')") tk.MustQuery("select * from performance_schema.events_statements_summary_by_digest").Check(testkit.Rows()) - tk.MustExec("set session tidb_enable_stmt_summary = 1") + tk.MustExec("set global tidb_enable_stmt_summary = 1") defer func() { - tk.MustExec("set session tidb_enable_stmt_summary = 0") + tk.MustExec("set global tidb_enable_stmt_summary = 0") }() - tk.MustQuery("select @@session.tidb_enable_stmt_summary").Check(testkit.Rows("1")) + tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) + + // Invalidate the cache manually so that tidb_enable_stmt_summary works immediately. + s.dom.GetGlobalVarsCache().Disable() + + // Create a new session to test + tk = testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("use test") + // Test INSERT tk.MustExec("insert into t values(1, 'a')") tk.MustExec("insert into t values(2, 'b')") tk.MustExec("insert into t VALUES(3, 'c')") @@ -93,6 +97,7 @@ func (s *testTableSuite) TestStmtSummaryTable(c *C) { where digest_text like 'insert into t%'`, ).Check(testkit.Rows("test 4 4 insert into t values(1, 'a')")) + // Test SELECT tk.MustQuery("select * from t where a=2") tk.MustQuery(`select schema_name, exec_count, sum_rows_affected, query_sample_text from performance_schema.events_statements_summary_by_digest From 37ebb33ee308e23b8bf1fede1bb5b1bcb8da3d80 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Mon, 9 Sep 2019 13:49:52 +0800 Subject: [PATCH 12/26] add test cases for statement summary. --- config/config_test.go | 5 ++++ infoschema/perfschema/tables.go | 3 +++ infoschema/perfschema/tables_test.go | 12 +++++++++ sessionctx/variable/session_test.go | 1 + util/kvcache/simple_lru.go | 1 + util/kvcache/simple_lru_test.go | 38 ++++++++++++++++++++++++++++ 6 files changed, 60 insertions(+) diff --git a/config/config_test.go b/config/config_test.go index 0d4f4cdc4ed0e..a5e0c6e79e04f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -70,6 +70,9 @@ txn-total-size-limit=2000 [tikv-client] commit-timeout="41s" max-batch-size=128 +[stmt-summary] +max-stmt-count=1000 +max-sql-length=1024 `) c.Assert(err, IsNil) @@ -90,6 +93,8 @@ max-batch-size=128 c.Assert(conf.EnableTableLock, IsTrue) c.Assert(conf.DelayCleanTableLock, Equals, uint64(5)) c.Assert(conf.SplitRegionMaxNum, Equals, uint64(10000)) + c.Assert(conf.StmtSummary.MaxStmtCount, Equals, uint(1000)) + c.Assert(conf.StmtSummary.MaxSQLLength, Equals, uint(1024)) c.Assert(f.Close(), IsNil) c.Assert(os.Remove(configFile), IsNil) diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go index ffc1e1d8d4b72..a115fd60f2795 100644 --- a/infoschema/perfschema/tables.go +++ b/infoschema/perfschema/tables.go @@ -14,6 +14,7 @@ package perfschema import ( + "fmt" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" @@ -174,6 +175,7 @@ func (vt *perfSchemaTable) RecordKey(h int64) kv.Key { // AddRecord implements table.Table AddRecord interface. func (vt *perfSchemaTable) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...table.AddRecordOption) (recordID int64, err error) { + fmt.Println("add record") return 0, table.ErrUnsupportedOp } @@ -184,6 +186,7 @@ func (vt *perfSchemaTable) RemoveRecord(ctx sessionctx.Context, h int64, r []typ // UpdateRecord implements table.Table UpdateRecord interface. func (vt *perfSchemaTable) UpdateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datum, touched []bool) error { + fmt.Println("update record") return table.ErrUnsupportedOp } diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index 41c225fb66db7..9cc4fb48030ce 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" + "strings" "testing" ) @@ -63,6 +64,17 @@ func (s *testTableSuite) TestPerfSchemaTables(c *C) { tk.MustQuery("select * from events_stages_history_long").Check(testkit.Rows()) } +func (s *testTableSuite) TestPerfSchemaTablesError(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("use performance_schema") + _, err := tk.Exec("insert into events_statements_summary_by_digest values('', '', '', 0, 0, 0, 0, 0, 0, now(), now(), '')") + c.Assert(strings.Contains(err.Error(), "operation not supported"), IsTrue) + + _, err = tk.Exec("update events_statements_summary_by_digest set exec_count=exec_count+1") + c.Assert(strings.Contains(err.Error(), "operation not supported"), IsTrue) +} + // Test events_statements_summary_by_digest func (s *testTableSuite) TestStmtSummaryTable(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) diff --git a/sessionctx/variable/session_test.go b/sessionctx/variable/session_test.go index 7e1a5716a20c1..7efb93315df55 100644 --- a/sessionctx/variable/session_test.go +++ b/sessionctx/variable/session_test.go @@ -51,6 +51,7 @@ func (*testSessionSuite) TestSetSystemVariable(c *C) { {variable.TIDBMemQuotaIndexLookupReader, "1024", false}, {variable.TIDBMemQuotaIndexLookupJoin, "1024", false}, {variable.TIDBMemQuotaNestedLoopApply, "1024", false}, + {variable.TiDBEnableStmtSummary, "1", false}, } for _, t := range tests { err := variable.SetSessionSystemVar(v, t.key, types.NewDatum(t.value)) diff --git a/util/kvcache/simple_lru.go b/util/kvcache/simple_lru.go index 9dd87f9eeaf71..b3f18f2871d42 100644 --- a/util/kvcache/simple_lru.go +++ b/util/kvcache/simple_lru.go @@ -89,6 +89,7 @@ func (l *SimpleLRUCache) Put(key Key, value Value) { l.elements[hash] = element l.size++ + // Getting used memory is expensive and can be avoided by setting quota to 0. if l.quota <= 0 { if l.size > l.capacity { lru := l.cache.Back() diff --git a/util/kvcache/simple_lru_test.go b/util/kvcache/simple_lru_test.go index 1aa17a646f8ed..14e3d629bec61 100644 --- a/util/kvcache/simple_lru_test.go +++ b/util/kvcache/simple_lru_test.go @@ -106,6 +106,22 @@ func (s *testLRUCacheSuite) TestPut(c *C) { c.Assert(root, IsNil) } +func (s *testLRUCacheSuite) TestZeroQuota(c *C) { + lru := NewSimpleLRUCache(100, 0, 0) + c.Assert(lru.capacity, Equals, uint(100)) + + keys := make([]*mockCacheKey, 100) + vals := make([]int64, 100) + + for i := 0; i < 100; i++ { + keys[i] = newMockHashKey(int64(i)) + vals[i] = int64(i) + lru.Put(keys[i], vals[i]) + } + c.Assert(lru.size, Equals, lru.capacity) + c.Assert(lru.size, Equals, uint(100)) +} + func (s *testLRUCacheSuite) TestOOMGuard(c *C) { maxMem, err := memory.MemTotal() c.Assert(err, IsNil) @@ -228,3 +244,25 @@ func (s *testLRUCacheSuite) TestDeleteAll(c *C) { c.Assert(int(lru.size), Equals, 0) } } + +func (s *testLRUCacheSuite) TestValues(c *C) { + maxMem, err := memory.MemTotal() + c.Assert(err, IsNil) + + lru := NewSimpleLRUCache(5, 0, maxMem) + + keys := make([]*mockCacheKey, 5) + vals := make([]int64, 5) + + for i := 0; i < 5; i++ { + keys[i] = newMockHashKey(int64(i)) + vals[i] = int64(i) + lru.Put(keys[i], vals[i]) + } + + values := lru.Values() + c.Assert(len(values), Equals, 5) + for i := 0; i < 5; i++ { + c.Assert(values[i], Equals, int64(4-i)) + } +} From 3706cb00469195be29ae44db148fd9a4c8232776 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Mon, 9 Sep 2019 22:06:51 +0800 Subject: [PATCH 13/26] set default of tidb_enable_stmt_summary to True to enable benchmark test. --- infoschema/perfschema/tables_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index 9cc4fb48030ce..41c225fb66db7 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" - "strings" "testing" ) @@ -64,17 +63,6 @@ func (s *testTableSuite) TestPerfSchemaTables(c *C) { tk.MustQuery("select * from events_stages_history_long").Check(testkit.Rows()) } -func (s *testTableSuite) TestPerfSchemaTablesError(c *C) { - tk := testkit.NewTestKit(c, s.store) - - tk.MustExec("use performance_schema") - _, err := tk.Exec("insert into events_statements_summary_by_digest values('', '', '', 0, 0, 0, 0, 0, 0, now(), now(), '')") - c.Assert(strings.Contains(err.Error(), "operation not supported"), IsTrue) - - _, err = tk.Exec("update events_statements_summary_by_digest set exec_count=exec_count+1") - c.Assert(strings.Contains(err.Error(), "operation not supported"), IsTrue) -} - // Test events_statements_summary_by_digest func (s *testTableSuite) TestStmtSummaryTable(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) From 0b9ed2370a44fec16d10a801bab5a211adca485c Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Tue, 10 Sep 2019 11:04:23 +0800 Subject: [PATCH 14/26] format import in tables_test --- infoschema/perfschema/tables_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index 41c225fb66db7..a596d97b03155 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -14,6 +14,8 @@ package perfschema_test import ( + "testing" + . "github.com/pingcap/check" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" @@ -21,7 +23,6 @@ import ( "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" - "testing" ) func TestT(t *testing.T) { From b0e1f0f401f8894509f8dcad31096d20bd8602d9 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Tue, 10 Sep 2019 11:15:35 +0800 Subject: [PATCH 15/26] change the year in copyright to 2019. --- util/stmtsummary/statement_summary.go | 2 +- util/stmtsummary/statement_summary_test.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index 9c26801c15f74..7345097fecf6c 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -1,4 +1,4 @@ -// Copyright 2016 PingCAP, Inc. +// Copyright 2019 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index 472761c9c6a32..6226b6959651a 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -1,3 +1,16 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package stmtsummary import ( From bb82da5b057b3a1922df21ec495682d8a5d9f752 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Tue, 10 Sep 2019 11:24:29 +0800 Subject: [PATCH 16/26] format import in util/stmtsummary/statement_summary_test.go --- util/stmtsummary/statement_summary_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index 6226b6959651a..a6ba6110103a5 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -15,14 +15,14 @@ package stmtsummary import ( "fmt" - "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb/config" "strings" "sync" "testing" "time" . "github.com/pingcap/check" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/types" ) From 24829388f50a64373efee8573892189cbca4965d Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Tue, 10 Sep 2019 11:26:05 +0800 Subject: [PATCH 17/26] format imports in tables.go --- infoschema/perfschema/tables.go | 1 + 1 file changed, 1 insertion(+) diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go index a115fd60f2795..cbfdf4d12ce53 100644 --- a/infoschema/perfschema/tables.go +++ b/infoschema/perfschema/tables.go @@ -15,6 +15,7 @@ package perfschema import ( "fmt" + "github.com/pingcap/parser/model" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" From 6a3d2094bdbfbe88ea690b25a3c100d905019d1d Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Tue, 10 Sep 2019 12:40:48 +0800 Subject: [PATCH 18/26] remove debugging print in tables.go --- infoschema/perfschema/tables.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go index cbfdf4d12ce53..ffc1e1d8d4b72 100644 --- a/infoschema/perfschema/tables.go +++ b/infoschema/perfschema/tables.go @@ -14,8 +14,6 @@ package perfschema import ( - "fmt" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" @@ -176,7 +174,6 @@ func (vt *perfSchemaTable) RecordKey(h int64) kv.Key { // AddRecord implements table.Table AddRecord interface. func (vt *perfSchemaTable) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...table.AddRecordOption) (recordID int64, err error) { - fmt.Println("add record") return 0, table.ErrUnsupportedOp } @@ -187,7 +184,6 @@ func (vt *perfSchemaTable) RemoveRecord(ctx sessionctx.Context, h int64, r []typ // UpdateRecord implements table.Table UpdateRecord interface. func (vt *perfSchemaTable) UpdateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datum, touched []bool) error { - fmt.Println("update record") return table.ErrUnsupportedOp } From 02cedb13431be091e7e56319b3e1303ef3542c6e Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Tue, 10 Sep 2019 18:56:39 +0800 Subject: [PATCH 19/26] add a separate lock for each summary entry --- util/stmtsummary/statement_summary.go | 35 +++++++++++++++++++-------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index 7345097fecf6c..9bad1ce8fefed 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -46,6 +46,7 @@ func (key *stmtSummaryCacheKey) Hash() []byte { // A LRU cache that stores statement summary. type stmtSummaryByDigest struct { + // It's rare to read concurrently, so RWMutex is not needed. sync.RWMutex summaryMap *kvcache.SimpleLRUCache } @@ -55,6 +56,8 @@ var StmtSummary = NewStmtSummaryByDigest() // Summary of each type of statements. type stmtSummary struct { + // It's rare to read concurrently, so RWMutex is not needed. + sync.Mutex schemaName string digest string normalizedSQL string @@ -124,6 +127,8 @@ func convertStmtExecInfoToSummary(sei *StmtExecInfo) *stmtSummary { // Add a new StmtExecInfo to stmtSummary func addStmtExecInfoToSummary(sei *StmtExecInfo, ss *stmtSummary) { + ss.Lock() + ss.sumLatency += sei.TotalLatency ss.execCount++ if sei.TotalLatency > ss.maxLatency { @@ -141,6 +146,8 @@ func addStmtExecInfoToSummary(sei *StmtExecInfo, ss *stmtSummary) { if ss.lastSeen.Before(sei.StartTime) { ss.lastSeen = sei.StartTime } + + ss.Unlock() } // Add a statement to statement summary. @@ -150,15 +157,21 @@ func (ss *stmtSummaryByDigest) AddStatement(sei *StmtExecInfo) { digest: sei.Digest, } + var summary *stmtSummary ss.Lock() - summary, ok := ss.summaryMap.Get(key) + value, ok := ss.summaryMap.Get(key) if ok { - addStmtExecInfoToSummary(sei, summary.(*stmtSummary)) + summary = value.(*stmtSummary) } else { - summary = convertStmtExecInfoToSummary(sei) - ss.summaryMap.Put(key, summary) + newSummary := convertStmtExecInfoToSummary(sei) + ss.summaryMap.Put(key, newSummary) } ss.Unlock() + + // Lock a single entry, not the whole cache. + if summary != nil { + addStmtExecInfoToSummary(sei, summary) + } } // Remove all statement summaries. @@ -170,12 +183,14 @@ func (ss *stmtSummaryByDigest) Clear() { // Convert statement summary to Datum func (ss *stmtSummaryByDigest) ToDatum() [][]types.Datum { - ss.RLock() - rows := make([][]types.Datum, 0, ss.summaryMap.Size()) - // Summary of each statement may be modified at the same time, - // so surround the whole block with read lock. - for _, value := range ss.summaryMap.Values() { + ss.Lock() + values := ss.summaryMap.Values() + ss.Unlock() + + rows := make([][]types.Datum, 0, len(values)) + for _, value := range values { summary := value.(*stmtSummary) + summary.Lock() record := types.MakeDatums( summary.schemaName, summary.digest, @@ -190,9 +205,9 @@ func (ss *stmtSummaryByDigest) ToDatum() [][]types.Datum { types.Time{Time: types.FromGoTime(summary.lastSeen), Type: mysql.TypeTimestamp}, summary.sampleSQL, ) + summary.Unlock() rows = append(rows, record) } - ss.RUnlock() return rows } From 681e099c017f538f4285747ed15b67a8dd346337 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 11 Sep 2019 11:37:30 +0800 Subject: [PATCH 20/26] remove unsupported methods from perfSchemaTable --- infoschema/perfschema/tables.go | 91 +-------------------------------- 1 file changed, 1 insertion(+), 90 deletions(-) diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go index ffc1e1d8d4b72..33b8e5bf3ef0a 100644 --- a/infoschema/perfschema/tables.go +++ b/infoschema/perfschema/tables.go @@ -89,7 +89,7 @@ func (vt *perfSchemaTable) Meta() *model.TableInfo { func (vt *perfSchemaTable) getRows(ctx sessionctx.Context, cols []*table.Column) (fullRows [][]types.Datum, err error) { switch vt.meta.Name.O { case tableNameEventsStatementsSummaryByDigest: - fullRows = dataForEventsStatementsSummaryByDigest(ctx) + fullRows = stmtsummary.StmtSummary.ToDatum() } if len(cols) == len(vt.cols) { return @@ -126,92 +126,3 @@ func (vt *perfSchemaTable) IterRecords(ctx sessionctx.Context, startKey kv.Key, } return nil } - -// RowWithCols implements table.Table RowWithCols interface. -func (vt *perfSchemaTable) RowWithCols(ctx sessionctx.Context, h int64, cols []*table.Column) ([]types.Datum, error) { - return nil, table.ErrUnsupportedOp -} - -// Row implements table.Table Row interface. -func (vt *perfSchemaTable) Row(ctx sessionctx.Context, h int64) ([]types.Datum, error) { - return nil, table.ErrUnsupportedOp -} - -// Indices implements table.Table Indices interface. -func (vt *perfSchemaTable) Indices() []table.Index { - return nil -} - -// WritableIndices implements table.Table WritableIndices interface. -func (vt *perfSchemaTable) WritableIndices() []table.Index { - return nil -} - -// DeletableIndices implements table.Table DeletableIndices interface. -func (vt *perfSchemaTable) DeletableIndices() []table.Index { - return nil -} - -// RecordPrefix implements table.Table RecordPrefix interface. -func (vt *perfSchemaTable) RecordPrefix() kv.Key { - return nil -} - -// IndexPrefix implements table.Table IndexPrefix interface. -func (vt *perfSchemaTable) IndexPrefix() kv.Key { - return nil -} - -// FirstKey implements table.Table FirstKey interface. -func (vt *perfSchemaTable) FirstKey() kv.Key { - return nil -} - -// RecordKey implements table.Table RecordKey interface. -func (vt *perfSchemaTable) RecordKey(h int64) kv.Key { - return nil -} - -// AddRecord implements table.Table AddRecord interface. -func (vt *perfSchemaTable) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...table.AddRecordOption) (recordID int64, err error) { - return 0, table.ErrUnsupportedOp -} - -// RemoveRecord implements table.Table RemoveRecord interface. -func (vt *perfSchemaTable) RemoveRecord(ctx sessionctx.Context, h int64, r []types.Datum) error { - return table.ErrUnsupportedOp -} - -// UpdateRecord implements table.Table UpdateRecord interface. -func (vt *perfSchemaTable) UpdateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datum, touched []bool) error { - return table.ErrUnsupportedOp -} - -// AllocHandle implements table.Table AllocHandle interface. -func (vt *perfSchemaTable) AllocHandle(ctx sessionctx.Context) (int64, error) { - return 0, table.ErrUnsupportedOp -} - -// Allocator implements table.Table Allocator interface. -func (vt *perfSchemaTable) Allocator(ctx sessionctx.Context) autoid.Allocator { - return nil -} - -// RebaseAutoID implements table.Table RebaseAutoID interface. -func (vt *perfSchemaTable) RebaseAutoID(ctx sessionctx.Context, newBase int64, isSetStep bool) error { - return table.ErrUnsupportedOp -} - -// Seek implements table.Table Seek interface. -func (vt *perfSchemaTable) Seek(ctx sessionctx.Context, h int64) (int64, bool, error) { - return 0, false, table.ErrUnsupportedOp -} - -// Type implements table.Table Type interface. -func (vt *perfSchemaTable) Type() table.Type { - return table.VirtualTable -} - -func dataForEventsStatementsSummaryByDigest(ctx sessionctx.Context) [][]types.Datum { - return stmtsummary.StmtSummary.ToDatum() -} From 9dfb9735bff74356f102e451ae44059ed7eaa86a Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 11 Sep 2019 13:53:23 +0800 Subject: [PATCH 21/26] use Mutex in stmtSummaryByDigest instead of RWMutex --- util/stmtsummary/statement_summary.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index 9bad1ce8fefed..edb8fe6a58a1d 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -47,7 +47,7 @@ func (key *stmtSummaryCacheKey) Hash() []byte { // A LRU cache that stores statement summary. type stmtSummaryByDigest struct { // It's rare to read concurrently, so RWMutex is not needed. - sync.RWMutex + sync.Mutex summaryMap *kvcache.SimpleLRUCache } From 8b2b2981c8472861c8ee84ad88cfbc67ee342d9b Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 11 Sep 2019 15:31:33 +0800 Subject: [PATCH 22/26] rename class names related to statement summary --- executor/adapter.go | 2 +- infoschema/perfschema/tables.go | 2 +- util/stmtsummary/statement_summary.go | 112 ++++++++++----------- util/stmtsummary/statement_summary_test.go | 76 +++++++------- 4 files changed, 96 insertions(+), 96 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index e39b378cbf10e..f92a356fe6c6b 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -790,7 +790,7 @@ func (a *ExecStmt) SummaryStmt() { stmtCtx := sessVars.StmtCtx normalizedSQL, digest := stmtCtx.SQLDigest() costTime := time.Since(sessVars.StartTime) - stmtsummary.StmtSummary.AddStatement(&stmtsummary.StmtExecInfo{ + stmtsummary.StmtSummaryByDigestMap.AddStatement(&stmtsummary.StmtExecInfo{ SchemaName: sessVars.CurrentDB, OriginalSQL: a.Text, NormalizedSQL: normalizedSQL, diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go index 33b8e5bf3ef0a..65684416e3401 100644 --- a/infoschema/perfschema/tables.go +++ b/infoschema/perfschema/tables.go @@ -89,7 +89,7 @@ func (vt *perfSchemaTable) Meta() *model.TableInfo { func (vt *perfSchemaTable) getRows(ctx sessionctx.Context, cols []*table.Column) (fullRows [][]types.Datum, err error) { switch vt.meta.Name.O { case tableNameEventsStatementsSummaryByDigest: - fullRows = stmtsummary.StmtSummary.ToDatum() + fullRows = stmtsummary.StmtSummaryByDigestMap.ToDatum() } if len(cols) == len(vt.cols) { return diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index edb8fe6a58a1d..dac79f862e530 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -25,7 +25,11 @@ import ( "github.com/pingcap/tidb/util/kvcache" ) -type stmtSummaryCacheKey struct { +// There're many types of statement summary tables in MySQL, but we have +// only implemented events_statement_summary_by_digest for now. + +// stmtSummaryByDigestKey defines key for stmtSummaryByDigestMap.summaryMap +type stmtSummaryByDigestKey struct { // Same statements may appear in different schema, but they refer to different tables. schemaName string digest string @@ -35,7 +39,7 @@ type stmtSummaryCacheKey struct { } // Hash implements SimpleLRUCache.Key -func (key *stmtSummaryCacheKey) Hash() []byte { +func (key *stmtSummaryByDigestKey) Hash() []byte { if len(key.hash) == 0 { key.hash = make([]byte, 0, len(key.schemaName)+len(key.digest)) key.hash = append(key.hash, hack.Slice(key.digest)...) @@ -44,18 +48,18 @@ func (key *stmtSummaryCacheKey) Hash() []byte { return key.hash } -// A LRU cache that stores statement summary. -type stmtSummaryByDigest struct { +// stmtSummaryByDigestMap is a LRU cache that stores statement summaries. +type stmtSummaryByDigestMap struct { // It's rare to read concurrently, so RWMutex is not needed. sync.Mutex summaryMap *kvcache.SimpleLRUCache } -// StmtSummary is the global object for statement summary. -var StmtSummary = NewStmtSummaryByDigest() +// StmtSummaryByDigestMap is a global map containing all statement summaries. +var StmtSummaryByDigestMap = newStmtSummaryByDigestMap() -// Summary of each type of statements. -type stmtSummary struct { +// stmtSummaryByDigest is the summary for each type of statements. +type stmtSummaryByDigest struct { // It's rare to read concurrently, so RWMutex is not needed. sync.Mutex schemaName string @@ -75,7 +79,7 @@ type stmtSummary struct { lastSeen time.Time } -// StmtExecInfo records execution status of each statement. +// StmtExecInfo records execution information of each statement. type StmtExecInfo struct { SchemaName string OriginalSQL string @@ -88,16 +92,16 @@ type StmtExecInfo struct { StartTime time.Time } -// NewStmtSummaryByDigest creates a stmtSummaryByDigest. -func NewStmtSummaryByDigest() *stmtSummaryByDigest { +// newStmtSummaryByDigestMap creates an empty stmtSummaryByDigestMap. +func newStmtSummaryByDigestMap() *stmtSummaryByDigestMap { maxStmtCount := config.GetGlobalConfig().StmtSummary.MaxStmtCount - return &stmtSummaryByDigest{ + return &stmtSummaryByDigestMap{ summaryMap: kvcache.NewSimpleLRUCache(maxStmtCount, 0, 0), } } -// Convert StmtExecInfo to stmtSummary -func convertStmtExecInfoToSummary(sei *StmtExecInfo) *stmtSummary { +// newStmtSummaryByDigest creates a stmtSummaryByDigest from StmtExecInfo +func newStmtSummaryByDigest(sei *StmtExecInfo) *stmtSummaryByDigest { // Trim SQL to size MaxSQLLength maxSQLLength := config.GetGlobalConfig().StmtSummary.MaxSQLLength normalizedSQL := sei.NormalizedSQL @@ -109,7 +113,7 @@ func convertStmtExecInfoToSummary(sei *StmtExecInfo) *stmtSummary { sampleSQL = sampleSQL[:maxSQLLength] } - return &stmtSummary{ + return &stmtSummaryByDigest{ schemaName: sei.SchemaName, digest: sei.Digest, normalizedSQL: normalizedSQL, @@ -125,71 +129,67 @@ func convertStmtExecInfoToSummary(sei *StmtExecInfo) *stmtSummary { } } -// Add a new StmtExecInfo to stmtSummary -func addStmtExecInfoToSummary(sei *StmtExecInfo, ss *stmtSummary) { - ss.Lock() +// Add a StmtExecInfo to stmtSummary +func (ssbd *stmtSummaryByDigest) add(sei *StmtExecInfo) { + ssbd.Lock() - ss.sumLatency += sei.TotalLatency - ss.execCount++ - if sei.TotalLatency > ss.maxLatency { - ss.maxLatency = sei.TotalLatency + ssbd.sumLatency += sei.TotalLatency + ssbd.execCount++ + if sei.TotalLatency > ssbd.maxLatency { + ssbd.maxLatency = sei.TotalLatency } - if sei.TotalLatency < ss.minLatency { - ss.minLatency = sei.TotalLatency + if sei.TotalLatency < ssbd.minLatency { + ssbd.minLatency = sei.TotalLatency } - // TODO: uint64 overflow - ss.sumAffectedRows += sei.AffectedRows - ss.sumSentRows += sei.SentRows - if sei.StartTime.Before(ss.firstSeen) { - ss.firstSeen = sei.StartTime + ssbd.sumAffectedRows += sei.AffectedRows + ssbd.sumSentRows += sei.SentRows + if sei.StartTime.Before(ssbd.firstSeen) { + ssbd.firstSeen = sei.StartTime } - if ss.lastSeen.Before(sei.StartTime) { - ss.lastSeen = sei.StartTime + if ssbd.lastSeen.Before(sei.StartTime) { + ssbd.lastSeen = sei.StartTime } - ss.Unlock() + ssbd.Unlock() } -// Add a statement to statement summary. -func (ss *stmtSummaryByDigest) AddStatement(sei *StmtExecInfo) { - key := &stmtSummaryCacheKey{ +// AddStatement adds a statement to StmtSummaryByDigestMap. +func (ssMap *stmtSummaryByDigestMap) AddStatement(sei *StmtExecInfo) { + key := &stmtSummaryByDigestKey{ schemaName: sei.SchemaName, digest: sei.Digest, } - var summary *stmtSummary - ss.Lock() - value, ok := ss.summaryMap.Get(key) - if ok { - summary = value.(*stmtSummary) - } else { - newSummary := convertStmtExecInfoToSummary(sei) - ss.summaryMap.Put(key, newSummary) + ssMap.Lock() + value, ok := ssMap.summaryMap.Get(key) + if !ok { + newSummary := newStmtSummaryByDigest(sei) + ssMap.summaryMap.Put(key, newSummary) } - ss.Unlock() + ssMap.Unlock() // Lock a single entry, not the whole cache. - if summary != nil { - addStmtExecInfoToSummary(sei, summary) + if ok { + value.(*stmtSummaryByDigest).add(sei) } } -// Remove all statement summaries. -func (ss *stmtSummaryByDigest) Clear() { - ss.Lock() - ss.summaryMap.DeleteAll() - ss.Unlock() +// Clear removes all statement summaries. +func (ssMap *stmtSummaryByDigestMap) Clear() { + ssMap.Lock() + ssMap.summaryMap.DeleteAll() + ssMap.Unlock() } // Convert statement summary to Datum -func (ss *stmtSummaryByDigest) ToDatum() [][]types.Datum { - ss.Lock() - values := ss.summaryMap.Values() - ss.Unlock() +func (ssMap *stmtSummaryByDigestMap) ToDatum() [][]types.Datum { + ssMap.Lock() + values := ssMap.summaryMap.Values() + ssMap.Unlock() rows := make([][]types.Datum, 0, len(values)) for _, value := range values { - summary := value.(*stmtSummary) + summary := value.(*stmtSummaryByDigest) summary.Lock() record := types.MakeDatums( summary.schemaName, diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index a6ba6110103a5..ca6615bcab47c 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -29,11 +29,11 @@ import ( var _ = Suite(&testStmtSummarySuite{}) type testStmtSummarySuite struct { - stmtSummaryByDigest *stmtSummaryByDigest + ssMap *stmtSummaryByDigestMap } func (s *testStmtSummarySuite) SetUpSuite(c *C) { - s.stmtSummaryByDigest = NewStmtSummaryByDigest() + s.ssMap = newStmtSummaryByDigestMap() } func TestT(t *testing.T) { @@ -43,7 +43,7 @@ func TestT(t *testing.T) { // Test stmtSummaryByDigest.AddStatement func (s *testStmtSummarySuite) TestAddStatement(c *C) { - s.stmtSummaryByDigest.Clear() + s.ssMap.Clear() // First statement stmtExecInfo1 := &StmtExecInfo{ @@ -56,11 +56,11 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { SentRows: 100, StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), } - key := &stmtSummaryCacheKey{ + key := &stmtSummaryByDigestKey{ schemaName: stmtExecInfo1.SchemaName, digest: stmtExecInfo1.Digest, } - expectedSummary := stmtSummary{ + expectedSummary := stmtSummaryByDigest{ schemaName: stmtExecInfo1.SchemaName, digest: stmtExecInfo1.Digest, normalizedSQL: stmtExecInfo1.NormalizedSQL, @@ -75,10 +75,10 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { lastSeen: stmtExecInfo1.StartTime, } - s.stmtSummaryByDigest.AddStatement(stmtExecInfo1) - summary, ok := s.stmtSummaryByDigest.summaryMap.Get(key) + s.ssMap.AddStatement(stmtExecInfo1) + summary, ok := s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) - c.Assert(*summary.(*stmtSummary), Equals, expectedSummary) + c.Assert(*summary.(*stmtSummaryByDigest), Equals, expectedSummary) // Second statement stmtExecInfo2 := &StmtExecInfo{ @@ -98,10 +98,10 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { expectedSummary.sumSentRows += stmtExecInfo2.SentRows expectedSummary.lastSeen = stmtExecInfo2.StartTime - s.stmtSummaryByDigest.AddStatement(stmtExecInfo2) - summary, ok = s.stmtSummaryByDigest.summaryMap.Get(key) + s.ssMap.AddStatement(stmtExecInfo2) + summary, ok = s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) - c.Assert(*summary.(*stmtSummary), Equals, expectedSummary) + c.Assert(*summary.(*stmtSummaryByDigest), Equals, expectedSummary) // Third statement stmtExecInfo3 := &StmtExecInfo{ @@ -121,10 +121,10 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { expectedSummary.sumSentRows += stmtExecInfo3.SentRows expectedSummary.firstSeen = stmtExecInfo3.StartTime - s.stmtSummaryByDigest.AddStatement(stmtExecInfo3) - summary, ok = s.stmtSummaryByDigest.summaryMap.Get(key) + s.ssMap.AddStatement(stmtExecInfo3) + summary, ok = s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) - c.Assert(*summary.(*stmtSummary), Equals, expectedSummary) + c.Assert(*summary.(*stmtSummaryByDigest), Equals, expectedSummary) // Fourth statement that in a different schema stmtExecInfo4 := &StmtExecInfo{ @@ -137,14 +137,14 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { SentRows: 10, StartTime: time.Date(2019, 1, 1, 10, 10, 0, 10, time.UTC), } - key = &stmtSummaryCacheKey{ + key = &stmtSummaryByDigestKey{ schemaName: stmtExecInfo4.SchemaName, digest: stmtExecInfo4.Digest, } - s.stmtSummaryByDigest.AddStatement(stmtExecInfo4) - c.Assert(s.stmtSummaryByDigest.summaryMap.Size(), Equals, 2) - _, ok = s.stmtSummaryByDigest.summaryMap.Get(key) + s.ssMap.AddStatement(stmtExecInfo4) + c.Assert(s.ssMap.summaryMap.Size(), Equals, 2) + _, ok = s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) // Fifth statement that has a different digest @@ -158,14 +158,14 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { SentRows: 10, StartTime: time.Date(2019, 1, 1, 10, 10, 0, 10, time.UTC), } - key = &stmtSummaryCacheKey{ + key = &stmtSummaryByDigestKey{ schemaName: stmtExecInfo5.SchemaName, digest: stmtExecInfo5.Digest, } - s.stmtSummaryByDigest.AddStatement(stmtExecInfo5) - c.Assert(s.stmtSummaryByDigest.summaryMap.Size(), Equals, 3) - _, ok = s.stmtSummaryByDigest.summaryMap.Get(key) + s.ssMap.AddStatement(stmtExecInfo5) + c.Assert(s.ssMap.summaryMap.Size(), Equals, 3) + _, ok = s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) } @@ -180,7 +180,7 @@ func match(c *C, row []types.Datum, expected ...interface{}) { // Test stmtSummaryByDigest.ToDatum func (s *testStmtSummarySuite) TestToDatum(c *C) { - s.stmtSummaryByDigest.Clear() + s.ssMap.Clear() stmtExecInfo1 := &StmtExecInfo{ SchemaName: "schema_name", @@ -192,8 +192,8 @@ func (s *testStmtSummarySuite) TestToDatum(c *C) { SentRows: 100, StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), } - s.stmtSummaryByDigest.AddStatement(stmtExecInfo1) - datums := s.stmtSummaryByDigest.ToDatum() + s.ssMap.AddStatement(stmtExecInfo1) + datums := s.ssMap.ToDatum() c.Assert(len(datums), Equals, 1) t := types.Time{Time: types.FromGoTime(stmtExecInfo1.StartTime), Type: mysql.TypeTimestamp} match(c, datums[0], stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, stmtExecInfo1.NormalizedSQL, @@ -203,7 +203,7 @@ func (s *testStmtSummarySuite) TestToDatum(c *C) { // Test AddStatement and ToDatum parallel func (s *testStmtSummarySuite) TestAddStatementParallel(c *C) { - s.stmtSummaryByDigest.Clear() + s.ssMap.Clear() threads := 8 loops := 32 @@ -226,11 +226,11 @@ func (s *testStmtSummarySuite) TestAddStatementParallel(c *C) { // Add 32 times with different digest for i := 0; i < loops; i++ { stmtExecInfo1.Digest = fmt.Sprintf("digest%d", i) - s.stmtSummaryByDigest.AddStatement(stmtExecInfo1) + s.ssMap.AddStatement(stmtExecInfo1) } // There would be 32 summaries - datums := s.stmtSummaryByDigest.ToDatum() + datums := s.ssMap.ToDatum() c.Assert(len(datums), Equals, loops) } @@ -239,13 +239,13 @@ func (s *testStmtSummarySuite) TestAddStatementParallel(c *C) { } wg.Wait() - datums := s.stmtSummaryByDigest.ToDatum() + datums := s.ssMap.ToDatum() c.Assert(len(datums), Equals, loops) } // Test max number of statement count. func (s *testStmtSummarySuite) TestMaxStmtCount(c *C) { - s.stmtSummaryByDigest.Clear() + s.ssMap.Clear() stmtExecInfo1 := &StmtExecInfo{ SchemaName: "schema_name", @@ -264,16 +264,16 @@ func (s *testStmtSummarySuite) TestMaxStmtCount(c *C) { loops := int(maxStmtCount) * 10 for i := 0; i < loops; i++ { stmtExecInfo1.Digest = fmt.Sprintf("digest%d", i) - s.stmtSummaryByDigest.AddStatement(stmtExecInfo1) + s.ssMap.AddStatement(stmtExecInfo1) } // Summary count should be MaxStmtCount - sm := s.stmtSummaryByDigest.summaryMap + sm := s.ssMap.summaryMap c.Assert(sm.Size(), Equals, int(maxStmtCount)) // LRU cache should work for i := loops - int(maxStmtCount); i < loops; i++ { - key := &stmtSummaryCacheKey{ + key := &stmtSummaryByDigestKey{ schemaName: stmtExecInfo1.SchemaName, digest: fmt.Sprintf("digest%d", i), } @@ -284,7 +284,7 @@ func (s *testStmtSummarySuite) TestMaxStmtCount(c *C) { // Test max length of normalized and sample SQL. func (s *testStmtSummarySuite) TestMaxSQLLength(c *C) { - s.stmtSummaryByDigest.Clear() + s.ssMap.Clear() // Create a long SQL maxSQLLength := config.GetGlobalConfig().StmtSummary.MaxSQLLength @@ -302,15 +302,15 @@ func (s *testStmtSummarySuite) TestMaxSQLLength(c *C) { StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), } - s.stmtSummaryByDigest.AddStatement(stmtExecInfo1) - key := &stmtSummaryCacheKey{ + s.ssMap.AddStatement(stmtExecInfo1) + key := &stmtSummaryByDigestKey{ schemaName: stmtExecInfo1.SchemaName, digest: stmtExecInfo1.Digest, } - value, ok := s.stmtSummaryByDigest.summaryMap.Get(key) + value, ok := s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) // Length of normalizedSQL and sampleSQL should be maxSQLLength - summary := value.(*stmtSummary) + summary := value.(*stmtSummaryByDigest) c.Assert(len(summary.normalizedSQL), Equals, int(maxSQLLength)) c.Assert(len(summary.sampleSQL), Equals, int(maxSQLLength)) } From f60ad41bce8f3447c8a9626cc101e8a7dbdb97de Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 12 Sep 2019 15:56:46 +0800 Subject: [PATCH 23/26] clear events_statement_summary_by_digest when tidb_enable_stmt_summary is set to false --- executor/adapter.go | 4 +- infoschema/perfschema/tables_test.go | 24 +++++++++-- session/session.go | 6 +++ sessionctx/variable/session.go | 6 --- sessionctx/variable/sysvar.go | 8 ++++ sessionctx/variable/sysvar_observer_test.go | 48 +++++++++++++++++++++ sessionctx/variable/sysvar_observers.go | 46 ++++++++++++++++++++ sessionctx/variable/sysvar_test.go | 5 +++ sessionctx/variable/tidb_vars.go | 1 + tidb-server/main.go | 3 ++ util/stmtsummary/statement_summary.go | 18 ++++++++ util/stmtsummary/statement_summary_test.go | 30 +++++++++++++ 12 files changed, 187 insertions(+), 12 deletions(-) create mode 100644 sessionctx/variable/sysvar_observer_test.go create mode 100644 sessionctx/variable/sysvar_observers.go diff --git a/executor/adapter.go b/executor/adapter.go index f92a356fe6c6b..7e66dea831e5f 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -781,10 +781,10 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool) { } } -// SummaryStmt Collects statements for performance_schema.events_statements_summary_by_digest +// SummaryStmt collects statements for performance_schema.events_statements_summary_by_digest func (a *ExecStmt) SummaryStmt() { sessVars := a.Ctx.GetSessionVars() - if !sessVars.EnableStmtSummary || sessVars.InRestrictedSQL { + if sessVars.InRestrictedSQL || atomic.LoadInt32(&variable.EnableStmtSummary) == 0 { return } stmtCtx := sessVars.StmtCtx diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index a596d97b03155..15c7292d3823d 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -20,7 +20,9 @@ import ( "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/mockstore" + "github.com/pingcap/tidb/util/stmtsummary" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" ) @@ -38,6 +40,8 @@ type testTableSuite struct { } func (s *testTableSuite) SetUpSuite(c *C) { + variable.RegisterGlobalSysVarObserver(variable.TiDBEnableStmtSummary, stmtsummary.OnEnableStmtSummaryModified) + testleak.BeforeTest() var err error @@ -71,15 +75,12 @@ func (s *testTableSuite) TestStmtSummaryTable(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b varchar(10))") - // Statement summary is disabled in default + // Statement summary is disabled by default tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("0")) tk.MustExec("insert into t values(1, 'a')") tk.MustQuery("select * from performance_schema.events_statements_summary_by_digest").Check(testkit.Rows()) tk.MustExec("set global tidb_enable_stmt_summary = 1") - defer func() { - tk.MustExec("set global tidb_enable_stmt_summary = 0") - }() tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) // Invalidate the cache manually so that tidb_enable_stmt_summary works immediately. @@ -110,4 +111,19 @@ func (s *testTableSuite) TestStmtSummaryTable(c *C) { from performance_schema.events_statements_summary_by_digest order by exec_count desc limit 1`, ).Check(testkit.Rows("test 4 4 insert into t values(1, 'a')")) + + // Disable it again + tk.MustExec("set global tidb_enable_stmt_summary = 0") + tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("0")) + + // Create a new session to test + tk = testkit.NewTestKitWithInit(c, s.store) + + // This statement shouldn't be summarized + tk.MustQuery("select * from t where a=2") + + // The table should be cleared + tk.MustQuery(`select schema_name, exec_count, sum_rows_affected, query_sample_text + from performance_schema.events_statements_summary_by_digest`, + ).Check(testkit.Rows()) } diff --git a/session/session.go b/session/session.go index 4c25828592d63..144777be8d6a6 100644 --- a/session/session.go +++ b/session/session.go @@ -1776,6 +1776,12 @@ func (s *session) loadCommonGlobalVariablesIfNeeded() error { return err } } + if !succ { + err = variable.NotifyGlobalSysVarModify(varName, varVal) + if err != nil { + return err + } + } } // when client set Capability Flags CLIENT_INTERACTIVE, init wait_timeout with interactive_timeout diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 6d424070a3d8b..e402b060e02cf 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -420,9 +420,6 @@ type SessionVars struct { // AllowRemoveAutoInc indicates whether a user can drop the auto_increment column attribute or not. AllowRemoveAutoInc bool - - // Enable statement summary - EnableStmtSummary bool } // ConnectionInfo present connection used by audit. @@ -480,7 +477,6 @@ func NewSessionVars() *SessionVars { EnableNoopFuncs: DefTiDBEnableNoopFuncs, ReplicaRead: kv.ReplicaReadLeader, AllowRemoveAutoInc: DefTiDBAllowRemoveAutoInc, - EnableStmtSummary: DefTiDBEnableStmtSummary, } vars.Concurrency = Concurrency{ IndexLookupConcurrency: DefIndexLookupConcurrency, @@ -867,8 +863,6 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { } case TiDBAllowRemoveAutoInc: s.AllowRemoveAutoInc = TiDBOptOn(val) - case TiDBEnableStmtSummary: - s.EnableStmtSummary = TiDBOptOn(val) } s.systems[name] = val return nil diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 283be87f0aafc..7d68607a92b96 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -117,6 +117,14 @@ func BoolToIntStr(b bool) string { return "0" } +// BoolToInt32 converts bool to int32 +func BoolToInt32(b bool) int32 { + if b { + return 1 + } + return 0 +} + // we only support MySQL now var defaultSysVars = []*SysVar{ {ScopeGlobal, "gtid_mode", "OFF"}, diff --git a/sessionctx/variable/sysvar_observer_test.go b/sessionctx/variable/sysvar_observer_test.go new file mode 100644 index 0000000000000..57f7c5a3ddbdd --- /dev/null +++ b/sessionctx/variable/sysvar_observer_test.go @@ -0,0 +1,48 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package variable + +import ( + "testing" + + . "github.com/pingcap/check" + "github.com/pingcap/tidb/types" +) + +var _ = Suite(&testSysVarObserverSuite{}) + +type testSysVarObserverSuite struct { +} + +func TestSysVarObserver(t *testing.T) { + CustomVerboseFlag = true + TestingT(t) +} + +func (s *testSysVarObserverSuite) TestNotifyModify(c *C) { + var varName = "test_var_name" + var value = "old_value" + RegisterGlobalSysVarObserver(varName, func(newValue string) error { + value = newValue + return nil + }) + + datum := types.NewDatum("new_value") + NotifyGlobalSysVarModify(varName, datum) + c.Assert(value, Equals, "new_value") + + datum = types.NewDatum("old_value") + NotifyGlobalSysVarModify("another_name", datum) + c.Assert(value, Equals, "new_value") +} diff --git a/sessionctx/variable/sysvar_observers.go b/sessionctx/variable/sysvar_observers.go new file mode 100644 index 0000000000000..10a7ffea62729 --- /dev/null +++ b/sessionctx/variable/sysvar_observers.go @@ -0,0 +1,46 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package variable + +import ( + "github.com/pingcap/tidb/types" +) + +// globalSysVarObservers is a map who observes modifying of system values. +var globalSysVarObservers = map[string]func(newValue string) error{} + +// RegisterGlobalSysVarObserver registers a new observer for sys var. +// All observers must be registered before any session is created. +func RegisterGlobalSysVarObserver(varName string, observer func(newValue string) error) { + globalSysVarObservers[varName] = observer +} + +// NotifyGlobalSysVarModify triggers an observer if it's registered. +func NotifyGlobalSysVarModify(varName string, newValue types.Datum) error { + observer, ok := globalSysVarObservers[varName] + if !ok { + return nil + } + + sVal := "" + var err error + if !newValue.IsNull() { + sVal, err = newValue.ToString() + if err != nil { + return err + } + } + + return observer(sVal) +} diff --git a/sessionctx/variable/sysvar_test.go b/sessionctx/variable/sysvar_test.go index 702db0ac7fce1..5a23978f98f56 100644 --- a/sessionctx/variable/sysvar_test.go +++ b/sessionctx/variable/sysvar_test.go @@ -64,3 +64,8 @@ func (*testSysVarSuite) TestTxnMode(c *C) { err = seVar.setTxnMode("something else") c.Assert(err, NotNil) } + +func (*testSysVarSuite) TestBoolToInt32(c *C) { + c.Assert(BoolToInt32(true), Equals, int32(1)) + c.Assert(BoolToInt32(false), Equals, int32(0)) +} diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 83dfe6b8dd909..86c2091f62606 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -385,4 +385,5 @@ var ( MaxOfMaxAllowedPacket uint64 = 1073741824 ExpensiveQueryTimeThreshold uint64 = DefTiDBExpensiveQueryTimeThreshold MinExpensiveQueryTimeThreshold uint64 = 10 //10s + EnableStmtSummary int32 = BoolToInt32(DefTiDBEnableStmtSummary) ) diff --git a/tidb-server/main.go b/tidb-server/main.go index c2e2d80fbe531..d7c6be258781b 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -54,6 +54,7 @@ import ( "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/printer" "github.com/pingcap/tidb/util/signal" + "github.com/pingcap/tidb/util/stmtsummary" "github.com/pingcap/tidb/util/sys/linux" "github.com/pingcap/tidb/util/systimemon" "github.com/prometheus/client_golang/prometheus" @@ -511,6 +512,8 @@ func setGlobalVars() { variable.SysVars[variable.DataDir].Value = cfg.Path variable.SysVars[variable.TiDBSlowQueryFile].Value = cfg.Log.SlowQueryFile + variable.RegisterGlobalSysVarObserver(variable.TiDBEnableStmtSummary, stmtsummary.OnEnableStmtSummaryModified) + // For CI environment we default enable prepare-plan-cache. plannercore.SetPreparedPlanCache(config.CheckTableBeforeDrop || cfg.PreparedPlanCache.Enabled) if plannercore.PreparedPlanCacheEnabled() { diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index dac79f862e530..0ef9a581954e3 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -16,10 +16,12 @@ package stmtsummary import ( "strings" "sync" + "sync/atomic" "time" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/kvcache" @@ -161,6 +163,11 @@ func (ssMap *stmtSummaryByDigestMap) AddStatement(sei *StmtExecInfo) { } ssMap.Lock() + // Check again. Statements could be added before disabling the flag and after Clear() + if atomic.LoadInt32(&variable.EnableStmtSummary) == 0 { + ssMap.Unlock() + return + } value, ok := ssMap.summaryMap.Get(key) if !ok { newSummary := newStmtSummaryByDigest(sei) @@ -211,3 +218,14 @@ func (ssMap *stmtSummaryByDigestMap) ToDatum() [][]types.Datum { return rows } + +// OnEnableStmtSummaryModified is triggered once EnableStmtSummary is modified. +func OnEnableStmtSummaryModified(newValue string) error { + if variable.TiDBOptOn(newValue) { + atomic.StoreInt32(&variable.EnableStmtSummary, 1) + } else { + atomic.StoreInt32(&variable.EnableStmtSummary, 0) + StmtSummaryByDigestMap.Clear() + } + return nil +} diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index ca6615bcab47c..1f21ba434eb94 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -17,12 +17,14 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "testing" "time" . "github.com/pingcap/check" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" ) @@ -34,6 +36,7 @@ type testStmtSummarySuite struct { func (s *testStmtSummarySuite) SetUpSuite(c *C) { s.ssMap = newStmtSummaryByDigestMap() + atomic.StoreInt32(&variable.EnableStmtSummary, 1) } func TestT(t *testing.T) { @@ -314,3 +317,30 @@ func (s *testStmtSummarySuite) TestMaxSQLLength(c *C) { c.Assert(len(summary.normalizedSQL), Equals, int(maxSQLLength)) c.Assert(len(summary.sampleSQL), Equals, int(maxSQLLength)) } + +// Test setting EnableStmtSummary to 0 +func (s *testStmtSummarySuite) TestDisableStmtSummary(c *C) { + s.ssMap.Clear() + OnEnableStmtSummaryModified("0") + + stmtExecInfo1 := &StmtExecInfo{ + SchemaName: "schema_name", + OriginalSQL: "original_sql1", + NormalizedSQL: "normalized_sql", + Digest: "digest", + TotalLatency: 10000, + AffectedRows: 100, + SentRows: 100, + StartTime: time.Date(2019, 1, 1, 10, 10, 10, 10, time.UTC), + } + + s.ssMap.AddStatement(stmtExecInfo1) + datums := s.ssMap.ToDatum() + c.Assert(len(datums), Equals, 0) + + OnEnableStmtSummaryModified("1") + + s.ssMap.AddStatement(stmtExecInfo1) + datums = s.ssMap.ToDatum() + c.Assert(len(datums), Equals, 1) +} From 2fde3ad1253d47900bd783662c1442cfceeacff9 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 12 Sep 2019 17:37:55 +0800 Subject: [PATCH 24/26] remove sysvar_observers to simplify code --- domain/global_vars_cache.go | 26 +++++++++++ domain/global_vars_cache_test.go | 46 ++++++++++++++++++++ infoschema/perfschema/tables_test.go | 4 -- session/session.go | 6 --- sessionctx/variable/sysvar_observer_test.go | 48 --------------------- sessionctx/variable/sysvar_observers.go | 46 -------------------- tidb-server/main.go | 3 -- util/stmtsummary/statement_summary.go | 3 +- 8 files changed, 73 insertions(+), 109 deletions(-) delete mode 100644 sessionctx/variable/sysvar_observer_test.go delete mode 100644 sessionctx/variable/sysvar_observers.go diff --git a/domain/global_vars_cache.go b/domain/global_vars_cache.go index 89cc772f8fec7..68a8863a745ee 100644 --- a/domain/global_vars_cache.go +++ b/domain/global_vars_cache.go @@ -18,7 +18,9 @@ import ( "time" "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/stmtsummary" ) // GlobalVariableCache caches global variables. @@ -41,6 +43,8 @@ func (gvc *GlobalVariableCache) Update(rows []chunk.Row, fields []*ast.ResultFie gvc.rows = rows gvc.fields = fields gvc.Unlock() + + checkEnableStmtSummary(rows, fields) } // Get gets the global variables from cache. @@ -63,6 +67,28 @@ func (gvc *GlobalVariableCache) Disable() { return } +// checkEnableStmtSummary looks for TiDBEnableStmtSummary and notifies StmtSummary +func checkEnableStmtSummary(rows []chunk.Row, fields []*ast.ResultField) { + for _, row := range rows { + varName := row.GetString(0) + if varName == variable.TiDBEnableStmtSummary { + varVal := row.GetDatum(1, &fields[1].Column.FieldType) + + sVal := "" + if !varVal.IsNull() { + var err error + sVal, err = varVal.ToString() + if err != nil { + return + } + } + + stmtsummary.OnEnableStmtSummaryModified(sVal) + break + } + } +} + // GetGlobalVarsCache gets the global variable cache. func (do *Domain) GetGlobalVarsCache() *GlobalVariableCache { return &do.gvc diff --git a/domain/global_vars_cache_test.go b/domain/global_vars_cache_test.go index 11cb0c95a32f1..455cb30fcc28d 100644 --- a/domain/global_vars_cache_test.go +++ b/domain/global_vars_cache_test.go @@ -14,6 +14,7 @@ package domain import ( + "sync/atomic" "time" . "github.com/pingcap/check" @@ -21,6 +22,7 @@ import ( "github.com/pingcap/parser/charset" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" @@ -96,3 +98,47 @@ func getResultField(colName string, id, offset int) *ast.ResultField { DBName: model.NewCIStr("test"), } } + +func (gvcSuite *testGVCSuite) TestCheckEnableStmtSummary(c *C) { + defer testleak.AfterTest(c)() + testleak.BeforeTest() + + store, err := mockstore.NewMockTikvStore() + c.Assert(err, IsNil) + defer store.Close() + ddlLease := 50 * time.Millisecond + dom := NewDomain(store, ddlLease, 0, mockFactory) + err = dom.Init(ddlLease, sysMockFactory) + c.Assert(err, IsNil) + defer dom.Close() + + gvc := dom.GetGlobalVarsCache() + + rf := getResultField("c", 1, 0) + rf1 := getResultField("c1", 2, 1) + ft := &types.FieldType{ + Tp: mysql.TypeString, + Charset: charset.CharsetBin, + Collate: charset.CollationBin, + } + ft1 := &types.FieldType{ + Tp: mysql.TypeString, + Charset: charset.CharsetBin, + Collate: charset.CollationBin, + } + + atomic.StoreInt32(&variable.EnableStmtSummary, 0) + ck := chunk.NewChunkWithCapacity([]*types.FieldType{ft, ft1}, 1024) + ck.AppendString(0, variable.TiDBEnableStmtSummary) + ck.AppendString(1, "1") + row := ck.GetRow(0) + gvc.Update([]chunk.Row{row}, []*ast.ResultField{rf, rf1}) + c.Assert(atomic.LoadInt32(&variable.EnableStmtSummary), Equals, int32(1)) + + ck = chunk.NewChunkWithCapacity([]*types.FieldType{ft, ft1}, 1024) + ck.AppendString(0, variable.TiDBEnableStmtSummary) + ck.AppendString(1, "0") + row = ck.GetRow(0) + gvc.Update([]chunk.Row{row}, []*ast.ResultField{rf, rf1}) + c.Assert(atomic.LoadInt32(&variable.EnableStmtSummary), Equals, int32(0)) +} diff --git a/infoschema/perfschema/tables_test.go b/infoschema/perfschema/tables_test.go index 15c7292d3823d..5f7d044431570 100644 --- a/infoschema/perfschema/tables_test.go +++ b/infoschema/perfschema/tables_test.go @@ -20,9 +20,7 @@ import ( "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/session" - "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/mockstore" - "github.com/pingcap/tidb/util/stmtsummary" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" ) @@ -40,8 +38,6 @@ type testTableSuite struct { } func (s *testTableSuite) SetUpSuite(c *C) { - variable.RegisterGlobalSysVarObserver(variable.TiDBEnableStmtSummary, stmtsummary.OnEnableStmtSummaryModified) - testleak.BeforeTest() var err error diff --git a/session/session.go b/session/session.go index 144777be8d6a6..4c25828592d63 100644 --- a/session/session.go +++ b/session/session.go @@ -1776,12 +1776,6 @@ func (s *session) loadCommonGlobalVariablesIfNeeded() error { return err } } - if !succ { - err = variable.NotifyGlobalSysVarModify(varName, varVal) - if err != nil { - return err - } - } } // when client set Capability Flags CLIENT_INTERACTIVE, init wait_timeout with interactive_timeout diff --git a/sessionctx/variable/sysvar_observer_test.go b/sessionctx/variable/sysvar_observer_test.go deleted file mode 100644 index 57f7c5a3ddbdd..0000000000000 --- a/sessionctx/variable/sysvar_observer_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package variable - -import ( - "testing" - - . "github.com/pingcap/check" - "github.com/pingcap/tidb/types" -) - -var _ = Suite(&testSysVarObserverSuite{}) - -type testSysVarObserverSuite struct { -} - -func TestSysVarObserver(t *testing.T) { - CustomVerboseFlag = true - TestingT(t) -} - -func (s *testSysVarObserverSuite) TestNotifyModify(c *C) { - var varName = "test_var_name" - var value = "old_value" - RegisterGlobalSysVarObserver(varName, func(newValue string) error { - value = newValue - return nil - }) - - datum := types.NewDatum("new_value") - NotifyGlobalSysVarModify(varName, datum) - c.Assert(value, Equals, "new_value") - - datum = types.NewDatum("old_value") - NotifyGlobalSysVarModify("another_name", datum) - c.Assert(value, Equals, "new_value") -} diff --git a/sessionctx/variable/sysvar_observers.go b/sessionctx/variable/sysvar_observers.go deleted file mode 100644 index 10a7ffea62729..0000000000000 --- a/sessionctx/variable/sysvar_observers.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package variable - -import ( - "github.com/pingcap/tidb/types" -) - -// globalSysVarObservers is a map who observes modifying of system values. -var globalSysVarObservers = map[string]func(newValue string) error{} - -// RegisterGlobalSysVarObserver registers a new observer for sys var. -// All observers must be registered before any session is created. -func RegisterGlobalSysVarObserver(varName string, observer func(newValue string) error) { - globalSysVarObservers[varName] = observer -} - -// NotifyGlobalSysVarModify triggers an observer if it's registered. -func NotifyGlobalSysVarModify(varName string, newValue types.Datum) error { - observer, ok := globalSysVarObservers[varName] - if !ok { - return nil - } - - sVal := "" - var err error - if !newValue.IsNull() { - sVal, err = newValue.ToString() - if err != nil { - return err - } - } - - return observer(sVal) -} diff --git a/tidb-server/main.go b/tidb-server/main.go index d7c6be258781b..c2e2d80fbe531 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -54,7 +54,6 @@ import ( "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/printer" "github.com/pingcap/tidb/util/signal" - "github.com/pingcap/tidb/util/stmtsummary" "github.com/pingcap/tidb/util/sys/linux" "github.com/pingcap/tidb/util/systimemon" "github.com/prometheus/client_golang/prometheus" @@ -512,8 +511,6 @@ func setGlobalVars() { variable.SysVars[variable.DataDir].Value = cfg.Path variable.SysVars[variable.TiDBSlowQueryFile].Value = cfg.Log.SlowQueryFile - variable.RegisterGlobalSysVarObserver(variable.TiDBEnableStmtSummary, stmtsummary.OnEnableStmtSummaryModified) - // For CI environment we default enable prepare-plan-cache. plannercore.SetPreparedPlanCache(config.CheckTableBeforeDrop || cfg.PreparedPlanCache.Enabled) if plannercore.PreparedPlanCacheEnabled() { diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index 0ef9a581954e3..75e931db5a7c3 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -220,12 +220,11 @@ func (ssMap *stmtSummaryByDigestMap) ToDatum() [][]types.Datum { } // OnEnableStmtSummaryModified is triggered once EnableStmtSummary is modified. -func OnEnableStmtSummaryModified(newValue string) error { +func OnEnableStmtSummaryModified(newValue string) { if variable.TiDBOptOn(newValue) { atomic.StoreInt32(&variable.EnableStmtSummary, 1) } else { atomic.StoreInt32(&variable.EnableStmtSummary, 0) StmtSummaryByDigestMap.Clear() } - return nil } From 06beeac2855c2719c856d8ce0e486cdf8e10138f Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 12 Sep 2019 18:03:01 +0800 Subject: [PATCH 25/26] modify TestStmtSummaryTable to avoid copying lock value --- util/stmtsummary/statement_summary_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index 1f21ba434eb94..614f6f7800146 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -81,7 +81,7 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { s.ssMap.AddStatement(stmtExecInfo1) summary, ok := s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) - c.Assert(*summary.(*stmtSummaryByDigest), Equals, expectedSummary) + c.Assert(*summary.(*stmtSummaryByDigest) == expectedSummary, IsTrue) // Second statement stmtExecInfo2 := &StmtExecInfo{ @@ -104,7 +104,7 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { s.ssMap.AddStatement(stmtExecInfo2) summary, ok = s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) - c.Assert(*summary.(*stmtSummaryByDigest), Equals, expectedSummary) + c.Assert(*summary.(*stmtSummaryByDigest) == expectedSummary, IsTrue) // Third statement stmtExecInfo3 := &StmtExecInfo{ @@ -127,7 +127,7 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { s.ssMap.AddStatement(stmtExecInfo3) summary, ok = s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) - c.Assert(*summary.(*stmtSummaryByDigest), Equals, expectedSummary) + c.Assert(*summary.(*stmtSummaryByDigest) == expectedSummary, IsTrue) // Fourth statement that in a different schema stmtExecInfo4 := &StmtExecInfo{ From 5e59a4193a5807adfec2c57491df7c86aefa8245 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 12 Sep 2019 19:05:45 +0800 Subject: [PATCH 26/26] rephrase comments for statement summary --- config/config.go | 4 ++-- config/config.toml.example | 2 +- sessionctx/variable/tidb_vars.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index c37253cd1d8ee..adeb4cc1c623c 100644 --- a/config/config.go +++ b/config/config.go @@ -319,9 +319,9 @@ type PessimisticTxn struct { // StmtSummary is the config for statement summary. type StmtSummary struct { - // How many statements can be kept in the table + // The maximum number of statements kept in memory. MaxStmtCount uint `toml:"max-stmt-count" json:"max-stmt-count"` - // How long can a normalizedSQL / sampleSQL be + // The maximum length of displayed normalized SQL and sample SQL. MaxSQLLength uint `toml:"max-sql-length" json:"max-sql-length"` } diff --git a/config/config.toml.example b/config/config.toml.example index 28dab02b95159..f4891850d7690 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -317,7 +317,7 @@ max-retry-count = 256 ttl = "40s" [stmt-summary] -# max count of statement summary kept in memory. +# max number of statements kept in memory. max-stmt-count = 100 # max length of displayed normalized sql and sample sql. diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 86c2091f62606..af9c936dc4575 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -297,7 +297,7 @@ const ( // TiDBEnableNoopFuncs set true will enable using fake funcs(like get_lock release_lock) TiDBEnableNoopFuncs = "tidb_enable_noop_functions" - // TiDBEnableStmtSummary indicates to enable statement summary + // TiDBEnableStmtSummary indicates whether the statement summary is enabled. TiDBEnableStmtSummary = "tidb_enable_stmt_summary" )