Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

stmtsummary: add persistence implementation #40814

Merged
merged 54 commits into from
Feb 9, 2023
Merged
Show file tree
Hide file tree
Changes from 42 commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
5828537
stmtsummary: add persistent v2 implementation
mornyx Jan 29, 2023
70b3bd5
Merge branch 'master' into stmtsummary
mornyx Jan 29, 2023
8308dc2
resolve comments
mornyx Jan 30, 2023
85c5f0d
Merge branch 'master' into stmtsummary
mornyx Jan 31, 2023
2bc61d8
update bazel file
mornyx Jan 31, 2023
4659ce8
fix stmtsummary global var value
mornyx Jan 31, 2023
3302e54
use BoolToOnOff
mornyx Jan 31, 2023
e01dd47
optimize time window processing
mornyx Jan 31, 2023
8c85d12
Merge branch 'master' into stmtsummary
mornyx Jan 31, 2023
4a6ba00
defaultRotateCheckInterval 5s -> 1s
mornyx Jan 31, 2023
e9e6fa0
fix linting
mornyx Jan 31, 2023
9bf28ff
fix explain info
mornyx Jan 31, 2023
4ca4405
unify
zhongzc Jan 31, 2023
a64f1b4
Merge remote-tracking branch 'mornyx/stmtsummary' into HEAD
zhongzc Jan 31, 2023
fa60ecc
Merge pull request #3 from zhongzc/stmtsummary
mornyx Jan 31, 2023
dc67841
fix linting & bazel
mornyx Jan 31, 2023
c768c8c
fix build
zhongzc Jan 31, 2023
7d92952
Merge pull request #4 from zhongzc/stmtsummary
mornyx Jan 31, 2023
34ff06b
fix predicates
mornyx Feb 1, 2023
3d982ce
Merge branch 'master' into stmtsummary
mornyx Feb 2, 2023
c56cc01
refine stmt v1 retriever
zhongzc Feb 2, 2023
f014848
Merge pull request #5 from zhongzc/stmtsummary
mornyx Feb 2, 2023
f29ba58
remove field 'Enable' in stmtsummary extractor
mornyx Feb 2, 2023
c76ebad
refine: use coarse time range
zhongzc Feb 2, 2023
c97bbd2
refine stmt summary v2 retriever
zhongzc Feb 3, 2023
d7ce5b0
tiny
zhongzc Feb 3, 2023
db456e6
Merge pull request #6 from zhongzc/stmtsummary
mornyx Feb 3, 2023
2d1bd06
simplify locks
mornyx Feb 6, 2023
84846b0
fix window creation in rotate loop
mornyx Feb 6, 2023
e665687
use warning instead of stop while parsing file meta
mornyx Feb 6, 2023
1dddf0f
fix stmt files close
mornyx Feb 6, 2023
8252148
some refine
zhongzc Feb 6, 2023
3d77170
Merge pull request #7 from zhongzc/stmtsummary
mornyx Feb 6, 2023
a99f5ad
concurrent history reader
zhongzc Feb 7, 2023
7c7fc1f
tiny
zhongzc Feb 7, 2023
53e7ae1
Merge pull request #8 from zhongzc/stmtsummary
mornyx Feb 7, 2023
ba7d449
comment
zhongzc Feb 7, 2023
ab93000
Merge pull request #9 from zhongzc/stmtsummary
mornyx Feb 7, 2023
838a6e3
Merge branch 'master' into stmtsummary
mornyx Feb 7, 2023
774de35
fix stmt window lock
mornyx Feb 7, 2023
e851cba
fix unlock
mornyx Feb 7, 2023
487aacd
Merge branch 'master' into stmtsummary
mornyx Feb 7, 2023
aa1ca21
fix
mornyx Feb 8, 2023
127c1ae
update copyright
mornyx Feb 8, 2023
198d960
optimize time ranges overlap
mornyx Feb 8, 2023
3b55994
Update util/stmtsummary/v2/reader.go
crazycs520 Feb 8, 2023
a359ae9
Update executor/stmtsummary.go
crazycs520 Feb 8, 2023
0c9c01c
misc update
mornyx Feb 8, 2023
eed9d83
rounding time window
mornyx Feb 8, 2023
583d619
make summary_end_time compatible
mornyx Feb 8, 2023
4548bfb
revert time window rounding & fix ut
mornyx Feb 9, 2023
d12f573
Merge branch 'master' into stmtsummary
crazycs520 Feb 9, 2023
8712341
Merge branch 'master' into stmtsummary
ti-chi-bot Feb 9, 2023
c5914a7
Merge branch 'master' into stmtsummary
ti-chi-bot Feb 9, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion bindinfo/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ go_library(
"//util/memory",
"//util/parser",
"//util/sqlexec",
"//util/stmtsummary",
"//util/table-filter",
"//util/timeutil",
"@org_golang_x_exp//maps",
Expand Down
3 changes: 1 addition & 2 deletions bindinfo/handle.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ import (
"github.com/pingcap/tidb/util/logutil"
utilparser "github.com/pingcap/tidb/util/parser"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/pingcap/tidb/util/stmtsummary"
tablefilter "github.com/pingcap/tidb/util/table-filter"
"github.com/pingcap/tidb/util/timeutil"
"go.uber.org/zap"
Expand Down Expand Up @@ -892,7 +891,7 @@ func (h *BindHandle) CaptureBaselines() {
parser4Capture := parser.New()
captureFilter := h.extractCaptureFilterFromStorage()
emptyCaptureFilter := captureFilter.isEmpty()
bindableStmts := stmtsummary.StmtSummaryByDigestMap.GetMoreThanCntBindableStmt(captureFilter.frequency)
bindableStmts := h.sctx.GetSessionVars().StmtSummary.GetBindableStmts(captureFilter.frequency)
for _, bindableStmt := range bindableStmts {
stmt, err := parser4Capture.ParseOneStmt(bindableStmt.Query, bindableStmt.Charset, bindableStmt.Collation)
if err != nil {
Expand Down
19 changes: 19 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -502,6 +502,20 @@ type Instance struct {
DDLSlowOprThreshold uint32 `toml:"ddl_slow_threshold" json:"ddl_slow_threshold"`
// ExpensiveQueryTimeThreshold indicates the time threshold of expensive query.
ExpensiveQueryTimeThreshold uint64 `toml:"tidb_expensive_query_time_threshold" json:"tidb_expensive_query_time_threshold"`
// StmtSummaryEnablePersistent indicates whether to enable file persistence for stmtsummary.
StmtSummaryEnablePersistent bool `toml:"tidb_stmt_summary_enable_persistent" json:"tidb_stmt_summary_enable_persistent"`
// StmtSummaryFilename indicates the file name written by stmtsummary
// when StmtSummaryEnablePersistent is true.
StmtSummaryFilename string `toml:"tidb_stmt_summary_filename" json:"tidb_stmt_summary_filename"`
// StmtSummaryFileMaxDays indicates how many days the files written by
// stmtsummary will be kept when StmtSummaryEnablePersistent is true.
StmtSummaryFileMaxDays int `toml:"tidb_stmt_summary_file_max_days" json:"tidb_stmt_summary_file_max_days"`
// StmtSummaryFileMaxSize indicates the maximum size (in mb) of a single file
// written by stmtsummary when StmtSummaryEnablePersistent is true.
StmtSummaryFileMaxSize int `toml:"tidb_stmt_summary_file_max_size" json:"tidb_stmt_summary_file_max_size"`
// StmtSummaryFileMaxBackups indicates the maximum number of files written
// by stmtsummary when StmtSummaryEnablePersistent is true.
StmtSummaryFileMaxBackups int `toml:"tidb_stmt_summary_file_max_backups" json:"tidb_stmt_summary_file_max_backups"`

// These variables exist in both 'instance' section and another place.
// The configuration in 'instance' section takes precedence.
Expand Down Expand Up @@ -901,6 +915,11 @@ var defaultConf = Config{
EnablePProfSQLCPU: false,
DDLSlowOprThreshold: DefDDLSlowOprThreshold,
ExpensiveQueryTimeThreshold: DefExpensiveQueryTimeThreshold,
StmtSummaryEnablePersistent: false,
StmtSummaryFilename: "tidb-statements.log",
StmtSummaryFileMaxDays: 3,
StmtSummaryFileMaxSize: 64,
StmtSummaryFileMaxBackups: 0,
EnableSlowLog: *NewAtomicBool(logutil.DefaultTiDBEnableSlowLog),
SlowThreshold: logutil.DefaultSlowThreshold,
RecordPlanInSlowLog: logutil.DefaultRecordPlanInSlowLog,
Expand Down
4 changes: 4 additions & 0 deletions executor/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ go_library(
"slow_query.go",
"sort.go",
"split.go",
"stmtsummary.go",
"table_reader.go",
"trace.go",
"union_scan.go",
Expand Down Expand Up @@ -194,6 +195,7 @@ go_library(
"//util/size",
"//util/sqlexec",
"//util/stmtsummary",
"//util/stmtsummary/v2:stmtsummary",
"//util/stringutil",
"//util/table-filter",
"//util/timeutil",
Expand Down Expand Up @@ -326,6 +328,7 @@ go_test(
"split_test.go",
"stale_txn_test.go",
"statement_context_test.go",
"stmtsummary_test.go",
"table_readers_required_rows_test.go",
"temporary_table_test.go",
"tikv_regions_peers_table_test.go",
Expand Down Expand Up @@ -419,6 +422,7 @@ go_test(
"//util/rowcodec",
"//util/set",
"//util/sqlexec",
"//util/stmtsummary/v2:stmtsummary",
"//util/stringutil",
"//util/tableutil",
"//util/timeutil",
Expand Down
6 changes: 4 additions & 2 deletions executor/adapter.go
Original file line number Diff line number Diff line change
Expand Up @@ -1802,8 +1802,10 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
userString = sessVars.User.Username
}

enabled := sessVars.StmtSummary.Enabled()
enabledInternalQuery := sessVars.StmtSummary.EnabledInternal()
// Internal SQLs must also be recorded to keep the consistency of `PrevStmt` and `PrevStmtDigest`.
if !stmtsummary.StmtSummaryByDigestMap.Enabled() || ((sessVars.InRestrictedSQL || len(userString) == 0) && !stmtsummary.StmtSummaryByDigestMap.EnabledInternal()) {
if !enabled || ((sessVars.InRestrictedSQL || len(userString) == 0) && !enabledInternalQuery) {
sessVars.SetPrevStmtDigest("")
return
}
Expand Down Expand Up @@ -1920,7 +1922,7 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
if a.retryCount > 0 {
stmtExecInfo.ExecRetryTime = costTime - sessVars.DurationParse - sessVars.DurationCompile - time.Since(a.retryStartTime)
}
stmtsummary.StmtSummaryByDigestMap.AddStatement(stmtExecInfo)
sessVars.StmtSummary.Add(stmtExecInfo)
}

// GetTextToLog return the query text to log.
Expand Down
17 changes: 8 additions & 9 deletions executor/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -1937,8 +1937,6 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo
strings.ToLower(infoschema.TableTiFlashReplica),
strings.ToLower(infoschema.TableTiDBServersInfo),
strings.ToLower(infoschema.TableTiKVStoreStatus),
strings.ToLower(infoschema.TableStatementsSummaryEvicted),
strings.ToLower(infoschema.ClusterTableStatementsSummaryEvicted),
strings.ToLower(infoschema.TableClientErrorsSummaryGlobal),
strings.ToLower(infoschema.TableClientErrorsSummaryByUser),
strings.ToLower(infoschema.TableClientErrorsSummaryByHost),
Expand Down Expand Up @@ -1993,16 +1991,18 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo
}
case strings.ToLower(infoschema.TableStatementsSummary),
strings.ToLower(infoschema.TableStatementsSummaryHistory),
strings.ToLower(infoschema.TableStatementsSummaryEvicted),
strings.ToLower(infoschema.ClusterTableStatementsSummary),
strings.ToLower(infoschema.ClusterTableStatementsSummaryHistory),
strings.ToLower(infoschema.ClusterTableStatementsSummary):
strings.ToLower(infoschema.ClusterTableStatementsSummaryEvicted):
var extractor *plannercore.StatementsSummaryExtractor
if v.Extractor != nil {
extractor = v.Extractor.(*plannercore.StatementsSummaryExtractor)
}
Comment on lines +1997 to +2001
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Beside the difference between stmtSummaryRetrieverV2 and stmtSummaryRetriever, there are other changes:

  1. moved tables
  2. the nil check for v.Extractor

Feeling curious about these.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. Previously TableStatementsSummaryEvicted and ClusterTableStatementsSummaryEvicted were handled separately, and the code organization was different from other tables. Here we just group them together in stmtsummary.go.
  2. I constructed the case where v.Extractor is nil in my tests, and I found that v.Extractor.(*plannercore.StatementsSummaryExtractor) will panic due to a type mismatch.

Copy link
Contributor

@zhongzc zhongzc Jan 30, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Previously, to retrieve TableStatementsSummaryEvicted and ClusterTableStatementsSummaryEvicted we use memtableRetriever, but stmtSummaryRetriever here. Do they act the same?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I just migrated this part of code(just a bit: https://github.com/pingcap/tidb/pull/40814/files#diff-feafb829b9dc329222267c9010f60ed7ce8800012c1e6dbcf6beb27d99e71b49L2308-L2323) to executor/stmtsummary.go. I think this will make stmtsummary code more independent.

return &MemTableReaderExec{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()),
table: v.Table,
retriever: &stmtSummaryTableRetriever{
table: v.Table,
columns: v.Columns,
extractor: v.Extractor.(*plannercore.StatementsSummaryExtractor),
},
retriever: buildStmtSummaryRetriever(b.ctx, v.Table, v.Columns, extractor),
}
case strings.ToLower(infoschema.TableColumns):
return &MemTableReaderExec{
Expand All @@ -2016,7 +2016,6 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo
viewOutputNamesMap: make(map[int64]types.NameSlice),
},
}

case strings.ToLower(infoschema.TableSlowQuery), strings.ToLower(infoschema.ClusterTableSlowLog):
memTracker := memory.NewTracker(v.ID(), -1)
memTracker.AttachTo(b.ctx.GetSessionVars().StmtCtx.MemTracker)
Expand Down
67 changes: 0 additions & 67 deletions executor/infoschema_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ import (
"github.com/pingcap/tidb/util/servermemorylimit"
"github.com/pingcap/tidb/util/set"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/pingcap/tidb/util/stmtsummary"
"github.com/pingcap/tidb/util/stringutil"
"github.com/tikv/client-go/v2/txnkv/txnlock"
"go.uber.org/zap"
Expand Down Expand Up @@ -158,9 +157,6 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
e.dataForTableTiFlashReplica(sctx, dbs)
case infoschema.TableTiKVStoreStatus:
err = e.dataForTiKVStoreStatus(sctx)
case infoschema.TableStatementsSummaryEvicted,
infoschema.ClusterTableStatementsSummaryEvicted:
err = e.setDataForStatementsSummaryEvicted(sctx)
case infoschema.TableClientErrorsSummaryGlobal,
infoschema.TableClientErrorsSummaryByUser,
infoschema.TableClientErrorsSummaryByHost:
Expand Down Expand Up @@ -2306,22 +2302,6 @@ func (e *memtableRetriever) dataForTableTiFlashReplica(ctx sessionctx.Context, s
e.rows = rows
}

func (e *memtableRetriever) setDataForStatementsSummaryEvicted(ctx sessionctx.Context) error {
if !hasPriv(ctx, mysql.ProcessPriv) {
return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS")
}
e.rows = stmtsummary.StmtSummaryByDigestMap.ToEvictedCountDatum()
switch e.table.Name.O {
case infoschema.ClusterTableStatementsSummaryEvicted:
rows, err := infoschema.AppendHostInfoToRows(ctx, e.rows)
if err != nil {
return err
}
e.rows = rows
}
return nil
}

func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context, tableName string) error {
// Seeing client errors should require the PROCESS privilege, with the exception of errors for your own user.
// This is similar to information_schema.processlist, which is the closest comparison.
Expand Down Expand Up @@ -2476,50 +2456,6 @@ func (e *memtableRetriever) setDataForClusterMemoryUsageOpsHistory(ctx sessionct
return nil
}

type stmtSummaryTableRetriever struct {
dummyCloser
table *model.TableInfo
columns []*model.ColumnInfo
retrieved bool
extractor *plannercore.StatementsSummaryExtractor
}

// retrieve implements the infoschemaRetriever interface
func (e *stmtSummaryTableRetriever) retrieve(ctx context.Context, sctx sessionctx.Context) ([][]types.Datum, error) {
if e.extractor.SkipRequest || e.retrieved {
return nil, nil
}
e.retrieved = true

var err error
var instanceAddr string
switch e.table.Name.O {
case infoschema.ClusterTableStatementsSummary,
infoschema.ClusterTableStatementsSummaryHistory:
instanceAddr, err = infoschema.GetInstanceAddr(sctx)
if err != nil {
return nil, err
}
}
user := sctx.GetSessionVars().User
reader := stmtsummary.NewStmtSummaryReader(user, hasPriv(sctx, mysql.ProcessPriv), e.columns, instanceAddr, sctx.GetSessionVars().StmtCtx.TimeZone)
if e.extractor.Enable {
checker := stmtsummary.NewStmtSummaryChecker(e.extractor.Digests)
reader.SetChecker(checker)
}
var rows [][]types.Datum
switch e.table.Name.O {
case infoschema.TableStatementsSummary,
infoschema.ClusterTableStatementsSummary:
rows = reader.GetStmtSummaryCurrentRows()
case infoschema.TableStatementsSummaryHistory,
infoschema.ClusterTableStatementsSummaryHistory:
rows = reader.GetStmtSummaryHistoryRows()
}

return rows, nil
}

// tidbTrxTableRetriever is the memtable retriever for the TIDB_TRX and CLUSTER_TIDB_TRX table.
type tidbTrxTableRetriever struct {
dummyCloser
Expand Down Expand Up @@ -3018,9 +2954,6 @@ func (e *hugeMemTableRetriever) retrieve(ctx context.Context, sctx sessionctx.Co
}

func adjustColumns(input [][]types.Datum, outColumns []*model.ColumnInfo, table *model.TableInfo) [][]types.Datum {
if table.Name.O == infoschema.TableStatementsSummary {
return input
}
if len(outColumns) == len(table.Columns) {
return input
}
Expand Down
Loading