From 33e5baade9f7d9a304a9db0ab96595ec9582685a Mon Sep 17 00:00:00 2001 From: Bin <49082129+songzhibin97@users.noreply.github.com> Date: Tue, 13 Dec 2022 10:24:51 +0800 Subject: [PATCH 1/3] fix: update ioutil.readall (#39850) --- br/pkg/storage/memstore_test.go | 5 ++--- executor/set_test.go | 3 +-- parser/consistent_test.go | 4 ++-- parser/format/format_test.go | 4 ++-- parser/reserved_words_test.go | 4 ++-- planner/core/binary_plan_test.go | 4 ++-- server/plan_replayer.go | 6 +++--- server/plan_replayer_test.go | 3 +-- statistics/handle/dump.go | 4 ++-- util/cpuprofile/cpuprofile_test.go | 3 +-- 10 files changed, 18 insertions(+), 22 deletions(-) diff --git a/br/pkg/storage/memstore_test.go b/br/pkg/storage/memstore_test.go index a85a2ff467fa1..3ae9a08d168bc 100644 --- a/br/pkg/storage/memstore_test.go +++ b/br/pkg/storage/memstore_test.go @@ -17,7 +17,6 @@ import ( "bytes" "context" "io" - "io/ioutil" "sync" "testing" "time" @@ -70,7 +69,7 @@ func TestMemStoreBasic(t *testing.T) { require.Nil(t, err) r2, err := store.Open(ctx, "/hello.txt") require.Nil(t, err) - fileContent, err = ioutil.ReadAll(r) + fileContent, err = io.ReadAll(r) require.Nil(t, err) require.True(t, bytes.Equal([]byte("hello world 3"), fileContent)) require.Nil(t, r.Close()) @@ -83,7 +82,7 @@ func TestMemStoreBasic(t *testing.T) { _, err = r2.Seek(5, io.SeekStart) require.Nil(t, err) - fileContent, err = ioutil.ReadAll(r2) + fileContent, err = io.ReadAll(r2) require.Nil(t, err) require.True(t, bytes.Equal([]byte(" world 3"), fileContent)) diff --git a/executor/set_test.go b/executor/set_test.go index 75b6c210146b7..c24e8226a03bf 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -20,7 +20,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "strconv" "testing" @@ -1802,7 +1801,7 @@ func TestSetClusterConfig(t *testing.T) { httpCnt = 0 tk.Session().SetValue(executor.TestSetConfigHTTPHandlerKey, func(req *http.Request) (*http.Response, error) { httpCnt++ - body, err := ioutil.ReadAll(req.Body) + body, err := io.ReadAll(req.Body) require.NoError(t, err) // The `raftstore.` prefix is stripped. require.JSONEq(t, `{"server.snap-max-write-bytes-per-sec":"500MB"}`, string(body)) diff --git a/parser/consistent_test.go b/parser/consistent_test.go index e78b7f31ddddd..1acc1a58bc850 100644 --- a/parser/consistent_test.go +++ b/parser/consistent_test.go @@ -14,7 +14,7 @@ package parser import ( - "io/ioutil" + gio "io" "os" "sort" "strings" @@ -27,7 +27,7 @@ func TestKeywordConsistent(t *testing.T) { parserFilename := "parser.y" parserFile, err := os.Open(parserFilename) requires.NoError(t, err) - data, err := ioutil.ReadAll(parserFile) + data, err := gio.ReadAll(parserFile) requires.NoError(t, err) content := string(data) diff --git a/parser/format/format_test.go b/parser/format/format_test.go index 429c2c27c19d1..bc04033214cdd 100644 --- a/parser/format/format_test.go +++ b/parser/format/format_test.go @@ -15,7 +15,7 @@ package format import ( "bytes" - "io/ioutil" + "io" "strings" "testing" @@ -26,7 +26,7 @@ import ( func checkFormat(t *testing.T, f Formatter, buf *bytes.Buffer, str, expect string) { _, err := f.Format(str, 3) require.NoError(t, err) - b, err := ioutil.ReadAll(buf) + b, err := io.ReadAll(buf) require.NoError(t, err) require.Equal(t, expect, string(b)) } diff --git a/parser/reserved_words_test.go b/parser/reserved_words_test.go index f58359a7d20a3..6896167741284 100644 --- a/parser/reserved_words_test.go +++ b/parser/reserved_words_test.go @@ -28,7 +28,7 @@ import ( // needed to connect to MySQL dbsql "database/sql" - "io/ioutil" + gio "io" "os" "testing" @@ -41,7 +41,7 @@ func TestCompareReservedWordsWithMySQL(t *testing.T) { parserFilename := "parser.y" parserFile, err := os.Open(parserFilename) requires.NoError(t, err) - data, err := ioutil.ReadAll(parserFile) + data, err := gio.ReadAll(parserFile) requires.NoError(t, err) content := string(data) diff --git a/planner/core/binary_plan_test.go b/planner/core/binary_plan_test.go index 806e750d33fac..be8bf75378dc7 100644 --- a/planner/core/binary_plan_test.go +++ b/planner/core/binary_plan_test.go @@ -17,7 +17,7 @@ package core_test import ( "encoding/base64" "fmt" - "io/ioutil" + "io" "os" "regexp" "strings" @@ -521,7 +521,7 @@ func TestUnnecessaryBinaryPlanInSlowLog(t *testing.T) { tk.MustExec("drop table if exists th") tk.MustExec("set global tidb_slow_log_threshold = 1;") tk.MustExec("create table th (i int, a int,b int, c int, index (a)) partition by hash (a) partitions 100;") - slowLogBytes, err := ioutil.ReadAll(f) + slowLogBytes, err := io.ReadAll(f) require.NoError(t, err) require.NotContains(t, string(slowLogBytes), `tidb_decode_binary_plan('')`) } diff --git a/server/plan_replayer.go b/server/plan_replayer.go index 30a00e4eda112..7bfb680a756ba 100644 --- a/server/plan_replayer.go +++ b/server/plan_replayer.go @@ -16,7 +16,7 @@ package server import ( "fmt" - "io/ioutil" + "io" "net/http" "os" "path/filepath" @@ -83,7 +83,7 @@ func handleDownloadFile(handler downloadFileHandler, w http.ResponseWriter, req writeError(w, err) return } - content, err := ioutil.ReadAll(file) + content, err := io.ReadAll(file) if err != nil { writeError(w, err) return @@ -137,7 +137,7 @@ func handleDownloadFile(handler downloadFileHandler, w http.ResponseWriter, req zap.String("remote-addr", remoteAddr), zap.Int("status-code", resp.StatusCode)) continue } - content, err := ioutil.ReadAll(resp.Body) + content, err := io.ReadAll(resp.Body) if err != nil { writeError(w, err) return diff --git a/server/plan_replayer_test.go b/server/plan_replayer_test.go index 2a00bc0db04de..2f2308efdd1a9 100644 --- a/server/plan_replayer_test.go +++ b/server/plan_replayer_test.go @@ -18,7 +18,6 @@ import ( "bytes" "database/sql" "io" - "io/ioutil" "os" "path/filepath" "testing" @@ -69,7 +68,7 @@ func TestDumpPlanReplayerAPI(t *testing.T) { require.NoError(t, resp0.Body.Close()) }() - body, err := ioutil.ReadAll(resp0.Body) + body, err := io.ReadAll(resp0.Body) require.NoError(t, err) path := "/tmp/plan_replayer.zip" diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go index 546916751c32d..f663ab9d726f9 100644 --- a/statistics/handle/dump.go +++ b/statistics/handle/dump.go @@ -19,7 +19,7 @@ import ( "compress/gzip" "encoding/json" "fmt" - "io/ioutil" + "io" "time" "github.com/pingcap/errors" @@ -501,7 +501,7 @@ func BlocksToJSONTable(blocks [][]byte) (*JSONTable, error) { if err := gzipReader.Close(); err != nil { return nil, err } - jsonStr, err := ioutil.ReadAll(gzipReader) + jsonStr, err := io.ReadAll(gzipReader) if err != nil { return nil, errors.Trace(err) } diff --git a/util/cpuprofile/cpuprofile_test.go b/util/cpuprofile/cpuprofile_test.go index 2d400264ede10..b20428dcf21fd 100644 --- a/util/cpuprofile/cpuprofile_test.go +++ b/util/cpuprofile/cpuprofile_test.go @@ -18,7 +18,6 @@ import ( "bytes" "context" "io" - "io/ioutil" "net" "net/http" "runtime/pprof" @@ -237,7 +236,7 @@ func TestProfileHTTPHandler(t *testing.T) { resp, err = http.Get("http://" + address + "/debug/pprof/profile?seconds=100000") require.NoError(t, err) require.Equal(t, 400, resp.StatusCode) - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Equal(t, "profile duration exceeds server's WriteTimeout\n", string(body)) require.NoError(t, resp.Body.Close()) From 3f86a11492f47f12bec770786303ca83edda4168 Mon Sep 17 00:00:00 2001 From: fzzf678 <108643977+fzzf678@users.noreply.github.com> Date: Tue, 13 Dec 2022 11:10:51 +0800 Subject: [PATCH 2/3] planner: add three fields to statement summary table (#39828) ref pingcap/tidb#39199 --- cmd/explaintest/r/index_merge.result | 6 +++--- infoschema/tables.go | 3 +++ infoschema/tables_test.go | 30 ++++++++++++++++++++++++++++ util/stmtsummary/reader.go | 12 +++++++++++ 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/cmd/explaintest/r/index_merge.result b/cmd/explaintest/r/index_merge.result index 0233dbdb55f52..6b44c6122987e 100644 --- a/cmd/explaintest/r/index_merge.result +++ b/cmd/explaintest/r/index_merge.result @@ -455,9 +455,9 @@ c1 c2 c3 ///// MEMORY Table explain select count(c1) from (select /*+ use_index_merge(t_alias), stream_agg() */ count(1) c1 from information_schema.statements_summary where sum_latency >= 0 or max_latency >= 0 order by 1) dt; id estRows task access object operator info -StreamAgg_10 1.00 root funcs:count(Column#93)->Column#94 -└─Sort_11 1.00 root Column#93 - └─StreamAgg_14 1.00 root funcs:count(1)->Column#93 +StreamAgg_10 1.00 root funcs:count(Column#96)->Column#97 +└─Sort_11 1.00 root Column#96 + └─StreamAgg_14 1.00 root funcs:count(1)->Column#96 └─MemTableScan_18 10000.00 root table:STATEMENTS_SUMMARY show warnings; Level Code Message diff --git a/infoschema/tables.go b/infoschema/tables.go index d76d8b8be60b0..2f481d4e004b5 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -1323,6 +1323,9 @@ var tableStatementsSummaryCols = []columnInfo{ {name: stmtsummary.PlanDigestStr, tp: mysql.TypeVarchar, size: 64, comment: "Digest of its execution plan"}, {name: stmtsummary.PlanStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "Sampled execution plan"}, {name: stmtsummary.BinaryPlan, tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "Sampled binary plan"}, + {name: stmtsummary.Charset, tp: mysql.TypeVarchar, size: 64, comment: "Sampled charset"}, + {name: stmtsummary.Collation, tp: mysql.TypeVarchar, size: 64, comment: "Sampled collation"}, + {name: stmtsummary.PlanHint, tp: mysql.TypeVarchar, size: 64, comment: "Sampled plan hint"}, } var tableStorageStatsCols = []columnInfo{ diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index bd6168d035873..ebf3e9b535893 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -1661,3 +1661,33 @@ func TestMemoryUsageAndOpsHistory(t *testing.T) { require.Equal(t, row[10], "e3237ec256015a3566757e0c2742507cd30ae04e4cac2fbc14d269eafe7b067b") // SQL_DIGEST require.Equal(t, row[11], "explain analyze select * from t t1 join t t2 join t t3 on t1.a=t2.a and t1.a=t3.a order by t1.a") // SQL_TEXT } + +func TestAddFieldsForBinding(t *testing.T) { + s := new(clusterTablesSuite) + s.store, s.dom = testkit.CreateMockStoreAndDomain(t) + s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil) + s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer() + s.startTime = time.Now() + defer s.httpServer.Close() + defer s.rpcserver.Stop() + tk := s.newTestKitWithRoot(t) + + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil)) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, key(a))") + tk.MustExec("select /*+ ignore_index(t, a)*/ * from t where a = 1") + planDigest := "4e3159169cc63c14b139a4e7d72eae1759875c9a9581f94bb2079aae961189cb" + rows := tk.MustQuery(fmt.Sprintf("select stmt_type, prepared, sample_user, schema_name, query_sample_text, charset, collation, plan_hint, digest_text "+ + "from information_schema.cluster_statements_summary where plan_digest = '%s'", planDigest)).Rows() + + require.Equal(t, rows[0][0], "Select") + require.Equal(t, rows[0][1], "0") + require.Equal(t, rows[0][2], "root") + require.Equal(t, rows[0][3], "test") + require.Equal(t, rows[0][4], "select /*+ ignore_index(t, a)*/ * from t where a = 1") + require.Equal(t, rows[0][5], "utf8mb4") + require.Equal(t, rows[0][6], "utf8mb4_bin") + require.Equal(t, rows[0][7], "use_index(@`sel_1` `test`.`t` ), ignore_index(`t` `a`)") + require.Equal(t, rows[0][8], "select * from `t` where `a` = ?") +} diff --git a/util/stmtsummary/reader.go b/util/stmtsummary/reader.go index eca48ae4e82eb..e5e9a2db39705 100644 --- a/util/stmtsummary/reader.go +++ b/util/stmtsummary/reader.go @@ -310,6 +310,9 @@ const ( PlanDigestStr = "PLAN_DIGEST" PlanStr = "PLAN" BinaryPlan = "BINARY_PLAN" + Charset = "CHARSET" + Collation = "COLLATION" + PlanHint = "PLAN_HINT" ) type columnValueFactory func(reader *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest) interface{} @@ -620,4 +623,13 @@ var columnValueFactoryMap = map[string]columnValueFactory{ BinaryPlan: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} { return ssElement.sampleBinaryPlan }, + Charset: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} { + return ssElement.charset + }, + Collation: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} { + return ssElement.collation + }, + PlanHint: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} { + return ssElement.planHint + }, } From b73eb4bf4ce3775583963a1815c8940c780e66e4 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Tue, 13 Dec 2022 11:52:51 +0800 Subject: [PATCH 3/3] ddl: fix unexpect fail when create expression index (#39822) close pingcap/tidb#39784 --- ddl/backfilling.go | 5 ++--- ddl/column.go | 6 +++++- ddl/db_change_test.go | 8 ++++++++ ddl/index.go | 6 +++++- ddl/partition.go | 6 +++++- ddl/reorg.go | 7 +++++-- 6 files changed, 30 insertions(+), 8 deletions(-) diff --git a/ddl/backfilling.go b/ddl/backfilling.go index 92d5ee943c647..7c966591016d9 100644 --- a/ddl/backfilling.go +++ b/ddl/backfilling.go @@ -626,8 +626,7 @@ func loadDDLReorgVars(ctx context.Context, sessPool *sessionPool) error { return ddlutil.LoadDDLReorgVars(ctx, sCtx) } -func makeupDecodeColMap(sessCtx sessionctx.Context, t table.Table) (map[int64]decoder.Column, error) { - dbName := model.NewCIStr(sessCtx.GetSessionVars().CurrentDB) +func makeupDecodeColMap(sessCtx sessionctx.Context, dbName model.CIStr, t table.Table) (map[int64]decoder.Column, error) { writableColInfos := make([]*model.ColumnInfo, 0, len(t.WritableCols())) for _, col := range t.WritableCols() { writableColInfos = append(writableColInfos, col.ColumnInfo) @@ -860,7 +859,7 @@ func (dc *ddlCtx) writePhysicalTableRecord(sessPool *sessionPool, t table.Physic startKey, endKey := reorgInfo.StartKey, reorgInfo.EndKey sessCtx := newContext(reorgInfo.d.store) - decodeColMap, err := makeupDecodeColMap(sessCtx, t) + decodeColMap, err := makeupDecodeColMap(sessCtx, reorgInfo.dbInfo.Name, t) if err != nil { return errors.Trace(err) } diff --git a/ddl/column.go b/ddl/column.go index e9c353aacf2f5..d9425ceabac2c 100644 --- a/ddl/column.go +++ b/ddl/column.go @@ -807,7 +807,11 @@ func doReorgWorkForModifyColumn(w *worker, d *ddlCtx, t *meta.Meta, job *model.J oldCol, changingCol *model.ColumnInfo, changingIdxs []*model.IndexInfo) (done bool, ver int64, err error) { job.ReorgMeta.ReorgTp = model.ReorgTypeTxn rh := newReorgHandler(t, w.sess, w.concurrentDDL) - reorgInfo, err := getReorgInfo(d.jobContext(job), d, rh, job, tbl, BuildElements(changingCol, changingIdxs), false) + dbInfo, err := t.GetDatabase(job.SchemaID) + if err != nil { + return false, ver, errors.Trace(err) + } + reorgInfo, err := getReorgInfo(d.jobContext(job), d, rh, job, dbInfo, tbl, BuildElements(changingCol, changingIdxs), false) if err != nil || reorgInfo.first { // If we run reorg firstly, we should update the job snapshot version // and then run the reorg next time. diff --git a/ddl/db_change_test.go b/ddl/db_change_test.go index cf11036a9935e..da49688ccc608 100644 --- a/ddl/db_change_test.go +++ b/ddl/db_change_test.go @@ -1712,6 +1712,14 @@ func TestCreateExpressionIndex(t *testing.T) { require.NoError(t, checkErr) tk.MustExec("admin check table t") tk.MustQuery("select * from t order by a, b").Check(testkit.Rows("0 9", "0 11", "0 11", "1 7", "2 7", "5 7", "8 8", "10 10", "10 10")) + + // https://github.com/pingcap/tidb/issues/39784 + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(name varchar(20))") + tk.MustExec("insert into t values ('Abc'), ('Bcd'), ('abc')") + tk.MustExec("create index idx on test.t((lower(test.t.name)))") + tk.MustExec("admin check table t") } func TestCreateUniqueExpressionIndex(t *testing.T) { diff --git a/ddl/index.go b/ddl/index.go index 8ecc916e7b3f9..0f70b73b61046 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -870,7 +870,11 @@ func runReorgJobAndHandleErr(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, tbl table.Table, indexInfo *model.IndexInfo, mergingTmpIdx bool) (done bool, ver int64, err error) { elements := []*meta.Element{{ID: indexInfo.ID, TypeKey: meta.IndexElementKey}} rh := newReorgHandler(t, w.sess, w.concurrentDDL) - reorgInfo, err := getReorgInfo(d.jobContext(job), d, rh, job, tbl, elements, mergingTmpIdx) + dbInfo, err := t.GetDatabase(job.SchemaID) + if err != nil { + return false, ver, errors.Trace(err) + } + reorgInfo, err := getReorgInfo(d.jobContext(job), d, rh, job, dbInfo, tbl, elements, mergingTmpIdx) if err != nil || reorgInfo.first { // If we run reorg firstly, we should update the job snapshot version // and then run the reorg next time. diff --git a/ddl/partition.go b/ddl/partition.go index cf4bd7aed962f..0a1ea4e6fbe66 100644 --- a/ddl/partition.go +++ b/ddl/partition.go @@ -1743,6 +1743,10 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) ( if err != nil { return ver, errors.Trace(err) } + dbInfo, err := t.GetDatabase(job.SchemaID) + if err != nil { + return ver, errors.Trace(err) + } // If table has global indexes, we need reorg to clean up them. if pt, ok := tbl.(table.PartitionedTable); ok && hasGlobalIndex(tblInfo) { // Build elements for compatible with modify column type. elements will not be used when reorganizing. @@ -1753,7 +1757,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) ( } } rh := newReorgHandler(t, w.sess, w.concurrentDDL) - reorgInfo, err := getReorgInfoFromPartitions(d.jobContext(job), d, rh, job, tbl, physicalTableIDs, elements) + reorgInfo, err := getReorgInfoFromPartitions(d.jobContext(job), d, rh, job, dbInfo, tbl, physicalTableIDs, elements) if err != nil || reorgInfo.first { // If we run reorg firstly, we should update the job snapshot version diff --git a/ddl/reorg.go b/ddl/reorg.go index a03cf417177dc..d7671031f64d1 100644 --- a/ddl/reorg.go +++ b/ddl/reorg.go @@ -386,6 +386,7 @@ type reorgInfo struct { // PhysicalTableID is used to trace the current partition we are handling. // If the table is not partitioned, PhysicalTableID would be TableID. PhysicalTableID int64 + dbInfo *model.DBInfo elements []*meta.Element currElement *meta.Element } @@ -585,7 +586,7 @@ func getValidCurrentVersion(store kv.Storage) (ver kv.Version, err error) { return ver, nil } -func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, +func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, dbInfo *model.DBInfo, tbl table.Table, elements []*meta.Element, mergingTmpIdx bool) (*reorgInfo, error) { var ( element *meta.Element @@ -685,11 +686,12 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, info.currElement = element info.elements = elements info.mergingTmpIdx = mergingTmpIdx + info.dbInfo = dbInfo return &info, nil } -func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, tbl table.Table, partitionIDs []int64, elements []*meta.Element) (*reorgInfo, error) { +func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, dbInfo *model.DBInfo, tbl table.Table, partitionIDs []int64, elements []*meta.Element) (*reorgInfo, error) { var ( element *meta.Element start kv.Key @@ -745,6 +747,7 @@ func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, jo info.PhysicalTableID = pid info.currElement = element info.elements = elements + info.dbInfo = dbInfo return &info, nil }