diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2e69534502f49..abdd9b114b948 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,5 @@ /expression @pingcap/co-expression +/planner @pingcap/co-planner +/statistics @pingcap/co-planner +/util/ranger @pingcap/co-planner +/bindinfo @pingcap/co-planner diff --git a/bindinfo/bind_test.go b/bindinfo/bind_test.go index 2361c0d1a1dc0..c0bb2d6118ddb 100644 --- a/bindinfo/bind_test.go +++ b/bindinfo/bind_test.go @@ -528,3 +528,25 @@ func (s *testSuite) TestAddEvolveTasks(c *C) { status := rows[1][3].(string) c.Assert(status == "using" || status == "rejected", IsTrue) } + +func (s *testSuite) TestBindingCache(c *C) { + tk := testkit.NewTestKit(c, s.store) + s.cleanBindingEnv(tk) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b int, index idx(a))") + tk.MustExec("create global binding for select * from t using select * from t use index(idx)") + tk.MustExec("create database tmp") + tk.MustExec("use tmp") + tk.MustExec("create table t(a int, b int, index idx(a))") + tk.MustExec("create global binding for select * from t using select * from t use index(idx)") + + c.Assert(s.domain.BindHandle().Update(false), IsNil) + c.Assert(s.domain.BindHandle().Update(false), IsNil) + res := tk.MustQuery("show global bindings") + c.Assert(len(res.Rows()), Equals, 2) + + tk.MustExec("drop global binding for select * from t") + c.Assert(s.domain.BindHandle().Update(false), IsNil) + c.Assert(len(s.domain.BindHandle().GetAllBindRecord()), Equals, 1) +} diff --git a/bindinfo/handle.go b/bindinfo/handle.go index 9272a439d0f5b..badd9012a5d6a 100644 --- a/bindinfo/handle.go +++ b/bindinfo/handle.go @@ -126,7 +126,7 @@ func (h *BindHandle) Update(fullLoad bool) (err error) { sql := "select original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation from mysql.bind_info" if !fullLoad { - sql += " where update_time >= \"" + lastUpdateTime.String() + "\"" + sql += " where update_time > \"" + lastUpdateTime.String() + "\"" } // We need to apply the updates by order, wrong apply order of same original sql may cause inconsistent state. sql += " order by update_time" @@ -154,7 +154,7 @@ func (h *BindHandle) Update(fullLoad bool) (err error) { lastUpdateTime = meta.Bindings[0].UpdateTime } if err != nil { - logutil.BgLogger().Error("update bindinfo failed", zap.Error(err)) + logutil.BgLogger().Info("update bindinfo failed", zap.Error(err)) continue } @@ -163,7 +163,7 @@ func (h *BindHandle) Update(fullLoad bool) (err error) { if len(newRecord.Bindings) > 0 { newCache.setBindRecord(hash, newRecord) } else { - newCache.removeDeletedBindRecord(hash, oldRecord) + newCache.removeDeletedBindRecord(hash, newRecord) } updateMetrics(metrics.ScopeGlobal, oldRecord, newCache.getBindRecord(hash, meta.OriginalSQL, meta.Db), true) } @@ -459,6 +459,7 @@ func (c cache) removeDeletedBindRecord(hash string, meta *BindRecord) { } } } + c[hash] = metas } func (c cache) setBindRecord(hash string, meta *BindRecord) { diff --git a/config/config.go b/config/config.go index 5c6a437c4892d..8fbeca484b585 100644 --- a/config/config.go +++ b/config/config.go @@ -29,6 +29,7 @@ import ( "github.com/BurntSushi/toml" "github.com/pingcap/errors" zaplog "github.com/pingcap/log" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/util/logutil" tracing "github.com/uber/jaeger-client-go/config" "go.uber.org/atomic" @@ -38,7 +39,7 @@ import ( const ( MaxLogFileSize = 4096 // MB // DefTxnTotalSizeLimit is the default value of TxnTxnTotalSizeLimit. - DefTxnTotalSizeLimit = 100 * 1024 * 1024 + DefTxnTotalSizeLimit = 1024 * 1024 * 1024 ) // Valid config maps @@ -74,8 +75,8 @@ type Config struct { TxnLocalLatches TxnLocalLatches `toml:"txn-local-latches" json:"txn-local-latches"` // Set sys variable lower-case-table-names, ref: https://dev.mysql.com/doc/refman/5.7/en/identifier-case-sensitivity.html. // TODO: We actually only support mode 2, which keeps the original case, but the comparison is case-insensitive. - LowerCaseTableNames int `toml:"lower-case-table-names" json:"lower-case-table-names"` - + LowerCaseTableNames int `toml:"lower-case-table-names" json:"lower-case-table-names"` + ServerVersion string `toml:"server-version" json:"server-version"` Log Log `toml:"log" json:"log"` Security Security `toml:"security" json:"security"` Status Status `toml:"status" json:"status"` @@ -100,6 +101,9 @@ type Config struct { DelayCleanTableLock uint64 `toml:"delay-clean-table-lock" json:"delay-clean-table-lock"` SplitRegionMaxNum uint64 `toml:"split-region-max-num" json:"split-region-max-num"` StmtSummary StmtSummary `toml:"stmt-summary" json:"stmt-summary"` + // RepairMode indicates that the TiDB is in the repair mode for table meta. + RepairMode bool `toml:"repair-mode" json:"repair-mode"` + RepairTableList []string `toml:"repair-table-list" json:"repair-table-list"` } // nullableBool defaults unset bool options to unset instead of false, which enables us to know if the user has set 2 @@ -440,11 +444,14 @@ var defaultConf = Config{ EnableTableLock: false, DelayCleanTableLock: 0, SplitRegionMaxNum: 1000, + RepairMode: false, + RepairTableList: []string{}, TxnLocalLatches: TxnLocalLatches{ Enabled: false, Capacity: 2048000, }, LowerCaseTableNames: 2, + ServerVersion: "", Log: Log{ Level: "info", Format: "text", @@ -638,6 +645,9 @@ func (c *Config) Load(confFile string) error { if c.TokenLimit == 0 { c.TokenLimit = 1000 } + if len(c.ServerVersion) > 0 { + mysql.ServerVersion = c.ServerVersion + } // If any items in confFile file are not mapped into the Config struct, issue // an error and stop the server from starting. undecoded := metaData.Undecoded() @@ -700,6 +710,10 @@ func (c *Config) Valid() error { if c.TiKVClient.GrpcConnectionCount == 0 { return fmt.Errorf("grpc-connection-count should be greater than 0") } + + if c.Performance.TxnTotalSizeLimit > (10 << 30) { + return fmt.Errorf("txn-total-size-limit should be less than %d", 10<<30) + } return nil } diff --git a/config/config.toml.example b/config/config.toml.example index b1c2d77799fa0..f17a5aaa1509f 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -74,6 +74,19 @@ split-region-max-num = 1000 # In order to support "drop primary key" operation , this flag must be true and the table does not have the pkIsHandle flag. alter-primary-key = false +# server-version is used to change the version string of TiDB in the following scenarios: +# 1. the server version returned by builtin-function `VERSION()`. +# 2. the server version filled in handshake packets of MySQL Connection Protocol, see https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake for more details. +# if server-version = "", the default value(original TiDB version string) is used. +server-version = "" + +# repair mode is used to repair the broken table meta in TiKV in extreme cases. +repair-mode = false + +# Repair table list is used to list the tables in repair mode with the format like ["db.table",]. +# In repair mode, repairing table which is not in repair list will get wrong database or wrong table error. +repair-table-list = [] + [log] # Log level: debug, info, warn, error, fatal. level = "info" @@ -201,7 +214,7 @@ bind-info-lease = "3s" # If using TiKV as the storage, the entry represents a key/value pair. # WARNING: Do not set the value too large, otherwise it will make a very large impact on the TiKV cluster. # Please adjust this configuration carefully. -txn-total-size-limit = 104857600 +txn-total-size-limit = 1073741824 [proxy-protocol] # PROXY protocol acceptable client networks. diff --git a/config/config_test.go b/config/config_test.go index f8d3a3dea03f2..b6e7afb7410e2 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -23,6 +23,7 @@ import ( "github.com/BurntSushi/toml" . "github.com/pingcap/check" zaplog "github.com/pingcap/log" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/util/logutil" tracing "github.com/uber/jaeger-client-go/config" ) @@ -179,6 +180,8 @@ alter-primary-key = true delay-clean-table-lock = 5 split-region-max-num=10000 enable-batch-dml = true +server-version = "test_version" +repair-mode = true [performance] txn-total-size-limit=2000 [tikv-client] @@ -196,6 +199,8 @@ max-sql-length=1024 c.Assert(conf.Load(configFile), IsNil) + c.Assert(conf.ServerVersion, Equals, "test_version") + c.Assert(mysql.ServerVersion, Equals, conf.ServerVersion) // Test that the original value will not be clear by load the config file that does not contain the option. c.Assert(conf.Binlog.Enable, Equals, true) c.Assert(conf.Binlog.Strategy, Equals, "hash") @@ -215,6 +220,7 @@ max-sql-length=1024 c.Assert(conf.StmtSummary.MaxStmtCount, Equals, uint(1000)) c.Assert(conf.StmtSummary.MaxSQLLength, Equals, uint(1024)) c.Assert(conf.EnableBatchDML, Equals, true) + c.Assert(conf.RepairMode, Equals, true) c.Assert(f.Close(), IsNil) c.Assert(os.Remove(configFile), IsNil) @@ -224,7 +230,7 @@ max-sql-length=1024 // Make sure the example config is the same as default config. c.Assert(conf, DeepEquals, GetGlobalConfig()) - // Test for lof config. + // Test for log config. c.Assert(conf.Log.ToLogConfig(), DeepEquals, logutil.NewLogConfig("info", "text", "tidb-slow.log", conf.Log.File, false, func(config *zaplog.Config) { config.DisableErrorVerbose = conf.Log.getDisableErrorStack() })) // Test for tracing config. @@ -349,3 +355,20 @@ func (s *testConfigSuite) TestOOMActionValid(c *C) { c.Assert(c1.Valid() == nil, Equals, tt.valid) } } + +func (s *testConfigSuite) TestTxnTotalSizeLimitValid(c *C) { + conf := NewConfig() + tests := []struct { + limit uint64 + valid bool + }{ + {4 << 10, true}, + {10 << 30, true}, + {10<<30 + 1, false}, + } + + for _, tt := range tests { + conf.Performance.TxnTotalSizeLimit = tt.limit + c.Assert(conf.Valid() == nil, Equals, tt.valid) + } +} diff --git a/ddl/column_test.go b/ddl/column_test.go index 7e1b9641dbbe4..16f682543950e 100644 --- a/ddl/column_test.go +++ b/ddl/column_test.go @@ -800,7 +800,7 @@ func (s *testColumnSuite) TestAddColumn(c *C) { hookErr = errors.Trace(err1) return } - newCol := table.FindCol(t.(*tables.Table).Columns, newColName) + newCol := table.FindCol(t.(*tables.TableCommon).Columns, newColName) if newCol == nil { return } @@ -891,7 +891,7 @@ func (s *testColumnSuite) TestDropColumn(c *C) { hookErr = errors.Trace(err1) return } - col := table.FindCol(t.(*tables.Table).Columns, colName) + col := table.FindCol(t.(*tables.TableCommon).Columns, colName) if col == nil { checkOK = true return diff --git a/ddl/db_change_test.go b/ddl/db_change_test.go index 35d473642c30d..c0e25e59b1a04 100644 --- a/ddl/db_change_test.go +++ b/ddl/db_change_test.go @@ -23,6 +23,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" @@ -37,14 +38,24 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/admin" + "github.com/pingcap/tidb/util/gcutil" "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tidb/util/testkit" "go.uber.org/zap" ) var _ = Suite(&testStateChangeSuite{}) +var _ = SerialSuites(&serialTestStateChangeSuite{}) + +type serialTestStateChangeSuite struct { + testStateChangeSuiteBase +} type testStateChangeSuite struct { + testStateChangeSuiteBase +} + +type testStateChangeSuiteBase struct { lease time.Duration store kv.Storage dom *domain.Domain @@ -53,7 +64,7 @@ type testStateChangeSuite struct { preSQL string } -func (s *testStateChangeSuite) SetUpSuite(c *C) { +func (s *testStateChangeSuiteBase) SetUpSuite(c *C) { s.lease = 200 * time.Millisecond ddl.WaitTimeWhenErrorOccured = 1 * time.Microsecond var err error @@ -71,7 +82,7 @@ func (s *testStateChangeSuite) SetUpSuite(c *C) { s.p = parser.New() } -func (s *testStateChangeSuite) TearDownSuite(c *C) { +func (s *testStateChangeSuiteBase) TearDownSuite(c *C) { s.se.Execute(context.Background(), "drop database if exists test_db_state") s.se.Close() s.dom.Close() @@ -534,7 +545,7 @@ func (s *testStateChangeSuite) TestDeleteOnly(c *C) { s.runTestInSchemaState(c, model.StateDeleteOnly, "", dropColumnSQL, sqls, nil) } -func (s *testStateChangeSuite) runTestInSchemaState(c *C, state model.SchemaState, tableName, alterTableSQL string, +func (s *testStateChangeSuiteBase) runTestInSchemaState(c *C, state model.SchemaState, tableName, alterTableSQL string, sqlWithErrs []sqlWithErr, expectQuery *expectQuery) { _, err := s.se.Execute(context.Background(), `create table t ( c1 varchar(64), @@ -592,7 +603,7 @@ func (s *testStateChangeSuite) runTestInSchemaState(c *C, state model.SchemaStat } } -func (s *testStateChangeSuite) execQuery(tk *testkit.TestKit, sql string, args ...interface{}) (*testkit.Result, error) { +func (s *testStateChangeSuiteBase) execQuery(tk *testkit.TestKit, sql string, args ...interface{}) (*testkit.Result, error) { comment := Commentf("sql:%s, args:%v", sql, args) rs, err := tk.Exec(sql, args...) if err != nil { @@ -611,7 +622,7 @@ func checkResult(result *testkit.Result, expected [][]interface{}) error { return nil } -func (s *testStateChangeSuite) CheckResult(tk *testkit.TestKit, sql string, args ...interface{}) (*testkit.Result, error) { +func (s *testStateChangeSuiteBase) CheckResult(tk *testkit.TestKit, sql string, args ...interface{}) (*testkit.Result, error) { comment := Commentf("sql:%s, args:%v", sql, args) rs, err := tk.Exec(sql, args...) if err != nil { @@ -829,7 +840,7 @@ func (s *testStateChangeSuite) TestParallelCreateAndRename(c *C) { type checkRet func(c *C, err1, err2 error) -func (s *testStateChangeSuite) testControlParallelExecSQL(c *C, sql1, sql2 string, f checkRet) { +func (s *testStateChangeSuiteBase) testControlParallelExecSQL(c *C, sql1, sql2 string, f checkRet) { _, err := s.se.Execute(context.Background(), "use test_db_state") c.Assert(err, IsNil) _, err = s.se.Execute(context.Background(), "create table t(a int, b int, c int, d int auto_increment,e int, index idx1(d), index idx2(d,e))") @@ -1136,3 +1147,60 @@ func (s *testStateChangeSuite) TestParallelTruncateTableAndAddColumn(c *C) { } s.testControlParallelExecSQL(c, sql1, sql2, f) } + +// TestParallelFlashbackTable tests parallel flashback table. +func (s *serialTestStateChangeSuite) TestParallelFlashbackTable(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) + defer func(originGC bool) { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil) + if originGC { + ddl.EmulatorGCEnable() + } else { + ddl.EmulatorGCDisable() + } + }(ddl.IsEmulatorGCEnable()) + + // disable emulator GC. + // Disable emulator GC, otherwise, emulator GC will delete table record as soon as possible after executing drop table DDL. + ddl.EmulatorGCDisable() + gcTimeFormat := "20060102-15:04:05 -0700 MST" + timeBeforeDrop := time.Now().Add(0 - time.Duration(48*60*60*time.Second)).Format(gcTimeFormat) + safePointSQL := `INSERT HIGH_PRIORITY INTO mysql.tidb VALUES ('tikv_gc_safe_point', '%[1]s', '') + ON DUPLICATE KEY + UPDATE variable_value = '%[1]s'` + tk := testkit.NewTestKit(c, s.store) + // clear GC variables first. + tk.MustExec("delete from mysql.tidb where variable_name in ( 'tikv_gc_safe_point','tikv_gc_enable' )") + // set GC safe point + tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop)) + // set GC enable. + err := gcutil.EnableGC(tk.Se) + c.Assert(err, IsNil) + + // prepare dropped table. + tk.MustExec("use test_db_state") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int);") + tk.MustExec("drop table if exists t") + // Test parallel flashback table. + ts := getDDLJobStartTime(tk, "test_db_state", "t") + sql1 := fmt.Sprintf("flashback table t until timestamp '%v' to t_flashback", ts) + f := func(c *C, err1, err2 error) { + c.Assert(err1, IsNil) + c.Assert(err2, NotNil) + c.Assert(err2.Error(), Equals, "[schema:1050]Table 't_flashback' already exists") + + } + s.testControlParallelExecSQL(c, sql1, sql1, f) +} + +func getDDLJobStartTime(tk *testkit.TestKit, dbName, tblName string) string { + re := tk.MustQuery("admin show ddl jobs 100") + rows := re.Rows() + for _, row := range rows { + if row[1] == dbName && row[2] == tblName && (row[3] == "drop table" || row[3] == "truncate table") { + return row[8].(string) + } + } + return "" +} diff --git a/ddl/db_test.go b/ddl/db_test.go index 5ab37e81fd952..52ed09b91cec6 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -47,6 +47,7 @@ import ( "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/admin" + "github.com/pingcap/tidb/util/domainutil" "github.com/pingcap/tidb/util/israce" "github.com/pingcap/tidb/util/mock" "github.com/pingcap/tidb/util/testkit" @@ -2003,6 +2004,241 @@ func (s *testDBSuite1) TestCreateTable(c *C) { c.Assert(err.Error(), Equals, "[types:1291]Column 'a' has duplicated value 'B' in ENUM") } +func (s *testDBSuite5) TestRepairTable(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/infoschema/repairFetchCreateTable", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/infoschema/repairFetchCreateTable"), IsNil) + }() + s.tk = testkit.NewTestKit(c, s.store) + s.tk.MustExec("use test") + s.tk.MustExec("drop table if exists t, other_table, origin") + + // Test repair table when TiDB is not in repair mode. + s.tk.MustExec("CREATE TABLE t (a int primary key, b varchar(10));") + _, err := s.tk.Exec("admin repair table t CREATE TABLE t (a float primary key, b varchar(5));") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: TiDB is not in REPAIR MODE") + + // Test repair table when the repaired list is empty. + domainutil.RepairInfo.SetRepairMode(true) + _, err = s.tk.Exec("admin repair table t CREATE TABLE t (a float primary key, b varchar(5));") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: repair list is empty") + + // Test repair table when it's database isn't in repairInfo. + domainutil.RepairInfo.SetRepairTableList([]string{"test.other_table"}) + _, err = s.tk.Exec("admin repair table t CREATE TABLE t (a float primary key, b varchar(5));") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: database test is not in repair") + + // Test repair table when the table isn't in repairInfo. + s.tk.MustExec("CREATE TABLE other_table (a int, b varchar(1), key using hash(b));") + _, err = s.tk.Exec("admin repair table t CREATE TABLE t (a float primary key, b varchar(5));") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: table t is not in repair") + + // Test user can't access to the repaired table. + _, err = s.tk.Exec("select * from other_table") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[schema:1146]Table 'test.other_table' doesn't exist") + + // Test create statement use the same name with what is in repaired. + _, err = s.tk.Exec("CREATE TABLE other_table (a int);") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:1103]Incorrect table name 'other_table'%!(EXTRA string=this table is in repair)") + + // Test column lost in repair table. + _, err = s.tk.Exec("admin repair table other_table CREATE TABLE other_table (a int, c char(1));") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: Column c has lost") + + // Test column type should be the same. + _, err = s.tk.Exec("admin repair table other_table CREATE TABLE other_table (a bigint, b varchar(1), key using hash(b));") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: Column a type should be the same") + + // Test index lost in repair table. + _, err = s.tk.Exec("admin repair table other_table CREATE TABLE other_table (a int unique);") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: Index a has lost") + + // Test index type should be the same. + _, err = s.tk.Exec("admin repair table other_table CREATE TABLE other_table (a int, b varchar(2) unique)") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: Index b type should be the same") + + // Test sub create statement in repair statement with the same name. + _, err = s.tk.Exec("admin repair table other_table CREATE TABLE other_table (a int);") + c.Assert(err, IsNil) + + // Test whether repair table name is case sensitive. + domainutil.RepairInfo.SetRepairMode(true) + domainutil.RepairInfo.SetRepairTableList([]string{"test.other_table2"}) + s.tk.MustExec("CREATE TABLE otHer_tAblE2 (a int, b varchar(1));") + _, err = s.tk.Exec("admin repair table otHer_tAblE2 CREATE TABLE otHeR_tAbLe (a int, b varchar(2));") + c.Assert(err, IsNil) + repairTable := testGetTableByName(c, s.s, "test", "otHeR_tAbLe") + c.Assert(repairTable.Meta().Name.O, Equals, "otHeR_tAbLe") + + // Test memory and system database is not for repair. + domainutil.RepairInfo.SetRepairMode(true) + domainutil.RepairInfo.SetRepairTableList([]string{"test.xxx"}) + _, err = s.tk.Exec("admin repair table performance_schema.xxx CREATE TABLE yyy (a int);") + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: memory or system database is not for repair") + + // Test the repair detail. + turnRepairModeAndInit(true) + defer turnRepairModeAndInit(false) + // Domain reload the tableInfo and add it into repairInfo. + s.tk.MustExec("CREATE TABLE origin (a int primary key, b varchar(10), c int auto_increment);") + // Repaired tableInfo has been filtered by `domain.InfoSchema()`, so get it in repairInfo. + originTableInfo, _ := domainutil.RepairInfo.GetRepairedTableInfoByTableName("test", "origin") + + hook := &ddl.TestDDLCallback{} + var repairErr error + hook.OnJobRunBeforeExported = func(job *model.Job) { + if job.Type != model.ActionRepairTable { + return + } + if job.TableID != originTableInfo.ID { + repairErr = errors.New("table id should be the same") + return + } + if job.SchemaState != model.StateNone { + repairErr = errors.New("repair job state should be the none") + return + } + // Test whether it's readable, when repaired table is still stateNone. + tkInternal := testkit.NewTestKitWithInit(c, s.store) + _, repairErr = tkInternal.Exec("select * from origin") + // Repaired tableInfo has been filtered by `domain.InfoSchema()`, here will get an error cause user can't get access to it. + if repairErr != nil && terror.ErrorEqual(repairErr, infoschema.ErrTableNotExists) { + repairErr = nil + } + } + originalHook := s.dom.DDL().GetHook() + defer s.dom.DDL().(ddl.DDLForTest).SetHook(originalHook) + s.dom.DDL().(ddl.DDLForTest).SetHook(hook) + + // Exec the repair statement to override the tableInfo. + s.tk.MustExec("admin repair table origin CREATE TABLE origin (a int primary key, b varchar(5), c int auto_increment);") + c.Assert(repairErr, IsNil) + + // Check the repaired tableInfo is exactly the same with old one in tableID, indexID, colID. + // testGetTableByName will extract the Table from `domain.InfoSchema()` directly. + repairTable = testGetTableByName(c, s.s, "test", "origin") + c.Assert(repairTable.Meta().ID, Equals, originTableInfo.ID) + c.Assert(len(repairTable.Meta().Columns), Equals, 3) + c.Assert(repairTable.Meta().Columns[0].ID, Equals, originTableInfo.Columns[0].ID) + c.Assert(repairTable.Meta().Columns[1].ID, Equals, originTableInfo.Columns[1].ID) + c.Assert(repairTable.Meta().Columns[2].ID, Equals, originTableInfo.Columns[2].ID) + c.Assert(len(repairTable.Meta().Indices), Equals, 1) + c.Assert(repairTable.Meta().Indices[0].ID, Equals, originTableInfo.Columns[0].ID) + c.Assert(repairTable.Meta().AutoIncID, Equals, originTableInfo.AutoIncID) + + c.Assert(repairTable.Meta().Columns[0].Tp, Equals, mysql.TypeLong) + c.Assert(repairTable.Meta().Columns[1].Tp, Equals, mysql.TypeVarchar) + c.Assert(repairTable.Meta().Columns[1].Flen, Equals, 5) + c.Assert(repairTable.Meta().Columns[2].Tp, Equals, mysql.TypeLong) + + // Exec the show create table statement to make sure new tableInfo has been set. + result := s.tk.MustQuery("show create table origin") + c.Assert(result.Rows()[0][1], Equals, "CREATE TABLE `origin` (\n `a` int(11) NOT NULL,\n `b` varchar(5) DEFAULT NULL,\n `c` int(11) NOT NULL AUTO_INCREMENT,\n PRIMARY KEY (`a`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin") + +} + +func turnRepairModeAndInit(on bool) { + list := make([]string, 0, 0) + if on { + list = append(list, "test.origin") + } + domainutil.RepairInfo.SetRepairMode(on) + domainutil.RepairInfo.SetRepairTableList(list) +} + +func (s *testDBSuite5) TestRepairTableWithPartition(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/infoschema/repairFetchCreateTable", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/infoschema/repairFetchCreateTable"), IsNil) + }() + s.tk = testkit.NewTestKit(c, s.store) + s.tk.MustExec("use test") + s.tk.MustExec("drop table if exists origin") + + turnRepairModeAndInit(true) + defer turnRepairModeAndInit(false) + // Domain reload the tableInfo and add it into repairInfo. + s.tk.MustExec("create table origin (a int not null) partition by RANGE(a) (" + + "partition p10 values less than (10)," + + "partition p30 values less than (30)," + + "partition p50 values less than (50)," + + "partition p70 values less than (70)," + + "partition p90 values less than (90));") + // Test for some old partition has lost. + _, err := s.tk.Exec("admin repair table origin create table origin (a int not null) partition by RANGE(a) (" + + "partition p10 values less than (10)," + + "partition p30 values less than (30)," + + "partition p50 values less than (50)," + + "partition p90 values less than (90)," + + "partition p100 values less than (100));") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: Partition p100 has lost") + + // Test for some partition changed the condition. + _, err = s.tk.Exec("admin repair table origin create table origin (a int not null) partition by RANGE(a) (" + + "partition p10 values less than (10)," + + "partition p20 values less than (25)," + + "partition p50 values less than (50)," + + "partition p90 values less than (90));") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: Partition p20 has lost") + + // Test for some partition changed the partition name. + _, err = s.tk.Exec("admin repair table origin create table origin (a int not null) partition by RANGE(a) (" + + "partition p10 values less than (10)," + + "partition p30 values less than (30)," + + "partition pNew values less than (50)," + + "partition p90 values less than (90));") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: Partition pnew has lost") + + originTableInfo, _ := domainutil.RepairInfo.GetRepairedTableInfoByTableName("test", "origin") + s.tk.MustExec("admin repair table origin create table origin_rename (a int not null) partition by RANGE(a) (" + + "partition p10 values less than (10)," + + "partition p30 values less than (30)," + + "partition p50 values less than (50)," + + "partition p90 values less than (90));") + repairTable := testGetTableByName(c, s.s, "test", "origin_rename") + c.Assert(repairTable.Meta().ID, Equals, originTableInfo.ID) + c.Assert(len(repairTable.Meta().Columns), Equals, 1) + c.Assert(repairTable.Meta().Columns[0].ID, Equals, originTableInfo.Columns[0].ID) + c.Assert(len(repairTable.Meta().Partition.Definitions), Equals, 4) + c.Assert(repairTable.Meta().Partition.Definitions[0].ID, Equals, originTableInfo.Partition.Definitions[0].ID) + c.Assert(repairTable.Meta().Partition.Definitions[1].ID, Equals, originTableInfo.Partition.Definitions[1].ID) + c.Assert(repairTable.Meta().Partition.Definitions[2].ID, Equals, originTableInfo.Partition.Definitions[2].ID) + c.Assert(repairTable.Meta().Partition.Definitions[3].ID, Equals, originTableInfo.Partition.Definitions[4].ID) + + // Test hash partition. + s.tk.MustExec("drop table if exists origin") + domainutil.RepairInfo.SetRepairMode(true) + domainutil.RepairInfo.SetRepairTableList([]string{"test.origin"}) + s.tk.MustExec("create table origin (a varchar(1), b int not null, c int, key idx(c)) partition by hash(b) partitions 30") + + // Test partition num in repair should be exactly same with old one, other wise will cause partition semantic problem. + _, err = s.tk.Exec("admin repair table origin create table origin (a varchar(2), b int not null, c int, key idx(c)) partition by hash(b) partitions 20") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:8215]Failed to repair table: Hash partition num should be the same") + + originTableInfo, _ = domainutil.RepairInfo.GetRepairedTableInfoByTableName("test", "origin") + s.tk.MustExec("admin repair table origin create table origin (a varchar(3), b int not null, c int, key idx(c)) partition by hash(b) partitions 30") + repairTable = testGetTableByName(c, s.s, "test", "origin") + c.Assert(repairTable.Meta().ID, Equals, originTableInfo.ID) + c.Assert(len(repairTable.Meta().Partition.Definitions), Equals, 30) + c.Assert(repairTable.Meta().Partition.Definitions[0].ID, Equals, originTableInfo.Partition.Definitions[0].ID) + c.Assert(repairTable.Meta().Partition.Definitions[1].ID, Equals, originTableInfo.Partition.Definitions[1].ID) + c.Assert(repairTable.Meta().Partition.Definitions[29].ID, Equals, originTableInfo.Partition.Definitions[29].ID) +} + func (s *testDBSuite2) TestCreateTableWithSetCol(c *C) { s.tk = testkit.NewTestKitWithInit(c, s.store) s.tk.MustExec("create table t_set (a int, b set('e') default '');") diff --git a/ddl/ddl.go b/ddl/ddl.go index b8b657974c9b1..fcacfbb6b1750 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -84,6 +84,8 @@ var ( errRunMultiSchemaChanges = terror.ClassDDL.New(mysql.ErrUnsupportedDDLOperation, fmt.Sprintf(mysql.MySQLErrName[mysql.ErrUnsupportedDDLOperation], "multi schema change")) errWaitReorgTimeout = terror.ClassDDL.New(mysql.ErrLockWaitTimeout, mysql.MySQLErrName[mysql.ErrWaitReorgTimeout]) errInvalidStoreVer = terror.ClassDDL.New(mysql.ErrInvalidStoreVersion, mysql.MySQLErrName[mysql.ErrInvalidStoreVersion]) + // ErrRepairTableFail is used to repair tableInfo in repair mode. + ErrRepairTableFail = terror.ClassDDL.New(mysql.ErrRepairTable, mysql.MySQLErrName[mysql.ErrRepairTable]) // We don't support dropping column with index covered now. errCantDropColWithIndex = terror.ClassDDL.New(mysql.ErrUnsupportedDDLOperation, fmt.Sprintf(mysql.MySQLErrName[mysql.ErrUnsupportedDDLOperation], "drop column with index")) @@ -244,6 +246,7 @@ type DDL interface { UnlockTables(ctx sessionctx.Context, lockedTables []model.TableLockTpInfo) error CleanupTableLock(ctx sessionctx.Context, tables []*ast.TableName) error UpdateTableReplicaInfo(ctx sessionctx.Context, tid int64, available bool) error + RepairTable(ctx sessionctx.Context, table *ast.TableName, createStmt *ast.CreateTableStmt) error // GetLease returns current schema lease time. GetLease() time.Duration diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 546d20524f277..db16bcb555ead 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -47,6 +47,7 @@ import ( "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/domainutil" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/mock" "github.com/pingcap/tidb/util/set" @@ -3931,3 +3932,80 @@ func extractCollateFromOption(def *ast.ColumnDef) []string { } return specifiedCollates } + +func (d *ddl) RepairTable(ctx sessionctx.Context, table *ast.TableName, createStmt *ast.CreateTableStmt) error { + // Existence of DB and table has been checked in the preprocessor. + oldTableInfo, ok := (ctx.Value(domainutil.RepairedTable)).(*model.TableInfo) + if !ok || oldTableInfo == nil { + return ErrRepairTableFail.GenWithStack("Failed to get the repaired table") + } + oldDBInfo, ok := (ctx.Value(domainutil.RepairedDatabase)).(*model.DBInfo) + if !ok || oldDBInfo == nil { + return ErrRepairTableFail.GenWithStack("Failed to get the repaired database") + } + // By now only support same DB repair. + if createStmt.Table.Schema.L != oldDBInfo.Name.L { + return ErrRepairTableFail.GenWithStack("Repaired table should in same database with the old one") + } + // It is necessary to specify the table.ID and partition.ID manually. + newTableInfo, err := buildTableInfoWithCheck(ctx, d, createStmt, oldTableInfo.Charset, oldTableInfo.Collate) + if err != nil { + return errors.Trace(err) + } + + // Override newTableInfo with oldTableInfo's element necessary. + // TODO: There may be more element assignments here, and the new TableInfo should be verified with the actual data. + newTableInfo.ID = oldTableInfo.ID + if err = checkAndOverridePartitionID(newTableInfo, oldTableInfo); err != nil { + return err + } + newTableInfo.AutoIncID = oldTableInfo.AutoIncID + // If any old columnInfo has lost, that means the old column ID lost too, repair failed. + for i, newOne := range newTableInfo.Columns { + old := getColumnInfoByName(oldTableInfo, newOne.Name.L) + if old == nil { + return ErrRepairTableFail.GenWithStackByArgs("Column " + newOne.Name.L + " has lost") + } + if newOne.Tp != old.Tp { + return ErrRepairTableFail.GenWithStackByArgs("Column " + newOne.Name.L + " type should be the same") + } + if newOne.Flen != old.Flen { + logutil.BgLogger().Warn("[ddl] admin repair table : Column " + newOne.Name.L + " flen is not equal to the old one") + } + newTableInfo.Columns[i].ID = old.ID + } + // If any old indexInfo has lost, that means the index ID lost too, so did the data, repair failed. + for i, newOne := range newTableInfo.Indices { + old := getIndexInfoByNameAndColumn(oldTableInfo, newOne) + if old == nil { + return ErrRepairTableFail.GenWithStackByArgs("Index " + newOne.Name.L + " has lost") + } + if newOne.Tp != old.Tp { + return ErrRepairTableFail.GenWithStackByArgs("Index " + newOne.Name.L + " type should be the same") + } + newTableInfo.Indices[i].ID = old.ID + } + + newTableInfo.State = model.StatePublic + err = checkTableInfoValid(newTableInfo) + if err != nil { + return err + } + newTableInfo.State = model.StateNone + + job := &model.Job{ + SchemaID: oldDBInfo.ID, + TableID: newTableInfo.ID, + SchemaName: oldDBInfo.Name.L, + Type: model.ActionRepairTable, + BinlogInfo: &model.HistoryInfo{}, + Args: []interface{}{newTableInfo}, + } + err = d.doDDLJob(ctx, job) + if err == nil { + // Remove the old TableInfo from repairInfo before domain reload. + domainutil.RepairInfo.RemoveFromRepairInfo(oldDBInfo.Name.L, oldTableInfo.Name.L) + } + err = d.callHookOnChanged(err) + return errors.Trace(err) +} diff --git a/ddl/ddl_worker.go b/ddl/ddl_worker.go index 2cfd37b2c4832..54d7ff7d4ed30 100644 --- a/ddl/ddl_worker.go +++ b/ddl/ddl_worker.go @@ -527,6 +527,8 @@ func (w *worker) runDDLJob(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, ver, err = onDropSchema(t, job) case model.ActionCreateTable: ver, err = onCreateTable(d, t, job) + case model.ActionRepairTable: + ver, err = onRepairTable(d, t, job) case model.ActionCreateView: ver, err = onCreateView(d, t, job) case model.ActionDropTable, model.ActionDropView: diff --git a/ddl/delete_range.go b/ddl/delete_range.go index e053619bf9f27..81179a7c2606f 100644 --- a/ddl/delete_range.go +++ b/ddl/delete_range.go @@ -49,7 +49,7 @@ type delRangeManager interface { addDelRangeJob(job *model.Job) error // removeFromGCDeleteRange removes the deleting table job from gc_delete_range table by jobID and tableID. // It's use for recover the table that was mistakenly deleted. - removeFromGCDeleteRange(jobID, tableID int64) error + removeFromGCDeleteRange(jobID int64, tableID []int64) error start() clear() } @@ -100,13 +100,13 @@ func (dr *delRange) addDelRangeJob(job *model.Job) error { } // removeFromGCDeleteRange implements delRangeManager interface. -func (dr *delRange) removeFromGCDeleteRange(jobID, tableID int64) error { +func (dr *delRange) removeFromGCDeleteRange(jobID int64, tableIDs []int64) error { ctx, err := dr.sessPool.get() if err != nil { return errors.Trace(err) } defer dr.sessPool.put(ctx) - err = util.RemoveFromGCDeleteRange(ctx, jobID, tableID) + err = util.RemoveMultiFromGCDeleteRange(ctx, jobID, tableIDs) return errors.Trace(err) } diff --git a/ddl/index.go b/ddl/index.go index dcf23e93a67ac..a030e428e8fd9 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -1518,3 +1518,30 @@ func iterateSnapshotRows(store kv.Storage, priority int, t table.Table, version return nil } + +func getIndexInfoByNameAndColumn(oldTableInfo *model.TableInfo, newOne *model.IndexInfo) *model.IndexInfo { + for _, oldOne := range oldTableInfo.Indices { + if newOne.Name.L == oldOne.Name.L && indexColumnSliceEqual(newOne.Columns, oldOne.Columns) { + return oldOne + } + } + return nil +} + +func indexColumnSliceEqual(a, b []*model.IndexColumn) bool { + if len(a) != len(b) { + return false + } + if len(a) == 0 { + logutil.BgLogger().Warn("[ddl] admin repair table : index's columns length equal to 0") + return true + } + // Accelerate the compare by eliminate index bound check. + b = b[:len(a)] + for i, v := range a { + if v.Name.L != b[i].Name.L { + return false + } + } + return true +} diff --git a/ddl/mock.go b/ddl/mock.go index 32c244618ec30..33f420d510fea 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -137,7 +137,7 @@ func (dr *mockDelRange) addDelRangeJob(job *model.Job) error { } // removeFromGCDeleteRange implements delRangeManager interface. -func (dr *mockDelRange) removeFromGCDeleteRange(jobID, tableID int64) error { +func (dr *mockDelRange) removeFromGCDeleteRange(jobID int64, tableIDs []int64) error { return nil } diff --git a/ddl/partition.go b/ddl/partition.go index 2a5bc0806c119..75241571f637f 100644 --- a/ddl/partition.go +++ b/ddl/partition.go @@ -138,7 +138,7 @@ func buildHashPartitionDefinitions(ctx sessionctx.Context, d *ddl, s *ast.Create func buildRangePartitionDefinitions(ctx sessionctx.Context, d *ddl, s *ast.CreateTableStmt, pi *model.PartitionInfo) error { genIDs, err := d.genGlobalIDs(len(s.Partition.Definitions)) if err != nil { - return err + return errors.Trace(err) } for ith, def := range s.Partition.Definitions { comment, _ := def.Comment() @@ -190,6 +190,56 @@ func checkAddPartitionNameUnique(tbInfo *model.TableInfo, pi *model.PartitionInf return nil } +func checkAndOverridePartitionID(newTableInfo, oldTableInfo *model.TableInfo) error { + // If any old partitionInfo has lost, that means the partition ID lost too, so did the data, repair failed. + if newTableInfo.Partition == nil { + return nil + } + if oldTableInfo.Partition == nil { + return ErrRepairTableFail.GenWithStackByArgs("Old table doesn't have partitions") + } + if newTableInfo.Partition.Type != oldTableInfo.Partition.Type { + return ErrRepairTableFail.GenWithStackByArgs("Partition type should be the same") + } + // Check whether partitionType is hash partition. + if newTableInfo.Partition.Type == model.PartitionTypeHash { + if newTableInfo.Partition.Num != oldTableInfo.Partition.Num { + return ErrRepairTableFail.GenWithStackByArgs("Hash partition num should be the same") + } + } + for i, newOne := range newTableInfo.Partition.Definitions { + found := false + for _, oldOne := range oldTableInfo.Partition.Definitions { + if newOne.Name.L == oldOne.Name.L && stringSliceEqual(newOne.LessThan, oldOne.LessThan) { + newTableInfo.Partition.Definitions[i].ID = oldOne.ID + found = true + break + } + } + if !found { + return ErrRepairTableFail.GenWithStackByArgs("Partition " + newOne.Name.L + " has lost") + } + } + return nil +} + +func stringSliceEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + if len(a) == 0 { + return true + } + // Accelerate the compare by eliminate index bound check. + b = b[:len(a)] + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} + // See https://github.com/mysql/mysql-server/blob/5.7/sql/item_func.h#L387 func hasTimestampField(ctx sessionctx.Context, tblInfo *model.TableInfo, expr ast.ExprNode) (bool, error) { partCols, err := checkPartitionColumns(tblInfo, expr) diff --git a/ddl/reorg.go b/ddl/reorg.go index 42d2dc2c2bffa..16d92b609b1a6 100644 --- a/ddl/reorg.go +++ b/ddl/reorg.go @@ -220,9 +220,8 @@ func constructLimitPB(count uint64) *tipb.Executor { return &tipb.Executor{Tp: tipb.ExecType_TypeLimit, Limit: limitExec} } -func buildDescTableScanDAG(ctx sessionctx.Context, startTS uint64, tbl table.PhysicalTable, columns []*model.ColumnInfo, limit uint64) (*tipb.DAGRequest, error) { +func buildDescTableScanDAG(ctx sessionctx.Context, tbl table.PhysicalTable, columns []*model.ColumnInfo, limit uint64) (*tipb.DAGRequest, error) { dagReq := &tipb.DAGRequest{} - dagReq.StartTs = startTS _, timeZoneOffset := time.Now().In(time.UTC).Zone() dagReq.TimeZoneOffset = int64(timeZoneOffset) for i := range columns { @@ -249,7 +248,7 @@ func getColumnsTypes(columns []*model.ColumnInfo) []*types.FieldType { // buildDescTableScan builds a desc table scan upon tblInfo. func (d *ddlCtx) buildDescTableScan(ctx context.Context, startTS uint64, tbl table.PhysicalTable, columns []*model.ColumnInfo, limit uint64) (distsql.SelectResult, error) { sctx := newContext(d.store) - dagPB, err := buildDescTableScanDAG(sctx, startTS, tbl, columns, limit) + dagPB, err := buildDescTableScanDAG(sctx, tbl, columns, limit) if err != nil { return nil, errors.Trace(err) } @@ -257,6 +256,7 @@ func (d *ddlCtx) buildDescTableScan(ctx context.Context, startTS uint64, tbl tab var builder distsql.RequestBuilder builder.SetTableRanges(tbl.GetPhysicalID(), ranges, nil). SetDAGRequest(dagPB). + SetStartTS(startTS). SetKeepOrder(true). SetConcurrency(1).SetDesc(true) diff --git a/ddl/rollingback.go b/ddl/rollingback.go index b8af05a80203e..360743249ba86 100644 --- a/ddl/rollingback.go +++ b/ddl/rollingback.go @@ -295,7 +295,7 @@ func convertJob2RollbackJob(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) model.ActionModifyColumn, model.ActionAddForeignKey, model.ActionDropForeignKey, model.ActionRenameTable, model.ActionModifyTableCharsetAndCollate, model.ActionTruncateTablePartition, - model.ActionModifySchemaCharsetAndCollate: + model.ActionModifySchemaCharsetAndCollate, model.ActionRepairTable: ver, err = cancelOnlyNotHandledJob(job) default: job.State = model.JobStateCancelled diff --git a/ddl/table.go b/ddl/table.go index 155e208114e7d..16cdc99527200 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -92,6 +92,15 @@ func createTableOrViewWithCheck(t *meta.Meta, job *model.Job, schemaID int64, tb return t.CreateTableOrView(schemaID, tbInfo) } +func repairTableOrViewWithCheck(t *meta.Meta, job *model.Job, schemaID int64, tbInfo *model.TableInfo) error { + err := checkTableInfoValid(tbInfo) + if err != nil { + job.State = model.JobStateCancelled + return errors.Trace(err) + } + return t.UpdateTable(schemaID, tbInfo) +} + func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) { schemaID := job.SchemaID tbInfo := &model.TableInfo{} @@ -264,7 +273,13 @@ func (w *worker) onRecoverTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in return ver, errors.Trace(err) } // Remove dropped table DDL job from gc_delete_range table. - err = w.delRangeManager.removeFromGCDeleteRange(dropJobID, tblInfo.ID) + var tids []int64 + if tblInfo.GetPartitionInfo() != nil { + tids = getPartitionIDs(tblInfo) + } else { + tids = []int64{tblInfo.ID} + } + err = w.delRangeManager.removeFromGCDeleteRange(dropJobID, tids) if err != nil { return ver, errors.Trace(err) } @@ -896,3 +911,46 @@ func checkAddPartitionValue(meta *model.TableInfo, part *model.PartitionInfo) er } return nil } + +func onRepairTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) { + schemaID := job.SchemaID + tblInfo := &model.TableInfo{} + + if err := job.DecodeArgs(tblInfo); err != nil { + // Invalid arguments, cancel this job. + job.State = model.JobStateCancelled + return ver, errors.Trace(err) + } + + tblInfo.State = model.StateNone + + // Check the old DB and old table exist. + _, err := getTableInfoAndCancelFaultJob(t, job, schemaID) + if err != nil { + return ver, errors.Trace(err) + } + + // When in repair mode, the repaired table in a server is not access to user, + // the table after repairing will be removed from repair list. Other server left + // behind alive may need to restart to get the latest schema version. + ver, err = updateSchemaVersion(t, job) + if err != nil { + return ver, errors.Trace(err) + } + switch tblInfo.State { + case model.StateNone: + // none -> public + tblInfo.State = model.StatePublic + tblInfo.UpdateTS = t.StartTS + err = repairTableOrViewWithCheck(t, job, schemaID, tblInfo) + if err != nil { + return ver, errors.Trace(err) + } + // Finish this job. + job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo) + asyncNotifyEvent(d, &util.Event{Tp: model.ActionRepairTable, TableInfo: tblInfo}) + return ver, nil + default: + return ver, ErrInvalidDDLState.GenWithStackByArgs("table", tblInfo.State) + } +} diff --git a/ddl/util/util.go b/ddl/util/util.go index 08ecdc9d8b1ca..69a6d3ced56f7 100644 --- a/ddl/util/util.go +++ b/ddl/util/util.go @@ -14,9 +14,11 @@ package util import ( + "bytes" "context" "encoding/hex" "fmt" + "strconv" "github.com/pingcap/errors" "github.com/pingcap/parser/terror" @@ -28,13 +30,14 @@ import ( ) const ( - deleteRangesTable = `gc_delete_range` - doneDeleteRangesTable = `gc_delete_range_done` - loadDeleteRangeSQL = `SELECT HIGH_PRIORITY job_id, element_id, start_key, end_key FROM mysql.%s WHERE ts < %v` - recordDoneDeletedRangeSQL = `INSERT IGNORE INTO mysql.gc_delete_range_done SELECT * FROM mysql.gc_delete_range WHERE job_id = %d AND element_id = %d` - completeDeleteRangeSQL = `DELETE FROM mysql.gc_delete_range WHERE job_id = %d AND element_id = %d` - updateDeleteRangeSQL = `UPDATE mysql.gc_delete_range SET start_key = "%s" WHERE job_id = %d AND element_id = %d AND start_key = "%s"` - deleteDoneRecordSQL = `DELETE FROM mysql.gc_delete_range_done WHERE job_id = %d AND element_id = %d` + deleteRangesTable = `gc_delete_range` + doneDeleteRangesTable = `gc_delete_range_done` + loadDeleteRangeSQL = `SELECT HIGH_PRIORITY job_id, element_id, start_key, end_key FROM mysql.%s WHERE ts < %v` + recordDoneDeletedRangeSQL = `INSERT IGNORE INTO mysql.gc_delete_range_done SELECT * FROM mysql.gc_delete_range WHERE job_id = %d AND element_id = %d` + completeDeleteRangeSQL = `DELETE FROM mysql.gc_delete_range WHERE job_id = %d AND element_id = %d` + completeDeleteMultiRangesSQL = `DELETE FROM mysql.gc_delete_range WHERE job_id = %d AND element_id in (%v)` + updateDeleteRangeSQL = `UPDATE mysql.gc_delete_range SET start_key = "%s" WHERE job_id = %d AND element_id = %d AND start_key = "%s"` + deleteDoneRecordSQL = `DELETE FROM mysql.gc_delete_range_done WHERE job_id = %d AND element_id = %d` ) // DelRangeTask is for run delete-range command in gc_worker. @@ -119,6 +122,20 @@ func RemoveFromGCDeleteRange(ctx sessionctx.Context, jobID, elementID int64) err return errors.Trace(err) } +// RemoveMultiFromGCDeleteRange is exported for ddl pkg to use. +func RemoveMultiFromGCDeleteRange(ctx sessionctx.Context, jobID int64, elementIDs []int64) error { + var buf bytes.Buffer + for i, elementID := range elementIDs { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(strconv.FormatInt(elementID, 10)) + } + sql := fmt.Sprintf(completeDeleteMultiRangesSQL, jobID, buf.String()) + _, err := ctx.(sqlexec.SQLExecutor).Execute(context.TODO(), sql) + return errors.Trace(err) +} + // DeleteDoneRecord removes a record from gc_delete_range_done table. func DeleteDoneRecord(ctx sessionctx.Context, dr DelRangeTask) error { sql := fmt.Sprintf(deleteDoneRecordSQL, dr.JobID, dr.ElementID) diff --git a/distsql/request_builder.go b/distsql/request_builder.go index dbde317178447..651e7c3bfa5fa 100644 --- a/distsql/request_builder.go +++ b/distsql/request_builder.go @@ -76,7 +76,6 @@ func (builder *RequestBuilder) SetTableHandles(tid int64, handles []int64) *Requ func (builder *RequestBuilder) SetDAGRequest(dag *tipb.DAGRequest) *RequestBuilder { if builder.err == nil { builder.Request.Tp = kv.ReqTypeDAG - builder.Request.StartTs = dag.StartTs builder.Request.Data, builder.err = dag.Marshal() } @@ -87,7 +86,6 @@ func (builder *RequestBuilder) SetDAGRequest(dag *tipb.DAGRequest) *RequestBuild func (builder *RequestBuilder) SetAnalyzeRequest(ana *tipb.AnalyzeReq) *RequestBuilder { if builder.err == nil { builder.Request.Tp = kv.ReqTypeAnalyze - builder.Request.StartTs = ana.StartTs builder.Request.Data, builder.err = ana.Marshal() builder.Request.NotFillCache = true builder.Request.IsolationLevel = kv.RC @@ -101,7 +99,6 @@ func (builder *RequestBuilder) SetAnalyzeRequest(ana *tipb.AnalyzeReq) *RequestB func (builder *RequestBuilder) SetChecksumRequest(checksum *tipb.ChecksumRequest) *RequestBuilder { if builder.err == nil { builder.Request.Tp = kv.ReqTypeChecksum - builder.Request.StartTs = checksum.StartTs builder.Request.Data, builder.err = checksum.Marshal() builder.Request.NotFillCache = true } @@ -115,6 +112,12 @@ func (builder *RequestBuilder) SetKeyRanges(keyRanges []kv.KeyRange) *RequestBui return builder } +// SetStartTS sets "StartTS" for "kv.Request". +func (builder *RequestBuilder) SetStartTS(startTS uint64) *RequestBuilder { + builder.Request.StartTs = startTS + return builder +} + // SetDesc sets "Desc" for "kv.Request". func (builder *RequestBuilder) SetDesc(desc bool) *RequestBuilder { builder.Request.Desc = desc diff --git a/distsql/request_builder_test.go b/distsql/request_builder_test.go index 62c25cfc3e5f2..1e4800898b788 100644 --- a/distsql/request_builder_test.go +++ b/distsql/request_builder_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/mock" @@ -55,7 +56,8 @@ type testSuite struct { func (s *testSuite) SetUpSuite(c *C) { ctx := mock.NewContext() ctx.GetSessionVars().StmtCtx = &stmtctx.StatementContext{ - MemTracker: memory.NewTracker(stringutil.StringerStr("testSuite"), variable.DefTiDBMemQuotaDistSQL), + MemTracker: memory.NewTracker(stringutil.StringerStr("testSuite"), variable.DefTiDBMemQuotaDistSQL), + DiskTracker: disk.NewTracker(stringutil.StringerStr("testSuite"), -1), } ctx.Store = &mock.Store{ Client: &mock.Client{ @@ -285,7 +287,7 @@ func (s *testSuite) TestRequestBuilder1(c *C) { expect := &kv.Request{ Tp: 103, StartTs: 0x0, - Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0}, + Data: []uint8{0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0}, KeyRanges: []kv.KeyRange{ { StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, @@ -360,7 +362,7 @@ func (s *testSuite) TestRequestBuilder2(c *C) { expect := &kv.Request{ Tp: 103, StartTs: 0x0, - Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0}, + Data: []uint8{0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0}, KeyRanges: []kv.KeyRange{ { StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, @@ -409,7 +411,7 @@ func (s *testSuite) TestRequestBuilder3(c *C) { expect := &kv.Request{ Tp: 103, StartTs: 0x0, - Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0}, + Data: []uint8{0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0}, KeyRanges: []kv.KeyRange{ { StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, @@ -472,7 +474,7 @@ func (s *testSuite) TestRequestBuilder4(c *C) { expect := &kv.Request{ Tp: 103, StartTs: 0x0, - Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0}, + Data: []uint8{0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0}, KeyRanges: keyRanges, KeepOrder: false, Desc: false, @@ -516,7 +518,7 @@ func (s *testSuite) TestRequestBuilder5(c *C) { expect := &kv.Request{ Tp: 104, StartTs: 0x0, - Data: []uint8{0x8, 0x0, 0x10, 0x0, 0x18, 0x0, 0x20, 0x0}, + Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0}, KeyRanges: keyRanges, KeepOrder: true, Desc: false, @@ -549,7 +551,7 @@ func (s *testSuite) TestRequestBuilder6(c *C) { expect := &kv.Request{ Tp: 105, StartTs: 0x0, - Data: []uint8{0x8, 0x0, 0x10, 0x0, 0x18, 0x0}, + Data: []uint8{0x10, 0x0, 0x18, 0x0}, KeyRanges: keyRanges, KeepOrder: false, Desc: false, diff --git a/docs/design/2019-11-14-tidb-builtin-diagnostics-zh_CN.md b/docs/design/2019-11-14-tidb-builtin-diagnostics-zh_CN.md index 6d612ff9622e9..0d235da7f9496 100644 --- a/docs/design/2019-11-14-tidb-builtin-diagnostics-zh_CN.md +++ b/docs/design/2019-11-14-tidb-builtin-diagnostics-zh_CN.md @@ -132,7 +132,7 @@ TiDB/TiKV/PD 产生的日志都保存在各自的节点上，并且 TiDB 集群 - `start_time`: 日志检索的开始时间（unix 时间戳，单位毫秒），如果没有该谓词，则默认为 0。 - `end_time`: 日志检索的开始时间（unix 时间戳，单位毫秒），如果没有该谓词，则默认为 `int64::MAX`。 -- `pattern`: 如 SELECT * FROM tidb_cluster_log WHERE pattern LIKE "%gc%" 中的 %gc% 即为过滤的关键字 +- `pattern`: 如 SELECT * FROM cluster_log WHERE pattern LIKE "%gc%" 中的 %gc% 即为过滤的关键字 - `level`: 日志等级，可以选为 DEBUG/INFO/WARN/WARNING/TRACE/CRITICAL/ERROR - `limit`: 返回日志的条数，如果没有指定，则限制为 64k 条，防止日质量太大占用大量网络 @@ -301,7 +301,7 @@ message ServerInfoResponse { mysql> use information_schema; Database changed -mysql> desc TIDB_CLUSTER_INFO; +mysql> desc CLUSTER_INFO; +----------------+---------------------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +----------------+---------------------+------+------+---------+-------+ @@ -313,7 +313,7 @@ mysql> desc TIDB_CLUSTER_INFO; +----------------+---------------------+------+------+---------+-------+ 5 rows in set (0.00 sec) -mysql> select TYPE, ADDRESS, STATUS_ADDRESS,VERSION from TIDB_CLUSTER_INFO; +mysql> select TYPE, ADDRESS, STATUS_ADDRESS,VERSION from CLUSTER_INFO; +------+-----------------+-----------------+-----------------------------------------------+ | TYPE | ADDRESS | STATUS_ADDRESS | VERSION | +------+-----------------+-----------------+-----------------------------------------------+ @@ -492,9 +492,9 @@ mysql> select address, type, value from pd_client_cmd_ops where start_time='2019 | 表名 | 描述 | |------|-----| -| tidb_cluster_slow_query | 所有 TiDB 节点的 slow_query 表数据 | -| tidb_cluster_statements_summary | 所有 TiDB 节点的 statements summary 表数据 | -| tidb_cluster_processlist | 所有 TiDB 节点的 processlist 表数据 | +| cluster_slow_query | 所有 TiDB 节点的 slow_query 表数据 | +| cluster_statements_summary | 所有 TiDB 节点的 statements summary 表数据 | +| cluster_processlist | 所有 TiDB 节点的 processlist 表数据 | #### 所有节点的配置信息 @@ -506,7 +506,7 @@ mysql> select address, type, value from pd_client_cmd_ops where start_time='2019 mysql> use information_schema; Database changed -mysql> select * from tidb_cluster_config where `key` like 'log%'; +mysql> select * from cluster_config where `key` like 'log%'; +------+-----------------+-----------------------------+---------------+ | TYPE | ADDRESS | KEY | VALUE | +------+-----------------+-----------------------------+---------------+ @@ -546,7 +546,7 @@ mysql> select * from tidb_cluster_config where `key` like 'log%'; +------+-----------------+-----------------------------+---------------+ 33 rows in set (0.00 sec) -mysql> select * from tidb_cluster_config where type='tikv' and `key` like 'raftdb.wal%'; +mysql> select * from cluster_config where type='tikv' and `key` like 'raftdb.wal%'; +------+-----------------+---------------------------+--------+ | TYPE | ADDRESS | KEY | VALUE | +------+-----------------+---------------------------+--------+ @@ -567,7 +567,7 @@ mysql> select * from tidb_cluster_config where type='tikv' and `key` like 'raftd mysql> use information_schema; Database changed -mysql> select * from tidb_cluster_hardware +mysql> select * from cluster_hardware +------+-----------------+----------+----------+-------------+--------+ | TYPE | ADDRESS | HW_TYPE | HW_NAME | KEY | VALUE | +------+-----------------+----------+----------+-------------+--------+ @@ -584,7 +584,7 @@ mysql> select * from tidb_cluster_hardware +------+-----------------+----------+----------+-------------+--------+ 10 rows in set (0.01 sec) -mysql> select * from tidb_cluster_systeminfo +mysql> select * from cluster_systeminfo +------+-----------------+----------+--------------+--------+ | TYPE | ADDRESS | MODULE | KEY | VALUE | +------+-----------------+----------+--------------+--------+ @@ -594,7 +594,7 @@ mysql> select * from tidb_cluster_systeminfo +------+-----------------+----------+--------------+--------+ 20 rows in set (0.01 sec) -mysql> select * from tidb_cluster_load +mysql> select * from cluster_load +------+-----------------+----------+-------------+--------+ | TYPE | ADDRESS | MODULE | KEY | VALUE | +------+-----------------+----------+-------------+--------+ @@ -606,7 +606,7 @@ mysql> select * from tidb_cluster_load #### 全链路日志系统表 -当前日志搜索需要登陆多台机器分别进行检索，并且没有简单的办法对多个机器的检索结果按照时间全排序。本提案新建一个 `tidb_cluster_log` 系统表用于提供全链路日志，简化通过日志排查问题的方式以及提高效率。实现方式为：通过 gRPC Diagnosis Service 的 `search_log` 接口，将日志过滤的谓词下推到各个节点，并最终按照时间进行归并。 +当前日志搜索需要登陆多台机器分别进行检索，并且没有简单的办法对多个机器的检索结果按照时间全排序。本提案新建一个 `cluster_log` 系统表用于提供全链路日志，简化通过日志排查问题的方式以及提高效率。实现方式为：通过 gRPC Diagnosis Service 的 `search_log` 接口，将日志过滤的谓词下推到各个节点，并最终按照时间进行归并。 如下示例是实现本提案后的预期结果： @@ -614,7 +614,7 @@ mysql> select * from tidb_cluster_load mysql> use information_schema; Database changed -mysql> desc tidb_cluster_log; +mysql> desc cluster_log; +---------+-------------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +---------+-------------+------+------+---------+-------+ @@ -626,7 +626,7 @@ mysql> desc tidb_cluster_log; +---------+-------------+------+------+---------+-------+ 5 rows in set (0.00 sec) -mysql> select * from tidb_cluster_log where content like '%412134239937495042%'; -- 查询 TSO 为 412134239937495042 全链路日志 +mysql> select * from cluster_log where content like '%412134239937495042%'; -- 查询 TSO 为 412134239937495042 全链路日志 +------+--------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | TYPE | ADDRESS | LEVEL | CONTENT | +------+------------------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -665,9 +665,9 @@ mysql> select * from tidb_cluster_log where content like '%412134239937495042%'; +------+--------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ 31 rows in set (0.01 sec) -mysql> select * from tidb_cluster_log where type='pd' and content like '%scheduler%'; -- 查询 PD 的调度日志 +mysql> select * from cluster_log where type='pd' and content like '%scheduler%'; -- 查询 PD 的调度日志 -mysql> select * from tidb_cluster_log where type='tidb' and content like '%ddl%'; -- 查询 TiDB 的 DDL 日志 +mysql> select * from cluster_log where type='tidb' and content like '%ddl%'; -- 查询 TiDB 的 DDL 日志 ``` ### 集群诊断 diff --git a/docs/design/2019-11-14-tidb-builtin-diagnostics.md b/docs/design/2019-11-14-tidb-builtin-diagnostics.md index 98a2e7757858e..4223c17f08a09 100644 --- a/docs/design/2019-11-14-tidb-builtin-diagnostics.md +++ b/docs/design/2019-11-14-tidb-builtin-diagnostics.md @@ -135,7 +135,7 @@ The following are the predicates that the log interface needs to process: - `start_time`: start time of the log retrieval (Unix timestamp, in milliseconds). If there is no such predicate, the default is 0. - `end_time:`: end time of the log retrieval (Unix timestamp, in milliseconds). If there is no such predicate, the default is `int64::MAX`. -- `pattern`: filter pattern determined by the keyword. For example, `SELECT * FROM tidb_cluster_log` WHERE "%gc%" `%gc%` is the filtered keyword. +- `pattern`: filter pattern determined by the keyword. For example, `SELECT * FROM cluster_log` WHERE "%gc%" `%gc%` is the filtered keyword. - `level`: log level; can be selected as DEBUG/INFO/WARN/WARNING/TRACE/CRITICAL/ERROR - `limit`: the maximum of logs items to return, preventing the log from being too large and occupying a large bandwidth of the network.. If not specified, the default limit is 64k. @@ -306,7 +306,7 @@ The implementation of this proposal can query the following results through SQL: mysql> use information_schema; Database changed -mysql> desc TIDB_CLUSTER_INFO; +mysql> desc CLUSTER_INFO; +----------------+---------------------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +----------------+---------------------+------+------+---------+-------+ @@ -318,7 +318,7 @@ mysql> desc TIDB_CLUSTER_INFO; +----------------+---------------------+------+------+---------+-------+ 5 rows in set (0.00 sec) -mysql> select TYPE, ADDRESS, STATUS_ADDRESS,VERSION from TIDB_CLUSTER_INFO; +mysql> select TYPE, ADDRESS, STATUS_ADDRESS,VERSION from CLUSTER_INFO; +------+-----------------+-----------------+-----------------------------------------------+ | TYPE | ADDRESS | STATUS_ADDRESS | VERSION | +------+-----------------+-----------------+-----------------------------------------------+ @@ -484,9 +484,9 @@ Current the `slow_query`/`events_statements_summary_by_digest`/`processlist` mem | Table Name | Description | |------|-----| -| tidb_cluster_slow_query | slow_query table data for all TiDB nodes | -| tidb_cluster_statements_summary | statements summary table Data for all TiDB nodes | -| tidb_cluster_processlist | processlist table data for all TiDB nodes | +| cluster_slow_query | slow_query table data for all TiDB nodes | +| cluster_statements_summary | statements summary table Data for all TiDB nodes | +| cluster_processlist | processlist table data for all TiDB nodes | #### Configuration information of all nodes @@ -498,7 +498,7 @@ See the following example for some expected results of this proposal: mysql> use information_schema; Database changed -mysql> select * from tidb_cluster_config where `key` like 'log%'; +mysql> select * from cluster_config where `key` like 'log%'; +------+-----------------+-----------------------------+---------------+ | TYPE | ADDRESS | KEY | VALUE | +------+-----------------+-----------------------------+---------------+ @@ -538,7 +538,7 @@ mysql> select * from tidb_cluster_config where `key` like 'log%'; +------+-----------------+-----------------------------+---------------+ 33 rows in set (0.00 sec) -mysql> select * from tidb_cluster_config where type='tikv' and `key` like 'raftdb.wal%'; +mysql> select * from cluster_config where type='tikv' and `key` like 'raftdb.wal%'; +------+-----------------+---------------------------+--------+ | TYPE | ADDRESS | KEY | VALUE | +------+-----------------+---------------------------+--------+ @@ -559,7 +559,7 @@ According to the definition of `gRPC Service` protocol, each `ServerInfoItem` co mysql> use information_schema; Database changed -mysql> select * from tidb_cluster_hardware +mysql> select * from cluster_hardware +------+-----------------+----------+----------+-------------+--------+ | TYPE | ADDRESS | HW_TYPE | HW_NAME | KEY | VALUE | +------+-----------------+----------+----------+-------------+--------+ @@ -576,7 +576,7 @@ mysql> select * from tidb_cluster_hardware +------+-----------------+----------+----------+-------------+--------+ 10 rows in set (0.01 sec) -mysql> select * from tidb_cluster_systeminfo +mysql> select * from cluster_systeminfo +------+-----------------+----------+--------------+--------+ | TYPE | ADDRESS | MODULE | KEY | VALUE | +------+-----------------+----------+--------------+--------+ @@ -586,7 +586,7 @@ mysql> select * from tidb_cluster_systeminfo +------+-----------------+----------+--------------+--------+ 20 rows in set (0.01 sec) -mysql> select * from tidb_cluster_load +mysql> select * from cluster_load +------+-----------------+----------+-------------+--------+ | TYPE | ADDRESS | MODULE | KEY | VALUE | +------+-----------------+----------+-------------+--------+ @@ -599,7 +599,7 @@ mysql> select * from tidb_cluster_load #### Full-chain log system table -To search in the current log, users need to log in to multiple machines for retrieval respectively, and there is no easy way to sort the retrieval results of multiple machines by time. This proposal creates a new `tidb_cluster_log` system table to provide full-link logs, thereby simplifying the way to troubleshoot problems through logs and improving efficiency. This is achieved by pushing the log-filtering predicates down to the nodes through the `search_log` interface of the gRPC Diagnosis Service. The filtered logs will be eventually merged by time. +To search in the current log, users need to log in to multiple machines for retrieval respectively, and there is no easy way to sort the retrieval results of multiple machines by time. This proposal creates a new `cluster_log` system table to provide full-link logs, thereby simplifying the way to troubleshoot problems through logs and improving efficiency. This is achieved by pushing the log-filtering predicates down to the nodes through the `search_log` interface of the gRPC Diagnosis Service. The filtered logs will be eventually merged by time. The following example shows the expected results of this proposal: @@ -607,7 +607,7 @@ The following example shows the expected results of this proposal: mysql> use information_schema; Database changed -mysql> desc tidb_cluster_log; +mysql> desc cluster_log; +---------+-------------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +---------+-------------+------+------+---------+-------+ @@ -619,7 +619,7 @@ mysql> desc tidb_cluster_log; +---------+-------------+------+------+---------+-------+ 5 rows in set (0.00 sec) -mysql> select * from tidb_cluster_log where content like '%412134239937495042%'; -- Query the full link log related to TSO 412134239937495042 +mysql> select * from cluster_log where content like '%412134239937495042%'; -- Query the full link log related to TSO 412134239937495042 +------+--------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | TYPE | ADDRESS | LEVEL | CONTENT | +------+------------------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -658,9 +658,9 @@ mysql> select * from tidb_cluster_log where content like '%412134239937495042%'; +------+--------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ 31 rows in set (0.01 sec) -mysql> select * from tidb_cluster_log where type='pd' and content like '%scheduler%'; -- Query scheduler logs of PD +mysql> select * from cluster_log where type='pd' and content like '%scheduler%'; -- Query scheduler logs of PD -mysql> select * from tidb_cluster_log where type='tidb' and content like '%ddl%'; -- Query DDL logs of TiDB +mysql> select * from cluster_log where type='tidb' and content like '%ddl%'; -- Query DDL logs of TiDB ``` ### Cluster diagnostics @@ -700,4 +700,4 @@ Therefore, implementing a diagnostic system that supports hot rule loading is ne - Advantages: easy to write, load and execute - Disadvantages: need to run on the machine where the MySQL client is installed -This proposal temporarily adopts the third option to write diagnostic rules using Shell. There is no intrusion into TiDB, and it also provides scalability for subsequent implementations of better solutions. \ No newline at end of file +This proposal temporarily adopts the third option to write diagnostic rules using Shell. There is no intrusion into TiDB, and it also provides scalability for subsequent implementations of better solutions. diff --git a/domain/domain.go b/domain/domain.go index 0373111eb1c71..207a5f7e6fd51 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -45,6 +45,7 @@ import ( "github.com/pingcap/tidb/statistics/handle" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/util" + "github.com/pingcap/tidb/util/domainutil" "github.com/pingcap/tidb/util/expensivequery" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/sqlexec" @@ -207,6 +208,10 @@ func (do *Domain) fetchSchemasWithTables(schemas []*model.DBInfo, m *meta.Meta, continue } infoschema.ConvertCharsetCollateToLowerCaseIfNeed(tbl) + // Check whether the table is in repair mode. + if domainutil.RepairInfo.InRepairMode() && domainutil.RepairInfo.CheckAndFetchRepairedTable(di, tbl) { + continue + } di.Tables = append(di.Tables, tbl) } } diff --git a/domain/domain_test.go b/domain/domain_test.go index 9e6e6d9fddace..3b3644b860bd1 100644 --- a/domain/domain_test.go +++ b/domain/domain_test.go @@ -317,23 +317,33 @@ func (*testSuite) TestT(c *C) { res := dom.ShowSlowQuery(&ast.ShowSlow{Tp: ast.ShowSlowTop, Count: 2}) c.Assert(res, HasLen, 2) - c.Assert(*res[0], Equals, SlowQueryInfo{SQL: "bbb", Duration: 3 * time.Second}) - c.Assert(*res[1], Equals, SlowQueryInfo{SQL: "ccc", Duration: 2 * time.Second}) + c.Assert(res[0].SQL, Equals, "bbb") + c.Assert(res[0].Duration, Equals, 3*time.Second) + c.Assert(res[1].SQL, Equals, "ccc") + c.Assert(res[1].Duration, Equals, 2*time.Second) res = dom.ShowSlowQuery(&ast.ShowSlow{Tp: ast.ShowSlowTop, Count: 2, Kind: ast.ShowSlowKindInternal}) c.Assert(res, HasLen, 1) - c.Assert(*res[0], Equals, SlowQueryInfo{SQL: "aaa", Duration: time.Second, Internal: true}) + c.Assert(res[0].SQL, Equals, "aaa") + c.Assert(res[0].Duration, Equals, time.Second) + c.Assert(res[0].Internal, Equals, true) res = dom.ShowSlowQuery(&ast.ShowSlow{Tp: ast.ShowSlowTop, Count: 4, Kind: ast.ShowSlowKindAll}) c.Assert(res, HasLen, 3) - c.Assert(*res[0], Equals, SlowQueryInfo{SQL: "bbb", Duration: 3 * time.Second}) - c.Assert(*res[1], Equals, SlowQueryInfo{SQL: "ccc", Duration: 2 * time.Second}) - c.Assert(*res[2], Equals, SlowQueryInfo{SQL: "aaa", Duration: time.Second, Internal: true}) + c.Assert(res[0].SQL, Equals, "bbb") + c.Assert(res[0].Duration, Equals, 3*time.Second) + c.Assert(res[1].SQL, Equals, "ccc") + c.Assert(res[1].Duration, Equals, 2*time.Second) + c.Assert(res[2].SQL, Equals, "aaa") + c.Assert(res[2].Duration, Equals, time.Second) + c.Assert(res[2].Internal, Equals, true) res = dom.ShowSlowQuery(&ast.ShowSlow{Tp: ast.ShowSlowRecent, Count: 2}) c.Assert(res, HasLen, 2) - c.Assert(*res[0], Equals, SlowQueryInfo{SQL: "ccc", Duration: 2 * time.Second}) - c.Assert(*res[1], Equals, SlowQueryInfo{SQL: "bbb", Duration: 3 * time.Second}) + c.Assert(res[0].SQL, Equals, "ccc") + c.Assert(res[0].Duration, Equals, 2*time.Second) + c.Assert(res[1].SQL, Equals, "bbb") + c.Assert(res[1].Duration, Equals, 3*time.Second) metrics.PanicCounter.Reset() // Since the stats lease is 0 now, so create a new ticker will panic. diff --git a/domain/topn_slow_query_test.go b/domain/topn_slow_query_test.go index dbf7ea0a8c6fb..583d82651e9c7 100644 --- a/domain/topn_slow_query_test.go +++ b/domain/topn_slow_query_test.go @@ -116,14 +116,14 @@ func (t *testTopNSlowQuerySuite) TestQueue(c *C) { q.Append(&SlowQueryInfo{SQL: "ccc"}) query := q.recent.Query(1) - c.Assert(*query[0], Equals, SlowQueryInfo{SQL: "ccc"}) + c.Assert(query[0].SQL, Equals, "ccc") query = q.recent.Query(2) - c.Assert(*query[0], Equals, SlowQueryInfo{SQL: "ccc"}) - c.Assert(*query[1], Equals, SlowQueryInfo{SQL: "bbb"}) + c.Assert(query[0].SQL, Equals, "ccc") + c.Assert(query[1].SQL, Equals, "bbb") query = q.recent.Query(6) - c.Assert(*query[0], Equals, SlowQueryInfo{SQL: "ccc"}) - c.Assert(*query[1], Equals, SlowQueryInfo{SQL: "bbb"}) - c.Assert(*query[2], Equals, SlowQueryInfo{SQL: "aaa"}) + c.Assert(query[0].SQL, Equals, "ccc") + c.Assert(query[1].SQL, Equals, "bbb") + c.Assert(query[2].SQL, Equals, "aaa") q.Append(&SlowQueryInfo{SQL: "ddd"}) q.Append(&SlowQueryInfo{SQL: "eee"}) @@ -131,13 +131,13 @@ func (t *testTopNSlowQuerySuite) TestQueue(c *C) { q.Append(&SlowQueryInfo{SQL: "ggg"}) query = q.recent.Query(3) - c.Assert(*query[0], Equals, SlowQueryInfo{SQL: "ggg"}) - c.Assert(*query[1], Equals, SlowQueryInfo{SQL: "fff"}) - c.Assert(*query[2], Equals, SlowQueryInfo{SQL: "eee"}) + c.Assert(query[0].SQL, Equals, "ggg") + c.Assert(query[1].SQL, Equals, "fff") + c.Assert(query[2].SQL, Equals, "eee") query = q.recent.Query(6) - c.Assert(*query[0], Equals, SlowQueryInfo{SQL: "ggg"}) - c.Assert(*query[1], Equals, SlowQueryInfo{SQL: "fff"}) - c.Assert(*query[2], Equals, SlowQueryInfo{SQL: "eee"}) - c.Assert(*query[3], Equals, SlowQueryInfo{SQL: "ddd"}) - c.Assert(*query[4], Equals, SlowQueryInfo{SQL: "ccc"}) + c.Assert(query[0].SQL, Equals, "ggg") + c.Assert(query[1].SQL, Equals, "fff") + c.Assert(query[2].SQL, Equals, "eee") + c.Assert(query[3].SQL, Equals, "ddd") + c.Assert(query[4].SQL, Equals, "ccc") } diff --git a/executor/adapter.go b/executor/adapter.go index e8e027122d2f6..9e2cb5f3134c5 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -254,11 +254,6 @@ func (a *ExecStmt) IsReadOnly(vars *variable.SessionVars) bool { // RebuildPlan rebuilds current execute statement plan. // It returns the current information schema version that 'a' is using. func (a *ExecStmt) RebuildPlan(ctx context.Context) (int64, error) { - startTime := time.Now() - defer func() { - a.Ctx.GetSessionVars().DurationCompile = time.Since(startTime) - }() - is := GetInfoSchema(a.Ctx) a.InfoSchema = is if err := plannercore.Preprocess(a.Ctx, a.StmtNode, is, plannercore.InTxnRetry); err != nil { @@ -619,9 +614,6 @@ func (a *ExecStmt) handlePessimisticLockError(ctx context.Context, err error) (E // Rollback the statement change before retry it. a.Ctx.StmtRollback() a.Ctx.GetSessionVars().StmtCtx.ResetForRetry() - a.Ctx.GetSessionVars().StartTime = time.Now() - a.Ctx.GetSessionVars().DurationCompile = time.Duration(0) - a.Ctx.GetSessionVars().DurationParse = time.Duration(0) if err = e.Open(ctx); err != nil { return nil, err @@ -752,7 +744,7 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) { sessVars := a.Ctx.GetSessionVars() level := log.GetLevel() cfg := config.GetGlobalConfig() - costTime := time.Since(a.Ctx.GetSessionVars().StartTime) + costTime := time.Since(sessVars.StartTime) + sessVars.DurationParse threshold := time.Duration(atomic.LoadUint64(&cfg.Log.SlowThreshold)) * time.Millisecond if costTime < threshold && level > zapcore.DebugLevel { return @@ -761,10 +753,10 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) { var tableIDs, indexNames string if len(sessVars.StmtCtx.TableIDs) > 0 { - tableIDs = strings.Replace(fmt.Sprintf("%v", a.Ctx.GetSessionVars().StmtCtx.TableIDs), " ", ",", -1) + tableIDs = strings.Replace(fmt.Sprintf("%v", sessVars.StmtCtx.TableIDs), " ", ",", -1) } if len(sessVars.StmtCtx.IndexNames) > 0 { - indexNames = strings.Replace(fmt.Sprintf("%v", a.Ctx.GetSessionVars().StmtCtx.IndexNames), " ", ",", -1) + indexNames = strings.Replace(fmt.Sprintf("%v", sessVars.StmtCtx.IndexNames), " ", ",", -1) } execDetail := sessVars.StmtCtx.GetExecDetails() copTaskInfo := sessVars.StmtCtx.CopTasksDetails() @@ -776,8 +768,8 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) { SQL: sql.String(), Digest: digest, TimeTotal: costTime, - TimeParse: a.Ctx.GetSessionVars().DurationParse, - TimeCompile: a.Ctx.GetSessionVars().DurationCompile, + TimeParse: sessVars.DurationParse, + TimeCompile: sessVars.DurationCompile, IndexNames: indexNames, StatsInfos: statsInfos, CopTasks: copTaskInfo, @@ -805,7 +797,7 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) { domain.GetDomain(a.Ctx).LogSlowQuery(&domain.SlowQueryInfo{ SQL: sql.String(), Digest: digest, - Start: a.Ctx.GetSessionVars().StartTime, + Start: sessVars.StartTime, Duration: costTime, Detail: sessVars.StmtCtx.GetExecDetails(), Succ: succ, @@ -826,23 +818,7 @@ func getPlanTree(p plannercore.Plan) string { if atomic.LoadUint32(&cfg.Log.RecordPlanInSlowLog) == 0 { return "" } - var selectPlan plannercore.PhysicalPlan - if physicalPlan, ok := p.(plannercore.PhysicalPlan); ok { - selectPlan = physicalPlan - } else { - switch x := p.(type) { - case *plannercore.Delete: - selectPlan = x.SelectPlan - case *plannercore.Update: - selectPlan = x.SelectPlan - case *plannercore.Insert: - selectPlan = x.SelectPlan - } - } - if selectPlan == nil { - return "" - } - planTree := plannercore.EncodePlan(selectPlan) + planTree := plannercore.EncodePlan(p) if len(planTree) == 0 { return planTree } diff --git a/executor/admin.go b/executor/admin.go index 0f58ae187c8c5..73ce92dad6444 100644 --- a/executor/admin.go +++ b/executor/admin.go @@ -108,9 +108,14 @@ func (e *CheckIndexRangeExec) Open(ctx context.Context) error { return err } sc := e.ctx.GetSessionVars().StmtCtx + txn, err := e.ctx.Txn(true) + if err != nil { + return nil + } var builder distsql.RequestBuilder kvReq, err := builder.SetIndexRanges(sc, e.table.ID, e.index.ID, ranger.FullRange()). SetDAGRequest(dagPB). + SetStartTS(txn.StartTS()). SetKeepOrder(true). SetFromSessionVars(e.ctx.GetSessionVars()). Build() @@ -128,11 +133,6 @@ func (e *CheckIndexRangeExec) Open(ctx context.Context) error { func (e *CheckIndexRangeExec) buildDAGPB() (*tipb.DAGRequest, error) { dagReq := &tipb.DAGRequest{} - txn, err := e.ctx.Txn(true) - if err != nil { - return nil, err - } - dagReq.StartTs = txn.StartTS() dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(e.ctx.GetSessionVars().Location()) sc := e.ctx.GetSessionVars().StmtCtx dagReq.Flags = sc.PushDownFlags() @@ -142,7 +142,7 @@ func (e *CheckIndexRangeExec) buildDAGPB() (*tipb.DAGRequest, error) { execPB := e.constructIndexScanPB() dagReq.Executors = append(dagReq.Executors, execPB) - err = plannercore.SetPBColumnsDefaultValue(e.ctx, dagReq.Executors[0].IdxScan.Columns, e.cols) + err := plannercore.SetPBColumnsDefaultValue(e.ctx, dagReq.Executors[0].IdxScan.Columns, e.cols) if err != nil { return nil, err } @@ -231,7 +231,6 @@ func (e *RecoverIndexExec) constructLimitPB(count uint64) *tipb.Executor { func (e *RecoverIndexExec) buildDAGPB(txn kv.Transaction, limitCnt uint64) (*tipb.DAGRequest, error) { dagReq := &tipb.DAGRequest{} - dagReq.StartTs = txn.StartTS() dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(e.ctx.GetSessionVars().Location()) sc := e.ctx.GetSessionVars().StmtCtx dagReq.Flags = sc.PushDownFlags() @@ -264,6 +263,7 @@ func (e *RecoverIndexExec) buildTableScan(ctx context.Context, txn kv.Transactio var builder distsql.RequestBuilder kvReq, err := builder.SetTableRanges(tblInfo.ID, ranges, nil). SetDAGRequest(dagPB). + SetStartTS(txn.StartTS()). SetKeepOrder(true). SetFromSessionVars(e.ctx.GetSessionVars()). Build() @@ -631,6 +631,7 @@ func (e *CleanupIndexExec) buildIndexScan(ctx context.Context, txn kv.Transactio ranges := ranger.FullRange() kvReq, err := builder.SetIndexRanges(sc, e.table.Meta().ID, e.index.Meta().ID, ranges). SetDAGRequest(dagPB). + SetStartTS(txn.StartTS()). SetKeepOrder(true). SetFromSessionVars(e.ctx.GetSessionVars()). Build() @@ -668,7 +669,6 @@ func (e *CleanupIndexExec) Open(ctx context.Context) error { func (e *CleanupIndexExec) buildIdxDAGPB(txn kv.Transaction) (*tipb.DAGRequest, error) { dagReq := &tipb.DAGRequest{} - dagReq.StartTs = txn.StartTS() dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(e.ctx.GetSessionVars().Location()) sc := e.ctx.GetSessionVars().StmtCtx dagReq.Flags = sc.PushDownFlags() diff --git a/executor/aggregate.go b/executor/aggregate.go index 9020d82168bb9..61c55b5a3584d 100644 --- a/executor/aggregate.go +++ b/executor/aggregate.go @@ -949,64 +949,6 @@ func (e *StreamAggExec) appendResult2Chunk(chk *chunk.Chunk) error { return nil } -type groupChecker struct { - StmtCtx *stmtctx.StatementContext - GroupByItems []expression.Expression - curGroupKey []types.Datum - tmpGroupKey []types.Datum -} - -func newGroupChecker(stmtCtx *stmtctx.StatementContext, items []expression.Expression) *groupChecker { - return &groupChecker{ - StmtCtx: stmtCtx, - GroupByItems: items, - } -} - -// meetNewGroup returns a value that represents if the new group is different from last group. -// TODO: Since all the group by items are only a column reference, guaranteed by building projection below aggregation, we can directly compare data in a chunk. -func (e *groupChecker) meetNewGroup(row chunk.Row) (bool, error) { - if len(e.GroupByItems) == 0 { - return false, nil - } - e.tmpGroupKey = e.tmpGroupKey[:0] - matched, firstGroup := true, false - if len(e.curGroupKey) == 0 { - matched, firstGroup = false, true - } - for i, item := range e.GroupByItems { - v, err := item.Eval(row) - if err != nil { - return false, err - } - if matched { - c, err := v.CompareDatum(e.StmtCtx, &e.curGroupKey[i]) - if err != nil { - return false, err - } - matched = c == 0 - } - e.tmpGroupKey = append(e.tmpGroupKey, v) - } - if matched { - return false, nil - } - e.curGroupKey = e.curGroupKey[:0] - for _, v := range e.tmpGroupKey { - e.curGroupKey = append(e.curGroupKey, *((&v).Copy())) - } - return !firstGroup, nil -} - -func (e *groupChecker) reset() { - if e.curGroupKey != nil { - e.curGroupKey = e.curGroupKey[:0] - } - if e.tmpGroupKey != nil { - e.tmpGroupKey = e.tmpGroupKey[:0] - } -} - // vecGroupChecker is used to split a given chunk according to the `group by` expression in a vectorized manner // It is usually used for streamAgg type vecGroupChecker struct { @@ -1036,7 +978,6 @@ type vecGroupChecker struct { } func newVecGroupChecker(ctx sessionctx.Context, items []expression.Expression) *vecGroupChecker { - return &vecGroupChecker{ ctx: ctx, GroupByItems: items, diff --git a/executor/analyze.go b/executor/analyze.go index 1aa62d1baaabb..7d91bdaff42d9 100755 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -259,6 +259,7 @@ func (e *AnalyzeIndexExec) fetchAnalyzeResult(ranges []*ranger.Range, isNullRang var builder distsql.RequestBuilder kvReq, err := builder.SetIndexRanges(e.ctx.GetSessionVars().StmtCtx, e.physicalTableID, e.idxInfo.ID, ranges). SetAnalyzeRequest(e.analyzePB). + SetStartTS(math.MaxUint64). SetKeepOrder(true). SetConcurrency(e.concurrency). Build() @@ -429,6 +430,7 @@ func (e *AnalyzeColumnsExec) buildResp(ranges []*ranger.Range) (distsql.SelectRe // correct `correlation` of columns. kvReq, err := builder.SetTableRanges(e.physicalTableID, ranges, nil). SetAnalyzeRequest(e.analyzePB). + SetStartTS(math.MaxUint64). SetKeepOrder(true). SetConcurrency(e.concurrency). Build() diff --git a/executor/benchmark_test.go b/executor/benchmark_test.go index a120c2b6d1850..3260ce2cac7b0 100644 --- a/executor/benchmark_test.go +++ b/executor/benchmark_test.go @@ -33,6 +33,7 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/mock" "github.com/pingcap/tidb/util/stringutil" @@ -558,6 +559,7 @@ func defaultHashJoinTestCase(cols []*types.FieldType, joinType core.JoinType, us ctx.GetSessionVars().InitChunkSize = variable.DefInitChunkSize ctx.GetSessionVars().MaxChunkSize = variable.DefMaxChunkSize ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(nil, -1) + ctx.GetSessionVars().StmtCtx.DiskTracker = disk.NewTracker(nil, -1) ctx.GetSessionVars().IndexLookupJoinConcurrency = 4 tc := &hashJoinTestCase{rows: 100000, concurrency: 4, ctx: ctx, keyIdx: []int{0, 1}} tc.cols = cols @@ -603,7 +605,9 @@ func prepare4HashJoin(testCase *hashJoinTestCase, innerExec, outerExec Executor) } t := memory.NewTracker(stringutil.StringerStr("root of prepare4HashJoin"), memLimit) t.SetActionOnExceed(nil) + t2 := disk.NewTracker(stringutil.StringerStr("root of prepare4HashJoin"), -1) e.ctx.GetSessionVars().StmtCtx.MemTracker = t + e.ctx.GetSessionVars().StmtCtx.DiskTracker = t2 return e } @@ -865,6 +869,7 @@ func defaultIndexJoinTestCase() *indexJoinTestCase { ctx.GetSessionVars().MaxChunkSize = variable.DefMaxChunkSize ctx.GetSessionVars().SnapshotTS = 1 ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(nil, -1) + ctx.GetSessionVars().StmtCtx.DiskTracker = disk.NewTracker(nil, -1) tc := &indexJoinTestCase{ outerRows: 100000, innerRows: variable.DefMaxChunkSize * 100, diff --git a/executor/builder.go b/executor/builder.go index f748fddd82041..48d5c21e40a83 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1245,9 +1245,9 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo e := &TableScanExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()), t: tb, - columns: v.Table.Columns, + columns: v.Columns, seekHandle: math.MinInt64, - isVirtualTable: tb.Type() == table.VirtualTable, + isVirtualTable: !tb.Type().IsNormalTable(), } return e } @@ -1452,7 +1452,6 @@ func (b *executorBuilder) buildAnalyzeIndexPushdown(task plannercore.AnalyzeInde concurrency: b.ctx.GetSessionVars().IndexSerialScanConcurrency, analyzePB: &tipb.AnalyzeReq{ Tp: tipb.AnalyzeType_TypeIndex, - StartTs: math.MaxUint64, Flags: sc.PushDownFlags(), TimeZoneOffset: offset, }, @@ -1520,7 +1519,6 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeCo concurrency: b.ctx.GetSessionVars().DistSQLScanConcurrency, analyzePB: &tipb.AnalyzeReq{ Tp: tipb.AnalyzeType_TypeColumn, - StartTs: math.MaxUint64, Flags: sc.PushDownFlags(), TimeZoneOffset: offset, }, @@ -1698,10 +1696,6 @@ func constructDistExec(sctx sessionctx.Context, plans []plannercore.PhysicalPlan func (b *executorBuilder) constructDAGReq(plans []plannercore.PhysicalPlan) (dagReq *tipb.DAGRequest, streaming bool, err error) { dagReq = &tipb.DAGRequest{} - dagReq.StartTs, err = b.getStartTS() - if err != nil { - return nil, false, err - } dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(b.ctx.GetSessionVars().Location()) sc := b.ctx.GetSessionVars().StmtCtx dagReq.Flags = sc.PushDownFlags() @@ -1935,9 +1929,14 @@ func buildNoRangeTableReader(b *executorBuilder, v *plannercore.PhysicalTableRea pt := tbl.(table.PartitionedTable) tbl = pt.GetPartition(physicalTableID) } + startTS, err := b.getStartTS() + if err != nil { + return nil, err + } e := &TableReaderExecutor{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()), dagPB: dagReq, + startTS: startTS, table: tbl, keepOrder: ts.KeepOrder, desc: ts.Desc, @@ -1997,9 +1996,14 @@ func buildNoRangeIndexReader(b *executorBuilder, v *plannercore.PhysicalIndexRea } else { physicalTableID = is.Table.ID } + startTS, err := b.getStartTS() + if err != nil { + return nil, err + } e := &IndexReaderExecutor{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()), dagPB: dagReq, + startTS: startTS, physicalTableID: physicalTableID, table: tbl, index: is.Index, @@ -2068,9 +2072,14 @@ func buildNoRangeIndexLookUpReader(b *executorBuilder, v *plannercore.PhysicalIn pt := tbl.(table.PartitionedTable) tbl = pt.GetPartition(physicalTableID) } + startTS, err := b.getStartTS() + if err != nil { + return nil, err + } e := &IndexLookUpExecutor{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()), dagPB: indexReq, + startTS: startTS, table: tbl, index: is.Index, keepOrder: is.KeepOrder, @@ -2199,11 +2208,16 @@ func (builder *dataReaderBuilder) buildTableReaderFromHandles(ctx context.Contex colExec := true e.dagPB.CollectExecutionSummaries = &colExec } + startTS, err := builder.getStartTS() + if err != nil { + return nil, err + } sort.Sort(sortutil.Int64Slice(handles)) var b distsql.RequestBuilder kvReq, err := b.SetTableHandles(getPhysicalTableID(e.table), handles). SetDAGRequest(e.dagPB). + SetStartTS(startTS). SetDesc(e.desc). SetKeepOrder(e.keepOrder). SetStreaming(e.streaming). @@ -2384,7 +2398,7 @@ func (b *executorBuilder) buildWindow(v *plannercore.PhysicalWindow) *WindowExec } return &WindowExec{baseExecutor: base, processor: processor, - groupChecker: newGroupChecker(b.ctx.GetSessionVars().StmtCtx, groupByItems), + groupChecker: newVecGroupChecker(b.ctx, groupByItems), numWindowFuncs: len(v.WindowFuncDescs), } } diff --git a/executor/checksum.go b/executor/checksum.go index c84579fe85ee8..5ff1b07841902 100644 --- a/executor/checksum.go +++ b/executor/checksum.go @@ -229,7 +229,6 @@ func (c *checksumContext) appendRequest(ctx sessionctx.Context, tableID int64, r func (c *checksumContext) buildTableRequest(ctx sessionctx.Context, tableID int64) (*kv.Request, error) { checksum := &tipb.ChecksumRequest{ - StartTs: c.StartTs, ScanOn: tipb.ChecksumScanOn_Table, Algorithm: tipb.ChecksumAlgorithm_Crc64_Xor, } @@ -239,13 +238,13 @@ func (c *checksumContext) buildTableRequest(ctx sessionctx.Context, tableID int6 var builder distsql.RequestBuilder return builder.SetTableRanges(tableID, ranges, nil). SetChecksumRequest(checksum). + SetStartTS(c.StartTs). SetConcurrency(ctx.GetSessionVars().DistSQLScanConcurrency). Build() } func (c *checksumContext) buildIndexRequest(ctx sessionctx.Context, tableID int64, indexInfo *model.IndexInfo) (*kv.Request, error) { checksum := &tipb.ChecksumRequest{ - StartTs: c.StartTs, ScanOn: tipb.ChecksumScanOn_Index, Algorithm: tipb.ChecksumAlgorithm_Crc64_Xor, } @@ -255,6 +254,7 @@ func (c *checksumContext) buildIndexRequest(ctx sessionctx.Context, tableID int6 var builder distsql.RequestBuilder return builder.SetIndexRanges(ctx.GetSessionVars().StmtCtx, tableID, indexInfo.ID, ranges). SetChecksumRequest(checksum). + SetStartTS(c.StartTs). SetConcurrency(ctx.GetSessionVars().DistSQLScanConcurrency). Build() } diff --git a/executor/coprocessor.go b/executor/coprocessor.go new file mode 100644 index 0000000000000..3c95ddf55158d --- /dev/null +++ b/executor/coprocessor.go @@ -0,0 +1,183 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "context" + + "github.com/gogo/protobuf/proto" + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/coprocessor" + "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/timeutil" + "github.com/pingcap/tipb/go-tipb" +) + +// CoprocessorDAGHandler uses to handle cop dag request. +type CoprocessorDAGHandler struct { + sctx sessionctx.Context + resp *coprocessor.Response + selResp *tipb.SelectResponse + dagReq *tipb.DAGRequest +} + +// NewCoprocessorDAGHandler creates a new CoprocessorDAGHandler. +func NewCoprocessorDAGHandler(sctx sessionctx.Context) *CoprocessorDAGHandler { + return &CoprocessorDAGHandler{ + sctx: sctx, + resp: &coprocessor.Response{}, + selResp: &tipb.SelectResponse{}, + } +} + +// HandleRequest handles the coprocessor request. +func (h *CoprocessorDAGHandler) HandleRequest(ctx context.Context, req *coprocessor.Request) *coprocessor.Response { + e, err := h.buildDAGExecutor(req) + if err != nil { + return h.buildResponse(err) + } + + err = e.Open(ctx) + if err != nil { + return h.buildResponse(err) + } + + chk := newFirstChunk(e) + tps := e.base().retFieldTypes + for { + chk.Reset() + err = Next(ctx, e, chk) + if err != nil { + break + } + if chk.NumRows() == 0 { + break + } + err = h.appendChunk(chk, tps) + if err != nil { + break + } + } + return h.buildResponse(err) +} + +func (h *CoprocessorDAGHandler) buildDAGExecutor(req *coprocessor.Request) (Executor, error) { + if req.GetTp() != kv.ReqTypeDAG { + return nil, errors.Errorf("unsupported request type %d", req.GetTp()) + } + dagReq := new(tipb.DAGRequest) + err := proto.Unmarshal(req.Data, dagReq) + if err != nil { + return nil, errors.Trace(err) + } + + stmtCtx := h.sctx.GetSessionVars().StmtCtx + stmtCtx.SetFlagsFromPBFlag(dagReq.Flags) + stmtCtx.TimeZone, err = timeutil.ConstructTimeZone(dagReq.TimeZoneName, int(dagReq.TimeZoneOffset)) + if err != nil { + return nil, errors.Trace(err) + } + h.dagReq = dagReq + is := h.sctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) + // Build physical plan. + bp := core.NewPBPlanBuilder(h.sctx, is) + plan, err := bp.Build(dagReq.Executors) + if err != nil { + return nil, errors.Trace(err) + } + // Build executor. + b := newExecutorBuilder(h.sctx, is) + return b.build(plan), nil +} + +func (h *CoprocessorDAGHandler) appendChunk(chk *chunk.Chunk, tps []*types.FieldType) error { + var err error + switch h.dagReq.EncodeType { + case tipb.EncodeType_TypeDefault: + err = h.encodeDefault(chk, tps) + case tipb.EncodeType_TypeChunk: + err = h.encodeChunk(chk, tps) + default: + return errors.Errorf("unknown DAG encode type: %v", h.dagReq.EncodeType) + } + return err +} + +func (h *CoprocessorDAGHandler) buildResponse(err error) *coprocessor.Response { + if err != nil { + h.resp.OtherError = err.Error() + return h.resp + } + h.selResp.EncodeType = h.dagReq.EncodeType + data, err := proto.Marshal(h.selResp) + if err != nil { + h.resp.OtherError = err.Error() + return h.resp + } + h.resp.Data = data + return h.resp +} + +func (h *CoprocessorDAGHandler) encodeChunk(chk *chunk.Chunk, colTypes []*types.FieldType) error { + colOrdinal := h.dagReq.OutputOffsets + chunks := h.selResp.Chunks + respColTypes := make([]*types.FieldType, 0, len(colOrdinal)) + for _, ordinal := range colOrdinal { + respColTypes = append(respColTypes, colTypes[ordinal]) + } + encoder := chunk.NewCodec(respColTypes) + chunks = append(chunks, tipb.Chunk{}) + cur := &chunks[len(chunks)-1] + cur.RowsData = append(cur.RowsData, encoder.Encode(chk)...) + h.selResp.Chunks = chunks + return nil +} + +func (h *CoprocessorDAGHandler) encodeDefault(chk *chunk.Chunk, tps []*types.FieldType) error { + colOrdinal := h.dagReq.OutputOffsets + chunks := h.selResp.Chunks + stmtCtx := h.sctx.GetSessionVars().StmtCtx + requestedRow := make([]byte, 0) + for i := 0; i < chk.NumRows(); i++ { + requestedRow = requestedRow[:0] + row := chk.GetRow(i) + for _, ordinal := range colOrdinal { + data, err := codec.EncodeValue(stmtCtx, nil, row.GetDatum(int(ordinal), tps[ordinal])) + if err != nil { + return err + } + requestedRow = append(requestedRow, data...) + } + chunks = h.appendRow(chunks, requestedRow, i) + } + h.selResp.Chunks = chunks + return nil +} + +const rowsPerChunk = 64 + +func (h *CoprocessorDAGHandler) appendRow(chunks []tipb.Chunk, data []byte, rowCnt int) []tipb.Chunk { + if rowCnt%rowsPerChunk == 0 { + chunks = append(chunks, tipb.Chunk{}) + } + cur := &chunks[len(chunks)-1] + cur.RowsData = append(cur.RowsData, data...) + return chunks +} diff --git a/executor/ddl.go b/executor/ddl.go index e7756c2d9c030..f897c060d90d4 100644 --- a/executor/ddl.go +++ b/executor/ddl.go @@ -16,6 +16,7 @@ package executor import ( "context" "fmt" + "strconv" "strings" "github.com/pingcap/errors" @@ -98,6 +99,8 @@ func (e *DDLExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { err = e.executeDropTableOrView(x) case *ast.RecoverTableStmt: err = e.executeRecoverTable(x) + case *ast.FlashBackTableStmt: + err = e.executeFlashbackTable(x) case *ast.RenameTableStmt: err = e.executeRenameTable(x) case *ast.TruncateTableStmt: @@ -108,6 +111,8 @@ func (e *DDLExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { err = e.executeUnlockTables(x) case *ast.CleanupTableLockStmt: err = e.executeCleanupTableLock(x) + case *ast.RepairTableStmt: + err = e.executeRepairTable(x) } if err != nil { @@ -328,23 +333,32 @@ func (e *DDLExec) executeRecoverTable(s *ast.RecoverTableStmt) error { if s.JobID != 0 { job, tblInfo, err = e.getRecoverTableByJobID(s, t, dom) } else { - job, tblInfo, err = e.getRecoverTableByTableName(s, t, dom) + job, tblInfo, err = e.getRecoverTableByTableName(s.Table, "") } if err != nil { return err } + autoID, err := e.getTableAutoIDFromSnapshot(job) + if err != nil { + return err + } + // Call DDL RecoverTable. + err = domain.GetDomain(e.ctx).DDL().RecoverTable(e.ctx, tblInfo, job.SchemaID, autoID, job.ID, job.StartTS) + return err +} + +func (e *DDLExec) getTableAutoIDFromSnapshot(job *model.Job) (int64, error) { // Get table original autoID before table drop. + dom := domain.GetDomain(e.ctx) m, err := dom.GetSnapshotMeta(job.StartTS) if err != nil { - return err + return 0, err } autoID, err := m.GetAutoTableID(job.SchemaID, job.TableID) if err != nil { - return errors.Errorf("recover table_id: %d, get original autoID from snapshot meta err: %s", job.TableID, err.Error()) + return 0, errors.Errorf("recover table_id: %d, get original autoID from snapshot meta err: %s", job.TableID, err.Error()) } - // Call DDL RecoverTable - err = domain.GetDomain(e.ctx).DDL().RecoverTable(e.ctx, tblInfo, job.SchemaID, autoID, job.ID, job.StartTS) - return err + return autoID, nil } func (e *DDLExec) getRecoverTableByJobID(s *ast.RecoverTableStmt, t *meta.Meta, dom *domain.Domain) (*model.Job, *model.TableInfo, error) { @@ -381,7 +395,12 @@ func (e *DDLExec) getRecoverTableByJobID(s *ast.RecoverTableStmt, t *meta.Meta, return job, table.Meta(), nil } -func (e *DDLExec) getRecoverTableByTableName(s *ast.RecoverTableStmt, t *meta.Meta, dom *domain.Domain) (*model.Job, *model.TableInfo, error) { +func (e *DDLExec) getRecoverTableByTableName(tableName *ast.TableName, ts string) (*model.Job, *model.TableInfo, error) { + txn, err := e.ctx.Txn(true) + if err != nil { + return nil, nil, err + } + t := meta.NewMeta(txn) jobs, err := t.GetAllHistoryDDLJobs() if err != nil { return nil, nil, err @@ -392,17 +411,18 @@ func (e *DDLExec) getRecoverTableByTableName(s *ast.RecoverTableStmt, t *meta.Me if err != nil { return nil, nil, err } - schemaName := s.Table.Schema.L + schemaName := tableName.Schema.L if schemaName == "" { schemaName = e.ctx.GetSessionVars().CurrentDB } if schemaName == "" { return nil, nil, errors.Trace(core.ErrNoDB) } + dom := domain.GetDomain(e.ctx) // TODO: only search recent `e.JobNum` DDL jobs. for i := len(jobs) - 1; i > 0; i-- { job = jobs[i] - if job.Type != model.ActionDropTable { + if job.Type != model.ActionDropTable && job.Type != model.ActionTruncateTable { continue } // Check GC safe point for getting snapshot infoSchema. @@ -410,6 +430,9 @@ func (e *DDLExec) getRecoverTableByTableName(s *ast.RecoverTableStmt, t *meta.Me if err != nil { return nil, nil, err } + if len(ts) != 0 && ts != model.TSConvert2Time(job.StartTS).String() { + continue + } // Get the snapshot infoSchema before drop table. snapInfo, err := dom.GetSnapshotInfoSchema(job.StartTS) if err != nil { @@ -423,7 +446,7 @@ func (e *DDLExec) getRecoverTableByTableName(s *ast.RecoverTableStmt, t *meta.Me fmt.Sprintf("(Table ID %d)", job.TableID), ) } - if table.Meta().Name.L == s.Table.Name.L { + if table.Meta().Name.L == tableName.Name.L { schema, ok := dom.InfoSchema().SchemaByID(job.SchemaID) if !ok { return nil, nil, infoschema.ErrDatabaseNotExists.GenWithStackByArgs( @@ -437,11 +460,39 @@ func (e *DDLExec) getRecoverTableByTableName(s *ast.RecoverTableStmt, t *meta.Me } } if tblInfo == nil { - return nil, nil, errors.Errorf("Can't found drop table: %v in ddl history jobs", s.Table.Name) + return nil, nil, errors.Errorf("Can't find dropped table: %v in ddl history jobs", tableName.Name) } return job, tblInfo, nil } +func (e *DDLExec) executeFlashbackTable(s *ast.FlashBackTableStmt) error { + ts := s.Timestamp.GetString() + if len(ts) == 0 { + return errors.Errorf("The timestamp in flashback statement should be consistent with the drop/truncate DDL start time") + } + job, tblInfo, err := e.getRecoverTableByTableName(s.Table, ts) + if err != nil { + return err + } + if len(s.NewName) != 0 { + tblInfo.Name = model.NewCIStr(s.NewName) + } + // Check the table ID was not exists. + is := domain.GetDomain(e.ctx).InfoSchema() + _, ok := is.TableByID(tblInfo.ID) + if ok { + return infoschema.ErrTableExists.GenWithStackByArgs("tableID:" + strconv.FormatInt(tblInfo.ID, 10)) + } + + autoID, err := e.getTableAutoIDFromSnapshot(job) + if err != nil { + return err + } + // Call DDL RecoverTable. + err = domain.GetDomain(e.ctx).DDL().RecoverTable(e.ctx, tblInfo, job.SchemaID, autoID, job.ID, job.StartTS) + return err +} + func (e *DDLExec) executeLockTables(s *ast.LockTablesStmt) error { if !config.TableLockEnabled() { return nil @@ -458,6 +509,9 @@ func (e *DDLExec) executeUnlockTables(s *ast.UnlockTablesStmt) error { } func (e *DDLExec) executeCleanupTableLock(s *ast.CleanupTableLockStmt) error { - err := domain.GetDomain(e.ctx).DDL().CleanupTableLock(e.ctx, s.Tables) - return err + return domain.GetDomain(e.ctx).DDL().CleanupTableLock(e.ctx, s.Tables) +} + +func (e *DDLExec) executeRepairTable(s *ast.RepairTableStmt) error { + return domain.GetDomain(e.ctx).DDL().RepairTable(e.ctx, s.Table, s.CreateStmt) } diff --git a/executor/distsql.go b/executor/distsql.go index b57403c4e3900..82e71a9b7c199 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -213,6 +213,7 @@ type IndexReaderExecutor struct { // kvRanges are only used for union scan. kvRanges []kv.KeyRange dagPB *tipb.DAGRequest + startTS uint64 // result returns one or more distsql.PartialResult and each PartialResult is returned by one region. result distsql.SelectResult @@ -292,6 +293,7 @@ func (e *IndexReaderExecutor) open(ctx context.Context, kvRanges []kv.KeyRange) var builder distsql.RequestBuilder kvReq, err := builder.SetKeyRanges(kvRanges). SetDAGRequest(e.dagPB). + SetStartTS(e.startTS). SetDesc(e.desc). SetKeepOrder(e.keepOrder). SetStreaming(e.streaming). @@ -321,6 +323,7 @@ type IndexLookUpExecutor struct { desc bool ranges []*ranger.Range dagPB *tipb.DAGRequest + startTS uint64 // handleIdx is the index of handle, which is only used for case of keeping order. handleIdx int tableRequest *tipb.DAGRequest @@ -438,6 +441,7 @@ func (e *IndexLookUpExecutor) startIndexWorker(ctx context.Context, kvRanges []k var builder distsql.RequestBuilder kvReq, err := builder.SetKeyRanges(kvRanges). SetDAGRequest(e.dagPB). + SetStartTS(e.startTS). SetDesc(e.desc). SetKeepOrder(e.keepOrder). SetStreaming(e.indexStreaming). @@ -528,6 +532,7 @@ func (e *IndexLookUpExecutor) buildTableReader(ctx context.Context, handles []in baseExecutor: newBaseExecutor(e.ctx, e.schema, stringutil.MemoizeStr(func() string { return e.id.String() + "_tableReader" })), table: e.table, dagPB: e.tableRequest, + startTS: e.startTS, columns: e.columns, streaming: e.tableStreaming, feedback: statistics.NewQueryFeedback(0, nil, 0, false), diff --git a/executor/errors.go b/executor/errors.go index a48152f0acdfe..cac87c2716872 100644 --- a/executor/errors.go +++ b/executor/errors.go @@ -18,26 +18,15 @@ import ( "github.com/pingcap/parser/terror" ) -// Error codes that are not mapping to mysql error codes. -const ( - codeUnknownPlan = iota - codePrepareMulti - codePrepareDDL - codeResultIsEmpty - codeErrBuildExec - codeBatchInsertFail - codeGetStartTS -) - // Error instances. var ( - ErrGetStartTS = terror.ClassExecutor.New(codeGetStartTS, "Can not get start ts") - ErrUnknownPlan = terror.ClassExecutor.New(codeUnknownPlan, "Unknown plan") - ErrPrepareMulti = terror.ClassExecutor.New(codePrepareMulti, "Can not prepare multiple statements") - ErrPrepareDDL = terror.ClassExecutor.New(codePrepareDDL, "Can not prepare DDL statements with parameters") - ErrResultIsEmpty = terror.ClassExecutor.New(codeResultIsEmpty, "result is empty") - ErrBuildExecutor = terror.ClassExecutor.New(codeErrBuildExec, "Failed to build executor") - ErrBatchInsertFail = terror.ClassExecutor.New(codeBatchInsertFail, "Batch insert failed, please clean the table and try again.") + ErrGetStartTS = terror.ClassExecutor.New(mysql.ErrGetStartTS, mysql.MySQLErrName[mysql.ErrGetStartTS]) + ErrUnknownPlan = terror.ClassExecutor.New(mysql.ErrUnknownPlan, mysql.MySQLErrName[mysql.ErrUnknownPlan]) + ErrPrepareMulti = terror.ClassExecutor.New(mysql.ErrPrepareMulti, mysql.MySQLErrName[mysql.ErrPrepareMulti]) + ErrPrepareDDL = terror.ClassExecutor.New(mysql.ErrPrepareDDL, mysql.MySQLErrName[mysql.ErrPrepareDDL]) + ErrResultIsEmpty = terror.ClassExecutor.New(mysql.ErrResultIsEmpty, mysql.MySQLErrName[mysql.ErrResultIsEmpty]) + ErrBuildExecutor = terror.ClassExecutor.New(mysql.ErrBuildExecutor, mysql.MySQLErrName[mysql.ErrBuildExecutor]) + ErrBatchInsertFail = terror.ClassExecutor.New(mysql.ErrBatchInsertFail, mysql.MySQLErrName[mysql.ErrBatchInsertFail]) ErrCantCreateUserWithGrant = terror.ClassExecutor.New(mysql.ErrCantCreateUserWithGrant, mysql.MySQLErrName[mysql.ErrCantCreateUserWithGrant]) ErrPasswordNoMatch = terror.ClassExecutor.New(mysql.ErrPasswordNoMatch, mysql.MySQLErrName[mysql.ErrPasswordNoMatch]) @@ -58,9 +47,17 @@ var ( func init() { // Map error codes to mysql error codes. tableMySQLErrCodes := map[terror.ErrCode]uint16{ + mysql.ErrGetStartTS: mysql.ErrGetStartTS, + mysql.ErrUnknownPlan: mysql.ErrUnknownPlan, + mysql.ErrPrepareMulti: mysql.ErrPrepareMulti, + mysql.ErrPrepareDDL: mysql.ErrPrepareDDL, + mysql.ErrResultIsEmpty: mysql.ErrResultIsEmpty, + mysql.ErrBuildExecutor: mysql.ErrBuildExecutor, + mysql.ErrBatchInsertFail: mysql.ErrBatchInsertFail, + + mysql.ErrCantCreateUserWithGrant: mysql.ErrCantCreateUserWithGrant, mysql.ErrPasswordNoMatch: mysql.ErrPasswordNoMatch, mysql.ErrCannotUser: mysql.ErrCannotUser, - mysql.ErrWrongValueCountOnRow: mysql.ErrWrongValueCountOnRow, mysql.ErrPasswordFormat: mysql.ErrPasswordFormat, mysql.ErrCantChangeTxCharacteristics: mysql.ErrCantChangeTxCharacteristics, mysql.ErrPsManyParam: mysql.ErrPsManyParam, @@ -69,8 +66,10 @@ func init() { mysql.ErrTableaccessDenied: mysql.ErrTableaccessDenied, mysql.ErrBadDB: mysql.ErrBadDB, mysql.ErrWrongObject: mysql.ErrWrongObject, + mysql.ErrRoleNotGranted: mysql.ErrRoleNotGranted, mysql.ErrLockDeadlock: mysql.ErrLockDeadlock, mysql.ErrQueryInterrupted: mysql.ErrQueryInterrupted, + mysql.ErrWrongValueCountOnRow: mysql.ErrWrongValueCountOnRow, } terror.ErrClassToMySQLCodes[terror.ClassExecutor] = tableMySQLErrCodes } diff --git a/executor/executor.go b/executor/executor.go index ff4acd00a981d..6a10c3620642d 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -46,6 +46,7 @@ import ( "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/admin" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" @@ -1493,9 +1494,10 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { memQuota = stmtHints.MemQuotaQuery } sc := &stmtctx.StatementContext{ - StmtHints: stmtHints, - TimeZone: vars.Location(), - MemTracker: memory.NewTracker(stringutil.MemoizeStr(s.Text), memQuota), + StmtHints: stmtHints, + TimeZone: vars.Location(), + MemTracker: memory.NewTracker(stringutil.MemoizeStr(s.Text), memQuota), + DiskTracker: disk.NewTracker(stringutil.MemoizeStr(s.Text), -1), } switch config.GetGlobalConfig().OOMAction { case config.OOMActionCancel: diff --git a/executor/executor_required_rows_test.go b/executor/executor_required_rows_test.go index 348b043f883de..76dcb742c1796 100644 --- a/executor/executor_required_rows_test.go +++ b/executor/executor_required_rows_test.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/mock" ) @@ -206,6 +207,7 @@ func defaultCtx() sessionctx.Context { ctx.GetSessionVars().MaxChunkSize = variable.DefMaxChunkSize ctx.GetSessionVars().MemQuotaSort = variable.DefTiDBMemQuotaSort ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(nil, ctx.GetSessionVars().MemQuotaQuery) + ctx.GetSessionVars().StmtCtx.DiskTracker = disk.NewTracker(nil, -1) ctx.GetSessionVars().SnapshotTS = uint64(1) return ctx } diff --git a/executor/executor_test.go b/executor/executor_test.go index 7fc12e6c08b7b..6294973172c02 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -115,7 +115,7 @@ var _ = Suite(&testUpdateSuite{}) var _ = Suite(&testOOMSuite{}) var _ = Suite(&testPointGetSuite{}) var _ = Suite(&testBatchPointGetSuite{}) -var _ = Suite(&testRecoverTable{}) +var _ = SerialSuites(&testRecoverTable{}) var _ = Suite(&testFlushSuite{}) type testSuite struct{ *baseTestSuite } @@ -4672,6 +4672,131 @@ func (s *testRecoverTable) TestRecoverTable(c *C) { c.Assert(gcEnable, Equals, false) } +func (s *testRecoverTable) TestFlashbackTable(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil) + }() + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test_flashback") + tk.MustExec("use test_flashback") + tk.MustExec("drop table if exists t_flashback") + tk.MustExec("create table t_flashback (a int);") + defer func(originGC bool) { + if originGC { + ddl.EmulatorGCEnable() + } else { + ddl.EmulatorGCDisable() + } + }(ddl.IsEmulatorGCEnable()) + + // Disable emulator GC. + // Otherwise emulator GC will delete table record as soon as possible after execute drop table ddl. + ddl.EmulatorGCDisable() + gcTimeFormat := "20060102-15:04:05 -0700 MST" + timeBeforeDrop := time.Now().Add(0 - time.Duration(48*60*60*time.Second)).Format(gcTimeFormat) + safePointSQL := `INSERT HIGH_PRIORITY INTO mysql.tidb VALUES ('tikv_gc_safe_point', '%[1]s', '') + ON DUPLICATE KEY + UPDATE variable_value = '%[1]s'` + // Clear GC variables first. + tk.MustExec("delete from mysql.tidb where variable_name in ( 'tikv_gc_safe_point','tikv_gc_enable' )") + // Set GC safe point + tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop)) + // Set GC enable. + err := gcutil.EnableGC(tk.Se) + c.Assert(err, IsNil) + + tk.MustExec("insert into t_flashback values (1),(2),(3)") + tk.MustExec("drop table t_flashback") + + // Test flash table with wrong time. + _, err = tk.Exec(fmt.Sprintf("flashback table t_flashback until timestamp '%v'", time.Now().String())) + c.Assert(err.Error(), Equals, "Can't find dropped table: t_flashback in ddl history jobs") + + // Test flashback table failed by there is already a new table with the same name. + ts := getDDLJobStartTime(tk, "test_flashback", "t_flashback") + // If there is a new table with the same name, should return failed. + tk.MustExec("create table t_flashback (a int);") + _, err = tk.Exec(fmt.Sprintf("flashback table t_flashback until timestamp '%v'", ts)) + c.Assert(err.Error(), Equals, infoschema.ErrTableExists.GenWithStackByArgs("t_flashback").Error()) + + // Drop the new table with the same name, then flashback table. + tk.MustExec("drop table t_flashback") + + // Test for flashback table. + tk.MustExec(fmt.Sprintf("flashback table t_flashback until timestamp '%v'", ts)) + // Check flashback table meta and data record. + tk.MustQuery("select * from t_flashback;").Check(testkit.Rows("1", "2", "3")) + // Check flashback table autoID. + tk.MustExec("insert into t_flashback values (4),(5),(6)") + tk.MustQuery("select * from t_flashback;").Check(testkit.Rows("1", "2", "3", "4", "5", "6")) + // Check rebase auto id. + tk.MustQuery("select a,_tidb_rowid from t_flashback;").Check(testkit.Rows("1 1", "2 2", "3 3", "4 5001", "5 5002", "6 5003")) + + // Test for flashback to new table. + tk.MustExec("drop table t_flashback") + ts = getDDLJobStartTime(tk, "test_flashback", "t_flashback") + tk.MustExec("create table t_flashback (a int);") + tk.MustExec(fmt.Sprintf("flashback table t_flashback until timestamp '%v' to t_flashback2", ts)) + // Check flashback table meta and data record. + tk.MustQuery("select * from t_flashback2;").Check(testkit.Rows("1", "2", "3", "4", "5", "6")) + // Check flashback table autoID. + tk.MustExec("insert into t_flashback2 values (7),(8),(9)") + tk.MustQuery("select * from t_flashback2;").Check(testkit.Rows("1", "2", "3", "4", "5", "6", "7", "8", "9")) + // Check rebase auto id. + tk.MustQuery("select a,_tidb_rowid from t_flashback2;").Check(testkit.Rows("1 1", "2 2", "3 3", "4 5001", "5 5002", "6 5003", "7 10001", "8 10002", "9 10003")) + + // Test for flashback one table multiple time. + _, err = tk.Exec(fmt.Sprintf("flashback table t_flashback until timestamp '%v' to t_flashback4", ts)) + c.Assert(infoschema.ErrTableExists.Equal(err), IsTrue) + + // Test for flashback truncated table to new table. + tk.MustExec("truncate table t_flashback2") + ts = getDDLJobStartTime(tk, "test_flashback", "t_flashback2") + tk.MustExec(fmt.Sprintf("flashback table t_flashback2 until timestamp '%v' to t_flashback3", ts)) + // Check flashback table meta and data record. + tk.MustQuery("select * from t_flashback3;").Check(testkit.Rows("1", "2", "3", "4", "5", "6", "7", "8", "9")) + // Check flashback table autoID. + tk.MustExec("insert into t_flashback3 values (10),(11)") + tk.MustQuery("select * from t_flashback3;").Check(testkit.Rows("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11")) + // Check rebase auto id. + tk.MustQuery("select a,_tidb_rowid from t_flashback3;").Check(testkit.Rows("1 1", "2 2", "3 3", "4 5001", "5 5002", "6 5003", "7 10001", "8 10002", "9 10003", "10 15001", "11 15002")) + + // Test for flashback drop partition table. + tk.MustExec("drop table if exists t_p_flashback") + tk.MustExec("create table t_p_flashback (a int) partition by hash(a) partitions 4;") + tk.MustExec("insert into t_p_flashback values (1),(2),(3)") + tk.MustExec("drop table t_p_flashback") + ts = getDDLJobStartTime(tk, "test_flashback", "t_p_flashback") + tk.MustExec(fmt.Sprintf("flashback table t_p_flashback until timestamp '%v'", ts)) + // Check flashback table meta and data record. + tk.MustQuery("select * from t_p_flashback order by a;").Check(testkit.Rows("1", "2", "3")) + // Check flashback table autoID. + tk.MustExec("insert into t_p_flashback values (4),(5)") + tk.MustQuery("select a,_tidb_rowid from t_p_flashback order by a;").Check(testkit.Rows("1 1", "2 2", "3 3", "4 5001", "5 5002")) + + // Test for flashback truncate partition table. + tk.MustExec("truncate table t_p_flashback") + ts = getDDLJobStartTime(tk, "test_flashback", "t_p_flashback") + tk.MustExec(fmt.Sprintf("flashback table t_p_flashback until timestamp '%v' to t_p_flashback1", ts)) + // Check flashback table meta and data record. + tk.MustQuery("select * from t_p_flashback1 order by a;").Check(testkit.Rows("1", "2", "3", "4", "5")) + // Check flashback table autoID. + tk.MustExec("insert into t_p_flashback1 values (6)") + tk.MustQuery("select a,_tidb_rowid from t_p_flashback1 order by a;").Check(testkit.Rows("1 1", "2 2", "3 3", "4 5001", "5 5002", "6 10001")) +} + +func getDDLJobStartTime(tk *testkit.TestKit, dbName, tblName string) string { + re := tk.MustQuery("admin show ddl jobs 100") + rows := re.Rows() + for _, row := range rows { + if row[1] == dbName && row[2] == tblName && (row[3] == "drop table" || row[3] == "truncate table") { + return row[8].(string) + } + } + return "" +} + func (s *testSuiteP2) TestPointGetPreparedPlan(c *C) { tk1 := testkit.NewTestKit(c, s.store) tk1.MustExec("drop database if exists ps_text") diff --git a/executor/hash_table.go b/executor/hash_table.go index 77e11576004c3..c453e781efd21 100644 --- a/executor/hash_table.go +++ b/executor/hash_table.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" + "github.com/pingcap/tidb/util/stringutil" "go.uber.org/zap" ) @@ -88,7 +89,8 @@ type hashRowContainer struct { // memTracker is the reference of records.GetMemTracker(). // records would be set to nil for garbage collection when spilling is activated // so we need this reference. - memTracker *memory.Tracker + memTracker *memory.Tracker + diskTracker *disk.Tracker // records stores the chunks in memory. records *chunk.List @@ -122,9 +124,10 @@ func newHashRowContainer(sCtx sessionctx.Context, estCount int, hCtx *hashContex sc: sCtx.GetSessionVars().StmtCtx, hCtx: hCtx, - hashTable: newRowHashMap(estCount), - memTracker: initList.GetMemTracker(), - records: initList, + hashTable: newRowHashMap(estCount), + memTracker: initList.GetMemTracker(), + diskTracker: disk.NewTracker(stringutil.StringerStr("hashRowContainer"), -1), + records: initList, } return c @@ -174,6 +177,7 @@ func (c *hashRowContainer) matchJoinKey(buildRow, probeRow chunk.Row, probeHCtx func (c *hashRowContainer) spillToDisk() (err error) { N := c.records.NumChunks() c.recordsInDisk = chunk.NewListInDisk(c.hCtx.allTypes) + c.recordsInDisk.GetDiskTracker().AttachTo(c.diskTracker) for i := 0; i < N; i++ { chk := c.records.GetChunk(i) err = c.recordsInDisk.Add(chk) @@ -271,7 +275,7 @@ func (c *hashRowContainer) Close() error { func (c *hashRowContainer) GetMemTracker() *memory.Tracker { return c.memTracker } // GetDiskTracker returns the underlying disk usage tracker in hashRowContainer. -func (c *hashRowContainer) GetDiskTracker() *disk.Tracker { return c.recordsInDisk.GetDiskTracker() } +func (c *hashRowContainer) GetDiskTracker() *disk.Tracker { return c.diskTracker } // ActionSpill returns a memory.ActionOnExceed for spilling over to disk. func (c *hashRowContainer) ActionSpill() memory.ActionOnExceed { diff --git a/executor/insert_common.go b/executor/insert_common.go index b9dca684c639a..69e4001359190 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -393,7 +393,7 @@ func insertRowsFromSelect(ctx context.Context, base insertCommon) error { } for innerChunkRow := iter.Begin(); innerChunkRow != iter.End(); innerChunkRow = iter.Next() { - innerRow := types.CloneRow(innerChunkRow.GetDatumRow(fields)) + innerRow := innerChunkRow.GetDatumRow(fields) e.rowCount++ row, err := e.getRow(ctx, innerRow) if err != nil { @@ -410,8 +410,14 @@ func insertRowsFromSelect(ctx context.Context, base insertCommon) error { } } } + + err = base.exec(ctx, rows) + if err != nil { + return err + } + rows = rows[:0] } - return base.exec(ctx, rows) + return nil } func (e *InsertValues) doBatchInsert(ctx context.Context) error { diff --git a/executor/join.go b/executor/join.go index d9d7c921dd226..8e6c95f339b2d 100644 --- a/executor/join.go +++ b/executor/join.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/util/bitmap" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/stringutil" ) @@ -70,6 +71,7 @@ type HashJoinExec struct { joinResultCh chan *hashjoinWorkerResult memTracker *memory.Tracker // track memory usage. + diskTracker *disk.Tracker // track disk usage. prepared bool isOuterJoin bool @@ -145,6 +147,9 @@ func (e *HashJoinExec) Open(ctx context.Context) error { e.memTracker = memory.NewTracker(e.id, -1) e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) + e.diskTracker = disk.NewTracker(e.id, -1) + e.diskTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.DiskTracker) + e.closeCh = make(chan struct{}) e.finished.Store(false) e.joinWorkerWaitGroup = sync.WaitGroup{} @@ -677,6 +682,8 @@ func (e *HashJoinExec) buildHashTableForList(buildSideResultCh <-chan *chunk.Chu e.rowContainer = newHashRowContainer(e.ctx, int(e.buildSideEstCount), hCtx) e.rowContainer.GetMemTracker().AttachTo(e.memTracker) e.rowContainer.GetMemTracker().SetLabel(buildSideResultLabel) + e.rowContainer.GetDiskTracker().AttachTo(e.diskTracker) + e.rowContainer.GetDiskTracker().SetLabel(buildSideResultLabel) if config.GetGlobalConfig().OOMUseTmpStorage { actionSpill := e.rowContainer.ActionSpill() e.ctx.GetSessionVars().StmtCtx.MemTracker.FallbackOldAndSetNewAction(actionSpill) diff --git a/executor/seqtest/seq_executor_test.go b/executor/seqtest/seq_executor_test.go index 72815eb039d87..cf85e0c17167c 100644 --- a/executor/seqtest/seq_executor_test.go +++ b/executor/seqtest/seq_executor_test.go @@ -755,6 +755,43 @@ func (s *seqTestSuite) TestAdminShowNextID(c *C) { r.Check(testkit.Rows("test1 tt id 41")) } +func (s *seqTestSuite) TestNoHistoryWhenDisableRetry(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists history") + tk.MustExec("create table history (a int)") + tk.MustExec("set @@autocommit = 0") + + // retry_limit = 0 will not add history. + tk.MustExec("set @@tidb_retry_limit = 0") + tk.MustExec("insert history values (1)") + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 0) + + // Disable auto_retry will add history for auto committed only + tk.MustExec("set @@autocommit = 1") + tk.MustExec("set @@tidb_retry_limit = 10") + tk.MustExec("set @@tidb_disable_txn_auto_retry = 1") + c.Assert(failpoint.Enable("github.com/pingcap/tidb/session/keepHistory", `return(true)`), IsNil) + tk.MustExec("insert history values (1)") + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 1) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/session/keepHistory"), IsNil) + tk.MustExec("begin") + tk.MustExec("insert history values (1)") + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 0) + tk.MustExec("commit") + + // Enable auto_retry will add history for both. + tk.MustExec("set @@tidb_disable_txn_auto_retry = 0") + c.Assert(failpoint.Enable("github.com/pingcap/tidb/session/keepHistory", `return(true)`), IsNil) + tk.MustExec("insert history values (1)") + c.Assert(failpoint.Disable("github.com/pingcap/tidb/session/keepHistory"), IsNil) + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 1) + tk.MustExec("begin") + tk.MustExec("insert history values (1)") + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 2) + tk.MustExec("commit") +} + func (s *seqTestSuite) TestPrepareMaxParamCountCheck(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") diff --git a/executor/simple.go b/executor/simple.go index bafd310d5ffe5..5952bf2491524 100644 --- a/executor/simple.go +++ b/executor/simple.go @@ -135,7 +135,12 @@ func (e *SimpleExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { } func (e *SimpleExec) setDefaultRoleNone(s *ast.SetDefaultRoleStmt) error { - sqlExecutor := e.ctx.(sqlexec.SQLExecutor) + restrictedCtx, err := e.getSysSession() + if err != nil { + return err + } + defer e.releaseSysSession(restrictedCtx) + sqlExecutor := restrictedCtx.(sqlexec.SQLExecutor) if _, err := sqlExecutor.Execute(context.Background(), "begin"); err != nil { return err } @@ -177,7 +182,13 @@ func (e *SimpleExec) setDefaultRoleRegular(s *ast.SetDefaultRoleStmt) error { return ErrCannotUser.GenWithStackByArgs("SET DEFAULT ROLE", role.String()) } } - sqlExecutor := e.ctx.(sqlexec.SQLExecutor) + + restrictedCtx, err := e.getSysSession() + if err != nil { + return err + } + defer e.releaseSysSession(restrictedCtx) + sqlExecutor := restrictedCtx.(sqlexec.SQLExecutor) if _, err := sqlExecutor.Execute(context.Background(), "begin"); err != nil { return err } @@ -291,7 +302,6 @@ func (e *SimpleExec) setDefaultRoleForCurrentUser(s *ast.SetDefaultRoleStmt) (er return err } defer e.releaseSysSession(restrictedCtx) - sqlExecutor := restrictedCtx.(sqlexec.SQLExecutor) if _, err := sqlExecutor.Execute(context.Background(), "begin"); err != nil { @@ -554,8 +564,15 @@ func (e *SimpleExec) executeRevokeRole(s *ast.RevokeRoleStmt) error { } } + restrictedCtx, err := e.getSysSession() + if err != nil { + return err + } + defer e.releaseSysSession(restrictedCtx) + sqlExecutor := restrictedCtx.(sqlexec.SQLExecutor) + // begin a transaction to insert role graph edges. - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), "begin"); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), "begin"); err != nil { return errors.Trace(err) } for _, user := range s.Users { @@ -564,7 +581,7 @@ func (e *SimpleExec) executeRevokeRole(s *ast.RevokeRoleStmt) error { return errors.Trace(err) } if !exists { - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), "rollback"); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), "rollback"); err != nil { return errors.Trace(err) } return ErrCannotUser.GenWithStackByArgs("REVOKE ROLE", user.String()) @@ -574,22 +591,22 @@ func (e *SimpleExec) executeRevokeRole(s *ast.RevokeRoleStmt) error { role.Hostname = "%" } sql := fmt.Sprintf(`DELETE IGNORE FROM %s.%s WHERE FROM_HOST='%s' and FROM_USER='%s' and TO_HOST='%s' and TO_USER='%s'`, mysql.SystemDB, mysql.RoleEdgeTable, role.Hostname, role.Username, user.Hostname, user.Username) - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), sql); err != nil { - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), "rollback"); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), sql); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), "rollback"); err != nil { return errors.Trace(err) } return ErrCannotUser.GenWithStackByArgs("REVOKE ROLE", role.String()) } sql = fmt.Sprintf(`DELETE IGNORE FROM %s.%s WHERE DEFAULT_ROLE_HOST='%s' and DEFAULT_ROLE_USER='%s' and HOST='%s' and USER='%s'`, mysql.SystemDB, mysql.DefaultRoleTable, role.Hostname, role.Username, user.Hostname, user.Username) - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), sql); err != nil { - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), "rollback"); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), sql); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), "rollback"); err != nil { return errors.Trace(err) } return ErrCannotUser.GenWithStackByArgs("REVOKE ROLE", role.String()) } } } - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), "commit"); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), "commit"); err != nil { return err } domain.GetDomain(e.ctx).NotifyUpdatePrivilege(e.ctx) @@ -770,29 +787,36 @@ func (e *SimpleExec) executeGrantRole(s *ast.GrantRoleStmt) error { } } + restrictedCtx, err := e.getSysSession() + if err != nil { + return err + } + defer e.releaseSysSession(restrictedCtx) + sqlExecutor := restrictedCtx.(sqlexec.SQLExecutor) + // begin a transaction to insert role graph edges. - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), "begin"); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), "begin"); err != nil { return err } for _, user := range s.Users { for _, role := range s.Roles { sql := fmt.Sprintf(`INSERT IGNORE INTO %s.%s (FROM_HOST, FROM_USER, TO_HOST, TO_USER) VALUES ('%s','%s','%s','%s')`, mysql.SystemDB, mysql.RoleEdgeTable, role.Hostname, role.Username, user.Hostname, user.Username) - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), sql); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), sql); err != nil { failedUsers = append(failedUsers, user.String()) logutil.BgLogger().Error(fmt.Sprintf("Error occur when executing %s", sql)) - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), "rollback"); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), "rollback"); err != nil { return err } return ErrCannotUser.GenWithStackByArgs("GRANT ROLE", user.String()) } } } - if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), "commit"); err != nil { + if _, err := sqlExecutor.Execute(context.Background(), "commit"); err != nil { return err } - err := domain.GetDomain(e.ctx).PrivilegeHandle().Update(e.ctx.(sessionctx.Context)) - return err + domain.GetDomain(e.ctx).NotifyUpdatePrivilege(e.ctx) + return nil } func (e *SimpleExec) executeDropUser(s *ast.DropUserStmt) error { diff --git a/executor/statement_context_test.go b/executor/statement_context_test.go index 46c73907ec4bb..3bb049c96889c 100644 --- a/executor/statement_context_test.go +++ b/executor/statement_context_test.go @@ -72,7 +72,7 @@ func (s *testSuite1) TestStatementContext(c *C) { tk.MustExec(strictModeSQL) _, err = tk.Exec("insert sc2 values (unhex('4040ffff'))") c.Assert(err, NotNil) - c.Assert(terror.ErrorEqual(err, table.ErrTruncateWrongValue), IsTrue, Commentf("err %v", err)) + c.Assert(terror.ErrorEqual(err, table.ErrTruncatedWrongValueForField), IsTrue, Commentf("err %v", err)) tk.MustExec("set @@tidb_skip_utf8_check = '1'") _, err = tk.Exec("insert sc2 values (unhex('4040ffff'))") @@ -98,10 +98,10 @@ func (s *testSuite1) TestStatementContext(c *C) { tk.MustExec(strictModeSQL) _, err = tk.Exec("insert t1 values (unhex('f09f8c80'))") c.Assert(err, NotNil) - c.Assert(terror.ErrorEqual(err, table.ErrTruncateWrongValue), IsTrue, Commentf("err %v", err)) + c.Assert(terror.ErrorEqual(err, table.ErrTruncatedWrongValueForField), IsTrue, Commentf("err %v", err)) _, err = tk.Exec("insert t1 values (unhex('F0A48BAE'))") c.Assert(err, NotNil) - c.Assert(terror.ErrorEqual(err, table.ErrTruncateWrongValue), IsTrue, Commentf("err %v", err)) + c.Assert(terror.ErrorEqual(err, table.ErrTruncatedWrongValueForField), IsTrue, Commentf("err %v", err)) old := config.GetGlobalConfig() conf := *old conf.CheckMb4ValueInUTF8 = false diff --git a/executor/table_reader.go b/executor/table_reader.go index 3b45dd9be5efc..4b7a0559b9451 100644 --- a/executor/table_reader.go +++ b/executor/table_reader.go @@ -61,6 +61,7 @@ type TableReaderExecutor struct { // kvRanges are only use for union scan. kvRanges []kv.KeyRange dagPB *tipb.DAGRequest + startTS uint64 // columns are only required by union scan and virtual column. columns []*model.ColumnInfo @@ -207,6 +208,7 @@ func (e *TableReaderExecutor) buildResp(ctx context.Context, ranges []*ranger.Ra var builder distsql.RequestBuilder kvReq, err := builder.SetTableRanges(getPhysicalTableID(e.table), ranges, e.feedback). SetDAGRequest(e.dagPB). + SetStartTS(e.startTS). SetDesc(e.desc). SetKeepOrder(e.keepOrder). SetStreaming(e.streaming). diff --git a/executor/table_readers_required_rows_test.go b/executor/table_readers_required_rows_test.go index a34b36f80c40d..9e5aab6b53fa1 100644 --- a/executor/table_readers_required_rows_test.go +++ b/executor/table_readers_required_rows_test.go @@ -115,7 +115,7 @@ func mockSelectResult(ctx context.Context, sctx sessionctx.Context, kvReq *kv.Re func buildTableReader(sctx sessionctx.Context) Executor { e := &TableReaderExecutor{ baseExecutor: buildMockBaseExec(sctx), - table: &tables.Table{}, + table: &tables.TableCommon{}, dagPB: buildMockDAGRequest(sctx), selectResultHook: selectResultHook{mockSelectResult}, } diff --git a/executor/window.go b/executor/window.go index 12cfb1797ee81..f99df0afc0dd3 100644 --- a/executor/window.go +++ b/executor/window.go @@ -30,9 +30,9 @@ import ( type WindowExec struct { baseExecutor - groupChecker *groupChecker - // inputIter is the iterator of child chunks - inputIter *chunk.Iterator4Chunk + groupChecker *vecGroupChecker + // childResult stores the child chunk + childResult *chunk.Chunk // executed indicates the child executor is drained or something unexpected happened. executed bool // resultChunks stores the chunks to return @@ -74,8 +74,8 @@ func (e *WindowExec) preparedChunkAvailable() bool { func (e *WindowExec) consumeOneGroup(ctx context.Context) error { var groupRows []chunk.Row - for { - eof, err := e.fetchChildIfNecessary(ctx) + if e.groupChecker.isExhausted() { + eof, err := e.fetchChild(ctx) if err != nil { return errors.Trace(err) } @@ -83,17 +83,41 @@ func (e *WindowExec) consumeOneGroup(ctx context.Context) error { e.executed = true return e.consumeGroupRows(groupRows) } - for inputRow := e.inputIter.Current(); inputRow != e.inputIter.End(); inputRow = e.inputIter.Next() { - meetNewGroup, err := e.groupChecker.meetNewGroup(inputRow) - if err != nil { - return errors.Trace(err) - } - if meetNewGroup { - return e.consumeGroupRows(groupRows) + _, err = e.groupChecker.splitIntoGroups(e.childResult) + if err != nil { + return errors.Trace(err) + } + } + begin, end := e.groupChecker.getNextGroup() + for i := begin; i < end; i++ { + groupRows = append(groupRows, e.childResult.GetRow(i)) + } + + for meetLastGroup := end == e.childResult.NumRows(); meetLastGroup; { + meetLastGroup = false + eof, err := e.fetchChild(ctx) + if err != nil { + return errors.Trace(err) + } + if eof { + e.executed = true + return e.consumeGroupRows(groupRows) + } + + isFirstGroupSameAsPrev, err := e.groupChecker.splitIntoGroups(e.childResult) + if err != nil { + return errors.Trace(err) + } + + if isFirstGroupSameAsPrev { + begin, end = e.groupChecker.getNextGroup() + for i := begin; i < end; i++ { + groupRows = append(groupRows, e.childResult.GetRow(i)) } - groupRows = append(groupRows, inputRow) + meetLastGroup = end == e.childResult.NumRows() } } + return e.consumeGroupRows(groupRows) } func (e *WindowExec) consumeGroupRows(groupRows []chunk.Row) (err error) { @@ -125,11 +149,7 @@ func (e *WindowExec) consumeGroupRows(groupRows []chunk.Row) (err error) { return nil } -func (e *WindowExec) fetchChildIfNecessary(ctx context.Context) (EOF bool, err error) { - if e.inputIter != nil && e.inputIter.Current() != e.inputIter.End() { - return false, nil - } - +func (e *WindowExec) fetchChild(ctx context.Context) (EOF bool, err error) { childResult := newFirstChunk(e.children[0]) err = Next(ctx, e.children[0], childResult) if err != nil { @@ -149,8 +169,7 @@ func (e *WindowExec) fetchChildIfNecessary(ctx context.Context) (EOF bool, err e e.resultChunks = append(e.resultChunks, resultChk) e.remainingRowsInChunk = append(e.remainingRowsInChunk, numRows) - e.inputIter = chunk.NewIterator4Chunk(childResult) - e.inputIter.Begin() + e.childResult = childResult return false, nil } diff --git a/expression/aggregation/agg_to_pb.go b/expression/aggregation/agg_to_pb.go index 59d09db237701..9fc89f19fd3e5 100644 --- a/expression/aggregation/agg_to_pb.go +++ b/expression/aggregation/agg_to_pb.go @@ -14,10 +14,12 @@ package aggregation import ( + "github.com/pingcap/errors" "github.com/pingcap/parser/ast" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/types" "github.com/pingcap/tipb/go-tipb" ) @@ -64,3 +66,47 @@ func AggFuncToPBExpr(sc *stmtctx.StatementContext, client kv.Client, aggFunc *Ag } return &tipb.Expr{Tp: tp, Children: children, FieldType: expression.ToPBFieldType(aggFunc.RetTp)} } + +// PBExprToAggFuncDesc converts pb to aggregate function. +func PBExprToAggFuncDesc(sc *stmtctx.StatementContext, aggFunc *tipb.Expr, fieldTps []*types.FieldType) (*AggFuncDesc, error) { + var name string + switch aggFunc.Tp { + case tipb.ExprType_Count: + name = ast.AggFuncCount + case tipb.ExprType_First: + name = ast.AggFuncFirstRow + case tipb.ExprType_GroupConcat: + name = ast.AggFuncGroupConcat + case tipb.ExprType_Max: + name = ast.AggFuncMax + case tipb.ExprType_Min: + name = ast.AggFuncMin + case tipb.ExprType_Sum: + name = ast.AggFuncSum + case tipb.ExprType_Avg: + name = ast.AggFuncAvg + case tipb.ExprType_Agg_BitOr: + name = ast.AggFuncBitOr + case tipb.ExprType_Agg_BitXor: + name = ast.AggFuncBitXor + case tipb.ExprType_Agg_BitAnd: + name = ast.AggFuncBitAnd + default: + return nil, errors.Errorf("unknown aggregation function type: %v", aggFunc.Tp) + } + + args, err := expression.PBToExprs(aggFunc.Children, fieldTps, sc) + if err != nil { + return nil, err + } + base := baseFuncDesc{ + Name: name, + Args: args, + RetTp: expression.FieldTypeFromPB(aggFunc.FieldType), + } + return &AggFuncDesc{ + baseFuncDesc: base, + Mode: Partial1Mode, + HasDistinct: false, + }, nil +} diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index ff556cfc09a3d..955c9f1a92364 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -1458,6 +1458,9 @@ func (b *builtinCastDurationAsDecimalSig) evalDecimal(row chunk.Row) (res *types if isNull || err != nil { return res, isNull, err } + if val.Fsp, err = types.CheckFsp(int(val.Fsp)); err != nil { + return res, false, err + } sc := b.ctx.GetSessionVars().StmtCtx res, err = types.ProduceDecWithSpecifiedTp(val.ToNumber(), b.tp, sc) return res, false, err diff --git a/expression/builtin_cast_vec.go b/expression/builtin_cast_vec.go index ccb800f4efb0f..8add713d6053a 100644 --- a/expression/builtin_cast_vec.go +++ b/expression/builtin_cast_vec.go @@ -881,11 +881,42 @@ func (b *builtinCastStringAsDurationSig) vecEvalDuration(input *chunk.Chunk, res } func (b *builtinCastDurationAsDecimalSig) vectorized() bool { - return false + return true } func (b *builtinCastDurationAsDecimalSig) vecEvalDecimal(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + buf, err := b.bufAllocator.get(types.ETDuration, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf) + if err := b.args[0].VecEvalDuration(b.ctx, input, buf); err != nil { + return err + } + result.ResizeDecimal(n, false) + result.MergeNulls(buf) + d64s := result.Decimals() + var duration types.Duration + ds := buf.GoDurations() + sc := b.ctx.GetSessionVars().StmtCtx + fsp := int8(b.args[0].GetType().Decimal) + if fsp, err = types.CheckFsp(int(fsp)); err != nil { + return err + } + for i := 0; i < n; i++ { + if result.IsNull(i) { + continue + } + duration.Duration = ds[i] + duration.Fsp = fsp + res, err := types.ProduceDecWithSpecifiedTp(duration.ToNumber(), b.tp, sc) + if err != nil { + return err + } + d64s[i] = *res + } + return nil } func (b *builtinCastIntAsDecimalSig) vectorized() bool { diff --git a/expression/builtin_cast_vec_test.go b/expression/builtin_cast_vec_test.go index edc30d01a5373..e34ce2e137e20 100644 --- a/expression/builtin_cast_vec_test.go +++ b/expression/builtin_cast_vec_test.go @@ -30,6 +30,7 @@ var vecBuiltinCastCases = map[string][]vecExprBenchCase{ {retEvalType: types.ETDecimal, childrenTypes: []types.EvalType{types.ETJson}, geners: []dataGenerator{&decimalJSONGener{}}}, {retEvalType: types.ETDecimal, childrenTypes: []types.EvalType{types.ETString}, geners: []dataGenerator{&decimalStringGener{}}}, {retEvalType: types.ETDecimal, childrenTypes: []types.EvalType{types.ETReal}}, + {retEvalType: types.ETDecimal, childrenTypes: []types.EvalType{types.ETDuration}}, {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETInt}}, {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETReal}}, {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETDecimal}}, diff --git a/expression/builtin_miscellaneous_vec.go b/expression/builtin_miscellaneous_vec.go index 266033fab53a6..3bcf50269495d 100644 --- a/expression/builtin_miscellaneous_vec.go +++ b/expression/builtin_miscellaneous_vec.go @@ -195,11 +195,20 @@ func (b *builtinNameConstDurationSig) vecEvalDuration(input *chunk.Chunk, result } func (b *builtinLockSig) vectorized() bool { - return false + return true } +// See https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html#function_get-lock +// The lock function will do nothing. +// Warning: get_lock() function is parsed but ignored. func (b *builtinLockSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + result.ResizeInt64(n, false) + i64s := result.Int64s() + for i := range i64s { + i64s[i] = 1 + } + return nil } func (b *builtinDurationAnyValueSig) vectorized() bool { @@ -527,9 +536,18 @@ func (b *builtinNameConstRealSig) vecEvalReal(input *chunk.Chunk, result *chunk. } func (b *builtinReleaseLockSig) vectorized() bool { - return false + return true } +// See https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html#function_release-lock +// The release lock function will do nothing. +// Warning: release_lock() function is parsed but ignored. func (b *builtinReleaseLockSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + result.ResizeInt64(n, false) + i64s := result.Int64s() + for i := range i64s { + i64s[i] = 1 + } + return nil } diff --git a/expression/distsql_builtin.go b/expression/distsql_builtin.go index 3e7665374e81b..7c929db4f578b 100644 --- a/expression/distsql_builtin.go +++ b/expression/distsql_builtin.go @@ -1038,6 +1038,22 @@ func newDistSQLFunctionBySig(sc *stmtctx.StatementContext, sigCode tipb.ScalarFu }, nil } +// PBToExprs converts pb structures to expressions. +func PBToExprs(pbExprs []*tipb.Expr, fieldTps []*types.FieldType, sc *stmtctx.StatementContext) ([]Expression, error) { + exprs := make([]Expression, 0, len(pbExprs)) + for _, expr := range pbExprs { + e, err := PBToExpr(expr, fieldTps, sc) + if err != nil { + return nil, errors.Trace(err) + } + if e == nil { + return nil, errors.Errorf("pb to expression failed, pb expression is %v", expr) + } + exprs = append(exprs, e) + } + return exprs, nil +} + // PBToExpr converts pb structure to expression. func PBToExpr(expr *tipb.Expr, tps []*types.FieldType, sc *stmtctx.StatementContext) (Expression, error) { switch expr.Tp { diff --git a/expression/errors.go b/expression/errors.go index 10a719d54230a..d0f7d128784fd 100644 --- a/expression/errors.go +++ b/expression/errors.go @@ -51,6 +51,7 @@ func init() { mysql.ErrWrongParamcountToNativeFct: mysql.ErrWrongParamcountToNativeFct, mysql.ErrDivisionByZero: mysql.ErrDivisionByZero, mysql.ErrSpDoesNotExist: mysql.ErrSpDoesNotExist, + mysql.ErrNotSupportedYet: mysql.ErrNotSupportedYet, mysql.ErrZlibZData: mysql.ErrZlibZData, mysql.ErrZlibZBuf: mysql.ErrZlibZBuf, mysql.ErrWrongArguments: mysql.ErrWrongArguments, diff --git a/expression/explain.go b/expression/explain.go index 8250a260b6605..b1994a426ba57 100644 --- a/expression/explain.go +++ b/expression/explain.go @@ -25,10 +25,18 @@ import ( // ExplainInfo implements the Expression interface. func (expr *ScalarFunction) ExplainInfo() string { + return expr.explainInfo(false) +} + +func (expr *ScalarFunction) explainInfo(normalized bool) string { var buffer bytes.Buffer fmt.Fprintf(&buffer, "%s(", expr.FuncName.L) for i, arg := range expr.GetArgs() { - buffer.WriteString(arg.ExplainInfo()) + if normalized { + buffer.WriteString(arg.ExplainNormalizedInfo()) + } else { + buffer.WriteString(arg.ExplainInfo()) + } if i+1 < len(expr.GetArgs()) { buffer.WriteString(", ") } @@ -37,11 +45,21 @@ func (expr *ScalarFunction) ExplainInfo() string { return buffer.String() } +// ExplainNormalizedInfo implements the Expression interface. +func (expr *ScalarFunction) ExplainNormalizedInfo() string { + return expr.explainInfo(true) +} + // ExplainInfo implements the Expression interface. func (col *Column) ExplainInfo() string { return col.String() } +// ExplainNormalizedInfo implements the Expression interface. +func (col *Column) ExplainNormalizedInfo() string { + return col.ExplainInfo() +} + // ExplainInfo implements the Expression interface. func (expr *Constant) ExplainInfo() string { dt, err := expr.Eval(chunk.Row{}) @@ -51,6 +69,11 @@ func (expr *Constant) ExplainInfo() string { return expr.format(dt) } +// ExplainNormalizedInfo implements the Expression interface. +func (expr *Constant) ExplainNormalizedInfo() string { + return "?" +} + func (expr *Constant) format(dt types.Datum) string { switch dt.Kind() { case types.KindNull: @@ -83,10 +106,18 @@ func ExplainExpressionList(exprs []Expression, schema *Schema) string { // In some scenarios, the expr's order may not be stable when executing multiple times. // So we add a sort to make its explain result stable. func SortedExplainExpressionList(exprs []Expression) []byte { + return sortedExplainExpressionList(exprs, false) +} + +func sortedExplainExpressionList(exprs []Expression, normalized bool) []byte { buffer := bytes.NewBufferString("") exprInfos := make([]string, 0, len(exprs)) for _, expr := range exprs { - exprInfos = append(exprInfos, expr.ExplainInfo()) + if normalized { + exprInfos = append(exprInfos, expr.ExplainNormalizedInfo()) + } else { + exprInfos = append(exprInfos, expr.ExplainInfo()) + } } sort.Strings(exprInfos) for i, info := range exprInfos { @@ -98,6 +129,20 @@ func SortedExplainExpressionList(exprs []Expression) []byte { return buffer.Bytes() } +// SortedExplainNormalizedExpressionList is same like SortedExplainExpressionList, but use for generating normalized information. +func SortedExplainNormalizedExpressionList(exprs []Expression) []byte { + return sortedExplainExpressionList(exprs, true) +} + +// SortedExplainNormalizedScalarFuncList is same like SortedExplainExpressionList, but use for generating normalized information. +func SortedExplainNormalizedScalarFuncList(exprs []*ScalarFunction) []byte { + expressions := make([]Expression, len(exprs)) + for i := range exprs { + expressions[i] = exprs[i] + } + return sortedExplainExpressionList(expressions, true) +} + // ExplainColumnList generates explain information for a list of columns. func ExplainColumnList(cols []*Column) []byte { buffer := bytes.NewBufferString("") diff --git a/expression/expr_to_pb.go b/expression/expr_to_pb.go index 515b2f5f13795..87c33019d43c8 100644 --- a/expression/expr_to_pb.go +++ b/expression/expr_to_pb.go @@ -186,6 +186,18 @@ func ToPBFieldType(ft *types.FieldType) *tipb.FieldType { } } +// FieldTypeFromPB converts *tipb.FieldType to *types.FieldType. +func FieldTypeFromPB(ft *tipb.FieldType) *types.FieldType { + return &types.FieldType{ + Tp: byte(ft.Tp), + Flag: uint(ft.Flag), + Flen: int(ft.Flen), + Decimal: int(ft.Decimal), + Charset: ft.Charset, + Collate: protoToCollation(ft.Collate), + } +} + func collationToProto(c string) int32 { v, ok := mysql.CollationNames[c] if ok { @@ -194,6 +206,14 @@ func collationToProto(c string) int32 { return int32(mysql.DefaultCollationID) } +func protoToCollation(c int32) string { + v, ok := mysql.Collations[uint8(c)] + if ok { + return v + } + return mysql.DefaultCollationName +} + func (pc PbConverter) columnToPBExpr(column *Column) *tipb.Expr { if !pc.client.IsRequestTypeSupported(kv.ReqTypeSelect, int64(tipb.ExprType_ColumnRef)) { return nil diff --git a/expression/expression.go b/expression/expression.go index bfdc2741e273b..f8ab93986b245 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -131,6 +131,9 @@ type Expression interface { // ExplainInfo returns operator information to be explained. ExplainInfo() string + // ExplainNormalizedInfo returns operator normalized information for generating digest. + ExplainNormalizedInfo() string + // HashCode creates the hashcode for expression which can be used to identify itself from other expression. // It generated as the following: // Constant: ConstantFlag+encoded value diff --git a/expression/util_test.go b/expression/util_test.go index ec71299ec451f..13bea0dc9ec8c 100644 --- a/expression/util_test.go +++ b/expression/util_test.go @@ -510,5 +510,6 @@ func (m *MockExpr) Decorrelate(schema *Schema) Expression { return m func (m *MockExpr) ResolveIndices(schema *Schema) (Expression, error) { return m, nil } func (m *MockExpr) resolveIndices(schema *Schema) error { return nil } func (m *MockExpr) ExplainInfo() string { return "" } +func (m *MockExpr) ExplainNormalizedInfo() string { return "" } func (m *MockExpr) HashCode(sc *stmtctx.StatementContext) []byte { return nil } func (m *MockExpr) Vectorized() bool { return false } diff --git a/go.mod b/go.mod index 58ab75dc12000..bbd17235fb0d9 100644 --- a/go.mod +++ b/go.mod @@ -34,13 +34,13 @@ require ( github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e - github.com/pingcap/kvproto v0.0.0-20191121022655-4c654046831d + github.com/pingcap/kvproto v0.0.0-20191202044712-32be31591b03 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 - github.com/pingcap/parser v0.0.0-20191120165920-d5c49d11cc64 + github.com/pingcap/parser v0.0.0-20191203075000-55f02bc42e92 github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 github.com/pingcap/sysutil v0.0.0-20191126040022-986c5b3ed9a3 github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible - github.com/pingcap/tipb v0.0.0-20191127084114-0820b784842f + github.com/pingcap/tipb v0.0.0-20191203131953-a35f738b4796 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7 // indirect @@ -60,9 +60,9 @@ require ( go.uber.org/atomic v1.5.0 go.uber.org/automaxprocs v1.2.0 go.uber.org/zap v1.12.0 - golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf // indirect + golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c // indirect golang.org/x/net v0.0.0-20190909003024-a7b16738d86b - golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c + golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 golang.org/x/text v0.3.2 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect golang.org/x/tools v0.0.0-20191107010934-f79515f33823 diff --git a/go.sum b/go.sum index 0cb5c7a16dba9..b70bd62ab358a 100644 --- a/go.sum +++ b/go.sum @@ -181,21 +181,22 @@ github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d/go.mod h1:fMRU1BA1y+r89 github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20190822090350-11ea838aedf7/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= -github.com/pingcap/kvproto v0.0.0-20191121022655-4c654046831d h1:aH7ZFzWEyBgUtG/YlLOU7pIx++PqtXlRT7zpHcEf2Rg= github.com/pingcap/kvproto v0.0.0-20191121022655-4c654046831d/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20191202044712-32be31591b03 h1:IyJl+qesVPf3UfFFmKtX69y1K5KC8uXlot3U0QgH7V4= +github.com/pingcap/kvproto v0.0.0-20191202044712-32be31591b03/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/parser v0.0.0-20191120165920-d5c49d11cc64 h1:jpLGhi9hEp6Px9NDKkSjpcWuBdkgSCTxGMlrw9bWipQ= -github.com/pingcap/parser v0.0.0-20191120165920-d5c49d11cc64/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20191203075000-55f02bc42e92 h1:lmNqIpKSYnSUZL00nTU/bNSKqT2XS3i1jMh/ujIiOaY= +github.com/pingcap/parser v0.0.0-20191203075000-55f02bc42e92/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 h1:GIEq+wZfrl2bcJxpuSrEH4H7/nlf5YdmpS+dU9lNIt8= github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0/go.mod h1:G/6rJpnYwM0LKMec2rI82/5Kg6GaZMvlfB+e6/tvYmI= github.com/pingcap/sysutil v0.0.0-20191126040022-986c5b3ed9a3 h1:HCNif3lukL83gNC2EBAoh2Qbz36+2p0bm0LjgnNfl1s= github.com/pingcap/sysutil v0.0.0-20191126040022-986c5b3ed9a3/go.mod h1:Futrrmuw98pEsbEmoPsjw8aKLCmixwHEmT2rF+AsXGw= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20191127084114-0820b784842f h1:ywyH6JKJIf+gEYg2kRlU8SaPMryHgu5SoXKtPdkeZPU= -github.com/pingcap/tipb v0.0.0-20191127084114-0820b784842f/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20191203131953-a35f738b4796 h1:VNxsATjjGSXYbLXYdwJMj4ah5oxkMbKtOg/kaoXUX64= +github.com/pingcap/tipb v0.0.0-20191203131953-a35f738b4796/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -300,8 +301,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf h1:fnPsqIDRbCSgumaMCRpoIoF2s4qxv0xSSS0BVZUE/ss= -golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c h1:/nJuwDLoL/zrqY6gf57vxC+Pi+pZ8bfhpPkicO5H7W4= +golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -336,8 +337,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c h1:S/FtSvpNLtFBgjTqcKsRpsa6aVsI6iztaz1bQd9BJwE= -golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX0Az1Bw5HF7PDraD+qU= +golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/infoschema/builder.go b/infoschema/builder.go index f089ceff730cb..0075696fa3c2a 100644 --- a/infoschema/builder.go +++ b/infoschema/builder.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/config" @@ -26,6 +27,7 @@ import ( "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" + "github.com/pingcap/tidb/util/domainutil" ) // Builder builds a new InfoSchema. @@ -46,7 +48,6 @@ func (b *Builder) ApplyDiff(m *meta.Meta, diff *model.SchemaDiff) ([]int64, erro } else if diff.Type == model.ActionModifySchemaCharsetAndCollate { return nil, b.applyModifySchemaCharsetAndCollate(m, diff) } - roDBInfo, ok := b.is.SchemaByID(diff.SchemaID) if !ok { return nil, ErrDatabaseNotExists.GenWithStackByArgs( @@ -56,7 +57,7 @@ func (b *Builder) ApplyDiff(m *meta.Meta, diff *model.SchemaDiff) ([]int64, erro var oldTableID, newTableID int64 tblIDs := make([]int64, 0, 2) switch diff.Type { - case model.ActionCreateTable, model.ActionRecoverTable: + case model.ActionCreateTable, model.ActionRecoverTable, model.ActionRepairTable: newTableID = diff.TableID tblIDs = append(tblIDs, newTableID) case model.ActionDropTable, model.ActionDropView: @@ -95,7 +96,7 @@ func (b *Builder) ApplyDiff(m *meta.Meta, diff *model.SchemaDiff) ([]int64, erro } if tableIDIsValid(newTableID) { // All types except DropTableOrView. - err := b.applyCreateTable(m, dbInfo, newTableID, alloc) + err := b.applyCreateTable(m, dbInfo, newTableID, alloc, diff.Type) if err != nil { return nil, errors.Trace(err) } @@ -179,7 +180,7 @@ func (b *Builder) copySortedTablesBucket(bucketIdx int) { b.is.sortedTablesBuckets[bucketIdx] = newSortedTables } -func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID int64, alloc autoid.Allocator) error { +func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID int64, alloc autoid.Allocator, tp model.ActionType) error { tblInfo, err := m.GetTable(dbInfo.ID, tableID) if err != nil { return errors.Trace(err) @@ -192,6 +193,16 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i fmt.Sprintf("(Table ID %d)", tableID), ) } + // Failpoint check whether tableInfo should be added to repairInfo. + // Typically used in repair table test to load mock `bad` tableInfo into repairInfo. + failpoint.Inject("repairFetchCreateTable", func(val failpoint.Value) { + if val.(bool) { + if domainutil.RepairInfo.InRepairMode() && tp != model.ActionRepairTable && domainutil.RepairInfo.CheckAndFetchRepairedTable(dbInfo, tblInfo) { + failpoint.Return(nil) + } + } + }) + ConvertCharsetCollateToLowerCaseIfNeed(tblInfo) ConvertOldVersionUTF8ToUTF8MB4IfNeed(tblInfo) diff --git a/infoschema/cluster.go b/infoschema/cluster.go new file mode 100644 index 0000000000000..98ca62fcacd5c --- /dev/null +++ b/infoschema/cluster.go @@ -0,0 +1,98 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package infoschema + +import ( + "strconv" + "strings" + + "github.com/pingcap/errors" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/domain/infosync" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util" +) + +// Cluster table list, attention: +// 1. the table name should be upper case. +// 2. clusterTableName should equal to "CLUSTER_" + memTableTableName. +const ( + clusterTableSlowLog = "CLUSTER_SLOW_QUERY" + clusterTableProcesslist = "CLUSTER_PROCESSLIST" +) + +// memTableToClusterTables means add memory table to cluster table. +var memTableToClusterTables = map[string]string{ + tableSlowLog: clusterTableSlowLog, + tableProcesslist: clusterTableProcesslist, +} + +func init() { + var addrCol = columnInfo{"ADDRESS", mysql.TypeVarchar, 64, 0, nil, nil} + for memTableName, clusterMemTableName := range memTableToClusterTables { + memTableCols := tableNameToColumns[memTableName] + if len(memTableCols) == 0 { + continue + } + cols := make([]columnInfo, 0, len(memTableCols)+1) + cols = append(cols, memTableCols...) + cols = append(cols, addrCol) + tableNameToColumns[clusterMemTableName] = cols + } +} + +// isClusterTableByName used to check whether the table is a cluster memory table. +func isClusterTableByName(dbName, tableName string) bool { + dbName = strings.ToUpper(dbName) + switch dbName { + case util.InformationSchemaName, util.PerformanceSchemaName: + break + default: + return false + } + tableName = strings.ToUpper(tableName) + for _, name := range memTableToClusterTables { + name = strings.ToUpper(name) + if name == tableName { + return true + } + } + return false +} + +func getClusterMemTableRows(ctx sessionctx.Context, tableName string) (rows [][]types.Datum, err error) { + tableName = strings.ToUpper(tableName) + switch tableName { + case clusterTableSlowLog: + rows, err = dataForSlowLog(ctx) + case clusterTableProcesslist: + rows = dataForProcesslist(ctx) + default: + err = errors.Errorf("unknown cluster table: %v", tableName) + } + if err != nil { + return nil, err + } + return appendHostInfoToRows(rows), nil +} + +func appendHostInfoToRows(rows [][]types.Datum) [][]types.Datum { + serverInfo := infosync.GetServerInfo() + addr := serverInfo.IP + ":" + strconv.FormatUint(uint64(serverInfo.StatusPort), 10) + for i := range rows { + rows[i] = append(rows[i], types.NewStringDatum(addr)) + } + return rows +} diff --git a/infoschema/infoschema.go b/infoschema/infoschema.go index c51c1507113ce..976234684d381 100644 --- a/infoschema/infoschema.go +++ b/infoschema/infoschema.go @@ -14,6 +14,7 @@ package infoschema import ( + "fmt" "sort" "sync/atomic" @@ -348,14 +349,18 @@ func init() { terror.ErrClassToMySQLCodes[terror.ClassSchema] = schemaMySQLErrCodes // Initialize the information shema database and register the driver to `drivers` - dbID := autoid.GenLocalSchemaID() + dbID := autoid.InformationSchemaDBID infoSchemaTables := make([]*model.TableInfo, 0, len(tableNameToColumns)) for name, cols := range tableNameToColumns { tableInfo := buildTableMeta(name, cols) infoSchemaTables = append(infoSchemaTables, tableInfo) - tableInfo.ID = autoid.GenLocalSchemaID() - for _, c := range tableInfo.Columns { - c.ID = autoid.GenLocalSchemaID() + var ok bool + tableInfo.ID, ok = tableIDMap[tableInfo.Name.O] + if !ok { + panic(fmt.Sprintf("get information_schema table id failed, unknown system table `%v`", tableInfo.Name.O)) + } + for i, c := range tableInfo.Columns { + c.ID = int64(i) + 1 } } infoSchemaDB := &model.DBInfo{ diff --git a/infoschema/perfschema/const.go b/infoschema/perfschema/const.go index 4f52d468c8460..532892c5fcdac 100644 --- a/infoschema/perfschema/const.go +++ b/infoschema/perfschema/const.go @@ -56,17 +56,17 @@ var perfSchemaTables = []string{ } // tableGlobalStatus contains the column name definitions for table global_status, same as MySQL. -const tableGlobalStatus = "CREATE TABLE performance_schema.global_status(" + +const tableGlobalStatus = "CREATE TABLE performance_schema." + tableNameGlobalStatus + " (" + "VARIABLE_NAME VARCHAR(64) not null," + "VARIABLE_VALUE VARCHAR(1024));" // tableSessionStatus contains the column name definitions for table session_status, same as MySQL. -const tableSessionStatus = "CREATE TABLE performance_schema.session_status(" + +const tableSessionStatus = "CREATE TABLE performance_schema." + tableNameSessionStatus + " (" + "VARIABLE_NAME VARCHAR(64) not null," + "VARIABLE_VALUE VARCHAR(1024));" // tableSetupActors contains the column name definitions for table setup_actors, same as MySQL. -const tableSetupActors = "CREATE TABLE if not exists performance_schema.setup_actors (" + +const tableSetupActors = "CREATE TABLE if not exists performance_schema." + tableNameSetupActors + " (" + "HOST CHAR(60) NOT NULL DEFAULT '%'," + "USER CHAR(32) NOT NULL DEFAULT '%'," + "ROLE CHAR(16) NOT NULL DEFAULT '%'," + @@ -74,7 +74,7 @@ const tableSetupActors = "CREATE TABLE if not exists performance_schema.setup_ac "HISTORY ENUM('YES','NO') NOT NULL DEFAULT 'YES');" // tableSetupObjects contains the column name definitions for table setup_objects, same as MySQL. -const tableSetupObjects = "CREATE TABLE if not exists performance_schema.setup_objects (" + +const tableSetupObjects = "CREATE TABLE if not exists performance_schema." + tableNameSetupObjects + " (" + "OBJECT_TYPE ENUM('EVENT','FUNCTION','TABLE') NOT NULL DEFAULT 'TABLE'," + "OBJECT_SCHEMA VARCHAR(64) DEFAULT '%'," + "OBJECT_NAME VARCHAR(64) NOT NULL DEFAULT '%'," + @@ -82,18 +82,18 @@ const tableSetupObjects = "CREATE TABLE if not exists performance_schema.setup_o "TIMED ENUM('YES','NO') NOT NULL DEFAULT 'YES');" // tableSetupInstruments contains the column name definitions for table setup_instruments, same as MySQL. -const tableSetupInstruments = "CREATE TABLE if not exists performance_schema.setup_instruments (" + +const tableSetupInstruments = "CREATE TABLE if not exists performance_schema." + tableNameSetupInstruments + " (" + "NAME VARCHAR(128) NOT NULL," + "ENABLED ENUM('YES','NO') NOT NULL," + "TIMED ENUM('YES','NO') NOT NULL);" // tableSetupConsumers contains the column name definitions for table setup_consumers, same as MySQL. -const tableSetupConsumers = "CREATE TABLE if not exists performance_schema.setup_consumers (" + +const tableSetupConsumers = "CREATE TABLE if not exists performance_schema." + tableNameSetupConsumers + " (" + "NAME VARCHAR(64) NOT NULL," + "ENABLED ENUM('YES','NO') NOT NULL);" // tableStmtsCurrent contains the column name definitions for table events_statements_current, same as MySQL. -const tableStmtsCurrent = "CREATE TABLE if not exists performance_schema.events_statements_current (" + +const tableStmtsCurrent = "CREATE TABLE if not exists performance_schema." + tableNameEventsStatementsCurrent + " (" + "THREAD_ID BIGINT(20) UNSIGNED NOT NULL," + "EVENT_ID BIGINT(20) UNSIGNED NOT NULL," + "END_EVENT_ID BIGINT(20) UNSIGNED," + @@ -137,7 +137,7 @@ const tableStmtsCurrent = "CREATE TABLE if not exists performance_schema.events_ "NESTING_EVENT_LEVEL INT(11));" // tableStmtsHistory contains the column name definitions for table events_statements_history, same as MySQL. -const tableStmtsHistory = "CREATE TABLE if not exists performance_schema.events_statements_history (" + +const tableStmtsHistory = "CREATE TABLE if not exists performance_schema." + tableNameEventsStatementsHistory + " (" + "THREAD_ID BIGINT(20) UNSIGNED NOT NULL," + "EVENT_ID BIGINT(20) UNSIGNED NOT NULL," + "END_EVENT_ID BIGINT(20) UNSIGNED," + @@ -181,7 +181,7 @@ const tableStmtsHistory = "CREATE TABLE if not exists performance_schema.events_ "NESTING_EVENT_LEVEL INT(11));" // tableStmtsHistoryLong contains the column name definitions for table events_statements_history_long, same as MySQL. -const tableStmtsHistoryLong = "CREATE TABLE if not exists performance_schema.events_statements_history_long (" + +const tableStmtsHistoryLong = "CREATE TABLE if not exists performance_schema." + tableNameEventsStatementsHistoryLong + " (" + "THREAD_ID BIGINT(20) UNSIGNED NOT NULL," + "EVENT_ID BIGINT(20) UNSIGNED NOT NULL," + "END_EVENT_ID BIGINT(20) UNSIGNED," + @@ -225,7 +225,7 @@ const tableStmtsHistoryLong = "CREATE TABLE if not exists performance_schema.eve "NESTING_EVENT_LEVEL INT(11));" // tablePreparedStmtsInstances contains the column name definitions for table prepared_statements_instances, same as MySQL. -const tablePreparedStmtsInstances = "CREATE TABLE if not exists performance_schema.prepared_statements_instances (" + +const tablePreparedStmtsInstances = "CREATE TABLE if not exists performance_schema." + tableNamePreparedStatementsInstances + " (" + "OBJECT_INSTANCE_BEGIN BIGINT(20) UNSIGNED NOT NULL," + "STATEMENT_ID BIGINT(20) UNSIGNED NOT NULL," + "STATEMENT_NAME VARCHAR(64)," + @@ -263,7 +263,7 @@ const tablePreparedStmtsInstances = "CREATE TABLE if not exists performance_sche "SUM_NO_GOOD_INDEX_USED BIGINT(20) UNSIGNED NOT NULL);" // tableTransCurrent contains the column name definitions for table events_transactions_current, same as MySQL. -const tableTransCurrent = "CREATE TABLE if not exists performance_schema.events_transactions_current (" + +const tableTransCurrent = "CREATE TABLE if not exists performance_schema." + tableNameEventsTransactionsCurrent + " (" + "THREAD_ID BIGINT(20) UNSIGNED NOT NULL," + "EVENT_ID BIGINT(20) UNSIGNED NOT NULL," + "END_EVENT_ID BIGINT(20) UNSIGNED," + @@ -291,7 +291,7 @@ const tableTransCurrent = "CREATE TABLE if not exists performance_schema.events_ // tableTransHistory contains the column name definitions for table events_transactions_history, same as MySQL. // -const tableTransHistory = "CREATE TABLE if not exists performance_schema.events_transactions_history (" + +const tableTransHistory = "CREATE TABLE if not exists performance_schema." + tableNameEventsTransactionsHistory + " (" + "THREAD_ID BIGINT(20) UNSIGNED NOT NULL," + "EVENT_ID BIGINT(20) UNSIGNED NOT NULL," + "END_EVENT_ID BIGINT(20) UNSIGNED," + @@ -318,7 +318,7 @@ const tableTransHistory = "CREATE TABLE if not exists performance_schema.events_ "NESTING_EVENT_TYPE ENUM('TRANSACTION','STATEMENT','STAGE'));" // tableTransHistoryLong contains the column name definitions for table events_transactions_history_long, same as MySQL. -const tableTransHistoryLong = "CREATE TABLE if not exists performance_schema.events_transactions_history_long (" + +const tableTransHistoryLong = "CREATE TABLE if not exists performance_schema." + tableNameEventsTransactionsHistoryLong + " (" + "THREAD_ID BIGINT(20) UNSIGNED NOT NULL," + "EVENT_ID BIGINT(20) UNSIGNED NOT NULL," + "END_EVENT_ID BIGINT(20) UNSIGNED," + @@ -345,7 +345,7 @@ const tableTransHistoryLong = "CREATE TABLE if not exists performance_schema.eve "NESTING_EVENT_TYPE ENUM('TRANSACTION','STATEMENT','STAGE'));" // tableStagesCurrent contains the column name definitions for table events_stages_current, same as MySQL. -const tableStagesCurrent = "CREATE TABLE if not exists performance_schema.events_stages_current (" + +const tableStagesCurrent = "CREATE TABLE if not exists performance_schema." + tableNameEventsStagesCurrent + " (" + "THREAD_ID BIGINT(20) UNSIGNED NOT NULL," + "EVENT_ID BIGINT(20) UNSIGNED NOT NULL," + "END_EVENT_ID BIGINT(20) UNSIGNED," + @@ -360,7 +360,7 @@ const tableStagesCurrent = "CREATE TABLE if not exists performance_schema.events "NESTING_EVENT_TYPE ENUM('TRANSACTION','STATEMENT','STAGE'));" // tableStagesHistory contains the column name definitions for table events_stages_history, same as MySQL. -const tableStagesHistory = "CREATE TABLE if not exists performance_schema.events_stages_history (" + +const tableStagesHistory = "CREATE TABLE if not exists performance_schema." + tableNameEventsStagesHistory + " (" + "THREAD_ID BIGINT(20) UNSIGNED NOT NULL," + "EVENT_ID BIGINT(20) UNSIGNED NOT NULL," + "END_EVENT_ID BIGINT(20) UNSIGNED," + @@ -375,7 +375,7 @@ const tableStagesHistory = "CREATE TABLE if not exists performance_schema.events "NESTING_EVENT_TYPE ENUM('TRANSACTION','STATEMENT','STAGE'));" // tableStagesHistoryLong contains the column name definitions for table events_stages_history_long, same as MySQL. -const tableStagesHistoryLong = "CREATE TABLE if not exists performance_schema.events_stages_history_long (" + +const tableStagesHistoryLong = "CREATE TABLE if not exists performance_schema." + tableNameEventsStagesHistoryLong + " (" + "THREAD_ID BIGINT(20) UNSIGNED NOT NULL," + "EVENT_ID BIGINT(20) UNSIGNED NOT NULL," + "END_EVENT_ID BIGINT(20) UNSIGNED," + @@ -391,7 +391,7 @@ const tableStagesHistoryLong = "CREATE TABLE if not exists performance_schema.ev // tableEventsStatementsSummaryByDigest contains the column name definitions for table // events_statements_summary_by_digest, same as MySQL. -const tableEventsStatementsSummaryByDigest = "CREATE TABLE if not exists events_statements_summary_by_digest (" + +const tableEventsStatementsSummaryByDigest = "CREATE TABLE if not exists " + tableNameEventsStatementsSummaryByDigest + " (" + "SUMMARY_BEGIN_TIME TIMESTAMP(6) NOT NULL," + "STMT_TYPE VARCHAR(64) NOT NULL," + "SCHEMA_NAME VARCHAR(64) DEFAULT NULL," + diff --git a/infoschema/perfschema/init.go b/infoschema/perfschema/init.go index 335556cf8ca5d..1cf549244ea3a 100644 --- a/infoschema/perfschema/init.go +++ b/infoschema/perfschema/init.go @@ -14,6 +14,7 @@ package perfschema import ( + "fmt" "sync" "github.com/pingcap/parser" @@ -39,8 +40,7 @@ func Init() { initOnce := func() { p := parser.New() tbls := make([]*model.TableInfo, 0) - dbID := autoid.GenLocalSchemaID() - + dbID := autoid.PerformanceSchemaDBID for _, sql := range perfSchemaTables { stmt, err := p.ParseOneStmt(sql, "", "") if err != nil { @@ -51,9 +51,13 @@ func Init() { panic(err) } tbls = append(tbls, meta) - meta.ID = autoid.GenLocalSchemaID() - for _, c := range meta.Columns { - c.ID = autoid.GenLocalSchemaID() + var ok bool + meta.ID, ok = tableIDMap[meta.Name.O] + if !ok { + panic(fmt.Sprintf("get performance_schema table id failed, unknown system table `%v`", meta.Name.O)) + } + for i, c := range meta.Columns { + c.ID = int64(i) + 1 } } dbInfo := &model.DBInfo{ diff --git a/infoschema/perfschema/tables.go b/infoschema/perfschema/tables.go index e217fbe8ecdb4..e9bc013f58487 100644 --- a/infoschema/perfschema/tables.go +++ b/infoschema/perfschema/tables.go @@ -37,6 +37,22 @@ import ( ) const ( + tableNameGlobalStatus = "global_status" + tableNameSessionStatus = "session_status" + tableNameSetupActors = "setup_actors" + tableNameSetupObjects = "setup_objects" + tableNameSetupInstruments = "setup_instruments" + tableNameSetupConsumers = "setup_consumers" + tableNameEventsStatementsCurrent = "events_statements_current" + tableNameEventsStatementsHistory = "events_statements_history" + tableNameEventsStatementsHistoryLong = "events_statements_history_long" + tableNamePreparedStatementsInstances = "prepared_statements_instances" + tableNameEventsTransactionsCurrent = "events_transactions_current" + tableNameEventsTransactionsHistory = "events_transactions_history" + tableNameEventsTransactionsHistoryLong = "events_transactions_history_long" + tableNameEventsStagesCurrent = "events_stages_current" + tableNameEventsStagesHistory = "events_stages_history" + tableNameEventsStagesHistoryLong = "events_stages_history_long" tableNameEventsStatementsSummaryByDigest = "events_statements_summary_by_digest" tableNameTiDBProfileCPU = "tidb_profile_cpu" tableNameTiDBProfileMemory = "tidb_profile_memory" @@ -53,6 +69,39 @@ const ( tableNamePDProfileGoroutines = "pd_profile_goroutines" ) +var tableIDMap = map[string]int64{ + tableNameGlobalStatus: autoid.PerformanceSchemaDBID + 1, + tableNameSessionStatus: autoid.PerformanceSchemaDBID + 2, + tableNameSetupActors: autoid.PerformanceSchemaDBID + 3, + tableNameSetupObjects: autoid.PerformanceSchemaDBID + 4, + tableNameSetupInstruments: autoid.PerformanceSchemaDBID + 5, + tableNameSetupConsumers: autoid.PerformanceSchemaDBID + 6, + tableNameEventsStatementsCurrent: autoid.PerformanceSchemaDBID + 7, + tableNameEventsStatementsHistory: autoid.PerformanceSchemaDBID + 8, + tableNameEventsStatementsHistoryLong: autoid.PerformanceSchemaDBID + 9, + tableNamePreparedStatementsInstances: autoid.PerformanceSchemaDBID + 10, + tableNameEventsTransactionsCurrent: autoid.PerformanceSchemaDBID + 11, + tableNameEventsTransactionsHistory: autoid.PerformanceSchemaDBID + 12, + tableNameEventsTransactionsHistoryLong: autoid.PerformanceSchemaDBID + 13, + tableNameEventsStagesCurrent: autoid.PerformanceSchemaDBID + 14, + tableNameEventsStagesHistory: autoid.PerformanceSchemaDBID + 15, + tableNameEventsStagesHistoryLong: autoid.PerformanceSchemaDBID + 16, + tableNameEventsStatementsSummaryByDigest: autoid.PerformanceSchemaDBID + 17, + tableNameTiDBProfileCPU: autoid.PerformanceSchemaDBID + 18, + tableNameTiDBProfileMemory: autoid.PerformanceSchemaDBID + 19, + tableNameTiDBProfileMutex: autoid.PerformanceSchemaDBID + 20, + tableNameTiDBProfileAllocs: autoid.PerformanceSchemaDBID + 21, + tableNameTiDBProfileBlock: autoid.PerformanceSchemaDBID + 22, + tableNameTiDBProfileGoroutines: autoid.PerformanceSchemaDBID + 23, + tableNameTiKVProfileCPU: autoid.PerformanceSchemaDBID + 24, + tableNamePDProfileCPU: autoid.PerformanceSchemaDBID + 25, + tableNamePDProfileMemory: autoid.PerformanceSchemaDBID + 26, + tableNamePDProfileMutex: autoid.PerformanceSchemaDBID + 27, + tableNamePDProfileAllocs: autoid.PerformanceSchemaDBID + 28, + tableNamePDProfileBlock: autoid.PerformanceSchemaDBID + 29, + tableNamePDProfileGoroutines: autoid.PerformanceSchemaDBID + 30, +} + // perfSchemaTable stands for the fake table all its data is in the memory. type perfSchemaTable struct { infoschema.VirtualTable diff --git a/infoschema/tables.go b/infoschema/tables.go index eedda6bd585b6..48e7922eef54b 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -98,12 +98,63 @@ const ( tableTiKVRegionStatus = "TIKV_REGION_STATUS" tableTiKVRegionPeers = "TIKV_REGION_PEERS" tableTiDBServersInfo = "TIDB_SERVERS_INFO" - tableTiDBClusterInfo = "TIDB_CLUSTER_INFO" - tableTiDBClusterConfig = "TIDB_CLUSTER_CONFIG" - tableTiDBClusterLoad = "TIDB_CLUSTER_LOAD" - tableTiFlashReplica = "TIFLASH_REPLICA" + tableTiDBClusterInfo = "CLUSTER_INFO" + // TableTiDBClusterConfig is the string constant of cluster configuration memory table + TableTiDBClusterConfig = "CLUSTER_CONFIG" + tableTiDBClusterLoad = "CLUSTER_LOAD" + tableTiFlashReplica = "TIFLASH_REPLICA" ) +var tableIDMap = map[string]int64{ + tableSchemata: autoid.InformationSchemaDBID + 1, + tableTables: autoid.InformationSchemaDBID + 2, + tableColumns: autoid.InformationSchemaDBID + 3, + tableColumnStatistics: autoid.InformationSchemaDBID + 4, + tableStatistics: autoid.InformationSchemaDBID + 5, + tableCharacterSets: autoid.InformationSchemaDBID + 6, + tableCollations: autoid.InformationSchemaDBID + 7, + tableFiles: autoid.InformationSchemaDBID + 8, + catalogVal: autoid.InformationSchemaDBID + 9, + tableProfiling: autoid.InformationSchemaDBID + 10, + tablePartitions: autoid.InformationSchemaDBID + 11, + tableKeyColumm: autoid.InformationSchemaDBID + 12, + tableReferConst: autoid.InformationSchemaDBID + 13, + tableSessionVar: autoid.InformationSchemaDBID + 14, + tablePlugins: autoid.InformationSchemaDBID + 15, + tableConstraints: autoid.InformationSchemaDBID + 16, + tableTriggers: autoid.InformationSchemaDBID + 17, + tableUserPrivileges: autoid.InformationSchemaDBID + 18, + tableSchemaPrivileges: autoid.InformationSchemaDBID + 19, + tableTablePrivileges: autoid.InformationSchemaDBID + 20, + tableColumnPrivileges: autoid.InformationSchemaDBID + 21, + tableEngines: autoid.InformationSchemaDBID + 22, + tableViews: autoid.InformationSchemaDBID + 23, + tableRoutines: autoid.InformationSchemaDBID + 24, + tableParameters: autoid.InformationSchemaDBID + 25, + tableEvents: autoid.InformationSchemaDBID + 26, + tableGlobalStatus: autoid.InformationSchemaDBID + 27, + tableGlobalVariables: autoid.InformationSchemaDBID + 28, + tableSessionStatus: autoid.InformationSchemaDBID + 29, + tableOptimizerTrace: autoid.InformationSchemaDBID + 30, + tableTableSpaces: autoid.InformationSchemaDBID + 31, + tableCollationCharacterSetApplicability: autoid.InformationSchemaDBID + 32, + tableProcesslist: autoid.InformationSchemaDBID + 33, + tableTiDBIndexes: autoid.InformationSchemaDBID + 34, + tableSlowLog: autoid.InformationSchemaDBID + 35, + tableTiDBHotRegions: autoid.InformationSchemaDBID + 36, + tableTiKVStoreStatus: autoid.InformationSchemaDBID + 37, + tableAnalyzeStatus: autoid.InformationSchemaDBID + 38, + tableTiKVRegionStatus: autoid.InformationSchemaDBID + 39, + tableTiKVRegionPeers: autoid.InformationSchemaDBID + 40, + tableTiDBServersInfo: autoid.InformationSchemaDBID + 41, + tableTiDBClusterInfo: autoid.InformationSchemaDBID + 42, + TableTiDBClusterConfig: autoid.InformationSchemaDBID + 43, + tableTiDBClusterLoad: autoid.InformationSchemaDBID + 44, + tableTiFlashReplica: autoid.InformationSchemaDBID + 45, + clusterTableSlowLog: autoid.InformationSchemaDBID + 46, + clusterTableProcesslist: autoid.InformationSchemaDBID + 47, +} + type columnInfo struct { name string tp byte @@ -983,7 +1034,7 @@ func dataForProcesslist(ctx sessionctx.Context) [][]types.Datum { for _, pi := range pl { // If you have the PROCESS privilege, you can see all threads. // Otherwise, you can see only your own threads. - if !hasProcessPriv && pi.User != loginUser.Username { + if !hasProcessPriv && loginUser != nil && pi.User != loginUser.Username { continue } @@ -2328,7 +2379,7 @@ var tableNameToColumns = map[string][]columnInfo{ tableTiKVRegionPeers: tableTiKVRegionPeersCols, tableTiDBServersInfo: tableTiDBServersInfoCols, tableTiDBClusterInfo: tableTiDBClusterInfoCols, - tableTiDBClusterConfig: tableTiDBClusterConfigCols, + TableTiDBClusterConfig: tableTiDBClusterConfigCols, tableTiDBClusterLoad: tableTiDBClusterLoadCols, tableTiFlashReplica: tableTableTiFlashReplicaCols, } @@ -2338,12 +2389,17 @@ func createInfoSchemaTable(_ autoid.Allocator, meta *model.TableInfo) (table.Tab for i, col := range meta.Columns { columns[i] = table.ToColumn(col) } - return &infoschemaTable{meta: meta, cols: columns}, nil + tp := table.VirtualTable + if isClusterTableByName(util.InformationSchemaName, meta.Name.L) { + tp = table.ClusterTable + } + return &infoschemaTable{meta: meta, cols: columns, tp: tp}, nil } type infoschemaTable struct { meta *model.TableInfo cols []*table.Column + tp table.Type } // schemasSorter implements the sort.Interface interface, sorts DBInfo by name. @@ -2432,12 +2488,15 @@ func (it *infoschemaTable) getRows(ctx sessionctx.Context, cols []*table.Column) fullRows, err = dataForServersInfo() case tableTiDBClusterInfo: fullRows, err = dataForTiDBClusterInfo(ctx) - case tableTiDBClusterConfig: + case TableTiDBClusterConfig: fullRows, err = dataForClusterConfig(ctx) case tableTiDBClusterLoad: fullRows, err = dataForClusterLoadInfo(ctx) case tableTiFlashReplica: fullRows = dataForTableTiFlashReplica(dbs) + // Data for cluster memory table. + case clusterTableSlowLog, clusterTableProcesslist: + fullRows, err = getClusterMemTableRows(ctx, it.meta.Name.O) } if err != nil { return nil, err @@ -2585,7 +2644,7 @@ func (it *infoschemaTable) Seek(ctx sessionctx.Context, h int64) (int64, bool, e // Type implements table.Table Type interface. func (it *infoschemaTable) Type() table.Type { - return table.VirtualTable + return it.tp } // VirtualTable is a dummy table.Table implementation. diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index 095135fc36d5b..f9ab43405d100 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -37,6 +37,7 @@ import ( "github.com/pingcap/tidb/domain/infosync" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/server" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/statistics" @@ -86,14 +87,22 @@ type testClusterTableSuite struct { func (s *testClusterTableSuite) SetUpSuite(c *C) { s.testTableSuite.SetUpSuite(c) - s.rpcserver = setUpRPCService(c, "0.0.0.0:10080") + s.rpcserver = s.setUpRPCService(c, "0.0.0.0:10080") s.httpServer, s.mockAddr = setUpMockPDHTTPSercer() } -func setUpRPCService(c *C, addr string) *grpc.Server { +func (s *testClusterTableSuite) setUpRPCService(c *C, addr string) *grpc.Server { lis, err := net.Listen("tcp", addr) c.Assert(err, IsNil) - srv := server.NewRPCServer(config.GetGlobalConfig().Security) + // Fix issue 9836 + sm := &mockSessionManager{make(map[uint64]*util.ProcessInfo, 1)} + sm.processInfoMap[1] = &util.ProcessInfo{ + ID: 1, + User: "root", + Host: "127.0.0.1", + Command: mysql.ComQuery, + } + srv := server.NewRPCServer(config.GetGlobalConfig().Security, s.dom, sm) go func() { err = srv.Serve(lis) c.Assert(err, IsNil) @@ -559,6 +568,35 @@ func (s *testTableSuite) TestTableIDAndIndexID(c *C) { tk.MustQuery("select * from information_schema.tidb_indexes where table_schema = 'test' and table_name = 't'").Check(testkit.Rows("test t 0 PRIMARY 1 a 0", "test t 1 k1 1 b 1")) } +func prepareSlowLogfile(c *C, slowLogFileName string) { + f, err := os.OpenFile(slowLogFileName, os.O_CREATE|os.O_WRONLY, 0644) + c.Assert(err, IsNil) + _, err = f.Write([]byte(`# Time: 2019-02-12T19:33:56.571953+08:00 +# Txn_start_ts: 406315658548871171 +# User: root@127.0.0.1 +# Conn_ID: 6 +# Query_time: 4.895492 +# Parse_time: 0.4 +# Compile_time: 0.2 +# Request_count: 1 Prewrite_time: 0.19 Commit_time: 0.01 Commit_backoff_time: 0.18 Backoff_types: [txnLock] Resolve_lock_time: 0.03 Write_keys: 15 Write_size: 480 Prewrite_region: 1 Txn_retry: 8 +# Process_time: 0.161 Request_count: 1 Total_keys: 100001 Process_keys: 100000 +# Wait_time: 0.101 +# Backoff_time: 0.092 +# DB: test +# Is_internal: false +# Digest: 42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772 +# Stats: t1:1,t2:2 +# Cop_proc_avg: 0.1 Cop_proc_p90: 0.2 Cop_proc_max: 0.03 Cop_proc_addr: 127.0.0.1:20160 +# Cop_wait_avg: 0.05 Cop_wait_p90: 0.6 Cop_wait_max: 0.8 Cop_wait_addr: 0.0.0.0:20160 +# Mem_max: 70724 +# Succ: true +# Plan: abcd +# Prev_stmt: update t set i = 2; +select * from t_slim;`)) + c.Assert(f.Sync(), IsNil) + c.Assert(err, IsNil) +} + func (s *testTableSuite) TestTableRowIDShardingInfo(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("DROP DATABASE IF EXISTS `sharding_info_test_db`") @@ -605,33 +643,8 @@ func (s *testTableSuite) TestSlowQuery(c *C) { tk := testkit.NewTestKit(c, s.store) // Prepare slow log file. slowLogFileName := "tidb_slow.log" - f, err := os.OpenFile(slowLogFileName, os.O_CREATE|os.O_WRONLY, 0644) - c.Assert(err, IsNil) + prepareSlowLogfile(c, slowLogFileName) defer os.Remove(slowLogFileName) - _, err = f.Write([]byte(`# Time: 2019-02-12T19:33:56.571953+08:00 -# Txn_start_ts: 406315658548871171 -# User: root@127.0.0.1 -# Conn_ID: 6 -# Query_time: 4.895492 -# Parse_time: 0.4 -# Compile_time: 0.2 -# Request_count: 1 Prewrite_time: 0.19 Commit_time: 0.01 Commit_backoff_time: 0.18 Backoff_types: [txnLock] Resolve_lock_time: 0.03 Write_keys: 15 Write_size: 480 Prewrite_region: 1 Txn_retry: 8 -# Process_time: 0.161 Request_count: 1 Total_keys: 100001 Process_keys: 100000 -# Wait_time: 0.101 -# Backoff_time: 0.092 -# DB: test -# Is_internal: false -# Digest: 42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772 -# Stats: t1:1,t2:2 -# Cop_proc_avg: 0.1 Cop_proc_p90: 0.2 Cop_proc_max: 0.03 Cop_proc_addr: 127.0.0.1:20160 -# Cop_wait_avg: 0.05 Cop_wait_p90: 0.6 Cop_wait_max: 0.8 Cop_wait_addr: 0.0.0.0:20160 -# Mem_max: 70724 -# Succ: true -# Plan: abcd -# Prev_stmt: update t set i = 2; -select * from t_slim;`)) - c.Assert(f.Sync(), IsNil) - c.Assert(err, IsNil) tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", slowLogFileName)) tk.MustExec("set time_zone = '+08:00';") @@ -643,6 +656,9 @@ select * from t_slim;`)) re.Check(testutil.RowsWithSep("|", "2019-02-12 11:33:56.571953|406315658548871171|root|127.0.0.1|6|4.895492|0.4|0.2|0.19|0.01|0|0.18|[txnLock]|0.03|0|15|480|1|8|0.161|0.101|0.092|1|100001|100000|test||0|42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772|t1:1,t2:2|0.1|0.2|0.03|127.0.0.1:20160|0.05|0.6|0.8|0.0.0.0:20160|70724|1|abcd|update t set i = 2;|select * from t_slim;")) // Test for long query. + f, err := os.OpenFile(slowLogFileName, os.O_CREATE|os.O_WRONLY, 0644) + c.Assert(err, IsNil) + defer f.Close() _, err = f.Write([]byte(` # Time: 2019-02-13T19:33:56.571953+08:00 `)) @@ -726,7 +742,7 @@ func (s *mockStore) StartGCWorker() error { panic("not implemented") } func (s *testClusterTableSuite) TestTiDBClusterInfo(c *C) { tk := testkit.NewTestKit(c, s.store) - err := tk.QueryToErr("select * from information_schema.tidb_cluster_info") + err := tk.QueryToErr("select * from information_schema.cluster_info") c.Assert(err, NotNil) mockAddr := s.mockAddr store := &mockStore{ @@ -734,7 +750,7 @@ func (s *testClusterTableSuite) TestTiDBClusterInfo(c *C) { mockAddr, } tk = testkit.NewTestKit(c, store) - tk.MustQuery("select * from information_schema.tidb_cluster_info").Check(testkit.Rows( + tk.MustQuery("select * from information_schema.cluster_info").Check(testkit.Rows( "tidb :4000 :10080 5.7.25-TiDB-None None", "pd "+mockAddr+" "+mockAddr+" 4.0.0-alpha mock-pd-githash", "tikv 127.0.0.1:20160 "+mockAddr+" 4.0.0-alpha mock-tikv-githash", @@ -748,7 +764,7 @@ func (s *testClusterTableSuite) TestTiDBClusterInfo(c *C) { fpExpr := `return("` + strings.Join(instances, ";") + `")` c.Assert(failpoint.Enable("github.com/pingcap/tidb/infoschema/mockClusterInfo", fpExpr), IsNil) defer func() { c.Assert(failpoint.Disable("github.com/pingcap/tidb/infoschema/mockClusterInfo"), IsNil) }() - tk.MustQuery("select * from information_schema.tidb_cluster_config").Check(testkit.Rows( + tk.MustQuery("select * from information_schema.cluster_config").Check(testkit.Rows( "pd 127.0.0.1:11080 key1 value1", "pd 127.0.0.1:11080 key2.nest1 n-value1", "pd 127.0.0.1:11080 key2.nest2 n-value2", @@ -771,7 +787,7 @@ func (s *testClusterTableSuite) TestTiDBClusterInfo(c *C) { "tikv 127.0.0.1:11080 key3.nest1 n-value1", "tikv 127.0.0.1:11080 key3.nest2 n-value2", )) - tk.MustQuery("select TYPE, `KEY`, VALUE from information_schema.tidb_cluster_config where `key`='key3.key4.nest4' order by type").Check(testkit.Rows( + tk.MustQuery("select TYPE, `KEY`, VALUE from information_schema.cluster_config where `key`='key3.key4.nest4' order by type").Check(testkit.Rows( "pd key3.key4.nest4 n-value5", "tidb key3.key4.nest4 n-value5", "tikv key3.key4.nest4 n-value5", @@ -822,7 +838,7 @@ func (s *testClusterTableSuite) TestForClusterServerInfo(c *C) { c.Assert(failpoint.Enable("github.com/pingcap/tidb/infoschema/mockClusterInfo", fpExpr), IsNil) defer func() { c.Assert(failpoint.Disable("github.com/pingcap/tidb/infoschema/mockClusterInfo"), IsNil) }() - re := tk.MustQuery("select * from information_schema.TIDB_CLUSTER_LOAD;") + re := tk.MustQuery("select * from information_schema.CLUSTER_LOAD;") rows := re.Rows() c.Assert(len(rows), Greater, 0) @@ -852,3 +868,45 @@ func (s *testClusterTableSuite) TestForClusterServerInfo(c *C) { c.Assert(len(addrMap), Equals, 0) c.Assert(len(nameMap), Equals, 0) } + +func (s *testTableSuite) TestSystemSchemaID(c *C) { + uniqueIDMap := make(map[int64]string) + s.checkSystemSchemaTableID(c, "information_schema", autoid.SystemSchemaIDFlag|1, 1, 10000, uniqueIDMap) + s.checkSystemSchemaTableID(c, "performance_schema", autoid.SystemSchemaIDFlag|10000, 10000, 20000, uniqueIDMap) +} + +func (s *testTableSuite) checkSystemSchemaTableID(c *C, dbName string, dbID, start, end int64, uniqueIDMap map[int64]string) { + is := s.dom.InfoSchema() + c.Assert(is, NotNil) + db, ok := is.SchemaByName(model.NewCIStr(dbName)) + c.Assert(ok, IsTrue) + c.Assert(db.ID, Equals, dbID) + // Test for information_schema table id. + tables := is.SchemaTables(model.NewCIStr(dbName)) + c.Assert(len(tables), Greater, 0) + for _, tbl := range tables { + tid := tbl.Meta().ID + comment := Commentf("table name is %v", tbl.Meta().Name) + c.Assert(tid&autoid.SystemSchemaIDFlag, Greater, int64(0), comment) + c.Assert(tid&^autoid.SystemSchemaIDFlag, Greater, start, comment) + c.Assert(tid&^autoid.SystemSchemaIDFlag, Less, end, comment) + name, ok := uniqueIDMap[tid] + c.Assert(ok, IsFalse, Commentf("schema id of %v is duplicate with %v, both is %v", name, tbl.Meta().Name, tid)) + uniqueIDMap[tid] = tbl.Meta().Name.O + } +} + +func (s *testClusterTableSuite) TestSelectClusterTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + slowLogFileName := "tidb-slow.log" + prepareSlowLogfile(c, slowLogFileName) + defer os.Remove(slowLogFileName) + tk.MustExec("use information_schema") + tk.MustQuery("select count(*) from `CLUSTER_SLOW_QUERY`").Check(testkit.Rows("1")) + tk.MustQuery("select count(*) from `CLUSTER_PROCESSLIST`").Check(testkit.Rows("1")) + tk.MustQuery("select * from `CLUSTER_PROCESSLIST`").Check(testkit.Rows("1 root 127.0.0.1 Query 9223372036 0 0 :10080")) + tk.MustQuery("select query_time, conn_id from `CLUSTER_SLOW_QUERY` order by time limit 1").Check(testkit.Rows("4.895492 6")) + tk.MustQuery("select count(*) from `CLUSTER_SLOW_QUERY` group by digest").Check(testkit.Rows("1")) + tk.MustQuery("select digest, count(*) from `CLUSTER_SLOW_QUERY` group by digest").Check(testkit.Rows("42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772 1")) + tk.MustQuery("select count(*) from `CLUSTER_SLOW_QUERY` where time > now() group by digest").Check(testkit.Rows()) +} diff --git a/kv/kv.go b/kv/kv.go index c1e3d55c9e5a0..137064a1d4fc0 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -226,12 +226,16 @@ const ( TiKV StoreType = iota // TiFlash means the type of a store is TiFlash. TiFlash + // TiDB means the type of a store is TiDB. + TiDB ) // Name returns the name of store type. func (t StoreType) Name() string { if t == TiFlash { return "tiflash" + } else if t == TiDB { + return "tidb" } return "tikv" } diff --git a/meta/autoid/autoid.go b/meta/autoid/autoid.go index 0b65c84595e13..5e7084c322aca 100755 --- a/meta/autoid/autoid.go +++ b/meta/autoid/autoid.go @@ -17,7 +17,6 @@ import ( "context" "math" "sync" - "sync/atomic" "time" "github.com/cznic/mathutil" @@ -31,6 +30,19 @@ import ( "go.uber.org/zap" ) +// Attention: +// For reading cluster TiDB memory tables, the system schema/table should be same. +// Once the system schema/table id been allocated, it can't be changed any more. +// Change the system schema/table id may have the compatibility problem. +const ( + // SystemSchemaIDFlag is the system schema/table id flag, uses the highest bit position as system schema ID flag, it's exports for test. + SystemSchemaIDFlag = 1 << 62 + // InformationSchemaDBID is the information_schema schema id, it's exports for test. + InformationSchemaDBID int64 = SystemSchemaIDFlag | 1 + // PerformanceSchemaDBID is the performance_schema schema id, it's exports for test. + PerformanceSchemaDBID int64 = SystemSchemaIDFlag | 10000 +) + const ( minStep = 30000 maxStep = 2000000 @@ -250,16 +262,9 @@ func NewAllocator(store kv.Storage, dbID int64, isUnsigned bool) Allocator { } } -//codeInvalidTableID is the code of autoid error. +// codeInvalidTableID is the code of autoid error. const codeInvalidTableID terror.ErrCode = 1 -var localSchemaID = int64(math.MaxInt64) - -// GenLocalSchemaID generates a local schema ID. -func GenLocalSchemaID() int64 { - return atomic.AddInt64(&localSchemaID, -1) -} - // Alloc implements autoid.Allocator Alloc interface. func (alloc *allocator) Alloc(tableID int64, n uint64) (int64, int64, error) { if tableID == 0 { diff --git a/planner/cascades/enforcer_rules_test.go b/planner/cascades/enforcer_rules_test.go index 6aa337dc5ed11..94cc96bd31991 100644 --- a/planner/cascades/enforcer_rules_test.go +++ b/planner/cascades/enforcer_rules_test.go @@ -22,7 +22,7 @@ import ( func (s *testCascadesSuite) TestGetEnforcerRules(c *C) { prop := &property.PhysicalProperty{} - group := memo.NewGroupWithSchema(nil, nil) + group := memo.NewGroupWithSchema(nil, expression.NewSchema()) enforcers := GetEnforcerRules(group, prop) c.Assert(enforcers, IsNil) col := &expression.Column{} @@ -37,7 +37,7 @@ func (s *testCascadesSuite) TestGetEnforcerRules(c *C) { func (s *testCascadesSuite) TestNewProperties(c *C) { prop := &property.PhysicalProperty{} col := &expression.Column{} - group := memo.NewGroupWithSchema(nil, nil) + group := memo.NewGroupWithSchema(nil, expression.NewSchema()) prop.Items = append(prop.Items, property.Item{Col: col}) enforcers := GetEnforcerRules(group, prop) orderEnforcer, _ := enforcers[0].(*OrderEnforcer) diff --git a/planner/cascades/implementation_rules.go b/planner/cascades/implementation_rules.go index 84d3319336f9c..b54e12f2f60f8 100644 --- a/planner/cascades/implementation_rules.go +++ b/planner/cascades/implementation_rules.go @@ -42,8 +42,11 @@ var defaultImplementationMap = map[memo.Operand][]ImplementationRule{ memo.OperandTableScan: { &ImplTableScan{}, }, - memo.OperandTableGather: { - &ImplTableGather{}, + memo.OperandIndexScan: { + &ImplIndexScan{}, + }, + memo.OperandTiKVSingleGather: { + &ImplTiKVSingleReadGather{}, }, memo.OperandShow: { &ImplShow{}, @@ -120,21 +123,26 @@ func (r *ImplProjection) OnImplement(expr *memo.GroupExpr, reqProp *property.Phy return impl.NewProjectionImpl(proj), nil } -// ImplTableGather implements TableGather as PhysicalTableReader. -type ImplTableGather struct { +// ImplTiKVSingleReadGather implements TiKVSingleGather +// as PhysicalTableReader or PhysicalIndexReader. +type ImplTiKVSingleReadGather struct { } // Match implements ImplementationRule Match interface. -func (r *ImplTableGather) Match(expr *memo.GroupExpr, prop *property.PhysicalProperty) (matched bool) { +func (r *ImplTiKVSingleReadGather) Match(expr *memo.GroupExpr, prop *property.PhysicalProperty) (matched bool) { return true } // OnImplement implements ImplementationRule OnImplement interface. -func (r *ImplTableGather) OnImplement(expr *memo.GroupExpr, reqProp *property.PhysicalProperty) (memo.Implementation, error) { +func (r *ImplTiKVSingleReadGather) OnImplement(expr *memo.GroupExpr, reqProp *property.PhysicalProperty) (memo.Implementation, error) { logicProp := expr.Group.Prop - tg := expr.ExprNode.(*plannercore.TableGather) - reader := tg.GetPhysicalReader(logicProp.Schema, logicProp.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt), reqProp) - return impl.NewTableReaderImpl(reader, tg.Source.TblColHists), nil + sg := expr.ExprNode.(*plannercore.TiKVSingleGather) + if sg.IsIndexGather { + reader := sg.GetPhysicalIndexReader(logicProp.Schema, logicProp.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt), reqProp) + return impl.NewIndexReaderImpl(reader, sg.Source.TblColHists), nil + } + reader := sg.GetPhysicalTableReader(logicProp.Schema, logicProp.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt), reqProp) + return impl.NewTableReaderImpl(reader, sg.Source.TblColHists), nil } // ImplTableScan implements TableScan as PhysicalTableScan. @@ -143,14 +151,14 @@ type ImplTableScan struct { // Match implements ImplementationRule Match interface. func (r *ImplTableScan) Match(expr *memo.GroupExpr, prop *property.PhysicalProperty) (matched bool) { - ts := expr.ExprNode.(*plannercore.TableScan) + ts := expr.ExprNode.(*plannercore.LogicalTableScan) return prop.IsEmpty() || (len(prop.Items) == 1 && ts.Handle != nil && prop.Items[0].Col.Equal(nil, ts.Handle)) } // OnImplement implements ImplementationRule OnImplement interface. func (r *ImplTableScan) OnImplement(expr *memo.GroupExpr, reqProp *property.PhysicalProperty) (memo.Implementation, error) { logicProp := expr.Group.Prop - logicalScan := expr.ExprNode.(*plannercore.TableScan) + logicalScan := expr.ExprNode.(*plannercore.LogicalTableScan) ts := logicalScan.GetPhysicalScan(logicProp.Schema, logicProp.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt)) if !reqProp.IsEmpty() { ts.KeepOrder = true @@ -160,6 +168,29 @@ func (r *ImplTableScan) OnImplement(expr *memo.GroupExpr, reqProp *property.Phys return impl.NewTableScanImpl(ts, tblCols, tblColHists), nil } +// ImplIndexScan implements IndexScan as PhysicalIndexScan. +type ImplIndexScan struct { +} + +// Match implements ImplementationRule Match interface. +func (r *ImplIndexScan) Match(expr *memo.GroupExpr, prop *property.PhysicalProperty) (matched bool) { + is := expr.ExprNode.(*plannercore.LogicalIndexScan) + return is.MatchIndexProp(prop) +} + +// OnImplement implements ImplementationRule OnImplement interface. +func (r *ImplIndexScan) OnImplement(expr *memo.GroupExpr, reqProp *property.PhysicalProperty) (memo.Implementation, error) { + logicalScan := expr.ExprNode.(*plannercore.LogicalIndexScan) + is := logicalScan.GetPhysicalIndexScan(expr.Group.Prop.Schema, expr.Group.Prop.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt)) + if !reqProp.IsEmpty() { + is.KeepOrder = true + if reqProp.Items[0].Desc { + is.Desc = true + } + } + return impl.NewIndexScanImpl(is, logicalScan.Source.TblColHists), nil +} + // ImplShow is the implementation rule which implements LogicalShow to // PhysicalShow. type ImplShow struct { diff --git a/planner/cascades/integration_test.go b/planner/cascades/integration_test.go index 564db6ef20da9..8e29d97544001 100644 --- a/planner/cascades/integration_test.go +++ b/planner/cascades/integration_test.go @@ -88,6 +88,30 @@ func (s *testIntegrationSuite) TestPKIsHandleRangeScan(c *C) { } } +func (s *testIntegrationSuite) TestIndexScan(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int primary key, b int, c int, d int, index idx_b(b), index idx_c_b(c, b))") + tk.MustExec("insert into t values(1,2,3,100),(4,5,6,200),(7,8,9,300)") + tk.MustExec("set session tidb_enable_cascades_planner = 1") + var input []string + var output []struct { + SQL string + Plan []string + Result []string + } + s.testData.GetTestCases(c, &input, &output) + for i, sql := range input { + s.testData.OnRecord(func() { + output[i].SQL = sql + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery("explain " + sql).Rows()) + output[i].Result = s.testData.ConvertRowsToStrings(tk.MustQuery(sql).Rows()) + }) + tk.MustQuery("explain " + sql).Check(testkit.Rows(output[i].Plan...)) + tk.MustQuery(sql).Check(testkit.Rows(output[i].Result...)) + } +} + func (s *testIntegrationSuite) TestBasicShow(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("drop table if exists t") diff --git a/planner/cascades/optimize.go b/planner/cascades/optimize.go index f4a318b2efb0c..c4ddff045b230 100644 --- a/planner/cascades/optimize.go +++ b/planner/cascades/optimize.go @@ -107,7 +107,7 @@ func (opt *Optimizer) FindBestPlan(sctx sessionctx.Context, logical plannercore. if err != nil { return nil, 0, err } - rootGroup := convert2Group(logical) + rootGroup := memo.Convert2Group(logical) err = opt.onPhaseExploration(sctx, rootGroup) if err != nil { return nil, 0, err @@ -120,32 +120,11 @@ func (opt *Optimizer) FindBestPlan(sctx sessionctx.Context, logical plannercore. return p, cost, err } -// convert2GroupExpr converts a logical plan to a GroupExpr. -func convert2GroupExpr(node plannercore.LogicalPlan) *memo.GroupExpr { - e := memo.NewGroupExpr(node) - e.Children = make([]*memo.Group, 0, len(node.Children())) - for _, child := range node.Children() { - childGroup := convert2Group(child) - e.Children = append(e.Children, childGroup) - } - return e -} - -// convert2Group converts a logical plan to a Group. -func convert2Group(node plannercore.LogicalPlan) *memo.Group { - e := convert2GroupExpr(node) - g := memo.NewGroupWithSchema(e, node.Schema()) - // Stats property for `Group` would be computed after exploration phase. - return g -} - func (opt *Optimizer) onPhasePreprocessing(sctx sessionctx.Context, plan plannercore.LogicalPlan) (plannercore.LogicalPlan, error) { err := plan.PruneColumns(plan.Schema().Columns) if err != nil { return nil, err } - // TODO: Build key info when convert LogicalPlan to GroupExpr. - plan.BuildKeyInfo() return plan, nil } diff --git a/planner/cascades/optimize_test.go b/planner/cascades/optimize_test.go index 9bc08fdf82418..73d5d6ae33365 100644 --- a/planner/cascades/optimize_test.go +++ b/planner/cascades/optimize_test.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/tidb/infoschema" plannercore "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/planner/memo" "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/util/testleak" @@ -61,7 +62,7 @@ func (s *testCascadesSuite) TestImplGroupZeroCost(c *C) { c.Assert(err, IsNil) logic, ok := p.(plannercore.LogicalPlan) c.Assert(ok, IsTrue) - rootGroup := convert2Group(logic) + rootGroup := memo.Convert2Group(logic) prop := &property.PhysicalProperty{ ExpectedCnt: math.MaxFloat64, } @@ -77,7 +78,7 @@ func (s *testCascadesSuite) TestInitGroupSchema(c *C) { c.Assert(err, IsNil) logic, ok := p.(plannercore.LogicalPlan) c.Assert(ok, IsTrue) - g := convert2Group(logic) + g := memo.Convert2Group(logic) c.Assert(g, NotNil) c.Assert(g.Prop, NotNil) c.Assert(g.Prop.Schema.Len(), Equals, 1) @@ -91,7 +92,7 @@ func (s *testCascadesSuite) TestFillGroupStats(c *C) { c.Assert(err, IsNil) logic, ok := p.(plannercore.LogicalPlan) c.Assert(ok, IsTrue) - rootGroup := convert2Group(logic) + rootGroup := memo.Convert2Group(logic) err = s.optimizer.fillGroupStats(rootGroup) c.Assert(err, IsNil) c.Assert(rootGroup.Prop.Stats, NotNil) diff --git a/planner/cascades/stringer_test.go b/planner/cascades/stringer_test.go index da5e2ec62a562..ae961832d605c 100644 --- a/planner/cascades/stringer_test.go +++ b/planner/cascades/stringer_test.go @@ -53,7 +53,7 @@ func (s *testStringerSuite) TearDownSuite(c *C) { func (s *testStringerSuite) TestGroupStringer(c *C) { s.optimizer.ResetTransformationRules(map[memo.Operand][]Transformation{ memo.OperandSelection: { - NewRulePushSelDownTableGather(), + NewRulePushSelDownTiKVSingleGather(), NewRulePushSelDownTableScan(), }, memo.OperandDataSource: { @@ -78,9 +78,10 @@ func (s *testStringerSuite) TestGroupStringer(c *C) { c.Assert(ok, IsTrue) logic, err = s.optimizer.onPhasePreprocessing(s.sctx, logic) c.Assert(err, IsNil) - group := convert2Group(logic) + group := memo.Convert2Group(logic) err = s.optimizer.onPhaseExploration(s.sctx, group) c.Assert(err, IsNil) + group.BuildKeyInfo() s.testData.OnRecord(func() { output[i].SQL = sql output[i].Result = ToString(group) diff --git a/planner/cascades/testdata/integration_suite_in.json b/planner/cascades/testdata/integration_suite_in.json index 84ec85bba9bfc..e4d537155c0c1 100644 --- a/planner/cascades/testdata/integration_suite_in.json +++ b/planner/cascades/testdata/integration_suite_in.json @@ -41,6 +41,15 @@ "select * from ((select a as aa from t t1) union all (select b as aa from t t2)) as t3 order by aa" ] }, + { + "name": "TestIndexScan", + "cases": [ + "select b from t", + "select a from t order by b", + "select c from t", + "select a from t order by c" + ] + }, { "name": "TestJoin", "cases": [ diff --git a/planner/cascades/testdata/integration_suite_out.json b/planner/cascades/testdata/integration_suite_out.json index c9bde66d59e6e..2f6f775b4e28e 100644 --- a/planner/cascades/testdata/integration_suite_out.json +++ b/planner/cascades/testdata/integration_suite_out.json @@ -279,10 +279,10 @@ { "SQL": "select b from t order by b limit 3", "Plan": [ - "Projection_8 3.00 root test.t.b", - "└─TopN_9 3.00 root test.t.b:asc, offset:0, count:3", - " └─TableReader_11 10000.00 root data:TableScan_12", - " └─TableScan_12 10000.00 cop[tikv] table:t, range:[-inf,+inf], keep order:false, stats:pseudo" + "Projection_9 3.00 root test.t.b", + "└─TopN_10 3.00 root test.t.b:asc, offset:0, count:3", + " └─TableReader_12 10000.00 root data:TableScan_13", + " └─TableScan_13 10000.00 cop[tikv] table:t, range:[-inf,+inf], keep order:false, stats:pseudo" ], "Result": [ "11", @@ -293,10 +293,10 @@ { "SQL": "select a from t order by a limit 1 offset 2", "Plan": [ - "Projection_8 1.00 root test.t.a", - "└─Limit_10 1.00 root offset:2, count:1", - " └─TableReader_13 3.00 root data:TableScan_14", - " └─TableScan_14 3.00 cop[tikv] table:t, range:[-inf,+inf], keep order:true, stats:pseudo" + "Projection_9 1.00 root test.t.a", + "└─Limit_11 1.00 root offset:2, count:1", + " └─TableReader_14 3.00 root data:TableScan_15", + " └─TableScan_15 3.00 cop[tikv] table:t, range:[-inf,+inf], keep order:true, stats:pseudo" ], "Result": [ "3" @@ -330,6 +330,65 @@ } ] }, + { + "Name": "TestIndexScan", + "Cases": [ + { + "SQL": "select b from t", + "Plan": [ + "Projection_9 10000.00 root test.t.b", + "└─IndexReader_14 10000.00 root index:IndexScan_15", + " └─IndexScan_15 10000.00 cop[tikv] table:t, index:b, range:[NULL,+inf], keep order:false, stats:pseudo" + ], + "Result": [ + "2", + "5", + "8" + ] + }, + { + "SQL": "select a from t order by b", + "Plan": [ + "Projection_11 10000.00 root test.t.a", + "└─Projection_13 10000.00 root test.t.a, test.t.b", + " └─IndexReader_16 10000.00 root index:IndexScan_17", + " └─IndexScan_17 10000.00 cop[tikv] table:t, index:b, range:[NULL,+inf], keep order:true, stats:pseudo" + ], + "Result": [ + "1", + "4", + "7" + ] + }, + { + "SQL": "select c from t", + "Plan": [ + "Projection_7 10000.00 root test.t.c", + "└─IndexReader_10 10000.00 root index:IndexScan_11", + " └─IndexScan_11 10000.00 cop[tikv] table:t, index:c, b, range:[NULL,+inf], keep order:false, stats:pseudo" + ], + "Result": [ + "3", + "6", + "9" + ] + }, + { + "SQL": "select a from t order by c", + "Plan": [ + "Projection_9 10000.00 root test.t.a", + "└─Projection_11 10000.00 root test.t.a, test.t.c", + " └─IndexReader_13 10000.00 root index:IndexScan_14", + " └─IndexScan_14 10000.00 cop[tikv] table:t, index:c, b, range:[NULL,+inf], keep order:true, stats:pseudo" + ], + "Result": [ + "1", + "4", + "7" + ] + } + ] + }, { "Name": "TestJoin", "Cases": [ diff --git a/planner/cascades/testdata/stringer_suite_out.json b/planner/cascades/testdata/stringer_suite_out.json index 1e6da38cd12e9..9daac9bdc98d5 100644 --- a/planner/cascades/testdata/stringer_suite_out.json +++ b/planner/cascades/testdata/stringer_suite_out.json @@ -8,7 +8,7 @@ "Group#0 Schema:[test.t.b]", " Projection_3 input:[Group#1], test.t.b", "Group#1 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_5 input:[Group#2]", + " TiKVSingleGather_5 input:[Group#2], table:t", "Group#2 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", " Selection_8 input:[Group#3], lt(test.t.b, 1)", "Group#3 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", @@ -34,13 +34,31 @@ "Group#2 Schema:[test.t.a,test.t.b,test.t.a]", " Join_3 input:[Group#3,Group#4], inner join", "Group#3 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_7 input:[Group#5]", + " TiKVSingleGather_7 input:[Group#5], table:t1", "Group#5 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", " TableScan_6 table:t1, pk col:test.t.a", "Group#4 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableGather_9 input:[Group#6]", + " TiKVSingleGather_9 input:[Group#6], table:t2", + " TiKVSingleGather_21 input:[Group#7], table:t2, index:e_d_c_str_prefix", + " TiKVSingleGather_19 input:[Group#8], table:t2, index:c_d_e_str", + " TiKVSingleGather_17 input:[Group#9], table:t2, index:f_g", + " TiKVSingleGather_15 input:[Group#10], table:t2, index:g", + " TiKVSingleGather_13 input:[Group#11], table:t2, index:f", + " TiKVSingleGather_11 input:[Group#12], table:t2, index:c_d_e", "Group#6 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableScan_8 table:t2, pk col:test.t.a" + " TableScan_8 table:t2, pk col:test.t.a", + "Group#7 Schema:[test.t.a]", + " IndexScan_20 table:t2, index:e_str, d_str, c_str", + "Group#8 Schema:[test.t.a]", + " IndexScan_18 table:t2, index:c_str, d_str, e_str", + "Group#9 Schema:[test.t.a]", + " IndexScan_16 table:t2, index:f, g", + "Group#10 Schema:[test.t.a]", + " IndexScan_14 table:t2, index:g", + "Group#11 Schema:[test.t.a]", + " IndexScan_12 table:t2, index:f", + "Group#12 Schema:[test.t.a]", + " IndexScan_10 table:t2, index:c, d, e" ] }, { @@ -51,7 +69,7 @@ "Group#1 Schema:[Column#13,Column#14]", " Aggregation_3 input:[Group#2], group by:test.t.d, funcs:max(test.t.b), sum(test.t.a)", "Group#2 Schema:[test.t.a,test.t.b,test.t.c,test.t.d], UniqueKey:[test.t.a]", - " TableGather_6 input:[Group#3]", + " TiKVSingleGather_6 input:[Group#3], table:t", "Group#3 Schema:[test.t.a,test.t.b,test.t.c,test.t.d], UniqueKey:[test.t.a]", " Selection_7 input:[Group#4], gt(test.t.c, 10)", "Group#4 Schema:[test.t.a,test.t.b,test.t.c,test.t.d], UniqueKey:[test.t.a]", @@ -66,7 +84,7 @@ "Group#1 Schema:[Column#13]", " Aggregation_3 input:[Group#2], funcs:avg(test.t.b)", "Group#2 Schema:[test.t.b]", - " TableGather_6 input:[Group#3]", + " TiKVSingleGather_6 input:[Group#3], table:t", "Group#3 Schema:[test.t.b]", " Selection_7 input:[Group#4], gt(test.t.b, 10)", "Group#4 Schema:[test.t.b]", @@ -83,15 +101,45 @@ "Group#2 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", " Apply_6 input:[Group#3,Group#4], semi join", "Group#3 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_9 input:[Group#5]", + " TiKVSingleGather_9 input:[Group#5], table:t1", "Group#5 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", " TableScan_8 table:t1, pk col:test.t.a", "Group#4 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableGather_11 input:[Group#6]", + " TiKVSingleGather_11 input:[Group#6], table:t2", + " TiKVSingleGather_13 input:[Group#7], table:t2, index:c_d_e", + " TiKVSingleGather_15 input:[Group#8], table:t2, index:f", + " TiKVSingleGather_17 input:[Group#9], table:t2, index:g", + " TiKVSingleGather_19 input:[Group#10], table:t2, index:f_g", + " TiKVSingleGather_21 input:[Group#11], table:t2, index:c_d_e_str", + " TiKVSingleGather_23 input:[Group#12], table:t2, index:e_d_c_str_prefix", "Group#6 Schema:[test.t.a], UniqueKey:[test.t.a]", - " Selection_12 input:[Group#7], lt(test.t.a, test.t.b)", - "Group#7 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableScan_10 table:t2, pk col:test.t.a" + " Selection_24 input:[Group#13], lt(test.t.a, test.t.b)", + "Group#13 Schema:[test.t.a], UniqueKey:[test.t.a]", + " TableScan_10 table:t2, pk col:test.t.a", + "Group#7 Schema:[test.t.a]", + " Selection_30 input:[Group#14], lt(test.t.a, test.t.b)", + "Group#14 Schema:[test.t.a]", + " IndexScan_12 table:t2, index:c, d, e", + "Group#8 Schema:[test.t.a]", + " Selection_29 input:[Group#15], lt(test.t.a, test.t.b)", + "Group#15 Schema:[test.t.a]", + " IndexScan_14 table:t2, index:f", + "Group#9 Schema:[test.t.a]", + " Selection_28 input:[Group#16], lt(test.t.a, test.t.b)", + "Group#16 Schema:[test.t.a]", + " IndexScan_16 table:t2, index:g", + "Group#10 Schema:[test.t.a]", + " Selection_27 input:[Group#17], lt(test.t.a, test.t.b)", + "Group#17 Schema:[test.t.a]", + " IndexScan_18 table:t2, index:f, g", + "Group#11 Schema:[test.t.a]", + " Selection_26 input:[Group#18], lt(test.t.a, test.t.b)", + "Group#18 Schema:[test.t.a]", + " IndexScan_20 table:t2, index:c_str, d_str, e_str", + "Group#12 Schema:[test.t.a]", + " Selection_25 input:[Group#19], lt(test.t.a, test.t.b)", + "Group#19 Schema:[test.t.a]", + " IndexScan_22 table:t2, index:e_str, d_str, c_str" ] }, { @@ -106,13 +154,31 @@ "Group#3 Schema:[test.t.a,test.t.b]", " Join_3 input:[Group#4,Group#5], inner join", "Group#4 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableGather_8 input:[Group#6]", + " TiKVSingleGather_8 input:[Group#6], table:t1", + " TiKVSingleGather_20 input:[Group#7], table:t1, index:e_d_c_str_prefix", + " TiKVSingleGather_18 input:[Group#8], table:t1, index:c_d_e_str", + " TiKVSingleGather_16 input:[Group#9], table:t1, index:f_g", + " TiKVSingleGather_14 input:[Group#10], table:t1, index:g", + " TiKVSingleGather_12 input:[Group#11], table:t1, index:f", + " TiKVSingleGather_10 input:[Group#12], table:t1, index:c_d_e", "Group#6 Schema:[test.t.a], UniqueKey:[test.t.a]", " TableScan_7 table:t1, pk col:test.t.a", + "Group#7 Schema:[test.t.a]", + " IndexScan_19 table:t1, index:e_str, d_str, c_str", + "Group#8 Schema:[test.t.a]", + " IndexScan_17 table:t1, index:c_str, d_str, e_str", + "Group#9 Schema:[test.t.a]", + " IndexScan_15 table:t1, index:f, g", + "Group#10 Schema:[test.t.a]", + " IndexScan_13 table:t1, index:g", + "Group#11 Schema:[test.t.a]", + " IndexScan_11 table:t1, index:f", + "Group#12 Schema:[test.t.a]", + " IndexScan_9 table:t1, index:c, d, e", "Group#5 Schema:[test.t.b]", - " TableGather_10 input:[Group#7]", - "Group#7 Schema:[test.t.b]", - " TableScan_9 table:t2" + " TiKVSingleGather_22 input:[Group#13], table:t2", + "Group#13 Schema:[test.t.b]", + " TableScan_21 table:t2" ] }, { @@ -123,9 +189,39 @@ "Group#1 Schema:[test.t.a], UniqueKey:[test.t.a]", " Projection_3 input:[Group#2], test.t.a", "Group#2 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableGather_6 input:[Group#3]", + " TiKVSingleGather_6 input:[Group#3], table:t", + " TiKVSingleGather_8 input:[Group#4], table:t, index:c_d_e", + " TiKVSingleGather_10 input:[Group#5], table:t, index:f", + " TiKVSingleGather_12 input:[Group#6], table:t, index:g", + " TiKVSingleGather_14 input:[Group#7], table:t, index:f_g", + " TiKVSingleGather_16 input:[Group#8], table:t, index:c_d_e_str", + " TiKVSingleGather_18 input:[Group#9], table:t, index:e_d_c_str_prefix", "Group#3 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableScan_8 table:t, pk col:test.t.a, cond:[gt(test.t.a, 10)]" + " TableScan_26 table:t, pk col:test.t.a, cond:[gt(test.t.a, 10)]", + "Group#4 Schema:[test.t.a]", + " Selection_25 input:[Group#10], gt(test.t.a, 10)", + "Group#10 Schema:[test.t.a]", + " IndexScan_7 table:t, index:c, d, e", + "Group#5 Schema:[test.t.a]", + " Selection_24 input:[Group#11], gt(test.t.a, 10)", + "Group#11 Schema:[test.t.a]", + " IndexScan_9 table:t, index:f", + "Group#6 Schema:[test.t.a]", + " Selection_23 input:[Group#12], gt(test.t.a, 10)", + "Group#12 Schema:[test.t.a]", + " IndexScan_11 table:t, index:g", + "Group#7 Schema:[test.t.a]", + " Selection_22 input:[Group#13], gt(test.t.a, 10)", + "Group#13 Schema:[test.t.a]", + " IndexScan_13 table:t, index:f, g", + "Group#8 Schema:[test.t.a]", + " Selection_21 input:[Group#14], gt(test.t.a, 10)", + "Group#14 Schema:[test.t.a]", + " IndexScan_15 table:t, index:c_str, d_str, e_str", + "Group#9 Schema:[test.t.a]", + " Selection_20 input:[Group#15], gt(test.t.a, 10)", + "Group#15 Schema:[test.t.a]", + " IndexScan_17 table:t, index:e_str, d_str, c_str" ] }, { @@ -138,7 +234,7 @@ "Group#2 Schema:[test.t.a,test.t.c], UniqueKey:[test.t.a]", " Projection_3 input:[Group#3], test.t.a, test.t.c", "Group#3 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]", - " TableGather_7 input:[Group#4]", + " TiKVSingleGather_7 input:[Group#4], table:t", "Group#4 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]", " Selection_8 input:[Group#5], gt(test.t.b, 1)", "Group#5 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]", @@ -157,19 +253,37 @@ "Group#4 Schema:[Column#26]", " Aggregation_5 input:[Group#5], funcs:avg(test.t.a)", "Group#5 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableGather_11 input:[Group#6]", + " TiKVSingleGather_11 input:[Group#6], table:t", + " TiKVSingleGather_23 input:[Group#7], table:t, index:e_d_c_str_prefix", + " TiKVSingleGather_21 input:[Group#8], table:t, index:c_d_e_str", + " TiKVSingleGather_19 input:[Group#9], table:t, index:f_g", + " TiKVSingleGather_17 input:[Group#10], table:t, index:g", + " TiKVSingleGather_15 input:[Group#11], table:t, index:f", + " TiKVSingleGather_13 input:[Group#12], table:t, index:c_d_e", "Group#6 Schema:[test.t.a], UniqueKey:[test.t.a]", " TableScan_10 table:t, pk col:test.t.a", + "Group#7 Schema:[test.t.a]", + " IndexScan_22 table:t, index:e_str, d_str, c_str", + "Group#8 Schema:[test.t.a]", + " IndexScan_20 table:t, index:c_str, d_str, e_str", + "Group#9 Schema:[test.t.a]", + " IndexScan_18 table:t, index:f, g", + "Group#10 Schema:[test.t.a]", + " IndexScan_16 table:t, index:g", + "Group#11 Schema:[test.t.a]", + " IndexScan_14 table:t, index:f", + "Group#12 Schema:[test.t.a]", + " IndexScan_12 table:t, index:c, d, e", "Group#2 Schema:[Column#27]", - " Projection_9 input:[Group#7], Column#13", - "Group#7 Schema:[Column#13]", - " Projection_3 input:[Group#8], Column#13", - "Group#8 Schema:[Column#13]", - " Aggregation_2 input:[Group#9], funcs:avg(test.t.b)", - "Group#9 Schema:[test.t.b]", - " TableGather_13 input:[Group#10]", - "Group#10 Schema:[test.t.b]", - " TableScan_12 table:t" + " Projection_9 input:[Group#13], Column#13", + "Group#13 Schema:[Column#13]", + " Projection_3 input:[Group#14], Column#13", + "Group#14 Schema:[Column#13]", + " Aggregation_2 input:[Group#15], funcs:avg(test.t.b)", + "Group#15 Schema:[test.t.b]", + " TiKVSingleGather_25 input:[Group#16], table:t", + "Group#16 Schema:[test.t.b]", + " TableScan_24 table:t" ] } ] diff --git a/planner/cascades/testdata/transformation_rules_suite_in.json b/planner/cascades/testdata/transformation_rules_suite_in.json index fe04e14bb0b31..f8d7d9c8eebee 100644 --- a/planner/cascades/testdata/transformation_rules_suite_in.json +++ b/planner/cascades/testdata/transformation_rules_suite_in.json @@ -5,8 +5,8 @@ "select a, b from (select a, b from t as t1 order by a) as t2 where t2.b > 10", "select a, b from (select a, b from t as t1 order by a) as t2 where t2.a > 10", "select a, b from (select a, b, a+b as a_b from t as t1) as t2 where a_b > 10 and b = 1", - "select a, @i:=@i+1 as ii from (select a, @i:=0 from t as t1) as t2 where @i < 10", - "select a, @i:=@i+1 as ii from (select a, @i:=0 from t as t1) as t2 where @i < 10 and a > 10", + "select b, @i:=@i+1 as ii from (select b, @i:=0 from t as t1) as t2 where @i < 10", + "select b, @i:=@i+1 as ii from (select a, b, @i:=0 from t as t1) as t2 where @i < 10 and a > 10", "select a, max(b) from t group by a having a > 1", "select a, avg(b) from t group by a having a > 1 and max(b) > 10", "select t1.a, t1.b, t2.b from t t1, t t2 where t1.a = t2.a and t2.b = t1.b and t1.a > 10 and t2.b > 10 and t1.a > t2.b", @@ -16,9 +16,9 @@ { "name": "TestAggPushDownGather", "cases": [ - "select a, sum(b) from t group by a", - "select a, sum(b) from t group by a+c, a", - "select a, sum(b) from t group by sin(a)+sin(c)" + "select b, sum(a) from t group by b", + "select b, sum(a) from t group by c, b", + "select b, sum(a) from t group by sin(b)+sin(c), b" ] }, { diff --git a/planner/cascades/testdata/transformation_rules_suite_out.json b/planner/cascades/testdata/transformation_rules_suite_out.json index d9a3d7cf1790a..53a72766631f5 100644 --- a/planner/cascades/testdata/transformation_rules_suite_out.json +++ b/planner/cascades/testdata/transformation_rules_suite_out.json @@ -5,107 +5,107 @@ { "SQL": "select a, b from (select a, b from t as t1 order by a) as t2 where t2.b > 10", "Result": [ - "Group#0 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#0 Schema:[test.t.a,test.t.b]", " Projection_5 input:[Group#1], test.t.a, test.t.b", - "Group#1 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#1 Schema:[test.t.a,test.t.b]", " Sort_3 input:[Group#2], test.t.a:asc", - "Group#2 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#2 Schema:[test.t.a,test.t.b]", " Projection_2 input:[Group#3], test.t.a, test.t.b", - "Group#3 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_7 input:[Group#4]", - "Group#4 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#3 Schema:[test.t.a,test.t.b]", + " TiKVSingleGather_7 input:[Group#4], table:t1", + "Group#4 Schema:[test.t.a,test.t.b]", " Selection_9 input:[Group#5], gt(test.t.b, 10)", - "Group#5 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#5 Schema:[test.t.a,test.t.b]", " TableScan_6 table:t1, pk col:test.t.a" ] }, { "SQL": "select a, b from (select a, b from t as t1 order by a) as t2 where t2.a > 10", "Result": [ - "Group#0 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#0 Schema:[test.t.a,test.t.b]", " Projection_5 input:[Group#1], test.t.a, test.t.b", - "Group#1 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#1 Schema:[test.t.a,test.t.b]", " Sort_3 input:[Group#2], test.t.a:asc", - "Group#2 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#2 Schema:[test.t.a,test.t.b]", " Projection_2 input:[Group#3], test.t.a, test.t.b", - "Group#3 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_7 input:[Group#4]", - "Group#4 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#3 Schema:[test.t.a,test.t.b]", + " TiKVSingleGather_7 input:[Group#4], table:t1", + "Group#4 Schema:[test.t.a,test.t.b]", " TableScan_10 table:t1, pk col:test.t.a, cond:[gt(test.t.a, 10)]" ] }, { "SQL": "select a, b from (select a, b, a+b as a_b from t as t1) as t2 where a_b > 10 and b = 1", "Result": [ - "Group#0 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#0 Schema:[test.t.a,test.t.b]", " Projection_4 input:[Group#1], test.t.a, test.t.b", - "Group#1 Schema:[test.t.a,test.t.b,Column#13], UniqueKey:[test.t.a]", + "Group#1 Schema:[test.t.a,test.t.b,Column#13]", " Projection_2 input:[Group#2], test.t.a, test.t.b, plus(test.t.a, test.t.b)->Column#13", - "Group#2 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_6 input:[Group#3]", - "Group#3 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#2 Schema:[test.t.a,test.t.b]", + " TiKVSingleGather_6 input:[Group#3], table:t1", + "Group#3 Schema:[test.t.a,test.t.b]", " Selection_8 input:[Group#4], eq(test.t.b, 1), gt(plus(test.t.a, test.t.b), 10)", - "Group#4 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#4 Schema:[test.t.a,test.t.b]", " TableScan_5 table:t1, pk col:test.t.a" ] }, { - "SQL": "select a, @i:=@i+1 as ii from (select a, @i:=0 from t as t1) as t2 where @i < 10", + "SQL": "select b, @i:=@i+1 as ii from (select b, @i:=0 from t as t1) as t2 where @i < 10", "Result": [ - "Group#0 Schema:[test.t.a,Column#14], UniqueKey:[test.t.a]", - " Projection_4 input:[Group#1], test.t.a, setvar(i, cast(plus(cast(getvar(i)), 1)))->Column#14", - "Group#1 Schema:[test.t.a,Column#13], UniqueKey:[test.t.a]", + "Group#0 Schema:[test.t.b,Column#14]", + " Projection_4 input:[Group#1], test.t.b, setvar(i, cast(plus(cast(getvar(i)), 1)))->Column#14", + "Group#1 Schema:[test.t.b,Column#13]", " Selection_3 input:[Group#2], lt(cast(getvar(\"i\")), 10)", - "Group#2 Schema:[test.t.a,Column#13], UniqueKey:[test.t.a]", - " Projection_2 input:[Group#3], test.t.a, setvar(i, 0)->Column#13", - "Group#3 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableGather_6 input:[Group#4]", - "Group#4 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableScan_5 table:t1, pk col:test.t.a" + "Group#2 Schema:[test.t.b,Column#13]", + " Projection_2 input:[Group#3], test.t.b, setvar(i, 0)->Column#13", + "Group#3 Schema:[test.t.b]", + " TiKVSingleGather_6 input:[Group#4], table:t1", + "Group#4 Schema:[test.t.b]", + " TableScan_5 table:t1" ] }, { - "SQL": "select a, @i:=@i+1 as ii from (select a, @i:=0 from t as t1) as t2 where @i < 10 and a > 10", + "SQL": "select b, @i:=@i+1 as ii from (select a, b, @i:=0 from t as t1) as t2 where @i < 10 and a > 10", "Result": [ - "Group#0 Schema:[test.t.a,Column#14], UniqueKey:[test.t.a]", - " Projection_4 input:[Group#1], test.t.a, setvar(i, cast(plus(cast(getvar(i)), 1)))->Column#14", - "Group#1 Schema:[test.t.a,Column#13], UniqueKey:[test.t.a]", + "Group#0 Schema:[test.t.b,Column#14]", + " Projection_4 input:[Group#1], test.t.b, setvar(i, cast(plus(cast(getvar(i)), 1)))->Column#14", + "Group#1 Schema:[test.t.a,test.t.b,Column#13]", " Selection_8 input:[Group#2], lt(cast(getvar(\"i\")), 10)", - "Group#2 Schema:[test.t.a,Column#13], UniqueKey:[test.t.a]", - " Projection_2 input:[Group#3], test.t.a, setvar(i, 0)->Column#13", - "Group#3 Schema:[test.t.a], UniqueKey:[test.t.a]", - " TableGather_6 input:[Group#4]", - "Group#4 Schema:[test.t.a], UniqueKey:[test.t.a]", + "Group#2 Schema:[test.t.a,test.t.b,Column#13]", + " Projection_2 input:[Group#3], test.t.a, test.t.b, setvar(i, 0)->Column#13", + "Group#3 Schema:[test.t.a,test.t.b]", + " TiKVSingleGather_6 input:[Group#4], table:t1", + "Group#4 Schema:[test.t.a,test.t.b]", " TableScan_10 table:t1, pk col:test.t.a, cond:[gt(test.t.a, 10)]" ] }, { "SQL": "select a, max(b) from t group by a having a > 1", "Result": [ - "Group#0 Schema:[test.t.a,Column#13], UniqueKey:[test.t.a,test.t.a]", + "Group#0 Schema:[test.t.a,Column#13]", " Projection_3 input:[Group#1], test.t.a, Column#13", - "Group#1 Schema:[Column#13,test.t.a], UniqueKey:[test.t.a,test.t.a]", + "Group#1 Schema:[Column#13,test.t.a]", " Aggregation_2 input:[Group#2], group by:test.t.a, funcs:max(test.t.b), firstrow(test.t.a)", - "Group#2 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_6 input:[Group#3]", - "Group#3 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#2 Schema:[test.t.a,test.t.b]", + " TiKVSingleGather_6 input:[Group#3], table:t", + "Group#3 Schema:[test.t.a,test.t.b]", " TableScan_10 table:t, pk col:test.t.a, cond:[gt(test.t.a, 1)]" ] }, { "SQL": "select a, avg(b) from t group by a having a > 1 and max(b) > 10", "Result": [ - "Group#0 Schema:[test.t.a,Column#16], UniqueKey:[test.t.a,test.t.a]", + "Group#0 Schema:[test.t.a,Column#16]", " Projection_5 input:[Group#1], test.t.a, Column#13", - "Group#1 Schema:[test.t.a,Column#13,Column#14], UniqueKey:[test.t.a,test.t.a]", + "Group#1 Schema:[test.t.a,Column#13,Column#14]", " Projection_3 input:[Group#2], test.t.a, Column#13, Column#14", - "Group#2 Schema:[Column#13,Column#14,test.t.a], UniqueKey:[test.t.a,test.t.a]", + "Group#2 Schema:[Column#13,Column#14,test.t.a]", " Selection_10 input:[Group#3], gt(Column#14, 10)", - "Group#3 Schema:[Column#13,Column#14,test.t.a], UniqueKey:[test.t.a,test.t.a]", + "Group#3 Schema:[Column#13,Column#14,test.t.a]", " Aggregation_2 input:[Group#4], group by:test.t.a, funcs:avg(test.t.b), max(test.t.b), firstrow(test.t.a)", - "Group#4 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_7 input:[Group#5]", - "Group#5 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#4 Schema:[test.t.a,test.t.b]", + " TiKVSingleGather_7 input:[Group#5], table:t", + "Group#5 Schema:[test.t.a,test.t.b]", " TableScan_12 table:t, pk col:test.t.a, cond:[gt(test.t.a, 1)]" ] }, @@ -116,17 +116,17 @@ " Projection_5 input:[Group#1], test.t.a, test.t.b, test.t.b", "Group#1 Schema:[test.t.a,test.t.b,test.t.a,test.t.b]", " Join_3 input:[Group#2,Group#3], inner join, equal:[eq(test.t.a, test.t.a) eq(test.t.b, test.t.b)], other cond:gt(test.t.a, test.t.b)", - "Group#2 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_7 input:[Group#4]", - "Group#4 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#2 Schema:[test.t.a,test.t.b]", + " TiKVSingleGather_7 input:[Group#4], table:t1", + "Group#4 Schema:[test.t.a,test.t.b]", " Selection_14 input:[Group#5], gt(test.t.a, test.t.b), gt(test.t.b, 10)", - "Group#5 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#5 Schema:[test.t.a,test.t.b]", " TableScan_13 table:t1, pk col:test.t.a, cond:[gt(test.t.a, 10)]", - "Group#3 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_9 input:[Group#6]", - "Group#6 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#3 Schema:[test.t.a,test.t.b]", + " TiKVSingleGather_9 input:[Group#6], table:t2", + "Group#6 Schema:[test.t.a,test.t.b]", " Selection_17 input:[Group#7], gt(test.t.a, test.t.b), gt(test.t.b, 10)", - "Group#7 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#7 Schema:[test.t.a,test.t.b]", " TableScan_16 table:t2, pk col:test.t.a, cond:[gt(test.t.a, 10)]" ] }, @@ -136,7 +136,7 @@ "Group#0 Schema:[test.t.a,test.t.b]", " Projection_5 input:[Group#1], test.t.a, test.t.b", "Group#1 Schema:[test.t.a,test.t.b,test.t.a]", - " TableDual_10 rowcount:0" + " TableDual_22 rowcount:0" ] } ] @@ -145,50 +145,50 @@ "Name": "TestAggPushDownGather", "Cases": [ { - "SQL": "select a, sum(b) from t group by a", + "SQL": "select b, sum(a) from t group by b", "Result": [ - "Group#0 Schema:[test.t.a,Column#13], UniqueKey:[test.t.a,test.t.a]", - " Projection_3 input:[Group#1], test.t.a, Column#13", - "Group#1 Schema:[Column#13,test.t.a], UniqueKey:[test.t.a,test.t.a]", - " Aggregation_2 input:[Group#2], group by:test.t.a, funcs:sum(test.t.b), firstrow(test.t.a)", - " Aggregation_7 input:[Group#3], group by:test.t.a, funcs:sum(Column#14), firstrow(test.t.a)", + "Group#0 Schema:[test.t.b,Column#13], UniqueKey:[test.t.b]", + " Projection_3 input:[Group#1], test.t.b, Column#13", + "Group#1 Schema:[Column#13,test.t.b], UniqueKey:[test.t.b]", + " Aggregation_2 input:[Group#2], group by:test.t.b, funcs:sum(test.t.a), firstrow(test.t.b)", + " Aggregation_7 input:[Group#3], group by:test.t.b, funcs:sum(Column#14), firstrow(test.t.b)", "Group#2 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_5 input:[Group#4]", + " TiKVSingleGather_5 input:[Group#4], table:t", "Group#4 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", " TableScan_4 table:t, pk col:test.t.a", - "Group#3 Schema:[Column#14,test.t.a]", - " TableGather_5 input:[Group#5]", - "Group#5 Schema:[Column#14,test.t.a]", - " Aggregation_6 input:[Group#4], group by:test.t.a, funcs:sum(test.t.b)" + "Group#3 Schema:[Column#14,test.t.b]", + " TiKVSingleGather_5 input:[Group#5], table:t", + "Group#5 Schema:[Column#14,test.t.b]", + " Aggregation_6 input:[Group#4], group by:test.t.b, funcs:sum(test.t.a)" ] }, { - "SQL": "select a, sum(b) from t group by a+c, a", + "SQL": "select b, sum(a) from t group by c, b", "Result": [ - "Group#0 Schema:[test.t.a,Column#13], UniqueKey:[test.t.a]", - " Projection_3 input:[Group#1], test.t.a, Column#13", - "Group#1 Schema:[Column#13,test.t.a], UniqueKey:[test.t.a]", - " Aggregation_2 input:[Group#2], group by:plus(test.t.a, test.t.c), test.t.a, funcs:sum(test.t.b), firstrow(test.t.a)", - " Aggregation_7 input:[Group#3], group by:Column#16, test.t.a, funcs:sum(Column#14), firstrow(test.t.a)", + "Group#0 Schema:[test.t.b,Column#13]", + " Projection_3 input:[Group#1], test.t.b, Column#13", + "Group#1 Schema:[Column#13,test.t.b]", + " Aggregation_2 input:[Group#2], group by:test.t.b, test.t.c, funcs:sum(test.t.a), firstrow(test.t.b)", + " Aggregation_7 input:[Group#3], group by:test.t.b, test.t.c, funcs:sum(Column#14), firstrow(test.t.b)", "Group#2 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]", - " TableGather_5 input:[Group#4]", + " TiKVSingleGather_5 input:[Group#4], table:t", "Group#4 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]", " TableScan_4 table:t, pk col:test.t.a", - "Group#3 Schema:[Column#14,Column#16,test.t.a]", - " TableGather_5 input:[Group#5]", - "Group#5 Schema:[Column#14,Column#16,test.t.a]", - " Aggregation_6 input:[Group#4], group by:plus(test.t.a, test.t.c), test.t.a, funcs:sum(test.t.b)" + "Group#3 Schema:[Column#14,test.t.c,test.t.b]", + " TiKVSingleGather_5 input:[Group#5], table:t", + "Group#5 Schema:[Column#14,test.t.c,test.t.b]", + " Aggregation_6 input:[Group#4], group by:test.t.b, test.t.c, funcs:sum(test.t.a)" ] }, { - "SQL": "select a, sum(b) from t group by sin(a)+sin(c)", + "SQL": "select b, sum(a) from t group by sin(b)+sin(c), b", "Result": [ - "Group#0 Schema:[test.t.a,Column#13], UniqueKey:[test.t.a]", - " Projection_3 input:[Group#1], test.t.a, Column#13", - "Group#1 Schema:[Column#13,test.t.a], UniqueKey:[test.t.a]", - " Aggregation_2 input:[Group#2], group by:plus(sin(cast(test.t.a)), sin(cast(test.t.c))), funcs:sum(test.t.b), firstrow(test.t.a)", + "Group#0 Schema:[test.t.b,Column#13]", + " Projection_3 input:[Group#1], test.t.b, Column#13", + "Group#1 Schema:[Column#13,test.t.b]", + " Aggregation_2 input:[Group#2], group by:plus(sin(cast(test.t.b)), sin(cast(test.t.c))), test.t.b, funcs:sum(test.t.a), firstrow(test.t.b)", "Group#2 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]", - " TableGather_5 input:[Group#3]", + " TiKVSingleGather_5 input:[Group#3], table:t", "Group#3 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]", " TableScan_4 table:t, pk col:test.t.a" ] @@ -203,13 +203,13 @@ "Result": [ "Group#0 Schema:[test.t.b]", " Projection_5 input:[Group#1], test.t.b", - "Group#1 Schema:[test.t.b,test.t.a], UniqueKey:[test.t.a]", + "Group#1 Schema:[test.t.b,test.t.a]", " Projection_2 input:[Group#2], test.t.b, test.t.a", - "Group#2 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TopN_8 input:[Group#3], test.t.a:asc, offset:0, count:2", - "Group#3 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_7 input:[Group#4]", - "Group#4 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#2 Schema:[test.t.a,test.t.b]", + " TopN_9 input:[Group#3], test.t.a:asc, offset:0, count:2", + "Group#3 Schema:[test.t.a,test.t.b]", + " TiKVSingleGather_7 input:[Group#4], table:t", + "Group#4 Schema:[test.t.a,test.t.b]", " TableScan_6 table:t, pk col:test.t.a" ] }, @@ -218,13 +218,13 @@ "Result": [ "Group#0 Schema:[Column#14]", " Projection_5 input:[Group#1], Column#13", - "Group#1 Schema:[Column#13,test.t.a], UniqueKey:[test.t.a]", + "Group#1 Schema:[Column#13,test.t.a]", " Projection_2 input:[Group#2], plus(test.t.a, test.t.b)->Column#13, test.t.a", - "Group#2 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TopN_8 input:[Group#3], test.t.a:asc, offset:2, count:1", - "Group#3 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", - " TableGather_7 input:[Group#4]", - "Group#4 Schema:[test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#2 Schema:[test.t.a,test.t.b]", + " TopN_9 input:[Group#3], test.t.a:asc, offset:2, count:1", + "Group#3 Schema:[test.t.a,test.t.b]", + " TiKVSingleGather_7 input:[Group#4], table:t", + "Group#4 Schema:[test.t.a,test.t.b]", " TableScan_6 table:t, pk col:test.t.a" ] }, @@ -233,14 +233,17 @@ "Result": [ "Group#0 Schema:[test.t.c]", " Projection_5 input:[Group#1], test.t.c", - "Group#1 Schema:[test.t.c,test.t.a], UniqueKey:[test.t.a]", + "Group#1 Schema:[test.t.c,test.t.a]", " Projection_2 input:[Group#2], test.t.c, test.t.a", - "Group#2 Schema:[test.t.a,test.t.c], UniqueKey:[test.t.a]", - " TopN_8 input:[Group#3], test.t.a:asc, offset:0, count:1", - "Group#3 Schema:[test.t.a,test.t.c], UniqueKey:[test.t.a]", - " TableGather_7 input:[Group#4]", - "Group#4 Schema:[test.t.a,test.t.c], UniqueKey:[test.t.a]", - " TableScan_6 table:t, pk col:test.t.a" + "Group#2 Schema:[test.t.a,test.t.c]", + " TopN_11 input:[Group#3], test.t.a:asc, offset:0, count:1", + "Group#3 Schema:[test.t.a,test.t.c]", + " TiKVSingleGather_7 input:[Group#4], table:t", + " TiKVSingleGather_9 input:[Group#5], table:t, index:c_d_e", + "Group#4 Schema:[test.t.a,test.t.c]", + " TableScan_6 table:t, pk col:test.t.a", + "Group#5 Schema:[test.t.a,test.t.c]", + " IndexScan_8 table:t, index:c, d, e" ] }, { @@ -248,13 +251,13 @@ "Result": [ "Group#0 Schema:[test.t.c]", " Projection_5 input:[Group#1], test.t.c", - "Group#1 Schema:[test.t.c,test.t.a,test.t.b], UniqueKey:[test.t.a]", + "Group#1 Schema:[test.t.c,test.t.a,test.t.b]", " Projection_2 input:[Group#2], test.t.c, test.t.a, test.t.b", - "Group#2 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]", - " TopN_8 input:[Group#3], plus(test.t.a, test.t.b):asc, offset:0, count:1", - "Group#3 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]", - " TableGather_7 input:[Group#4]", - "Group#4 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]", + "Group#2 Schema:[test.t.a,test.t.b,test.t.c]", + " TopN_9 input:[Group#3], plus(test.t.a, test.t.b):asc, offset:0, count:1", + "Group#3 Schema:[test.t.a,test.t.b,test.t.c]", + " TiKVSingleGather_7 input:[Group#4], table:t", + "Group#4 Schema:[test.t.a,test.t.b,test.t.c]", " TableScan_6 table:t, pk col:test.t.a" ] } diff --git a/planner/cascades/transformation_rules.go b/planner/cascades/transformation_rules.go index 0c5fd8b949376..04a93dc730e69 100644 --- a/planner/cascades/transformation_rules.go +++ b/planner/cascades/transformation_rules.go @@ -45,7 +45,7 @@ type Transformation interface { var defaultTransformationMap = map[memo.Operand][]Transformation{ memo.OperandSelection: { NewRulePushSelDownTableScan(), - NewRulePushSelDownTableGather(), + NewRulePushSelDownTiKVSingleGather(), NewRulePushSelDownSort(), NewRulePushSelDownProjection(), NewRulePushSelDownAggregation(), @@ -104,7 +104,7 @@ func NewRulePushSelDownTableScan() Transformation { // the key ranges of the `ts` operator. func (r *PushSelDownTableScan) OnTransform(old *memo.ExprIter) (newExprs []*memo.GroupExpr, eraseOld bool, eraseAll bool, err error) { sel := old.GetExpr().ExprNode.(*plannercore.LogicalSelection) - ts := old.Children[0].GetExpr().ExprNode.(*plannercore.TableScan) + ts := old.Children[0].GetExpr().ExprNode.(*plannercore.LogicalTableScan) if ts.Handle == nil { return nil, false, false, nil } @@ -112,7 +112,7 @@ func (r *PushSelDownTableScan) OnTransform(old *memo.ExprIter) (newExprs []*memo if accesses == nil { return nil, false, false, nil } - newTblScan := plannercore.TableScan{ + newTblScan := plannercore.LogicalTableScan{ Source: ts.Source, Handle: ts.Handle, AccessConds: ts.AccessConds.Shallow(), @@ -132,19 +132,19 @@ func (r *PushSelDownTableScan) OnTransform(old *memo.ExprIter) (newExprs []*memo return []*memo.GroupExpr{selExpr}, true, false, nil } -// PushSelDownTableGather pushes the selection down to child of TableGather. -type PushSelDownTableGather struct { +// PushSelDownTiKVSingleGather pushes the selection down to child of TiKVSingleGather. +type PushSelDownTiKVSingleGather struct { baseRule } -// NewRulePushSelDownTableGather creates a new Transformation PushSelDownTableGather. -// The pattern of this rule is `Selection -> TableGather -> Any`. -func NewRulePushSelDownTableGather() Transformation { +// NewRulePushSelDownTiKVSingleGather creates a new Transformation PushSelDownTiKVSingleGather. +// The pattern of this rule is `Selection -> TiKVSingleGather -> Any`. +func NewRulePushSelDownTiKVSingleGather() Transformation { any := memo.NewPattern(memo.OperandAny, memo.EngineTiKVOrTiFlash) - tg := memo.BuildPattern(memo.OperandTableGather, memo.EngineTiDBOnly, any) + tg := memo.BuildPattern(memo.OperandTiKVSingleGather, memo.EngineTiDBOnly, any) p := memo.BuildPattern(memo.OperandSelection, memo.EngineTiDBOnly, tg) - rule := &PushSelDownTableGather{} + rule := &PushSelDownTiKVSingleGather{} rule.pattern = p return rule } @@ -154,12 +154,12 @@ func NewRulePushSelDownTableGather() Transformation { // It transforms `oldSel -> oldTg -> any` to one of the following new exprs: // 1. `newTg -> pushedSel -> any` // 2. `remainedSel -> newTg -> pushedSel -> any` -func (r *PushSelDownTableGather) OnTransform(old *memo.ExprIter) (newExprs []*memo.GroupExpr, eraseOld bool, eraseAll bool, err error) { +func (r *PushSelDownTiKVSingleGather) OnTransform(old *memo.ExprIter) (newExprs []*memo.GroupExpr, eraseOld bool, eraseAll bool, err error) { sel := old.GetExpr().ExprNode.(*plannercore.LogicalSelection) - tg := old.Children[0].GetExpr().ExprNode.(*plannercore.TableGather) + sg := old.Children[0].GetExpr().ExprNode.(*plannercore.TiKVSingleGather) childGroup := old.Children[0].Children[0].Group var pushed, remained []expression.Expression - sctx := tg.SCtx() + sctx := sg.SCtx() _, pushed, remained = expression.ExpressionsToPB(sctx.GetSessionVars().StmtCtx, sel.Conditions, sctx.GetClient()) if len(pushed) == 0 { return nil, false, false, nil @@ -168,12 +168,12 @@ func (r *PushSelDownTableGather) OnTransform(old *memo.ExprIter) (newExprs []*me pushedSelExpr := memo.NewGroupExpr(pushedSel) pushedSelExpr.Children = append(pushedSelExpr.Children, childGroup) pushedSelGroup := memo.NewGroupWithSchema(pushedSelExpr, childGroup.Prop.Schema).SetEngineType(childGroup.EngineType) - // The field content of TableGather would not be modified currently, so we + // The field content of TiKVSingleGather would not be modified currently, so we // just reference the same tg instead of making a copy of it. // - // TODO: if we save pushed filters later in TableGather, in order to do partition - // pruning or skyline pruning, we need to make a copy of the TableGather here. - tblGatherExpr := memo.NewGroupExpr(tg) + // TODO: if we save pushed filters later in TiKVSingleGather, in order to do partition + // pruning or skyline pruning, we need to make a copy of the TiKVSingleGather here. + tblGatherExpr := memo.NewGroupExpr(sg) tblGatherExpr.Children = append(tblGatherExpr.Children, pushedSelGroup) if len(remained) == 0 { // `oldSel -> oldTg -> any` is transformed to `newTg -> pushedSel -> any`. @@ -205,7 +205,7 @@ func (r *EnumeratePaths) OnTransform(old *memo.ExprIter) (newExprs []*memo.Group ds := old.GetExpr().ExprNode.(*plannercore.DataSource) gathers := ds.Convert2Gathers() for _, gather := range gathers { - expr := convert2GroupExpr(gather) + expr := memo.Convert2GroupExpr(gather) expr.Children[0].SetEngineType(memo.EngineTiKV) newExprs = append(newExprs, expr) } @@ -213,19 +213,19 @@ func (r *EnumeratePaths) OnTransform(old *memo.ExprIter) (newExprs []*memo.Group } // PushAggDownGather splits Aggregation to two stages, final and partial1, -// and pushed the partial Aggregation down to the child of TableGather. +// and pushed the partial Aggregation down to the child of TiKVSingleGather. type PushAggDownGather struct { baseRule } // NewRulePushAggDownGather creates a new Transformation PushAggDownGather. -// The pattern of this rule is: `Aggregation -> TableGather`. +// The pattern of this rule is: `Aggregation -> TiKVSingleGather`. func NewRulePushAggDownGather() Transformation { rule := &PushAggDownGather{} rule.pattern = memo.BuildPattern( memo.OperandAggregation, memo.EngineTiDBOnly, - memo.NewPattern(memo.OperandTableGather, memo.EngineTiDBOnly), + memo.NewPattern(memo.OperandTiKVSingleGather, memo.EngineTiDBOnly), ) return rule } @@ -251,7 +251,7 @@ func (r *PushAggDownGather) Match(expr *memo.ExprIter) bool { func (r *PushAggDownGather) OnTransform(old *memo.ExprIter) (newExprs []*memo.GroupExpr, eraseOld bool, eraseAll bool, err error) { agg := old.GetExpr().ExprNode.(*plannercore.LogicalAggregation) aggSchema := old.GetExpr().Group.Prop.Schema - gather := old.Children[0].GetExpr().ExprNode.(*plannercore.TableGather) + gather := old.Children[0].GetExpr().ExprNode.(*plannercore.TiKVSingleGather) childGroup := old.Children[0].GetExpr().Children[0] // The old Aggregation should stay unchanged for other transformation. // So we build a new LogicalAggregation for the partialAgg. @@ -359,6 +359,7 @@ func NewRulePushSelDownProjection() Transformation { func (r *PushSelDownProjection) OnTransform(old *memo.ExprIter) (newExprs []*memo.GroupExpr, eraseOld bool, eraseAll bool, err error) { sel := old.GetExpr().ExprNode.(*plannercore.LogicalSelection) proj := old.Children[0].GetExpr().ExprNode.(*plannercore.LogicalProjection) + projSchema := old.Children[0].Prop.Schema childGroup := old.Children[0].GetExpr().Children[0] for _, expr := range proj.Exprs { if expression.HasAssignSetVarFunc(expr) { @@ -369,7 +370,7 @@ func (r *PushSelDownProjection) OnTransform(old *memo.ExprIter) (newExprs []*mem canNotBePushed := make([]expression.Expression, 0, len(sel.Conditions)) for _, cond := range sel.Conditions { if !expression.HasGetSetVarFunc(cond) { - canBePushed = append(canBePushed, expression.ColumnSubstitute(cond, proj.Schema(), proj.Exprs)) + canBePushed = append(canBePushed, expression.ColumnSubstitute(cond, projSchema, proj.Exprs)) } else { canNotBePushed = append(canNotBePushed, cond) } @@ -386,7 +387,7 @@ func (r *PushSelDownProjection) OnTransform(old *memo.ExprIter) (newExprs []*mem if len(canNotBePushed) == 0 { return []*memo.GroupExpr{newProjExpr}, true, false, nil } - newProjGroup := memo.NewGroupWithSchema(newProjExpr, proj.Schema()) + newProjGroup := memo.NewGroupWithSchema(newProjExpr, projSchema) newTopSel := plannercore.LogicalSelection{Conditions: canNotBePushed}.Init(sel.SCtx(), sel.SelectBlockOffset()) newTopSelExpr := memo.NewGroupExpr(newTopSel) newTopSelExpr.SetChildren(newProjGroup) @@ -416,6 +417,7 @@ func NewRulePushSelDownAggregation() Transformation { func (r *PushSelDownAggregation) OnTransform(old *memo.ExprIter) (newExprs []*memo.GroupExpr, eraseOld bool, eraseAll bool, err error) { sel := old.GetExpr().ExprNode.(*plannercore.LogicalSelection) agg := old.Children[0].GetExpr().ExprNode.(*plannercore.LogicalAggregation) + aggSchema := old.Children[0].Prop.Schema var pushedExprs []expression.Expression var remainedExprs []expression.Expression exprsOriginal := make([]expression.Expression, 0, len(agg.AggFuncs)) @@ -442,7 +444,7 @@ func (r *PushSelDownAggregation) OnTransform(old *memo.ExprIter) (newExprs []*me } if canPush { // TODO: Don't substitute since they should be the same column. - newCond := expression.ColumnSubstitute(cond, agg.Schema(), exprsOriginal) + newCond := expression.ColumnSubstitute(cond, aggSchema, exprsOriginal) pushedExprs = append(pushedExprs, newCond) } else { remainedExprs = append(remainedExprs, cond) @@ -469,7 +471,7 @@ func (r *PushSelDownAggregation) OnTransform(old *memo.ExprIter) (newExprs []*me return []*memo.GroupExpr{aggGroupExpr}, true, false, nil } - aggGroup := memo.NewGroupWithSchema(aggGroupExpr, agg.Schema()) + aggGroup := memo.NewGroupWithSchema(aggGroupExpr, aggSchema) remainedSel := plannercore.LogicalSelection{Conditions: remainedExprs}.Init(sctx, sel.SelectBlockOffset()) remainedGroupExpr := memo.NewGroupExpr(remainedSel) remainedGroupExpr.SetChildren(aggGroup) @@ -615,18 +617,26 @@ func (r *PushTopNDownProjection) OnTransform(old *memo.ExprIter) (newExprs []*me topN := old.GetExpr().ExprNode.(*plannercore.LogicalTopN) proj := old.Children[0].GetExpr().ExprNode.(*plannercore.LogicalProjection) childGroup := old.Children[0].GetExpr().Children[0] - for _, by := range topN.ByItems { + + newTopN := plannercore.LogicalTopN{ + Offset: topN.Offset, + Count: topN.Count, + }.Init(topN.SCtx(), topN.SelectBlockOffset()) + + newTopN.ByItems = append(newTopN.ByItems, topN.ByItems...) + for _, by := range newTopN.ByItems { by.Expr = expression.ColumnSubstitute(by.Expr, old.Children[0].Group.Prop.Schema, proj.Exprs) } + // remove meaningless constant sort items. - for i := len(topN.ByItems) - 1; i >= 0; i-- { - switch topN.ByItems[i].Expr.(type) { + for i := len(newTopN.ByItems) - 1; i >= 0; i-- { + switch newTopN.ByItems[i].Expr.(type) { case *expression.Constant, *expression.CorrelatedColumn: - topN.ByItems = append(topN.ByItems[:i], topN.ByItems[i+1:]...) + topN.ByItems = append(newTopN.ByItems[:i], newTopN.ByItems[i+1:]...) } } projExpr := memo.NewGroupExpr(proj) - topNExpr := memo.NewGroupExpr(topN) + topNExpr := memo.NewGroupExpr(newTopN) topNExpr.SetChildren(childGroup) topNGroup := memo.NewGroupWithSchema(topNExpr, childGroup.Prop.Schema) projExpr.SetChildren(topNGroup) diff --git a/planner/cascades/transformation_rules_test.go b/planner/cascades/transformation_rules_test.go index 397a3624b89f6..1aeeb99b2dc4f 100644 --- a/planner/cascades/transformation_rules_test.go +++ b/planner/cascades/transformation_rules_test.go @@ -63,7 +63,7 @@ func testGroupToString(input []string, output []struct { c.Assert(ok, IsTrue) logic, err = s.optimizer.onPhasePreprocessing(s.sctx, logic) c.Assert(err, IsNil) - group := convert2Group(logic) + group := memo.Convert2Group(logic) err = s.optimizer.onPhaseExploration(s.sctx, group) c.Assert(err, IsNil) s.testData.OnRecord(func() { @@ -92,14 +92,33 @@ func (s *testTransformationRuleSuite) TestAggPushDownGather(c *C) { Result []string } s.testData.GetTestCases(c, &input, &output) - testGroupToString(input, output, s, c) + for i, sql := range input { + stmt, err := s.ParseOneStmt(sql, "", "") + c.Assert(err, IsNil) + p, _, err := plannercore.BuildLogicalPlan(context.Background(), s.sctx, stmt, s.is) + c.Assert(err, IsNil) + logic, ok := p.(plannercore.LogicalPlan) + c.Assert(ok, IsTrue) + logic, err = s.optimizer.onPhasePreprocessing(s.sctx, logic) + c.Assert(err, IsNil) + group := memo.Convert2Group(logic) + err = s.optimizer.onPhaseExploration(s.sctx, group) + c.Assert(err, IsNil) + // BuildKeyInfo here to test the KeyInfo for partialAgg. + group.BuildKeyInfo() + s.testData.OnRecord(func() { + output[i].SQL = sql + output[i].Result = ToString(group) + }) + c.Assert(ToString(group), DeepEquals, output[i].Result) + } } func (s *testTransformationRuleSuite) TestPredicatePushDown(c *C) { s.optimizer.ResetTransformationRules(map[memo.Operand][]Transformation{ memo.OperandSelection: { NewRulePushSelDownTableScan(), - NewRulePushSelDownTableGather(), + NewRulePushSelDownTiKVSingleGather(), NewRulePushSelDownSort(), NewRulePushSelDownProjection(), NewRulePushSelDownAggregation(), diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index bf3f591dd8c63..878eff8d1a8df 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -94,7 +94,7 @@ func (s *testAnalyzeSuite) TestExplainAnalyze(c *C) { rs := tk.MustQuery("explain analyze select t1.a, t1.b, sum(t1.c) from t1 join t2 on t1.a = t2.b where t1.a > 1") c.Assert(len(rs.Rows()), Equals, 10) for _, row := range rs.Rows() { - c.Assert(len(row), Equals, 6) + c.Assert(len(row), Equals, 7) execInfo := row[4].(string) c.Assert(strings.Contains(execInfo, "time"), Equals, true) c.Assert(strings.Contains(execInfo, "loops"), Equals, true) @@ -131,6 +131,8 @@ func (s *testAnalyzeSuite) TestCBOWithoutAnalyze(c *C) { " └─Selection_14 5.99 cop[tikv] not(isnull(test.t2.a))", " └─TableScan_13 6.00 cop[tikv] table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) + testKit.MustQuery("explain format = 'hint' select * from t1, t2 where t1.a = t2.a").Check(testkit.Rows( + "USE_INDEX(@`sel_1` `test`.`t1` ), USE_INDEX(@`sel_1` `test`.`t2` ), HASH_JOIN(@`sel_1` `test`.`t1`)")) } func (s *testAnalyzeSuite) TestStraightJoin(c *C) { diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 3687edab2d0ac..e5299ab8395ec 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -675,9 +675,11 @@ func (e *Explain) prepareSchema() error { case format == ast.ExplainFormatROW && !e.Analyze: fieldNames = []string{"id", "count", "task", "operator info"} case format == ast.ExplainFormatROW && e.Analyze: - fieldNames = []string{"id", "count", "task", "operator info", "execution info", "memory"} + fieldNames = []string{"id", "count", "task", "operator info", "execution info", "memory", "disk"} case format == ast.ExplainFormatDOT: fieldNames = []string{"dot contents"} + case format == ast.ExplainFormatHint: + fieldNames = []string{"hint"} default: return errors.Errorf("explain format '%s' is not supported now", e.Format) } @@ -709,6 +711,8 @@ func (e *Explain) RenderResult() error { } case ast.ExplainFormatDOT: e.prepareDotInfo(e.TargetPlan.(PhysicalPlan)) + case ast.ExplainFormatHint: + e.Rows = append(e.Rows, []string{GenHintsFromPhysicalPlan(e.TargetPlan)}) default: return errors.Errorf("explain format '%s' is not supported now", e.Format) } @@ -739,14 +743,12 @@ func (e *Explain) explainPlanInRowFormat(p Plan, taskType, indent string, isLast case *PhysicalTableReader: var storeType string switch x.StoreType { - case kv.TiKV: - storeType = kv.TiKV.Name() - case kv.TiFlash: - storeType = kv.TiFlash.Name() + case kv.TiKV, kv.TiFlash, kv.TiDB: + // expected do nothing default: - err = errors.Errorf("the store type %v is unknown", x.StoreType) - return + return errors.Errorf("the store type %v is unknown", x.StoreType) } + storeType = x.StoreType.Name() err = e.explainPlanInRowFormat(x.tablePlan, "cop["+storeType+"]", childIndent, true) case *PhysicalIndexReader: err = e.explainPlanInRowFormat(x.indexPlan, "cop[tikv]", childIndent, true) @@ -788,7 +790,6 @@ func (e *Explain) explainPlanInRowFormat(p Plan, taskType, indent string, isLast // operator id, task type, operator info, and the estemated row count. func (e *Explain) prepareOperatorInfo(p Plan, taskType string, indent string, isLastChild bool) { operatorInfo := p.ExplainInfo() - count := "N/A" if si := p.statsInfo(); si != nil { count = strconv.FormatFloat(si.RowCount, 'f', 2, 64) @@ -815,9 +816,16 @@ func (e *Explain) prepareOperatorInfo(p Plan, taskType string, indent string, is } row = append(row, analyzeInfo) - tracker := e.ctx.GetSessionVars().StmtCtx.MemTracker.SearchTracker(p.ExplainID().String()) - if tracker != nil { - row = append(row, tracker.BytesToString(tracker.MaxConsumed())) + memTracker := e.ctx.GetSessionVars().StmtCtx.MemTracker.SearchTracker(p.ExplainID().String()) + if memTracker != nil { + row = append(row, memTracker.BytesToString(memTracker.MaxConsumed())) + } else { + row = append(row, "N/A") + } + + diskTracker := e.ctx.GetSessionVars().StmtCtx.DiskTracker.SearchTracker(p.ExplainID().String()) + if diskTracker != nil { + row = append(row, diskTracker.BytesToString(diskTracker.MaxConsumed())) } else { row = append(row, "N/A") } diff --git a/planner/core/encode.go b/planner/core/encode.go index 4abb3ed9b7cd9..f7e4e665856a5 100644 --- a/planner/core/encode.go +++ b/planner/core/encode.go @@ -15,6 +15,9 @@ package core import ( "bytes" + "crypto/sha256" + "fmt" + "hash" "sync" "github.com/pingcap/tidb/util/plancodec" @@ -32,10 +35,14 @@ type planEncoder struct { } // EncodePlan is used to encodePlan the plan to the plan tree with compressing. -func EncodePlan(p PhysicalPlan) string { +func EncodePlan(p Plan) string { pn := encoderPool.Get().(*planEncoder) defer encoderPool.Put(pn) - return pn.encodePlanTree(p) + selectPlan := getSelectPlan(p) + if selectPlan == nil { + return "" + } + return pn.encodePlanTree(selectPlan) } func (pn *planEncoder) encodePlanTree(p PhysicalPlan) string { @@ -64,5 +71,95 @@ func (pn *planEncoder) encodePlan(p PhysicalPlan, isRoot bool, depth int) { case *PhysicalIndexLookUpReader: pn.encodePlan(copPlan.indexPlan, false, depth) pn.encodePlan(copPlan.tablePlan, false, depth) + case *PhysicalIndexMergeReader: + for _, p := range copPlan.partialPlans { + pn.encodePlan(p, false, depth) + } + if copPlan.tablePlan != nil { + pn.encodePlan(copPlan.tablePlan, false, depth) + } + } +} + +var digesterPool = sync.Pool{ + New: func() interface{} { + return &planDigester{ + hasher: sha256.New(), + } + }, +} + +type planDigester struct { + buf bytes.Buffer + encodedPlans map[int]bool + hasher hash.Hash +} + +// NormalizePlan is used to normalize the plan and generate plan digest. +func NormalizePlan(p Plan) (normalized, digest string) { + selectPlan := getSelectPlan(p) + if selectPlan == nil { + return "", "" + } + d := digesterPool.Get().(*planDigester) + defer digesterPool.Put(d) + d.normalizePlanTree(selectPlan) + normalized = string(d.buf.Bytes()) + d.hasher.Write(d.buf.Bytes()) + d.buf.Reset() + digest = fmt.Sprintf("%x", d.hasher.Sum(nil)) + d.hasher.Reset() + return +} + +func (d *planDigester) normalizePlanTree(p PhysicalPlan) { + d.encodedPlans = make(map[int]bool) + d.buf.Reset() + d.normalizePlan(p, true, 0) +} + +func (d *planDigester) normalizePlan(p PhysicalPlan, isRoot bool, depth int) { + plancodec.NormalizePlanNode(depth, p.ID(), p.TP(), isRoot, p.ExplainNormalizedInfo(), &d.buf) + d.encodedPlans[p.ID()] = true + + depth++ + for _, child := range p.Children() { + if d.encodedPlans[child.ID()] { + continue + } + d.normalizePlan(child.(PhysicalPlan), isRoot, depth) + } + switch x := p.(type) { + case *PhysicalTableReader: + d.normalizePlan(x.tablePlan, false, depth) + case *PhysicalIndexReader: + d.normalizePlan(x.indexPlan, false, depth) + case *PhysicalIndexLookUpReader: + d.normalizePlan(x.indexPlan, false, depth) + d.normalizePlan(x.tablePlan, false, depth) + case *PhysicalIndexMergeReader: + for _, p := range x.partialPlans { + d.normalizePlan(p, false, depth) + } + if x.tablePlan != nil { + d.normalizePlan(x.tablePlan, false, depth) + } + } +} + +func getSelectPlan(p Plan) PhysicalPlan { + var selectPlan PhysicalPlan + if physicalPlan, ok := p.(PhysicalPlan); ok { + selectPlan = physicalPlan + } else { + switch x := p.(type) { + case *Delete: + selectPlan = x.SelectPlan + case *Update: + selectPlan = x.SelectPlan + case *Insert: + selectPlan = x.SelectPlan + } } + return selectPlan } diff --git a/planner/core/errors.go b/planner/core/errors.go index 72d75a9a5f47a..8097e50285a0e 100644 --- a/planner/core/errors.go +++ b/planner/core/errors.go @@ -18,187 +18,133 @@ import ( "github.com/pingcap/parser/terror" ) -const ( - codeUnsupportedType terror.ErrCode = 1 - codeAnalyzeMissIndex = 2 - codeUnsupported = 3 - codeStmtNotFound = 4 - codeWrongParamCount = 5 - codeSchemaChanged = 6 - - codeNotSupportedYet = mysql.ErrNotSupportedYet - codeWrongUsage = mysql.ErrWrongUsage - codeAmbiguous = mysql.ErrNonUniq - codeUnknown = mysql.ErrUnknown - codeUnknownColumn = mysql.ErrBadField - codeUnknownTable = mysql.ErrUnknownTable - codeWrongArguments = mysql.ErrWrongArguments - codeBadGeneratedColumn = mysql.ErrBadGeneratedColumn - codeFieldNotInGroupBy = mysql.ErrFieldNotInGroupBy - codeBadTable = mysql.ErrBadTable - codeKeyDoesNotExist = mysql.ErrKeyDoesNotExist - codeOperandColumns = mysql.ErrOperandColumns - codeInvalidWildCard = mysql.ErrParse - codeInvalidGroupFuncUse = mysql.ErrInvalidGroupFuncUse - codeIllegalReference = mysql.ErrIllegalReference - codeNoDB = mysql.ErrNoDB - codeUnknownExplainFormat = mysql.ErrUnknownExplainFormat - codeWrongGroupField = mysql.ErrWrongGroupField - codeDupFieldName = mysql.ErrDupFieldName - codeNonUpdatableTable = mysql.ErrNonUpdatableTable - codeInternal = mysql.ErrInternal - codeMixOfGroupFuncAndFields = mysql.ErrMixOfGroupFuncAndFields - codeNonUniqTable = mysql.ErrNonuniqTable - codeWrongNumberOfColumnsInSelect = mysql.ErrWrongNumberOfColumnsInSelect - codeWrongValueCountOnRow = mysql.ErrWrongValueCountOnRow - codeTablenameNotAllowedHere = mysql.ErrTablenameNotAllowedHere - codePrivilegeCheckFail = mysql.ErrUnknown - codeWindowInvalidWindowFuncUse = mysql.ErrWindowInvalidWindowFuncUse - codeWindowInvalidWindowFuncAliasUse = mysql.ErrWindowInvalidWindowFuncAliasUse - codeWindowNoSuchWindow = mysql.ErrWindowNoSuchWindow - codeWindowCircularityInWindowGraph = mysql.ErrWindowCircularityInWindowGraph - codeWindowNoChildPartitioning = mysql.ErrWindowNoChildPartitioning - codeWindowNoInherentFrame = mysql.ErrWindowNoInherentFrame - codeWindowNoRedefineOrderBy = mysql.ErrWindowNoRedefineOrderBy - codeWindowDuplicateName = mysql.ErrWindowDuplicateName - codeErrTooBigPrecision = mysql.ErrTooBigPrecision - codePartitionClauseOnNonpartitioned = mysql.ErrPartitionClauseOnNonpartitioned - codeDBaccessDenied = mysql.ErrDBaccessDenied - codeTableaccessDenied = mysql.ErrTableaccessDenied - codeSpecificAccessDenied = mysql.ErrSpecificAccessDenied - codeViewNoExplain = mysql.ErrViewNoExplain - codeWindowFrameStartIllegal = mysql.ErrWindowFrameStartIllegal - codeWindowFrameEndIllegal = mysql.ErrWindowFrameEndIllegal - codeWindowFrameIllegal = mysql.ErrWindowFrameIllegal - codeWindowRangeFrameOrderType = mysql.ErrWindowRangeFrameOrderType - codeWindowRangeFrameTemporalType = mysql.ErrWindowRangeFrameTemporalType - codeWindowRangeFrameNumericType = mysql.ErrWindowRangeFrameNumericType - codeWindowRangeBoundNotConstant = mysql.ErrWindowRangeBoundNotConstant - codeWindowRowsIntervalUse = mysql.ErrWindowRowsIntervalUse - codeWindowFunctionIgnoresFrame = mysql.ErrWindowFunctionIgnoresFrame - codeUnsupportedOnGeneratedColumn = mysql.ErrUnsupportedOnGeneratedColumn -) - // error definitions. var ( - ErrUnsupportedType = terror.ClassOptimizer.New(codeUnsupportedType, "Unsupported type %T") - ErrAnalyzeMissIndex = terror.ClassOptimizer.New(codeAnalyzeMissIndex, "Index '%s' in field list does not exist in table '%s'") - ErrCartesianProductUnsupported = terror.ClassOptimizer.New(codeUnsupported, "Cartesian product is unsupported") - ErrStmtNotFound = terror.ClassOptimizer.New(codeStmtNotFound, "Prepared statement not found") - ErrWrongParamCount = terror.ClassOptimizer.New(codeWrongParamCount, "Wrong parameter count") - ErrSchemaChanged = terror.ClassOptimizer.New(codeSchemaChanged, "Schema has changed") - ErrTablenameNotAllowedHere = terror.ClassOptimizer.New(codeTablenameNotAllowedHere, "Table '%s' from one of the %ss cannot be used in %s") - - ErrNotSupportedYet = terror.ClassOptimizer.New(codeNotSupportedYet, mysql.MySQLErrName[mysql.ErrNotSupportedYet]) - ErrWrongUsage = terror.ClassOptimizer.New(codeWrongUsage, mysql.MySQLErrName[mysql.ErrWrongUsage]) - ErrAmbiguous = terror.ClassOptimizer.New(codeAmbiguous, mysql.MySQLErrName[mysql.ErrNonUniq]) - ErrUnknown = terror.ClassOptimizer.New(codeUnknown, mysql.MySQLErrName[mysql.ErrUnknown]) - ErrUnknownColumn = terror.ClassOptimizer.New(codeUnknownColumn, mysql.MySQLErrName[mysql.ErrBadField]) - ErrUnknownTable = terror.ClassOptimizer.New(codeUnknownTable, mysql.MySQLErrName[mysql.ErrUnknownTable]) - ErrWrongArguments = terror.ClassOptimizer.New(codeWrongArguments, mysql.MySQLErrName[mysql.ErrWrongArguments]) - ErrWrongNumberOfColumnsInSelect = terror.ClassOptimizer.New(codeWrongNumberOfColumnsInSelect, mysql.MySQLErrName[mysql.ErrWrongNumberOfColumnsInSelect]) - ErrBadGeneratedColumn = terror.ClassOptimizer.New(codeBadGeneratedColumn, mysql.MySQLErrName[mysql.ErrBadGeneratedColumn]) - ErrFieldNotInGroupBy = terror.ClassOptimizer.New(codeFieldNotInGroupBy, mysql.MySQLErrName[mysql.ErrFieldNotInGroupBy]) - ErrBadTable = terror.ClassOptimizer.New(codeBadTable, mysql.MySQLErrName[mysql.ErrBadTable]) - ErrKeyDoesNotExist = terror.ClassOptimizer.New(codeKeyDoesNotExist, mysql.MySQLErrName[mysql.ErrKeyDoesNotExist]) - ErrOperandColumns = terror.ClassOptimizer.New(codeOperandColumns, mysql.MySQLErrName[mysql.ErrOperandColumns]) - ErrInvalidWildCard = terror.ClassOptimizer.New(codeInvalidWildCard, "Wildcard fields without any table name appears in wrong place") - ErrInvalidGroupFuncUse = terror.ClassOptimizer.New(codeInvalidGroupFuncUse, mysql.MySQLErrName[mysql.ErrInvalidGroupFuncUse]) - ErrIllegalReference = terror.ClassOptimizer.New(codeIllegalReference, mysql.MySQLErrName[mysql.ErrIllegalReference]) - ErrNoDB = terror.ClassOptimizer.New(codeNoDB, mysql.MySQLErrName[mysql.ErrNoDB]) - ErrUnknownExplainFormat = terror.ClassOptimizer.New(codeUnknownExplainFormat, mysql.MySQLErrName[mysql.ErrUnknownExplainFormat]) - ErrWrongGroupField = terror.ClassOptimizer.New(codeWrongGroupField, mysql.MySQLErrName[mysql.ErrWrongGroupField]) - ErrDupFieldName = terror.ClassOptimizer.New(codeDupFieldName, mysql.MySQLErrName[mysql.ErrDupFieldName]) - ErrNonUpdatableTable = terror.ClassOptimizer.New(codeNonUpdatableTable, mysql.MySQLErrName[mysql.ErrNonUpdatableTable]) - ErrInternal = terror.ClassOptimizer.New(codeInternal, mysql.MySQLErrName[mysql.ErrInternal]) - ErrMixOfGroupFuncAndFields = terror.ClassOptimizer.New(codeMixOfGroupFuncAndFields, "In aggregated query without GROUP BY, expression #%d of SELECT list contains nonaggregated column '%s'; this is incompatible with sql_mode=only_full_group_by") - ErrNonUniqTable = terror.ClassOptimizer.New(codeNonUniqTable, mysql.MySQLErrName[mysql.ErrNonuniqTable]) - ErrWrongValueCountOnRow = terror.ClassOptimizer.New(mysql.ErrWrongValueCountOnRow, mysql.MySQLErrName[mysql.ErrWrongValueCountOnRow]) - ErrViewInvalid = terror.ClassOptimizer.New(mysql.ErrViewInvalid, mysql.MySQLErrName[mysql.ErrViewInvalid]) - ErrPrivilegeCheckFail = terror.ClassOptimizer.New(codePrivilegeCheckFail, "privilege check fail") - ErrWindowInvalidWindowFuncUse = terror.ClassOptimizer.New(codeWindowInvalidWindowFuncUse, mysql.MySQLErrName[mysql.ErrWindowInvalidWindowFuncUse]) - ErrWindowInvalidWindowFuncAliasUse = terror.ClassOptimizer.New(codeWindowInvalidWindowFuncAliasUse, mysql.MySQLErrName[mysql.ErrWindowInvalidWindowFuncAliasUse]) - ErrWindowNoSuchWindow = terror.ClassOptimizer.New(codeWindowNoSuchWindow, mysql.MySQLErrName[mysql.ErrWindowNoSuchWindow]) - ErrWindowCircularityInWindowGraph = terror.ClassOptimizer.New(codeWindowCircularityInWindowGraph, mysql.MySQLErrName[mysql.ErrWindowCircularityInWindowGraph]) - ErrWindowNoChildPartitioning = terror.ClassOptimizer.New(codeWindowNoChildPartitioning, mysql.MySQLErrName[mysql.ErrWindowNoChildPartitioning]) - ErrWindowNoInherentFrame = terror.ClassOptimizer.New(codeWindowNoInherentFrame, mysql.MySQLErrName[mysql.ErrWindowNoInherentFrame]) - ErrWindowNoRedefineOrderBy = terror.ClassOptimizer.New(codeWindowNoRedefineOrderBy, mysql.MySQLErrName[mysql.ErrWindowNoRedefineOrderBy]) - ErrWindowDuplicateName = terror.ClassOptimizer.New(codeWindowDuplicateName, mysql.MySQLErrName[mysql.ErrWindowDuplicateName]) - ErrPartitionClauseOnNonpartitioned = terror.ClassOptimizer.New(codePartitionClauseOnNonpartitioned, mysql.MySQLErrName[mysql.ErrPartitionClauseOnNonpartitioned]) + ErrUnsupportedType = terror.ClassOptimizer.New(mysql.ErrUnsupportedType, mysql.MySQLErrName[mysql.ErrUnsupportedType]) + ErrAnalyzeMissIndex = terror.ClassOptimizer.New(mysql.ErrAnalyzeMissIndex, mysql.MySQLErrName[mysql.ErrAnalyzeMissIndex]) + ErrWrongParamCount = terror.ClassOptimizer.New(mysql.ErrWrongParamCount, mysql.MySQLErrName[mysql.ErrWrongParamCount]) + ErrSchemaChanged = terror.ClassOptimizer.New(mysql.ErrSchemaChanged, mysql.MySQLErrName[mysql.ErrSchemaChanged]) + ErrTablenameNotAllowedHere = terror.ClassOptimizer.New(mysql.ErrTablenameNotAllowedHere, mysql.MySQLErrName[mysql.ErrTablenameNotAllowedHere]) + ErrNotSupportedYet = terror.ClassOptimizer.New(mysql.ErrNotSupportedYet, mysql.MySQLErrName[mysql.ErrNotSupportedYet]) + ErrWrongUsage = terror.ClassOptimizer.New(mysql.ErrWrongUsage, mysql.MySQLErrName[mysql.ErrWrongUsage]) + ErrUnknown = terror.ClassOptimizer.New(mysql.ErrUnknown, mysql.MySQLErrName[mysql.ErrUnknown]) + ErrUnknownTable = terror.ClassOptimizer.New(mysql.ErrUnknownTable, mysql.MySQLErrName[mysql.ErrUnknownTable]) + ErrWrongArguments = terror.ClassOptimizer.New(mysql.ErrWrongArguments, mysql.MySQLErrName[mysql.ErrWrongArguments]) + ErrWrongNumberOfColumnsInSelect = terror.ClassOptimizer.New(mysql.ErrWrongNumberOfColumnsInSelect, mysql.MySQLErrName[mysql.ErrWrongNumberOfColumnsInSelect]) + ErrBadGeneratedColumn = terror.ClassOptimizer.New(mysql.ErrBadGeneratedColumn, mysql.MySQLErrName[mysql.ErrBadGeneratedColumn]) + ErrFieldNotInGroupBy = terror.ClassOptimizer.New(mysql.ErrFieldNotInGroupBy, mysql.MySQLErrName[mysql.ErrFieldNotInGroupBy]) + ErrBadTable = terror.ClassOptimizer.New(mysql.ErrBadTable, mysql.MySQLErrName[mysql.ErrBadTable]) + ErrKeyDoesNotExist = terror.ClassOptimizer.New(mysql.ErrKeyDoesNotExist, mysql.MySQLErrName[mysql.ErrKeyDoesNotExist]) + ErrOperandColumns = terror.ClassOptimizer.New(mysql.ErrOperandColumns, mysql.MySQLErrName[mysql.ErrOperandColumns]) + ErrInvalidGroupFuncUse = terror.ClassOptimizer.New(mysql.ErrInvalidGroupFuncUse, mysql.MySQLErrName[mysql.ErrInvalidGroupFuncUse]) + ErrIllegalReference = terror.ClassOptimizer.New(mysql.ErrIllegalReference, mysql.MySQLErrName[mysql.ErrIllegalReference]) + ErrNoDB = terror.ClassOptimizer.New(mysql.ErrNoDB, mysql.MySQLErrName[mysql.ErrNoDB]) + ErrUnknownExplainFormat = terror.ClassOptimizer.New(mysql.ErrUnknownExplainFormat, mysql.MySQLErrName[mysql.ErrUnknownExplainFormat]) + ErrWrongGroupField = terror.ClassOptimizer.New(mysql.ErrWrongGroupField, mysql.MySQLErrName[mysql.ErrWrongGroupField]) + ErrDupFieldName = terror.ClassOptimizer.New(mysql.ErrDupFieldName, mysql.MySQLErrName[mysql.ErrDupFieldName]) + ErrNonUpdatableTable = terror.ClassOptimizer.New(mysql.ErrNonUpdatableTable, mysql.MySQLErrName[mysql.ErrNonUpdatableTable]) + ErrInternal = terror.ClassOptimizer.New(mysql.ErrInternal, mysql.MySQLErrName[mysql.ErrInternal]) + ErrNonUniqTable = terror.ClassOptimizer.New(mysql.ErrNonuniqTable, mysql.MySQLErrName[mysql.ErrNonuniqTable]) + ErrWindowInvalidWindowFuncUse = terror.ClassOptimizer.New(mysql.ErrWindowInvalidWindowFuncUse, mysql.MySQLErrName[mysql.ErrWindowInvalidWindowFuncUse]) + ErrWindowInvalidWindowFuncAliasUse = terror.ClassOptimizer.New(mysql.ErrWindowInvalidWindowFuncAliasUse, mysql.MySQLErrName[mysql.ErrWindowInvalidWindowFuncAliasUse]) + ErrWindowNoSuchWindow = terror.ClassOptimizer.New(mysql.ErrWindowNoSuchWindow, mysql.MySQLErrName[mysql.ErrWindowNoSuchWindow]) + ErrWindowCircularityInWindowGraph = terror.ClassOptimizer.New(mysql.ErrWindowCircularityInWindowGraph, mysql.MySQLErrName[mysql.ErrWindowCircularityInWindowGraph]) + ErrWindowNoChildPartitioning = terror.ClassOptimizer.New(mysql.ErrWindowNoChildPartitioning, mysql.MySQLErrName[mysql.ErrWindowNoChildPartitioning]) + ErrWindowNoInherentFrame = terror.ClassOptimizer.New(mysql.ErrWindowNoInherentFrame, mysql.MySQLErrName[mysql.ErrWindowNoInherentFrame]) + ErrWindowNoRedefineOrderBy = terror.ClassOptimizer.New(mysql.ErrWindowNoRedefineOrderBy, mysql.MySQLErrName[mysql.ErrWindowNoRedefineOrderBy]) + ErrWindowDuplicateName = terror.ClassOptimizer.New(mysql.ErrWindowDuplicateName, mysql.MySQLErrName[mysql.ErrWindowDuplicateName]) + ErrPartitionClauseOnNonpartitioned = terror.ClassOptimizer.New(mysql.ErrPartitionClauseOnNonpartitioned, mysql.MySQLErrName[mysql.ErrPartitionClauseOnNonpartitioned]) + ErrWindowFrameStartIllegal = terror.ClassOptimizer.New(mysql.ErrWindowFrameStartIllegal, mysql.MySQLErrName[mysql.ErrWindowFrameStartIllegal]) + ErrWindowFrameEndIllegal = terror.ClassOptimizer.New(mysql.ErrWindowFrameEndIllegal, mysql.MySQLErrName[mysql.ErrWindowFrameEndIllegal]) + ErrWindowFrameIllegal = terror.ClassOptimizer.New(mysql.ErrWindowFrameIllegal, mysql.MySQLErrName[mysql.ErrWindowFrameIllegal]) + ErrWindowRangeFrameOrderType = terror.ClassOptimizer.New(mysql.ErrWindowRangeFrameOrderType, mysql.MySQLErrName[mysql.ErrWindowRangeFrameOrderType]) + ErrWindowRangeFrameTemporalType = terror.ClassOptimizer.New(mysql.ErrWindowRangeFrameTemporalType, mysql.MySQLErrName[mysql.ErrWindowRangeFrameTemporalType]) + ErrWindowRangeFrameNumericType = terror.ClassOptimizer.New(mysql.ErrWindowRangeFrameNumericType, mysql.MySQLErrName[mysql.ErrWindowRangeFrameNumericType]) + ErrWindowRangeBoundNotConstant = terror.ClassOptimizer.New(mysql.ErrWindowRangeBoundNotConstant, mysql.MySQLErrName[mysql.ErrWindowRangeBoundNotConstant]) + ErrWindowRowsIntervalUse = terror.ClassOptimizer.New(mysql.ErrWindowRowsIntervalUse, mysql.MySQLErrName[mysql.ErrWindowRowsIntervalUse]) + ErrWindowFunctionIgnoresFrame = terror.ClassOptimizer.New(mysql.ErrWindowFunctionIgnoresFrame, mysql.MySQLErrName[mysql.ErrWindowFunctionIgnoresFrame]) + ErrUnsupportedOnGeneratedColumn = terror.ClassOptimizer.New(mysql.ErrUnsupportedOnGeneratedColumn, mysql.MySQLErrName[mysql.ErrUnsupportedOnGeneratedColumn]) + ErrPrivilegeCheckFail = terror.ClassOptimizer.New(mysql.ErrPrivilegeCheckFail, mysql.MySQLErrName[mysql.ErrPrivilegeCheckFail]) + ErrInvalidWildCard = terror.ClassOptimizer.New(mysql.ErrInvalidWildCard, mysql.MySQLErrName[mysql.ErrInvalidWildCard]) + ErrMixOfGroupFuncAndFields = terror.ClassOptimizer.New(mysql.ErrMixOfGroupFuncAndFieldsIncompatible, mysql.MySQLErrName[mysql.ErrMixOfGroupFuncAndFieldsIncompatible]) errTooBigPrecision = terror.ClassExpression.New(mysql.ErrTooBigPrecision, mysql.MySQLErrName[mysql.ErrTooBigPrecision]) ErrDBaccessDenied = terror.ClassOptimizer.New(mysql.ErrDBaccessDenied, mysql.MySQLErrName[mysql.ErrDBaccessDenied]) ErrTableaccessDenied = terror.ClassOptimizer.New(mysql.ErrTableaccessDenied, mysql.MySQLErrName[mysql.ErrTableaccessDenied]) ErrSpecificAccessDenied = terror.ClassOptimizer.New(mysql.ErrSpecificAccessDenied, mysql.MySQLErrName[mysql.ErrSpecificAccessDenied]) ErrViewNoExplain = terror.ClassOptimizer.New(mysql.ErrViewNoExplain, mysql.MySQLErrName[mysql.ErrViewNoExplain]) - ErrWindowFrameStartIllegal = terror.ClassOptimizer.New(codeWindowFrameStartIllegal, mysql.MySQLErrName[mysql.ErrWindowFrameStartIllegal]) - ErrWindowFrameEndIllegal = terror.ClassOptimizer.New(codeWindowFrameEndIllegal, mysql.MySQLErrName[mysql.ErrWindowFrameEndIllegal]) - ErrWindowFrameIllegal = terror.ClassOptimizer.New(codeWindowFrameIllegal, mysql.MySQLErrName[mysql.ErrWindowFrameIllegal]) - ErrWindowRangeFrameOrderType = terror.ClassOptimizer.New(codeWindowRangeFrameOrderType, mysql.MySQLErrName[mysql.ErrWindowRangeFrameOrderType]) - ErrWindowRangeFrameTemporalType = terror.ClassOptimizer.New(codeWindowRangeFrameTemporalType, mysql.MySQLErrName[mysql.ErrWindowRangeFrameTemporalType]) - ErrWindowRangeFrameNumericType = terror.ClassOptimizer.New(codeWindowRangeFrameNumericType, mysql.MySQLErrName[mysql.ErrWindowRangeFrameNumericType]) - ErrWindowRangeBoundNotConstant = terror.ClassOptimizer.New(codeWindowRangeBoundNotConstant, mysql.MySQLErrName[mysql.ErrWindowRangeBoundNotConstant]) - ErrWindowRowsIntervalUse = terror.ClassOptimizer.New(codeWindowRowsIntervalUse, mysql.MySQLErrName[mysql.ErrWindowRowsIntervalUse]) - ErrWindowFunctionIgnoresFrame = terror.ClassOptimizer.New(codeWindowFunctionIgnoresFrame, mysql.MySQLErrName[mysql.ErrWindowFunctionIgnoresFrame]) - ErrUnsupportedOnGeneratedColumn = terror.ClassOptimizer.New(codeUnsupportedOnGeneratedColumn, mysql.MySQLErrName[mysql.ErrUnsupportedOnGeneratedColumn]) + ErrWrongValueCountOnRow = terror.ClassOptimizer.New(mysql.ErrWrongValueCountOnRow, mysql.MySQLErrName[mysql.ErrWrongValueCountOnRow]) + ErrViewInvalid = terror.ClassOptimizer.New(mysql.ErrViewInvalid, mysql.MySQLErrName[mysql.ErrViewInvalid]) ErrNoSuchThread = terror.ClassOptimizer.New(mysql.ErrNoSuchThread, mysql.MySQLErrName[mysql.ErrNoSuchThread]) + ErrUnknownColumn = terror.ClassOptimizer.New(mysql.ErrBadField, mysql.MySQLErrName[mysql.ErrBadField]) + ErrCartesianProductUnsupported = terror.ClassOptimizer.New(mysql.ErrCartesianProductUnsupported, mysql.MySQLErrName[mysql.ErrCartesianProductUnsupported]) + ErrStmtNotFound = terror.ClassOptimizer.New(mysql.ErrPreparedStmtNotFound, mysql.MySQLErrName[mysql.ErrPreparedStmtNotFound]) + ErrAmbiguous = terror.ClassOptimizer.New(mysql.ErrNonUniq, mysql.MySQLErrName[mysql.ErrNonUniq]) // Since we cannot know if user loggined with a password, use message of ErrAccessDeniedNoPassword instead ErrAccessDenied = terror.ClassOptimizer.New(mysql.ErrAccessDenied, mysql.MySQLErrName[mysql.ErrAccessDeniedNoPassword]) ) func init() { mysqlErrCodeMap := map[terror.ErrCode]uint16{ - codeNotSupportedYet: mysql.ErrNotSupportedYet, - codeWrongUsage: mysql.ErrWrongUsage, - codeAmbiguous: mysql.ErrNonUniq, - codeUnknownColumn: mysql.ErrBadField, - codeUnknownTable: mysql.ErrBadTable, - codeWrongArguments: mysql.ErrWrongArguments, - codeBadGeneratedColumn: mysql.ErrBadGeneratedColumn, - codeFieldNotInGroupBy: mysql.ErrFieldNotInGroupBy, - codeBadTable: mysql.ErrBadTable, - codeKeyDoesNotExist: mysql.ErrKeyDoesNotExist, - codeOperandColumns: mysql.ErrOperandColumns, - codeInvalidWildCard: mysql.ErrParse, - codeInvalidGroupFuncUse: mysql.ErrInvalidGroupFuncUse, - codeIllegalReference: mysql.ErrIllegalReference, - codeNoDB: mysql.ErrNoDB, - codeUnknownExplainFormat: mysql.ErrUnknownExplainFormat, - codeWrongGroupField: mysql.ErrWrongGroupField, - codeDupFieldName: mysql.ErrDupFieldName, - codeNonUpdatableTable: mysql.ErrUnknownTable, - codeInternal: mysql.ErrInternal, - codeMixOfGroupFuncAndFields: mysql.ErrMixOfGroupFuncAndFields, - codeNonUniqTable: mysql.ErrNonuniqTable, - codeWrongNumberOfColumnsInSelect: mysql.ErrWrongNumberOfColumnsInSelect, - codeWrongValueCountOnRow: mysql.ErrWrongValueCountOnRow, - - codeWindowInvalidWindowFuncUse: mysql.ErrWindowInvalidWindowFuncUse, - codeWindowInvalidWindowFuncAliasUse: mysql.ErrWindowInvalidWindowFuncAliasUse, - codeWindowNoSuchWindow: mysql.ErrWindowNoSuchWindow, - codeWindowCircularityInWindowGraph: mysql.ErrWindowCircularityInWindowGraph, - codeWindowNoChildPartitioning: mysql.ErrWindowNoChildPartitioning, - codeWindowNoInherentFrame: mysql.ErrWindowNoInherentFrame, - codeWindowNoRedefineOrderBy: mysql.ErrWindowNoRedefineOrderBy, - codeWindowDuplicateName: mysql.ErrWindowDuplicateName, - codePartitionClauseOnNonpartitioned: mysql.ErrPartitionClauseOnNonpartitioned, - codeErrTooBigPrecision: mysql.ErrTooBigPrecision, - codeDBaccessDenied: mysql.ErrDBaccessDenied, - codeTableaccessDenied: mysql.ErrTableaccessDenied, - codeSpecificAccessDenied: mysql.ErrSpecificAccessDenied, - codeViewNoExplain: mysql.ErrViewNoExplain, - codeWindowFrameStartIllegal: mysql.ErrWindowFrameStartIllegal, - codeWindowFrameEndIllegal: mysql.ErrWindowFrameEndIllegal, - codeWindowFrameIllegal: mysql.ErrWindowFrameIllegal, - codeWindowRangeFrameOrderType: mysql.ErrWindowRangeFrameOrderType, - codeWindowRangeFrameTemporalType: mysql.ErrWindowRangeFrameTemporalType, - codeWindowRangeFrameNumericType: mysql.ErrWindowRangeFrameNumericType, - codeWindowRangeBoundNotConstant: mysql.ErrWindowRangeBoundNotConstant, - codeWindowRowsIntervalUse: mysql.ErrWindowRowsIntervalUse, - codeWindowFunctionIgnoresFrame: mysql.ErrWindowFunctionIgnoresFrame, - codeUnsupportedOnGeneratedColumn: mysql.ErrUnsupportedOnGeneratedColumn, - - mysql.ErrNoSuchThread: mysql.ErrNoSuchThread, - mysql.ErrAccessDenied: mysql.ErrAccessDenied, + mysql.ErrViewInvalid: mysql.ErrViewInvalid, + mysql.ErrUnknown: mysql.ErrUnknown, + mysql.ErrTablenameNotAllowedHere: mysql.ErrTablenameNotAllowedHere, + mysql.ErrUnsupportedType: mysql.ErrUnsupportedType, + mysql.ErrAnalyzeMissIndex: mysql.ErrAnalyzeMissIndex, + mysql.ErrWrongParamCount: mysql.ErrWrongParamCount, + mysql.ErrSchemaChanged: mysql.ErrSchemaChanged, + mysql.ErrNotSupportedYet: mysql.ErrNotSupportedYet, + mysql.ErrWrongUsage: mysql.ErrWrongUsage, + mysql.ErrUnknownTable: mysql.ErrUnknownTable, + mysql.ErrWrongArguments: mysql.ErrWrongArguments, + mysql.ErrBadGeneratedColumn: mysql.ErrBadGeneratedColumn, + mysql.ErrFieldNotInGroupBy: mysql.ErrFieldNotInGroupBy, + mysql.ErrBadTable: mysql.ErrBadTable, + mysql.ErrKeyDoesNotExist: mysql.ErrKeyDoesNotExist, + mysql.ErrOperandColumns: mysql.ErrOperandColumns, + mysql.ErrInvalidGroupFuncUse: mysql.ErrInvalidGroupFuncUse, + mysql.ErrIllegalReference: mysql.ErrIllegalReference, + mysql.ErrNoDB: mysql.ErrNoDB, + mysql.ErrUnknownExplainFormat: mysql.ErrUnknownExplainFormat, + mysql.ErrWrongGroupField: mysql.ErrWrongGroupField, + mysql.ErrDupFieldName: mysql.ErrDupFieldName, + mysql.ErrNonUpdatableTable: mysql.ErrNonUpdatableTable, + mysql.ErrInternal: mysql.ErrInternal, + mysql.ErrMixOfGroupFuncAndFieldsIncompatible: mysql.ErrMixOfGroupFuncAndFieldsIncompatible, + mysql.ErrWrongNumberOfColumnsInSelect: mysql.ErrWrongNumberOfColumnsInSelect, + mysql.ErrWrongValueCountOnRow: mysql.ErrWrongValueCountOnRow, + mysql.ErrWindowInvalidWindowFuncUse: mysql.ErrWindowInvalidWindowFuncUse, + mysql.ErrWindowInvalidWindowFuncAliasUse: mysql.ErrWindowInvalidWindowFuncAliasUse, + mysql.ErrWindowNoSuchWindow: mysql.ErrWindowNoSuchWindow, + mysql.ErrWindowCircularityInWindowGraph: mysql.ErrWindowCircularityInWindowGraph, + mysql.ErrWindowNoChildPartitioning: mysql.ErrWindowNoChildPartitioning, + mysql.ErrWindowNoInherentFrame: mysql.ErrWindowNoInherentFrame, + mysql.ErrWindowNoRedefineOrderBy: mysql.ErrWindowNoRedefineOrderBy, + mysql.ErrWindowDuplicateName: mysql.ErrWindowDuplicateName, + mysql.ErrPartitionClauseOnNonpartitioned: mysql.ErrPartitionClauseOnNonpartitioned, + mysql.ErrDBaccessDenied: mysql.ErrDBaccessDenied, + mysql.ErrTableaccessDenied: mysql.ErrTableaccessDenied, + mysql.ErrSpecificAccessDenied: mysql.ErrSpecificAccessDenied, + mysql.ErrViewNoExplain: mysql.ErrViewNoExplain, + mysql.ErrWindowFrameStartIllegal: mysql.ErrWindowFrameStartIllegal, + mysql.ErrWindowFrameEndIllegal: mysql.ErrWindowFrameEndIllegal, + mysql.ErrWindowFrameIllegal: mysql.ErrWindowFrameIllegal, + mysql.ErrWindowRangeFrameOrderType: mysql.ErrWindowRangeFrameOrderType, + mysql.ErrWindowRangeFrameTemporalType: mysql.ErrWindowRangeFrameTemporalType, + mysql.ErrWindowRangeFrameNumericType: mysql.ErrWindowRangeFrameNumericType, + mysql.ErrWindowRangeBoundNotConstant: mysql.ErrWindowRangeBoundNotConstant, + mysql.ErrWindowRowsIntervalUse: mysql.ErrWindowRowsIntervalUse, + mysql.ErrWindowFunctionIgnoresFrame: mysql.ErrWindowFunctionIgnoresFrame, + mysql.ErrUnsupportedOnGeneratedColumn: mysql.ErrUnsupportedOnGeneratedColumn, + mysql.ErrNoSuchThread: mysql.ErrNoSuchThread, + mysql.ErrAccessDenied: mysql.ErrAccessDenied, + mysql.ErrPrivilegeCheckFail: mysql.ErrPrivilegeCheckFail, + mysql.ErrCartesianProductUnsupported: mysql.ErrCartesianProductUnsupported, + mysql.ErrPreparedStmtNotFound: mysql.ErrPreparedStmtNotFound, + mysql.ErrNonUniq: mysql.ErrNonUniq, + mysql.ErrBadField: mysql.ErrBadField, + mysql.ErrNonuniqTable: mysql.ErrNonuniqTable, + mysql.ErrTooBigPrecision: mysql.ErrTooBigPrecision, + mysql.ErrInvalidWildCard: mysql.ErrInvalidWildCard, } terror.ErrClassToMySQLCodes[terror.ClassOptimizer] = mysqlErrCodeMap } diff --git a/planner/core/errors_test.go b/planner/core/errors_test.go new file mode 100644 index 0000000000000..bce2a3563978d --- /dev/null +++ b/planner/core/errors_test.go @@ -0,0 +1,90 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + . "github.com/pingcap/check" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/parser/terror" +) + +type testErrorSuite struct{} + +var _ = Suite(testErrorSuite{}) + +func (s testErrorSuite) TestError(c *C) { + kvErrs := []*terror.Error{ + ErrUnsupportedType, + ErrAnalyzeMissIndex, + ErrWrongParamCount, + ErrSchemaChanged, + ErrTablenameNotAllowedHere, + ErrNotSupportedYet, + ErrWrongUsage, + ErrUnknownTable, + ErrWrongArguments, + ErrWrongNumberOfColumnsInSelect, + ErrBadGeneratedColumn, + ErrFieldNotInGroupBy, + ErrBadTable, + ErrKeyDoesNotExist, + ErrOperandColumns, + ErrInvalidGroupFuncUse, + ErrIllegalReference, + ErrNoDB, + ErrUnknownExplainFormat, + ErrWrongGroupField, + ErrDupFieldName, + ErrNonUpdatableTable, + ErrInternal, + ErrNonUniqTable, + ErrWindowInvalidWindowFuncUse, + ErrWindowInvalidWindowFuncAliasUse, + ErrWindowNoSuchWindow, + ErrWindowCircularityInWindowGraph, + ErrWindowNoChildPartitioning, + ErrWindowNoInherentFrame, + ErrWindowNoRedefineOrderBy, + ErrWindowDuplicateName, + ErrPartitionClauseOnNonpartitioned, + ErrWindowFrameStartIllegal, + ErrWindowFrameEndIllegal, + ErrWindowFrameIllegal, + ErrWindowRangeFrameOrderType, + ErrWindowRangeFrameTemporalType, + ErrWindowRangeFrameNumericType, + ErrWindowRangeBoundNotConstant, + ErrWindowRowsIntervalUse, + ErrWindowFunctionIgnoresFrame, + ErrUnsupportedOnGeneratedColumn, + ErrPrivilegeCheckFail, + ErrInvalidWildCard, + ErrMixOfGroupFuncAndFields, + ErrDBaccessDenied, + ErrTableaccessDenied, + ErrSpecificAccessDenied, + ErrViewNoExplain, + ErrWrongValueCountOnRow, + ErrViewInvalid, + ErrNoSuchThread, + ErrUnknownColumn, + ErrCartesianProductUnsupported, + ErrStmtNotFound, + ErrAmbiguous, + } + for _, err := range kvErrs { + code := err.ToSQLError().Code + c.Assert(code != mysql.ErrUnknown && code == uint16(err.Code()), IsTrue, Commentf("err: %v", err)) + } +} diff --git a/planner/core/explain.go b/planner/core/explain.go index d53cb3c104d10..526a051fcceba 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -33,6 +33,10 @@ func (p *PhysicalLock) ExplainInfo() string { // ExplainInfo implements Plan interface. func (p *PhysicalIndexScan) ExplainInfo() string { + return p.explainInfo(false) +} + +func (p *PhysicalIndexScan) explainInfo(normalized bool) string { buffer := bytes.NewBufferString("") tblName := p.Table.Name.O if p.TableAsName != nil && p.TableAsName.O != "" { @@ -64,13 +68,21 @@ func (p *PhysicalIndexScan) ExplainInfo() string { if len(p.rangeInfo) > 0 { fmt.Fprintf(buffer, ", range: decided by %v", p.rangeInfo) } else if haveCorCol { - fmt.Fprintf(buffer, ", range: decided by %v", p.AccessCondition) + if normalized { + fmt.Fprintf(buffer, ", range: decided by %s", expression.SortedExplainNormalizedExpressionList(p.AccessCondition)) + } else { + fmt.Fprintf(buffer, ", range: decided by %v", p.AccessCondition) + } } else if len(p.Ranges) > 0 { - fmt.Fprint(buffer, ", range:") - for i, idxRange := range p.Ranges { - fmt.Fprint(buffer, idxRange.String()) - if i+1 < len(p.Ranges) { - fmt.Fprint(buffer, ", ") + if normalized { + fmt.Fprint(buffer, ", range:[?,?]") + } else { + fmt.Fprint(buffer, ", range:") + for i, idxRange := range p.Ranges { + fmt.Fprint(buffer, idxRange.String()) + if i+1 < len(p.Ranges) { + fmt.Fprint(buffer, ", ") + } } } } @@ -78,14 +90,28 @@ func (p *PhysicalIndexScan) ExplainInfo() string { if p.Desc { buffer.WriteString(", desc") } - if p.stats.StatsVersion == statistics.PseudoVersion { + if p.stats.StatsVersion == statistics.PseudoVersion && !normalized { buffer.WriteString(", stats:pseudo") } return buffer.String() } +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalIndexScan) ExplainNormalizedInfo() string { + return p.explainInfo(true) +} + // ExplainInfo implements Plan interface. func (p *PhysicalTableScan) ExplainInfo() string { + return p.explainInfo(false) +} + +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalTableScan) ExplainNormalizedInfo() string { + return p.explainInfo(true) +} + +func (p *PhysicalTableScan) explainInfo(normalized bool) string { buffer := bytes.NewBufferString("") tblName := p.Table.Name.O if p.TableAsName != nil && p.TableAsName.O != "" { @@ -111,9 +137,15 @@ func (p *PhysicalTableScan) ExplainInfo() string { if len(p.rangeDecidedBy) > 0 { fmt.Fprintf(buffer, ", range: decided by %v", p.rangeDecidedBy) } else if haveCorCol { - fmt.Fprintf(buffer, ", range: decided by %v", p.AccessCondition) + if normalized { + fmt.Fprintf(buffer, ", range: decided by %s", expression.SortedExplainNormalizedExpressionList(p.AccessCondition)) + } else { + fmt.Fprintf(buffer, ", range: decided by %v", p.AccessCondition) + } } else if len(p.Ranges) > 0 { - if p.StoreType == kv.TiFlash { + if normalized { + fmt.Fprint(buffer, ", range:[?,?]") + } else if p.StoreType == kv.TiFlash { // TiFlash table always use full range scan for each region, // the ranges in p.Ranges is used to prune cop task fmt.Fprintf(buffer, ", range:"+ranger.FullIntRange(false)[0].String()) @@ -131,7 +163,7 @@ func (p *PhysicalTableScan) ExplainInfo() string { if p.Desc { buffer.WriteString(", desc") } - if p.stats.StatsVersion == statistics.PseudoVersion { + if p.stats.StatsVersion == statistics.PseudoVersion && !normalized { buffer.WriteString(", stats:pseudo") } return buffer.String() @@ -142,11 +174,21 @@ func (p *PhysicalTableReader) ExplainInfo() string { return "data:" + p.tablePlan.ExplainID().String() } +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalTableReader) ExplainNormalizedInfo() string { + return p.ExplainInfo() +} + // ExplainInfo implements Plan interface. func (p *PhysicalIndexReader) ExplainInfo() string { return "index:" + p.indexPlan.ExplainID().String() } +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalIndexReader) ExplainNormalizedInfo() string { + return p.ExplainInfo() +} + // ExplainInfo implements Plan interface. func (p *PhysicalIndexLookUpReader) ExplainInfo() string { // The children can be inferred by the relation symbol. @@ -171,11 +213,21 @@ func (p *PhysicalSelection) ExplainInfo() string { return string(expression.SortedExplainExpressionList(p.Conditions)) } +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalSelection) ExplainNormalizedInfo() string { + return string(expression.SortedExplainNormalizedExpressionList(p.Conditions)) +} + // ExplainInfo implements Plan interface. func (p *PhysicalProjection) ExplainInfo() string { return expression.ExplainExpressionList(p.Exprs, p.schema) } +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalProjection) ExplainNormalizedInfo() string { + return string(expression.SortedExplainNormalizedExpressionList(p.Exprs)) +} + // ExplainInfo implements Plan interface. func (p *PhysicalTableDual) ExplainInfo() string { return fmt.Sprintf("rows:%v", p.RowCount) @@ -194,10 +246,19 @@ func (p *PhysicalLimit) ExplainInfo() string { // ExplainInfo implements Plan interface. func (p *basePhysicalAgg) ExplainInfo() string { + return p.explainInfo(false) +} + +func (p *basePhysicalAgg) explainInfo(normalized bool) string { + sortedExplainExpressionList := expression.SortedExplainExpressionList + if normalized { + sortedExplainExpressionList = expression.SortedExplainNormalizedExpressionList + } + builder := &strings.Builder{} if len(p.GroupByItems) > 0 { fmt.Fprintf(builder, "group by:%s, ", - expression.SortedExplainExpressionList(p.GroupByItems)) + sortedExplainExpressionList(p.GroupByItems)) } for i := 0; i < len(p.AggFuncs); i++ { builder.WriteString("funcs:") @@ -209,8 +270,22 @@ func (p *basePhysicalAgg) ExplainInfo() string { return builder.String() } +// ExplainNormalizedInfo implements Plan interface. +func (p *basePhysicalAgg) ExplainNormalizedInfo() string { + return p.explainInfo(true) +} + // ExplainInfo implements Plan interface. func (p *PhysicalIndexJoin) ExplainInfo() string { + return p.explainInfo(false) +} + +func (p *PhysicalIndexJoin) explainInfo(normalized bool) string { + sortedExplainExpressionList := expression.SortedExplainExpressionList + if normalized { + sortedExplainExpressionList = expression.SortedExplainNormalizedExpressionList + } + buffer := bytes.NewBufferString(p.JoinType.String()) fmt.Fprintf(buffer, ", inner:%s", p.Children()[p.InnerChildIdx].ExplainID()) if len(p.OuterJoinKeys) > 0 { @@ -223,21 +298,40 @@ func (p *PhysicalIndexJoin) ExplainInfo() string { } if len(p.LeftConditions) > 0 { fmt.Fprintf(buffer, ", left cond:%s", - expression.SortedExplainExpressionList(p.LeftConditions)) + sortedExplainExpressionList(p.LeftConditions)) } if len(p.RightConditions) > 0 { fmt.Fprintf(buffer, ", right cond:%s", - expression.SortedExplainExpressionList(p.RightConditions)) + sortedExplainExpressionList(p.RightConditions)) } if len(p.OtherConditions) > 0 { fmt.Fprintf(buffer, ", other cond:%s", - expression.SortedExplainExpressionList(p.OtherConditions)) + sortedExplainExpressionList(p.OtherConditions)) } return buffer.String() } +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalIndexJoin) ExplainNormalizedInfo() string { + return p.explainInfo(true) +} + // ExplainInfo implements Plan interface. func (p *PhysicalHashJoin) ExplainInfo() string { + return p.explainInfo(false) +} + +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalHashJoin) ExplainNormalizedInfo() string { + return p.explainInfo(true) +} + +func (p *PhysicalHashJoin) explainInfo(normalized bool) string { + sortedExplainExpressionList := expression.SortedExplainExpressionList + if normalized { + sortedExplainExpressionList = expression.SortedExplainNormalizedExpressionList + } + buffer := new(bytes.Buffer) if len(p.EqualConditions) == 0 { @@ -255,24 +349,41 @@ func (p *PhysicalHashJoin) ExplainInfo() string { buffer.WriteString(" (REVERSED)") } if len(p.EqualConditions) > 0 { - fmt.Fprintf(buffer, ", equal:%v", p.EqualConditions) + if normalized { + fmt.Fprintf(buffer, ", equal:%s", expression.SortedExplainNormalizedScalarFuncList(p.EqualConditions)) + } else { + fmt.Fprintf(buffer, ", equal:%v", p.EqualConditions) + } } if len(p.LeftConditions) > 0 { - fmt.Fprintf(buffer, ", left cond:%s", p.LeftConditions) + if normalized { + fmt.Fprintf(buffer, ", left cond:%s", expression.SortedExplainNormalizedExpressionList(p.LeftConditions)) + } else { + fmt.Fprintf(buffer, ", left cond:%s", p.LeftConditions) + } } if len(p.RightConditions) > 0 { fmt.Fprintf(buffer, ", right cond:%s", - expression.SortedExplainExpressionList(p.RightConditions)) + sortedExplainExpressionList(p.RightConditions)) } if len(p.OtherConditions) > 0 { fmt.Fprintf(buffer, ", other cond:%s", - expression.SortedExplainExpressionList(p.OtherConditions)) + sortedExplainExpressionList(p.OtherConditions)) } return buffer.String() } // ExplainInfo implements Plan interface. func (p *PhysicalMergeJoin) ExplainInfo() string { + return p.explainInfo(false) +} + +func (p *PhysicalMergeJoin) explainInfo(normalized bool) string { + sortedExplainExpressionList := expression.SortedExplainExpressionList + if normalized { + sortedExplainExpressionList = expression.SortedExplainNormalizedExpressionList + } + buffer := bytes.NewBufferString(p.JoinType.String()) if len(p.LeftJoinKeys) > 0 { fmt.Fprintf(buffer, ", left key:%s", @@ -283,19 +394,28 @@ func (p *PhysicalMergeJoin) ExplainInfo() string { expression.ExplainColumnList(p.RightJoinKeys)) } if len(p.LeftConditions) > 0 { - fmt.Fprintf(buffer, ", left cond:%s", p.LeftConditions) + if normalized { + fmt.Fprintf(buffer, ", left cond:%s", expression.SortedExplainNormalizedExpressionList(p.LeftConditions)) + } else { + fmt.Fprintf(buffer, ", left cond:%s", p.LeftConditions) + } } if len(p.RightConditions) > 0 { fmt.Fprintf(buffer, ", right cond:%s", - expression.SortedExplainExpressionList(p.RightConditions)) + sortedExplainExpressionList(p.RightConditions)) } if len(p.OtherConditions) > 0 { fmt.Fprintf(buffer, ", other cond:%s", - expression.SortedExplainExpressionList(p.OtherConditions)) + sortedExplainExpressionList(p.OtherConditions)) } return buffer.String() } +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalMergeJoin) ExplainNormalizedInfo() string { + return p.explainInfo(true) +} + // ExplainInfo implements Plan interface. func (p *PhysicalTopN) ExplainInfo() string { buffer := bytes.NewBufferString("") @@ -304,6 +424,13 @@ func (p *PhysicalTopN) ExplainInfo() string { return buffer.String() } +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalTopN) ExplainNormalizedInfo() string { + buffer := bytes.NewBufferString("") + buffer = explainNormalizedByItems(buffer, p.ByItems) + return buffer.String() +} + func (p *PhysicalWindow) formatFrameBound(buffer *bytes.Buffer, bound *FrameBound) { if bound.Type == ast.CurrentRow { buffer.WriteString("current row") @@ -467,9 +594,6 @@ func (p *DataSource) ExplainInfo() string { fmt.Fprintf(buffer, ", partition:%s", partitionName) } } - if p.handleCol != nil { - fmt.Fprintf(buffer, ", pk col:%s", p.handleCol.ExplainInfo()) - } return buffer.String() } @@ -496,6 +620,20 @@ func explainByItems(buffer *bytes.Buffer, byItems []*ByItems) *bytes.Buffer { return buffer } +func explainNormalizedByItems(buffer *bytes.Buffer, byItems []*ByItems) *bytes.Buffer { + for i, item := range byItems { + order := "asc" + if item.Desc { + order = "desc" + } + fmt.Fprintf(buffer, "%s:%s", item.Expr.ExplainNormalizedInfo(), order) + if i+1 < len(byItems) { + buffer.WriteString(", ") + } + } + return buffer +} + // ExplainInfo implements Plan interface. func (p *LogicalSort) ExplainInfo() string { buffer := bytes.NewBufferString("") @@ -516,10 +654,41 @@ func (p *LogicalLimit) ExplainInfo() string { } // ExplainInfo implements Plan interface. -func (p *TableScan) ExplainInfo() string { +func (p *LogicalTableScan) ExplainInfo() string { + buffer := bytes.NewBufferString(p.Source.ExplainInfo()) + if p.Source.handleCol != nil { + fmt.Fprintf(buffer, ", pk col:%s", p.Source.handleCol.ExplainInfo()) + } + if len(p.AccessConds) > 0 { + fmt.Fprintf(buffer, ", cond:%v", p.AccessConds) + } + return buffer.String() +} + +// ExplainInfo implements Plan interface. +func (p *LogicalIndexScan) ExplainInfo() string { buffer := bytes.NewBufferString(p.Source.ExplainInfo()) + index := p.Index + if len(index.Columns) > 0 { + buffer.WriteString(", index:") + for i, idxCol := range index.Columns { + buffer.WriteString(idxCol.Name.O) + if i+1 < len(index.Columns) { + buffer.WriteString(", ") + } + } + } if len(p.AccessConds) > 0 { fmt.Fprintf(buffer, ", cond:%v", p.AccessConds) } return buffer.String() } + +// ExplainInfo implements Plan interface. +func (p *TiKVSingleGather) ExplainInfo() string { + buffer := bytes.NewBufferString(p.Source.ExplainInfo()) + if p.IsIndexGather { + buffer.WriteString(", index:" + p.Index.Name.String()) + } + return buffer.String() +} diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 7851582a801d4..e5633b82a1970 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -187,8 +187,8 @@ func (p *LogicalMemTable) findBestTask(prop *property.PhysicalProperty) (t task, return invalidTask, nil } memTable := PhysicalMemTable{ - DBName: p.dbName, - Table: p.tableInfo, + Table: p.tableInfo, + Columns: p.tableInfo.Columns, }.Init(p.ctx, p.stats, p.blockOffset) memTable.SetSchema(p.schema) return &rootTask{p: memTable}, nil @@ -740,7 +740,11 @@ func (is *PhysicalIndexScan) initSchema(idx *model.IndexInfo, idxExprCols []*exp // If it's double read case, the first index must return handle. So we should add extra handle column // if there isn't a handle column. if isDoubleRead && !setHandle { - indexCols = append(indexCols, &expression.Column{ID: model.ExtraHandleID, UniqueID: is.ctx.GetSessionVars().AllocPlanColumnID()}) + indexCols = append(indexCols, &expression.Column{ + RetType: types.NewFieldType(mysql.TypeLonglong), + ID: model.ExtraHandleID, + UniqueID: is.ctx.GetSessionVars().AllocPlanColumnID(), + }) } is.SetSchema(expression.NewSchema(indexCols...)) } @@ -960,8 +964,8 @@ func (ds *DataSource) crossEstimateRowCount(path *accessPath, expectedCnt float6 return scanCount, true, 0 } -// GetPhysicalScan returns PhysicalTableScan for the logical TableScan. -func (s *TableScan) GetPhysicalScan(schema *expression.Schema, stats *property.StatsInfo) *PhysicalTableScan { +// GetPhysicalScan returns PhysicalTableScan for the LogicalTableScan. +func (s *LogicalTableScan) GetPhysicalScan(schema *expression.Schema, stats *property.StatsInfo) *PhysicalTableScan { ds := s.Source ts := PhysicalTableScan{ Table: ds.tableInfo, @@ -985,6 +989,28 @@ func (s *TableScan) GetPhysicalScan(schema *expression.Schema, stats *property.S return ts } +// GetPhysicalIndexScan returns PhysicalIndexScan for the logical IndexScan. +func (s *LogicalIndexScan) GetPhysicalIndexScan(schema *expression.Schema, stats *property.StatsInfo) *PhysicalIndexScan { + ds := s.Source + is := PhysicalIndexScan{ + Table: ds.tableInfo, + TableAsName: ds.TableAsName, + DBName: ds.DBName, + Columns: s.Columns, + Index: s.Index, + IdxCols: s.idxCols, + IdxColLens: s.idxColLens, + AccessCondition: s.AccessConds, + Ranges: s.Ranges, + dataSourceSchema: ds.schema, + isPartition: ds.isPartition, + physicalTableID: ds.physicalTableID, + }.Init(ds.ctx, ds.blockOffset) + is.stats = stats + is.initSchema(s.Index, s.fullIdxCols, s.IsDoubleRead) + return is +} + // convertToTableScan converts the DataSource to table scan. func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candidate *candidatePath) (task task, err error) { // It will be handled in convertToIndexScan. diff --git a/planner/core/initialize.go b/planner/core/initialize.go index 04647e1d1361f..b3b0f1831f3d7 100644 --- a/planner/core/initialize.go +++ b/planner/core/initialize.go @@ -39,18 +39,24 @@ func (ds DataSource) Init(ctx sessionctx.Context, offset int) *DataSource { return &ds } -// Init initializes TableGather. -func (tg TableGather) Init(ctx sessionctx.Context, offset int) *TableGather { - tg.baseLogicalPlan = newBaseLogicalPlan(ctx, plancodec.TypeTableGather, &tg, offset) - return &tg +// Init initializes TiKVSingleGather. +func (sg TiKVSingleGather) Init(ctx sessionctx.Context, offset int) *TiKVSingleGather { + sg.baseLogicalPlan = newBaseLogicalPlan(ctx, plancodec.TypeTiKVSingleGather, &sg, offset) + return &sg } -// Init initializes TableScan. -func (ts TableScan) Init(ctx sessionctx.Context, offset int) *TableScan { +// Init initializes LogicalTableScan. +func (ts LogicalTableScan) Init(ctx sessionctx.Context, offset int) *LogicalTableScan { ts.baseLogicalPlan = newBaseLogicalPlan(ctx, plancodec.TypeTableScan, &ts, offset) return &ts } +// Init initializes LogicalIndexScan. +func (is LogicalIndexScan) Init(ctx sessionctx.Context, offset int) *LogicalIndexScan { + is.baseLogicalPlan = newBaseLogicalPlan(ctx, plancodec.TypeIdxScan, &is, offset) + return &is +} + // Init initializes LogicalApply. func (la LogicalApply) Init(ctx sessionctx.Context, offset int) *LogicalApply { la.baseLogicalPlan = newBaseLogicalPlan(ctx, plancodec.TypeApply, &la, offset) @@ -393,15 +399,7 @@ func (p PhysicalTableReader) Init(ctx sessionctx.Context, offset int) *PhysicalT // Init initializes PhysicalIndexReader. func (p PhysicalIndexReader) Init(ctx sessionctx.Context, offset int) *PhysicalIndexReader { p.basePhysicalPlan = newBasePhysicalPlan(ctx, plancodec.TypeIndexReader, &p, offset) - p.IndexPlans = flattenPushDownPlan(p.indexPlan) - switch p.indexPlan.(type) { - case *PhysicalHashAgg, *PhysicalStreamAgg: - p.schema = p.indexPlan.Schema() - default: - is := p.IndexPlans[0].(*PhysicalIndexScan) - p.schema = is.dataSourceSchema - } - p.OutputColumns = p.schema.Clone().Columns + p.SetSchema(nil) return &p } diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 78bf6e6968193..8a2a8f31c11fb 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2505,7 +2505,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as } b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SelectPriv, dbName.L, tableInfo.Name.L, "", authErr) - if tbl.Type() == table.VirtualTable { + if tbl.Type().IsVirtualTable() { return b.buildMemTable(ctx, dbName, tableInfo) } @@ -2530,7 +2530,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as if tblName.L == "" { tblName = tn.Name } - possiblePaths, err := b.getPossibleAccessPaths(tn.IndexHints, tableInfo, dbName, tblName) + possiblePaths, err := b.getPossibleAccessPaths(tn.IndexHints, tbl, dbName, tblName) if err != nil { return nil, err } @@ -2711,6 +2711,13 @@ func (b *PlanBuilder) buildMemTable(ctx context.Context, dbName model.CIStr, tab }.Init(b.ctx, b.getSelectOffset()) p.SetSchema(schema) p.names = names + + // Some memory tables can receive some predicates + switch tableInfo.Name.L { + case strings.ToLower(infoschema.TableTiDBClusterConfig): + p.Extractor = &ClusterConfigTableExtractor{} + } + return p, nil } diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index a9d4c656db41e..265713839186f 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -41,8 +41,9 @@ var ( _ LogicalPlan = &LogicalMaxOneRow{} _ LogicalPlan = &LogicalTableDual{} _ LogicalPlan = &DataSource{} - _ LogicalPlan = &TableGather{} - _ LogicalPlan = &TableScan{} + _ LogicalPlan = &TiKVSingleGather{} + _ LogicalPlan = &LogicalTableScan{} + _ LogicalPlan = &LogicalIndexScan{} _ LogicalPlan = &LogicalUnionAll{} _ LogicalPlan = &LogicalSort{} _ LogicalPlan = &LogicalLock{} @@ -282,6 +283,12 @@ func (la *LogicalAggregation) CopyAggHints(agg *LogicalAggregation) { la.aggHints = agg.aggHints } +// IsPartialModeAgg returns if all of the AggFuncs are partialMode. +func (la *LogicalAggregation) IsPartialModeAgg() bool { + // Since all of the AggFunc share the same AggMode, we only need to check the first one. + return la.AggFuncs[0].Mode == aggregation.Partial1Mode +} + // GetGroupByCols returns the groupByCols. If the groupByCols haven't be collected, // this method would collect them at first. If the GroupByItems have been changed, // we should explicitly collect GroupByColumns before this method. @@ -355,8 +362,8 @@ type LogicalTableDual struct { // LogicalMemTable represents a memory table or virtual table // Some memory tables wants to take the ownership of some predications // e.g -// SELECT * FROM tidb_cluster_log WHERE type='tikv' AND address='192.16.5.32' -// Assume that the table `tidb_cluster_log` is a memory table, which is used +// SELECT * FROM cluster_log WHERE type='tikv' AND address='192.16.5.32' +// Assume that the table `cluster_log` is a memory table, which is used // to retrieve logs from remote components. In the above situation we should // send log search request to the target TiKV (192.16.5.32) directly instead of // requesting all cluster components log search gRPC interface to retrieve @@ -364,6 +371,7 @@ type LogicalTableDual struct { type LogicalMemTable struct { logicalSchemaProducer + Extractor MemTablePredicateExtractor dbName model.CIStr tableInfo *model.TableInfo } @@ -420,15 +428,20 @@ type DataSource struct { preferStoreType int } -// TableGather is a leaf logical operator of TiDB layer to gather +// TiKVSingleGather is a leaf logical operator of TiDB layer to gather // tuples from TiKV regions. -type TableGather struct { +type TiKVSingleGather struct { logicalSchemaProducer Source *DataSource + // IsIndexGather marks if this TiKVSingleGather gathers tuples from an IndexScan. + // in implementation phase, we need this flag to determine whether to generate + // PhysicalTableReader or PhysicalIndexReader. + IsIndexGather bool + Index *model.IndexInfo } -// TableScan is the logical table scan operator for TiKV. -type TableScan struct { +// LogicalTableScan is the logical table scan operator for TiKV. +type LogicalTableScan struct { logicalSchemaProducer Source *DataSource Handle *expression.Column @@ -436,6 +449,43 @@ type TableScan struct { Ranges []*ranger.Range } +// LogicalIndexScan is the logical index scan operator for TiKV. +type LogicalIndexScan struct { + logicalSchemaProducer + // DataSource should be read-only here. + Source *DataSource + IsDoubleRead bool + + EqCondCount int + AccessConds expression.CNFExprs + Ranges []*ranger.Range + + Index *model.IndexInfo + Columns []*model.ColumnInfo + fullIdxCols []*expression.Column + fullIdxColLens []int + idxCols []*expression.Column + idxColLens []int +} + +// MatchIndexProp checks if the indexScan can match the required property. +func (p *LogicalIndexScan) MatchIndexProp(prop *property.PhysicalProperty) (match bool) { + if prop.IsEmpty() { + return true + } + if all, _ := prop.AllSameOrder(); !all { + return false + } + for i, col := range p.idxCols { + if col.Equal(nil, prop.Items[0].Col) { + return matchIndicesProp(p.idxCols[i:], p.idxColLens[i:], prop.Items) + } else if i >= p.EqCondCount { + break + } + } + return false +} + // accessPath indicates the way we access a table: by using single index, or by using multiple indexes, // or just by using table scan. type accessPath struct { @@ -474,21 +524,47 @@ func getTablePath(paths []*accessPath) *accessPath { } func (ds *DataSource) buildTableGather() LogicalPlan { - ts := TableScan{Source: ds, Handle: ds.getHandleCol()}.Init(ds.ctx, ds.blockOffset) + ts := LogicalTableScan{Source: ds, Handle: ds.getHandleCol()}.Init(ds.ctx, ds.blockOffset) ts.SetSchema(ds.Schema()) - tg := TableGather{Source: ds}.Init(ds.ctx, ds.blockOffset) - tg.SetSchema(ds.Schema()) - tg.SetChildren(ts) - return tg -} - -// Convert2Gathers builds logical TableGather and IndexGather(to be implemented) from DataSource. + sg := TiKVSingleGather{Source: ds, IsIndexGather: false}.Init(ds.ctx, ds.blockOffset) + sg.SetSchema(ds.Schema()) + sg.SetChildren(ts) + return sg +} + +func (ds *DataSource) buildIndexGather(path *accessPath) LogicalPlan { + is := LogicalIndexScan{ + Source: ds, + IsDoubleRead: false, + Index: path.index, + }.Init(ds.ctx, ds.blockOffset) + + is.Columns = make([]*model.ColumnInfo, len(ds.Columns)) + copy(is.Columns, ds.Columns) + is.SetSchema(ds.Schema()) + + sg := TiKVSingleGather{ + Source: ds, + IsIndexGather: true, + Index: path.index, + }.Init(ds.ctx, ds.blockOffset) + sg.SetSchema(ds.Schema()) + sg.SetChildren(is) + return sg +} + +// Convert2Gathers builds logical TiKVSingleGathers from DataSource. func (ds *DataSource) Convert2Gathers() (gathers []LogicalPlan) { tg := ds.buildTableGather() gathers = append(gathers, tg) for _, path := range ds.possibleAccessPaths { if !path.isTablePath { - // TODO: add IndexGather + path.fullIdxCols, path.fullIdxColLens = expression.IndexInfo2Cols(ds.Columns, ds.schema.Columns, path.index) + // If index columns can cover all of the needed columns, we can use a IndexGather + IndexScan. + if isCoveringIndex(ds.schema.Columns, path.fullIdxCols, path.fullIdxColLens, ds.tableInfo.PKIsHandle) { + gathers = append(gathers, ds.buildIndexGather(path)) + } + // TODO: If index columns can not cover the schema, use IndexLookUpGather. } } return gathers @@ -738,24 +814,35 @@ func isColEqCorColOrConstant(filter expression.Expression, col *expression.Colum return false } -func (ds *DataSource) getPKIsHandleCol() *expression.Column { - if !ds.tableInfo.PKIsHandle { +func getPKIsHandleColFromSchema(cols []*model.ColumnInfo, schema *expression.Schema, pkIsHandle bool) *expression.Column { + if !pkIsHandle { // If the PKIsHandle is false, return the ExtraHandleColumn. - for i, col := range ds.Columns { + for i, col := range cols { if col.ID == model.ExtraHandleID { - return ds.schema.Columns[i] + return schema.Columns[i] } } return nil } - for i, col := range ds.Columns { + for i, col := range cols { if mysql.HasPriKeyFlag(col.Flag) { - return ds.schema.Columns[i] + return schema.Columns[i] } } return nil } +func (ds *DataSource) getPKIsHandleCol() *expression.Column { + return getPKIsHandleColFromSchema(ds.Columns, ds.schema, ds.tableInfo.PKIsHandle) +} + +func (p *LogicalIndexScan) getPKIsHandleCol() *expression.Column { + // We cannot use p.Source.getPKIsHandleCol() here, + // Because we may re-prune p.Columns and p.schema during the transformation. + // That will make p.Columns different from p.Source.Columns. + return getPKIsHandleColFromSchema(p.Columns, p.schema, p.Source.tableInfo.PKIsHandle) +} + func (ds *DataSource) getHandleCol() *expression.Column { if ds.handleCol != nil { return ds.handleCol diff --git a/planner/core/memtable_predicate_extractor.go b/planner/core/memtable_predicate_extractor.go new file mode 100644 index 0000000000000..de03644cc2890 --- /dev/null +++ b/planner/core/memtable_predicate_extractor.go @@ -0,0 +1,201 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "fmt" + "strings" + + "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/set" +) + +// MemTablePredicateExtractor is used to extract some predicates from `WHERE` clause +// and push the predicates down to the data retrieving on reading memory table stage. +// +// e.g: +// SELECT * FROM cluster_config WHERE type='tikv' AND address='192.168.1.9:2379' +// We must request all components in the cluster via HTTP API for retrieving +// configurations and filter them by `type/address` columns. +// +// The purpose of defining a `MemTablePredicateExtractor` is to optimize this +// 1. Define a `ClusterConfigTablePredicateExtractor` +// 2. Extract the `type/address` columns on the logic optimizing stage and save them via fields. +// 3. Passing the extractor to the `ClusterConfigReaderExec` executor +// 4. Executor sends requests to the target components instead of all of the components +type MemTablePredicateExtractor interface { + // Extracts predicates which can be pushed down and returns the remained predicates + Extract(*expression.Schema, []*types.FieldName, []expression.Expression) (remained []expression.Expression) +} + +// extractHelper contains some common utililty functions for all extractor. +// define an individual struct instead of a bunch of un-exported functions +// to avoid polluting the global scope of current package. +type extractHelper struct{} + +func (helper extractHelper) extractColInConsExpr(extractCols map[int64]*types.FieldName, expr *expression.ScalarFunction) (string, []types.Datum) { + args := expr.GetArgs() + col, isCol := args[0].(*expression.Column) + if !isCol { + return "", nil + } + name, found := extractCols[col.UniqueID] + if !found { + return "", nil + } + // All expressions in IN must be a constant + // SELECT * FROM t1 WHERE c IN ('1', '2') + var results []types.Datum + for _, arg := range args[1:] { + constant, ok := arg.(*expression.Constant) + if !ok || constant.DeferredExpr != nil || constant.ParamMarker != nil { + return "", nil + } + results = append(results, constant.Value) + } + return name.ColName.L, results +} + +func (helper extractHelper) extractColEqConsExpr(extractCols map[int64]*types.FieldName, expr *expression.ScalarFunction) (string, []types.Datum) { + args := expr.GetArgs() + var col *expression.Column + var colIdx int + // c = 'rhs' + // 'lhs' = c + for i := 0; i < 2; i++ { + var isCol bool + col, isCol = args[i].(*expression.Column) + if isCol { + colIdx = i + break + } + } + if col == nil { + return "", nil + } + + name, found := extractCols[col.UniqueID] + if !found { + return "", nil + } + // The `lhs/rhs` of EQ expression must be a constant + // SELECT * FROM t1 WHERE c='rhs' + // SELECT * FROM t1 WHERE 'lhs'=c + constant, ok := args[1-colIdx].(*expression.Constant) + if !ok || constant.DeferredExpr != nil || constant.ParamMarker != nil { + return "", nil + } + return name.ColName.L, []types.Datum{constant.Value} +} + +func (helper extractHelper) intersection(lhs set.StringSet, datums []types.Datum, toLower bool) set.StringSet { + tmpNodeTypes := set.NewStringSet() + for _, datum := range datums { + var s string + if toLower { + s = strings.ToLower(datum.GetString()) + } else { + s = datum.GetString() + } + tmpNodeTypes.Insert(s) + } + if len(lhs) > 0 { + return lhs.Intersection(tmpNodeTypes) + } + return tmpNodeTypes +} + +// ClusterConfigTableExtractor is used to extract some predicates of `cluster_config` +type ClusterConfigTableExtractor struct { + extractHelper + + // SkipRequest means the where clause always false, we don't need to request any component + SkipRequest bool + + // NodeTypes represents all components types we should send request to. + // e.g: + // 1. SELECT * FROM cluster_config WHERE type='tikv' + // 2. SELECT * FROM cluster_config WHERE type in ('tikv', 'tidb') + NodeTypes set.StringSet + + // Addresses represents all components addresses we should send request to. + // e.g: + // 1. SELECT * FROM cluster_config WHERE address='192.168.1.7:2379' + // 2. SELECT * FROM cluster_config WHERE type in ('192.168.1.7:2379', '192.168.1.9:2379') + Addresses set.StringSet +} + +// Extract implements the MemTablePredicateExtractor Extract interface +func (e *ClusterConfigTableExtractor) Extract(schema *expression.Schema, names []*types.FieldName, predicates []expression.Expression) []expression.Expression { + remained := make([]expression.Expression, 0, len(predicates)) + // All columns can be pushed down to the memory table `cluster_config` + const ( + ColNameType = "type" + ColNameAddress = "address" + ) + extractCols := make(map[int64]*types.FieldName) + for i, name := range names { + if ln := name.ColName.L; ln == ColNameType || ln == ColNameAddress { + extractCols[schema.Columns[i].UniqueID] = name + } + } + // We use the column name literal (local constant) to find the column in `names` + // instead of using a global constant. So the assumption (named `type/address`) + // maybe not satisfied if the column name has been changed in the future. + // The purpose of the following assert is used to make sure our assumption doesn't + // be broken (or hint the author who refactors this part to change here too). + if len(extractCols) != 2 { + panic(fmt.Sprintf("push down columns `type/address` not found in schema, got: %+v", extractCols)) + } + + skipRequest := false + nodeTypes := set.NewStringSet() + addresses := set.NewStringSet() + + // We should use INTERSECTION of sets because of the predicates is CNF array + for _, expr := range predicates { + var colName string + var datums []types.Datum + switch x := expr.(type) { + case *expression.ScalarFunction: + switch x.FuncName.L { + case ast.EQ: + colName, datums = e.extractColEqConsExpr(extractCols, x) + case ast.In: + colName, datums = e.extractColInConsExpr(extractCols, x) + } + } + switch colName { + case ColNameType: + nodeTypes = e.intersection(nodeTypes, datums, true) + skipRequest = len(nodeTypes) == 0 + case ColNameAddress: + addresses = e.intersection(addresses, datums, false) + skipRequest = len(addresses) == 0 + default: + remained = append(remained, expr) + } + // There are no data if the low-level executor skip request, so the filter can be droped + if skipRequest { + remained = remained[:0] + break + } + } + e.SkipRequest = skipRequest + e.NodeTypes = nodeTypes + e.Addresses = addresses + return remained +} diff --git a/planner/core/memtable_predicate_extractor_test.go b/planner/core/memtable_predicate_extractor_test.go new file mode 100644 index 0000000000000..3c76e07294397 --- /dev/null +++ b/planner/core/memtable_predicate_extractor_test.go @@ -0,0 +1,217 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core_test + +import ( + "context" + + . "github.com/pingcap/check" + "github.com/pingcap/parser" + "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/kv" + plannercore "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/store/mockstore" + "github.com/pingcap/tidb/util/set" +) + +var _ = Suite(&extractorSuite{}) + +type extractorSuite struct { + store kv.Storage + dom *domain.Domain +} + +func (s *extractorSuite) SetUpSuite(c *C) { + store, err := mockstore.NewMockTikvStore() + c.Assert(err, IsNil) + c.Assert(store, NotNil) + + session.SetSchemaLease(0) + session.DisableStats4Test() + dom, err := session.BootstrapSession(store) + c.Assert(err, IsNil) + c.Assert(dom, NotNil) + + s.store = store + s.dom = dom +} + +func (s *extractorSuite) TearDownSuite(c *C) { + s.dom.Close() + s.store.Close() +} + +func (s *extractorSuite) TestClusterConfigTableExtractor(c *C) { + se, err := session.CreateSession4Test(s.store) + c.Assert(err, IsNil) + + parser := parser.New() + var cases = []struct { + sql string + nodeTypes set.StringSet + addresses set.StringSet + skipRequest bool + }{ + { + sql: "select * from information_schema.cluster_config", + nodeTypes: nil, + addresses: nil, + }, + { + sql: "select * from information_schema.cluster_config where type='tikv'", + nodeTypes: set.NewStringSet("tikv"), + addresses: set.NewStringSet(), + }, + { + sql: "select * from information_schema.cluster_config where 'tikv'=type", + nodeTypes: set.NewStringSet("tikv"), + addresses: set.NewStringSet(), + }, + { + sql: "select * from information_schema.cluster_config where 'TiKV'=type", + nodeTypes: set.NewStringSet("tikv"), + addresses: set.NewStringSet(), + }, + { + sql: "select * from information_schema.cluster_config where type in ('tikv', 'pd')", + nodeTypes: set.NewStringSet("tikv", "pd"), + addresses: set.NewStringSet(), + }, + { + sql: "select * from information_schema.cluster_config where type in ('tikv', 'pd') and address='123.1.1.2:1234'", + nodeTypes: set.NewStringSet("tikv", "pd"), + addresses: set.NewStringSet("123.1.1.2:1234"), + }, + { + sql: "select * from information_schema.cluster_config where type in ('tikv', 'pd') and address in ('123.1.1.2:1234', '123.1.1.4:1234')", + nodeTypes: set.NewStringSet("tikv", "pd"), + addresses: set.NewStringSet("123.1.1.2:1234", "123.1.1.4:1234"), + }, + { + sql: "select * from information_schema.cluster_config where type='tikv' and address in ('123.1.1.2:1234', '123.1.1.4:1234')", + nodeTypes: set.NewStringSet("tikv"), + addresses: set.NewStringSet("123.1.1.2:1234", "123.1.1.4:1234"), + }, + { + sql: "select * from information_schema.cluster_config where type='tikv' and address='123.1.1.4:1234'", + nodeTypes: set.NewStringSet("tikv"), + addresses: set.NewStringSet("123.1.1.4:1234"), + }, + { + sql: "select * from information_schema.cluster_config where type='tikv' and address='123.1.1.4:1234'", + nodeTypes: set.NewStringSet("tikv"), + addresses: set.NewStringSet("123.1.1.4:1234"), + }, + { + sql: "select * from information_schema.cluster_config where type='tikv' and address='cNs2dm.tikv.pingcap.com:1234'", + nodeTypes: set.NewStringSet("tikv"), + addresses: set.NewStringSet("cNs2dm.tikv.pingcap.com:1234"), + }, + { + sql: "select * from information_schema.cluster_config where type='TIKV' and address='cNs2dm.tikv.pingcap.com:1234'", + nodeTypes: set.NewStringSet("tikv"), + addresses: set.NewStringSet("cNs2dm.tikv.pingcap.com:1234"), + }, + { + sql: "select * from information_schema.cluster_config where type='tikv' and type='pd'", + nodeTypes: set.NewStringSet(), + addresses: set.NewStringSet(), + skipRequest: true, + }, + { + sql: "select * from information_schema.cluster_config where type='tikv' and type in ('pd', 'tikv')", + nodeTypes: set.NewStringSet("tikv"), + addresses: set.NewStringSet(), + }, + { + sql: "select * from information_schema.cluster_config where type='tikv' and type in ('pd', 'tidb')", + nodeTypes: set.NewStringSet(), + addresses: set.NewStringSet(), + skipRequest: true, + }, + { + sql: "select * from information_schema.cluster_config where type in ('tikv', 'tidb') and type in ('pd', 'tidb')", + nodeTypes: set.NewStringSet("tidb"), + addresses: set.NewStringSet(), + }, + { + sql: "select * from information_schema.cluster_config where address='123.1.1.4:1234' and address='123.1.1.5:1234'", + nodeTypes: set.NewStringSet(), + addresses: set.NewStringSet(), + skipRequest: true, + }, + { + sql: "select * from information_schema.cluster_config where address='123.1.1.4:1234' and address in ('123.1.1.5:1234', '123.1.1.4:1234')", + nodeTypes: set.NewStringSet(), + addresses: set.NewStringSet("123.1.1.4:1234"), + }, + { + sql: "select * from information_schema.cluster_config where address='123.1.1.4:1234' and address in ('123.1.1.5:1234', '123.1.1.6:1234')", + nodeTypes: set.NewStringSet(), + addresses: set.NewStringSet(), + skipRequest: true, + }, + { + sql: "select * from information_schema.cluster_config where address in ('123.1.1.5:1234', '123.1.1.4:1234') and address in ('123.1.1.5:1234', '123.1.1.6:1234')", + nodeTypes: set.NewStringSet(), + addresses: set.NewStringSet("123.1.1.5:1234"), + }, + { + sql: `select * from information_schema.cluster_config + where address in ('123.1.1.5:1234', '123.1.1.4:1234') + and address in ('123.1.1.5:1234', '123.1.1.6:1234') + and type in ('tikv', 'tidb') + and type in ('pd', 'tidb')`, + nodeTypes: set.NewStringSet("tidb"), + addresses: set.NewStringSet("123.1.1.5:1234"), + }, + { + sql: `select * from information_schema.cluster_config + where address in ('123.1.1.5:1234', '123.1.1.4:1234') + and address in ('123.1.1.5:1234', '123.1.1.6:1234') + and address in ('123.1.1.6:1234', '123.1.1.7:1234') + and address in ('123.1.1.7:1234', '123.1.1.8:1234')`, + nodeTypes: set.NewStringSet(), + addresses: set.NewStringSet(), + skipRequest: true, + }, + } + for _, ca := range cases { + stmt, err := parser.ParseOneStmt(ca.sql, "", "") + c.Assert(err, IsNil) + + ctx := context.Background() + builder := plannercore.NewPlanBuilder(se, s.dom.InfoSchema(), &plannercore.BlockHintProcessor{}) + plan, err := builder.Build(ctx, stmt) + c.Assert(err, IsNil) + + logicalPlan, err := plannercore.LogicalOptimize(ctx, builder.GetOptFlag(), plan.(plannercore.LogicalPlan)) + c.Assert(err, IsNil) + + // Obtain the leaf plan + leafPlan := logicalPlan + for len(leafPlan.Children()) > 0 { + leafPlan = leafPlan.Children()[0] + } + + logicalMemTable := leafPlan.(*plannercore.LogicalMemTable) + c.Assert(logicalMemTable.Extractor, NotNil) + + clusterConfigExtractor := logicalMemTable.Extractor.(*plannercore.ClusterConfigTableExtractor) + c.Assert(clusterConfigExtractor.NodeTypes, DeepEquals, ca.nodeTypes, Commentf("SQL: %v", ca.sql)) + c.Assert(clusterConfigExtractor.Addresses, DeepEquals, ca.addresses, Commentf("SQL: %v", ca.sql)) + c.Assert(clusterConfigExtractor.SkipRequest, DeepEquals, ca.skipRequest, Commentf("SQL: %v", ca.sql)) + } +} diff --git a/planner/core/optimizer_test.go b/planner/core/optimizer_test.go new file mode 100644 index 0000000000000..1a8b91ec682e2 --- /dev/null +++ b/planner/core/optimizer_test.go @@ -0,0 +1,18 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +// LogicalOptimize exports the `logicalOptimize` function for test packages and +// doesn't affect the normal package and access control of Golang (tricky ^_^) +var LogicalOptimize = logicalOptimize diff --git a/planner/core/pb_to_plan.go b/planner/core/pb_to_plan.go new file mode 100644 index 0000000000000..1302254686e49 --- /dev/null +++ b/planner/core/pb_to_plan.go @@ -0,0 +1,220 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "github.com/pingcap/errors" + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/expression/aggregation" + "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tipb/go-tipb" +) + +// PBPlanBuilder uses to build physical plan from dag protocol buffers. +type PBPlanBuilder struct { + sctx sessionctx.Context + tps []*types.FieldType + is infoschema.InfoSchema +} + +// NewPBPlanBuilder creates a new pb plan builder. +func NewPBPlanBuilder(sctx sessionctx.Context, is infoschema.InfoSchema) *PBPlanBuilder { + return &PBPlanBuilder{sctx: sctx, is: is} +} + +// Build builds physical plan from dag protocol buffers. +func (b *PBPlanBuilder) Build(executors []*tipb.Executor) (p PhysicalPlan, err error) { + var src PhysicalPlan + for i := 0; i < len(executors); i++ { + curr, err := b.pbToPhysicalPlan(executors[i]) + if err != nil { + return nil, errors.Trace(err) + } + curr.SetChildren(src) + src = curr + } + return src, nil +} + +func (b *PBPlanBuilder) pbToPhysicalPlan(e *tipb.Executor) (p PhysicalPlan, err error) { + switch e.Tp { + case tipb.ExecType_TypeTableScan: + p, err = b.pbToTableScan(e) + case tipb.ExecType_TypeSelection: + p, err = b.pbToSelection(e) + case tipb.ExecType_TypeTopN: + p, err = b.pbToTopN(e) + case tipb.ExecType_TypeLimit: + p, err = b.pbToLimit(e) + case tipb.ExecType_TypeAggregation: + p, err = b.pbToAgg(e, false) + case tipb.ExecType_TypeStreamAgg: + p, err = b.pbToAgg(e, true) + default: + // TODO: Support other types. + err = errors.Errorf("this exec type %v doesn't support yet.", e.GetTp()) + } + return p, err +} + +func (b *PBPlanBuilder) pbToTableScan(e *tipb.Executor) (PhysicalPlan, error) { + tblScan := e.TblScan + tbl, ok := b.is.TableByID(tblScan.TableId) + if !ok { + return nil, infoschema.ErrTableNotExists.GenWithStack("Table which ID = %d does not exist.", tblScan.TableId) + } + // Currently only support cluster table. + if !tbl.Type().IsClusterTable() { + return nil, errors.Errorf("table %s is not a cluster table", tbl.Meta().Name.L) + } + columns, err := b.convertColumnInfo(tbl.Meta(), tblScan.Columns) + if err != nil { + return nil, err + } + schema := b.buildTableScanSchema(tbl.Meta(), columns) + p := PhysicalMemTable{ + Table: tbl.Meta(), + Columns: columns, + }.Init(b.sctx, nil, 0) + p.SetSchema(schema) + return p, nil +} + +func (b *PBPlanBuilder) buildTableScanSchema(tblInfo *model.TableInfo, columns []*model.ColumnInfo) *expression.Schema { + schema := expression.NewSchema(make([]*expression.Column, 0, len(columns))...) + for _, col := range tblInfo.Columns { + for _, colInfo := range columns { + if col.ID != colInfo.ID { + continue + } + newCol := &expression.Column{ + UniqueID: b.sctx.GetSessionVars().AllocPlanColumnID(), + ID: col.ID, + RetType: &col.FieldType, + } + schema.Append(newCol) + } + } + return schema +} + +func (b *PBPlanBuilder) pbToSelection(e *tipb.Executor) (PhysicalPlan, error) { + conds, err := expression.PBToExprs(e.Selection.Conditions, b.tps, b.sctx.GetSessionVars().StmtCtx) + if err != nil { + return nil, err + } + p := PhysicalSelection{ + Conditions: conds, + }.Init(b.sctx, nil, 0) + return p, nil +} + +func (b *PBPlanBuilder) pbToTopN(e *tipb.Executor) (PhysicalPlan, error) { + topN := e.TopN + sc := b.sctx.GetSessionVars().StmtCtx + byItems := make([]*ByItems, 0, len(topN.OrderBy)) + for _, item := range topN.OrderBy { + expr, err := expression.PBToExpr(item.Expr, b.tps, sc) + if err != nil { + return nil, errors.Trace(err) + } + byItems = append(byItems, &ByItems{Expr: expr, Desc: item.Desc}) + } + p := PhysicalTopN{ + ByItems: byItems, + Count: topN.Limit, + }.Init(b.sctx, nil, 0) + return p, nil +} + +func (b *PBPlanBuilder) pbToLimit(e *tipb.Executor) (PhysicalPlan, error) { + p := PhysicalLimit{ + Count: e.Limit.Limit, + }.Init(b.sctx, nil, 0) + return p, nil +} + +func (b *PBPlanBuilder) pbToAgg(e *tipb.Executor, isStreamAgg bool) (PhysicalPlan, error) { + aggFuncs, groupBys, err := b.getAggInfo(e) + if err != nil { + return nil, errors.Trace(err) + } + schema := b.buildAggSchema(aggFuncs, groupBys) + baseAgg := basePhysicalAgg{ + AggFuncs: aggFuncs, + GroupByItems: groupBys, + } + baseAgg.schema = schema + var partialAgg PhysicalPlan + if isStreamAgg { + partialAgg = baseAgg.initForHash(b.sctx, nil, 0) + } else { + partialAgg = baseAgg.initForStream(b.sctx, nil, 0) + } + return partialAgg, nil +} + +func (b *PBPlanBuilder) buildAggSchema(aggFuncs []*aggregation.AggFuncDesc, groupBys []expression.Expression) *expression.Schema { + schema := expression.NewSchema(make([]*expression.Column, 0, len(aggFuncs)+len(groupBys))...) + for _, agg := range aggFuncs { + newCol := &expression.Column{ + UniqueID: b.sctx.GetSessionVars().AllocPlanColumnID(), + RetType: agg.RetTp, + } + schema.Append(newCol) + } + return schema +} + +func (b *PBPlanBuilder) getAggInfo(executor *tipb.Executor) ([]*aggregation.AggFuncDesc, []expression.Expression, error) { + var err error + aggFuncs := make([]*aggregation.AggFuncDesc, 0, len(executor.Aggregation.AggFunc)) + sc := b.sctx.GetSessionVars().StmtCtx + for _, expr := range executor.Aggregation.AggFunc { + aggFunc, err := aggregation.PBExprToAggFuncDesc(sc, expr, b.tps) + if err != nil { + return nil, nil, errors.Trace(err) + } + aggFuncs = append(aggFuncs, aggFunc) + } + groupBys, err := expression.PBToExprs(executor.Aggregation.GetGroupBy(), b.tps, b.sctx.GetSessionVars().StmtCtx) + if err != nil { + return nil, nil, errors.Trace(err) + } + return aggFuncs, groupBys, nil +} + +func (b *PBPlanBuilder) convertColumnInfo(tblInfo *model.TableInfo, pbColumns []*tipb.ColumnInfo) ([]*model.ColumnInfo, error) { + columns := make([]*model.ColumnInfo, 0, len(pbColumns)) + tps := make([]*types.FieldType, 0, len(pbColumns)) + for _, col := range pbColumns { + found := false + for _, colInfo := range tblInfo.Columns { + if col.ColumnId == colInfo.ID { + columns = append(columns, colInfo) + tps = append(tps, colInfo.FieldType.Clone()) + found = true + break + } + } + if !found { + return nil, errors.Errorf("Column ID %v of table %v not found", col.ColumnId, tblInfo.Name.L) + } + } + b.tps = tps + return columns, nil +} diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index db3d35a78584a..5257289563420 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -39,29 +39,33 @@ var _ = SerialSuites(&testPlanSerialSuite{}) type testPlanSuiteBase struct { *parser.Parser is infoschema.InfoSchema +} - testData testutil.TestData +func (s *testPlanSuiteBase) SetUpSuite(c *C) { + s.is = infoschema.MockInfoSchema([]*model.TableInfo{core.MockSignedTable(), core.MockUnsignedTable()}) + s.Parser = parser.New() + s.Parser.EnableWindowFunc(true) } -type testPlanSuite struct { +type testPlanSerialSuite struct { testPlanSuiteBase } -type testPlanSerialSuite struct { +type testPlanSuite struct { testPlanSuiteBase + + testData testutil.TestData } -func (s *testPlanSuiteBase) SetUpSuite(c *C) { - s.is = infoschema.MockInfoSchema([]*model.TableInfo{core.MockSignedTable(), core.MockUnsignedTable()}) - s.Parser = parser.New() - s.Parser.EnableWindowFunc(true) +func (s *testPlanSuite) SetUpSuite(c *C) { + s.testPlanSuiteBase.SetUpSuite(c) var err error s.testData, err = testutil.LoadTestSuiteData("testdata", "plan_suite") c.Assert(err, IsNil) } -func (s *testPlanSuiteBase) TearDownSuite(c *C) { +func (s *testPlanSuite) TearDownSuite(c *C) { c.Assert(s.testData.GenerateOutputIfNeeded(), IsNil) } diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 5a91ca6f93026..23b17a1e26786 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -67,9 +67,18 @@ type PhysicalTableReader struct { StoreType kv.StoreType } -// GetPhysicalReader returns PhysicalTableReader for logical TableGather. -func (tg *TableGather) GetPhysicalReader(schema *expression.Schema, stats *property.StatsInfo, props ...*property.PhysicalProperty) *PhysicalTableReader { - reader := PhysicalTableReader{}.Init(tg.ctx, tg.blockOffset) +// GetPhysicalTableReader returns PhysicalTableReader for logical TiKVSingleGather. +func (sg *TiKVSingleGather) GetPhysicalTableReader(schema *expression.Schema, stats *property.StatsInfo, props ...*property.PhysicalProperty) *PhysicalTableReader { + reader := PhysicalTableReader{}.Init(sg.ctx, sg.blockOffset) + reader.stats = stats + reader.SetSchema(schema) + reader.childrenReqProps = props + return reader +} + +// GetPhysicalIndexReader returns PhysicalIndexReader for logical TiKVSingleGather. +func (sg *TiKVSingleGather) GetPhysicalIndexReader(schema *expression.Schema, stats *property.StatsInfo, props ...*property.PhysicalProperty) *PhysicalIndexReader { + reader := PhysicalIndexReader{}.Init(sg.ctx, sg.blockOffset) reader.stats = stats reader.SetSchema(schema) reader.childrenReqProps = props @@ -94,6 +103,27 @@ type PhysicalIndexReader struct { OutputColumns []*expression.Column } +// SetSchema overrides PhysicalPlan SetSchema interface. +func (p *PhysicalIndexReader) SetSchema(_ *expression.Schema) { + if p.indexPlan != nil { + p.IndexPlans = flattenPushDownPlan(p.indexPlan) + switch p.indexPlan.(type) { + case *PhysicalHashAgg, *PhysicalStreamAgg: + p.schema = p.indexPlan.Schema() + default: + is := p.IndexPlans[0].(*PhysicalIndexScan) + p.schema = is.dataSourceSchema + } + p.OutputColumns = p.schema.Clone().Columns + } +} + +// SetChildren overrides PhysicalPlan SetChildren interface. +func (p *PhysicalIndexReader) SetChildren(children ...PhysicalPlan) { + p.indexPlan = children[0] + p.SetSchema(nil) +} + // PushedDownLimit is the limit operator pushed down into PhysicalIndexLookUpReader. type PushedDownLimit struct { Offset uint64 @@ -172,8 +202,8 @@ type PhysicalIndexScan struct { type PhysicalMemTable struct { physicalSchemaProducer - DBName model.CIStr - Table *model.TableInfo + Table *model.TableInfo + Columns []*model.ColumnInfo } // PhysicalTableScan represents a table scan plan. diff --git a/planner/core/plan.go b/planner/core/plan.go index 82814cbb399b5..6cdf570f7072e 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -95,7 +95,10 @@ type LogicalPlan interface { findBestTask(prop *property.PhysicalProperty) (task, error) // BuildKeyInfo will collect the information of unique keys into schema. - BuildKeyInfo() + // Because this method is also used in cascades planner, we cannot use + // things like `p.schema` or `p.children` inside it. We should use the `selfSchema` + // and `childSchema` instead. + BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) // pushDownTopN will push down the topN or limit operator during logical optimization. pushDownTopN(topN *LogicalTopN) LogicalPlan @@ -163,6 +166,9 @@ type PhysicalPlan interface { // Stats returns the StatsInfo of the plan. Stats() *property.StatsInfo + + // ExplainNormalizedInfo returns operator normalized information for generating digest. + ExplainNormalizedInfo() string } type baseLogicalPlan struct { @@ -196,6 +202,11 @@ func (p *basePhysicalPlan) ExplainInfo() string { return "" } +// ExplainInfo implements Plan interface. +func (p *basePhysicalPlan) ExplainNormalizedInfo() string { + return "" +} + func (p *basePhysicalPlan) GetChildReqProps(idx int) *property.PhysicalProperty { return p.childrenReqProps[idx] } @@ -210,17 +221,44 @@ func (p *baseLogicalPlan) storeTask(prop *property.PhysicalProperty, task task) p.taskMap[string(key)] = task } -// BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. -func (p *baseLogicalPlan) BuildKeyInfo() { - for _, child := range p.children { - child.BuildKeyInfo() +// HasMaxOneRow returns if the LogicalPlan will output at most one row. +func HasMaxOneRow(p LogicalPlan, childMaxOneRow []bool) bool { + if len(childMaxOneRow) == 0 { + // The reason why we use this check is that, this function + // is used both in planner/core and planner/cascades. + // In cascades planner, LogicalPlan may have no `children`. + return false } - switch p.self.(type) { - case *LogicalLock, *LogicalLimit, *LogicalSort, *LogicalSelection, *LogicalApply, *LogicalProjection: - p.maxOneRow = p.children[0].MaxOneRow() + switch x := p.(type) { + case *LogicalLock, *LogicalLimit, *LogicalSort, *LogicalSelection, + *LogicalApply, *LogicalProjection, *LogicalWindow, *LogicalAggregation: + return childMaxOneRow[0] case *LogicalMaxOneRow: - p.maxOneRow = true + return true + case *LogicalJoin: + switch x.JoinType { + case SemiJoin, AntiSemiJoin, LeftOuterSemiJoin, AntiLeftOuterSemiJoin: + return childMaxOneRow[0] + default: + return childMaxOneRow[0] && childMaxOneRow[1] + } } + return false +} + +// BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. +func (p *baseLogicalPlan) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + childMaxOneRow := make([]bool, len(p.children)) + for i := range p.children { + childMaxOneRow[i] = p.children[i].MaxOneRow() + } + p.maxOneRow = HasMaxOneRow(p.self, childMaxOneRow) +} + +// BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. +func (p *logicalSchemaProducer) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + selfSchema.Keys = nil + p.baseLogicalPlan.BuildKeyInfo(selfSchema, childSchema) } func newBasePlan(ctx sessionctx.Context, tp string, offset int) basePlan { diff --git a/planner/core/plan_test.go b/planner/core/plan_test.go new file mode 100644 index 0000000000000..39df6d3652873 --- /dev/null +++ b/planner/core/plan_test.go @@ -0,0 +1,210 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by aprettyPrintlicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core_test + +import ( + "strings" + + . "github.com/pingcap/check" + "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/util/plancodec" + "github.com/pingcap/tidb/util/testkit" + "github.com/pingcap/tidb/util/testleak" + "github.com/pingcap/tidb/util/testutil" +) + +var _ = Suite(&testPlanNormalize{}) + +type testPlanNormalize struct { + store kv.Storage + dom *domain.Domain + + testData testutil.TestData +} + +func (s *testPlanNormalize) SetUpSuite(c *C) { + testleak.BeforeTest() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + s.store = store + s.dom = dom + + s.testData, err = testutil.LoadTestSuiteData("testdata", "plan_normalized_suite") + c.Assert(err, IsNil) +} + +func (s *testPlanNormalize) TearDownSuite(c *C) { + c.Assert(s.testData.GenerateOutputIfNeeded(), IsNil) + s.dom.Close() + s.store.Close() + testleak.AfterTest(c)() +} + +func (s *testPlanNormalize) TestNormalizedPlan(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1,t2") + tk.MustExec("create table t1 (a int key,b int,c int, index (b));") + tk.MustExec("create table t2 (a int key,b int,c int, index (b));") + var input []string + var output []struct { + SQL string + Plan []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + tk.Se.GetSessionVars().PlanID = 0 + tk.MustExec(tt) + info := tk.Se.ShowProcess() + c.Assert(info, NotNil) + p, ok := info.Plan.(core.Plan) + c.Assert(ok, IsTrue) + normalized, _ := core.NormalizePlan(p) + normalizedPlan, err := plancodec.DecodeNormalizedPlan(normalized) + normalizedPlanRows := getPlanRows(normalizedPlan) + c.Assert(err, IsNil) + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = normalizedPlanRows + }) + compareStringSlice(c, normalizedPlanRows, output[i].Plan) + } +} + +func (s *testPlanNormalize) TestNormalizedDigest(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1,t2") + tk.MustExec("create table t1 (a int key,b int,c int, index (b));") + tk.MustExec("create table t2 (a int key,b int,c int, index (b));") + normalizedDigestCases := []struct { + sql1 string + sql2 string + isSame bool + }{ + { + sql1: "select * from t1;", + sql2: "select * from t2;", + isSame: false, + }, + { // test for tableReader and tableScan. + sql1: "select * from t1 where a<1", + sql2: "select * from t1 where a<2", + isSame: true, + }, + { + sql1: "select * from t1 where a<1", + sql2: "select * from t1 where a=2", + isSame: false, + }, + { // test for point get. + sql1: "select * from t1 where a=3", + sql2: "select * from t1 where a=2", + isSame: true, + }, + { // test for indexLookUp. + sql1: "select * from t1 use index(b) where b=3", + sql2: "select * from t1 use index(b) where b=1", + isSame: true, + }, + { // test for indexReader. + sql1: "select a+1,b+2 from t1 use index(b) where b=3", + sql2: "select a+2,b+3 from t1 use index(b) where b>2", + isSame: true, + }, + { // test for merge join. + sql1: "SELECT /*+ TIDB_SMJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + sql2: "SELECT /*+ TIDB_SMJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>2;", + isSame: true, + }, + { // test for indexLookUpJoin. + sql1: "SELECT /*+ TIDB_INLJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + sql2: "SELECT /*+ TIDB_INLJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>3;", + isSame: true, + }, + { // test for hashJoin. + sql1: "SELECT /*+ TIDB_HJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + sql2: "SELECT /*+ TIDB_HJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>3;", + isSame: true, + }, + { // test for diff join. + sql1: "SELECT /*+ TIDB_HJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + sql2: "SELECT /*+ TIDB_INLJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>3;", + isSame: false, + }, + { // test for diff join. + sql1: "SELECT /*+ TIDB_INLJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + sql2: "SELECT /*+ TIDB_SMJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>3;", + isSame: false, + }, + { // test for apply. + sql1: "select * from t1 where t1.b > 0 and t1.a in (select sum(t2.b) from t2 where t2.a=t1.a and t2.b is not null and t2.c >1)", + sql2: "select * from t1 where t1.b > 1 and t1.a in (select sum(t2.b) from t2 where t2.a=t1.a and t2.b is not null and t2.c >0)", + isSame: true, + }, + { // test for apply. + sql1: "select * from t1 where t1.b > 0 and t1.a in (select sum(t2.b) from t2 where t2.a=t1.a and t2.b is not null and t2.c >1)", + sql2: "select * from t1 where t1.b > 1 and t1.a in (select sum(t2.b) from t2 where t2.a=t1.a and t2.b is not null)", + isSame: false, + }, + { // test for topN. + sql1: "SELECT * from t1 where a!=1 order by c limit 1", + sql2: "SELECT * from t1 where a!=2 order by c limit 2", + isSame: true, + }, + } + for _, testCase := range normalizedDigestCases { + testNormalizeDigest(tk, c, testCase.sql1, testCase.sql2, testCase.isSame) + } +} + +func testNormalizeDigest(tk *testkit.TestKit, c *C, sql1, sql2 string, isSame bool) { + tk.Se.GetSessionVars().PlanID = 0 + tk.MustQuery(sql1) + info := tk.Se.ShowProcess() + c.Assert(info, NotNil) + physicalPlan, ok := info.Plan.(core.PhysicalPlan) + c.Assert(ok, IsTrue) + normalized1, digest1 := core.NormalizePlan(physicalPlan) + + tk.Se.GetSessionVars().PlanID = 0 + tk.MustQuery(sql2) + info = tk.Se.ShowProcess() + c.Assert(info, NotNil) + physicalPlan, ok = info.Plan.(core.PhysicalPlan) + c.Assert(ok, IsTrue) + normalized2, digest2 := core.NormalizePlan(physicalPlan) + comment := Commentf("sql1: %v, sql2: %v\n%v !=\n%v\n", sql1, sql2, normalized1, normalized2) + if isSame { + c.Assert(normalized1, Equals, normalized2, comment) + c.Assert(digest1, Equals, digest2, comment) + } else { + c.Assert(normalized1 != normalized2, IsTrue, comment) + c.Assert(digest1 != digest2, IsTrue, comment) + } +} + +func getPlanRows(planStr string) []string { + planStr = strings.Replace(planStr, "\t", " ", -1) + return strings.Split(planStr, "\n") +} + +func compareStringSlice(c *C, ss1, ss2 []string) { + c.Assert(len(ss1), Equals, len(ss2)) + for i, s := range ss1 { + c.Assert(s, Equals, ss2[i]) + } +} diff --git a/planner/core/plan_to_pb.go b/planner/core/plan_to_pb.go index 3b07385f9584f..ed1b3448ddd01 100644 --- a/planner/core/plan_to_pb.go +++ b/planner/core/plan_to_pb.go @@ -18,6 +18,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/tablecodec" @@ -90,10 +91,9 @@ func (p *PhysicalLimit) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) { // ToPB implements PhysicalPlan ToPB interface. func (p *PhysicalTableScan) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) { - columns := p.Columns tsExec := &tipb.TableScan{ TableId: p.Table.ID, - Columns: model.ColumnsToProto(columns, p.Table.PKIsHandle), + Columns: model.ColumnsToProto(p.Columns, p.Table.PKIsHandle), Desc: p.Desc, } err := SetPBColumnsDefaultValue(ctx, tsExec.Columns, p.Columns) @@ -180,9 +180,12 @@ func SetPBColumnsDefaultValue(ctx sessionctx.Context, pbColumns []*tipb.ColumnIn // Some plans are difficult (if possible) to implement streaming, and some are pointless to do so. // TODO: Support more kinds of physical plan. func SupportStreaming(p PhysicalPlan) bool { - switch p.(type) { - case *PhysicalTableScan, *PhysicalIndexScan, *PhysicalSelection: + switch x := p.(type) { + case *PhysicalIndexScan, *PhysicalSelection: return true + case *PhysicalTableScan: + // TODO: remove this after TiDB coprocessor support stream. + return x.StoreType != kv.TiDB } return false } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index adbbae9195438..7e1bbc5bbffa4 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -605,9 +605,14 @@ func isPrimaryIndex(indexName model.CIStr) bool { return indexName.L == "primary" } -func (b *PlanBuilder) getPossibleAccessPaths(indexHints []*ast.IndexHint, tblInfo *model.TableInfo, dbName, tblName model.CIStr) ([]*accessPath, error) { +func (b *PlanBuilder) getPossibleAccessPaths(indexHints []*ast.IndexHint, tbl table.Table, dbName, tblName model.CIStr) ([]*accessPath, error) { + tblInfo := tbl.Meta() publicPaths := make([]*accessPath, 0, len(tblInfo.Indices)+2) - publicPaths = append(publicPaths, &accessPath{isTablePath: true, storeType: kv.TiKV}) + tp := kv.TiKV + if tbl.Type().IsClusterTable() { + tp = kv.TiDB + } + publicPaths = append(publicPaths, &accessPath{isTablePath: true, storeType: tp}) if tblInfo.TiFlashReplica != nil && tblInfo.TiFlashReplica.Available { publicPaths = append(publicPaths, &accessPath{isTablePath: true, storeType: kv.TiFlash}) } @@ -2528,7 +2533,7 @@ func (b *PlanBuilder) buildDDL(ctx context.Context, node ast.DDLNode) (Plan, err } b.visitInfo = appendVisitInfo(b.visitInfo, mysql.InsertPriv, v.NewTable.Schema.L, v.NewTable.Name.L, "", authErr) - case *ast.RecoverTableStmt: + case *ast.RecoverTableStmt, *ast.FlashBackTableStmt: // Recover table command can only be executed by administrator. b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SuperPriv, "", "", "", nil) case *ast.LockTablesStmt, *ast.UnlockTablesStmt: @@ -2536,6 +2541,9 @@ func (b *PlanBuilder) buildDDL(ctx context.Context, node ast.DDLNode) (Plan, err case *ast.CleanupTableLockStmt: // This command can only be executed by administrator. b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SuperPriv, "", "", "", nil) + case *ast.RepairTableStmt: + // Repair table command can only be executed by administrator. + b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SuperPriv, "", "", "", nil) } p := &DDL{Statement: node} return p, nil diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index 31d68a7cb11d9..2b5b0496a8c6b 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -82,6 +82,11 @@ func (p *PointGetPlan) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) { // ExplainInfo returns operator information to be explained. func (p *PointGetPlan) ExplainInfo() string { + return p.explainInfo(false) +} + +// ExplainInfo returns operator information to be explained. +func (p *PointGetPlan) explainInfo(normalized bool) string { buffer := bytes.NewBufferString("") tblName := p.TblInfo.Name.O fmt.Fprintf(buffer, "table:%s", tblName) @@ -94,10 +99,14 @@ func (p *PointGetPlan) ExplainInfo() string { } } } else { - if p.UnsignedHandle { - fmt.Fprintf(buffer, ", handle:%d", uint64(p.Handle)) + if normalized { + fmt.Fprintf(buffer, ", handle:?") } else { - fmt.Fprintf(buffer, ", handle:%d", p.Handle) + if p.UnsignedHandle { + fmt.Fprintf(buffer, ", handle:%d", uint64(p.Handle)) + } else { + fmt.Fprintf(buffer, ", handle:%d", p.Handle) + } } } if p.Lock { @@ -106,6 +115,11 @@ func (p *PointGetPlan) ExplainInfo() string { return buffer.String() } +// ExplainNormalizedInfo returns normalized operator information to be explained. +func (p *PointGetPlan) ExplainNormalizedInfo() string { + return p.explainInfo(true) +} + // GetChildReqProps gets the required property by child index. func (p *PointGetPlan) GetChildReqProps(idx int) *property.PhysicalProperty { return nil @@ -192,6 +206,11 @@ func (p *BatchPointGetPlan) ExplainInfo() string { return buffer.String() } +// ExplainNormalizedInfo returns normalized operator information to be explained. +func (p *BatchPointGetPlan) ExplainNormalizedInfo() string { + return p.ExplainInfo() +} + // GetChildReqProps gets the required property by child index. func (p *BatchPointGetPlan) GetChildReqProps(idx int) *property.PhysicalProperty { return nil diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index dc478fdc1ac1b..8c2b192c17ea5 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -30,6 +30,8 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/parser_driver" + "github.com/pingcap/tidb/util" + "github.com/pingcap/tidb/util/domainutil" ) // PreprocessOpt presents optional parameters to `Preprocess` method. @@ -66,6 +68,8 @@ const ( inCreateOrDropTable // parentIsJoin is set when visiting node's parent is join. parentIsJoin + // inRepairTable is set when visiting a repair table statement. + inRepairTable ) // preprocessor is an ast.Visitor that preprocess @@ -121,11 +125,15 @@ func (p *preprocessor) Enter(in ast.Node) (out ast.Node, skipChildren bool) { if node.HintedSel != nil { p.checkBindGrammar(node.OriginSel, node.HintedSel) } - case *ast.RecoverTableStmt: + case *ast.RecoverTableStmt, *ast.FlashBackTableStmt: // The specified table in recover table statement maybe already been dropped. // So skip check table name here, otherwise, recover table [table_name] syntax will return // table not exists error. But recover table statement is use to recover the dropped table. So skip children here. return in, true + case *ast.RepairTableStmt: + // The RepairTable should consist of the logic for creating tables and renaming tables. + p.flag |= inRepairTable + p.checkRepairTableGrammar(node) default: p.flag &= ^parentIsJoin } @@ -203,6 +211,8 @@ func (p *preprocessor) Leave(in ast.Node) (out ast.Node, ok bool) { x.Args[0] = ast.NewValueExpr(0) } } + case *ast.RepairTableStmt: + p.flag &= ^inRepairTable } return in, p.err == nil @@ -497,6 +507,10 @@ func (p *preprocessor) checkRenameTableGrammar(stmt *ast.RenameTableStmt) { oldTable := stmt.OldTable.Name.String() newTable := stmt.NewTable.Name.String() + p.checkRenameTable(oldTable, newTable) +} + +func (p *preprocessor) checkRenameTable(oldTable, newTable string) { if isIncorrectName(oldTable) { p.err = ddl.ErrWrongTableName.GenWithStackByArgs(oldTable) return @@ -508,6 +522,23 @@ func (p *preprocessor) checkRenameTableGrammar(stmt *ast.RenameTableStmt) { } } +func (p *preprocessor) checkRepairTableGrammar(stmt *ast.RepairTableStmt) { + // Check create table stmt whether it's is in REPAIR MODE. + if !domainutil.RepairInfo.InRepairMode() { + p.err = ddl.ErrRepairTableFail.GenWithStackByArgs("TiDB is not in REPAIR MODE") + return + } + if len(domainutil.RepairInfo.GetRepairTableList()) == 0 { + p.err = ddl.ErrRepairTableFail.GenWithStackByArgs("repair list is empty") + return + } + + // Check rename action as the rename statement does. + oldTable := stmt.Table.Name.String() + newTable := stmt.CreateStmt.Table.Name.String() + p.checkRenameTable(oldTable, newTable) +} + func (p *preprocessor) checkAlterTableGrammar(stmt *ast.AlterTableStmt) { tName := stmt.Table.Name.String() if isIncorrectName(tName) { @@ -715,9 +746,23 @@ func (p *preprocessor) handleTableName(tn *ast.TableName) { } if p.flag&inCreateOrDropTable > 0 { // The table may not exist in create table or drop table statement. - // Skip resolving the table to avoid error. + if p.flag&inRepairTable > 0 { + // Create stmt is in repair stmt, skip resolving the table to avoid error. + return + } + // Create stmt is not in repair stmt, check the table not in repair list. + if domainutil.RepairInfo.InRepairMode() { + p.checkNotInRepair(tn) + } + return + } + // repairStmt: admin repair table A create table B ... + // repairStmt's tableName is whether `inCreateOrDropTable` or `inRepairTable` flag. + if p.flag&inRepairTable > 0 { + p.handleRepairName(tn) return } + table, err := p.is.TableByName(tn.Schema, tn.Name) if err != nil { p.err = err @@ -728,6 +773,36 @@ func (p *preprocessor) handleTableName(tn *ast.TableName) { tn.DBInfo = dbInfo } +func (p *preprocessor) checkNotInRepair(tn *ast.TableName) { + tableInfo, dbInfo := domainutil.RepairInfo.GetRepairedTableInfoByTableName(tn.Schema.L, tn.Name.L) + if dbInfo == nil { + return + } + if tableInfo != nil { + p.err = ddl.ErrWrongTableName.GenWithStackByArgs(tn.Name.L, "this table is in repair") + } +} + +func (p *preprocessor) handleRepairName(tn *ast.TableName) { + // Check the whether the repaired table is system table. + if util.IsMemOrSysDB(tn.Schema.L) { + p.err = ddl.ErrRepairTableFail.GenWithStackByArgs("memory or system database is not for repair") + return + } + tableInfo, dbInfo := domainutil.RepairInfo.GetRepairedTableInfoByTableName(tn.Schema.L, tn.Name.L) + // tableName here only has the schema rather than DBInfo. + if dbInfo == nil { + p.err = ddl.ErrRepairTableFail.GenWithStackByArgs("database " + tn.Schema.L + " is not in repair") + return + } + if tableInfo == nil { + p.err = ddl.ErrRepairTableFail.GenWithStackByArgs("table " + tn.Name.L + " is not in repair") + return + } + p.ctx.SetValue(domainutil.RepairedTable, tableInfo) + p.ctx.SetValue(domainutil.RepairedDatabase, dbInfo) +} + func (p *preprocessor) resolveShowStmt(node *ast.ShowStmt) { if node.DBName == "" { if node.Table != nil && node.Table.Schema.L != "" { diff --git a/planner/core/rule_aggregation_push_down.go b/planner/core/rule_aggregation_push_down.go index f82de643831a2..278c83261e3be 100644 --- a/planner/core/rule_aggregation_push_down.go +++ b/planner/core/rule_aggregation_push_down.go @@ -354,7 +354,7 @@ func (a *aggregationPushDownSolver) aggPushDown(p LogicalPlan) (_ LogicalPlan, e } join.SetChildren(lChild, rChild) join.SetSchema(expression.MergeSchema(lChild.Schema(), rChild.Schema())) - join.BuildKeyInfo() + buildKeyInfo(join) proj := a.tryToEliminateAggregation(agg) if proj != nil { p = proj diff --git a/planner/core/rule_build_key_info.go b/planner/core/rule_build_key_info.go index 45a7c45d33224..2e2a43f39e2c2 100644 --- a/planner/core/rule_build_key_info.go +++ b/planner/core/rule_build_key_info.go @@ -15,8 +15,8 @@ package core import ( "context" - "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" ) @@ -24,33 +24,47 @@ import ( type buildKeySolver struct{} func (s *buildKeySolver) optimize(ctx context.Context, lp LogicalPlan) (LogicalPlan, error) { - lp.BuildKeyInfo() + buildKeyInfo(lp) return lp, nil } +// buildKeyInfo recursively calls LogicalPlan's BuildKeyInfo method. +func buildKeyInfo(lp LogicalPlan) { + for _, child := range lp.Children() { + buildKeyInfo(child) + } + childSchema := make([]*expression.Schema, len(lp.Children())) + for i, child := range lp.Children() { + childSchema[i] = child.Schema() + } + lp.BuildKeyInfo(lp.Schema(), childSchema) +} + // BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. -func (la *LogicalAggregation) BuildKeyInfo() { - la.schema.Keys = nil - la.baseLogicalPlan.BuildKeyInfo() - for _, key := range la.Children()[0].Schema().Keys { - indices := la.schema.ColumnsIndices(key) +func (la *LogicalAggregation) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + if la.IsPartialModeAgg() { + return + } + la.logicalSchemaProducer.BuildKeyInfo(selfSchema, childSchema) + for _, key := range childSchema[0].Keys { + indices := selfSchema.ColumnsIndices(key) if indices == nil { continue } newKey := make([]*expression.Column, 0, len(key)) for _, i := range indices { - newKey = append(newKey, la.schema.Columns[i]) + newKey = append(newKey, selfSchema.Columns[i]) } - la.schema.Keys = append(la.schema.Keys, newKey) + selfSchema.Keys = append(selfSchema.Keys, newKey) } if len(la.groupByCols) == len(la.GroupByItems) && len(la.GroupByItems) > 0 { - indices := la.schema.ColumnsIndices(la.groupByCols) + indices := selfSchema.ColumnsIndices(la.groupByCols) if indices != nil { newKey := make([]*expression.Column, 0, len(indices)) for _, i := range indices { - newKey = append(newKey, la.schema.Columns[i]) + newKey = append(newKey, selfSchema.Columns[i]) } - la.schema.Keys = append(la.schema.Keys, newKey) + selfSchema.Keys = append(selfSchema.Keys, newKey) } } if len(la.GroupByItems) == 0 { @@ -60,12 +74,12 @@ func (la *LogicalAggregation) BuildKeyInfo() { // If a condition is the form of (uniqueKey = constant) or (uniqueKey = Correlated column), it returns at most one row. // This function will check it. -func (p *LogicalSelection) checkMaxOneRowCond(unique expression.Expression, constOrCorCol expression.Expression) bool { +func (p *LogicalSelection) checkMaxOneRowCond(unique expression.Expression, constOrCorCol expression.Expression, childSchema *expression.Schema) bool { col, ok := unique.(*expression.Column) if !ok { return false } - if !p.children[0].Schema().IsUniqueKey(col) { + if !childSchema.IsUniqueKey(col) { return false } _, okCon := constOrCorCol.(*expression.Constant) @@ -77,11 +91,11 @@ func (p *LogicalSelection) checkMaxOneRowCond(unique expression.Expression, cons } // BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. -func (p *LogicalSelection) BuildKeyInfo() { - p.baseLogicalPlan.BuildKeyInfo() +func (p *LogicalSelection) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + p.baseLogicalPlan.BuildKeyInfo(selfSchema, childSchema) for _, cond := range p.Conditions { if sf, ok := cond.(*expression.ScalarFunction); ok && sf.FuncName.L == ast.EQ { - if p.checkMaxOneRowCond(sf.GetArgs()[0], sf.GetArgs()[1]) || p.checkMaxOneRowCond(sf.GetArgs()[1], sf.GetArgs()[0]) { + if p.checkMaxOneRowCond(sf.GetArgs()[0], sf.GetArgs()[1], childSchema[0]) || p.checkMaxOneRowCond(sf.GetArgs()[1], sf.GetArgs()[0], childSchema[0]) { p.maxOneRow = true break } @@ -90,8 +104,8 @@ func (p *LogicalSelection) BuildKeyInfo() { } // BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. -func (p *LogicalLimit) BuildKeyInfo() { - p.baseLogicalPlan.BuildKeyInfo() +func (p *LogicalLimit) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + p.baseLogicalPlan.BuildKeyInfo(selfSchema, childSchema) if p.Count == 1 { p.maxOneRow = true } @@ -99,8 +113,8 @@ func (p *LogicalLimit) BuildKeyInfo() { // A bijection exists between columns of a projection's schema and this projection's Exprs. // Sometimes we need a schema made by expr of Exprs to convert a column in child's schema to a column in this projection's Schema. -func (p *LogicalProjection) buildSchemaByExprs() *expression.Schema { - schema := expression.NewSchema(make([]*expression.Column, 0, p.schema.Len())...) +func (p *LogicalProjection) buildSchemaByExprs(selfSchema *expression.Schema) *expression.Schema { + schema := expression.NewSchema(make([]*expression.Column, 0, selfSchema.Len())...) for _, expr := range p.Exprs { if col, isCol := expr.(*expression.Column); isCol { schema.Append(col) @@ -116,31 +130,28 @@ func (p *LogicalProjection) buildSchemaByExprs() *expression.Schema { } // BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. -func (p *LogicalProjection) BuildKeyInfo() { - p.schema.Keys = nil - p.baseLogicalPlan.BuildKeyInfo() - schema := p.buildSchemaByExprs() - for _, key := range p.Children()[0].Schema().Keys { +func (p *LogicalProjection) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + p.logicalSchemaProducer.BuildKeyInfo(selfSchema, childSchema) + schema := p.buildSchemaByExprs(selfSchema) + for _, key := range childSchema[0].Keys { indices := schema.ColumnsIndices(key) if indices == nil { continue } newKey := make([]*expression.Column, 0, len(key)) for _, i := range indices { - newKey = append(newKey, p.schema.Columns[i]) + newKey = append(newKey, selfSchema.Columns[i]) } - p.schema.Keys = append(p.schema.Keys, newKey) + selfSchema.Keys = append(selfSchema.Keys, newKey) } } // BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. -func (p *LogicalJoin) BuildKeyInfo() { - p.schema.Keys = nil - p.baseLogicalPlan.BuildKeyInfo() - p.maxOneRow = p.children[0].MaxOneRow() && p.children[1].MaxOneRow() +func (p *LogicalJoin) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + p.logicalSchemaProducer.BuildKeyInfo(selfSchema, childSchema) switch p.JoinType { case SemiJoin, LeftOuterSemiJoin, AntiSemiJoin, AntiLeftOuterSemiJoin: - p.schema.Keys = p.children[0].Schema().Clone().Keys + selfSchema.Keys = childSchema[0].Clone().Keys case InnerJoin, LeftOuterJoin, RightOuterJoin: // If there is no equal conditions, then cartesian product can't be prevented and unique key information will destroy. if len(p.EqualConditions) == 0 { @@ -155,13 +166,13 @@ func (p *LogicalJoin) BuildKeyInfo() { for _, expr := range p.EqualConditions { ln := expr.GetArgs()[0].(*expression.Column) rn := expr.GetArgs()[1].(*expression.Column) - for _, key := range p.children[0].Schema().Keys { + for _, key := range childSchema[0].Keys { if len(key) == 1 && key[0].Equal(p.ctx, ln) { lOk = true break } } - for _, key := range p.children[1].Schema().Keys { + for _, key := range childSchema[1].Keys { if len(key) == 1 && key[0].Equal(p.ctx, rn) { rOk = true break @@ -172,61 +183,94 @@ func (p *LogicalJoin) BuildKeyInfo() { // another side's unique key information will all be reserved. // If it's an outer join, NULL value will fill some position, which will destroy the unique key information. if lOk && p.JoinType != LeftOuterJoin { - p.schema.Keys = append(p.schema.Keys, p.children[1].Schema().Keys...) + selfSchema.Keys = append(selfSchema.Keys, childSchema[1].Keys...) } if rOk && p.JoinType != RightOuterJoin { - p.schema.Keys = append(p.schema.Keys, p.children[0].Schema().Keys...) + selfSchema.Keys = append(selfSchema.Keys, childSchema[0].Keys...) } } } -// BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. -func (ds *DataSource) BuildKeyInfo() { - ds.schema.Keys = nil - ds.baseLogicalPlan.BuildKeyInfo() - for _, path := range ds.possibleAccessPaths { - if path.isTablePath { - continue - } - idx := path.index - if !idx.Unique { - continue - } - newKey := make([]*expression.Column, 0, len(idx.Columns)) - ok := true - for _, idxCol := range idx.Columns { - // The columns of this index should all occur in column schema. - // Since null value could be duplicate in unique key. So we check NotNull flag of every column. - find := false - for i, col := range ds.Columns { - if idxCol.Name.L == col.Name.L { - if !mysql.HasNotNullFlag(ds.Columns[i].Flag) { - break - } - newKey = append(newKey, ds.schema.Columns[i]) - find = true +// checkIndexCanBeKey checks whether an Index can be a Key in schema. +func checkIndexCanBeKey(idx *model.IndexInfo, columns []*model.ColumnInfo, schema *expression.Schema) expression.KeyInfo { + if !idx.Unique { + return nil + } + newKey := make([]*expression.Column, 0, len(idx.Columns)) + ok := true + for _, idxCol := range idx.Columns { + // The columns of this index should all occur in column schema. + // Since null value could be duplicate in unique key. So we check NotNull flag of every column. + find := false + for i, col := range columns { + if idxCol.Name.L == col.Name.L { + if !mysql.HasNotNullFlag(col.Flag) { break } - } - if !find { - ok = false + newKey = append(newKey, schema.Columns[i]) + find = true break } } - if ok { - ds.schema.Keys = append(ds.schema.Keys, newKey) + if !find { + ok = false + break + } + } + if ok { + return newKey + } + return nil +} + +// BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. +func (ds *DataSource) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + selfSchema.Keys = nil + for _, path := range ds.possibleAccessPaths { + if path.isTablePath { + continue + } + if newKey := checkIndexCanBeKey(path.index, ds.Columns, selfSchema); newKey != nil { + selfSchema.Keys = append(selfSchema.Keys, newKey) } } if ds.tableInfo.PKIsHandle { for i, col := range ds.Columns { if mysql.HasPriKeyFlag(col.Flag) { - ds.schema.Keys = append(ds.schema.Keys, []*expression.Column{ds.schema.Columns[i]}) + selfSchema.Keys = append(selfSchema.Keys, []*expression.Column{selfSchema.Columns[i]}) break } } } } +// BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. +func (ts *LogicalTableScan) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + ts.Source.BuildKeyInfo(selfSchema, childSchema) +} + +// BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. +func (is *LogicalIndexScan) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + selfSchema.Keys = nil + for _, path := range is.Source.possibleAccessPaths { + if path.isTablePath { + continue + } + if newKey := checkIndexCanBeKey(path.index, is.Columns, selfSchema); newKey != nil { + selfSchema.Keys = append(selfSchema.Keys, newKey) + } + } + handle := is.getPKIsHandleCol() + if handle != nil { + selfSchema.Keys = append(selfSchema.Keys, []*expression.Column{handle}) + } +} + +// BuildKeyInfo implements LogicalPlan BuildKeyInfo interface. +func (tg *TiKVSingleGather) BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) { + selfSchema.Keys = childSchema[0].Keys +} + func (*buildKeySolver) name() string { return "build_keys" } diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index d1c0753d2bbcb..c90ff5cdfe236 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -209,6 +209,8 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column) error { handleCol = ds.handleCol handleColInfo = ds.Columns[ds.schema.ColumnIndex(handleCol)] } + originSchemaColumns := ds.schema.Columns + originColumns := ds.Columns for i := len(used) - 1; i >= 0; i-- { if !used[i] { ds.schema.Columns = append(ds.schema.Columns[:i], ds.schema.Columns[i+1:]...) @@ -218,7 +220,11 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column) error { // For SQL like `select 1 from t`, tikv's response will be empty if no column is in schema. // So we'll force to push one if schema doesn't have any column. if ds.schema.Len() == 0 { - if handleCol == nil { + if ds.table.Type().IsClusterTable() && len(originColumns) > 0 { + // use the first line. + handleCol = originSchemaColumns[0] + handleColInfo = originColumns[0] + } else if handleCol == nil { handleCol = ds.newExtraHandleSchemaCol() handleColInfo = model.NewExtraHandleColInfo() } diff --git a/planner/core/rule_predicate_push_down.go b/planner/core/rule_predicate_push_down.go index d6380d4dc1a48..f0baf57bc4e1d 100644 --- a/planner/core/rule_predicate_push_down.go +++ b/planner/core/rule_predicate_push_down.go @@ -204,7 +204,7 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret p.RightJoinKeys = append(p.RightJoinKeys, eqCond.GetArgs()[1].(*expression.Column)) } p.mergeSchema() - p.BuildKeyInfo() + buildKeyInfo(p) return ret, p.self } @@ -578,6 +578,14 @@ func (p *LogicalWindow) PredicatePushDown(predicates []expression.Expression) ([ return canNotBePushed, p } +// PredicatePushDown implements LogicalPlan PredicatePushDown interface. +func (p *LogicalMemTable) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) { + if p.Extractor != nil { + predicates = p.Extractor.Extract(p.schema, p.names, predicates) + } + return predicates, p.self +} + func (*ppdSolver) name() string { return "predicate_push_down" } diff --git a/planner/core/stats.go b/planner/core/stats.go index 6cceed0abf0b4..0a6bf02c1b8a0 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/statistics" + "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/ranger" "go.uber.org/zap" @@ -136,7 +137,7 @@ func (ds *DataSource) getColumnNDV(colID int64) (ndv float64) { return ndv } -func (ds *DataSource) deriveStatsByFilter(conds expression.CNFExprs) { +func (ds *DataSource) deriveStatsByFilter(conds expression.CNFExprs) *property.StatsInfo { if ds.tableStats == nil { tableStats := &property.StatsInfo{ RowCount: float64(ds.statisticTable.Count), @@ -158,19 +159,20 @@ func (ds *DataSource) deriveStatsByFilter(conds expression.CNFExprs) { logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err)) selectivity = selectionFactor } - ds.stats = ds.tableStats.Scale(selectivity) + stats := ds.tableStats.Scale(selectivity) if ds.ctx.GetSessionVars().OptimizerSelectivityLevel >= 1 { - ds.stats.HistColl = ds.stats.HistColl.NewHistCollBySelectivity(ds.ctx.GetSessionVars().StmtCtx, nodes) + stats.HistColl = stats.HistColl.NewHistCollBySelectivity(ds.ctx.GetSessionVars().StmtCtx, nodes) } + return stats } // DeriveStats implement LogicalPlan DeriveStats interface. func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema *expression.Schema, childSchema []*expression.Schema) (*property.StatsInfo, error) { // PushDownNot here can convert query 'not (a != 1)' to 'a = 1'. for i, expr := range ds.pushedDownConds { - ds.pushedDownConds[i] = expression.PushDownNot(nil, expr) + ds.pushedDownConds[i] = expression.PushDownNot(ds.ctx, expr) } - ds.deriveStatsByFilter(ds.pushedDownConds) + ds.stats = ds.deriveStatsByFilter(ds.pushedDownConds) for _, path := range ds.possibleAccessPaths { if path.isTablePath { noIntervalRanges, err := ds.deriveTablePathStats(path, ds.pushedDownConds, false) @@ -233,15 +235,15 @@ func (ds *DataSource) generateAndPruneIndexMergePath() { ds.possibleAccessPaths = ds.possibleAccessPaths[regularPathCount:] } -// DeriveStats implement LogicalPlan DeriveStats interface. -func (ts *TableScan) DeriveStats(childStats []*property.StatsInfo, selfSchema *expression.Schema, childSchema []*expression.Schema) (_ *property.StatsInfo, err error) { +// DeriveStats implements LogicalPlan DeriveStats interface. +func (ts *LogicalTableScan) DeriveStats(childStats []*property.StatsInfo, selfSchema *expression.Schema, childSchema []*expression.Schema) (_ *property.StatsInfo, err error) { // PushDownNot here can convert query 'not (a != 1)' to 'a = 1'. for i, expr := range ts.AccessConds { // TODO The expressions may be shared by TableScan and several IndexScans, there would be redundant // `PushDownNot` function call in multiple `DeriveStats` then. - ts.AccessConds[i] = expression.PushDownNot(nil, expr) + ts.AccessConds[i] = expression.PushDownNot(ts.ctx, expr) } - ts.Source.deriveStatsByFilter(ts.AccessConds) + ts.stats = ts.Source.deriveStatsByFilter(ts.AccessConds) sc := ts.SCtx().GetSessionVars().StmtCtx // ts.Handle could be nil if PK is Handle, and PK column has been pruned. if ts.Handle != nil { @@ -258,7 +260,30 @@ func (ts *TableScan) DeriveStats(childStats []*property.StatsInfo, selfSchema *e if err != nil { return nil, err } - return ts.Source.stats, nil + return ts.stats, nil +} + +// DeriveStats implements LogicalPlan DeriveStats interface. +func (is *LogicalIndexScan) DeriveStats(childStats []*property.StatsInfo, selfSchema *expression.Schema, childSchema []*expression.Schema) (*property.StatsInfo, error) { + for i, expr := range is.AccessConds { + is.AccessConds[i] = expression.PushDownNot(is.ctx, expr) + } + is.stats = is.Source.deriveStatsByFilter(is.AccessConds) + if len(is.AccessConds) == 0 { + is.Ranges = ranger.FullRange() + } + // TODO: If the AccessConds is not empty, we have set the range when push down the selection. + + is.idxCols, is.idxColLens = expression.IndexInfo2PrefixCols(is.Columns, is.schema.Columns, is.Index) + is.fullIdxCols, is.fullIdxColLens = expression.IndexInfo2Cols(is.Columns, is.schema.Columns, is.Index) + if !is.Index.Unique && !is.Index.Primary && len(is.Index.Columns) == len(is.idxCols) { + handleCol := is.getPKIsHandleCol() + if handleCol != nil && !mysql.HasUnsignedFlag(handleCol.RetType.Flag) { + is.idxCols = append(is.idxCols, handleCol) + is.idxColLens = append(is.idxColLens, types.UnspecifiedLength) + } + } + return is.stats, nil } // getIndexMergeOrPath generates all possible IndexMergeOrPaths. diff --git a/planner/core/task.go b/planner/core/task.go index 6becdc2b2e500..b280a09640b13 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -19,12 +19,14 @@ import ( "github.com/pingcap/parser/ast" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/plancodec" ) @@ -140,6 +142,20 @@ func (t *copTask) finishIndexPlan() { t.cst += cnt * rowSize * sessVars.ScanFactor } +func (t *copTask) getStoreType() kv.StoreType { + if t.tablePlan == nil { + return kv.TiKV + } + tp := t.tablePlan + for len(tp.Children()) > 0 { + tp = tp.Children()[0] + } + if ts, ok := tp.(*PhysicalTableScan); ok { + return ts.StoreType + } + return kv.TiKV +} + func (p *basePhysicalPlan) attach2Task(tasks ...task) task { t := finishCopTask(p.ctx, tasks[0].copy()) return attachPlan2Task(p.self, t) @@ -390,17 +406,38 @@ func (p *PhysicalIndexJoin) GetCost(outerTask, innerTask task) float64 { return outerTask.cost() + innerPlanCost + cpuCost + memoryCost } +func (p *PhysicalHashJoin) avgRowSize(inner PhysicalPlan) (size float64) { + padChar := p.ctx.GetSessionVars().StmtCtx.PadCharToFullLength + if inner.statsInfo().HistColl != nil { + size = inner.statsInfo().HistColl.GetAvgRowSizeListInDisk(inner.Schema().Columns, padChar) + } else { + // Estimate using just the type info. + cols := inner.Schema().Columns + for _, col := range cols { + size += float64(chunk.EstimateTypeWidth(padChar, col.GetType())) + } + } + return +} + // GetCost computes cost of hash join operator itself. func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64) float64 { - innerCnt, outerCnt := lCnt, rCnt + buildCnt, probeCnt := lCnt, rCnt + build := p.children[0] // Taking the right as the inner for right join or using the outer to build a hash table. if (p.InnerChildIdx == 1 && !p.UseOuterToBuild) || (p.InnerChildIdx == 0 && p.UseOuterToBuild) { - innerCnt, outerCnt = rCnt, lCnt + buildCnt, probeCnt = rCnt, lCnt + build = p.children[1] } sessVars := p.ctx.GetSessionVars() + oomUseTmpStorage := config.GetGlobalConfig().OOMUseTmpStorage + memQuota := sessVars.StmtCtx.MemTracker.GetBytesLimit() // sessVars.MemQuotaQuery && hint + rowSize := p.avgRowSize(build) + spill := oomUseTmpStorage && memQuota > 0 && rowSize*buildCnt > float64(memQuota) // Cost of building hash table. - cpuCost := innerCnt * sessVars.CPUFactor - memoryCost := innerCnt * sessVars.MemoryFactor + cpuCost := buildCnt * sessVars.CPUFactor + memoryCost := buildCnt * sessVars.MemoryFactor + diskCost := buildCnt * sessVars.DiskFactor * rowSize // Number of matched row pairs regarding the equal join conditions. helper := &fullJoinRowCountHelper{ cartesian: false, @@ -428,23 +465,38 @@ func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64) float64 { numPairs = 0 } } - // Cost of quering hash table is cheap actually, so we just compute the cost of + // Cost of querying hash table is cheap actually, so we just compute the cost of // evaluating `OtherConditions` and joining row pairs. probeCost := numPairs * sessVars.CPUFactor + probeDiskCost := numPairs * sessVars.DiskFactor * rowSize // Cost of evaluating outer filter. if len(p.LeftConditions)+len(p.RightConditions) > 0 { // Input outer count for the above compution should be adjusted by selectionFactor. probeCost *= selectionFactor - probeCost += outerCnt * sessVars.CPUFactor + probeDiskCost *= selectionFactor + probeCost += probeCnt * sessVars.CPUFactor } + diskCost += probeDiskCost probeCost /= float64(p.Concurrency) // Cost of additional concurrent goroutines. cpuCost += probeCost + float64(p.Concurrency+1)*sessVars.ConcurrencyFactor // Cost of traveling the hash table to resolve missing matched cases when building the hash table from the outer table if p.UseOuterToBuild { - cpuCost += innerCnt * sessVars.CPUFactor / float64(p.Concurrency) + if spill { + // It runs in sequence when build data is on disk. See handleUnmatchedRowsFromHashTableInDisk + cpuCost += buildCnt * sessVars.CPUFactor + } else { + cpuCost += buildCnt * sessVars.CPUFactor / float64(p.Concurrency) + } + diskCost += buildCnt * sessVars.DiskFactor * rowSize } - return cpuCost + memoryCost + + if spill { + memoryCost *= float64(memQuota) / (rowSize * buildCnt) + } else { + diskCost = 0 + } + return cpuCost + memoryCost + diskCost } func (p *PhysicalHashJoin) attach2Task(tasks ...task) task { @@ -968,14 +1020,24 @@ func BuildFinalModeAggregation( return } -func (p *basePhysicalAgg) newPartialAggregate(copToFlash bool) (partial, final PhysicalPlan) { +func (p *basePhysicalAgg) newPartialAggregate(copTaskType kv.StoreType) (partial, final PhysicalPlan) { // Check if this aggregation can push down. - if !CheckAggCanPushCop(p.ctx, p.AggFuncs, p.GroupByItems, copToFlash) { + if !CheckAggCanPushCop(p.ctx, p.AggFuncs, p.GroupByItems, copTaskType == kv.TiFlash) { return nil, p.self } finalAggFuncs, finalGbyItems, partialSchema := BuildFinalModeAggregation(p.ctx, p.AggFuncs, p.GroupByItems, p.schema) // Remove unnecessary FirstRow. p.AggFuncs = RemoveUnnecessaryFirstRow(p.ctx, finalAggFuncs, finalGbyItems, p.AggFuncs, p.GroupByItems, partialSchema) + if copTaskType == kv.TiDB { + // For partial agg of TiDB cop task, since TiDB coprocessor reuse the TiDB executor, + // and TiDB aggregation executor won't output the group by value, + // so we need add `firstrow` aggregation function to output the group by value. + aggFuncs, err := genFirstRowAggForGroupBy(p.ctx, p.GroupByItems) + if err != nil { + return nil, p.self + } + p.AggFuncs = append(p.AggFuncs, aggFuncs...) + } finalSchema := p.schema p.schema = partialSchema partialAgg := p.self @@ -997,6 +1059,18 @@ func (p *basePhysicalAgg) newPartialAggregate(copToFlash bool) (partial, final P return partialAgg, finalAgg } +func genFirstRowAggForGroupBy(ctx sessionctx.Context, groupByItems []expression.Expression) ([]*aggregation.AggFuncDesc, error) { + aggFuncs := make([]*aggregation.AggFuncDesc, 0, len(groupByItems)) + for _, groupBy := range groupByItems { + agg, err := aggregation.NewAggFuncDesc(ctx, ast.AggFuncFirstRow, []expression.Expression{groupBy}, false) + if err != nil { + return nil, err + } + aggFuncs = append(aggFuncs, agg) + } + return aggFuncs, nil +} + // RemoveUnnecessaryFirstRow removes unnecessary FirstRow of the aggregation. This function can be // used for both LogicalAggregation and PhysicalAggregation. // When the select column is same with the group by key, the column can be removed and gets value from the group by key. @@ -1047,8 +1121,8 @@ func (p *PhysicalStreamAgg) attach2Task(tasks ...task) task { // The `extraHandleCol` is added if the double read needs to keep order. So we just use it to decided // whether the following plan is double read with order reserved. if cop.extraHandleCol == nil { - copToFlash := isFlashCopTask(cop) - partialAgg, finalAgg := p.newPartialAggregate(copToFlash) + copTaskType := cop.getStoreType() + partialAgg, finalAgg := p.newPartialAggregate(copTaskType) if partialAgg != nil { if cop.tablePlan != nil { cop.finishIndexPlan() @@ -1108,27 +1182,12 @@ func (p *PhysicalHashAgg) cpuCostDivisor(hasDistinct bool) (float64, float64) { return math.Min(float64(finalCon), float64(partialCon)), float64(finalCon + partialCon) } -func isFlashCopTask(cop *copTask) bool { - if cop.tablePlan == nil { - return false - } - tp := cop.tablePlan - for len(tp.Children()) > 0 { - tp = tp.Children()[0] - } - if ts, ok := tp.(*PhysicalTableScan); ok { - return ts.StoreType == kv.TiFlash - } - return false -} - func (p *PhysicalHashAgg) attach2Task(tasks ...task) task { t := tasks[0].copy() inputRows := t.count() if cop, ok := t.(*copTask); ok { - // copToFlash means whether the cop task is running on flash storage - copToFlash := isFlashCopTask(cop) - partialAgg, finalAgg := p.newPartialAggregate(copToFlash) + copTaskType := cop.getStoreType() + partialAgg, finalAgg := p.newPartialAggregate(copTaskType) if partialAgg != nil { if cop.tablePlan != nil { cop.finishIndexPlan() diff --git a/planner/core/testdata/plan_normalized_suite_in.json b/planner/core/testdata/plan_normalized_suite_in.json new file mode 100644 index 0000000000000..3753c984adb00 --- /dev/null +++ b/planner/core/testdata/plan_normalized_suite_in.json @@ -0,0 +1,27 @@ +[ + { + "name": "TestNormalizedPlan", + "cases": [ + "select * from t1;", + "select * from t1 where a<1;", + "select * from t1 where a>1", + "select * from t1 where a=1", + "select * from t1 where a in (1,2,3)", + "select * from t1 where b=1", + "select a+1,b+2 from t1 use index(b) where b=3", + "select * from t1 where t1.b > 1 and t1.a in (select sum(t2.b) from t2 where t2.a=t1.a and t2.b is not null)", + "SELECT * from t1 where a!=1 order by c limit 1", + "SELECT /*+ TIDB_SMJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + "SELECT /*+ TIDB_INLJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + "SELECT /*+ TIDB_HJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + "SELECT /*+ TIDB_HJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + "SELECT /*+ TIDB_INLJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + "insert into t1 values (1,1,1)", + "insert into t1 select * from t2 where t2.a>0 and t2.b!=0", + "update t1 set a=a+1", + "update t1 set a=a+1 where a>0", + "delete from t1", + "delete from t1 where a>0 and b=1 and c!=2" + ] + } +] diff --git a/planner/core/testdata/plan_normalized_suite_out.json b/planner/core/testdata/plan_normalized_suite_out.json new file mode 100644 index 0000000000000..23ba1d16b6863 --- /dev/null +++ b/planner/core/testdata/plan_normalized_suite_out.json @@ -0,0 +1,180 @@ +[ + { + "Name": "TestNormalizedPlan", + "Cases": [ + { + "SQL": "select * from t1;", + "Plan": [ + " TableReader_5 root data:TableScan_4", + " └─TableScan_4 cop table:t1, range:[?,?], keep order:false" + ] + }, + { + "SQL": "select * from t1 where a<1;", + "Plan": [ + " TableReader_6 root data:TableScan_5", + " └─TableScan_5 cop table:t1, range:[?,?], keep order:false" + ] + }, + { + "SQL": "select * from t1 where a>1", + "Plan": [ + " TableReader_6 root data:TableScan_5", + " └─TableScan_5 cop table:t1, range:[?,?], keep order:false" + ] + }, + { + "SQL": "select * from t1 where a=1", + "Plan": [ + " Point_Get_1 root table:t1, handle:?" + ] + }, + { + "SQL": "select * from t1 where a in (1,2,3)", + "Plan": [ + " Batch_Point_Get_1 root table:t1" + ] + }, + { + "SQL": "select * from t1 where b=1", + "Plan": [ + " IndexLookUp_10 root ", + " ├─IndexScan_8 cop table:t1, index:b, range:[?,?], keep order:false", + " └─TableScan_9 cop table:t1, keep order:false" + ] + }, + { + "SQL": "select a+1,b+2 from t1 use index(b) where b=3", + "Plan": [ + " Projection_4 root plus(test.t1.a, ?), plus(test.t1.b, ?)", + " └─IndexReader_6 root index:IndexScan_5", + " └─IndexScan_5 cop table:t1, index:b, range:[?,?], keep order:false" + ] + }, + { + "SQL": "select * from t1 where t1.b > 1 and t1.a in (select sum(t2.b) from t2 where t2.a=t1.a and t2.b is not null)", + "Plan": [ + " Projection_10 root test.t1.a, test.t1.b, test.t1.c", + " └─Apply_12 root semi join, inner:StreamAgg_34, equal:eq(Column#8, Column#7)", + " ├─Projection_13 root cast(test.t1.a), test.t1.a, test.t1.b, test.t1.c", + " │ └─TableReader_16 root data:Selection_15", + " │ └─Selection_15 cop gt(test.t1.b, ?)", + " │ └─TableScan_14 cop table:t1, range:[?,?], keep order:false", + " └─StreamAgg_34 root funcs:sum(Column#11)->Column#7", + " └─TableReader_35 root data:StreamAgg_23", + " └─StreamAgg_23 cop funcs:sum(test.t2.b)->Column#11", + " └─Selection_33 cop not(isnull(test.t2.b))", + " └─TableScan_32 cop table:t2, range: decided by eq(test.t2.a, test.t1.a), keep order:false" + ] + }, + { + "SQL": "SELECT * from t1 where a!=1 order by c limit 1", + "Plan": [ + " TopN_8 root test.t1.c:asc", + " └─TableReader_16 root data:TopN_15", + " └─TopN_15 cop test.t1.c:asc", + " └─TableScan_14 cop table:t1, range:[?,?], keep order:false" + ] + }, + { + "SQL": "SELECT /*+ TIDB_SMJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + "Plan": [ + " MergeJoin_7 root inner join, left key:test.t1.a, right key:test.t2.a", + " ├─TableReader_11 root data:Selection_10", + " │ └─Selection_10 cop gt(test.t1.c, ?)", + " │ └─TableScan_9 cop table:t1, range:[?,?], keep order:true", + " └─TableReader_13 root data:TableScan_12", + " └─TableScan_12 cop table:t2, range:[?,?], keep order:true" + ] + }, + { + "SQL": "SELECT /*+ TIDB_INLJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + "Plan": [ + " IndexJoin_10 root inner join, inner:TableReader_9, outer key:test.t1.a, inner key:test.t2.a", + " ├─TableReader_19 root data:Selection_18", + " │ └─Selection_18 cop gt(test.t1.c, ?)", + " │ └─TableScan_17 cop table:t1, range:[?,?], keep order:false", + " └─TableReader_9 root data:TableScan_8", + " └─TableScan_8 cop table:t2, range: decided by [test.t1.a], keep order:false" + ] + }, + { + "SQL": "SELECT /*+ TIDB_HJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + "Plan": [ + " HashRightJoin_18 root inner join, inner:TableReader_21, equal:eq(test.t1.a, test.t2.a)", + " ├─TableReader_21 root data:Selection_20", + " │ └─Selection_20 cop gt(test.t1.c, ?)", + " │ └─TableScan_19 cop table:t1, range:[?,?], keep order:false", + " └─TableReader_23 root data:TableScan_22", + " └─TableScan_22 cop table:t2, range:[?,?], keep order:false" + ] + }, + { + "SQL": "SELECT /*+ TIDB_HJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + "Plan": [ + " HashRightJoin_18 root inner join, inner:TableReader_21, equal:eq(test.t1.a, test.t2.a)", + " ├─TableReader_21 root data:Selection_20", + " │ └─Selection_20 cop gt(test.t1.c, ?)", + " │ └─TableScan_19 cop table:t1, range:[?,?], keep order:false", + " └─TableReader_23 root data:TableScan_22", + " └─TableScan_22 cop table:t2, range:[?,?], keep order:false" + ] + }, + { + "SQL": "SELECT /*+ TIDB_INLJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", + "Plan": [ + " IndexJoin_10 root inner join, inner:TableReader_9, outer key:test.t1.a, inner key:test.t2.a", + " ├─TableReader_19 root data:Selection_18", + " │ └─Selection_18 cop gt(test.t1.c, ?)", + " │ └─TableScan_17 cop table:t1, range:[?,?], keep order:false", + " └─TableReader_9 root data:TableScan_8", + " └─TableScan_8 cop table:t2, range: decided by [test.t1.a], keep order:false" + ] + }, + { + "SQL": "insert into t1 values (1,1,1)", + "Plan": [ + "" + ] + }, + { + "SQL": "insert into t1 select * from t2 where t2.a>0 and t2.b!=0", + "Plan": [ + " TableReader_9 root data:Selection_8", + " └─Selection_8 cop ne(test.t2.b, ?)", + " └─TableScan_7 cop table:t2, range:[?,?], keep order:false" + ] + }, + { + "SQL": "update t1 set a=a+1", + "Plan": [ + " TableReader_6 root data:TableScan_5", + " └─TableScan_5 cop table:t1, range:[?,?], keep order:false" + ] + }, + { + "SQL": "update t1 set a=a+1 where a>0", + "Plan": [ + " TableReader_7 root data:TableScan_6", + " └─TableScan_6 cop table:t1, range:[?,?], keep order:false" + ] + }, + { + "SQL": "delete from t1", + "Plan": [ + " TableReader_6 root data:TableScan_5", + " └─TableScan_5 cop table:t1, range:[?,?], keep order:false" + ] + }, + { + "SQL": "delete from t1 where a>0 and b=1 and c!=2", + "Plan": [ + " IndexLookUp_12 root ", + " ├─IndexScan_9 cop table:t1, index:b, range:[?,?], keep order:false", + " └─Selection_11 cop ne(test.t1.c, ?)", + " └─TableScan_10 cop table:t1, keep order:false" + ] + } + ] + } +] diff --git a/planner/implementation/datasource.go b/planner/implementation/datasource.go index a5fea2af32409..783d78d3fcbc6 100644 --- a/planner/implementation/datasource.go +++ b/planner/implementation/datasource.go @@ -74,7 +74,7 @@ func (impl *TableReaderImpl) ScaleCostLimit(costLimit float64) float64 { sessVars := reader.SCtx().GetSessionVars() copIterWorkers := float64(sessVars.DistSQLScanConcurrency) if math.MaxFloat64/copIterWorkers < costLimit { - return costLimit + return math.MaxFloat64 } return costLimit * copIterWorkers } @@ -108,3 +108,66 @@ func (impl *TableScanImpl) CalcCost(outCount float64, children ...memo.Implement } return impl.cost } + +// IndexReaderImpl is the implementation of PhysicalIndexReader. +type IndexReaderImpl struct { + baseImpl + tblColHists *statistics.HistColl +} + +// ScaleCostLimit implements Implementation interface. +func (impl *IndexReaderImpl) ScaleCostLimit(costLimit float64) float64 { + reader := impl.plan.(*plannercore.PhysicalIndexReader) + sessVars := reader.SCtx().GetSessionVars() + copIterWorkers := float64(sessVars.DistSQLScanConcurrency) + if math.MaxFloat64/copIterWorkers < costLimit { + return math.MaxFloat64 + } + return costLimit * copIterWorkers +} + +// CalcCost implements Implementation interface. +func (impl *IndexReaderImpl) CalcCost(outCount float64, children ...memo.Implementation) float64 { + reader := impl.plan.(*plannercore.PhysicalIndexReader) + sessVars := reader.SCtx().GetSessionVars() + networkCost := outCount * sessVars.NetworkFactor * impl.tblColHists.GetAvgRowSize(children[0].GetPlan().Schema().Columns, true) + copIterWorkers := float64(sessVars.DistSQLScanConcurrency) + impl.cost = (networkCost + children[0].GetCost()) / copIterWorkers + return impl.cost +} + +// NewIndexReaderImpl creates a new IndexReader Implementation. +func NewIndexReaderImpl(reader *plannercore.PhysicalIndexReader, tblColHists *statistics.HistColl) *IndexReaderImpl { + return &IndexReaderImpl{ + baseImpl: baseImpl{plan: reader}, + tblColHists: tblColHists, + } +} + +// IndexScanImpl is the Implementation of PhysicalIndexScan. +type IndexScanImpl struct { + baseImpl + tblColHists *statistics.HistColl +} + +// CalcCost implements Implementation interface. +func (impl *IndexScanImpl) CalcCost(outCount float64, children ...memo.Implementation) float64 { + is := impl.plan.(*plannercore.PhysicalIndexScan) + sessVars := is.SCtx().GetSessionVars() + rowSize := impl.tblColHists.GetIndexAvgRowSize(is.Schema().Columns, is.Index.Unique) + cost := outCount * rowSize * sessVars.ScanFactor + if is.Desc { + cost = outCount * rowSize * sessVars.DescScanFactor + } + cost += float64(len(is.Ranges)) * sessVars.SeekFactor + impl.cost = cost + return impl.cost +} + +// NewIndexScanImpl creates a new IndexScan Implementation. +func NewIndexScanImpl(scan *plannercore.PhysicalIndexScan, tblColHists *statistics.HistColl) *IndexScanImpl { + return &IndexScanImpl{ + baseImpl: baseImpl{plan: scan}, + tblColHists: tblColHists, + } +} diff --git a/planner/memo/expr_iterator_test.go b/planner/memo/expr_iterator_test.go index fceaa75f4580a..baf7f39fdaec8 100644 --- a/planner/memo/expr_iterator_test.go +++ b/planner/memo/expr_iterator_test.go @@ -19,12 +19,12 @@ import ( ) func (s *testMemoSuite) TestNewExprIterFromGroupElem(c *C) { - g0 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), nil) + g0 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), s.schema) g0.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g0.Insert(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0))) g0.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) - g1 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), nil) + g1 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), s.schema) g1.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g1.Insert(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0))) g1.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) @@ -32,7 +32,7 @@ func (s *testMemoSuite) TestNewExprIterFromGroupElem(c *C) { expr := NewGroupExpr(plannercore.LogicalJoin{}.Init(s.sctx, 0)) expr.Children = append(expr.Children, g0) expr.Children = append(expr.Children, g1) - g2 := NewGroupWithSchema(expr, nil) + g2 := NewGroupWithSchema(expr, s.schema) pattern := BuildPattern(OperandJoin, EngineAll, BuildPattern(OperandProjection, EngineAll), BuildPattern(OperandSelection, EngineAll)) iter := NewExprIterFromGroupElem(g2.Equivalents.Front(), pattern) @@ -58,13 +58,13 @@ func (s *testMemoSuite) TestNewExprIterFromGroupElem(c *C) { } func (s *testMemoSuite) TestExprIterNext(c *C) { - g0 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0)), nil) + g0 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0)), s.schema) g0.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g0.Insert(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0))) g0.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g0.Insert(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0))) - g1 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), nil) + g1 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), s.schema) g1.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g1.Insert(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0))) g1.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) @@ -73,7 +73,7 @@ func (s *testMemoSuite) TestExprIterNext(c *C) { expr := NewGroupExpr(plannercore.LogicalJoin{}.Init(s.sctx, 0)) expr.Children = append(expr.Children, g0) expr.Children = append(expr.Children, g1) - g2 := NewGroupWithSchema(expr, nil) + g2 := NewGroupWithSchema(expr, s.schema) pattern := BuildPattern(OperandJoin, EngineAll, BuildPattern(OperandProjection, EngineAll), BuildPattern(OperandSelection, EngineAll)) iter := NewExprIterFromGroupElem(g2.Equivalents.Front(), pattern) @@ -102,7 +102,7 @@ func (s *testMemoSuite) TestExprIterNext(c *C) { } func (s *testMemoSuite) TestExprIterReset(c *C) { - g0 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0)), nil) + g0 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0)), s.schema) g0.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g0.Insert(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0))) g0.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) @@ -111,13 +111,13 @@ func (s *testMemoSuite) TestExprIterReset(c *C) { sel1 := NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)) sel2 := NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)) sel3 := NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)) - g1 := NewGroupWithSchema(sel1, nil) + g1 := NewGroupWithSchema(sel1, s.schema) g1.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g1.Insert(sel2) g1.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g1.Insert(sel3) - g2 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), nil) + g2 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), s.schema) g2.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g2.Insert(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0))) g2.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) @@ -127,7 +127,7 @@ func (s *testMemoSuite) TestExprIterReset(c *C) { expr := NewGroupExpr(plannercore.LogicalJoin{}.Init(s.sctx, 0)) expr.Children = append(expr.Children, g0) expr.Children = append(expr.Children, g1) - g3 := NewGroupWithSchema(expr, nil) + g3 := NewGroupWithSchema(expr, s.schema) // link sel 1~3 with Group 2 sel1.Children = append(sel1.Children, g2) @@ -185,34 +185,34 @@ func countMatchedIter(group *Group, pattern *Pattern) int { } func (s *testMemoSuite) TestExprIterWithEngineType(c *C) { - g1 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), nil).SetEngineType(EngineTiFlash) + g1 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), s.schema).SetEngineType(EngineTiFlash) g1.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g1.Insert(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0))) g1.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) - g2 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), nil).SetEngineType(EngineTiKV) + g2 := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)), s.schema).SetEngineType(EngineTiKV) g2.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) g2.Insert(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0))) g2.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))) - flashGather := NewGroupExpr(plannercore.TableGather{}.Init(s.sctx, 0)) + flashGather := NewGroupExpr(plannercore.TiKVSingleGather{}.Init(s.sctx, 0)) flashGather.Children = append(flashGather.Children, g1) - g3 := NewGroupWithSchema(flashGather, nil).SetEngineType(EngineTiDB) + g3 := NewGroupWithSchema(flashGather, s.schema).SetEngineType(EngineTiDB) - tikvGather := NewGroupExpr(plannercore.TableGather{}.Init(s.sctx, 0)) + tikvGather := NewGroupExpr(plannercore.TiKVSingleGather{}.Init(s.sctx, 0)) tikvGather.Children = append(tikvGather.Children, g2) g3.Insert(tikvGather) join := NewGroupExpr(plannercore.LogicalJoin{}.Init(s.sctx, 0)) join.Children = append(join.Children, g3, g3) - g4 := NewGroupWithSchema(join, nil).SetEngineType(EngineTiDB) + g4 := NewGroupWithSchema(join, s.schema).SetEngineType(EngineTiDB) // The Groups look like this: // Group 4 // Join input:[Group3, Group3] // Group 3 - // TableGather input:[Group2] EngineTiKV - // TableGather input:[Group1] EngineTiFlash + // TiKVSingleGather input:[Group2] EngineTiKV + // TiKVSingleGather input:[Group1] EngineTiFlash // Group 2 // Selection // Projection @@ -224,36 +224,36 @@ func (s *testMemoSuite) TestExprIterWithEngineType(c *C) { // Limit // Limit - p0 := BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOnly)) + p0 := BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOnly)) c.Assert(countMatchedIter(g3, p0), Equals, 2) - p1 := BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiFlashOnly)) + p1 := BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiFlashOnly)) c.Assert(countMatchedIter(g3, p1), Equals, 2) - p2 := BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOrTiFlash)) + p2 := BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOrTiFlash)) c.Assert(countMatchedIter(g3, p2), Equals, 4) - p3 := BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandSelection, EngineTiFlashOnly)) + p3 := BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandSelection, EngineTiFlashOnly)) c.Assert(countMatchedIter(g3, p3), Equals, 1) - p4 := BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandProjection, EngineTiKVOnly)) + p4 := BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandProjection, EngineTiKVOnly)) c.Assert(countMatchedIter(g3, p4), Equals, 1) p5 := BuildPattern( OperandJoin, EngineTiDBOnly, - BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOnly)), - BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOnly)), + BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOnly)), + BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOnly)), ) c.Assert(countMatchedIter(g4, p5), Equals, 4) p6 := BuildPattern( OperandJoin, EngineTiDBOnly, - BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiFlashOnly)), - BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOnly)), + BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiFlashOnly)), + BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOnly)), ) c.Assert(countMatchedIter(g4, p6), Equals, 4) p7 := BuildPattern( OperandJoin, EngineTiDBOnly, - BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOrTiFlash)), - BuildPattern(OperandTableGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOrTiFlash)), + BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOrTiFlash)), + BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly, BuildPattern(OperandLimit, EngineTiKVOrTiFlash)), ) c.Assert(countMatchedIter(g4, p7), Equals, 16) @@ -263,8 +263,8 @@ func (s *testMemoSuite) TestExprIterWithEngineType(c *C) { p8 := BuildPattern( OperandJoin, EngineTiDBOnly, - BuildPattern(OperandTableGather, EngineTiDBOnly), - BuildPattern(OperandTableGather, EngineTiDBOnly), + BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly), + BuildPattern(OperandTiKVSingleGather, EngineTiDBOnly), ) c.Assert(countMatchedIter(g4, p8), Equals, 4) } diff --git a/planner/memo/group.go b/planner/memo/group.go index 036254fd538f4..da73641dc38c0 100644 --- a/planner/memo/group.go +++ b/planner/memo/group.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/pingcap/tidb/expression" + plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/planner/property" ) @@ -85,11 +86,17 @@ type Group struct { Prop *property.LogicalProperty EngineType EngineType + + //hasBuiltKeyInfo indicates whether this group has called `BuildKeyInfo`. + // BuildKeyInfo is lazily called when a rule needs information of + // unique key or maxOneRow (in LogicalProp). For each Group, we only need + // to collect these information once. + hasBuiltKeyInfo bool } // NewGroupWithSchema creates a new Group with given schema. func NewGroupWithSchema(e *GroupExpr, s *expression.Schema) *Group { - prop := &property.LogicalProperty{Schema: s} + prop := &property.LogicalProperty{Schema: expression.NewSchema(s.Columns...)} g := &Group{ Equivalents: list.New(), Fingerprints: make(map[string]*list.Element), @@ -197,3 +204,46 @@ func (g *Group) InsertImpl(prop *property.PhysicalProperty, impl Implementation) key := prop.HashCode() g.ImplMap[string(key)] = impl } + +// Convert2GroupExpr converts a logical plan to a GroupExpr. +func Convert2GroupExpr(node plannercore.LogicalPlan) *GroupExpr { + e := NewGroupExpr(node) + e.Children = make([]*Group, 0, len(node.Children())) + for _, child := range node.Children() { + childGroup := Convert2Group(child) + e.Children = append(e.Children, childGroup) + } + return e +} + +// Convert2Group converts a logical plan to a Group. +func Convert2Group(node plannercore.LogicalPlan) *Group { + e := Convert2GroupExpr(node) + g := NewGroupWithSchema(e, node.Schema()) + // Stats property for `Group` would be computed after exploration phase. + return g +} + +// BuildKeyInfo recursively builds UniqueKey and MaxOneRow info in the LogicalProperty. +func (g *Group) BuildKeyInfo() { + if g.hasBuiltKeyInfo { + return + } + g.hasBuiltKeyInfo = true + + e := g.Equivalents.Front().Value.(*GroupExpr) + childSchema := make([]*expression.Schema, len(e.Children)) + childMaxOneRow := make([]bool, len(e.Children)) + for i := range e.Children { + e.Children[i].BuildKeyInfo() + childSchema[i] = e.Children[i].Prop.Schema + childMaxOneRow[i] = e.Children[i].Prop.MaxOneRow + } + if len(childSchema) == 1 { + // For UnaryPlan(such as Selection, Limit ...), we can set the child's unique key as its unique key. + // If the GroupExpr is a schemaProducer, schema.Keys will be reset below in `BuildKeyInfo()`. + g.Prop.Schema.Keys = childSchema[0].Keys + } + e.ExprNode.BuildKeyInfo(g.Prop.Schema, childSchema) + g.Prop.MaxOneRow = e.ExprNode.MaxOneRow() || plannercore.HasMaxOneRow(e.ExprNode, childMaxOneRow) +} diff --git a/planner/memo/group_test.go b/planner/memo/group_test.go index e77ea3e22844e..fb8ff8a60c274 100644 --- a/planner/memo/group_test.go +++ b/planner/memo/group_test.go @@ -14,6 +14,7 @@ package memo import ( + "context" "testing" . "github.com/pingcap/check" @@ -36,8 +37,9 @@ var _ = Suite(&testMemoSuite{}) type testMemoSuite struct { *parser.Parser - is infoschema.InfoSchema - sctx sessionctx.Context + is infoschema.InfoSchema + schema *expression.Schema + sctx sessionctx.Context } func (s *testMemoSuite) SetUpSuite(c *C) { @@ -45,6 +47,7 @@ func (s *testMemoSuite) SetUpSuite(c *C) { s.is = infoschema.MockInfoSchema([]*model.TableInfo{plannercore.MockSignedTable()}) s.sctx = plannercore.MockContext() s.Parser = parser.New() + s.schema = expression.NewSchema() } func (s *testMemoSuite) TearDownSuite(c *C) { @@ -54,7 +57,7 @@ func (s *testMemoSuite) TearDownSuite(c *C) { func (s *testMemoSuite) TestNewGroup(c *C) { p := &plannercore.LogicalLimit{} expr := NewGroupExpr(p) - g := NewGroupWithSchema(expr, nil) + g := NewGroupWithSchema(expr, s.schema) c.Assert(g.Equivalents.Len(), Equals, 1) c.Assert(g.Equivalents.Front().Value.(*GroupExpr), Equals, expr) @@ -65,7 +68,7 @@ func (s *testMemoSuite) TestNewGroup(c *C) { func (s *testMemoSuite) TestGroupInsert(c *C) { p := &plannercore.LogicalLimit{} expr := NewGroupExpr(p) - g := NewGroupWithSchema(expr, nil) + g := NewGroupWithSchema(expr, s.schema) c.Assert(g.Insert(expr), IsFalse) expr.selfFingerprint = "1" c.Assert(g.Insert(expr), IsTrue) @@ -74,7 +77,7 @@ func (s *testMemoSuite) TestGroupInsert(c *C) { func (s *testMemoSuite) TestGroupDelete(c *C) { p := &plannercore.LogicalLimit{} expr := NewGroupExpr(p) - g := NewGroupWithSchema(expr, nil) + g := NewGroupWithSchema(expr, s.schema) c.Assert(g.Equivalents.Len(), Equals, 1) g.Delete(expr) @@ -86,7 +89,7 @@ func (s *testMemoSuite) TestGroupDelete(c *C) { func (s *testMemoSuite) TestGroupDeleteAll(c *C) { expr := NewGroupExpr(plannercore.LogicalSelection{}.Init(s.sctx, 0)) - g := NewGroupWithSchema(expr, nil) + g := NewGroupWithSchema(expr, s.schema) c.Assert(g.Insert(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0))), IsTrue) c.Assert(g.Insert(NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0))), IsTrue) c.Assert(g.Equivalents.Len(), Equals, 3) @@ -102,7 +105,7 @@ func (s *testMemoSuite) TestGroupDeleteAll(c *C) { func (s *testMemoSuite) TestGroupExists(c *C) { p := &plannercore.LogicalLimit{} expr := NewGroupExpr(p) - g := NewGroupWithSchema(expr, nil) + g := NewGroupWithSchema(expr, s.schema) c.Assert(g.Exists(expr), IsTrue) g.Delete(expr) @@ -116,7 +119,7 @@ func (s *testMemoSuite) TestGroupGetFirstElem(c *C) { expr3 := NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0)) expr4 := NewGroupExpr(plannercore.LogicalProjection{}.Init(s.sctx, 0)) - g := NewGroupWithSchema(expr0, nil) + g := NewGroupWithSchema(expr0, s.schema) g.Insert(expr1) g.Insert(expr2) g.Insert(expr3) @@ -139,7 +142,7 @@ func (impl *fakeImpl) GetPlan() plannercore.PhysicalPlan { return func (impl *fakeImpl) AttachChildren(...Implementation) Implementation { return nil } func (impl *fakeImpl) ScaleCostLimit(float64) float64 { return 0 } func (s *testMemoSuite) TestGetInsertGroupImpl(c *C) { - g := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0)), nil) + g := NewGroupWithSchema(NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0)), s.schema) emptyProp := &property.PhysicalProperty{} orderProp := &property.PhysicalProperty{Items: []property.Item{{Col: &expression.Column{}}}} @@ -180,7 +183,7 @@ func (s *testMemoSuite) TestEngineTypeSet(c *C) { func (s *testMemoSuite) TestFirstElemAfterDelete(c *C) { oldExpr := NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0)) - g := NewGroupWithSchema(oldExpr, nil) + g := NewGroupWithSchema(oldExpr, s.schema) newExpr := NewGroupExpr(plannercore.LogicalLimit{}.Init(s.sctx, 0)) g.Insert(newExpr) c.Assert(g.GetFirstElem(OperandLimit), NotNil) @@ -191,3 +194,45 @@ func (s *testMemoSuite) TestFirstElemAfterDelete(c *C) { g.Delete(newExpr) c.Assert(g.GetFirstElem(OperandLimit), IsNil) } + +func (s *testMemoSuite) TestBuildKeyInfo(c *C) { + // case 1: primary key has constant constraint + stmt1, err := s.ParseOneStmt("select a from t where a = 10", "", "") + c.Assert(err, IsNil) + p1, _, err := plannercore.BuildLogicalPlan(context.Background(), s.sctx, stmt1, s.is) + c.Assert(err, IsNil) + logic1, ok := p1.(plannercore.LogicalPlan) + c.Assert(ok, IsTrue) + group1 := Convert2Group(logic1) + group1.BuildKeyInfo() + c.Assert(group1.Prop.MaxOneRow, IsTrue) + c.Assert(len(group1.Prop.Schema.Keys), Equals, 1) + + // case 2: group by column is key + stmt2, err := s.ParseOneStmt("select b, sum(a) from t group by b", "", "") + c.Assert(err, IsNil) + p2, _, err := plannercore.BuildLogicalPlan(context.Background(), s.sctx, stmt2, s.is) + c.Assert(err, IsNil) + logic2, ok := p2.(plannercore.LogicalPlan) + c.Assert(ok, IsTrue) + group2 := Convert2Group(logic2) + group2.BuildKeyInfo() + c.Assert(group2.Prop.MaxOneRow, IsFalse) + c.Assert(len(group2.Prop.Schema.Keys), Equals, 1) + + // case 3: build key info for new Group + newSel := plannercore.LogicalSelection{}.Init(s.sctx, 0) + newExpr1 := NewGroupExpr(newSel) + newExpr1.SetChildren(group2) + newGroup1 := NewGroupWithSchema(newExpr1, group2.Prop.Schema) + newGroup1.BuildKeyInfo() + c.Assert(len(newGroup1.Prop.Schema.Keys), Equals, 1) + + // case 4: build maxOneRow for new Group + newLimit := plannercore.LogicalLimit{Count: 1}.Init(s.sctx, 0) + newExpr2 := NewGroupExpr(newLimit) + newExpr2.SetChildren(group2) + newGroup2 := NewGroupWithSchema(newExpr2, group2.Prop.Schema) + newGroup2.BuildKeyInfo() + c.Assert(newGroup2.Prop.MaxOneRow, IsTrue) +} diff --git a/planner/memo/implementation.go b/planner/memo/implementation.go index 56980c67bf64f..dd5a430f181b5 100644 --- a/planner/memo/implementation.go +++ b/planner/memo/implementation.go @@ -28,7 +28,7 @@ type Implementation interface { AttachChildren(children ...Implementation) Implementation // ScaleCostLimit scales costLimit by the Implementation's concurrency factor. - // Implementation like TableGather may divide the cost by its scan concurrency, + // Implementation like TiKVSingleGather may divide the cost by its scan concurrency, // so when we pass the costLimit for pruning the search space, we have to scale // the costLimit by its concurrency factor. ScaleCostLimit(costLimit float64) float64 diff --git a/planner/memo/pattern.go b/planner/memo/pattern.go index fff3529bd4fba..28f94ffc41cf5 100644 --- a/planner/memo/pattern.go +++ b/planner/memo/pattern.go @@ -55,10 +55,12 @@ const ( OperandLock // OperandLimit is the operand for LogicalLimit. OperandLimit - // OperandTableGather is the operand for TableGather. - OperandTableGather + // OperandTiKVSingleGather is the operand for TiKVSingleGather. + OperandTiKVSingleGather // OperandTableScan is the operand for TableScan. OperandTableScan + // OperandIndexScan is the operand for IndexScan. + OperandIndexScan // OperandShow is the operand for Show. OperandShow // OperandUnsupported is the operand for unsupported operators. @@ -96,10 +98,12 @@ func GetOperand(p plannercore.LogicalPlan) Operand { return OperandLock case *plannercore.LogicalLimit: return OperandLimit - case *plannercore.TableGather: - return OperandTableGather - case *plannercore.TableScan: + case *plannercore.TiKVSingleGather: + return OperandTiKVSingleGather + case *plannercore.LogicalTableScan: return OperandTableScan + case *plannercore.LogicalIndexScan: + return OperandIndexScan case *plannercore.LogicalShow: return OperandShow default: diff --git a/planner/property/logical_property.go b/planner/property/logical_property.go index a770e83755312..2a45e55c73c23 100644 --- a/planner/property/logical_property.go +++ b/planner/property/logical_property.go @@ -21,6 +21,7 @@ import ( // or statistics of columns in schema for output of Group. // All group expressions in a group share same logical property. type LogicalProperty struct { - Stats *StatsInfo - Schema *expression.Schema + Stats *StatsInfo + Schema *expression.Schema + MaxOneRow bool } diff --git a/privilege/privileges/errors.go b/privilege/privileges/errors.go index f4cdca47e41a8..55fbc0a3d3842 100644 --- a/privilege/privileges/errors.go +++ b/privilege/privileges/errors.go @@ -20,19 +20,17 @@ import ( // privilege error codes. const ( - codeInvalidPrivilegeType terror.ErrCode = 1 - codeInvalidUserNameFormat = 2 - codeErrNonexistingGrant = mysql.ErrNonexistingGrant + codeInvalidPrivilegeType terror.ErrCode = 1 ) var ( errInvalidPrivilegeType = terror.ClassPrivilege.New(codeInvalidPrivilegeType, "unknown privilege type %s") - errNonexistingGrant = terror.ClassPrivilege.New(codeErrNonexistingGrant, mysql.MySQLErrName[mysql.ErrNonexistingGrant]) + errNonexistingGrant = terror.ClassPrivilege.New(mysql.ErrNonexistingGrant, mysql.MySQLErrName[mysql.ErrNonexistingGrant]) ) func init() { privilegeMySQLErrCodes := map[terror.ErrCode]uint16{ - codeErrNonexistingGrant: mysql.ErrNonexistingGrant, + mysql.ErrNonexistingGrant: mysql.ErrNonexistingGrant, } terror.ErrClassToMySQLCodes[terror.ClassPrivilege] = privilegeMySQLErrCodes } diff --git a/server/http_handler.go b/server/http_handler.go index df44009cb1022..809ef31d791aa 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -77,8 +77,12 @@ const ( ) // For query string -const qTableID = "table_id" -const qLimit = "limit" +const ( + qTableID = "table_id" + qLimit = "limit" + qOperation = "op" + qSeconds = "seconds" +) const ( headerContentType = "Content-Type" @@ -669,7 +673,27 @@ func (h configReloadHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) // ServeHTTP recovers binlog service. func (h binlogRecover) ServeHTTP(w http.ResponseWriter, req *http.Request) { - binloginfo.DisableSkipBinlogFlag() + op := req.FormValue(qOperation) + switch op { + case "reset": + binloginfo.ResetSkippedCommitterCounter() + case "nowait": + binloginfo.DisableSkipBinlogFlag() + case "status": + default: + sec, err := strconv.ParseInt(req.FormValue(qSeconds), 10, 64) + if sec <= 0 || err != nil { + sec = 1800 + } + binloginfo.DisableSkipBinlogFlag() + timeout := time.Duration(sec) * time.Second + err = binloginfo.WaitBinlogRecover(timeout) + if err != nil { + writeError(w, err) + return + } + } + writeData(w, binloginfo.GetBinlogStatus()) } type tableFlashReplicaInfo struct { diff --git a/server/http_handler_test.go b/server/http_handler_test.go index 5ec65ab54ca6b..0b0ff73439511 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -40,6 +40,7 @@ import ( "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/binloginfo" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/helper" @@ -237,6 +238,73 @@ func (ts *HTTPHandlerTestSuite) TestGetRegionByIDWithError(c *C) { defer resp.Body.Close() } +func (ts *HTTPHandlerTestSuite) TestBinlogRecover(c *C) { + ts.startServer(c) + defer ts.stopServer(c) + binloginfo.EnableSkipBinlogFlag() + c.Assert(binloginfo.IsBinlogSkipped(), Equals, true) + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:10090/binlog/recover")) + c.Assert(err, IsNil) + defer resp.Body.Close() + c.Assert(resp.StatusCode, Equals, http.StatusOK) + c.Assert(binloginfo.IsBinlogSkipped(), Equals, false) + + // Invalid operation will use the default operation. + binloginfo.EnableSkipBinlogFlag() + c.Assert(binloginfo.IsBinlogSkipped(), Equals, true) + resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:10090/binlog/recover?op=abc")) + c.Assert(err, IsNil) + defer resp.Body.Close() + c.Assert(resp.StatusCode, Equals, http.StatusOK) + c.Assert(binloginfo.IsBinlogSkipped(), Equals, false) + + binloginfo.EnableSkipBinlogFlag() + c.Assert(binloginfo.IsBinlogSkipped(), Equals, true) + resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:10090/binlog/recover?op=abc&seconds=1")) + c.Assert(err, IsNil) + defer resp.Body.Close() + c.Assert(resp.StatusCode, Equals, http.StatusOK) + c.Assert(binloginfo.IsBinlogSkipped(), Equals, false) + + binloginfo.EnableSkipBinlogFlag() + c.Assert(binloginfo.IsBinlogSkipped(), Equals, true) + binloginfo.AddOneSkippedCommitter() + resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:10090/binlog/recover?op=abc&seconds=1")) + c.Assert(err, IsNil) + defer resp.Body.Close() + c.Assert(resp.StatusCode, Equals, http.StatusBadRequest) + c.Assert(binloginfo.IsBinlogSkipped(), Equals, false) + binloginfo.RemoveOneSkippedCommitter() + + binloginfo.AddOneSkippedCommitter() + c.Assert(binloginfo.SkippedCommitterCount(), Equals, int32(1)) + resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:10090/binlog/recover?op=reset")) + c.Assert(err, IsNil) + defer resp.Body.Close() + c.Assert(resp.StatusCode, Equals, http.StatusOK) + c.Assert(binloginfo.SkippedCommitterCount(), Equals, int32(0)) + + binloginfo.EnableSkipBinlogFlag() + resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:10090/binlog/recover?op=nowait")) + c.Assert(err, IsNil) + defer resp.Body.Close() + c.Assert(resp.StatusCode, Equals, http.StatusOK) + c.Assert(binloginfo.IsBinlogSkipped(), Equals, false) + + // Only the first should work. + binloginfo.EnableSkipBinlogFlag() + resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:10090/binlog/recover?op=nowait&op=reset")) + c.Assert(err, IsNil) + defer resp.Body.Close() + c.Assert(resp.StatusCode, Equals, http.StatusOK) + c.Assert(binloginfo.IsBinlogSkipped(), Equals, false) + + resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:10090/binlog/recover?op=status")) + c.Assert(err, IsNil) + defer resp.Body.Close() + c.Assert(resp.StatusCode, Equals, http.StatusOK) +} + func (ts *HTTPHandlerTestSuite) TestRegionsFromMeta(c *C) { ts.startServer(c) defer ts.stopServer(c) diff --git a/server/http_status.go b/server/http_status.go index d2dd4832531e2..90db784cbab2d 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -280,7 +280,7 @@ func (s *Server) setupStatuServerAndRPCServer(addr string, serverMux *http.Serve grpcL := m.Match(cmux.Any()) s.statusServer = &http.Server{Addr: addr, Handler: CorsHandler{handler: serverMux, cfg: s.cfg}} - s.grpcServer = NewRPCServer(s.cfg.Security) + s.grpcServer = NewRPCServer(s.cfg.Security, s.dom, s) go util.WithRecovery(func() { err := s.grpcServer.Serve(grpcL) diff --git a/server/rpc_server.go b/server/rpc_server.go index e195be0cb35a5..8ce8a8f126090 100644 --- a/server/rpc_server.go +++ b/server/rpc_server.go @@ -14,17 +14,29 @@ package server import ( + "context" + "fmt" + + "github.com/pingcap/kvproto/pkg/coprocessor" "github.com/pingcap/kvproto/pkg/diagnosticspb" + "github.com/pingcap/kvproto/pkg/tikvpb" "github.com/pingcap/sysutil" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/executor" + "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/store/mockstore/mocktikv" + "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tidb/util/memory" + "github.com/pingcap/tidb/util/stringutil" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) // NewRPCServer creates a new rpc server. -func NewRPCServer(security config.Security) *grpc.Server { +func NewRPCServer(security config.Security, dom *domain.Domain, sm util.SessionManager) *grpc.Server { defer func() { if v := recover(); v != nil { logutil.BgLogger().Error("panic in TiDB RPC server", zap.Any("stack", v)) @@ -41,6 +53,69 @@ func NewRPCServer(security config.Security) *grpc.Server { if s == nil { s = grpc.NewServer() } - diagnosticspb.RegisterDiagnosticsServer(s, &sysutil.DiagnoseServer{}) + rpcSrv := &rpcServer{ + dom: dom, + sm: sm, + } + // For redirection the cop task. + mocktikv.TiDBRPCServerCoprocessorHandler = rpcSrv.handleCopRequest + diagnosticspb.RegisterDiagnosticsServer(s, rpcSrv) + tikvpb.RegisterTikvServer(s, rpcSrv) return s } + +// rpcServer contains below 2 services: +// 1. Diagnose service, it's used for SQL diagnose. +// 2. Coprocessor service, it reuse the TikvServer interface, but only support the Coprocessor interface now. +// Coprocessor service will handle the cop task from other TiDB server. Currently, it's only use for read the cluster memory table. +type rpcServer struct { + sysutil.DiagnoseServer + tikvpb.TikvServer + dom *domain.Domain + sm util.SessionManager +} + +// Coprocessor implements the TiKVServer interface. +func (s *rpcServer) Coprocessor(ctx context.Context, in *coprocessor.Request) (resp *coprocessor.Response, err error) { + resp = &coprocessor.Response{} + defer func() { + if v := recover(); v != nil { + logutil.BgLogger().Error("panic in TiDB RPC server coprocessor", zap.Any("stack", v)) + resp.OtherError = fmt.Sprintf("rpc coprocessor panic, :%v", v) + } + }() + resp = s.handleCopRequest(ctx, in) + return resp, nil +} + +// handleCopRequest handles the cop dag request. +func (s *rpcServer) handleCopRequest(ctx context.Context, req *coprocessor.Request) *coprocessor.Response { + resp := &coprocessor.Response{} + se, err := s.createSession() + if err != nil { + resp.OtherError = err.Error() + return resp + } + defer se.Close() + + h := executor.NewCoprocessorDAGHandler(se) + return h.HandleRequest(ctx, req) +} + +func (s *rpcServer) createSession() (session.Session, error) { + se, err := session.CreateSessionWithDomain(s.dom.Store(), s.dom) + if err != nil { + return nil, err + } + do := domain.GetDomain(se) + is := do.InfoSchema() + // TODO: Need user and host to do privilege check. + se.GetSessionVars().TxnCtx.InfoSchema = is + // This is for disable parallel hash agg. + // TODO: remove this. + se.GetSessionVars().HashAggPartialConcurrency = 1 + se.GetSessionVars().HashAggFinalConcurrency = 1 + se.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(stringutil.StringerStr("coprocessor"), -1) + se.SetSessionManager(s.sm) + return se, nil +} diff --git a/server/server.go b/server/server.go index b83d7b945f43e..74f09b2bdb020 100644 --- a/server/server.go +++ b/server/server.go @@ -52,6 +52,7 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/plugin" "github.com/pingcap/tidb/sessionctx/variable" @@ -87,8 +88,8 @@ var ( errUnknownFieldType = terror.ClassServer.New(codeUnknownFieldType, "unknown field type") errInvalidSequence = terror.ClassServer.New(codeInvalidSequence, "invalid sequence") errInvalidType = terror.ClassServer.New(codeInvalidType, "invalid type") - errNotAllowedCommand = terror.ClassServer.New(codeNotAllowedCommand, "the used command is not allowed with this TiDB version") - errAccessDenied = terror.ClassServer.New(codeAccessDenied, mysql.MySQLErrName[mysql.ErrAccessDenied]) + errNotAllowedCommand = terror.ClassServer.New(mysql.ErrNotAllowedCommand, mysql.MySQLErrName[mysql.ErrNotAllowedCommand]) + errAccessDenied = terror.ClassServer.New(mysql.ErrAccessDenied, mysql.MySQLErrName[mysql.ErrAccessDenied]) ) // DefaultCapability is the capability of the server when it is created using the default configuration. @@ -110,6 +111,7 @@ type Server struct { concurrentLimiter *TokenLimiter clients map[uint32]*clientConn capability uint32 + dom *domain.Domain // stopListenerCh is used when a critical error occurred, we don't want to exit the process, because there may be // a supervisor automatically restart it, then new client connection will be created, but we can't server it. @@ -139,6 +141,11 @@ func (s *Server) releaseToken(token *Token) { s.concurrentLimiter.Put(token) } +// SetDomain use to set the server domain. +func (s *Server) SetDomain(dom *domain.Domain) { + s.dom = dom +} + // newConn creates a new *clientConn from a net.Conn. // It allocates a connection ID and random salt data for authentication. func (s *Server) newConn(conn net.Conn) *clientConn { @@ -617,21 +624,15 @@ func (s *Server) kickIdleConnection() { // Server error codes. const ( - codeUnknownFieldType = 1 - codeInvalidPayloadLen = 2 - codeInvalidSequence = 3 - codeInvalidType = 4 - - codeNotAllowedCommand = 1148 - codeAccessDenied = mysql.ErrAccessDenied - codeMaxExecTimeExceeded = mysql.ErrMaxExecTimeExceeded + codeUnknownFieldType = 1 + codeInvalidSequence = 3 + codeInvalidType = 4 ) func init() { serverMySQLErrCodes := map[terror.ErrCode]uint16{ - codeNotAllowedCommand: mysql.ErrNotAllowedCommand, - codeAccessDenied: mysql.ErrAccessDenied, - codeMaxExecTimeExceeded: mysql.ErrMaxExecTimeExceeded, + mysql.ErrNotAllowedCommand: mysql.ErrNotAllowedCommand, + mysql.ErrAccessDenied: mysql.ErrAccessDenied, } terror.ErrClassToMySQLCodes[terror.ClassServer] = serverMySQLErrCodes } diff --git a/server/server_test.go b/server/server_test.go index f0bcc9a95e680..e4dcdc039c3cc 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -808,7 +808,7 @@ func runTestErrorCode(c *C) { // Optimizer errors _, err = txn2.Exec("select *, * from test;") - checkErrorCode(c, err, tmysql.ErrParse) + checkErrorCode(c, err, tmysql.ErrInvalidWildCard) _, err = txn2.Exec("select row(1, 2) > 1;") checkErrorCode(c, err, tmysql.ErrOperandColumns) _, err = txn2.Exec("select * from test order by row(c, c);") diff --git a/session/bootstrap.go b/session/bootstrap.go index 04e8e439a0b70..fa5314d97e537 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -351,6 +351,7 @@ const ( version34 = 34 version35 = 35 version36 = 36 + version37 = 37 ) func checkBootstrapped(s Session) (bool, error) { @@ -555,6 +556,10 @@ func upgrade(s Session) { upgradeToVer36(s) } + if ver < version37 { + upgradeToVer37(s) + } + updateBootstrapVer(s) _, err = s.Execute(context.Background(), "COMMIT") @@ -877,6 +882,13 @@ func upgradeToVer36(s Session) { mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Shutdown_priv='Y' where User = 'root'") } +func upgradeToVer37(s Session) { + // when upgrade from old tidb and no 'tidb_enable_window_function' in GLOBAL_VARIABLES, init it with 0. + sql := fmt.Sprintf("INSERT IGNORE INTO %s.%s (`VARIABLE_NAME`, `VARIABLE_VALUE`) VALUES ('%s', '%d')", + mysql.SystemDB, mysql.GlobalVariablesTable, variable.TiDBEnableWindowFunction, 0) + mustExecute(s, sql) +} + // updateBootstrapVer updates bootstrap version variable in mysql.TiDB table. func updateBootstrapVer(s Session) { // Update bootstrap version. diff --git a/session/pessimistic_test.go b/session/pessimistic_test.go index f9648887abc9d..dd8dd3c7e5725 100644 --- a/session/pessimistic_test.go +++ b/session/pessimistic_test.go @@ -48,7 +48,7 @@ type testPessimisticSuite struct { func (s *testPessimisticSuite) SetUpSuite(c *C) { testleak.BeforeTest() // Set it to 300ms for testing lock resolve. - tikv.PessimisticLockTTL = 300 + tikv.ManagedLockTTL = 300 tikv.PrewriteMaxBackoff = 500 s.cluster = mocktikv.NewCluster() mocktikv.BootstrapWithSingleStore(s.cluster) diff --git a/session/session.go b/session/session.go index 4c144fe260864..424481349e89a 100644 --- a/session/session.go +++ b/session/session.go @@ -596,7 +596,7 @@ func (s *session) String() string { const sqlLogMaxLen = 1024 // SchemaChangedWithoutRetry is used for testing. -var SchemaChangedWithoutRetry bool +var SchemaChangedWithoutRetry uint32 func (s *session) getSQLLabel() string { if s.sessionVars.InRestrictedSQL { @@ -610,7 +610,7 @@ func (s *session) isInternal() bool { } func (s *session) isTxnRetryableError(err error) bool { - if SchemaChangedWithoutRetry { + if atomic.LoadUint32(&SchemaChangedWithoutRetry) == 1 { return kv.IsTxnRetryableError(err) } return kv.IsTxnRetryableError(err) || domain.ErrInfoSchemaChanged.Equal(err) @@ -662,9 +662,6 @@ func (s *session) retry(ctx context.Context, maxCnt uint) (err error) { for i, sr := range nh.history { st := sr.st s.sessionVars.StmtCtx = sr.stmtCtx - s.sessionVars.StartTime = time.Now() - s.sessionVars.DurationCompile = time.Duration(0) - s.sessionVars.DurationParse = time.Duration(0) s.sessionVars.StmtCtx.ResetForRetry() s.sessionVars.PreparedParams = s.sessionVars.PreparedParams[:0] schemaVersion, err = st.RebuildPlan(ctx) @@ -863,7 +860,7 @@ func createSessionFunc(store kv.Storage) pools.Factory { func createSessionWithDomainFunc(store kv.Storage) func(*domain.Domain) (pools.Resource, error) { return func(dom *domain.Domain) (pools.Resource, error) { - se, err := createSessionWithDomain(store, dom) + se, err := CreateSessionWithDomain(store, dom) if err != nil { return nil, err } @@ -1083,8 +1080,7 @@ func (s *session) execute(ctx context.Context, sql string) (recordSets []sqlexec charsetInfo, collation := s.sessionVars.GetCharsetInfo() // Step1: Compile query string to abstract syntax trees(ASTs). - startTS := time.Now() - s.GetSessionVars().StartTime = startTS + parseStartTime := time.Now() stmtNodes, warns, err := s.ParseSQL(ctx, sql, charsetInfo, collation) if err != nil { s.rollbackOnError(ctx) @@ -1093,7 +1089,7 @@ func (s *session) execute(ctx context.Context, sql string) (recordSets []sqlexec zap.String("SQL", sql)) return nil, util.SyntaxError(err) } - durParse := time.Since(startTS) + durParse := time.Since(parseStartTime) s.GetSessionVars().DurationParse = durParse isInternal := s.isInternal() if isInternal { @@ -1105,10 +1101,10 @@ func (s *session) execute(ctx context.Context, sql string) (recordSets []sqlexec compiler := executor.Compiler{Ctx: s} multiQuery := len(stmtNodes) > 1 for _, stmtNode := range stmtNodes { + s.sessionVars.StartTime = time.Now() s.PrepareTxnCtx(ctx) // Step2: Transform abstract syntax tree to a physical plan(stored in executor.ExecStmt). - startTS = time.Now() // Some executions are done in compile stage, so we reset them before compile. if err := executor.ResetContextOfStmt(s, stmtNode); err != nil { return nil, err @@ -1121,7 +1117,7 @@ func (s *session) execute(ctx context.Context, sql string) (recordSets []sqlexec zap.String("SQL", sql)) return nil, err } - durCompile := time.Since(startTS) + durCompile := time.Since(s.sessionVars.StartTime) s.GetSessionVars().DurationCompile = durCompile if isInternal { sessionExecuteCompileDurationInternal.Observe(durCompile.Seconds()) @@ -1710,11 +1706,11 @@ func createSession(store kv.Storage) (*session, error) { return s, nil } -// createSessionWithDomain creates a new Session and binds it with a Domain. +// CreateSessionWithDomain creates a new Session and binds it with a Domain. // We need this because when we start DDL in Domain, the DDL need a session // to change some system tables. But at that time, we have been already in // a lock context, which cause we can't call createSesion directly. -func createSessionWithDomain(store kv.Storage, dom *domain.Domain) (*session, error) { +func CreateSessionWithDomain(store kv.Storage, dom *domain.Domain) (*session, error) { s := &session{ store: store, parser: parser.New(), @@ -1736,7 +1732,7 @@ func createSessionWithDomain(store kv.Storage, dom *domain.Domain) (*session, er const ( notBootstrapped = 0 - currentBootstrapVersion = version36 + currentBootstrapVersion = version37 ) func getStoreBootstrapVersion(store kv.Storage) int64 { @@ -1836,6 +1832,7 @@ var builtinGlobalVariable = []string{ variable.TiDBOptScanFactor, variable.TiDBOptDescScanFactor, variable.TiDBOptMemoryFactor, + variable.TiDBOptDiskFactor, variable.TiDBOptConcurrencyFactor, variable.TiDBDistSQLScanConcurrency, variable.TiDBInitChunkSize, diff --git a/session/session_test.go b/session/session_test.go index 48df212964ba8..bcac4686392ff 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -587,41 +587,6 @@ func (s *testSessionSuite) TestReadOnlyNotInHistory(c *C) { c.Assert(history.Count(), Equals, 0) } -func (s *testSessionSuite) TestNoHistoryWhenDisableRetry(c *C) { - tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("create table history (a int)") - tk.MustExec("set @@autocommit = 0") - - // retry_limit = 0 will not add history. - tk.MustExec("set @@tidb_retry_limit = 0") - tk.MustExec("insert history values (1)") - c.Assert(session.GetHistory(tk.Se).Count(), Equals, 0) - - // Disable auto_retry will add history for auto committed only - tk.MustExec("set @@autocommit = 1") - tk.MustExec("set @@tidb_retry_limit = 10") - tk.MustExec("set @@tidb_disable_txn_auto_retry = 1") - c.Assert(failpoint.Enable("github.com/pingcap/tidb/session/keepHistory", `1*return(true)->return(false)`), IsNil) - tk.MustExec("insert history values (1)") - c.Assert(session.GetHistory(tk.Se).Count(), Equals, 1) - c.Assert(failpoint.Disable("github.com/pingcap/tidb/session/keepHistory"), IsNil) - tk.MustExec("begin") - tk.MustExec("insert history values (1)") - c.Assert(session.GetHistory(tk.Se).Count(), Equals, 0) - tk.MustExec("commit") - - // Enable auto_retry will add history for both. - tk.MustExec("set @@tidb_disable_txn_auto_retry = 0") - c.Assert(failpoint.Enable("github.com/pingcap/tidb/session/keepHistory", `1*return(true)->return(false)`), IsNil) - tk.MustExec("insert history values (1)") - c.Assert(failpoint.Disable("github.com/pingcap/tidb/session/keepHistory"), IsNil) - c.Assert(session.GetHistory(tk.Se).Count(), Equals, 1) - tk.MustExec("begin") - tk.MustExec("insert history values (1)") - c.Assert(session.GetHistory(tk.Se).Count(), Equals, 2) - tk.MustExec("commit") -} - func (s *testSessionSuite) TestNoRetryForCurrentTxn(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk1 := testkit.NewTestKitWithInit(c, s.store) @@ -1901,9 +1866,9 @@ func (s *testSchemaSerialSuite) TestSchemaCheckerSQL(c *C) { tk.MustExec(`commit;`) // The schema version is out of date in the first transaction, and the SQL can't be retried. - session.SchemaChangedWithoutRetry = true + atomic.StoreUint32(&session.SchemaChangedWithoutRetry, 1) defer func() { - session.SchemaChangedWithoutRetry = false + atomic.StoreUint32(&session.SchemaChangedWithoutRetry, 0) }() tk.MustExec(`begin;`) tk1.MustExec(`alter table t modify column c bigint;`) @@ -2857,7 +2822,7 @@ func (s *testSessionSuite2) TestIsolationRead(c *C) { tk := testkit.NewTestKit(c, s.store) tk.Se, err = session.CreateSession4Test(s.store) c.Assert(err, IsNil) - c.Assert(len(tk.Se.GetSessionVars().GetIsolationReadEngines()), Equals, 2) + c.Assert(len(tk.Se.GetSessionVars().GetIsolationReadEngines()), Equals, 3) tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash';") engines := tk.Se.GetSessionVars().GetIsolationReadEngines() c.Assert(len(engines), Equals, 1) diff --git a/session/tidb.go b/session/tidb.go index e50098d669330..a556bfbcbdb2a 100644 --- a/session/tidb.go +++ b/session/tidb.go @@ -367,17 +367,12 @@ func ResultSetToStringSlice(ctx context.Context, s Session, rs sqlexec.RecordSet // Session errors. var ( - ErrForUpdateCantRetry = terror.ClassSession.New(codeForUpdateCantRetry, - mysql.MySQLErrName[mysql.ErrForUpdateCantRetry]) -) - -const ( - codeForUpdateCantRetry terror.ErrCode = mysql.ErrForUpdateCantRetry + ErrForUpdateCantRetry = terror.ClassSession.New(mysql.ErrForUpdateCantRetry, mysql.MySQLErrName[mysql.ErrForUpdateCantRetry]) ) func init() { sessionMySQLErrCodes := map[terror.ErrCode]uint16{ - codeForUpdateCantRetry: mysql.ErrForUpdateCantRetry, + mysql.ErrForUpdateCantRetry: mysql.ErrForUpdateCantRetry, } terror.ErrClassToMySQLCodes[terror.ClassSession] = sessionMySQLErrCodes } diff --git a/session/txn.go b/session/txn.go index 903e9f38dc855..45e89744ca3f1 100755 --- a/session/txn.go +++ b/session/txn.go @@ -292,7 +292,15 @@ func (st *TxnState) IterReverse(k kv.Key) (kv.Iterator, error) { } func (st *TxnState) cleanup() { - st.buf.Reset() + const sz4M = 4 << 20 + if st.buf.Size() > sz4M { + // The memory footprint for the large transaction could be huge here. + // Each active session has its own buffer, we should free the buffer to + // avoid memory leak. + st.buf = kv.NewMemDbBuffer(kv.DefaultTxnMembufCap) + } else { + st.buf.Reset() + } for key := range st.mutations { delete(st.mutations, key) } @@ -301,7 +309,12 @@ func (st *TxnState) cleanup() { for i := 0; i < len(st.dirtyTableOP); i++ { st.dirtyTableOP[i] = empty } - st.dirtyTableOP = st.dirtyTableOP[:0] + if len(st.dirtyTableOP) > 256 { + // Reduce memory footprint for the large transaction. + st.dirtyTableOP = nil + } else { + st.dirtyTableOP = st.dirtyTableOP[:0] + } } } diff --git a/sessionctx/binloginfo/binloginfo.go b/sessionctx/binloginfo/binloginfo.go index e20c0822bd64a..cf8a61ac6148a 100644 --- a/sessionctx/binloginfo/binloginfo.go +++ b/sessionctx/binloginfo/binloginfo.go @@ -112,12 +112,83 @@ var statusListener = func(_ BinlogStatus) error { return nil } +// EnableSkipBinlogFlag enables the skipBinlog flag. +// NOTE: it is used *ONLY* for test. +func EnableSkipBinlogFlag() { + atomic.StoreUint32(&skipBinlog, 1) + logutil.BgLogger().Warn("[binloginfo] enable the skipBinlog flag") +} + // DisableSkipBinlogFlag disable the skipBinlog flag. func DisableSkipBinlogFlag() { atomic.StoreUint32(&skipBinlog, 0) logutil.BgLogger().Warn("[binloginfo] disable the skipBinlog flag") } +// IsBinlogSkipped gets the skipBinlog flag. +func IsBinlogSkipped() bool { + return atomic.LoadUint32(&skipBinlog) > 0 +} + +// BinlogRecoverStatus is used for display the binlog recovered status after some operations. +type BinlogRecoverStatus struct { + Skipped bool + SkippedCommitterCounter int32 +} + +// GetBinlogStatus returns the binlog recovered status. +func GetBinlogStatus() *BinlogRecoverStatus { + return &BinlogRecoverStatus{ + Skipped: IsBinlogSkipped(), + SkippedCommitterCounter: SkippedCommitterCount(), + } +} + +var skippedCommitterCounter int32 + +// WaitBinlogRecover returns when all committing transaction finished. +func WaitBinlogRecover(timeout time.Duration) error { + logutil.BgLogger().Warn("[binloginfo] start waiting for binlog recovering") + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + start := time.Now() + for { + select { + case <-ticker.C: + if atomic.LoadInt32(&skippedCommitterCounter) == 0 { + logutil.BgLogger().Warn("[binloginfo] binlog recovered") + return nil + } + if time.Since(start) > timeout { + logutil.BgLogger().Warn("[binloginfo] waiting for binlog recovering timed out", + zap.Duration("duration", timeout)) + return errors.New("timeout") + } + } + } +} + +// SkippedCommitterCount returns the number of alive committers whick skipped the binlog writing. +func SkippedCommitterCount() int32 { + return atomic.LoadInt32(&skippedCommitterCounter) +} + +// ResetSkippedCommitterCounter is used to reset the skippedCommitterCounter. +func ResetSkippedCommitterCounter() { + atomic.StoreInt32(&skippedCommitterCounter, 0) + logutil.BgLogger().Warn("[binloginfo] skippedCommitterCounter is reset to 0") +} + +// AddOneSkippedCommitter adds one committer to skippedCommitterCounter. +func AddOneSkippedCommitter() { + atomic.AddInt32(&skippedCommitterCounter, 1) +} + +// RemoveOneSkippedCommitter removes one committer from skippedCommitterCounter. +func RemoveOneSkippedCommitter() { + atomic.AddInt32(&skippedCommitterCounter, -1) +} + // SetIgnoreError sets the ignoreError flag, this function called when TiDB start // up and find config.Binlog.IgnoreError is true. func SetIgnoreError(on bool) { @@ -146,16 +217,32 @@ func RegisterStatusListener(listener func(BinlogStatus) error) { statusListener = listener } +// WriteResult is used for the returned chan of WriteBinlog. +type WriteResult struct { + skipped bool + err error +} + +// Skipped if true stands for the binlog writing is skipped. +func (wr *WriteResult) Skipped() bool { + return wr.skipped +} + +// GetError gets the error of WriteBinlog. +func (wr *WriteResult) GetError() error { + return wr.err +} + // WriteBinlog writes a binlog to Pump. -func (info *BinlogInfo) WriteBinlog(clusterID uint64) error { +func (info *BinlogInfo) WriteBinlog(clusterID uint64) *WriteResult { skip := atomic.LoadUint32(&skipBinlog) if skip > 0 { metrics.CriticalErrorCounter.Add(1) - return nil + return &WriteResult{true, nil} } if info.Client == nil { - return errors.New("pumps client is nil") + return &WriteResult{false, errors.New("pumps client is nil")} } // it will retry in PumpsClient if write binlog fail. @@ -176,18 +263,18 @@ func (info *BinlogInfo) WriteBinlog(clusterID uint64) error { logutil.BgLogger().Warn("update binlog status failed", zap.Error(err)) } } - return nil + return &WriteResult{true, nil} } if strings.Contains(err.Error(), "received message larger than max") { // This kind of error is not critical, return directly. - return errors.Errorf("binlog data is too large (%s)", err.Error()) + return &WriteResult{false, errors.Errorf("binlog data is too large (%s)", err.Error())} } - return terror.ErrCritical.GenWithStackByArgs(err) + return &WriteResult{false, terror.ErrCritical.GenWithStackByArgs(err)} } - return nil + return &WriteResult{false, nil} } // SetDDLBinlog sets DDL binlog in the kv.Transaction. diff --git a/sessionctx/binloginfo/binloginfo_test.go b/sessionctx/binloginfo/binloginfo_test.go index 0f8dc4c3bb26e..b7e4c7669d736 100644 --- a/sessionctx/binloginfo/binloginfo_test.go +++ b/sessionctx/binloginfo/binloginfo_test.go @@ -270,7 +270,8 @@ func (s *testBinlogSuite) TestMaxRecvSize(c *C) { }, Client: s.client, } - err := info.WriteBinlog(1) + binlogWR := info.WriteBinlog(1) + err := binlogWR.GetError() c.Assert(err, NotNil) c.Assert(terror.ErrCritical.Equal(err), IsFalse, Commentf("%v", err)) } diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index b3b9807ec98d4..2c21f4fd0030f 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/parser" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/memory" "go.uber.org/zap" @@ -126,6 +127,7 @@ type StatementContext struct { Priority mysql.PriorityEnum NotFillCache bool MemTracker *memory.Tracker + DiskTracker *disk.Tracker RuntimeStatsColl *execdetails.RuntimeStatsColl TableIDs []int64 IndexNames []string @@ -518,7 +520,15 @@ func (sc *StatementContext) CopTasksDetails() *CopTasksDetails { sc.mu.Lock() defer sc.mu.Unlock() n := len(sc.mu.allExecDetails) - d := &CopTasksDetails{NumCopTasks: n} + d := &CopTasksDetails{ + NumCopTasks: n, + MaxBackoffTime: make(map[string]time.Duration), + AvgBackoffTime: make(map[string]time.Duration), + P90BackoffTime: make(map[string]time.Duration), + TotBackoffTime: make(map[string]time.Duration), + TotBackoffTimes: make(map[string]int), + MaxBackoffAddress: make(map[string]string), + } if n == 0 { return d } @@ -538,9 +548,60 @@ func (sc *StatementContext) CopTasksDetails() *CopTasksDetails { d.P90WaitTime = sc.mu.allExecDetails[n*9/10].WaitTime d.MaxWaitTime = sc.mu.allExecDetails[n-1].WaitTime d.MaxWaitAddress = sc.mu.allExecDetails[n-1].CalleeAddress + + // calculate backoff details + type backoffItem struct { + callee string + sleepTime time.Duration + times int + } + backoffInfo := make(map[string][]backoffItem) + for _, ed := range sc.mu.allExecDetails { + for backoff := range ed.BackoffTimes { + backoffInfo[backoff] = append(backoffInfo[backoff], backoffItem{ + callee: ed.CalleeAddress, + sleepTime: ed.BackoffSleep[backoff], + times: ed.BackoffTimes[backoff], + }) + } + } + for backoff, items := range backoffInfo { + if len(items) == 0 { + continue + } + sort.Slice(items, func(i, j int) bool { + return items[i].sleepTime < items[j].sleepTime + }) + n := len(items) + d.MaxBackoffAddress[backoff] = items[n-1].callee + d.MaxBackoffTime[backoff] = items[n-1].sleepTime + d.P90BackoffTime[backoff] = items[n*9/10].sleepTime + + var totalTime time.Duration + totalTimes := 0 + for _, it := range items { + totalTime += it.sleepTime + totalTimes += it.times + } + d.AvgBackoffTime[backoff] = totalTime / time.Duration(n) + d.TotBackoffTime[backoff] = totalTime + d.TotBackoffTimes[backoff] = totalTimes + } return d } +// SetFlagsFromPBFlag set the flag of StatementContext from a `tipb.SelectRequest.Flags`. +func (sc *StatementContext) SetFlagsFromPBFlag(flags uint64) { + sc.IgnoreTruncate = (flags & model.FlagIgnoreTruncate) > 0 + sc.TruncateAsWarning = (flags & model.FlagTruncateAsWarning) > 0 + sc.PadCharToFullLength = (flags & model.FlagPadCharToFullLength) > 0 + sc.InInsertStmt = (flags & model.FlagInInsertStmt) > 0 + sc.InSelectStmt = (flags & model.FlagInSelectStmt) > 0 + sc.OverflowAsWarning = (flags & model.FlagOverflowAsWarning) > 0 + sc.IgnoreZeroInDate = (flags & model.FlagIgnoreZeroInDate) > 0 + sc.DividedByZeroAsWarning = (flags & model.FlagDividedByZeroAsWarning) > 0 +} + //CopTasksDetails collects some useful information of cop-tasks during execution. type CopTasksDetails struct { NumCopTasks int @@ -554,6 +615,13 @@ type CopTasksDetails struct { P90WaitTime time.Duration MaxWaitAddress string MaxWaitTime time.Duration + + MaxBackoffTime map[string]time.Duration + MaxBackoffAddress map[string]string + AvgBackoffTime map[string]time.Duration + P90BackoffTime map[string]time.Duration + TotBackoffTime map[string]time.Duration + TotBackoffTimes map[string]int } // ToZapFields wraps the CopTasksDetails as zap.Fileds. diff --git a/sessionctx/stmtctx/stmtctx_test.go b/sessionctx/stmtctx/stmtctx_test.go index 6ba2e862f27dc..01926518c644a 100644 --- a/sessionctx/stmtctx/stmtctx_test.go +++ b/sessionctx/stmtctx/stmtctx_test.go @@ -33,11 +33,18 @@ var _ = Suite(&stmtctxSuit{}) func (s *stmtctxSuit) TestCopTasksDetails(c *C) { ctx := new(stmtctx.StatementContext) + backoffs := []string{"tikvRPC", "pdRPC", "regionMiss"} for i := 0; i < 100; i++ { d := &execdetails.ExecDetails{ CalleeAddress: fmt.Sprintf("%v", i+1), ProcessTime: time.Second * time.Duration(i+1), WaitTime: time.Millisecond * time.Duration(i+1), + BackoffSleep: make(map[string]time.Duration), + BackoffTimes: make(map[string]int), + } + for _, backoff := range backoffs { + d.BackoffSleep[backoff] = time.Millisecond * 100 * time.Duration(i+1) + d.BackoffTimes[backoff] = i + 1 } ctx.MergeExecDetails(d, nil) } @@ -53,6 +60,14 @@ func (s *stmtctxSuit) TestCopTasksDetails(c *C) { c.Assert(d.MaxWaitAddress, Equals, "100") fields := d.ToZapFields() c.Assert(len(fields), Equals, 9) + for _, backoff := range backoffs { + c.Assert(d.MaxBackoffAddress[backoff], Equals, "100") + c.Assert(d.MaxBackoffTime[backoff], Equals, 100*time.Millisecond*100) + c.Assert(d.P90BackoffTime[backoff], Equals, time.Millisecond*100*91) + c.Assert(d.AvgBackoffTime[backoff], Equals, time.Millisecond*100*101/2) + c.Assert(d.TotBackoffTimes[backoff], Equals, 101*50) + c.Assert(d.TotBackoffTime[backoff], Equals, 101*50*100*time.Millisecond) + } } func (s *stmtctxSuit) TestStatementContextPushDownFLags(c *C) { diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 3ec99b0d87cd2..df7ecb66679bf 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -17,6 +17,7 @@ import ( "bytes" "crypto/tls" "fmt" + "sort" "strconv" "strings" "sync" @@ -308,6 +309,8 @@ type SessionVars struct { SeekFactor float64 // MemoryFactor is the memory cost of storing one tuple. MemoryFactor float64 + // DiskFactor is the IO cost of reading/writing one byte to temporary disk. + DiskFactor float64 // ConcurrencyFactor is the CPU cost of additional one goroutine. ConcurrencyFactor float64 @@ -517,6 +520,7 @@ func NewSessionVars() *SessionVars { DescScanFactor: DefOptDescScanFactor, SeekFactor: DefOptSeekFactor, MemoryFactor: DefOptMemoryFactor, + DiskFactor: DefOptDiskFactor, ConcurrencyFactor: DefOptConcurrencyFactor, EnableRadixJoin: false, EnableVectorizedExpression: DefEnableVectorizedExpression, @@ -532,7 +536,7 @@ func NewSessionVars() *SessionVars { AllowRemoveAutoInc: DefTiDBAllowRemoveAutoInc, UsePlanBaselines: DefTiDBUsePlanBaselines, EvolvePlanBaselines: DefTiDBEvolvePlanBaselines, - isolationReadEngines: map[kv.StoreType]struct{}{kv.TiKV: {}, kv.TiFlash: {}}, + isolationReadEngines: map[kv.StoreType]struct{}{kv.TiKV: {}, kv.TiFlash: {}, kv.TiDB: {}}, LockWaitTimeout: DefInnodbLockWaitTimeout * 1000, } vars.Concurrency = Concurrency{ @@ -848,6 +852,8 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { s.SeekFactor = tidbOptFloat64(val, DefOptSeekFactor) case TiDBOptMemoryFactor: s.MemoryFactor = tidbOptFloat64(val, DefOptMemoryFactor) + case TiDBOptDiskFactor: + s.DiskFactor = tidbOptFloat64(val, DefOptDiskFactor) case TiDBOptConcurrencyFactor: s.ConcurrencyFactor = tidbOptFloat64(val, DefOptConcurrencyFactor) case TiDBIndexLookupConcurrency: @@ -985,6 +991,8 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { s.isolationReadEngines[kv.TiKV] = struct{}{} case kv.TiFlash.Name(): s.isolationReadEngines[kv.TiFlash] = struct{}{} + case kv.TiDB.Name(): + s.isolationReadEngines[kv.TiDB] = struct{}{} } } case TiDBStoreLimit: @@ -1181,6 +1189,8 @@ const ( SlowLogCopWaitMax = "Cop_wait_max" // SlowLogCopWaitAddr is the address of TiKV where the cop-task which cost wait process time run. SlowLogCopWaitAddr = "Cop_wait_addr" + // SlowLogCopBackoffPrefix contains backoff information. + SlowLogCopBackoffPrefix = "Cop_backoff_" // SlowLogMemMax is the max number bytes of memory used in this statement. SlowLogMemMax = "Mem_max" // SlowLogPrepared is used to indicate whether this sql execute in prepare. @@ -1294,6 +1304,13 @@ func (s *SessionVars) SlowLogFormat(logItems *SlowQueryLogItems) string { if logItems.CopTasks != nil { writeSlowLogItem(&buf, SlowLogNumCopTasksStr, strconv.FormatInt(int64(logItems.CopTasks.NumCopTasks), 10)) if logItems.CopTasks.NumCopTasks > 0 { + // make the result stable + backoffs := make([]string, 0, 3) + for backoff := range logItems.CopTasks.TotBackoffTimes { + backoffs = append(backoffs, backoff) + } + sort.Strings(backoffs) + if logItems.CopTasks.NumCopTasks == 1 { buf.WriteString(SlowLogRowPrefixStr + fmt.Sprintf("%v%v%v %v%v%v", SlowLogCopProcAvg, SlowLogSpaceMarkStr, logItems.CopTasks.AvgProcessTime.Seconds(), @@ -1301,7 +1318,13 @@ func (s *SessionVars) SlowLogFormat(logItems *SlowQueryLogItems) string { buf.WriteString(SlowLogRowPrefixStr + fmt.Sprintf("%v%v%v %v%v%v", SlowLogCopWaitAvg, SlowLogSpaceMarkStr, logItems.CopTasks.AvgWaitTime.Seconds(), SlowLogCopWaitAddr, SlowLogSpaceMarkStr, logItems.CopTasks.MaxWaitAddress) + "\n") - + for _, backoff := range backoffs { + backoffPrefix := SlowLogCopBackoffPrefix + backoff + "_" + buf.WriteString(SlowLogRowPrefixStr + fmt.Sprintf("%v%v%v %v%v%v\n", + backoffPrefix+"total_times", SlowLogSpaceMarkStr, logItems.CopTasks.TotBackoffTimes[backoff], + backoffPrefix+"total_time", SlowLogSpaceMarkStr, logItems.CopTasks.TotBackoffTime[backoff].Seconds(), + )) + } } else { buf.WriteString(SlowLogRowPrefixStr + fmt.Sprintf("%v%v%v %v%v%v %v%v%v %v%v%v", SlowLogCopProcAvg, SlowLogSpaceMarkStr, logItems.CopTasks.AvgProcessTime.Seconds(), @@ -1313,6 +1336,17 @@ func (s *SessionVars) SlowLogFormat(logItems *SlowQueryLogItems) string { SlowLogCopWaitP90, SlowLogSpaceMarkStr, logItems.CopTasks.P90WaitTime.Seconds(), SlowLogCopWaitMax, SlowLogSpaceMarkStr, logItems.CopTasks.MaxWaitTime.Seconds(), SlowLogCopWaitAddr, SlowLogSpaceMarkStr, logItems.CopTasks.MaxWaitAddress) + "\n") + for _, backoff := range backoffs { + backoffPrefix := SlowLogCopBackoffPrefix + backoff + "_" + buf.WriteString(SlowLogRowPrefixStr + fmt.Sprintf("%v%v%v %v%v%v %v%v%v %v%v%v %v%v%v %v%v%v\n", + backoffPrefix+"total_times", SlowLogSpaceMarkStr, logItems.CopTasks.TotBackoffTimes[backoff], + backoffPrefix+"total_time", SlowLogSpaceMarkStr, logItems.CopTasks.TotBackoffTime[backoff].Seconds(), + backoffPrefix+"max_time", SlowLogSpaceMarkStr, logItems.CopTasks.MaxBackoffTime[backoff].Seconds(), + backoffPrefix+"max_addr", SlowLogSpaceMarkStr, logItems.CopTasks.MaxBackoffAddress[backoff], + backoffPrefix+"avg_time", SlowLogSpaceMarkStr, logItems.CopTasks.AvgBackoffTime[backoff].Seconds(), + backoffPrefix+"p90_time", SlowLogSpaceMarkStr, logItems.CopTasks.P90BackoffTime[backoff].Seconds(), + )) + } } } } diff --git a/sessionctx/variable/session_test.go b/sessionctx/variable/session_test.go index 93829d27bc68e..53efd686298e2 100644 --- a/sessionctx/variable/session_test.go +++ b/sessionctx/variable/session_test.go @@ -151,7 +151,24 @@ func (*testSessionSuite) TestSlowLogFormat(c *C) { P90WaitTime: time.Millisecond * 20, MaxWaitTime: time.Millisecond * 30, MaxWaitAddress: "10.6.131.79", + MaxBackoffTime: make(map[string]time.Duration), + AvgBackoffTime: make(map[string]time.Duration), + P90BackoffTime: make(map[string]time.Duration), + TotBackoffTime: make(map[string]time.Duration), + TotBackoffTimes: make(map[string]int), + MaxBackoffAddress: make(map[string]string), } + + backoffs := []string{"rpcTiKV", "rpcPD", "regionMiss"} + for _, backoff := range backoffs { + copTasks.MaxBackoffTime[backoff] = time.Millisecond * 200 + copTasks.MaxBackoffAddress[backoff] = "127.0.0.1" + copTasks.AvgBackoffTime[backoff] = time.Millisecond * 200 + copTasks.P90BackoffTime[backoff] = time.Millisecond * 200 + copTasks.TotBackoffTime[backoff] = time.Millisecond * 200 + copTasks.TotBackoffTimes[backoff] = 200 + } + var memMax int64 = 2333 resultString := `# Txn_start_ts: 406649736972468225 # User: root@192.168.0.1 @@ -168,6 +185,9 @@ func (*testSessionSuite) TestSlowLogFormat(c *C) { # Num_cop_tasks: 10 # Cop_proc_avg: 1 Cop_proc_p90: 2 Cop_proc_max: 3 Cop_proc_addr: 10.6.131.78 # Cop_wait_avg: 0.01 Cop_wait_p90: 0.02 Cop_wait_max: 0.03 Cop_wait_addr: 10.6.131.79 +# Cop_backoff_regionMiss_total_times: 200 Cop_backoff_regionMiss_total_time: 0.2 Cop_backoff_regionMiss_max_time: 0.2 Cop_backoff_regionMiss_max_addr: 127.0.0.1 Cop_backoff_regionMiss_avg_time: 0.2 Cop_backoff_regionMiss_p90_time: 0.2 +# Cop_backoff_rpcPD_total_times: 200 Cop_backoff_rpcPD_total_time: 0.2 Cop_backoff_rpcPD_max_time: 0.2 Cop_backoff_rpcPD_max_addr: 127.0.0.1 Cop_backoff_rpcPD_avg_time: 0.2 Cop_backoff_rpcPD_p90_time: 0.2 +# Cop_backoff_rpcTiKV_total_times: 200 Cop_backoff_rpcTiKV_total_time: 0.2 Cop_backoff_rpcTiKV_max_time: 0.2 Cop_backoff_rpcTiKV_max_addr: 127.0.0.1 Cop_backoff_rpcTiKV_avg_time: 0.2 Cop_backoff_rpcTiKV_p90_time: 0.2 # Mem_max: 2333 # Prepared: true # Has_more_results: true diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 22b042941948e..153205921a428 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -648,6 +648,7 @@ var defaultSysVars = []*SysVar{ {ScopeGlobal | ScopeSession, TiDBOptDescScanFactor, strconv.FormatFloat(DefOptDescScanFactor, 'f', -1, 64)}, {ScopeGlobal | ScopeSession, TiDBOptSeekFactor, strconv.FormatFloat(DefOptSeekFactor, 'f', -1, 64)}, {ScopeGlobal | ScopeSession, TiDBOptMemoryFactor, strconv.FormatFloat(DefOptMemoryFactor, 'f', -1, 64)}, + {ScopeGlobal | ScopeSession, TiDBOptDiskFactor, strconv.FormatFloat(DefOptDiskFactor, 'f', -1, 64)}, {ScopeGlobal | ScopeSession, TiDBOptConcurrencyFactor, strconv.FormatFloat(DefOptConcurrencyFactor, 'f', -1, 64)}, {ScopeGlobal | ScopeSession, TiDBIndexJoinBatchSize, strconv.Itoa(DefIndexJoinBatchSize)}, {ScopeGlobal | ScopeSession, TiDBIndexLookupSize, strconv.Itoa(DefIndexLookupSize)}, @@ -724,7 +725,7 @@ var defaultSysVars = []*SysVar{ {ScopeGlobal, TiDBEvolvePlanTaskMaxTime, strconv.Itoa(DefTiDBEvolvePlanTaskMaxTime)}, {ScopeGlobal, TiDBEvolvePlanTaskStartTime, DefTiDBEvolvePlanTaskStartTime}, {ScopeGlobal, TiDBEvolvePlanTaskEndTime, DefTiDBEvolvePlanTaskEndTime}, - {ScopeGlobal | ScopeSession, TiDBIsolationReadEngines, "tikv,tiflash"}, + {ScopeGlobal | ScopeSession, TiDBIsolationReadEngines, "tikv,tiflash,tidb"}, {ScopeGlobal | ScopeSession, TiDBStoreLimit, strconv.FormatInt(atomic.LoadInt64(&config.GetGlobalConfig().TiKVClient.StoreLimit), 10)}, } diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 11ac625d62861..e50cbbd7ae5e3 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -200,6 +200,8 @@ const ( TiDBOptSeekFactor = "tidb_opt_seek_factor" // tidb_opt_memory_factor is the memory cost of storing one tuple. TiDBOptMemoryFactor = "tidb_opt_memory_factor" + // tidb_opt_disk_factor is the IO cost of reading/writing one byte to temporary disk. + TiDBOptDiskFactor = "tidb_opt_disk_factor" // tidb_opt_concurrency_factor is the CPU cost of additional one goroutine. TiDBOptConcurrencyFactor = "tidb_opt_concurrency_factor" @@ -382,6 +384,7 @@ const ( DefOptDescScanFactor = 3.0 DefOptSeekFactor = 20.0 DefOptMemoryFactor = 0.001 + DefOptDiskFactor = 1.5 DefOptConcurrencyFactor = 3.0 DefOptInSubqToJoinAndAgg = true DefBatchInsert = false diff --git a/sessionctx/variable/varsutil.go b/sessionctx/variable/varsutil.go index 954a597a4e5fe..6c3434be0a2fc 100644 --- a/sessionctx/variable/varsutil.go +++ b/sessionctx/variable/varsutil.go @@ -503,6 +503,7 @@ func ValidateSetSystemVar(vars *SessionVars, name string, value string) (string, TiDBOptDescScanFactor, TiDBOptSeekFactor, TiDBOptMemoryFactor, + TiDBOptDiskFactor, TiDBOptConcurrencyFactor: v, err := strconv.ParseFloat(value, 64) if err != nil { @@ -647,6 +648,8 @@ func ValidateSetSystemVar(vars *SessionVars, name string, value string) (string, formatVal += kv.TiKV.Name() case strings.EqualFold(engine, kv.TiFlash.Name()): formatVal += kv.TiFlash.Name() + case strings.EqualFold(engine, kv.TiDB.Name()): + formatVal += kv.TiDB.Name() default: return value, ErrWrongValueForVar.GenWithStackByArgs(name, value) } diff --git a/sessionctx/variable/varsutil_test.go b/sessionctx/variable/varsutil_test.go index f2451f62b39ad..66801cf3b9827 100644 --- a/sessionctx/variable/varsutil_test.go +++ b/sessionctx/variable/varsutil_test.go @@ -352,6 +352,14 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { c.Assert(val, Equals, "1.0") c.Assert(v.MemoryFactor, Equals, 1.0) + c.Assert(v.DiskFactor, Equals, 1.5) + err = SetSessionSystemVar(v, TiDBOptDiskFactor, types.NewStringDatum("1.1")) + c.Assert(err, IsNil) + val, err = GetSessionSystemVar(v, TiDBOptDiskFactor) + c.Assert(err, IsNil) + c.Assert(val, Equals, "1.1") + c.Assert(v.DiskFactor, Equals, 1.1) + c.Assert(v.ConcurrencyFactor, Equals, 3.0) err = SetSessionSystemVar(v, TiDBOptConcurrencyFactor, types.NewStringDatum("5.0")) c.Assert(err, IsNil) @@ -455,6 +463,8 @@ func (s *testVarsutilSuite) TestValidate(c *C) { {TiDBOptSeekFactor, "-2", true}, {TiDBOptMemoryFactor, "a", true}, {TiDBOptMemoryFactor, "-2", true}, + {TiDBOptDiskFactor, "a", true}, + {TiDBOptDiskFactor, "-2", true}, {TiDBOptConcurrencyFactor, "a", true}, {TiDBOptConcurrencyFactor, "-2", true}, {TxnIsolation, "READ-UNCOMMITTED", true}, diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 2be0e6a445f3c..a72b38851c5a2 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -17,6 +17,7 @@ import ( "fmt" "testing" "time" + "unsafe" . "github.com/pingcap/check" "github.com/pingcap/errors" @@ -209,11 +210,15 @@ func (s *testStatsSuite) TestAvgColLen(c *C) { tableInfo := tbl.Meta() statsTbl := do.StatsHandle().GetTableStats(tableInfo) c.Assert(statsTbl.Columns[tableInfo.Columns[0].ID].AvgColSize(statsTbl.Count, false), Equals, 1.0) + c.Assert(statsTbl.Columns[tableInfo.Columns[0].ID].AvgColSizeListInDisk(statsTbl.Count), Equals, 8.0) // The size of varchar type is LEN + BYTE, here is 1 + 7 = 8 c.Assert(statsTbl.Columns[tableInfo.Columns[1].ID].AvgColSize(statsTbl.Count, false), Equals, 8.0) c.Assert(statsTbl.Columns[tableInfo.Columns[2].ID].AvgColSize(statsTbl.Count, false), Equals, 8.0) c.Assert(statsTbl.Columns[tableInfo.Columns[3].ID].AvgColSize(statsTbl.Count, false), Equals, 8.0) + c.Assert(statsTbl.Columns[tableInfo.Columns[1].ID].AvgColSizeListInDisk(statsTbl.Count), Equals, 8.0-1) + c.Assert(statsTbl.Columns[tableInfo.Columns[2].ID].AvgColSizeListInDisk(statsTbl.Count), Equals, float64(unsafe.Sizeof(float32(12.3)))) + c.Assert(statsTbl.Columns[tableInfo.Columns[3].ID].AvgColSizeListInDisk(statsTbl.Count), Equals, float64(unsafe.Sizeof(types.Time{}))) testKit.MustExec("insert into t values(132, '123456789112', 1232.3, '2018-03-07 19:17:29')") testKit.MustExec("analyze table t") statsTbl = do.StatsHandle().GetTableStats(tableInfo) @@ -221,6 +226,10 @@ func (s *testStatsSuite) TestAvgColLen(c *C) { c.Assert(statsTbl.Columns[tableInfo.Columns[1].ID].AvgColSize(statsTbl.Count, false), Equals, 10.5) c.Assert(statsTbl.Columns[tableInfo.Columns[2].ID].AvgColSize(statsTbl.Count, false), Equals, 8.0) c.Assert(statsTbl.Columns[tableInfo.Columns[3].ID].AvgColSize(statsTbl.Count, false), Equals, 8.0) + c.Assert(statsTbl.Columns[tableInfo.Columns[0].ID].AvgColSizeListInDisk(statsTbl.Count), Equals, 8.0) + c.Assert(statsTbl.Columns[tableInfo.Columns[1].ID].AvgColSizeListInDisk(statsTbl.Count), Equals, 10.5-1) + c.Assert(statsTbl.Columns[tableInfo.Columns[2].ID].AvgColSizeListInDisk(statsTbl.Count), Equals, float64(unsafe.Sizeof(float32(12.3)))) + c.Assert(statsTbl.Columns[tableInfo.Columns[3].ID].AvgColSizeListInDisk(statsTbl.Count), Equals, float64(unsafe.Sizeof(types.Time{}))) } func (s *testStatsSuite) TestDurationToTS(c *C) { diff --git a/statistics/histogram.go b/statistics/histogram.go index 83497b4ed112e..346678ff6136f 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -140,6 +140,26 @@ func (c *Column) AvgColSize(count int64, isKey bool) float64 { return math.Round(float64(c.TotColSize)/float64(count)*100) / 100 } +// AvgColSizeListInDisk is the average column size of the histogram. These sizes are derived +// from `chunk.ListInDisk` so we need to update them if those 2 functions are changed. +func (c *Column) AvgColSizeListInDisk(count int64) float64 { + if count == 0 { + return 0 + } + histCount := c.TotalRowCount() + notNullRatio := 1.0 + if histCount > 0 { + notNullRatio = 1.0 - float64(c.NullCount)/histCount + } + size := chunk.GetFixedLen(c.Histogram.Tp) + if size != -1 { + return float64(size) * notNullRatio + } + // Keep two decimal place. + // size of varchar type is LEN + BYTE, so we minus 1 here. + return math.Round(float64(c.TotColSize)/float64(count)*100)/100 - 1 +} + // AppendBucket appends a bucket into `hg`. func (hg *Histogram) AppendBucket(lower *types.Datum, upper *types.Datum, count, repeat int64) { hg.Buckets = append(hg.Buckets, Bucket{Count: count, Repeat: repeat}) diff --git a/statistics/table.go b/statistics/table.go index c3d79ad621fa9..d056986afeca6 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/ranger" "go.uber.org/atomic" @@ -694,6 +695,28 @@ func (coll *HistColl) GetAvgRowSize(cols []*expression.Column, isEncodedKey bool return size + float64(len(cols)) } +// GetAvgRowSizeListInDisk computes average row size for given columns. +func (coll *HistColl) GetAvgRowSizeListInDisk(cols []*expression.Column, padChar bool) (size float64) { + if coll.Pseudo || len(coll.Columns) == 0 || coll.Count == 0 { + for _, col := range cols { + size += float64(chunk.EstimateTypeWidth(padChar, col.GetType())) + } + } else { + for _, col := range cols { + colHist, ok := coll.Columns[col.UniqueID] + // Normally this would not happen, it is for compatibility with old version stats which + // does not include TotColSize. + if !ok || (!colHist.IsHandle && colHist.TotColSize == 0 && (colHist.NullCount != coll.Count)) { + size += float64(chunk.EstimateTypeWidth(padChar, col.GetType())) + continue + } + size += colHist.AvgColSizeListInDisk(coll.Count) + } + } + // Add 8 byte for each column's size record. See `ListInDisk` for details. + return size + float64(8*len(cols)) +} + // GetTableAvgRowSize computes average row size for a table scan, exclude the index key-value pairs. func (coll *HistColl) GetTableAvgRowSize(cols []*expression.Column, storeType kv.StoreType, handleInCols bool) (size float64) { size = coll.GetAvgRowSize(cols, false) diff --git a/store/mockstore/mocktikv/analyze.go b/store/mockstore/mocktikv/analyze.go index 65a85da0f6bb3..64552a3eb65f0 100644 --- a/store/mockstore/mocktikv/analyze.go +++ b/store/mockstore/mocktikv/analyze.go @@ -64,10 +64,14 @@ func (h *rpcHandler) handleAnalyzeIndexReq(req *coprocessor.Request, analyzeReq if err != nil { return nil, errors.Trace(err) } + startTS := req.StartTs + if startTS == 0 { + startTS = analyzeReq.GetStartTsFallback() + } e := &indexScanExec{ colsLen: int(analyzeReq.IdxReq.NumColumns), kvRanges: ranges, - startTS: analyzeReq.StartTs, + startTS: startTS, isolationLevel: h.isolationLevel, mvccStore: h.mvccStore, IndexScan: &tipb.IndexScan{Desc: false}, @@ -131,12 +135,16 @@ func (h *rpcHandler) handleAnalyzeColumnsReq(req *coprocessor.Request, analyzeRe if err != nil { return nil, errors.Trace(err) } + startTS := req.StartTs + if startTS == 0 { + startTS = analyzeReq.GetStartTsFallback() + } e := &analyzeColumnsExec{ tblExec: &tableScanExec{ TableScan: &tipb.TableScan{Columns: columns}, kvRanges: ranges, colIDs: evalCtx.colIDs, - startTS: analyzeReq.GetStartTs(), + startTS: startTS, isolationLevel: h.isolationLevel, mvccStore: h.mvccStore, execDetail: new(execDetail), diff --git a/store/mockstore/mocktikv/cop_handler_dag.go b/store/mockstore/mocktikv/cop_handler_dag.go index 693d3a79bc7fc..c9a628e2b1bae 100644 --- a/store/mockstore/mocktikv/cop_handler_dag.go +++ b/store/mockstore/mocktikv/cop_handler_dag.go @@ -49,6 +49,7 @@ var dummySlice = make([]byte, 0) type dagContext struct { dagReq *tipb.DAGRequest keyRanges []*coprocessor.KeyRange + startTS uint64 evalCtx *evalContext } @@ -116,6 +117,7 @@ func (h *rpcHandler) buildDAGExecutor(req *coprocessor.Request) (*dagContext, ex ctx := &dagContext{ dagReq: dagReq, keyRanges: req.Ranges, + startTS: req.StartTs, evalCtx: &evalContext{sc: sc}, } e, err := h.buildDAG(ctx, dagReq.Executors) @@ -129,11 +131,7 @@ func (h *rpcHandler) buildDAGExecutor(req *coprocessor.Request) (*dagContext, ex // is set, the daylight saving problem must be considered. Otherwise the // timezone offset in seconds east of UTC is used to constructed the timezone. func constructTimeZone(name string, offset int) (*time.Location, error) { - if name != "" { - return timeutil.LoadLocation(name) - } - - return time.FixedZone("", offset), nil + return timeutil.ConstructTimeZone(name, offset) } func (h *rpcHandler) handleCopStream(ctx context.Context, req *coprocessor.Request) (tikvpb.Tikv_CoprocessorStreamClient, error) { @@ -197,11 +195,15 @@ func (h *rpcHandler) buildTableScan(ctx *dagContext, executor *tipb.Executor) (* return nil, errors.Trace(err) } + startTS := ctx.startTS + if startTS == 0 { + startTS = ctx.dagReq.GetStartTsFallback() + } e := &tableScanExec{ TableScan: executor.TblScan, kvRanges: ranges, colIDs: ctx.evalCtx.colIDs, - startTS: ctx.dagReq.GetStartTs(), + startTS: startTS, isolationLevel: h.isolationLevel, resolvedLocks: h.resolvedLocks, mvccStore: h.mvccStore, @@ -236,11 +238,15 @@ func (h *rpcHandler) buildIndexScan(ctx *dagContext, executor *tipb.Executor) (* return nil, errors.Trace(err) } + startTS := ctx.startTS + if startTS == 0 { + startTS = ctx.dagReq.GetStartTsFallback() + } e := &indexScanExec{ IndexScan: executor.IdxScan, kvRanges: ranges, colsLen: len(columns), - startTS: ctx.dagReq.GetStartTs(), + startTS: startTS, isolationLevel: h.isolationLevel, resolvedLocks: h.resolvedLocks, mvccStore: h.mvccStore, diff --git a/store/mockstore/mocktikv/rpc.go b/store/mockstore/mocktikv/rpc.go index e9eba4f11db75..00eb09dc8fa4a 100644 --- a/store/mockstore/mocktikv/rpc.go +++ b/store/mockstore/mocktikv/rpc.go @@ -722,6 +722,10 @@ func (c *RPCClient) checkArgs(ctx context.Context, addr string) (*rpcHandler, er return handler, nil } +// TiDBRPCServerCoprocessorHandler is the TiDB rpc server coprocessor handler. +// TODO: remove this global variable. +var TiDBRPCServerCoprocessorHandler func(context.Context, *coprocessor.Request) *coprocessor.Response + // SendRequest sends a request to mock cluster. func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { @@ -736,12 +740,18 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R } }) + reqCtx := &req.Context + resp := &tikvrpc.Response{} + // When the store type is TiDB, the request should handle over to TiDB rpc server to handle. + if req.Type == tikvrpc.CmdCop && req.StoreTp == kv.TiDB && TiDBRPCServerCoprocessorHandler != nil { + resp.Resp = TiDBRPCServerCoprocessorHandler(context.Background(), req.Cop()) + return resp, nil + } + handler, err := c.checkArgs(ctx, addr) if err != nil { return nil, err } - reqCtx := &req.Context - resp := &tikvrpc.Response{} switch req.Type { case tikvrpc.CmdGet: r := req.Get() diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index 73b676b6081a8..2e74fe816f8c7 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -70,7 +70,7 @@ var ( // Global variable set by config file. var ( - PessimisticLockTTL uint64 = 20000 // 20s + ManagedLockTTL uint64 = 20000 // 20s ) func (actionPrewrite) String() string { @@ -111,6 +111,7 @@ type twoPhaseCommitter struct { connID uint64 // connID is used for log. cleanWg sync.WaitGroup detail unsafe.Pointer + txnSize int primaryKey []byte forUpdateTS uint64 @@ -280,6 +281,7 @@ func (c *twoPhaseCommitter) initKeysAndMutations() error { if len(keys) == 0 { return nil } + c.txnSize = size if size > int(kv.TxnTotalSizeLimit) { return kv.ErrTxnTooLarge.GenWithStackByArgs(size) @@ -436,7 +438,14 @@ func (c *twoPhaseCommitter) doActionOnBatches(bo *Backoffer, action twoPhaseComm } return errors.Trace(e) } - rateLim := len(batches) // this will be used for LargeTxn, set rateLim here + rateLim := len(batches) + // Set rateLim here for the large transaction. + // If the rate limit is too high, tikv will report service is busy. + // If the rate limit is too low, we can't full utilize the tikv's throughput. + // TODO: Find a self-adaptive way to control the rate limit here. + if rateLim > 32 { + rateLim = 32 + } batchExecutor := newBatchExecutor(rateLim, c, action, bo) err := batchExecutor.process(batches) return errors.Trace(err) @@ -526,6 +535,13 @@ func (actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, bat prewriteResp := resp.Resp.(*pb.PrewriteResponse) keyErrs := prewriteResp.GetErrors() if len(keyErrs) == 0 { + if bytes.Equal(c.primary(), batch.keys[0]) { + // After writing the primary key, if the size of the transaction is large than 4M, + // start the ttlManager. The ttlManager will be closed in tikvTxn.Commit(). + if c.txnSize > 32*1024*1024 { + c.run(c, nil) + } + } return nil } var locks []*Lock @@ -597,8 +613,8 @@ func (tm *ttlManager) close() { } func (tm *ttlManager) keepAlive(c *twoPhaseCommitter) { - // Ticker is set to 1/2 of the PessimisticLockTTL. - ticker := time.NewTicker(time.Duration(PessimisticLockTTL) * time.Millisecond / 2) + // Ticker is set to 1/2 of the ManagedLockTTL. + ticker := time.NewTicker(time.Duration(ManagedLockTTL) * time.Millisecond / 2) defer ticker.Stop() for { select { @@ -631,7 +647,9 @@ func (tm *ttlManager) keepAlive(c *twoPhaseCommitter) { return } - newTTL := uptime + PessimisticLockTTL + newTTL := uptime + ManagedLockTTL + logutil.BgLogger().Info("send TxnHeartBeat", + zap.Uint64("startTS", c.startTS), zap.Uint64("newTTL", newTTL)) startTime := time.Now() _, err = sendTxnHeartBeat(bo, c.store, c.primary(), c.startTS, newTTL) if err != nil { @@ -659,15 +677,13 @@ func (action actionPessimisticLock) handleSingleBatch(c *twoPhaseCommitter, bo * } mutations[i] = mut } - - t0 := oracle.GetTimeFromTS(c.forUpdateTS) - elapsed := uint64(time.Since(t0) / time.Millisecond) + elapsed := uint64(time.Since(c.txn.startTime) / time.Millisecond) req := tikvrpc.NewRequest(tikvrpc.CmdPessimisticLock, &pb.PessimisticLockRequest{ Mutations: mutations, PrimaryLock: c.primary(), StartVersion: c.startTS, ForUpdateTs: c.forUpdateTS, - LockTtl: elapsed + PessimisticLockTTL, + LockTtl: elapsed + ManagedLockTTL, IsFirstLock: c.isFirstLock, WaitTimeout: action.lockWaitTime, }, pb.Context{Priority: c.priority, SyncLog: c.syncLog}) @@ -778,25 +794,23 @@ func (actionPessimisticRollback) handleSingleBatch(c *twoPhaseCommitter, bo *Bac ForUpdateTs: c.forUpdateTS, Keys: batch.keys, }) - for { - resp, err := c.store.SendReq(bo, req, batch.region, readTimeoutShort) - if err != nil { - return errors.Trace(err) - } - regionErr, err := resp.GetRegionError() + resp, err := c.store.SendReq(bo, req, batch.region, readTimeoutShort) + if err != nil { + return errors.Trace(err) + } + regionErr, err := resp.GetRegionError() + if err != nil { + return errors.Trace(err) + } + if regionErr != nil { + err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) if err != nil { return errors.Trace(err) } - if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) - if err != nil { - return errors.Trace(err) - } - err = c.pessimisticRollbackKeys(bo, batch.keys) - return errors.Trace(err) - } - return nil + err = c.pessimisticRollbackKeys(bo, batch.keys) + return errors.Trace(err) } + return nil } func getTxnPriority(txn *tikvTxn) pb.CommandPri { @@ -819,8 +833,9 @@ func kvPriorityToCommandPri(pri int) pb.CommandPri { return pb.CommandPri_Low case kv.PriorityHigh: return pb.CommandPri_High + default: + return pb.CommandPri_Normal } - return pb.CommandPri_Normal } func (c *twoPhaseCommitter) setDetail(d *execdetails.CommitDetails) { @@ -999,19 +1014,9 @@ func (c *twoPhaseCommitter) pessimisticRollbackKeys(bo *Backoffer, keys [][]byte return c.doActionOnKeys(bo, actionPessimisticRollback{}, keys) } -func (c *twoPhaseCommitter) executeAndWriteFinishBinlog(ctx context.Context) error { - err := c.execute(ctx) - if err != nil { - c.writeFinishBinlog(ctx, binlog.BinlogType_Rollback, 0) - } else { - c.txn.commitTS = c.commitTS - c.writeFinishBinlog(ctx, binlog.BinlogType_Commit, int64(c.commitTS)) - } - return errors.Trace(err) -} - // execute executes the two-phase commit protocol. -func (c *twoPhaseCommitter) execute(ctx context.Context) error { +func (c *twoPhaseCommitter) execute(ctx context.Context) (err error) { + var binlogSkipped bool defer func() { // Always clean up all written keys if the txn does not commit. c.mu.RLock() @@ -1035,12 +1040,22 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) error { c.cleanWg.Done() }() } + c.txn.commitTS = c.commitTS + if binlogSkipped { + binloginfo.RemoveOneSkippedCommitter() + } else { + if err != nil { + c.writeFinishBinlog(ctx, binlog.BinlogType_Rollback, 0) + } else { + c.writeFinishBinlog(ctx, binlog.BinlogType_Commit, int64(c.commitTS)) + } + } }() binlogChan := c.prewriteBinlog(ctx) prewriteBo := NewBackoffer(ctx, PrewriteMaxBackoff).WithVars(c.txn.vars) start := time.Now() - err := c.prewriteKeys(prewriteBo, c.keys) + err = c.prewriteKeys(prewriteBo, c.keys) commitDetail := c.getDetail() commitDetail.PrewriteTime = time.Since(start) if prewriteBo.totalSleep > 0 { @@ -1050,9 +1065,13 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) error { commitDetail.Mu.Unlock() } if binlogChan != nil { - binlogErr := <-binlogChan - if binlogErr != nil { - return errors.Trace(binlogErr) + binlogWriteResult := <-binlogChan + if binlogWriteResult != nil { + binlogSkipped = binlogWriteResult.Skipped() + binlogErr := binlogWriteResult.GetError() + if binlogErr != nil { + return binlogErr + } } } if err != nil { @@ -1139,11 +1158,11 @@ func (c *twoPhaseCommitter) checkSchemaValid() error { return nil } -func (c *twoPhaseCommitter) prewriteBinlog(ctx context.Context) chan error { +func (c *twoPhaseCommitter) prewriteBinlog(ctx context.Context) chan *binloginfo.WriteResult { if !c.shouldWriteBinlog() { return nil } - ch := make(chan error, 1) + ch := make(chan *binloginfo.WriteResult, 1) go func() { logutil.Eventf(ctx, "start prewrite binlog") binInfo := c.txn.us.GetOption(kv.BinlogInfo).(*binloginfo.BinlogInfo) @@ -1152,9 +1171,13 @@ func (c *twoPhaseCommitter) prewriteBinlog(ctx context.Context) chan error { if bin.Tp == binlog.BinlogType_Prewrite { bin.PrewriteKey = c.keys[0] } - err := binInfo.WriteBinlog(c.store.clusterID) + wr := binInfo.WriteBinlog(c.store.clusterID) + if wr.Skipped() { + binInfo.Data.PrewriteValue = nil + binloginfo.AddOneSkippedCommitter() + } logutil.Eventf(ctx, "finish prewrite binlog") - ch <- errors.Trace(err) + ch <- wr }() return ch } @@ -1169,7 +1192,8 @@ func (c *twoPhaseCommitter) writeFinishBinlog(ctx context.Context, tp binlog.Bin binInfo.Data.PrewriteValue = nil go func() { logutil.Eventf(ctx, "start write finish binlog") - err := binInfo.WriteBinlog(c.store.clusterID) + binlogWriteResult := binInfo.WriteBinlog(c.store.clusterID) + err := binlogWriteResult.GetError() if err != nil { logutil.BgLogger().Error("failed to write binlog", zap.Error(err)) diff --git a/store/tikv/2pc_test.go b/store/tikv/2pc_test.go index 29c46939267d0..f5cb8ad103831 100644 --- a/store/tikv/2pc_test.go +++ b/store/tikv/2pc_test.go @@ -39,7 +39,7 @@ type testCommitterSuite struct { var _ = Suite(&testCommitterSuite{}) func (s *testCommitterSuite) SetUpSuite(c *C) { - PessimisticLockTTL = 3000 // 3s + ManagedLockTTL = 3000 // 3s s.OneByOneSuite.SetUpSuite(c) } @@ -611,7 +611,7 @@ func (s *testCommitterSuite) TestPessimisticTTL(c *C) { expire := oracle.ExtractPhysical(txn.startTS) + int64(lockInfoNew.LockTtl) now := oracle.ExtractPhysical(currentTS) c.Assert(expire > now, IsTrue) - c.Assert(uint64(expire-now) <= PessimisticLockTTL, IsTrue) + c.Assert(uint64(expire-now) <= ManagedLockTTL, IsTrue) return } time.Sleep(100 * time.Millisecond) @@ -619,6 +619,21 @@ func (s *testCommitterSuite) TestPessimisticTTL(c *C) { c.Assert(false, IsTrue, Commentf("update pessimistic ttl fail")) } +// TestElapsedTTL tests that elapsed time is correct even if ts physical time is greater than local time. +func (s *testCommitterSuite) TestElapsedTTL(c *C) { + key := kv.Key("key") + txn := s.begin(c) + txn.startTS = oracle.ComposeTS(oracle.GetPhysical(time.Now().Add(time.Second*10)), 1) + txn.SetOption(kv.Pessimistic, true) + time.Sleep(time.Millisecond * 100) + forUpdateTS := oracle.ComposeTS(oracle.ExtractPhysical(txn.startTS)+100, 1) + err := txn.LockKeys(context.Background(), nil, forUpdateTS, kv.LockAlwaysWait, key) + c.Assert(err, IsNil) + lockInfo := s.getLockInfo(c, key) + c.Assert(lockInfo.LockTtl-ManagedLockTTL, GreaterEqual, uint64(100)) + c.Assert(lockInfo.LockTtl-ManagedLockTTL, Less, uint64(150)) +} + func (s *testCommitterSuite) getLockInfo(c *C, key []byte) *kvrpcpb.LockInfo { txn := s.begin(c) err := txn.Set(key, key) diff --git a/store/tikv/backoff.go b/store/tikv/backoff.go index 25d1ea8645e7b..18bd3414071c3 100644 --- a/store/tikv/backoff.go +++ b/store/tikv/backoff.go @@ -250,6 +250,9 @@ type Backoffer struct { types []fmt.Stringer vars *kv.Variables noop bool + + backoffSleepMS map[backoffType]int + backoffTimes map[backoffType]int } type txnStartCtxKeyType struct{} @@ -332,6 +335,14 @@ func (b *Backoffer) BackoffWithMaxSleep(typ backoffType, maxSleepMs int, err err realSleep := f(b.ctx, maxSleepMs) backoffDuration.Observe(float64(realSleep) / 1000) b.totalSleep += realSleep + if b.backoffSleepMS == nil { + b.backoffSleepMS = make(map[backoffType]int) + } + b.backoffSleepMS[typ] += realSleep + if b.backoffTimes == nil { + b.backoffTimes = make(map[backoffType]int) + } + b.backoffTimes[typ]++ var startTs interface{} if ts := b.ctx.Value(txnStartKey); ts != nil { diff --git a/store/tikv/client.go b/store/tikv/client.go index 64882a8b6d31e..24635272243fa 100644 --- a/store/tikv/client.go +++ b/store/tikv/client.go @@ -31,10 +31,12 @@ import ( "github.com/pingcap/kvproto/pkg/tikvpb" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/util/logutil" "google.golang.org/grpc" + "google.golang.org/grpc/backoff" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" @@ -132,7 +134,7 @@ func (a *connArray) Init(addr string, security config.Security, idleNotify *uint grpc.WithUnaryInterceptor(unaryInterceptor), grpc.WithStreamInterceptor(streamInterceptor), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(MaxRecvMsgSize)), - grpc.WithBackoffMaxDelay(time.Second*3), + grpc.WithConnectParams(grpc.ConnectParams{Backoff: backoff.Config{BaseDelay: time.Second * 3}}), grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: time.Duration(keepAlive) * time.Second, Timeout: time.Duration(keepAliveTimeout) * time.Second, @@ -287,7 +289,9 @@ func (c *rpcClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R return nil, errors.Trace(err) } - if config.GetGlobalConfig().TiKVClient.MaxBatchSize > 0 { + // TiDB RPC server not support batch RPC now. + // TODO: remove this store type check after TiDB RPC Server support stream. + if config.GetGlobalConfig().TiKVClient.MaxBatchSize > 0 && req.StoreTp != kv.TiDB { if batchReq := req.ToBatchCommandsRequest(); batchReq != nil { return sendBatchRequest(ctx, addr, connArray.batchConn, batchReq, timeout) } diff --git a/store/tikv/client_batch.go b/store/tikv/client_batch.go index fe2dc76c42deb..73167e710be22 100644 --- a/store/tikv/client_batch.go +++ b/store/tikv/client_batch.go @@ -491,7 +491,6 @@ func (a *batchConn) getClientAndSend(entries []*batchCommandsEntry, requests []* } cli.send(req, entries) - return } func (c *batchCommandsClient) initBatchClient() error { diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go index 2f60d280725c3..f4ba544903f61 100644 --- a/store/tikv/coprocessor.go +++ b/store/tikv/coprocessor.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "sort" + "strconv" "strings" "sync" "sync/atomic" @@ -31,6 +32,7 @@ import ( "github.com/pingcap/kvproto/pkg/coprocessor" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/distsql" + "github.com/pingcap/tidb/domain/infosync" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" @@ -217,7 +219,6 @@ const rangesPerTask = 25000 func buildCopTasks(bo *Backoffer, cache *RegionCache, ranges *copRanges, req *kv.Request) ([]*copTask, error) { start := time.Now() - rangesLen := ranges.len() cmdType := tikvrpc.CmdCop if req.Streaming { cmdType = tikvrpc.CmdCopStream @@ -230,6 +231,11 @@ func buildCopTasks(bo *Backoffer, cache *RegionCache, ranges *copRanges, req *kv tableStart, tableEnd = keyRange[0].StartKey, keyRange[0].EndKey } + if req.StoreType == kv.TiDB { + return buildTiDBMemCopTasks(ranges, req) + } + + rangesLen := ranges.len() var tasks []*copTask appendTask := func(regionWithRangeInfo *KeyLocation, ranges *copRanges) { if req.StoreType == kv.TiKV { @@ -291,6 +297,25 @@ func buildCopTasks(bo *Backoffer, cache *RegionCache, ranges *copRanges, req *kv return tasks, nil } +func buildTiDBMemCopTasks(ranges *copRanges, req *kv.Request) ([]*copTask, error) { + servers, err := infosync.GetAllServerInfo(context.Background()) + if err != nil { + return nil, err + } + tasks := make([]*copTask, 0, len(servers)) + for _, ser := range servers { + addr := ser.IP + ":" + strconv.FormatUint(uint64(ser.StatusPort), 10) + tasks = append(tasks, &copTask{ + ranges: ranges, + respChan: make(chan *copResponse, 2), + cmdType: tikvrpc.CmdCop, + storeType: req.StoreType, + storeAddr: addr, + }) + } + return tasks, nil +} + func splitRanges(bo *Backoffer, cache *RegionCache, ranges *copRanges, fn func(regionWithRangeInfo *KeyLocation, ranges *copRanges)) error { for ranges.len() > 0 { loc, err := cache.LocateKey(bo, ranges.at(0).StartKey) @@ -688,9 +713,10 @@ func (worker *copIteratorWorker) handleTaskOnce(bo *Backoffer, task *copTask, ch }) req := tikvrpc.NewReplicaReadRequest(task.cmdType, &coprocessor.Request{ - Tp: worker.req.Tp, - Data: worker.req.Data, - Ranges: task.ranges.toPBRanges(), + Tp: worker.req.Tp, + StartTs: worker.req.StartTs, + Data: worker.req.Data, + Ranges: task.ranges.toPBRanges(), }, worker.req.ReplicaRead, worker.replicaReadSeed, kvrpcpb.Context{ IsolationLevel: pbIsolationLevel(worker.req.IsolationLevel), Priority: kvPriorityToCommandPri(worker.req.Priority), @@ -698,8 +724,9 @@ func (worker *copIteratorWorker) handleTaskOnce(bo *Backoffer, task *copTask, ch HandleTime: true, ScanDetail: true, }) + req.StoreTp = task.storeType startTime := time.Now() - resp, rpcCtx, storeAddr, err := worker.SendReqCtx(bo, req, task.region, ReadTimeoutMedium, task.storeType) + resp, rpcCtx, storeAddr, err := worker.SendReqCtx(bo, req, task.region, ReadTimeoutMedium, task.storeType, task.storeAddr) if err != nil { return nil, errors.Trace(err) } @@ -774,8 +801,11 @@ func (ch *clientHelper) ResolveLocks(bo *Backoffer, callerStartTS uint64, locks } // SendReqCtx wraps the SendReqCtx function and use the resolved lock result in the kvrpcpb.Context. -func (ch *clientHelper) SendReqCtx(bo *Backoffer, req *tikvrpc.Request, regionID RegionVerID, timeout time.Duration, sType kv.StoreType) (*tikvrpc.Response, *RPCContext, string, error) { +func (ch *clientHelper) SendReqCtx(bo *Backoffer, req *tikvrpc.Request, regionID RegionVerID, timeout time.Duration, sType kv.StoreType, directStoreAddr string) (*tikvrpc.Response, *RPCContext, string, error) { sender := NewRegionRequestSender(ch.RegionCache, ch.Client) + if len(directStoreAddr) > 0 { + sender.storeAddr = directStoreAddr + } req.Context.ResolvedLocks = ch.minCommitTSPushed.Get() resp, ctx, err := sender.SendReqCtx(bo, req, regionID, timeout, sType) return resp, ctx, sender.storeAddr, err @@ -881,6 +911,11 @@ func (worker *copIteratorWorker) handleCopStreamResult(bo *Backoffer, rpcCtx *RP // successful response, otherwise it's nil. func (worker *copIteratorWorker) handleCopResponse(bo *Backoffer, rpcCtx *RPCContext, resp *copResponse, task *copTask, ch chan<- *copResponse, lastRange *coprocessor.KeyRange, costTime time.Duration) ([]*copTask, error) { if regionErr := resp.pbResp.GetRegionError(); regionErr != nil { + if rpcCtx != nil && task.storeType == kv.TiDB { + resp.err = errors.Errorf("error: %v", regionErr) + worker.sendToRespCh(resp, ch, true) + return nil, nil + } if err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String())); err != nil { return nil, errors.Trace(err) } @@ -913,13 +948,19 @@ func (worker *copIteratorWorker) handleCopResponse(bo *Backoffer, rpcCtx *RPCCon // When the request is using streaming API, the `Range` is not nil. if resp.pbResp.Range != nil { resp.startKey = resp.pbResp.Range.Start - } else { + } else if task.ranges != nil && task.ranges.len() > 0 { resp.startKey = task.ranges.at(0).StartKey } if resp.detail == nil { resp.detail = new(execdetails.ExecDetails) } resp.detail.BackoffTime = time.Duration(bo.totalSleep) * time.Millisecond + resp.detail.BackoffSleep, resp.detail.BackoffTimes = make(map[string]time.Duration), make(map[string]int) + for backoff := range bo.backoffTimes { + backoffName := backoff.String() + resp.detail.BackoffTimes[backoffName] = bo.backoffTimes[backoff] + resp.detail.BackoffSleep[backoffName] = time.Duration(bo.backoffSleepMS[backoff]) * time.Millisecond + } if rpcCtx != nil { resp.detail.CalleeAddress = rpcCtx.Addr } diff --git a/store/tikv/gcworker/gc_worker.go b/store/tikv/gcworker/gc_worker.go index 3689a4e941782..f1b33aea4ad3e 100644 --- a/store/tikv/gcworker/gc_worker.go +++ b/store/tikv/gcworker/gc_worker.go @@ -179,7 +179,7 @@ func (w *GCWorker) start(ctx context.Context, wg *sync.WaitGroup) { w.lastFinish = time.Now() if err != nil { logutil.Logger(ctx).Error("[gc worker] runGCJob", zap.Error(err)) - break + return } case <-ctx.Done(): logutil.Logger(ctx).Info("[gc worker] quit", zap.String("uuid", w.uuid)) diff --git a/store/tikv/lock_resolver.go b/store/tikv/lock_resolver.go index c59b269906326..cd33dd42364fe 100644 --- a/store/tikv/lock_resolver.go +++ b/store/tikv/lock_resolver.go @@ -358,7 +358,6 @@ func (t *txnExpireTime) update(lockExpire int64) { if lockExpire < t.txnExpire { t.txnExpire = lockExpire } - return } func (t *txnExpireTime) value() int64 { diff --git a/store/tikv/range_task.go b/store/tikv/range_task.go index 92c768c49faac..038e61585ec0d 100644 --- a/store/tikv/range_task.go +++ b/store/tikv/range_task.go @@ -267,12 +267,11 @@ type rangeTaskWorker struct { // run starts the worker. It collects all objects from `w.taskCh` and process them one by one. func (w *rangeTaskWorker) run(ctx context.Context, cancel context.CancelFunc) { defer w.wg.Done() - for r := range w.taskCh { select { case <-ctx.Done(): w.err = ctx.Err() - break + return default: } diff --git a/store/tikv/region_cache.go b/store/tikv/region_cache.go index 1f57855b28470..24f1ff1e32fd3 100644 --- a/store/tikv/region_cache.go +++ b/store/tikv/region_cache.go @@ -82,9 +82,7 @@ type RegionStore struct { // clone clones region store struct. func (r *RegionStore) clone() *RegionStore { storeFails := make([]uint32, len(r.stores)) - for i, e := range r.storeFails { - storeFails[i] = e - } + copy(storeFails, r.storeFails) return &RegionStore{ workTiFlashIdx: r.workTiFlashIdx, workTiKVIdx: r.workTiKVIdx, @@ -1150,7 +1148,6 @@ retry: if !r.compareAndSwapStore(oldRegionStore, newRegionStore) { goto retry } - return } // Contains checks whether the key is in the region, for the maximum region endKey is empty. @@ -1304,7 +1301,6 @@ retryMarkResolved: if !s.compareAndSwapState(oldState, newState) { goto retryMarkResolved } - return } func (s *Store) getResolveState() resolveState { diff --git a/store/tikv/region_request.go b/store/tikv/region_request.go index 55b15d088725c..6e101f766b6ca 100644 --- a/store/tikv/region_request.go +++ b/store/tikv/region_request.go @@ -20,8 +20,8 @@ import ( "time" "go.uber.org/zap" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -120,6 +120,10 @@ func (s *RegionRequestSender) SendReqCtx( rpcCtx, err = s.regionCache.GetTiKVRPCContext(bo, regionID, replicaRead, seed) case kv.TiFlash: rpcCtx, err = s.regionCache.GetTiFlashRPCContext(bo, regionID) + case kv.TiDB: + rpcCtx = &RPCContext{ + Addr: s.storeAddr, + } default: err = errors.Errorf("unsupported storage type: %v", sType) } @@ -218,7 +222,7 @@ func (s *RegionRequestSender) onSendFail(bo *Backoffer, ctx *RPCContext, err err } else if atomic.LoadUint32(&ShuttingDown) > 0 { return errTiDBShuttingDown } - if grpc.Code(errors.Cause(err)) == codes.Canceled { + if status.Code(errors.Cause(err)) == codes.Canceled { select { case <-bo.ctx.Done(): return errors.Trace(err) @@ -230,7 +234,9 @@ func (s *RegionRequestSender) onSendFail(bo *Backoffer, ctx *RPCContext, err err } } - s.regionCache.OnSendFail(bo, ctx, s.needReloadRegion(ctx), err) + if ctx.Meta != nil { + s.regionCache.OnSendFail(bo, ctx, s.needReloadRegion(ctx), err) + } // Retry on send request failure when it's not canceled. // When a store is not available, the leader of related region should be elected quickly. @@ -337,7 +343,11 @@ func (s *RegionRequestSender) onRegionError(bo *Backoffer, ctx *RPCContext, seed logutil.BgLogger().Debug("tikv reports region failed", zap.Stringer("regionErr", regionErr), zap.Stringer("ctx", ctx)) - s.regionCache.InvalidateCachedRegion(ctx.Region) + // When the request is sent to TiDB, there is no region in the request, so the region id will be 0. + // So when region id is 0, there is no business with region cache. + if ctx.Region.id != 0 { + s.regionCache.InvalidateCachedRegion(ctx.Region) + } return false, nil } diff --git a/store/tikv/snapshot.go b/store/tikv/snapshot.go index 0f47476f24f9e..3c8f8acd46bd1 100644 --- a/store/tikv/snapshot.go +++ b/store/tikv/snapshot.go @@ -214,7 +214,7 @@ func (s *tikvSnapshot) batchGetSingleRegion(bo *Backoffer, batch batchKeys, coll NotFillCache: s.notFillCache, }) - resp, _, _, err := cli.SendReqCtx(bo, req, batch.region, ReadTimeoutMedium, kv.TiKV) + resp, _, _, err := cli.SendReqCtx(bo, req, batch.region, ReadTimeoutMedium, kv.TiKV, "") if err != nil { return errors.Trace(err) @@ -323,7 +323,7 @@ func (s *tikvSnapshot) get(bo *Backoffer, k kv.Key) ([]byte, error) { if err != nil { return nil, errors.Trace(err) } - resp, _, _, err := cli.SendReqCtx(bo, req, loc.Region, readTimeoutShort, kv.TiKV) + resp, _, _, err := cli.SendReqCtx(bo, req, loc.Region, readTimeoutShort, kv.TiKV, "") if err != nil { return nil, errors.Trace(err) } diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 4e329d39f8022..65594c0e6950b 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -32,10 +32,7 @@ import ( ) func equalRegionStartKey(key, regionStartKey []byte) bool { - if bytes.Equal(key, regionStartKey) { - return true - } - return false + return bytes.Equal(key, regionStartKey) } func (s *tikvStore) splitBatchRegionsReq(bo *Backoffer, keys [][]byte, scatter bool) (*tikvrpc.Response, error) { diff --git a/store/tikv/tikvrpc/tikvrpc.go b/store/tikv/tikvrpc/tikvrpc.go index 986566e539230..6de639fce48d8 100644 --- a/store/tikv/tikvrpc/tikvrpc.go +++ b/store/tikv/tikvrpc/tikvrpc.go @@ -145,6 +145,7 @@ type Request struct { req interface{} kvrpcpb.Context ReplicaReadSeed uint32 + StoreTp kv.StoreType } // NewRequest returns new kv rpc request. @@ -461,8 +462,10 @@ type CopStreamResponse struct { // SetContext set the Context field for the given req to the specified ctx. func SetContext(req *Request, region *metapb.Region, peer *metapb.Peer) error { ctx := &req.Context - ctx.RegionId = region.Id - ctx.RegionEpoch = region.RegionEpoch + if region != nil { + ctx.RegionId = region.Id + ctx.RegionEpoch = region.RegionEpoch + } ctx.Peer = peer switch req.Type { diff --git a/store/tikv/txn.go b/store/tikv/txn.go index 5f8ce931f2057..eb878fd78fe54 100644 --- a/store/tikv/txn.go +++ b/store/tikv/txn.go @@ -310,7 +310,7 @@ func (txn *tikvTxn) Commit(ctx context.Context) error { // latches disabled // pessimistic transaction should also bypass latch. if txn.store.txnLatches == nil || txn.IsPessimistic() { - err = committer.executeAndWriteFinishBinlog(ctx) + err = committer.execute(ctx) logutil.Logger(ctx).Debug("[kv] txnLatches disabled, 2pc directly", zap.Error(err)) return errors.Trace(err) } @@ -328,7 +328,7 @@ func (txn *tikvTxn) Commit(ctx context.Context) error { if lock.IsStale() { return kv.ErrWriteConflictInTiDB.FastGenByArgs(txn.startTS) } - err = committer.executeAndWriteFinishBinlog(ctx) + err = committer.execute(ctx) if err == nil { lock.SetCommitTS(committer.commitTS) } diff --git a/structure/structure.go b/structure/structure.go index 3aba2214550a9..b736ec9daf981 100644 --- a/structure/structure.go +++ b/structure/structure.go @@ -20,11 +20,9 @@ import ( // structure error codes. const ( - codeInvalidHashKeyFlag terror.ErrCode = 1 - codeInvalidHashKeyPrefix terror.ErrCode = 2 - codeInvalidListIndex terror.ErrCode = 3 - codeInvalidListMetaData terror.ErrCode = 4 - codeWriteOnSnapshot terror.ErrCode = 5 + codeInvalidHashKeyFlag terror.ErrCode = 1 + codeInvalidListMetaData terror.ErrCode = 4 + codeWriteOnSnapshot terror.ErrCode = 5 ) var ( diff --git a/table/column.go b/table/column.go index c20b5a41ec025..32f439266d7f1 100644 --- a/table/column.go +++ b/table/column.go @@ -154,7 +154,7 @@ func CastValues(ctx sessionctx.Context, rec []types.Datum, cols []*Column) (err func handleWrongUtf8Value(ctx sessionctx.Context, col *model.ColumnInfo, casted *types.Datum, str string, i int) (types.Datum, error) { sc := ctx.GetSessionVars().StmtCtx - err := ErrTruncateWrongValue.FastGen("incorrect utf8 value %x(%s) for column %s", casted.GetBytes(), str, col.Name) + err := ErrTruncatedWrongValueForField.FastGen("incorrect utf8 value %x(%s) for column %s", casted.GetBytes(), str, col.Name) logutil.BgLogger().Error("incorrect UTF-8 value", zap.Uint64("conn", ctx.GetSessionVars().ConnectionID), zap.Error(err)) // Truncate to valid utf8 string. truncateVal := types.NewStringDatum(str[:i]) @@ -324,7 +324,7 @@ func CheckOnce(cols []*Column) error { name := col.Name _, ok := m[name.L] if ok { - return errDuplicateColumn.GenWithStack("column specified twice - %s", name) + return errDuplicateColumn.GenWithStackByArgs(name) } m[name.L] = struct{}{} @@ -416,8 +416,7 @@ func getColDefaultValue(ctx sessionctx.Context, col *model.ColumnInfo, defaultVa } value, err := expression.GetTimeValue(ctx, defaultVal, col.Tp, int8(col.Decimal)) if err != nil { - return types.Datum{}, errGetDefaultFailed.GenWithStack("Field '%s' get default value fail - %s", - col.Name, err) + return types.Datum{}, errGetDefaultFailed.GenWithStackByArgs(col.Name) } // If the column's default value is not ZeroDatetimeStr or CurrentTimestamp, convert the default value to the current session time zone. if needChangeTimeZone { diff --git a/table/table.go b/table/table.go index a467cb641dbf2..4c6d93568be53 100644 --- a/table/table.go +++ b/table/table.go @@ -38,8 +38,25 @@ const ( NormalTable Type = iota // VirtualTable , store no data, just extract data from the memory struct. VirtualTable + // ClusterTable , contain the `VirtualTable` in the all cluster tidb nodes. + ClusterTable ) +// IsNormalTable checks whether the table is a normal table type. +func (tp Type) IsNormalTable() bool { + return tp == NormalTable +} + +// IsVirtualTable checks whether the table is a virtual table type. +func (tp Type) IsVirtualTable() bool { + return tp == VirtualTable +} + +// IsClusterTable checks whether the table is a cluster table type. +func (tp Type) IsClusterTable() bool { + return tp == ClusterTable +} + const ( // DirtyTableAddRow is the constant for dirty table operation type. DirtyTableAddRow = iota @@ -49,41 +66,39 @@ const ( var ( // ErrColumnCantNull is used for inserting null to a not null column. - ErrColumnCantNull = terror.ClassTable.New(codeColumnCantNull, mysql.MySQLErrName[mysql.ErrBadNull]) - errUnknownColumn = terror.ClassTable.New(codeUnknownColumn, "unknown column") - errDuplicateColumn = terror.ClassTable.New(codeDuplicateColumn, "duplicate column") + ErrColumnCantNull = terror.ClassTable.New(mysql.ErrBadNull, mysql.MySQLErrName[mysql.ErrBadNull]) + errUnknownColumn = terror.ClassTable.New(mysql.ErrBadField, mysql.MySQLErrName[mysql.ErrBadField]) + errDuplicateColumn = terror.ClassTable.New(mysql.ErrFieldSpecifiedTwice, mysql.MySQLErrName[mysql.ErrFieldSpecifiedTwice]) - errGetDefaultFailed = terror.ClassTable.New(codeGetDefaultFailed, "get default value fail") + errGetDefaultFailed = terror.ClassTable.New(mysql.ErrFieldGetDefaultFailed, mysql.MySQLErrName[mysql.ErrFieldGetDefaultFailed]) // ErrNoDefaultValue is used when insert a row, the column value is not given, and the column has not null flag // and it doesn't have a default value. - ErrNoDefaultValue = terror.ClassTable.New(codeNoDefaultValue, mysql.MySQLErrName[mysql.ErrNoDefaultForField]) + ErrNoDefaultValue = terror.ClassTable.New(mysql.ErrNoDefaultForField, mysql.MySQLErrName[mysql.ErrNoDefaultForField]) // ErrIndexOutBound returns for index column offset out of bound. - ErrIndexOutBound = terror.ClassTable.New(codeIndexOutBound, "index column offset out of bound") + ErrIndexOutBound = terror.ClassTable.New(mysql.ErrIndexOutBound, mysql.MySQLErrName[mysql.ErrIndexOutBound]) // ErrUnsupportedOp returns for unsupported operation. - ErrUnsupportedOp = terror.ClassTable.New(codeUnsupportedOp, "operation not supported") + ErrUnsupportedOp = terror.ClassTable.New(mysql.ErrUnsupportedOp, mysql.MySQLErrName[mysql.ErrUnsupportedOp]) // ErrRowNotFound returns for row not found. - ErrRowNotFound = terror.ClassTable.New(codeRowNotFound, "can not find the row") + ErrRowNotFound = terror.ClassTable.New(mysql.ErrRowNotFound, mysql.MySQLErrName[mysql.ErrRowNotFound]) // ErrTableStateCantNone returns for table none state. - ErrTableStateCantNone = terror.ClassTable.New(codeTableStateCantNone, "table can not be in none state") + ErrTableStateCantNone = terror.ClassTable.New(mysql.ErrTableStateCantNone, mysql.MySQLErrName[mysql.ErrTableStateCantNone]) // ErrColumnStateCantNone returns for column none state. - ErrColumnStateCantNone = terror.ClassTable.New(codeColumnStateCantNone, "column can not be in none state") + ErrColumnStateCantNone = terror.ClassTable.New(mysql.ErrColumnStateCantNone, mysql.MySQLErrName[mysql.ErrColumnStateCantNone]) // ErrColumnStateNonPublic returns for column non-public state. - ErrColumnStateNonPublic = terror.ClassTable.New(codeColumnStateNonPublic, "can not use non-public column") + ErrColumnStateNonPublic = terror.ClassTable.New(mysql.ErrColumnStateNonPublic, mysql.MySQLErrName[mysql.ErrColumnStateNonPublic]) // ErrIndexStateCantNone returns for index none state. - ErrIndexStateCantNone = terror.ClassTable.New(codeIndexStateCantNone, "index can not be in none state") + ErrIndexStateCantNone = terror.ClassTable.New(mysql.ErrIndexStateCantNone, mysql.MySQLErrName[mysql.ErrIndexStateCantNone]) // ErrInvalidRecordKey returns for invalid record key. - ErrInvalidRecordKey = terror.ClassTable.New(codeInvalidRecordKey, "invalid record key") - // ErrTruncateWrongValue returns for truncate wrong value for field. - ErrTruncateWrongValue = terror.ClassTable.New(codeTruncateWrongValue, "incorrect value") + ErrInvalidRecordKey = terror.ClassTable.New(mysql.ErrInvalidRecordKey, mysql.MySQLErrName[mysql.ErrInvalidRecordKey]) // ErrTruncatedWrongValueForField returns for truncate wrong value for field. - ErrTruncatedWrongValueForField = terror.ClassTable.New(codeTruncateWrongValue, mysql.MySQLErrName[mysql.ErrTruncatedWrongValueForField]) + ErrTruncatedWrongValueForField = terror.ClassTable.New(mysql.ErrTruncatedWrongValueForField, mysql.MySQLErrName[mysql.ErrTruncatedWrongValueForField]) // ErrUnknownPartition returns unknown partition error. - ErrUnknownPartition = terror.ClassTable.New(codeUnknownPartition, mysql.MySQLErrName[mysql.ErrUnknownPartition]) + ErrUnknownPartition = terror.ClassTable.New(mysql.ErrUnknownPartition, mysql.MySQLErrName[mysql.ErrUnknownPartition]) // ErrNoPartitionForGivenValue returns table has no partition for value. - ErrNoPartitionForGivenValue = terror.ClassTable.New(codeNoPartitionForGivenValue, mysql.MySQLErrName[mysql.ErrNoPartitionForGivenValue]) + ErrNoPartitionForGivenValue = terror.ClassTable.New(mysql.ErrNoPartitionForGivenValue, mysql.MySQLErrName[mysql.ErrNoPartitionForGivenValue]) // ErrLockOrActiveTransaction returns when execute unsupported statement in a lock session or an active transaction. - ErrLockOrActiveTransaction = terror.ClassTable.New(codeLockOrActiveTransaction, mysql.MySQLErrName[mysql.ErrLockOrActiveTransaction]) + ErrLockOrActiveTransaction = terror.ClassTable.New(mysql.ErrLockOrActiveTransaction, mysql.MySQLErrName[mysql.ErrLockOrActiveTransaction]) ) // RecordIterFunc is used for low-level record iteration. @@ -166,7 +181,7 @@ type Table interface { // AllocHandle allocates a handle for a new row. AllocHandle(ctx sessionctx.Context) (int64, error) - // AllocHandleIds allocates multiple handle for rows. + // AllocHandleIDs allocates multiple handle for rows. AllocHandleIDs(ctx sessionctx.Context, n uint64) (int64, int64, error) // Allocator returns Allocator. @@ -232,31 +247,6 @@ var TableFromMeta func(alloc autoid.Allocator, tblInfo *model.TableInfo) (Table, // MockTableFromMeta only serves for test. var MockTableFromMeta func(tableInfo *model.TableInfo) Table -// Table error codes. -const ( - codeGetDefaultFailed = 1 - codeIndexOutBound = 2 - codeUnsupportedOp = 3 - codeRowNotFound = 4 - codeTableStateCantNone = 5 - codeColumnStateCantNone = 6 - codeColumnStateNonPublic = 7 - codeIndexStateCantNone = 8 - codeInvalidRecordKey = 9 - - codeColumnCantNull = mysql.ErrBadNull - codeUnknownColumn = 1054 - codeDuplicateColumn = 1110 - codeNoDefaultValue = 1364 - codeTruncateWrongValue = 1366 - // MySQL error code, "Trigger creation context of table `%-.64s`.`%-.64s` is invalid". - // It may happen when inserting some data outside of all table partitions. - - codeUnknownPartition = mysql.ErrUnknownPartition - codeNoPartitionForGivenValue = mysql.ErrNoPartitionForGivenValue - codeLockOrActiveTransaction = mysql.ErrLockOrActiveTransaction -) - // Slice is used for table sorting. type Slice []Table @@ -270,14 +260,23 @@ func (s Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func init() { tableMySQLErrCodes := map[terror.ErrCode]uint16{ - codeColumnCantNull: mysql.ErrBadNull, - codeUnknownColumn: mysql.ErrBadField, - codeDuplicateColumn: mysql.ErrFieldSpecifiedTwice, - codeNoDefaultValue: mysql.ErrNoDefaultForField, - codeTruncateWrongValue: mysql.ErrTruncatedWrongValueForField, - codeUnknownPartition: mysql.ErrUnknownPartition, - codeNoPartitionForGivenValue: mysql.ErrNoPartitionForGivenValue, - codeLockOrActiveTransaction: mysql.ErrLockOrActiveTransaction, + mysql.ErrBadNull: mysql.ErrBadNull, + mysql.ErrBadField: mysql.ErrBadField, + mysql.ErrFieldSpecifiedTwice: mysql.ErrFieldSpecifiedTwice, + mysql.ErrNoDefaultForField: mysql.ErrNoDefaultForField, + mysql.ErrTruncatedWrongValueForField: mysql.ErrTruncatedWrongValueForField, + mysql.ErrUnknownPartition: mysql.ErrUnknownPartition, + mysql.ErrNoPartitionForGivenValue: mysql.ErrNoPartitionForGivenValue, + mysql.ErrLockOrActiveTransaction: mysql.ErrLockOrActiveTransaction, + mysql.ErrIndexOutBound: mysql.ErrIndexOutBound, + mysql.ErrColumnStateNonPublic: mysql.ErrColumnStateNonPublic, + mysql.ErrFieldGetDefaultFailed: mysql.ErrFieldGetDefaultFailed, + mysql.ErrUnsupportedOp: mysql.ErrUnsupportedOp, + mysql.ErrRowNotFound: mysql.ErrRowNotFound, + mysql.ErrTableStateCantNone: mysql.ErrTableStateCantNone, + mysql.ErrColumnStateCantNone: mysql.ErrColumnStateCantNone, + mysql.ErrIndexStateCantNone: mysql.ErrIndexStateCantNone, + mysql.ErrInvalidRecordKey: mysql.ErrInvalidRecordKey, } terror.ErrClassToMySQLCodes[terror.ClassTable] = tableMySQLErrCodes } diff --git a/table/table_test.go b/table/table_test.go index e6e2da0e5f7e8..79d40680c2948 100644 --- a/table/table_test.go +++ b/table/table_test.go @@ -15,6 +15,7 @@ package table import ( . "github.com/pingcap/check" + "github.com/pingcap/parser/mysql" ) var _ = Suite(&testTableSuite{}) @@ -27,3 +28,23 @@ func (t *testTableSuite) TestSlice(c *C) { c.Assert(length, Equals, 2) sl.Swap(0, 1) } + +func (t *testTableSuite) TestErrorCode(c *C) { + c.Assert(int(ErrColumnCantNull.ToSQLError().Code), Equals, mysql.ErrBadNull) + c.Assert(int(errUnknownColumn.ToSQLError().Code), Equals, mysql.ErrBadField) + c.Assert(int(errDuplicateColumn.ToSQLError().Code), Equals, mysql.ErrFieldSpecifiedTwice) + c.Assert(int(errGetDefaultFailed.ToSQLError().Code), Equals, mysql.ErrFieldGetDefaultFailed) + c.Assert(int(ErrNoDefaultValue.ToSQLError().Code), Equals, mysql.ErrNoDefaultForField) + c.Assert(int(ErrIndexOutBound.ToSQLError().Code), Equals, mysql.ErrIndexOutBound) + c.Assert(int(ErrUnsupportedOp.ToSQLError().Code), Equals, mysql.ErrUnsupportedOp) + c.Assert(int(ErrRowNotFound.ToSQLError().Code), Equals, mysql.ErrRowNotFound) + c.Assert(int(ErrTableStateCantNone.ToSQLError().Code), Equals, mysql.ErrTableStateCantNone) + c.Assert(int(ErrColumnStateCantNone.ToSQLError().Code), Equals, mysql.ErrColumnStateCantNone) + c.Assert(int(ErrColumnStateNonPublic.ToSQLError().Code), Equals, mysql.ErrColumnStateNonPublic) + c.Assert(int(ErrIndexStateCantNone.ToSQLError().Code), Equals, mysql.ErrIndexStateCantNone) + c.Assert(int(ErrInvalidRecordKey.ToSQLError().Code), Equals, mysql.ErrInvalidRecordKey) + c.Assert(int(ErrTruncatedWrongValueForField.ToSQLError().Code), Equals, mysql.ErrTruncatedWrongValueForField) + c.Assert(int(ErrUnknownPartition.ToSQLError().Code), Equals, mysql.ErrUnknownPartition) + c.Assert(int(ErrNoPartitionForGivenValue.ToSQLError().Code), Equals, mysql.ErrNoPartitionForGivenValue) + c.Assert(int(ErrLockOrActiveTransaction.ToSQLError().Code), Equals, mysql.ErrLockOrActiveTransaction) +} diff --git a/table/tables/index.go b/table/tables/index.go index 3290d6b8b4b16..b77422b2ec505 100644 --- a/table/tables/index.go +++ b/table/tables/index.go @@ -373,8 +373,7 @@ func (c *index) FetchValues(r []types.Datum, vals []types.Datum) ([]types.Datum, vals = vals[:needLength] for i, ic := range c.idxInfo.Columns { if ic.Offset < 0 || ic.Offset >= len(r) { - return nil, table.ErrIndexOutBound.GenWithStack("Index column %s offset out of bound, offset: %d, row: %v", - ic.Name, ic.Offset, r) + return nil, table.ErrIndexOutBound.GenWithStackByArgs(ic.Name, ic.Offset, r) } vals[i] = r[ic.Offset] } diff --git a/table/tables/partition.go b/table/tables/partition.go index 878d81a97c58a..f3aa4629ae858 100644 --- a/table/tables/partition.go +++ b/table/tables/partition.go @@ -47,7 +47,7 @@ var _ table.PartitionedTable = &partitionedTable{} // partitions) is basically the same. // partition also implements the table.Table interface. type partition struct { - tableCommon + TableCommon } // GetPhysicalID implements table.Table GetPhysicalID interface. @@ -58,27 +58,27 @@ func (p *partition) GetPhysicalID() int64 { // partitionedTable implements the table.PartitionedTable interface. // partitionedTable is a table, it contains many Partitions. type partitionedTable struct { - Table + TableCommon partitionExpr *PartitionExpr partitions map[int64]*partition } -func newPartitionedTable(tbl *Table, tblInfo *model.TableInfo) (table.Table, error) { - ret := &partitionedTable{Table: *tbl} +func newPartitionedTable(tbl *TableCommon, tblInfo *model.TableInfo) (table.Table, error) { + ret := &partitionedTable{TableCommon: *tbl} partitionExpr, err := newPartitionExpr(tblInfo) if err != nil { return nil, errors.Trace(err) } ret.partitionExpr = partitionExpr - if err := initTableIndices(&ret.tableCommon); err != nil { + if err := initTableIndices(&ret.TableCommon); err != nil { return nil, errors.Trace(err) } partitions := make(map[int64]*partition) pi := tblInfo.GetPartitionInfo() for _, p := range pi.Definitions { var t partition - err := initTableCommonWithIndices(&t.tableCommon, tblInfo, p.ID, tbl.Columns, tbl.alloc) + err := initTableCommonWithIndices(&t.TableCommon, tblInfo, p.ID, tbl.Columns, tbl.alloc) if err != nil { return nil, errors.Trace(err) } diff --git a/table/tables/tables.go b/table/tables/tables.go index c569986078180..dda9f95cf9d07 100644 --- a/table/tables/tables.go +++ b/table/tables/tables.go @@ -43,8 +43,8 @@ import ( "go.uber.org/zap" ) -// tableCommon is shared by both Table and partition. -type tableCommon struct { +// TableCommon is shared by both Table and partition. +type TableCommon struct { tableID int64 // physicalTableID is a unique int64 to identify a physical table. physicalTableID int64 @@ -61,13 +61,6 @@ type tableCommon struct { indexPrefix kv.Key } -// Table implements table.Table interface. -type Table struct { - tableCommon -} - -var _ table.Table = &Table{} - // MockTableFromMeta only serves for test. func MockTableFromMeta(tblInfo *model.TableInfo) table.Table { columns := make([]*table.Column, 0, len(tblInfo.Columns)) @@ -76,10 +69,10 @@ func MockTableFromMeta(tblInfo *model.TableInfo) table.Table { columns = append(columns, col) } - var t Table - initTableCommon(&t.tableCommon, tblInfo, tblInfo.ID, columns, nil) + var t TableCommon + initTableCommon(&t, tblInfo, tblInfo.ID, columns, nil) if tblInfo.GetPartitionInfo() == nil { - if err := initTableIndices(&t.tableCommon); err != nil { + if err := initTableIndices(&t); err != nil { return nil } return &t @@ -95,14 +88,14 @@ func MockTableFromMeta(tblInfo *model.TableInfo) table.Table { // TableFromMeta creates a Table instance from model.TableInfo. func TableFromMeta(alloc autoid.Allocator, tblInfo *model.TableInfo) (table.Table, error) { if tblInfo.State == model.StateNone { - return nil, table.ErrTableStateCantNone.GenWithStack("table %s can't be in none state", tblInfo.Name) + return nil, table.ErrTableStateCantNone.GenWithStackByArgs(tblInfo.Name) } colsLen := len(tblInfo.Columns) columns := make([]*table.Column, 0, colsLen) for i, colInfo := range tblInfo.Columns { if colInfo.State == model.StateNone { - return nil, table.ErrColumnStateCantNone.GenWithStack("column %s can't be in none state", colInfo.Name) + return nil, table.ErrColumnStateCantNone.GenWithStackByArgs(colInfo.Name) } // Print some information when the column's offset isn't equal to i. @@ -125,10 +118,10 @@ func TableFromMeta(alloc autoid.Allocator, tblInfo *model.TableInfo) (table.Tabl columns = append(columns, col) } - var t Table - initTableCommon(&t.tableCommon, tblInfo, tblInfo.ID, columns, alloc) + var t TableCommon + initTableCommon(&t, tblInfo, tblInfo.ID, columns, alloc) if tblInfo.GetPartitionInfo() == nil { - if err := initTableIndices(&t.tableCommon); err != nil { + if err := initTableIndices(&t); err != nil { return nil, err } return &t, nil @@ -137,8 +130,8 @@ func TableFromMeta(alloc autoid.Allocator, tblInfo *model.TableInfo) (table.Tabl return newPartitionedTable(&t, tblInfo) } -// initTableCommon initializes a tableCommon struct. -func initTableCommon(t *tableCommon, tblInfo *model.TableInfo, physicalTableID int64, cols []*table.Column, alloc autoid.Allocator) { +// initTableCommon initializes a TableCommon struct. +func initTableCommon(t *TableCommon, tblInfo *model.TableInfo, physicalTableID int64, cols []*table.Column, alloc autoid.Allocator) { t.tableID = tblInfo.ID t.physicalTableID = physicalTableID t.alloc = alloc @@ -151,33 +144,33 @@ func initTableCommon(t *tableCommon, tblInfo *model.TableInfo, physicalTableID i t.indexPrefix = tablecodec.GenTableIndexPrefix(physicalTableID) } -// initTableIndices initializes the indices of the tableCommon. -func initTableIndices(t *tableCommon) error { +// initTableIndices initializes the indices of the TableCommon. +func initTableIndices(t *TableCommon) error { tblInfo := t.meta for _, idxInfo := range tblInfo.Indices { if idxInfo.State == model.StateNone { - return table.ErrIndexStateCantNone.GenWithStack("index %s can't be in none state", idxInfo.Name) + return table.ErrIndexStateCantNone.GenWithStackByArgs(idxInfo.Name) } - // Use partition ID for index, because tableCommon may be table or partition. + // Use partition ID for index, because TableCommon may be table or partition. idx := NewIndex(t.physicalTableID, tblInfo, idxInfo) t.indices = append(t.indices, idx) } return nil } -func initTableCommonWithIndices(t *tableCommon, tblInfo *model.TableInfo, physicalTableID int64, cols []*table.Column, alloc autoid.Allocator) error { +func initTableCommonWithIndices(t *TableCommon, tblInfo *model.TableInfo, physicalTableID int64, cols []*table.Column, alloc autoid.Allocator) error { initTableCommon(t, tblInfo, physicalTableID, cols, alloc) return initTableIndices(t) } // Indices implements table.Table Indices interface. -func (t *tableCommon) Indices() []table.Index { +func (t *TableCommon) Indices() []table.Index { return t.indices } // WritableIndices implements table.Table WritableIndices interface. -func (t *tableCommon) WritableIndices() []table.Index { +func (t *TableCommon) WritableIndices() []table.Index { if len(t.writableIndices) > 0 { return t.writableIndices } @@ -192,23 +185,23 @@ func (t *tableCommon) WritableIndices() []table.Index { } // DeletableIndices implements table.Table DeletableIndices interface. -func (t *tableCommon) DeletableIndices() []table.Index { +func (t *TableCommon) DeletableIndices() []table.Index { // All indices are deletable because we don't need to check StateNone. return t.indices } // Meta implements table.Table Meta interface. -func (t *tableCommon) Meta() *model.TableInfo { +func (t *TableCommon) Meta() *model.TableInfo { return t.meta } // GetPhysicalID implements table.Table GetPhysicalID interface. -func (t *Table) GetPhysicalID() int64 { +func (t *TableCommon) GetPhysicalID() int64 { return t.physicalTableID } // Cols implements table.Table Cols interface. -func (t *tableCommon) Cols() []*table.Column { +func (t *TableCommon) Cols() []*table.Column { if len(t.publicColumns) > 0 { return t.publicColumns } @@ -227,7 +220,7 @@ func (t *tableCommon) Cols() []*table.Column { } // WritableCols implements table WritableCols interface. -func (t *tableCommon) WritableCols() []*table.Column { +func (t *TableCommon) WritableCols() []*table.Column { if len(t.writableColumns) > 0 { return t.writableColumns } @@ -246,29 +239,29 @@ func (t *tableCommon) WritableCols() []*table.Column { } // RecordPrefix implements table.Table interface. -func (t *tableCommon) RecordPrefix() kv.Key { +func (t *TableCommon) RecordPrefix() kv.Key { return t.recordPrefix } // IndexPrefix implements table.Table interface. -func (t *tableCommon) IndexPrefix() kv.Key { +func (t *TableCommon) IndexPrefix() kv.Key { return t.indexPrefix } // RecordKey implements table.Table interface. -func (t *tableCommon) RecordKey(h int64) kv.Key { +func (t *TableCommon) RecordKey(h int64) kv.Key { return tablecodec.EncodeRecordKey(t.recordPrefix, h) } // FirstKey implements table.Table interface. -func (t *tableCommon) FirstKey() kv.Key { +func (t *TableCommon) FirstKey() kv.Key { return t.RecordKey(math.MinInt64) } // UpdateRecord implements table.Table UpdateRecord interface. // `touched` means which columns are really modified, used for secondary indices. // Length of `oldData` and `newData` equals to length of `t.WritableCols()`. -func (t *tableCommon) UpdateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datum, touched []bool) error { +func (t *TableCommon) UpdateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datum, touched []bool) error { txn, err := ctx.Txn(true) if err != nil { return err @@ -359,7 +352,7 @@ func (t *tableCommon) UpdateRecord(ctx sessionctx.Context, h int64, oldData, new return nil } -func (t *tableCommon) rebuildIndices(ctx sessionctx.Context, rm kv.RetrieverMutator, h int64, touched []bool, oldData []types.Datum, newData []types.Datum) error { +func (t *TableCommon) rebuildIndices(ctx sessionctx.Context, rm kv.RetrieverMutator, h int64, touched []bool, oldData []types.Datum, newData []types.Datum) error { txn, err := ctx.Txn(true) if err != nil { return err @@ -416,7 +409,7 @@ func adjustRowValuesBuf(writeBufs *variable.WriteStmtBufs, rowLen int) { // getRollbackableMemStore get a rollbackable BufferStore, when we are importing data, // Just add the kv to transaction's membuf directly. -func (t *tableCommon) getRollbackableMemStore(ctx sessionctx.Context) (kv.RetrieverMutator, error) { +func (t *TableCommon) getRollbackableMemStore(ctx sessionctx.Context) (kv.RetrieverMutator, error) { bs := ctx.GetSessionVars().GetWriteStmtBufs().BufStore if bs == nil { txn, err := ctx.Txn(true) @@ -431,7 +424,7 @@ func (t *tableCommon) getRollbackableMemStore(ctx sessionctx.Context) (kv.Retrie } // AddRecord implements table.Table AddRecord interface. -func (t *tableCommon) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...table.AddRecordOption) (recordID int64, err error) { +func (t *TableCommon) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...table.AddRecordOption) (recordID int64, err error) { var opt table.AddRecordOpt for _, fn := range opts { fn.ApplyOn(&opt) @@ -571,7 +564,7 @@ func (t *tableCommon) AddRecord(ctx sessionctx.Context, r []types.Datum, opts .. } // genIndexKeyStr generates index content string representation. -func (t *tableCommon) genIndexKeyStr(colVals []types.Datum) (string, error) { +func (t *TableCommon) genIndexKeyStr(colVals []types.Datum) (string, error) { // Pass pre-composed error to txn. strVals := make([]string, 0, len(colVals)) for _, cv := range colVals { @@ -589,7 +582,7 @@ func (t *tableCommon) genIndexKeyStr(colVals []types.Datum) (string, error) { } // addIndices adds data into indices. If any key is duplicated, returns the original handle. -func (t *tableCommon) addIndices(sctx sessionctx.Context, recordID int64, r []types.Datum, rm kv.RetrieverMutator, +func (t *TableCommon) addIndices(sctx sessionctx.Context, recordID int64, r []types.Datum, rm kv.RetrieverMutator, opts []table.CreateIdxOptFunc) (int64, error) { txn, err := sctx.Txn(true) if err != nil { @@ -645,7 +638,7 @@ func (t *tableCommon) addIndices(sctx sessionctx.Context, recordID int64, r []ty } // RowWithCols implements table.Table RowWithCols interface. -func (t *tableCommon) RowWithCols(ctx sessionctx.Context, h int64, cols []*table.Column) ([]types.Datum, error) { +func (t *TableCommon) RowWithCols(ctx sessionctx.Context, h int64, cols []*table.Column) ([]types.Datum, error) { // Get raw row data from kv. key := t.RecordKey(h) txn, err := ctx.Txn(true) @@ -708,12 +701,12 @@ func DecodeRawRowData(ctx sessionctx.Context, meta *model.TableInfo, h int64, co } // Row implements table.Table Row interface. -func (t *tableCommon) Row(ctx sessionctx.Context, h int64) ([]types.Datum, error) { +func (t *TableCommon) Row(ctx sessionctx.Context, h int64) ([]types.Datum, error) { return t.RowWithCols(ctx, h, t.Cols()) } // RemoveRecord implements table.Table RemoveRecord interface. -func (t *tableCommon) RemoveRecord(ctx sessionctx.Context, h int64, r []types.Datum) error { +func (t *TableCommon) RemoveRecord(ctx sessionctx.Context, h int64, r []types.Datum) error { err := t.removeRowData(ctx, h) if err != nil { return err @@ -754,7 +747,7 @@ func (t *tableCommon) RemoveRecord(ctx sessionctx.Context, h int64, r []types.Da return err } -func (t *tableCommon) addInsertBinlog(ctx sessionctx.Context, h int64, row []types.Datum, colIDs []int64) error { +func (t *TableCommon) addInsertBinlog(ctx sessionctx.Context, h int64, row []types.Datum, colIDs []int64) error { mutation := t.getMutation(ctx) pk, err := codec.EncodeValue(ctx.GetSessionVars().StmtCtx, nil, types.NewIntDatum(h)) if err != nil { @@ -770,7 +763,7 @@ func (t *tableCommon) addInsertBinlog(ctx sessionctx.Context, h int64, row []typ return nil } -func (t *tableCommon) addUpdateBinlog(ctx sessionctx.Context, oldRow, newRow []types.Datum, colIDs []int64) error { +func (t *TableCommon) addUpdateBinlog(ctx sessionctx.Context, oldRow, newRow []types.Datum, colIDs []int64) error { old, err := tablecodec.EncodeRow(ctx.GetSessionVars().StmtCtx, oldRow, colIDs, nil, nil) if err != nil { return err @@ -786,7 +779,7 @@ func (t *tableCommon) addUpdateBinlog(ctx sessionctx.Context, oldRow, newRow []t return nil } -func (t *tableCommon) addDeleteBinlog(ctx sessionctx.Context, r []types.Datum, colIDs []int64) error { +func (t *TableCommon) addDeleteBinlog(ctx sessionctx.Context, r []types.Datum, colIDs []int64) error { data, err := tablecodec.EncodeRow(ctx.GetSessionVars().StmtCtx, r, colIDs, nil, nil) if err != nil { return err @@ -797,7 +790,7 @@ func (t *tableCommon) addDeleteBinlog(ctx sessionctx.Context, r []types.Datum, c return nil } -func (t *tableCommon) removeRowData(ctx sessionctx.Context, h int64) error { +func (t *TableCommon) removeRowData(ctx sessionctx.Context, h int64) error { // Remove row data. txn, err := ctx.Txn(true) if err != nil { @@ -813,7 +806,7 @@ func (t *tableCommon) removeRowData(ctx sessionctx.Context, h int64) error { } // removeRowIndices removes all the indices of a row. -func (t *tableCommon) removeRowIndices(ctx sessionctx.Context, h int64, rec []types.Datum) error { +func (t *TableCommon) removeRowIndices(ctx sessionctx.Context, h int64, rec []types.Datum) error { txn, err := ctx.Txn(true) if err != nil { return err @@ -838,12 +831,12 @@ func (t *tableCommon) removeRowIndices(ctx sessionctx.Context, h int64, rec []ty } // removeRowIndex implements table.Table RemoveRowIndex interface. -func (t *tableCommon) removeRowIndex(sc *stmtctx.StatementContext, rm kv.RetrieverMutator, h int64, vals []types.Datum, idx table.Index, txn kv.Transaction) error { +func (t *TableCommon) removeRowIndex(sc *stmtctx.StatementContext, rm kv.RetrieverMutator, h int64, vals []types.Datum, idx table.Index, txn kv.Transaction) error { return idx.Delete(sc, rm, vals, h) } // buildIndexForRow implements table.Table BuildIndexForRow interface. -func (t *tableCommon) buildIndexForRow(ctx sessionctx.Context, rm kv.RetrieverMutator, h int64, vals []types.Datum, idx table.Index, txn kv.Transaction, untouched bool) error { +func (t *TableCommon) buildIndexForRow(ctx sessionctx.Context, rm kv.RetrieverMutator, h int64, vals []types.Datum, idx table.Index, txn kv.Transaction, untouched bool) error { var opts []table.CreateIdxOptFunc if untouched { opts = append(opts, table.IndexIsUntouched) @@ -865,7 +858,7 @@ func (t *tableCommon) buildIndexForRow(ctx sessionctx.Context, rm kv.RetrieverMu } // IterRecords implements table.Table IterRecords interface. -func (t *tableCommon) IterRecords(ctx sessionctx.Context, startKey kv.Key, cols []*table.Column, +func (t *TableCommon) IterRecords(ctx sessionctx.Context, startKey kv.Key, cols []*table.Column, fn table.RecordIterFunc) error { prefix := t.RecordPrefix() txn, err := ctx.Txn(true) @@ -960,13 +953,13 @@ func GetColDefaultValue(ctx sessionctx.Context, col *table.Column, defaultVals [ } // AllocHandle implements table.Table AllocHandle interface. -func (t *tableCommon) AllocHandle(ctx sessionctx.Context) (int64, error) { +func (t *TableCommon) AllocHandle(ctx sessionctx.Context) (int64, error) { _, rowID, err := t.AllocHandleIDs(ctx, 1) return rowID, err } -// AllocHandle implements table.Table AllocHandle interface. -func (t *tableCommon) AllocHandleIDs(ctx sessionctx.Context, n uint64) (int64, int64, error) { +// AllocHandleIDs implements table.Table AllocHandle interface. +func (t *TableCommon) AllocHandleIDs(ctx sessionctx.Context, n uint64) (int64, int64, error) { base, maxID, err := t.Allocator(ctx).Alloc(t.tableID, n) if err != nil { return 0, 0, err @@ -1000,7 +993,7 @@ func OverflowShardBits(rowID int64, shardRowIDBits uint64) bool { return rowID&int64(mask) > 0 } -func (t *tableCommon) calcShard(startTS uint64) int64 { +func (t *TableCommon) calcShard(startTS uint64) int64 { var buf [8]byte binary.LittleEndian.PutUint64(buf[:], startTS) hashVal := int64(murmur3.Sum32(buf[:])) @@ -1008,7 +1001,7 @@ func (t *tableCommon) calcShard(startTS uint64) int64 { } // Allocator implements table.Table Allocator interface. -func (t *tableCommon) Allocator(ctx sessionctx.Context) autoid.Allocator { +func (t *TableCommon) Allocator(ctx sessionctx.Context) autoid.Allocator { if ctx != nil { sessAlloc := ctx.GetSessionVars().IDAllocator if sessAlloc != nil { @@ -1019,12 +1012,12 @@ func (t *tableCommon) Allocator(ctx sessionctx.Context) autoid.Allocator { } // RebaseAutoID implements table.Table RebaseAutoID interface. -func (t *tableCommon) RebaseAutoID(ctx sessionctx.Context, newBase int64, isSetStep bool) error { +func (t *TableCommon) RebaseAutoID(ctx sessionctx.Context, newBase int64, isSetStep bool) error { return t.Allocator(ctx).Rebase(t.tableID, newBase, isSetStep) } // Seek implements table.Table Seek interface. -func (t *tableCommon) Seek(ctx sessionctx.Context, h int64) (int64, bool, error) { +func (t *TableCommon) Seek(ctx sessionctx.Context, h int64) (int64, bool, error) { txn, err := ctx.Txn(true) if err != nil { return 0, false, err @@ -1046,7 +1039,7 @@ func (t *tableCommon) Seek(ctx sessionctx.Context, h int64) (int64, bool, error) } // Type implements table.Table Type interface. -func (t *tableCommon) Type() table.Type { +func (t *TableCommon) Type() table.Type { return table.NormalTable } @@ -1057,11 +1050,11 @@ func shouldWriteBinlog(ctx sessionctx.Context) bool { return !ctx.GetSessionVars().InRestrictedSQL } -func (t *tableCommon) getMutation(ctx sessionctx.Context) *binlog.TableMutation { +func (t *TableCommon) getMutation(ctx sessionctx.Context) *binlog.TableMutation { return ctx.StmtGetMutation(t.tableID) } -func (t *tableCommon) canSkip(col *table.Column, value types.Datum) bool { +func (t *TableCommon) canSkip(col *table.Column, value types.Datum) bool { return CanSkip(t.Meta(), col, value) } @@ -1083,7 +1076,7 @@ func CanSkip(info *model.TableInfo, col *table.Column, value types.Datum) bool { } // canSkipUpdateBinlog checks whether the column can be skipped or not. -func (t *tableCommon) canSkipUpdateBinlog(col *table.Column, value types.Datum) bool { +func (t *TableCommon) canSkipUpdateBinlog(col *table.Column, value types.Datum) bool { if col.IsGenerated() && !col.GeneratedStored { return true } diff --git a/tidb-server/main.go b/tidb-server/main.go index e036a96ecc3c6..fc8108798610d 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -29,7 +29,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" - pd "github.com/pingcap/pd/client" + "github.com/pingcap/pd/client" pumpcli "github.com/pingcap/tidb-tools/tidb-binlog/pump_client" "github.com/pingcap/tidb/bindinfo" "github.com/pingcap/tidb/config" @@ -50,6 +50,7 @@ import ( "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/gcworker" + "github.com/pingcap/tidb/util/domainutil" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/printer" @@ -91,6 +92,8 @@ const ( nmTokenLimit = "token-limit" nmPluginDir = "plugin-dir" nmPluginLoad = "plugin-load" + nmRepairMode = "repair-mode" + nmRepairList = "repair-list" nmProxyProtocolNetworks = "proxy-protocol-networks" nmProxyProtocolHeaderTimeout = "proxy-protocol-header-timeout" @@ -118,6 +121,8 @@ var ( pluginDir = flag.String(nmPluginDir, "/data/deploy/plugin", "the folder that hold plugin") pluginLoad = flag.String(nmPluginLoad, "", "wait load plugin name(separated by comma)") affinityCPU = flag.String(nmAffinityCPU, "", "affinity cpu (cpu-no. separated by comma, e.g. 1,2,3)") + repairMode = flagBoolean(nmRepairMode, false, "enable admin repair mode") + repairList = flag.String(nmRepairList, "", "admin repair table list") // Log logLevel = flag.String(nmLogLevel, "info", "log level: info, debug, warn, error, fatal") @@ -470,6 +475,14 @@ func overrideConfig() { if actualFlags[nmPluginDir] { cfg.Plugin.Dir = *pluginDir } + if actualFlags[nmRepairMode] { + cfg.RepairMode = *repairMode + } + if actualFlags[nmRepairList] { + if cfg.RepairMode { + cfg.RepairTableList = stringToList(*repairList) + } + } // Log if actualFlags[nmLogLevel] { @@ -561,6 +574,8 @@ func setGlobalVars() { tikv.CommitMaxBackoff = int(parseDuration(cfg.TiKVClient.CommitTimeout).Seconds() * 1000) tikv.RegionCacheTTLSec = int64(cfg.TiKVClient.RegionCacheTTL) + domainutil.RepairInfo.SetRepairMode(cfg.RepairMode) + domainutil.RepairInfo.SetRepairTableList(cfg.RepairTableList) } func setupLog() { @@ -589,6 +604,7 @@ func createServer() { svr, err = server.NewServer(cfg, driver) // Both domain and storage have started, so we have to clean them before exiting. terror.MustNil(err, closeDomainAndStorage) + svr.SetDomain(dom) go dom.ExpensiveQueryHandle().SetSessionManager(svr).Run() dom.InfoSyncer().SetSessionManager(svr) } @@ -660,3 +676,15 @@ func cleanup() { plugin.Shutdown(context.Background()) closeDomainAndStorage() } + +func stringToList(repairString string) []string { + if len(repairString) <= 0 { + return []string{} + } + if repairString[0] == '[' && repairString[len(repairString)-1] == ']' { + repairString = repairString[1 : len(repairString)-1] + } + return strings.FieldsFunc(repairString, func(r rune) bool { + return r == ',' || r == ' ' || r == '"' + }) +} diff --git a/types/datum.go b/types/datum.go index 125313ec558bc..0b3e9148d1809 100644 --- a/types/datum.go +++ b/types/datum.go @@ -1659,23 +1659,6 @@ func invalidConv(d *Datum, tp byte) (Datum, error) { return Datum{}, errors.Errorf("cannot convert datum from %s to type %s.", KindStr(d.Kind()), TypeStr(tp)) } -func (d *Datum) convergeType(hasUint, hasDecimal, hasFloat *bool) (x Datum) { - x = *d - switch d.Kind() { - case KindUint64: - *hasUint = true - case KindFloat32: - f := d.GetFloat32() - x.SetFloat64(float64(f)) - *hasFloat = true - case KindFloat64: - *hasFloat = true - case KindMysqlDecimal: - *hasDecimal = true - } - return x -} - // NewDatum creates a new Datum from an interface{}. func NewDatum(in interface{}) (d Datum) { switch x := in.(type) { diff --git a/types/json/binary.go b/types/json/binary.go index 84b2a1e1cd7de..519b74a71253f 100644 --- a/types/json/binary.go +++ b/types/json/binary.go @@ -357,23 +357,6 @@ func marshalStringTo(buf, s []byte) []byte { return buf } -func (bj BinaryJSON) marshalValueEntryTo(buf []byte, entryOff int) ([]byte, error) { - tpCode := bj.Value[entryOff] - switch tpCode { - case TypeCodeLiteral: - buf = marshalLiteralTo(buf, bj.Value[entryOff+1]) - default: - offset := endian.Uint32(bj.Value[entryOff+1:]) - tmp := BinaryJSON{TypeCode: tpCode, Value: bj.Value[offset:]} - var err error - buf, err = tmp.marshalTo(buf) - if err != nil { - return nil, errors.Trace(err) - } - } - return buf, nil -} - func marshalLiteralTo(b []byte, litType byte) []byte { switch litType { case LiteralFalse: diff --git a/types/time.go b/types/time.go index c1850a085e42b..21110c2245ecb 100644 --- a/types/time.go +++ b/types/time.go @@ -2252,16 +2252,6 @@ func skipWhiteSpace(input string) string { return "" } -var weekdayAbbrev = map[string]gotime.Weekday{ - "Sun": gotime.Sunday, - "Mon": gotime.Monday, - "Tue": gotime.Tuesday, - "Wed": gotime.Wednesday, - "Thu": gotime.Tuesday, - "Fri": gotime.Friday, - "Sat": gotime.Saturday, -} - var monthAbbrev = map[string]gotime.Month{ "Jan": gotime.January, "Feb": gotime.February, @@ -2628,35 +2618,6 @@ func monthNumeric(t *MysqlTime, input string, ctx map[string]int) (string, bool) return input[length:], true } -func parseOrdinalNumbers(input string) (value int, remain string) { - for i, c := range input { - if !unicode.IsDigit(c) { - v, err := strconv.ParseUint(input[:i], 10, 64) - if err != nil { - return -1, input - } - value = int(v) - break - } - } - switch { - case strings.HasPrefix(remain, "st"): - if value == 1 { - remain = remain[2:] - return - } - case strings.HasPrefix(remain, "nd"): - if value == 2 { - remain = remain[2:] - return - } - case strings.HasPrefix(remain, "th"): - remain = remain[2:] - return - } - return -1, input -} - // DateFSP gets fsp from date string. func DateFSP(date string) (fsp int) { i := strings.LastIndex(date, ".") diff --git a/types/time_test.go b/types/time_test.go index be3ec53b8f6a2..a3979c19d4665 100644 --- a/types/time_test.go +++ b/types/time_test.go @@ -439,26 +439,6 @@ func (s *testTimeSuite) TestYear(c *C) { } } -func (s *testTimeSuite) getLocation(c *C) *time.Location { - locations := []string{"Asia/Shanghai", "Europe/Berlin"} - timeFormat := "Jan 2, 2006 at 3:04pm (MST)" - - z, err := time.LoadLocation(locations[0]) - c.Assert(err, IsNil) - - t1, err := time.ParseInLocation(timeFormat, "Jul 9, 2012 at 5:02am (CEST)", z) - c.Assert(err, IsNil) - t2, err := time.Parse(timeFormat, "Jul 9, 2012 at 5:02am (CEST)") - c.Assert(err, IsNil) - - if t1.Equal(t2) { - z, err = time.LoadLocation(locations[1]) - c.Assert(err, IsNil) - } - - return z -} - func (s *testTimeSuite) TestCodec(c *C) { defer testleak.AfterTest(c)() diff --git a/util/admin/admin.go b/util/admin/admin.go index 52c949958ec0f..fb5f302d4828f 100644 --- a/util/admin/admin.go +++ b/util/admin/admin.go @@ -104,7 +104,7 @@ func IsJobRollbackable(job *model.Job) bool { model.ActionTruncateTable, model.ActionAddForeignKey, model.ActionDropForeignKey, model.ActionRenameTable, model.ActionModifyTableCharsetAndCollate, model.ActionTruncateTablePartition, - model.ActionModifySchemaCharsetAndCollate: + model.ActionModifySchemaCharsetAndCollate, model.ActionRepairTable: return job.SchemaState == model.StateNone } return true diff --git a/util/admin/admin_test.go b/util/admin/admin_test.go index ce14fd41682c2..5aede319c0c9a 100644 --- a/util/admin/admin_test.go +++ b/util/admin/admin_test.go @@ -262,13 +262,21 @@ func (s *testSuite) TestCancelJobs(c *C) { TableID: 2, Type: model.ActionAddIndex, } + job3 := &model.Job{ + ID: 1003, + SchemaID: 1, + TableID: 2, + Type: model.ActionRepairTable, + } err = t.EnQueueDDLJob(job, meta.AddIndexJobListKey) c.Assert(err, IsNil) err = t.EnQueueDDLJob(job1) c.Assert(err, IsNil) err = t.EnQueueDDLJob(job2, meta.AddIndexJobListKey) c.Assert(err, IsNil) - errs, err = CancelJobs(txn, []int64{job1.ID, job.ID, job2.ID}) + err = t.EnQueueDDLJob(job3) + c.Assert(err, IsNil) + errs, err = CancelJobs(txn, []int64{job1.ID, job.ID, job2.ID, job3.ID}) c.Assert(err, IsNil) for _, err := range errs { c.Assert(err, IsNil) diff --git a/util/chunk/codec.go b/util/chunk/codec.go index f866ac34c6efc..51ef7b6baa718 100644 --- a/util/chunk/codec.go +++ b/util/chunk/codec.go @@ -184,6 +184,50 @@ func getFixedLen(colType *types.FieldType) int { } } +// GetFixedLen get the memory size of a fixed-length type. +// if colType is not fixed-length, it returns varElemLen, aka -1. +func GetFixedLen(colType *types.FieldType) int { + return getFixedLen(colType) +} + +// EstimateTypeWidth estimates the average width of values of the type. +// This is used by the planner, which doesn't require absolutely correct results; +// it's OK (and expected) to guess if we don't know for sure. +// +// mostly study from https://github.com/postgres/postgres/blob/REL_12_STABLE/src/backend/utils/cache/lsyscache.c#L2356 +func EstimateTypeWidth(padChar bool, colType *types.FieldType) int { + colLen := getFixedLen(colType) + // Easy if it's a fixed-width type + if colLen != varElemLen { + return colLen + } + + colLen = colType.Flen + if colLen > 0 { + /* + * If PAD_CHAR_TO_FULL_LENGTH is enabled, and type is CHAR, + * the colType.Flen is also the only width. + */ + if padChar && colType.Tp == mysql.TypeString { + return colLen + } + if colLen <= 32 { + return colLen + } + if colLen < 1000 { + return 32 + (colLen-32)/2 // assume 50% + } + /* + * Beyond 1000, assume we're looking at something like + * "varchar(10000)" where the limit isn't actually reached often, and + * use a fixed estimate. + */ + return 32 + (1000-32)/2 + } + // Oops, we have no idea ... wild guess time. + return 32 +} + func init() { for i := 0; i < 128; i++ { allNotNullBitmap[i] = 0xFF diff --git a/util/chunk/codec_test.go b/util/chunk/codec_test.go index b904ed9756f85..5e22a3c8f10cb 100644 --- a/util/chunk/codec_test.go +++ b/util/chunk/codec_test.go @@ -77,6 +77,28 @@ func (s *testCodecSuite) TestCodec(c *check.C) { } } +func (s *testCodecSuite) TestEstimateTypeWidth(c *check.C) { + var colType *types.FieldType + + colType = &types.FieldType{Tp: mysql.TypeLonglong} + c.Assert(EstimateTypeWidth(false, colType), check.Equals, 8) // fixed-witch type + + colType = &types.FieldType{Tp: mysql.TypeString, Flen: 100000} + c.Assert(EstimateTypeWidth(true, colType), check.Equals, 100000) // PAD_CHAR_TO_FULL_LENGTH + + colType = &types.FieldType{Tp: mysql.TypeString, Flen: 31} + c.Assert(EstimateTypeWidth(false, colType), check.Equals, 31) // colLen <= 32 + + colType = &types.FieldType{Tp: mysql.TypeString, Flen: 999} + c.Assert(EstimateTypeWidth(false, colType), check.Equals, 515) // colLen < 1000 + + colType = &types.FieldType{Tp: mysql.TypeString, Flen: 2000} + c.Assert(EstimateTypeWidth(false, colType), check.Equals, 516) // colLen < 1000 + + colType = &types.FieldType{Tp: mysql.TypeString} + c.Assert(EstimateTypeWidth(false, colType), check.Equals, 32) // value after guessing +} + func BenchmarkEncodeChunk(b *testing.B) { numCols := 4 numRows := 1024 diff --git a/util/domainutil/repair_vars.go b/util/domainutil/repair_vars.go new file mode 100644 index 0000000000000..f45081e96fa53 --- /dev/null +++ b/util/domainutil/repair_vars.go @@ -0,0 +1,170 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package domainutil + +import ( + "strings" + "sync" + + "github.com/pingcap/parser/model" +) + +type repairInfo struct { + sync.RWMutex + repairMode bool + repairTableList []string + repairDBInfoMap map[int64]*model.DBInfo +} + +// RepairInfo indicates the repaired table info. +var RepairInfo repairInfo + +// InRepairMode indicates whether TiDB is in repairMode. +func (r *repairInfo) InRepairMode() bool { + r.RLock() + defer r.RUnlock() + return r.repairMode +} + +// SetRepairMode sets whether TiDB is in repairMode. +func (r *repairInfo) SetRepairMode(mode bool) { + r.Lock() + defer r.Unlock() + r.repairMode = mode +} + +// GetRepairTableList gets repairing table list. +func (r *repairInfo) GetRepairTableList() []string { + r.RLock() + defer r.RUnlock() + return r.repairTableList +} + +// SetRepairTableList sets repairing table list. +func (r *repairInfo) SetRepairTableList(list []string) { + for i, one := range list { + list[i] = strings.ToLower(one) + } + r.Lock() + defer r.Unlock() + r.repairTableList = list +} + +// CheckAndFetchRepairedTable fetches the repairing table list from meta, true indicates fetch success. +func (r *repairInfo) CheckAndFetchRepairedTable(di *model.DBInfo, tbl *model.TableInfo) bool { + r.Lock() + defer r.Unlock() + if !r.repairMode { + return false + } + isRepair := false + for _, tn := range r.repairTableList { + // Use dbName and tableName to specify a table. + if strings.ToLower(tn) == di.Name.L+"."+tbl.Name.L { + isRepair = true + break + } + } + if isRepair { + // Record the repaired table in Map. + if repairedDB, ok := r.repairDBInfoMap[di.ID]; ok { + repairedDB.Tables = append(repairedDB.Tables, tbl) + } else { + // Shallow copy the DBInfo. + repairedDB := di.Copy() + // Clean the tables and set repaired table. + repairedDB.Tables = []*model.TableInfo{tbl} + r.repairDBInfoMap[di.ID] = repairedDB + } + return true + } + return false +} + +// GetRepairedTableInfoByTableName is exported for test. +func (r *repairInfo) GetRepairedTableInfoByTableName(schemaLowerName, tableLowerName string) (*model.TableInfo, *model.DBInfo) { + r.RLock() + defer r.RUnlock() + for _, db := range r.repairDBInfoMap { + if db.Name.L != schemaLowerName { + continue + } + for _, t := range db.Tables { + if t.Name.L == tableLowerName { + return t, db + } + } + return nil, db + } + return nil, nil +} + +// RemoveFromRepairInfo remove the table from repair info when repaired. +func (r *repairInfo) RemoveFromRepairInfo(schemaLowerName, tableLowerName string) { + repairedLowerName := schemaLowerName + "." + tableLowerName + // Remove from the repair list. + r.Lock() + defer r.Unlock() + for i, rt := range r.repairTableList { + if strings.ToLower(rt) == repairedLowerName { + r.repairTableList = append(r.repairTableList[:i], r.repairTableList[i+1:]...) + break + } + } + // Remove from the repair map. + for _, db := range r.repairDBInfoMap { + if db.Name.L == schemaLowerName { + for j, t := range db.Tables { + if t.Name.L == tableLowerName { + db.Tables = append(db.Tables[:j], db.Tables[j+1:]...) + break + } + } + if len(db.Tables) == 0 { + delete(r.repairDBInfoMap, db.ID) + } + break + } + } + if len(r.repairDBInfoMap) == 0 { + r.repairMode = false + } +} + +// repairKeyType is keyType for admin repair table. +type repairKeyType int + +const ( + // RepairedTable is the key type, caching the target repaired table in sessionCtx. + RepairedTable repairKeyType = iota + // RepairedDatabase is the key type, caching the target repaired database in sessionCtx. + RepairedDatabase +) + +func (t repairKeyType) String() (res string) { + switch t { + case RepairedTable: + res = "RepairedTable" + case RepairedDatabase: + res = "RepairedDatabase" + } + return res +} + +func init() { + RepairInfo = repairInfo{} + RepairInfo.repairMode = false + RepairInfo.repairTableList = []string{} + RepairInfo.repairDBInfoMap = make(map[int64]*model.DBInfo) +} diff --git a/util/execdetails/execdetails.go b/util/execdetails/execdetails.go index b80e025ea2bd7..6e1ae05f6b0ec 100644 --- a/util/execdetails/execdetails.go +++ b/util/execdetails/execdetails.go @@ -37,6 +37,8 @@ type ExecDetails struct { ProcessTime time.Duration WaitTime time.Duration BackoffTime time.Duration + BackoffSleep map[string]time.Duration + BackoffTimes map[string]int RequestCount int TotalKeys int64 ProcessedKeys int64 diff --git a/util/memory/tracker.go b/util/memory/tracker.go index a28116c85feec..400c91c8bac61 100644 --- a/util/memory/tracker.go +++ b/util/memory/tracker.go @@ -78,6 +78,12 @@ func (t *Tracker) SetBytesLimit(bytesLimit int64) { t.bytesLimit = bytesLimit } +// GetBytesLimit gets the bytes limit for this tracker. +// "bytesLimit <= 0" means no limit. +func (t *Tracker) GetBytesLimit() int64 { + return t.bytesLimit +} + // SetActionOnExceed sets the action when memory usage exceeds bytesLimit. func (t *Tracker) SetActionOnExceed(a ActionOnExceed) { t.actionMu.Lock() diff --git a/util/mock/context.go b/util/mock/context.go index de79e16dcc92c..41f5625cca6cd 100644 --- a/util/mock/context.go +++ b/util/mock/context.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util" + "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/kvcache" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/sqlexec" @@ -267,6 +268,7 @@ func NewContext() *Context { sctx.sessionVars.MaxChunkSize = 32 sctx.sessionVars.StmtCtx.TimeZone = time.UTC sctx.sessionVars.StmtCtx.MemTracker = memory.NewTracker(stringutil.StringerStr("mock.NewContext"), -1) + sctx.sessionVars.StmtCtx.DiskTracker = disk.NewTracker(stringutil.StringerStr("mock.NewContext"), -1) sctx.sessionVars.GlobalVarsAccessor = variable.NewMockGlobalAccessor() if err := sctx.GetSessionVars().SetSystemVar(variable.MaxAllowedPacket, "67108864"); err != nil { panic(err) diff --git a/util/plancodec/codec.go b/util/plancodec/codec.go index 31b77df1e84b7..971bb94d84f75 100644 --- a/util/plancodec/codec.go +++ b/util/plancodec/codec.go @@ -55,6 +55,17 @@ func DecodePlan(planString string) (string, error) { return pd.decode(planString) } +// DecodeNormalizedPlan decodes the string to plan tree. +func DecodeNormalizedPlan(planString string) (string, error) { + if len(planString) == 0 { + return "", nil + } + pd := decoderPool.Get().(*planDecoder) + defer decoderPool.Put(pd) + pd.buf.Reset() + return pd.buildPlanTree(planString) +} + type planDecoder struct { buf bytes.Buffer depths []int @@ -72,8 +83,11 @@ func (pd *planDecoder) decode(planString string) (string, error) { if err != nil { return "", err } + return pd.buildPlanTree(str) +} - nodes := strings.Split(str, lineBreakerStr) +func (pd *planDecoder) buildPlanTree(planString string) (string, error) { + nodes := strings.Split(planString, lineBreakerStr) if len(pd.depths) < len(nodes) { pd.depths = make([]int, 0, len(nodes)) pd.planInfos = make([]*planInfo, 0, len(nodes)) @@ -256,6 +270,22 @@ func EncodePlanNode(depth, pid int, planType string, isRoot bool, rowCount float buf.WriteByte(lineBreaker) } +// NormalizePlanNode is used to normalize the plan to a string. +func NormalizePlanNode(depth, pid int, planType string, isRoot bool, explainInfo string, buf *bytes.Buffer) { + buf.WriteString(strconv.Itoa(depth)) + buf.WriteByte(separator) + buf.WriteString(encodeID(planType, pid)) + buf.WriteByte(separator) + if isRoot { + buf.WriteString(rootTaskType) + } else { + buf.WriteString(copTaskType) + } + buf.WriteByte(separator) + buf.WriteString(explainInfo) + buf.WriteByte(lineBreaker) +} + func encodeID(planType string, id int) string { planID := TypeStringToPhysicalID(planType) return strconv.Itoa(planID) + idSeparator + strconv.Itoa(id) diff --git a/util/plancodec/id.go b/util/plancodec/id.go index 8369f7bfd0e59..583799b422cfa 100644 --- a/util/plancodec/id.go +++ b/util/plancodec/id.go @@ -84,8 +84,8 @@ const ( TypeIndexReader = "IndexReader" // TypeWindow is the type of Window. TypeWindow = "Window" - // TypeTableGather is the type of TableGather. - TypeTableGather = "TableGather" + // TypeTiKVSingleGather is the type of TiKVSingleGather. + TypeTiKVSingleGather = "TiKVSingleGather" // TypeIndexMerge is the type of IndexMergeReader TypeIndexMerge = "IndexMerge" // TypePointGet is the type of PointGetPlan. @@ -94,6 +94,8 @@ const ( TypeShowDDLJobs = "ShowDDLJobs" // TypeBatchPointGet is the type of BatchPointGetPlan. TypeBatchPointGet = "Batch_Point_Get" + // TypeClusterMemTableReader is the type of TableReader. + TypeClusterMemTableReader = "ClusterMemTableReader" ) // plan id. @@ -132,11 +134,12 @@ const ( typeTableReaderID typeIndexReaderID typeWindowID - typeTableGatherID + typeTiKVSingleGatherID typeIndexMergeID typePointGet typeShowDDLJobs typeBatchPointGet + typeClusterMemTableReader ) // TypeStringToPhysicalID converts the plan type string to plan id. @@ -210,8 +213,8 @@ func TypeStringToPhysicalID(tp string) int { return typeIndexReaderID case TypeWindow: return typeWindowID - case TypeTableGather: - return typeTableGatherID + case TypeTiKVSingleGather: + return typeTiKVSingleGatherID case TypeIndexMerge: return typeIndexMergeID case TypePointGet: @@ -220,6 +223,8 @@ func TypeStringToPhysicalID(tp string) int { return typeShowDDLJobs case TypeBatchPointGet: return typeBatchPointGet + case TypeClusterMemTableReader: + return typeClusterMemTableReader } // Should never reach here. return 0 @@ -296,8 +301,8 @@ func PhysicalIDToTypeString(id int) string { return TypeIndexReader case typeWindowID: return TypeWindow - case typeTableGatherID: - return TypeTableGather + case typeTiKVSingleGatherID: + return TypeTiKVSingleGather case typeIndexMergeID: return TypeIndexMerge case typePointGet: @@ -306,6 +311,8 @@ func PhysicalIDToTypeString(id int) string { return TypeShowDDLJobs case typeBatchPointGet: return TypeBatchPointGet + case typeClusterMemTableReader: + return TypeClusterMemTableReader } // Should never reach here. diff --git a/util/set/int_set.go b/util/set/int_set.go index 9fef718e0c42a..dc835d7a82e7e 100644 --- a/util/set/int_set.go +++ b/util/set/int_set.go @@ -36,8 +36,12 @@ func (s IntSet) Insert(val int) { type Int64Set map[int64]struct{} // NewInt64Set builds a Int64Set. -func NewInt64Set() Int64Set { - return make(map[int64]struct{}) +func NewInt64Set(xs ...int64) Int64Set { + set := make(Int64Set) + for _, x := range xs { + set.Insert(x) + } + return set } // Exist checks whether `val` exists in `s`. diff --git a/util/set/int_set_test.go b/util/set/int_set_test.go index 6f5a928964556..0fd14e68eb32d 100644 --- a/util/set/int_set_test.go +++ b/util/set/int_set_test.go @@ -57,4 +57,10 @@ func (s *intSetTestSuite) TestInt64Set(c *check.C) { } c.Assert(set.Exist(11), check.IsFalse) + + set = NewInt64Set(1, 2, 3, 4, 5, 6) + for i := 1; i < 7; i++ { + c.Assert(set.Exist(int64(i)), check.IsTrue) + } + c.Assert(set.Exist(7), check.IsFalse) } diff --git a/util/set/string_set.go b/util/set/string_set.go index 37745b4ecc6ae..019c25c472c23 100644 --- a/util/set/string_set.go +++ b/util/set/string_set.go @@ -17,8 +17,12 @@ package set type StringSet map[string]struct{} // NewStringSet builds a float64 set. -func NewStringSet() StringSet { - return make(map[string]struct{}) +func NewStringSet(ss ...string) StringSet { + set := make(StringSet) + for _, s := range ss { + set.Insert(s) + } + return set } // Exist checks whether `val` exists in `s`. @@ -31,3 +35,14 @@ func (s StringSet) Exist(val string) bool { func (s StringSet) Insert(val string) { s[val] = struct{}{} } + +// Intersection returns the intersection of two sets +func (s StringSet) Intersection(rhs StringSet) StringSet { + newSet := NewStringSet() + for elt := range s { + if rhs.Exist(elt) { + newSet.Insert(elt) + } + } + return newSet +} diff --git a/util/set/string_set_test.go b/util/set/string_set_test.go index 9b07d16828697..21db3fbdcac30 100644 --- a/util/set/string_set_test.go +++ b/util/set/string_set_test.go @@ -14,6 +14,8 @@ package set import ( + "fmt" + "github.com/pingcap/check" ) @@ -38,4 +40,24 @@ func (s *stringSetTestSuite) TestStringSet(c *check.C) { } c.Assert(set.Exist("11"), check.IsFalse) + + set = NewStringSet("1", "2", "3", "4", "5", "6") + for i := 1; i < 7; i++ { + c.Assert(set.Exist(fmt.Sprintf("%d", i)), check.IsTrue) + } + c.Assert(set.Exist("7"), check.IsFalse) + + s1 := NewStringSet("1", "2", "3") + s2 := NewStringSet("4", "2", "3") + s3 := s1.Intersection(s2) + c.Assert(s3, check.DeepEquals, NewStringSet("2", "3")) + + s4 := NewStringSet("4", "5", "3") + c.Assert(s3.Intersection(s4), check.DeepEquals, NewStringSet("3")) + + s5 := NewStringSet("4", "5") + c.Assert(s3.Intersection(s5), check.DeepEquals, NewStringSet()) + + s6 := NewStringSet() + c.Assert(s3.Intersection(s6), check.DeepEquals, NewStringSet()) } diff --git a/util/timeutil/time.go b/util/timeutil/time.go index b11b1dd5d127f..6b00468c625a4 100644 --- a/util/timeutil/time.go +++ b/util/timeutil/time.go @@ -161,6 +161,16 @@ func Zone(loc *time.Location) (string, int64) { return name, int64(offset) } +// ConstructTimeZone constructs timezone by name first. When the timezone name +// is set, the daylight saving problem must be considered. Otherwise the +// timezone offset in seconds east of UTC is used to constructed the timezone. +func ConstructTimeZone(name string, offset int) (*time.Location, error) { + if name != "" { + return LoadLocation(name) + } + return time.FixedZone("", offset), nil +} + // WithinDayTimePeriod tests whether `now` is between `start` and `end`. func WithinDayTimePeriod(start, end, now time.Time) bool { // Converts to UTC and only keeps the hour and minute info.