Skip to content
This repository has been archived by the owner on Nov 24, 2023. It is now read-only.

Commit

Permalink
tracker: fetch ref table at CREATE LIKE and more (#1105) (#1108)
Browse files Browse the repository at this point in the history
Co-authored-by: lance6716 <lance6716@gmail.com>
  • Loading branch information
ti-srebot and lance6716 authored Sep 28, 2020
1 parent a220d84 commit 0cecb73
Show file tree
Hide file tree
Showing 11 changed files with 240 additions and 131 deletions.
1 change: 1 addition & 0 deletions _utils/terror_gen/errors_release.txt
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ ErrInvalidV1WorkerMetaPath,[code=11115:class=functional:scope=internal:level=med
ErrFailUpdateV1DBSchema,[code=11116:class=functional:scope=internal:level=medium], "Message: fail to upgrade v1.0.x DB schema, Workaround: Please confirm that you have not violated any restrictions in the upgrade documentation."
ErrBinlogStatusVarsParse,[code=11117:class=functional:scope=internal:level=medium], "Message: fail to parse binglog status_vars: %v, offset: %d"
ErrVerifyHandleErrorArgs,[code=11118:class=functional:scope=internal:level=low], "Workaround: Please make sure the args are correct."
ErrRewriteSQL,[code=11119:class=functional:scope=internal:level=high], "Message: failed to rewrite SQL for target DB, stmt: %+v, targetTableNames: %+v"
ErrConfigCheckItemNotSupport,[code=20001:class=config:scope=internal:level=medium], "Message: checking item %s is not supported\n%s, Workaround: Please check `ignore-checking-items` config in task configuration file, which can be set including `all`/`dump_privilege`/`replication_privilege`/`version`/`binlog_enable`/`binlog_format`/`binlog_row_image`/`table_schema`/`schema_of_shard_tables`/`auto_increment_ID`."
ErrConfigTomlTransform,[code=20002:class=config:scope=internal:level=medium], "Message: %s, Workaround: Please check the configuration file has correct TOML format."
ErrConfigYamlTransform,[code=20003:class=config:scope=internal:level=medium], "Message: %s, Workaround: Please check the configuration file has correct YAML format."
Expand Down
6 changes: 6 additions & 0 deletions errors.toml
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,12 @@ description = ""
workaround = "Please make sure the args are correct."
tags = ["internal", "low"]

[error.DM-functional-11119]
message = "failed to rewrite SQL for target DB, stmt: %+v, targetTableNames: %+v"
description = ""
workaround = ""
tags = ["internal", "high"]

[error.DM-config-20001]
message = "checking item %s is not supported\n%s"
description = ""
Expand Down
181 changes: 90 additions & 91 deletions pkg/parser/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@ import (
"go.uber.org/zap"
)

const (
// SingleRenameTableNameNum stands for number of TableNames in a single table renaming. it's 4 not 2 because TiDB
// parser repeats first two TableNames at start
// ref https://github.com/pingcap/tidb/pull/3892
SingleRenameTableNameNum = 4
)

// Parse wraps parser.Parse(), makes `parser` suitable for dm
func Parse(p *parser.Parser, sql, charset, collation string) (stmt []ast.StmtNode, err error) {
stmts, warnings, err := p.Parse(sql, charset, collation)
Expand All @@ -42,122 +49,114 @@ func Parse(p *parser.Parser, sql, charset, collation string) (stmt []ast.StmtNod
return stmts, terror.ErrParseSQL.Delegate(err)
}

// FetchDDLTableNames returns table names in ddl
// the result contains [tableName] excepted create table like and rename table
// for `create table like` DDL, result contains [sourceTableName, sourceRefTableName]
// for rename table ddl, result contains [oldTableName, newTableName]
// ref: https://github.com/pingcap/tidb/blob/09feccb529be2830944e11f5fed474020f50370f/server/sql_info_fetcher.go#L46
type tableNameExtractor struct {
curDB string
names []*filter.Table
}

func (tne *tableNameExtractor) Enter(in ast.Node) (ast.Node, bool) {
if t, ok := in.(*ast.TableName); ok {
tb := &filter.Table{Schema: t.Schema.L, Name: t.Name.L}
if tb.Schema == "" {
tb.Schema = tne.curDB
}
tne.names = append(tne.names, tb)
return in, true
}
return in, false
}

func (tne *tableNameExtractor) Leave(in ast.Node) (ast.Node, bool) {
return in, true
}

// FetchDDLTableNames returns tableNames in ddl the result contains many tableName.
// Because we use visitor pattern, first tableName is always upper-most table in ast
// specifically, for `create table like` DDL, result contains [sourceTableName, sourceRefTableName]
// for rename table ddl, result contains [old1, new1, old1, new1, old2, new2, old3, new3, ...] because of TiDB parser
// for other DDL, order of tableName is the node visit order
func FetchDDLTableNames(schema string, stmt ast.StmtNode) ([]*filter.Table, error) {
var res []*filter.Table
switch stmt.(type) {
case ast.DDLNode:
default:
return nil, terror.ErrUnknownTypeDDL.Generate(stmt)
}

// special cases: schema related SQLs doesn't have tableName
switch v := stmt.(type) {
case *ast.AlterDatabaseStmt:
res = append(res, genTableName(v.Name, ""))
return []*filter.Table{genTableName(v.Name, "")}, nil
case *ast.CreateDatabaseStmt:
res = append(res, genTableName(v.Name, ""))
return []*filter.Table{genTableName(v.Name, "")}, nil
case *ast.DropDatabaseStmt:
res = append(res, genTableName(v.Name, ""))
case *ast.CreateTableStmt:
res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O))
if v.ReferTable != nil {
res = append(res, genTableName(v.ReferTable.Schema.O, v.ReferTable.Name.O))
}
case *ast.DropTableStmt:
if len(v.Tables) != 1 {
return res, terror.ErrDropMultipleTables.Generate()
}
res = append(res, genTableName(v.Tables[0].Schema.O, v.Tables[0].Name.O))
case *ast.TruncateTableStmt:
res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O))
case *ast.AlterTableStmt:
res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O))
if v.Specs[0].NewTable != nil {
res = append(res, genTableName(v.Specs[0].NewTable.Schema.O, v.Specs[0].NewTable.Name.O))
}
case *ast.RenameTableStmt:
res = append(res, genTableName(v.OldTable.Schema.O, v.OldTable.Name.O))
res = append(res, genTableName(v.NewTable.Schema.O, v.NewTable.Name.O))
case *ast.CreateIndexStmt:
res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O))
case *ast.DropIndexStmt:
res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O))
default:
return res, terror.ErrUnknownTypeDDL.Generate(stmt)
return []*filter.Table{genTableName(v.Name, "")}, nil
}

e := &tableNameExtractor{
curDB: schema,
names: make([]*filter.Table, 0),
}
stmt.Accept(e)

return e.names, nil
}

type tableRenameVisitor struct {
targetNames []*filter.Table
i int
hasErr bool
}

for i := range res {
if res[i].Schema == "" {
res[i].Schema = schema
func (v *tableRenameVisitor) Enter(in ast.Node) (ast.Node, bool) {
if v.hasErr {
return in, true
}
if t, ok := in.(*ast.TableName); ok {
if v.i >= len(v.targetNames) {
v.hasErr = true
return in, true
}
t.Schema = model.NewCIStr(v.targetNames[v.i].Schema)
t.Name = model.NewCIStr(v.targetNames[v.i].Name)
v.i++
return in, true
}
return in, false
}

return res, nil
func (v *tableRenameVisitor) Leave(in ast.Node) (ast.Node, bool) {
if v.hasErr {
return in, false
}
return in, true
}

// RenameDDLTable renames table names in ddl by given `targetTableNames`
// argument `targetTableNames` is same with return value of FetchDDLTableNames
// returned DDL is formatted like StringSingleQuotes, KeyWordUppercase and NameBackQuotes
func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string, error) {
switch stmt.(type) {
case ast.DDLNode:
default:
return "", terror.ErrUnknownTypeDDL.Generate(stmt)
}

switch v := stmt.(type) {
case *ast.AlterDatabaseStmt:
v.Name = targetTableNames[0].Schema

case *ast.CreateDatabaseStmt:
v.Name = targetTableNames[0].Schema

case *ast.DropDatabaseStmt:
v.Name = targetTableNames[0].Schema

case *ast.CreateTableStmt:
v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema)
v.Table.Name = model.NewCIStr(targetTableNames[0].Name)

if v.ReferTable != nil {
v.ReferTable.Schema = model.NewCIStr(targetTableNames[1].Schema)
v.ReferTable.Name = model.NewCIStr(targetTableNames[1].Name)
}

case *ast.DropTableStmt:
if len(v.Tables) > 1 {
return "", terror.ErrDropMultipleTables.Generate()
}

v.Tables[0].Schema = model.NewCIStr(targetTableNames[0].Schema)
v.Tables[0].Name = model.NewCIStr(targetTableNames[0].Name)

case *ast.TruncateTableStmt:
v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema)
v.Table.Name = model.NewCIStr(targetTableNames[0].Name)

case *ast.DropIndexStmt:
v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema)
v.Table.Name = model.NewCIStr(targetTableNames[0].Name)
case *ast.CreateIndexStmt:
v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema)
v.Table.Name = model.NewCIStr(targetTableNames[0].Name)
case *ast.RenameTableStmt:
if len(v.TableToTables) > 1 {
return "", terror.ErrRenameMultipleTables.Generate()
}

v.TableToTables[0].OldTable.Schema = model.NewCIStr(targetTableNames[0].Schema)
v.TableToTables[0].OldTable.Name = model.NewCIStr(targetTableNames[0].Name)
v.TableToTables[0].NewTable.Schema = model.NewCIStr(targetTableNames[1].Schema)
v.TableToTables[0].NewTable.Name = model.NewCIStr(targetTableNames[1].Name)

case *ast.AlterTableStmt:
if len(v.Specs) > 1 {
return "", terror.ErrAlterMultipleTables.Generate()
default:
visitor := &tableRenameVisitor{
targetNames: targetTableNames,
}

v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema)
v.Table.Name = model.NewCIStr(targetTableNames[0].Name)

if v.Specs[0].Tp == ast.AlterTableRenameTable {
v.Specs[0].NewTable.Schema = model.NewCIStr(targetTableNames[1].Schema)
v.Specs[0].NewTable.Name = model.NewCIStr(targetTableNames[1].Name)
stmt.Accept(visitor)
if visitor.hasErr {
return "", terror.ErrRewriteSQL.Generate(stmt, targetTableNames)
}

default:
return "", terror.ErrUnknownTypeDDL.Generate(stmt)
}

var b []byte
Expand Down
39 changes: 31 additions & 8 deletions pkg/parser/common_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import (
. "github.com/pingcap/check"
"github.com/pingcap/parser"
"github.com/pingcap/tidb-tools/pkg/filter"

"github.com/pingcap/dm/pkg/terror"
)

var _ = Suite(&testParserSuite{})
Expand Down Expand Up @@ -93,6 +95,28 @@ func (t *testParserSuite) TestParser(c *C) {
}
}

func (t *testParserSuite) TestError(c *C) {
p := parser.New()

// DML will report ErrUnknownTypeDDL
dml := "INSERT INTO `t1` VALUES (1)"

stmts, err := Parse(p, dml, "", "")
c.Assert(err, IsNil)
_, err = FetchDDLTableNames("test", stmts[0])
c.Assert(terror.ErrUnknownTypeDDL.Equal(err), IsTrue)

_, err = RenameDDLTable(stmts[0], nil)
c.Assert(terror.ErrUnknownTypeDDL.Equal(err), IsTrue)

// tableRenameVisitor with less `targetNames` won't panic
ddl := "create table `s1`.`t1` (id int)"
stmts, err = Parse(p, ddl, "", "")
c.Assert(err, IsNil)
_, err = RenameDDLTable(stmts[0], nil)
c.Assert(terror.ErrRewriteSQL.Equal(err), IsTrue)
}

func (t *testParserSuite) TestResolveDDL(c *C) {
p := parser.New()
expectedSQLs := [][]string{
Expand Down Expand Up @@ -154,8 +178,8 @@ func (t *testParserSuite) TestResolveDDL(c *C) {
{{genTableName("test", "t1"), genTableName("xx", "t2")}},
{{genTableName("test", "t1")}},
{{genTableName("s1", "t1")}},
{{genTableName("s1", "t1"), genTableName("s2", "t2")}},
{{genTableName("test", "t1"), genTableName("test", "t2")}, {genTableName("s1", "t1"), genTableName("test", "t2")}},
{{genTableName("s1", "t1"), genTableName("s2", "t2"), genTableName("s1", "t1"), genTableName("s2", "t2")}},
{{genTableName("test", "t1"), genTableName("test", "t2"), genTableName("test", "t1"), genTableName("test", "t2")}, {genTableName("s1", "t1"), genTableName("test", "t2"), genTableName("s1", "t1"), genTableName("test", "t2")}},
{{genTableName("s1", "t1")}},
{{genTableName("test", "t1")}},
{{genTableName("test", "t1")}},
Expand All @@ -165,7 +189,7 @@ func (t *testParserSuite) TestResolveDDL(c *C) {
{{genTableName("s1", "t1")}, {genTableName("s1", "t1"), genTableName("xx", "t2")}, {genTableName("xx", "t2")}},
{{genTableName("test", "t1")}},
{{genTableName("test", "t1")}},
{{genTableName("test", "t1")}},
{{genTableName("test", "t1"), genTableName("test", "t2")}},
{{genTableName("test", "t1")}},
{{genTableName("test", "t1")}},
{{genTableName("test", "t1")}},
Expand Down Expand Up @@ -198,8 +222,8 @@ func (t *testParserSuite) TestResolveDDL(c *C) {
{{genTableName("xtest", "xt1"), genTableName("xxx", "xt2")}},
{{genTableName("xtest", "xt1")}},
{{genTableName("xs1", "xt1")}},
{{genTableName("xs1", "xt1"), genTableName("xs2", "xt2")}},
{{genTableName("xtest", "xt1"), genTableName("xtest", "xt2")}, {genTableName("xs1", "xt1"), genTableName("xtest", "xt2")}},
{{genTableName("xs1", "xt1"), genTableName("xs2", "xt2"), genTableName("xs1", "xt1"), genTableName("xs2", "xt2")}},
{{genTableName("xtest", "xt1"), genTableName("xtest", "xt2"), genTableName("xtest", "xt1"), genTableName("xtest", "xt2")}, {genTableName("xs1", "xt1"), genTableName("xtest", "xt2"), genTableName("xs1", "xt1"), genTableName("xtest", "xt2")}},
{{genTableName("xs1", "xt1")}},
{{genTableName("xtest", "xt1")}},
{{genTableName("xtest", "xt1")}},
Expand All @@ -209,7 +233,7 @@ func (t *testParserSuite) TestResolveDDL(c *C) {
{{genTableName("xs1", "xt1")}, {genTableName("xs1", "xt1"), genTableName("xxx", "xt2")}, {genTableName("xxx", "xt2")}},
{{genTableName("xtest", "xt1")}},
{{genTableName("xtest", "xt1")}},
{{genTableName("xtest", "xt1")}},
{{genTableName("xtest", "xt1"), genTableName("xtest", "xt2")}},
{{genTableName("xtest", "xt1")}},
{{genTableName("xtest", "xt1")}},
{{genTableName("xtest", "xt1")}},
Expand Down Expand Up @@ -253,7 +277,7 @@ func (t *testParserSuite) TestResolveDDL(c *C) {
{"ALTER TABLE `xs1`.`xt1` ADD COLUMN `c1` INT", "ALTER TABLE `xs1`.`xt1` RENAME AS `xxx`.`xt2`", "ALTER TABLE `xxx`.`xt2` DROP COLUMN `c2`"},
{"ALTER TABLE `xtest`.`xt1` ADD COLUMN IF NOT EXISTS `c1` INT"},
{"ALTER TABLE `xtest`.`xt1` ADD INDEX IF NOT EXISTS(`a`) USING BTREE COMMENT 'a'"},
{"ALTER TABLE `xtest`.`xt1` ADD CONSTRAINT `fk_t2_id` FOREIGN KEY IF NOT EXISTS (`t2_id`) REFERENCES `t2`(`id`)"},
{"ALTER TABLE `xtest`.`xt1` ADD CONSTRAINT `fk_t2_id` FOREIGN KEY IF NOT EXISTS (`t2_id`) REFERENCES `xtest`.`xt2`(`id`)"},
{"CREATE INDEX IF NOT EXISTS `i1` ON `xtest`.`xt1` (`c1`)"},
{"ALTER TABLE `xtest`.`xt1` ADD PARTITION IF NOT EXISTS (PARTITION `p2` VALUES LESS THAN (MAXVALUE))"},
{"ALTER TABLE `xtest`.`xt1` DROP COLUMN IF EXISTS `c2`"},
Expand Down Expand Up @@ -295,5 +319,4 @@ func (t *testParserSuite) TestResolveDDL(c *C) {
c.Assert(targetSQL, Equals, targetSQLs[i][j])
}
}

}
6 changes: 6 additions & 0 deletions pkg/terror/error_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,9 @@ const (

// dm/command
codeVerifyHandleErrorArgs

// pkg/parser
codeRewriteSQL
)

// Config related error code list
Expand Down Expand Up @@ -755,6 +758,9 @@ var (
// Functional error
ErrVerifyHandleErrorArgs = New(codeVerifyHandleErrorArgs, ClassFunctional, ScopeInternal, LevelLow, "", "Please make sure the args are correct.")

// pkg/parser
ErrRewriteSQL = New(codeRewriteSQL, ClassFunctional, ScopeInternal, LevelHigh, "failed to rewrite SQL for target DB, stmt: %+v, targetTableNames: %+v", "")

// Config related error
ErrConfigCheckItemNotSupport = New(codeConfigCheckItemNotSupport, ClassConfig, ScopeInternal, LevelMedium, "checking item %s is not supported\n%s", "Please check `ignore-checking-items` config in task configuration file, which can be set including `all`/`dump_privilege`/`replication_privilege`/`version`/`binlog_enable`/`binlog_format`/`binlog_row_image`/`table_schema`/`schema_of_shard_tables`/`auto_increment_ID`.")
ErrConfigTomlTransform = New(codeConfigTomlTransform, ClassConfig, ScopeInternal, LevelMedium, "%s", "Please check the configuration file has correct TOML format.")
Expand Down
2 changes: 0 additions & 2 deletions pkg/utils/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ var (
"^ALTER\\s+PROCEDURE",

// view
"^CREATE\\s*(OR REPLACE)?\\s+(ALGORITHM\\s?=.+?)?(DEFINER\\s?=.+?)?\\s+(SQL SECURITY DEFINER)?VIEW",
"^DROP\\s+VIEW",
"^ALTER\\s+(ALGORITHM\\s?=.+?)?(DEFINER\\s?=.+?)?(SQL SECURITY DEFINER)?VIEW",

// function
Expand Down
6 changes: 3 additions & 3 deletions syncer/filter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,10 @@ BEGIN
END`, true},

// view
{"CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v` AS SELECT qty, price, qty*price AS value FROM t", true},
{"CREATE OR REPLACE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v` AS SELECT qty, price, qty*price AS value FROM t", true},
{"CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v` AS SELECT qty, price, qty*price AS value FROM t", false},
{"CREATE OR REPLACE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v` AS SELECT qty, price, qty*price AS value FROM t", false},
{"ALTER ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v` AS SELECT qty, price, qty*price AS value FROM t", true},
{"DROP VIEW v", true},
{"DROP VIEW v", false},
{"CREATE TABLE `VIEW`(id int)", false},
{"ALTER TABLE `VIEW`(id int)", false},

Expand Down
Loading

0 comments on commit 0cecb73

Please sign in to comment.