Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix ApplySchema --batch-size with --allow-zero-in-date #13951

Merged
merged 3 commits into from
Sep 11, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions go/test/endtoend/vtgate/schema/schema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,18 @@ func testApplySchemaBatch(t *testing.T) {
require.NoError(t, err)
checkTables(t, totalTableCount)
}
{
sqls := "create table batch1(id int primary key);create table batch2(id int primary key);create table batch3(id int primary key);create table batch4(id int primary key);create table batch5(id int primary key);"
_, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ApplySchema", "--", "--ddl_strategy", "direct --allow-zero-in-date", "--sql", sqls, "--batch_size", "2", keyspaceName)
require.NoError(t, err)
checkTables(t, totalTableCount+5)
}
{
sqls := "drop table batch1; drop table batch2; drop table batch3; drop table batch4; drop table batch5"
_, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ApplySchema", "--", "--sql", sqls, keyspaceName)
require.NoError(t, err)
checkTables(t, totalTableCount)
}
}

// checkTables checks the number of tables in the first two shards.
Expand Down
36 changes: 29 additions & 7 deletions go/vt/schemamanager/tablet_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,33 @@ func (exec *TabletExecutor) executeOnAllTablets(ctx context.Context, execResult
}
}

// applyAllowZeroInDate takes a SQL string which may contain one or more statements,
// and, assuming those are DDLs, adds a /*vt+ allowZeroInDate=true */ directive to all of them,
// returning the result again as one long SQL.
func applyAllowZeroInDate(sql string) (string, error) {
// sql may be a batch of multiple statements
sqls, err := sqlparser.SplitStatementToPieces(sql)
if err != nil {
return sql, err
}
modifiedSqls := []string{}
shlomi-noach marked this conversation as resolved.
Show resolved Hide resolved
for _, singleSQL := range sqls {
// --allow-zero-in-date Applies to DDLs
stmt, err := sqlparser.Parse(singleSQL)
if err != nil {
return sql, err
}
if ddlStmt, ok := stmt.(sqlparser.DDLStatement); ok {
// Add comments directive to allow zero in date
const directive = `/*vt+ allowZeroInDate=true */`
ddlStmt.SetComments(ddlStmt.GetParsedComments().Prepend(directive))
singleSQL = sqlparser.String(ddlStmt)
}
modifiedSqls = append(modifiedSqls, singleSQL)
}
return strings.Join(modifiedSqls, ";"), err
}

func (exec *TabletExecutor) executeOneTablet(
ctx context.Context,
tablet *topodatapb.Tablet,
Expand All @@ -459,22 +486,17 @@ func (exec *TabletExecutor) executeOneTablet(
} else {
if exec.ddlStrategySetting != nil && exec.ddlStrategySetting.IsAllowZeroInDateFlag() {
// --allow-zero-in-date Applies to DDLs
stmt, err := sqlparser.Parse(string(sql))
sql, err = applyAllowZeroInDate(sql)
if err != nil {
errChan <- ShardWithError{Shard: tablet.Shard, Err: err.Error()}
return
}
if ddlStmt, ok := stmt.(sqlparser.DDLStatement); ok {
// Add comments directive to allow zero in date
const directive = `/*vt+ allowZeroInDate=true */`
ddlStmt.SetComments(ddlStmt.GetParsedComments().Prepend(directive))
sql = sqlparser.String(ddlStmt)
}
}
result, err = exec.tmc.ExecuteFetchAsDba(ctx, tablet, false, &tabletmanagerdatapb.ExecuteFetchAsDbaRequest{
Query: []byte(sql),
MaxRows: 10,
})

}
if err != nil {
errChan <- ShardWithError{Shard: tablet.Shard, Err: err.Error()}
Expand Down
35 changes: 35 additions & 0 deletions go/vt/schemamanager/tablet_executor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -408,3 +408,38 @@ func TestAllSQLsAreCreateQueries(t *testing.T) {
})
}
}

func TestApplyAllowZeroInDate(t *testing.T) {
tcases := []struct {
sql string
expect string
}{
{
"create table t1(id int primary key); ",
"create /*vt+ allowZeroInDate=true */ table t1 (\n\tid int primary key\n)",
},
{
"create table t1(id int primary key)",
"create /*vt+ allowZeroInDate=true */ table t1 (\n\tid int primary key\n)",
},
{
"create table t1(id int primary key);select 1 from dual",
"create /*vt+ allowZeroInDate=true */ table t1 (\n\tid int primary key\n);select 1 from dual",
},
{
"create table t1(id int primary key); alter table t2 add column id2 int",
"create /*vt+ allowZeroInDate=true */ table t1 (\n\tid int primary key\n);alter /*vt+ allowZeroInDate=true */ table t2 add column id2 int",
},
{
" ; ; ;;; create table t1(id int primary key); ;; alter table t2 add column id2 int ;;",
"create /*vt+ allowZeroInDate=true */ table t1 (\n\tid int primary key\n);alter /*vt+ allowZeroInDate=true */ table t2 add column id2 int",
},
}
for _, tcase := range tcases {
t.Run(tcase.sql, func(t *testing.T) {
result, err := applyAllowZeroInDate(tcase.sql)
assert.NoError(t, err)
assert.Equal(t, tcase.expect, result)
})
}
}
17 changes: 16 additions & 1 deletion go/vt/sqlparser/ast_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,22 @@ func TestSplitStatementToPieces(t *testing.T) {
"`createtime` datetime NOT NULL DEFAULT NOW() COMMENT 'create time;'," +
"`comment` varchar(100) NOT NULL DEFAULT '' COMMENT 'comment'," +
"PRIMARY KEY (`id`))",
}}
}, {
input: "create table t1 (id int primary key); create table t2 (id int primary key);",
output: "create table t1 (id int primary key); create table t2 (id int primary key)",
}, {
input: ";;; create table t1 (id int primary key);;; ;create table t2 (id int primary key);",
output: " create table t1 (id int primary key);create table t2 (id int primary key)",
}, {
// The input doesn't have to be valid SQL statements!
input: ";create table t1 ;create table t2 (id;",
output: "create table t1 ;create table t2 (id",
}, {
// Ignore quoted semicolon
input: ";create table t1 ';';;;create table t2 (id;",
output: "create table t1 ';';create table t2 (id",
},
}

for _, tcase := range testcases {
t.Run(tcase.input, func(t *testing.T) {
Expand Down