Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace into statement plan with foreign keys : unsharded #14396

Merged
merged 20 commits into from
Nov 15, 2023
Merged

Replace into statement plan with foreign keys : unsharded #14396

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
95a6673
feat: retrieve and store indexes from ddl on table info in schema tra…
harshit-gangal Oct 27, 2023
97a1462
update vschema using schema tracking indexes information
harshit-gangal Oct 27, 2023
f3c6e92
serialize engine and operator
GuptaManan100 Oct 23, 2023
2a01008
feat: add support for replace into query with foreign key in unsharde…
harshit-gangal Oct 30, 2023
77e966e
prevent auto-commit for unsharded dml when using sequential engine
harshit-gangal Nov 1, 2023
31e8881
omit empty in vschema table json marshal
harshit-gangal Nov 1, 2023
f569ddf
fix replace planner and allow only when primarykey or uniquekey is known
harshit-gangal Nov 1, 2023
612d513
sizegen: update
harshit-gangal Nov 1, 2023
150bdc5
fix the sequential condition
harshit-gangal Nov 1, 2023
d1ef281
track column default
harshit-gangal Nov 1, 2023
4b6c0a1
fix tracker test after key formatting change
harshit-gangal Nov 2, 2023
39ef21d
use column default when column not present in replace into query
harshit-gangal Nov 2, 2023
c661fda
fix: sequential engine to append the rowsAffected, take insertID and …
harshit-gangal Nov 2, 2023
e792ac6
unique keys to add when can return rows on delete, added tests
harshit-gangal Nov 2, 2023
6fca56a
add fuzzer test for replace into
harshit-gangal Nov 6, 2023
7806d1e
addressed review comments
harshit-gangal Nov 9, 2023
601133f
added replace fk fuzz test
harshit-gangal Nov 13, 2023
63a9075
address review comments
harshit-gangal Nov 13, 2023
f6576ee
disable replace into statement with foreign keys
harshit-gangal Nov 13, 2023
80760a7
review comments
harshit-gangal Nov 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go/mysql/sqlerror/sql_error.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ var stateToMysqlCode = map[vterrors.State]mysqlCode{
vterrors.CharacterSetMismatch: {num: ERCharacterSetMismatch, state: SSUnknownSQLState},
vterrors.WrongParametersToNativeFct: {num: ERWrongParametersToNativeFct, state: SSUnknownSQLState},
vterrors.KillDeniedError: {num: ERKillDenied, state: SSUnknownSQLState},
vterrors.BadNullError: {num: ERBadNullError, state: SSConstraintViolation},
}

func getStateToMySQLState(state vterrors.State) mysqlCode {
Expand Down
164 changes: 96 additions & 68 deletions go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,9 @@ func (fz *fuzzer) generateQuery() []string {
if val < fz.insertShare {
switch fz.queryFormat {
case OlapSQLQueries, SQLQueries:
return []string{fz.generateInsertDMLQuery()}
return []string{fz.generateInsertDMLQuery(getInsertType())}
case PreparedStatmentQueries:
return fz.getPreparedInsertQueries()
return fz.getPreparedInsertQueries(getInsertType())
default:
panic("Unknown query type")
}
Expand All @@ -122,22 +122,26 @@ func (fz *fuzzer) generateQuery() []string {
}
}

func getInsertType() string {
harshit-gangal marked this conversation as resolved.
Show resolved Hide resolved
return "insert"
}

// generateInsertDMLQuery generates an INSERT query from the parameters for the fuzzer.
func (fz *fuzzer) generateInsertDMLQuery() string {
func (fz *fuzzer) generateInsertDMLQuery(insertType string) string {
tableId := rand.Intn(len(fkTables))
idValue := 1 + rand.Intn(fz.maxValForId)
tableName := fkTables[tableId]
if tableName == "fk_t20" {
colValue := rand.Intn(1 + fz.maxValForCol)
col2Value := rand.Intn(1 + fz.maxValForCol)
return fmt.Sprintf("insert into %v (id, col, col2) values (%v, %v, %v)", tableName, idValue, convertIntValueToString(colValue), convertIntValueToString(col2Value))
return fmt.Sprintf("%s into %v (id, col, col2) values (%v, %v, %v)", insertType, tableName, idValue, convertIntValueToString(colValue), convertIntValueToString(col2Value))
} else if isMultiColFkTable(tableName) {
colaValue := rand.Intn(1 + fz.maxValForCol)
colbValue := rand.Intn(1 + fz.maxValForCol)
return fmt.Sprintf("insert into %v (id, cola, colb) values (%v, %v, %v)", tableName, idValue, convertIntValueToString(colaValue), convertIntValueToString(colbValue))
return fmt.Sprintf("%s into %v (id, cola, colb) values (%v, %v, %v)", insertType, tableName, idValue, convertIntValueToString(colaValue), convertIntValueToString(colbValue))
} else {
colValue := rand.Intn(1 + fz.maxValForCol)
return fmt.Sprintf("insert into %v (id, col) values (%v, %v)", tableName, idValue, convertIntValueToString(colValue))
return fmt.Sprintf("%s into %v (id, col) values (%v, %v)", insertType, tableName, idValue, convertIntValueToString(colValue))
}
}

Expand Down Expand Up @@ -332,15 +336,15 @@ func (fz *fuzzer) getPreparedDeleteQueries() []string {
}

// getPreparedInsertQueries gets the list of queries to run for executing an INSERT using prepared statements.
func (fz *fuzzer) getPreparedInsertQueries() []string {
func (fz *fuzzer) getPreparedInsertQueries(insertType string) []string {
tableId := rand.Intn(len(fkTables))
idValue := 1 + rand.Intn(fz.maxValForId)
tableName := fkTables[tableId]
if tableName == "fk_t20" {
colValue := rand.Intn(1 + fz.maxValForCol)
col2Value := rand.Intn(1 + fz.maxValForCol)
return []string{
"prepare stmt_insert from 'insert into fk_t20 (id, col, col2) values (?, ?, ?)'",
fmt.Sprintf("prepare stmt_insert from '%s into fk_t20 (id, col, col2) values (?, ?, ?)'", insertType),
fmt.Sprintf("SET @id = %v", idValue),
fmt.Sprintf("SET @col = %v", convertIntValueToString(colValue)),
fmt.Sprintf("SET @col2 = %v", convertIntValueToString(col2Value)),
Expand All @@ -350,7 +354,7 @@ func (fz *fuzzer) getPreparedInsertQueries() []string {
colaValue := rand.Intn(1 + fz.maxValForCol)
colbValue := rand.Intn(1 + fz.maxValForCol)
return []string{
fmt.Sprintf("prepare stmt_insert from 'insert into %v (id, cola, colb) values (?, ?, ?)'", tableName),
fmt.Sprintf("prepare stmt_insert from '%s into %v (id, cola, colb) values (?, ?, ?)'", insertType, tableName),
fmt.Sprintf("SET @id = %v", idValue),
fmt.Sprintf("SET @cola = %v", convertIntValueToString(colaValue)),
fmt.Sprintf("SET @colb = %v", convertIntValueToString(colbValue)),
Expand All @@ -359,7 +363,7 @@ func (fz *fuzzer) getPreparedInsertQueries() []string {
} else {
colValue := rand.Intn(1 + fz.maxValForCol)
return []string{
fmt.Sprintf("prepare stmt_insert from 'insert into %v (id, col) values (?, ?)'", tableName),
fmt.Sprintf("prepare stmt_insert from '%s into %v (id, col) values (?, ?)'", insertType, tableName),
fmt.Sprintf("SET @id = %v", idValue),
fmt.Sprintf("SET @col = %v", convertIntValueToString(colValue)),
"execute stmt_insert using @id, @col",
Expand Down Expand Up @@ -407,7 +411,7 @@ func (fz *fuzzer) getPreparedUpdateQueries() []string {
func (fz *fuzzer) generateParameterizedQuery() (query string, params []any) {
val := rand.Intn(fz.insertShare + fz.updateShare + fz.deleteShare)
if val < fz.insertShare {
return fz.generateParameterizedInsertQuery()
return fz.generateParameterizedInsertQuery(getInsertType())
}
if val < fz.insertShare+fz.updateShare {
return fz.generateParameterizedUpdateQuery()
Expand All @@ -416,21 +420,21 @@ func (fz *fuzzer) generateParameterizedQuery() (query string, params []any) {
}

// generateParameterizedInsertQuery generates a parameterized INSERT query for the query format PreparedStatementPacket.
func (fz *fuzzer) generateParameterizedInsertQuery() (query string, params []any) {
func (fz *fuzzer) generateParameterizedInsertQuery(insertType string) (query string, params []any) {
tableId := rand.Intn(len(fkTables))
idValue := 1 + rand.Intn(fz.maxValForId)
tableName := fkTables[tableId]
if tableName == "fk_t20" {
colValue := rand.Intn(1 + fz.maxValForCol)
col2Value := rand.Intn(1 + fz.maxValForCol)
return fmt.Sprintf("insert into %v (id, col, col2) values (?, ?, ?)", tableName), []any{idValue, convertIntValueToString(colValue), convertIntValueToString(col2Value)}
return fmt.Sprintf("%s into %v (id, col, col2) values (?, ?, ?)", insertType, tableName), []any{idValue, convertIntValueToString(colValue), convertIntValueToString(col2Value)}
} else if isMultiColFkTable(tableName) {
colaValue := rand.Intn(1 + fz.maxValForCol)
colbValue := rand.Intn(1 + fz.maxValForCol)
return fmt.Sprintf("insert into %v (id, cola, colb) values (?, ?, ?)", tableName), []any{idValue, convertIntValueToString(colaValue), convertIntValueToString(colbValue)}
return fmt.Sprintf("%s into %v (id, cola, colb) values (?, ?, ?)", insertType, tableName), []any{idValue, convertIntValueToString(colaValue), convertIntValueToString(colbValue)}
} else {
colValue := rand.Intn(1 + fz.maxValForCol)
return fmt.Sprintf("insert into %v (id, col) values (?, ?)", tableName), []any{idValue, convertIntValueToString(colValue)}
return fmt.Sprintf("%s into %v (id, col) values (?, ?)", insertType, tableName), []any{idValue, convertIntValueToString(colValue)}
}
}

Expand Down Expand Up @@ -555,58 +559,61 @@ func TestFkFuzzTest(t *testing.T) {
insertShare int
deleteShare int
updateShare int
}{
{
name: "Single Thread - Only Inserts",
concurrency: 1,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 10,
insertShare: 100,
deleteShare: 0,
updateShare: 0,
},
{
name: "Single Thread - Balanced Inserts and Deletes",
concurrency: 1,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 10,
insertShare: 50,
deleteShare: 50,
updateShare: 0,
},
{
name: "Single Thread - Balanced Inserts and Updates",
concurrency: 1,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 10,
insertShare: 50,
deleteShare: 0,
updateShare: 50,
},
{
name: "Single Thread - Balanced Inserts, Updates and Deletes",
concurrency: 1,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 10,
insertShare: 50,
deleteShare: 50,
updateShare: 50,
},
{
name: "Multi Thread - Balanced Inserts, Updates and Deletes",
concurrency: 30,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 30,
insertShare: 50,
deleteShare: 50,
updateShare: 50,
},
}
}{{
name: "Single Thread - Only Inserts",
concurrency: 1,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 10,
insertShare: 100,
deleteShare: 0,
updateShare: 0,
}, {
name: "Single Thread - Balanced Inserts and Deletes",
concurrency: 1,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 10,
insertShare: 50,
deleteShare: 50,
updateShare: 0,
}, {
name: "Single Thread - Balanced Inserts and Updates",
concurrency: 1,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 10,
insertShare: 50,
deleteShare: 0,
updateShare: 50,
}, {
name: "Single Thread - Balanced Inserts, Updates and Deletes",
concurrency: 1,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 10,
insertShare: 50,
deleteShare: 50,
updateShare: 50,
}, {
name: "Multi Thread - Only Inserts",
concurrency: 30,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 30,
insertShare: 100,
deleteShare: 0,
updateShare: 0,
}, {
name: "Multi Thread - Balanced Inserts, Updates and Deletes",
concurrency: 30,
timeForTesting: 5 * time.Second,
maxValForCol: 5,
maxValForId: 30,
insertShare: 50,
deleteShare: 50,
updateShare: 50,
}}
Comment on lines +607 to +616
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like this test is failing on CI:

    --- FAIL: TestFkFuzzTest/Unsharded_-_Multi_Thread_-_Balanced_Inserts,_Updates_and_Deletes_QueryFormat_-_PreparedStatementPacket (4.78s)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The failure looks unrelated to this PR. Investigating it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PR that fixes the issue #14524


for _, tt := range testcases {
for _, testSharded := range []bool{false, true} {
Expand All @@ -632,7 +639,16 @@ func TestFkFuzzTest(t *testing.T) {
fz.start(t, testSharded)

// Wait for the timeForTesting so that the threads continue to run.
time.Sleep(tt.timeForTesting)
totalTime := time.After(tt.timeForTesting)
done := false
for !done {
select {
case <-totalTime:
done = true
case <-time.After(10 * time.Millisecond):
validateReplication(t)
}
}

fz.stop()

Expand All @@ -654,3 +670,15 @@ func TestFkFuzzTest(t *testing.T) {
}
}
}

func validateReplication(t *testing.T) {
for _, keyspace := range clusterInstance.Keyspaces {
for _, shard := range keyspace.Shards {
for _, vttablet := range shard.Vttablets {
if vttablet.Type != "primary" {
checkReplicationHealthy(t, vttablet)
}
}
}
}
}
Loading
Loading