From 37584b4b2e7fb4d9a4267adc0deae697e1c79985 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 18 Jun 2023 16:41:02 -0400 Subject: [PATCH] Remove CI endtoend test for VReplication copy throttling (#13343) Improve the unit test and rely on that for testing the feature. This never really needed to have its own e2e test as it's a minor feature and pretty basic behavior. Signed-off-by: Matt Lord --- .../vreplication/vreplication_test.go | 7 ++ .../tabletserver/vstreamer/engine_test.go | 64 +++++++++++-------- test/config.json | 9 --- 3 files changed, 46 insertions(+), 34 deletions(-) diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 5693dfd14a7..38a1747103a 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -221,6 +221,13 @@ func TestVReplicationDDLHandling(t *testing.T) { moveTablesAction(t, "Cancel", defaultCellName, workflow, sourceKs, targetKs, table) } +// TestVreplicationCopyThrottling tests the logic that is used +// by vstreamer when starting a copy phase cycle. +// This logic today supports waiting for MySQL replication lag +// and/or InnoDB MVCC history to be below a certain threshold +// before starting the next copy phase. This test focuses on +// the innodb history list length check. +// NOTE: this is a manual test. It is not executed in the CI. func TestVreplicationCopyThrottling(t *testing.T) { workflow := "copy-throttling" cell := "zone1" diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine_test.go b/go/vt/vttablet/tabletserver/vstreamer/engine_test.go index e73912b6088..7324e59c3b9 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine_test.go @@ -164,8 +164,30 @@ func expectUpdateCount(t *testing.T, wantCount int64) int64 { panic("unreachable") } +// TestVStreamerWaitForMySQL tests the wait for MySQL to catch-up +// logic that is used by vstreamer when starting a copy phase cycle. +// This logic today supports waiting for MySQL replication lag +// and/or InnoDB MVCC history to be below a certain threshold before +// starting the next copy phase. func TestVStreamerWaitForMySQL(t *testing.T) { tableName := "test" + expectedWaits := int64(0) + testDB := fakesqldb.New(t) + hostres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "hostname|port", + "varchar|int64"), + "localhost|3306", + ) + thlres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "history_len", + "int64"), + "1000", + ) + sbmres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "Seconds_Behind_Master", + "int64"), + "10", + ) type fields struct { vse *Engine cp dbconfigs.Connector @@ -175,19 +197,21 @@ func TestVStreamerWaitForMySQL(t *testing.T) { maxMySQLReplLagSecs int64 } tests := []struct { - name string - fields fields - wantErr bool + name string + fields fields + shouldWait bool + wantErr bool }{ { name: "Small InnoDB MVCC impact limit", fields: fields{ vse: engine, se: engine.se, - maxInnoDBTrxHistLen: 100, + maxInnoDBTrxHistLen: 100, // Should wait on this maxMySQLReplLagSecs: 5000, }, - wantErr: true, + shouldWait: true, + wantErr: true, }, { name: "Small Repl Lag impact limit", @@ -195,9 +219,10 @@ func TestVStreamerWaitForMySQL(t *testing.T) { vse: engine, se: engine.se, maxInnoDBTrxHistLen: 10000, - maxMySQLReplLagSecs: 5, + maxMySQLReplLagSecs: 5, // Should wait on this }, - wantErr: true, + shouldWait: true, + wantErr: true, }, { name: "Large impact limits", @@ -210,25 +235,11 @@ func TestVStreamerWaitForMySQL(t *testing.T) { wantErr: false, }, } - testDB := fakesqldb.New(t) - hostres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( - "hostname|port", - "varchar|int64"), - "localhost|3306", - ) - thlres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( - "history_len", - "int64"), - "1000", - ) - sbmres := sqltypes.MakeTestResult(sqltypes.MakeTestFields( - "Seconds_Behind_Master", - "int64"), - "10", - ) + testDB.AddQuery(hostQuery, hostres) testDB.AddQuery(trxHistoryLenQuery, thlres) testDB.AddQuery(replicaLagQuery, sbmres) + for _, tt := range tests { tt.fields.cp = testDB.ConnParams() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -247,9 +258,12 @@ func TestVStreamerWaitForMySQL(t *testing.T) { if err := uvs.vse.waitForMySQL(ctx, uvs.cp, tableName); (err != nil) != tt.wantErr { t.Errorf("vstreamer.waitForMySQL() error = %v, wantErr %v", err, tt.wantErr) } + if tt.shouldWait { + expectedWaits++ + } }) } - require.Equal(t, engine.rowStreamerWaits.Counts()["VStreamerTest.waitForMySQL"], int64(2)) - require.Equal(t, engine.vstreamerPhaseTimings.Counts()["VStreamerTest."+tableName+":waitForMySQL"], int64(2)) + require.Equal(t, engine.rowStreamerWaits.Counts()["VStreamerTest.waitForMySQL"], expectedWaits) + require.Equal(t, engine.vstreamerPhaseTimings.Counts()["VStreamerTest."+tableName+":waitForMySQL"], expectedWaits) } diff --git a/test/config.json b/test/config.json index 0bf5c2ab31a..0070720f1a0 100644 --- a/test/config.json +++ b/test/config.json @@ -1035,15 +1035,6 @@ "RetryMax": 1, "Tags": [] }, - "vreplication_copy_throttling": { - "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestVreplicationCopyThrottling"], - "Command": [], - "Manual": false, - "Shard": "vreplication_basic", - "RetryMax": 1, - "Tags": [] - }, "vreplication_copy_parallel": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestVreplicationCopyParallel"],