Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

VTOrc: Update the primary key for all the tables from hostname, port to alias #13243

Merged
merged 21 commits into from
Jun 20, 2023
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
dd2d727
feat: remove unused code and rename orchestrator file
GuptaManan100 Jun 6, 2023
514357a
feat: remove code for hostname resolution since it is unrequired
GuptaManan100 Jun 6, 2023
14a7980
feat: remove more unused code and tables
GuptaManan100 Jun 6, 2023
bcaecb9
feat: remove database_instance_downtime from VTOrc as we weren't inse…
GuptaManan100 Jun 6, 2023
f4c0431
feat: get rid of group replication information that isn't required
GuptaManan100 Jun 13, 2023
d26b242
feat: fix bugs
GuptaManan100 Jun 14, 2023
9b85890
test: fix insert queries for test
GuptaManan100 Jun 14, 2023
06e4026
feat: introduce alias in all the tables and use it as the primary key…
GuptaManan100 Jun 15, 2023
d49b36e
Merge remote-tracking branch 'upstream/main' into vtorc-alias-primary…
GuptaManan100 Jun 15, 2023
72affbf
feat: fix all the unit tests
GuptaManan100 Jun 15, 2023
a2ed8e6
feat: fix end to end test expectations
GuptaManan100 Jun 15, 2023
6f27c67
feat: remove unused parameters
GuptaManan100 Jun 16, 2023
0e2c037
feat: add test for auditInstanceAnalysisInChangelog
GuptaManan100 Jun 16, 2023
33d1369
test: augment test to also verify reading of audit operations works a…
GuptaManan100 Jun 16, 2023
3885b63
feat: remove generics code in VTOrc around durability policies since …
GuptaManan100 Jun 16, 2023
1929395
feat: add tests for more functions changed in this PR
GuptaManan100 Jun 16, 2023
347dc87
feat: refactor forgetting instance code a little and add tests for it
GuptaManan100 Jun 16, 2023
e574ccb
feat: fix snapshot topologies code and add test for it
GuptaManan100 Jun 16, 2023
6c45fe4
test: add tests for savetablet and readtablet
GuptaManan100 Jun 16, 2023
703da07
feat: fix remaining tests
GuptaManan100 Jun 16, 2023
05bf273
feat: fix comment
GuptaManan100 Jun 20, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions go/test/endtoend/vtorc/api/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,20 +115,20 @@ func TestAPIEndpoints(t *testing.T) {
return response == "[]"
})
assert.Equal(t, 200, status, resp)
assert.Contains(t, resp, fmt.Sprintf(`"Port": %d`, replica.MySQLPort))
assert.Contains(t, resp, fmt.Sprintf(`"AnalyzedInstanceAlias": "%s"`, replica.Alias))
assert.Contains(t, resp, `"Analysis": "ReplicationStopped"`)

// Verify that filtering also works in the API as intended
status, resp, err = utils.MakeAPICall(t, vtorc, "/api/replication-analysis?keyspace=ks&shard=0")
require.NoError(t, err)
assert.Equal(t, 200, status, resp)
assert.Contains(t, resp, fmt.Sprintf(`"Port": %d`, replica.MySQLPort))
assert.Contains(t, resp, fmt.Sprintf(`"AnalyzedInstanceAlias": "%s"`, replica.Alias))

// Verify that filtering by keyspace also works in the API as intended
status, resp, err = utils.MakeAPICall(t, vtorc, "/api/replication-analysis?keyspace=ks")
require.NoError(t, err)
assert.Equal(t, 200, status, resp)
assert.Contains(t, resp, fmt.Sprintf(`"Port": %d`, replica.MySQLPort))
assert.Contains(t, resp, fmt.Sprintf(`"AnalyzedInstanceAlias": "%s"`, replica.Alias))

// Check that filtering using keyspace and shard works
status, resp, err = utils.MakeAPICall(t, vtorc, "/api/replication-analysis?keyspace=ks&shard=80-")
Expand Down
16 changes: 8 additions & 8 deletions go/test/endtoend/vtorc/readtopologyinstance/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,11 @@ func TestReadTopologyInstanceBufferable(t *testing.T) {
}
}

primaryInstance, err := inst.ReadTopologyInstanceBufferable(&inst.InstanceKey{
Hostname: utils.Hostname,
Port: primary.MySQLPort,
}, nil)
primaryInstance, err := inst.ReadTopologyInstanceBufferable(primary.Alias, nil)
require.NoError(t, err)
require.NotNil(t, primaryInstance)
assert.Equal(t, utils.Hostname, primaryInstance.Hostname)
assert.Equal(t, primary.MySQLPort, primaryInstance.Port)
assert.Contains(t, primaryInstance.InstanceAlias, "zone1")
assert.NotEqual(t, 0, primaryInstance.ServerID)
assert.Greater(t, len(primaryInstance.ServerUUID), 10)
Expand Down Expand Up @@ -120,12 +119,11 @@ func TestReadTopologyInstanceBufferable(t *testing.T) {
err = logic.EnableRecovery()
require.NoError(t, err)

replicaInstance, err := inst.ReadTopologyInstanceBufferable(&inst.InstanceKey{
Hostname: utils.Hostname,
Port: replica.MySQLPort,
}, nil)
replicaInstance, err := inst.ReadTopologyInstanceBufferable(replica.Alias, nil)
require.NoError(t, err)
require.NotNil(t, replicaInstance)
assert.Equal(t, utils.Hostname, replicaInstance.Hostname)
assert.Equal(t, replica.MySQLPort, replicaInstance.Port)
assert.Contains(t, replicaInstance.InstanceAlias, "zone1")
assert.NotEqual(t, 0, replicaInstance.ServerID)
assert.Greater(t, len(replicaInstance.ServerUUID), 10)
Expand All @@ -137,6 +135,8 @@ func TestReadTopologyInstanceBufferable(t *testing.T) {
assert.Equal(t, "ROW", replicaInstance.BinlogFormat)
assert.Equal(t, "ON", replicaInstance.GTIDMode)
assert.Equal(t, "FULL", replicaInstance.BinlogRowImage)
assert.Equal(t, utils.Hostname, replicaInstance.SourceHost)
assert.Equal(t, primary.MySQLPort, replicaInstance.SourcePort)
assert.Contains(t, replicaInstance.SelfBinlogCoordinates.LogFile, fmt.Sprintf("vt-0000000%d-bin", replica.TabletUID))
assert.Greater(t, replicaInstance.SelfBinlogCoordinates.LogPos, uint32(0))
assert.False(t, replicaInstance.SemiSyncPrimaryEnabled)
Expand Down
26 changes: 26 additions & 0 deletions go/vt/topotools/tablet.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,3 +245,29 @@ func TabletIdent(tablet *topodatapb.Tablet) string {
func TargetIdent(target *querypb.Target) string {
return fmt.Sprintf("%s/%s (%s)", target.Keyspace, target.Shard, target.TabletType)
}

// TabletEquality returns true iff two Tablets are identical for testing purposes
func TabletEquality(left, right *topodatapb.Tablet) bool {
if left.Keyspace != right.Keyspace {
return false
}
if left.Shard != right.Shard {
return false
}
if left.Hostname != right.Hostname {
return false
}
if left.Type != right.Type {
return false
}
if left.MysqlHostname != right.MysqlHostname {
return false
}
if left.MysqlPort != right.MysqlPort {
return false
}
if left.PrimaryTermStartTime.String() != right.PrimaryTermStartTime.String() {
return false
}
return topoproto.TabletAliasString(left.Alias) == topoproto.TabletAliasString(right.Alias)
}
2 changes: 0 additions & 2 deletions go/vt/vtorc/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,7 @@ const (
DiscoveryQueueCapacity = 100000
DiscoveryQueueMaxStatisticsSize = 120
DiscoveryCollectionRetentionSeconds = 120
HostnameResolveMethod = "default"
UnseenInstanceForgetHours = 240 // Number of hours after which an unseen instance is forgotten
ExpiryHostnameResolvesMinutes = 60 // Number of minutes after which to expire hostname-resolves
CandidateInstanceExpireMinutes = 60 // Minutes after which a suggestion to use an instance as a candidate replica (to be preferably promoted on primary failover) is expired.
FailureDetectionPeriodBlockMinutes = 60 // The time for which an instance's failure discovery is kept "active", so as to avoid concurrent "discoveries" of the instance's failure; this preceeds any recovery process, if any.
)
Expand Down
16 changes: 2 additions & 14 deletions go/vt/vtorc/db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func translateStatement(statement string) string {
return sqlutils.ToSqlite3Dialect(statement)
}

// registerVTOrcDeployment updates the vtorc_metadata table upon successful deployment
// registerVTOrcDeployment updates the vtorc_db_deployments table upon successful deployment
func registerVTOrcDeployment(db *sql.DB) error {
query := `
replace into vtorc_db_deployments (
Expand All @@ -82,7 +82,7 @@ func registerVTOrcDeployment(db *sql.DB) error {
)
`
if _, err := execInternal(db, query, ""); err != nil {
log.Fatalf("Unable to write to vtorc_metadata: %+v", err)
log.Fatalf("Unable to write to vtorc_db_deployments: %+v", err)
}
return nil
}
Expand Down Expand Up @@ -188,15 +188,3 @@ func QueryVTOrc(query string, argsArray []any, onRow func(sqlutils.RowMap) error

return err
}

// ReadTimeNow reads and returns the current timestamp as string. This is an unfortunate workaround
// to support both MySQL and SQLite in all possible timezones. SQLite only speaks UTC where MySQL has
// timezone support. By reading the time as string we get the database's de-facto notion of the time,
// which we can then feed back to it.
func ReadTimeNow() (timeNow string, err error) {
err = QueryVTOrc(`select now() as time_now`, nil, func(m sqlutils.RowMap) error {
timeNow = m.GetString("time_now")
return nil
})
return timeNow, err
}
Loading