vttablet sidecar schema:use schemadiff to reach desired schema on tab…

…let init replacing the withDDL-based approach (#11520)

* Add sidecardb module based schema init. Remove metatables. Conditionally disable withddl/health streamer schema init. Fix tests as needed.

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Implement review comments

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* update to column type

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

* Fix bug where we were using _vt before creating it. Added e2e test to CI

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Reinstate creating vtdatabase optionally

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Address review comments

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Fix missed function rename

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Use schemadiff's new ignore charset/collate hints

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Add comments to schema files. Normalize schema while reading from file to get rid of the comments and make related changes to tests

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Refactor e2e test

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Addressed more review comments

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Moved sidecar test helpers back to package since it is used in vtexplain

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Fix issues with setting up fakesqldb test queries

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Address new review comments

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

* Refactor the way we first create tables

Signed-off-by: Rohit Nayak <rohit@planetscale.com>

---------

Signed-off-by: Rohit Nayak <rohit@planetscale.com>
Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
Co-authored-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

config/init_db.sql

@@ -28,23 +28,6 @@ DROP DATABASE IF EXISTS test;
 # Vitess defaults
 ###############################################################################
 
-# Vitess-internal database.
-CREATE DATABASE IF NOT EXISTS _vt;
-# Note that definitions of local_metadata and shard_metadata should be the same
-# as in production which is defined in go/vt/mysqlctl/metadata_tables.go.
-CREATE TABLE IF NOT EXISTS _vt.local_metadata (
-  name VARCHAR(255) NOT NULL,
-  value VARCHAR(255) NOT NULL,
-  db_name VARBINARY(255) NOT NULL,
-  PRIMARY KEY (db_name, name)
-  ) ENGINE=InnoDB;
-CREATE TABLE IF NOT EXISTS _vt.shard_metadata (
-  name VARCHAR(255) NOT NULL,
-  value MEDIUMBLOB NOT NULL,
-  db_name VARBINARY(255) NOT NULL,
-  PRIMARY KEY (db_name, name)
-  ) ENGINE=InnoDB;
-
 # Admin user with all privileges.
 CREATE USER 'vt_dba'@'localhost';
 GRANT ALL ON *.* TO 'vt_dba'@'localhost';

doc/releasenotes/16_0_0_summary.md

@@ -280,6 +280,8 @@ is now fixed. The full issue can be found [here](https://github.com/vitessio/vit
 
 - vtbackup flag `--backup_storage_hook` has been removed, use one of the builtin compression algorithms or `--external-compressor` and `--external-decompressor` instead.
 
+- The VTTablet flag `--init_populate_metadata` has been deprecated, since we have deleted the `local_metadata` and `shard_metadata` sidecar database tables.
+
 - The dead legacy Workflow Manager related code was removed in [#12085](https://github.com/vitessio/vitess/pull/12085). This included the following `vtctl` client commands: `WorkflowAction`, `WorkflowCreate`, `WorkflowWait`, `WorkflowStart`, `WorkflowStop`, `WorkflowTree`, `WorkflowDelete`.
 
 - VTAdmin's `VTExplain` endpoint has been deprecated. Users can use the new `vexplain` query format instead. The endpoint will be deleted in a future release.
@@ -394,3 +396,19 @@ BenchmarkCompressLz4Builtin
     PASS
     cleaning up "/var/folders/96/k7gzd7q10zdb749vr02q7sjh0000gn/T/ee7d47b45ef09786c54fa2d7354d2a68.dat"
 ```
+
+## Refactor
+
+### VTTablet sidecar schema maintenance refactor
+
+This is an internal refactor and should not change the behavior of Vitess as seen by users. 
+
+Developers will see a difference though: v16 changes the way we maintain vttablet's sidecar database schema (also referred to as the `_vt`
+database). Instead of using the `WithDDL` package, introduced in #6348, we use a declarative approach. Users will now have to update
+the desired schema in the `go/vt/sidecardb/schema` directory.
+
+The desired schema is specified, one per table. A new module `sidecardb`, compares this to the existing schema and
+performs the required `create` or `alter` to reach it. This is done whenever a primary vttablet starts up.
+
+The sidecar tables `local_metadata` and `shard_metadata` are no longer in use and all references to them are removed as
+part of this refactor. There were used previously for Orchestrator support, which has been superseded by `vtorc`.

go.mod

@@ -80,13 +80,13 @@ require (
 	golang.org/x/crypto v0.3.0 // indirect
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
 	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/net v0.2.0
+	golang.org/x/net v0.3.0
 	golang.org/x/oauth2 v0.2.0
-	golang.org/x/sys v0.2.0 // indirect
-	golang.org/x/term v0.2.0
-	golang.org/x/text v0.4.0
+	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/term v0.3.0
+	golang.org/x/text v0.5.0
 	golang.org/x/time v0.2.0
-	golang.org/x/tools v0.3.0
+	golang.org/x/tools v0.4.0
 	google.golang.org/api v0.103.0
 	google.golang.org/genproto v0.0.0-20221116193143-41c2ba794472 // indirect
 	google.golang.org/grpc v1.50.1

go.sum

@@ -953,8 +953,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
-golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk=
+golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1061,13 +1061,13 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
-golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM=
-golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
+golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1078,8 +1078,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
+golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1146,8 +1147,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM=
-golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
+golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4=
+golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

go/cmd/vtbackup/vtbackup.go

@@ -74,7 +74,6 @@ import (
 	"vitess.io/vitess/go/cmd"
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/mysql"
-	"vitess.io/vitess/go/sqlescape"
 	"vitess.io/vitess/go/vt/dbconfigs"
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/logutil"
@@ -159,7 +158,6 @@ func main() {
 
 	servenv.ParseFlags("vtbackup")
 	servenv.Init()
-
 	ctx, cancel := context.WithCancel(context.Background())
 	servenv.OnClose(func() {
 		cancel()
@@ -304,16 +302,11 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		if err := mysqld.ResetReplication(ctx); err != nil {
 			return fmt.Errorf("can't reset replication: %v", err)
 		}
-		cmds := mysqlctl.CreateReparentJournal()
-		cmds = append(cmds, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", sqlescape.EscapeID(dbName)))
-		if err := mysqld.ExecuteSuperQueryList(ctx, cmds); err != nil {
-			return fmt.Errorf("can't initialize database: %v", err)
+		cmd := mysqlctl.GenerateInitialBinlogEntry()
+		if err := mysqld.ExecuteSuperQueryList(ctx, []string{cmd}); err != nil {
+			return err
 		}
 
-		// Execute Alter commands on reparent_journal and ignore errors
-		cmds = mysqlctl.AlterReparentJournal()
-		_ = mysqld.ExecuteSuperQueryList(ctx, cmds)
-
 		backupParams.BackupTime = time.Now()
 		// Now we're ready to take the backup.
 		if err := mysqlctl.Backup(ctx, backupParams); err != nil {
@@ -331,7 +324,6 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		Logger:              logutil.NewConsoleLogger(),
 		Concurrency:         concurrency,
 		HookExtraEnv:        extraEnv,
-		LocalMetadata:       map[string]string{},
 		DeleteBeforeRestore: true,
 		DbName:              dbName,
 		Keyspace:            initKeyspace,

go/cmd/vttablet/vttablet.go

@@ -123,7 +123,6 @@ func main() {
 		UpdateStream:        binlog.NewUpdateStream(ts, tablet.Keyspace, tabletAlias.Cell, qsc.SchemaEngine()),
 		VREngine:            vreplication.NewEngine(config, ts, tabletAlias.Cell, mysqld, qsc.LagThrottler()),
 		VDiffEngine:         vdiff.NewEngine(config, ts, tablet),
-		MetadataManager:     &mysqlctl.MetadataManager{},
 	}
 	if err := tm.Start(tablet, config.Healthcheck.IntervalSeconds.Get()); err != nil {
 		log.Exitf("failed to parse --tablet-path or initialize DB credentials: %v", err)

go/flags/endtoend/vttablet.txt

@@ -151,7 +151,6 @@ Usage of vttablet:
       --hot_row_protection_max_queue_size int                            Maximum number of BeginExecute RPCs which will be queued for the same row (range). (default 20)
       --init_db_name_override string                                     (init parameter) override the name of the db used by vttablet. Without this flag, the db name defaults to vt_<keyspacename>
       --init_keyspace string                                             (init parameter) keyspace to use for this tablet
-      --init_populate_metadata                                           (init parameter) populate metadata tables even if restore_from_backup is disabled. If restore_from_backup is enabled, metadata tables are always populated regardless of this flag.
       --init_shard string                                                (init parameter) shard to use for this tablet
       --init_tablet_type string                                          (init parameter) the tablet type to use for this tablet.
       --init_tags StringMap                                              (init parameter) comma separated list of key:value pairs used to tag the tablet

go/mysql/endtoend/schema_change_test.go

@@ -32,7 +32,6 @@ import (
 var ctx = context.Background()
 
 const (
-	createDb        = `create database if not exists _vt`
 	createUserTable = `create table vttest.product (id bigint(20) primary key, name char(10) CHARACTER SET utf8 COLLATE utf8_unicode_ci, created bigint(20))`
 	dropTestTable   = `drop table if exists product`
 )
@@ -42,11 +41,6 @@ func TestChangeSchemaIsNoticed(t *testing.T) {
 	require.NoError(t, err)
 	defer conn.Close()
 
-	_, err = conn.ExecuteFetch(createDb, 1000, true)
-	require.NoError(t, err)
-	_, err = conn.ExecuteFetch(mysql.CreateSchemaCopyTable, 1000, true)
-	require.NoError(t, err)
-
 	tests := []struct {
 		name    string
 		changeQ string

go/mysql/fakesqldb/server.go

@@ -23,12 +23,13 @@ import (
 	"os"
 	"path"
 	"regexp"
-	"runtime/debug"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"vitess.io/vitess/go/vt/sqlparser"
+
 	"vitess.io/vitess/go/vt/log"
 
 	"vitess.io/vitess/go/mysql"
@@ -137,9 +138,10 @@ type ExpectedResult struct {
 }
 
 type exprResult struct {
-	expr   *regexp.Regexp
-	result *sqltypes.Result
-	err    string
+	queryPattern string
+	expr         *regexp.Regexp
+	result       *sqltypes.Result
+	err          string
 }
 
 // ExpectedExecuteFetch defines for an expected query the to be faked output.
@@ -414,8 +416,9 @@ func (db *DB) HandleQuery(c *mysql.Conn, query string, callback func(*sqltypes.R
 		return callback(&sqltypes.Result{})
 	}
 	// Nothing matched.
-	err := fmt.Errorf("fakesqldb:: query: '%s' is not supported on %v", query, db.name)
-	log.Errorf("Query not found: %s:%s", query, debug.Stack())
+	err := fmt.Errorf("fakesqldb:: query: '%s' is not supported on %v",
+		sqlparser.TruncateForUI(query), db.name)
+	log.Errorf("Query not found: %s", sqlparser.TruncateForUI(query))
 
 	return err
 }
@@ -557,15 +560,15 @@ func (db *DB) AddQueryPattern(queryPattern string, expectedResult *sqltypes.Resu
 	result := *expectedResult
 	db.mu.Lock()
 	defer db.mu.Unlock()
-	db.patternData[queryPattern] = exprResult{expr: expr, result: &result}
+	db.patternData[queryPattern] = exprResult{queryPattern: queryPattern, expr: expr, result: &result}
 }
 
 // RejectQueryPattern allows a query pattern to be rejected with an error
 func (db *DB) RejectQueryPattern(queryPattern, error string) {
 	expr := regexp.MustCompile("(?is)^" + queryPattern + "$")
 	db.mu.Lock()
 	defer db.mu.Unlock()
-	db.patternData[queryPattern] = exprResult{expr: expr, err: error}
+	db.patternData[queryPattern] = exprResult{queryPattern: queryPattern, expr: expr, err: error}
 }
 
 // ClearQueryPattern removes all query patterns set up

go/mysql/schema.go

@@ -40,22 +40,6 @@ const (
 	// ShowRowsRead is the query used to find the number of rows read.
 	ShowRowsRead = "show status like 'Innodb_rows_read'"
 
-	// CreateVTDatabase creates the _vt database
-	CreateVTDatabase = `CREATE DATABASE IF NOT EXISTS _vt`
-
-	// CreateSchemaCopyTable query creates schemacopy table in _vt schema.
-	CreateSchemaCopyTable = `
-CREATE TABLE if not exists _vt.schemacopy (
-	table_schema varchar(64) NOT NULL,
-	table_name varchar(64) NOT NULL,
-	column_name varchar(64) NOT NULL,
-	ordinal_position bigint(21) unsigned NOT NULL,
-	character_set_name varchar(32) DEFAULT NULL,
-	collation_name varchar(32) DEFAULT NULL,
-	data_type varchar(64) NOT NULL,
-	column_key varchar(3) NOT NULL,
-	PRIMARY KEY (table_schema, table_name, ordinal_position))`
-
 	// DetectSchemaChange query detects if there is any schema change from previous copy.
 	DetectSchemaChange = `
 SELECT DISTINCT table_name
@@ -103,13 +87,6 @@ order by table_name, ordinal_position`
 	GetColumnNamesQueryPatternForTable = `SELECT COLUMN_NAME.*TABLE_NAME.*%s.*`
 
 	// Views
-	CreateViewsTable = `CREATE TABLE IF NOT EXISTS _vt.views (
-	TABLE_NAME varchar(64) NOT NULL,
-	VIEW_DEFINITION longtext NOT NULL,
-	CREATE_STATEMENT longtext NOT NULL,
-	UPDATED_AT TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-	PRIMARY KEY (TABLE_NAME))`
-
 	InsertIntoViewsTable = `insert into _vt.views (
 	table_name,
 	view_definition,
@@ -137,13 +114,6 @@ order by table_name, ordinal_position`
 	FetchViews = `select table_name, view_definition, create_statement from _vt.views`
 )
 
-// VTDatabaseInit contains all the schema creation queries needed to
-var VTDatabaseInit = []string{
-	CreateVTDatabase,
-	CreateSchemaCopyTable,
-	CreateViewsTable,
-}
-
 // BaseShowTablesFields contains the fields returned by a BaseShowTables or a BaseShowTablesForTable command.
 // They are validated by the
 // testBaseShowTables test.

go/test/endtoend/backup/vtbackup/backup_only_test.go

@@ -36,7 +36,7 @@ import (
 
 var (
 	vtInsertTest = `
-		create table vt_insert_test (
+		create table if not exists vt_insert_test (
 		id bigint auto_increment,
 		msg varchar(64),
 		primary key (id)
@@ -63,8 +63,14 @@ func TestTabletInitialBackup(t *testing.T) {
 	initTablets(t, false, false)
 
 	// Restore the Tablets
+
 	restore(t, primary, "replica", "NOT_SERVING")
+	// Vitess expects that the user has set the database into ReadWrite mode before calling
+	// TabletExternallyReparented
 	err := localCluster.VtctlclientProcess.ExecuteCommand(
+		"SetReadWrite", primary.Alias)
+	require.Nil(t, err)
+	err = localCluster.VtctlclientProcess.ExecuteCommand(
 		"TabletExternallyReparented", primary.Alias)
 	require.Nil(t, err)
 	restore(t, replica1, "replica", "SERVING")
@@ -137,7 +143,7 @@ func firstBackupTest(t *testing.T, tabletType string) {
 	require.Nil(t, err)
 	cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 2)
 
-	// eventhough we change the value of compression it won't effect
+	// even though we change the value of compression it won't affect
 	// decompression since it gets its value from MANIFEST file, created
 	// as part of backup.
 	mysqlctl.CompressionEngineName = "lz4"
@@ -151,20 +157,6 @@ func firstBackupTest(t *testing.T, tabletType string) {
 	//check the new replica has the data
 	cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2)
 
-	// check that the restored replica has the right local_metadata
-	result, err := replica2.VttabletProcess.QueryTabletWithDB("select * from local_metadata", "_vt")
-	require.Nil(t, err)
-	require.NotNil(t, result)
-	require.NotEmpty(t, result.Rows)
-	assert.Equal(t, replica2.Alias, result.Rows[0][1].ToString(), "Alias")
-	assert.Equal(t, "ks.0", result.Rows[1][1].ToString(), "ClusterAlias")
-	assert.Equal(t, cell, result.Rows[2][1].ToString(), "DataCenter")
-	if tabletType == "replica" {
-		assert.Equal(t, "neutral", result.Rows[3][1].ToString(), "PromotionRule")
-	} else {
-		assert.Equal(t, "must_not", result.Rows[3][1].ToString(), "PromotionRule")
-	}
-
 	removeBackups(t)
 	verifyBackupCount(t, shardKsName, 0)
 }

go/test/endtoend/backup/vtctlbackup/backup_utils.go

@@ -409,7 +409,6 @@ func primaryBackup(t *testing.T) {
 	// Verify that we have all the new data -- we should have 2 records now...
 	// And only 1 record after we restore using the first backup timestamp
 	cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2)
-	cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell)
 
 	err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", "--", "--allow_primary=true", primary.Alias)
 	require.Nil(t, err)
@@ -441,7 +440,6 @@ func primaryBackup(t *testing.T) {
 
 	// Verify that we don't have the record created after the older/first backup
 	cluster.VerifyRowsInTablet(t, primary, keyspaceName, 1)
-	cluster.VerifyLocalMetadata(t, primary, keyspaceName, shardName, cell)
 
 	verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups)
 	require.Nil(t, err)
@@ -793,7 +791,6 @@ func vtctlBackup(t *testing.T, tabletType string) {
 	require.Nil(t, err)
 	cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2)
 
-	cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell)
 	verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups)
 
 	// Stop VTOrc
@@ -944,8 +941,6 @@ func vtctlBackupReplicaNoDestroyNoWrites(t *testing.T, tabletType string) (backu
 	err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second)
 	require.Nil(t, err)
 
-	cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell)
-
 	err = replica2.VttabletProcess.TearDown()
 	require.Nil(t, err)