Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Backup & Restore: vtctldclient to support PITR flags #13513

Merged
merged 5 commits into from
Jul 31, 2023
Merged

Backup & Restore: vtctldclient to support PITR flags #13513

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
afec270
Backup & Restore: vtctldclient to support PITR flags
shlomi-noach Jul 16, 2023
3e0226e
resolved conflict
shlomi-noach Jul 25, 2023
2ffde82
Merge branch 'main' into vtctldclient-backup-incremental-flags
shlomi-noach Jul 25, 2023
59b558e
vtctld support for 'RestoreFromBackup --restore-to-timestamp' flag
shlomi-noach Jul 25, 2023
ef4c270
endtoend test to use 'vtctldclient RestoreFromBackup --restore-to-tim…
shlomi-noach Jul 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 37 additions & 11 deletions go/cmd/vtctldclient/command/backups.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (

"vitess.io/vitess/go/cmd/vtctldclient/cli"
"vitess.io/vitess/go/protoutil"
"vitess.io/vitess/go/vt/logutil"
"vitess.io/vitess/go/vt/mysqlctl"
"vitess.io/vitess/go/vt/topo/topoproto"

Expand All @@ -35,7 +36,7 @@ import (
var (
// Backup makes a Backup gRPC call to a vtctld.
Backup = &cobra.Command{
Use: "Backup [--concurrency <concurrency>] [--allow-primary] [--upgrade-safe] <tablet_alias>",
Use: "Backup [--concurrency <concurrency>] [--allow-primary] [--incremental-from-pos=<pos>|auto] [--upgrade-safe] <tablet_alias>",
Short: "Uses the BackupStorage service on the given tablet to create and store a new backup.",
DisableFlagsInUseLine: true,
Args: cobra.ExactArgs(1),
Expand Down Expand Up @@ -70,7 +71,7 @@ If no replica-type tablet can be found, the backup can be taken on the primary i
}
// RestoreFromBackup makes a RestoreFromBackup gRPC call to a vtctld.
RestoreFromBackup = &cobra.Command{
Use: "RestoreFromBackup [--backup-timestamp|-t <YYYY-mm-DD.HHMMSS>] <tablet_alias>",
Use: "RestoreFromBackup [--backup-timestamp|-t <YYYY-mm-DD.HHMMSS>] [--restore-to-pos <pos>] [--dry-run] <tablet_alias>",
Short: "Stops mysqld on the specified tablet and restores the data from either the latest backup or closest before `backup-timestamp`.",
DisableFlagsInUseLine: true,
Args: cobra.ExactArgs(1),
Expand All @@ -79,9 +80,10 @@ If no replica-type tablet can be found, the backup can be taken on the primary i
)

var backupOptions = struct {
AllowPrimary bool
Concurrency uint64
UpgradeSafe bool
AllowPrimary bool
Concurrency uint64
IncrementalFromPos string
UpgradeSafe bool
}{}

func commandBackup(cmd *cobra.Command, args []string) error {
Expand All @@ -93,10 +95,11 @@ func commandBackup(cmd *cobra.Command, args []string) error {
cli.FinishedParsing(cmd)

stream, err := client.Backup(commandCtx, &vtctldatapb.BackupRequest{
TabletAlias: tabletAlias,
AllowPrimary: backupOptions.AllowPrimary,
Concurrency: backupOptions.Concurrency,
UpgradeSafe: backupOptions.UpgradeSafe,
TabletAlias: tabletAlias,
AllowPrimary: backupOptions.AllowPrimary,
Concurrency: backupOptions.Concurrency,
IncrementalFromPos: backupOptions.IncrementalFromPos,
UpgradeSafe: backupOptions.UpgradeSafe,
})
if err != nil {
return err
Expand Down Expand Up @@ -214,7 +217,10 @@ func commandRemoveBackup(cmd *cobra.Command, args []string) error {
}

var restoreFromBackupOptions = struct {
BackupTimestamp string
BackupTimestamp string
RestoreToPos string
RestoreToTimestamp string
DryRun bool
}{}

func commandRestoreFromBackup(cmd *cobra.Command, args []string) error {
Expand All @@ -223,8 +229,23 @@ func commandRestoreFromBackup(cmd *cobra.Command, args []string) error {
return err
}

if restoreFromBackupOptions.RestoreToPos != "" && restoreFromBackupOptions.RestoreToTimestamp != "" {
return fmt.Errorf("--restore-to-pos and --restore-to-timestamp are mutually exclusive")
}

var restoreToTimestamp time.Time
if restoreFromBackupOptions.RestoreToTimestamp != "" {
restoreToTimestamp, err = mysqlctl.ParseRFC3339(restoreFromBackupOptions.RestoreToTimestamp)
if err != nil {
return err
}
}

req := &vtctldatapb.RestoreFromBackupRequest{
TabletAlias: alias,
TabletAlias: alias,
RestoreToPos: restoreFromBackupOptions.RestoreToPos,
RestoreToTimestamp: logutil.TimeToProto(restoreToTimestamp),
DryRun: restoreFromBackupOptions.DryRun,
}

if restoreFromBackupOptions.BackupTimestamp != "" {
Expand Down Expand Up @@ -259,6 +280,8 @@ func commandRestoreFromBackup(cmd *cobra.Command, args []string) error {
func init() {
Backup.Flags().BoolVar(&backupOptions.AllowPrimary, "allow-primary", false, "Allow the primary of a shard to be used for the backup. WARNING: If using the builtin backup engine, this will shutdown mysqld on the primary and stop writes for the duration of the backup.")
Backup.Flags().Uint64Var(&backupOptions.Concurrency, "concurrency", 4, "Specifies the number of compression/checksum jobs to run simultaneously.")
Backup.Flags().StringVar(&backupOptions.IncrementalFromPos, "incremental-from-pos", "", "Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position")

Backup.Flags().BoolVar(&backupOptions.UpgradeSafe, "upgrade-safe", false, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.")
Root.AddCommand(Backup)

Expand All @@ -274,5 +297,8 @@ func init() {
Root.AddCommand(RemoveBackup)

RestoreFromBackup.Flags().StringVarP(&restoreFromBackupOptions.BackupTimestamp, "backup-timestamp", "t", "", "Use the backup taken at, or closest before, this timestamp. Omit to use the latest backup. Timestamp format is \"YYYY-mm-DD.HHMMSS\".")
RestoreFromBackup.Flags().StringVar(&restoreFromBackupOptions.RestoreToPos, "restore-to-pos", "", "Run a point in time recovery that ends with the given position. This will attempt to use one full backup followed by zero or more incremental backups")
RestoreFromBackup.Flags().StringVar(&restoreFromBackupOptions.RestoreToTimestamp, "restore-to-timestamp", "", "Run a point in time recovery that restores up to, and excluding, given timestamp in RFC3339 format (`2006-01-02T15:04:05Z07:00`). This will attempt to use one full backup followed by zero or more incremental backups")
RestoreFromBackup.Flags().BoolVar(&restoreFromBackupOptions.DryRun, "dry-run", false, "Only validate restore steps, do not actually restore data")
Root.AddCommand(RestoreFromBackup)
}
8 changes: 4 additions & 4 deletions go/test/endtoend/backup/vtctlbackup/backup_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1072,7 +1072,7 @@ func vtctlBackupReplicaNoDestroyNoWrites(t *testing.T, tabletType string) (backu
restoreWaitForBackup(t, tabletType, nil, true)
verifyInitialReplication(t)

err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias)
err := localCluster.VtctldClientProcess.ExecuteCommand("Backup", replica1.Alias)
require.Nil(t, err)

backups = localCluster.VerifyBackupCount(t, shardKsName, 1)
Expand Down Expand Up @@ -1162,7 +1162,7 @@ func TestReplicaIncrementalBackup(t *testing.T, incrementalFromPos mysql.Positio
if !incrementalFromPos.IsZero() {
incrementalFromPosArg = mysql.EncodePosition(incrementalFromPos)
}
output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("Backup", "--", "--incremental_from_pos", incrementalFromPosArg, replica1.Alias)
output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("Backup", "--incremental-from-pos", incrementalFromPosArg, replica1.Alias)
if expectError != "" {
require.Errorf(t, err, "expected: %v", expectError)
require.Contains(t, output, expectError)
Expand All @@ -1181,7 +1181,7 @@ func TestReplicaIncrementalBackup(t *testing.T, incrementalFromPos mysql.Positio
func TestReplicaRestoreToPos(t *testing.T, restoreToPos mysql.Position, expectError string) {
require.False(t, restoreToPos.IsZero())
restoreToPosArg := mysql.EncodePosition(restoreToPos)
output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--", "--restore_to_pos", restoreToPosArg, replica1.Alias)
output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--restore-to-pos", restoreToPosArg, replica1.Alias)
if expectError != "" {
require.Errorf(t, err, "expected: %v", expectError)
require.Contains(t, output, expectError)
Expand All @@ -1194,7 +1194,7 @@ func TestReplicaRestoreToPos(t *testing.T, restoreToPos mysql.Position, expectEr
func TestReplicaRestoreToTimestamp(t *testing.T, restoreToTimestamp time.Time, expectError string) {
require.False(t, restoreToTimestamp.IsZero())
restoreToTimestampArg := mysqlctl.FormatRFC3339(restoreToTimestamp)
output, err := localCluster.VtctlclientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--", "--restore_to_timestamp", restoreToTimestampArg, replica1.Alias)
output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("RestoreFromBackup", "--restore-to-timestamp", restoreToTimestampArg, replica1.Alias)
if expectError != "" {
require.Errorf(t, err, "expected: %v", expectError)
require.Contains(t, output, expectError)
Expand Down