Added ability to show extra details during restore process

When working with a larger restore using the `restore-dump` command by default there is only the "Restoring database ..." indicator by default which provides minimal insight into how things are progressing.

On the other hand, if you choose to add the `--debug` flag the amount of information can then be too much as the large `INSERT` queries are then also included in the output.

The new `--show-details` flag optionally allows for users to see how things are going with their restore more easily.

Additionally, a `--start-from` flag was added that can be provided with a table name. This allows a user to skip earlier tables and start the import from a later point easily without having to create a separate copy of their dump folder and manually remove those files.

The new `--allow-different-destination` flag primarily helps with simplifying the process of taking a folder that was created for one database, e.g. the files contain a prefix for "first-database", and allows you to restore into a second database without having to adjust the database prefix on all of those existing files.

Also incorporated a check to prevent an issue for custom generated files where users might accidentally provide an `INSERT` query that is larger than 16777216 bytes. This provides some useful feedback rather than the `pkt` error that was being received previously.

internal/cmd/database/restore.go

@@ -18,11 +18,14 @@ import (
 )
 
 type restoreFlags struct {
-	localAddr  string
-	remoteAddr string
-	dir        string
-	overwrite  bool
-	threads    int
+	localAddr                 string
+	remoteAddr                string
+	dir                       string
+	overwrite                 bool
+	showDetails               bool
+	startFrom                 string
+	allowDifferentDestination bool
+	threads                   int
 }
 
 // RestoreCmd encapsulates the commands for restore a database
@@ -42,7 +45,10 @@ func RestoreCmd(ch *cmdutil.Helper) *cobra.Command {
 	cmd.PersistentFlags().StringVar(&f.dir, "dir", "",
 		"Directory containing the files to be used for the restore (required)")
 	cmd.PersistentFlags().BoolVar(&f.overwrite, "overwrite-tables", false, "If true, will attempt to DROP TABLE before restoring.")
-
+	cmd.PersistentFlags().BoolVar(&f.showDetails, "show-details", false, "If true, will add extra output during the restore process.")
+	cmd.PersistentFlags().StringVar(&f.startFrom, "start-from", "",
+		"Table to start from for the restore (useful for restarting from a certain point)")
+	cmd.PersistentFlags().BoolVar(&f.allowDifferentDestination, "allow-different-destination", false, "If true, will allow you to restore the files to a database with a different name without needing to rename the existing dump's files.")
 	cmd.PersistentFlags().IntVar(&f.threads, "threads", 1, "Number of concurrent threads to use to restore the database.")
 	return cmd
 }
@@ -151,20 +157,30 @@ func restore(ch *cmdutil.Helper, cmd *cobra.Command, flags *restoreFlags, args [
 	cfg.IntervalMs = 10 * 1000
 	cfg.Outdir = flags.dir
 	cfg.OverwriteTables = flags.overwrite
+	cfg.ShowDetails = flags.showDetails
+	cfg.AllowDifferentDestination = flags.allowDifferentDestination
+	cfg.Database = database // Needs to be passed in to allow for allowDifferentDestination flag to work
+	cfg.StartFrom = flags.startFrom
 
 	loader, err := dumper.NewLoader(cfg)
 	if err != nil {
 		return err
 	}
 
+	end := func() {}
+
 	ch.Printer.Printf("Starting to restore database %s from folder %s\n",
 		printer.BoldBlue(database), printer.BoldBlue(flags.dir))
 
-	end := ch.Printer.PrintProgress("Restoring database ...")
+	if flags.showDetails {
+		ch.Printer.Println("Restoring database ...")
+	} else {
+		end = ch.Printer.PrintProgress("Restoring database ...\n")
+	}
 	defer end()
 
 	start := time.Now()
-	err = loader.Run(ctx)
+	err = loader.Run(ctx, ch)
 	if err != nil {
 		return fmt.Errorf("failed to restore database: %s", err)
 	}

internal/dumper/dumper.go

@@ -22,32 +22,35 @@ const VITESS_GHOST_TABLE_REGEX = "_vt_EVAC_.*|_vt_DROP_.*|_vt_PURGE_.*|_vt_HOLD_
 
 // Config describes the settings to dump from a database.
 type Config struct {
-	User                 string
-	Password             string
-	Address              string
-	ToUser               string
-	ToPassword           string
-	ToAddress            string
-	ToDatabase           string
-	ToEngine             string
-	Database             string
-	DatabaseRegexp       string
-	DatabaseInvertRegexp bool
-	Shard                string
-	Table                string
-	Outdir               string
-	SessionVars          []string
-	Threads              int
-	ChunksizeInMB        int
-	StmtSize             int
-	Allbytes             uint64
-	Allrows              uint64
-	OverwriteTables      bool
-	UseReplica           bool
-	UseRdonly            bool
-	Wheres               map[string]string
-	Selects              map[string]map[string]string
-	Filters              map[string]map[string]string
+	User                      string
+	Password                  string
+	Address                   string
+	ToUser                    string
+	ToPassword                string
+	ToAddress                 string
+	ToDatabase                string
+	ToEngine                  string
+	Database                  string
+	DatabaseRegexp            string
+	DatabaseInvertRegexp      bool
+	Shard                     string
+	Table                     string
+	Outdir                    string
+	SessionVars               []string
+	Threads                   int
+	ChunksizeInMB             int
+	StmtSize                  int
+	Allbytes                  uint64
+	Allrows                   uint64
+	OverwriteTables           bool
+	ShowDetails               bool
+	StartFrom                 string
+	AllowDifferentDestination bool
+	UseReplica                bool
+	UseRdonly                 bool
+	Wheres                    map[string]string
+	Selects                   map[string]map[string]string
+	Filters                   map[string]map[string]string
 
 	// Interval in millisecond.
 	IntervalMs int