Updated dbmigrate to obtain source db type from config.toml (#1258)

* Updated dbmigrator to no longer detect the backend, but instead read from the config.toml. This was done because it was hard to distinguish between goleveldb and cleveldb.

* Added TODO comment as a reminder.

* Removed DetectDBType since it isn't used anymore, and updated tests to test for SourceDBType.

* Moved setting the SourceDB outside of the migrator so that it follows the same pattern as TargetDB. This also reverts some of the changes to Initialize because they are no longer needed.

* Added source db type to the logging.

* Updated changelog.

* Updated long help to provide how we are obtaining the source database type.

* Linted files and moved fileExists into test suite since it is only used there.

* Reverted most of the changes and applied Danny's suggestion of having a flag instead.

* Added logging of source back.

CHANGELOG.md

@@ -92,6 +92,7 @@ The `v1.13.0` release includes minor bug fixes and enhancements along with a res
 * Added two new Makefile targets to install and start the relayer [#1051] (https://github.com/provenance-io/provenance/pull/1051)
 * Updated relayer scripts to make them headless for external services [#1068] (https://github.com/provenance-io/provenance/pull/1068)
 * Added docker environment for testing IBC and added Makefile targets to bring this environment up/down [#1248] (https://github.com/provenance-io/provenance/pull/1248).
+* Updated the dbmigrate tool to allow the user to force the source database type with the source-db-backend option [#1258] (https://github.com/provenance-io/provenance/pull/1258)
 * Updated provenance-io/blockchain image to work with arm64 [#1261]. (https://github.com/provenance-io/provenance/pull/1261)
 
 ### Bug Fixes

cmd/dbmigrate/cmd/dbmigrate.go

@@ -21,10 +21,11 @@ import (
 )
 
 const (
-	FlagBackupDir  = "backup-dir"
-	FlagBatchSize  = "batch-size"
-	FlagStagingDir = "staging-dir"
-	FlagStageOnly  = "stage-only"
+	FlagBackupDir       = "backup-dir"
+	FlagBatchSize       = "batch-size"
+	FlagStagingDir      = "staging-dir"
+	FlagStageOnly       = "stage-only"
+	FlagSourceDbBackend = "source-db-backend"
 )
 
 // NewDBMigrateCmd creates a command for migrating the provenanced database from one underlying type to another.
@@ -98,10 +99,17 @@ Migration process:
 			if err != nil {
 				return fmt.Errorf("could not parse --%s option: %w", FlagBatchSize, err)
 			}
+
+			sourceDB, err := command.Flags().GetString(FlagSourceDbBackend)
+			if err != nil {
+				return fmt.Errorf("could not parse --%s option: %w", FlagSourceDbBackend, err)
+			}
+
 			migrator := &utils.Migrator{
 				TargetDBType: strings.ToLower(args[0]),
 				HomePath:     client.GetClientContextFromCmd(command).HomeDir,
 				BatchSize:    batchSizeMB * utils.BytesPerMB,
+				SourceDBType: sourceDB,
 			}
 
 			migrator.StageOnly, err = command.Flags().GetBool(FlagStageOnly)
@@ -134,6 +142,7 @@ Migration process:
 	rv.Flags().String(FlagStagingDir, "", "directory to hold the staging directory (default {home})")
 	rv.Flags().Uint(FlagBatchSize, 2_048, "(in megabytes) after a batch reaches this size it is written and a new one is started (0 = unlimited)")
 	rv.Flags().Bool(FlagStageOnly, false, "only migrate/copy the data (do not backup and replace the data directory and do not update the config)")
+	rv.Flags().String(FlagSourceDbBackend, "", "forces a source database type instead of trying to detect it.")
 	return rv
 }
 
@@ -152,6 +161,7 @@ func Execute(command *cobra.Command) error {
 func DoMigrateCmd(command *cobra.Command, migrator *utils.Migrator) error {
 	logger := server.GetServerContextFromCmd(command).Logger
 	logger.Info("Setting up database migrations.")
+
 	err := migrator.Initialize()
 	if err != nil {
 		return err

cmd/dbmigrate/utils/migrator.go

@@ -73,6 +73,9 @@ type Migrator struct {
 	// TargetDBType is the type of the target (new) DB.
 	TargetDBType string
 
+	// SourceDBType is the type of the source (old) DB.
+	SourceDBType string
+
 	// SourceDataDir is the path to the source (current) data directory.
 	// Default is { HomePath }/data
 	SourceDataDir string
@@ -233,6 +236,9 @@ func (m Migrator) ValidateBasic() error {
 	if len(m.DirDateFormat) == 0 {
 		return errors.New("no DirDateFormat defined")
 	}
+	if len(m.SourceDBType) > 0 && !IsPossibleDBType(m.SourceDBType) {
+		return fmt.Errorf("invalid SourceDBType: %q - must be one of: %s", m.SourceDBType, strings.Join(GetPossibleDBTypes(), ", "))
+	}
 	return nil
 }
 
@@ -460,11 +466,14 @@ func (m *migrationManager) MigrateDBDir(dbDir string) (summary string, err error
 	}()
 
 	m.Status = "detecting db type"
-	var ok bool
-	sourceDBType, ok = DetectDBType(dbName, sourceDir)
+	sourceDBType, ok := tmdb.BackendType(m.Migrator.SourceDBType), true
+	if len(m.Migrator.SourceDBType) == 0 {
+		sourceDBType, ok = DetectDBType(dbName, sourceDir)
+	}
 	if !ok {
 		return summaryError, fmt.Errorf("could not determine db type: %s", filepath.Join(m.SourceDataDir, dbDir))
 	}
+
 	if !IsPossibleDBType(string(sourceDBType)) {
 		return summaryError, fmt.Errorf("cannot read source db of type %q", sourceDBType)
 	}
@@ -644,6 +653,7 @@ func (m migrationManager) MakeSummaryString() string {
 		addLine("%16s: %s", "Backup Dir", m.BackupDataDir)
 	}
 	addLine("%16s: %s megabytes", "Batch Size", commaString(m.BatchSize/BytesPerMB))
+	addLine("%16s: %s", "Source DB Type", m.SourceDBType)
 	addLine("%16s: %s", "New DB Type", m.TargetDBType)
 	addLine("%16s: %s", fmt.Sprintf("%s (%d)", copyHead, len(m.ToCopy)), strings.Join(m.ToCopy, "  "))
 	if len(m.Summaries) == 0 {

cmd/dbmigrate/utils/migrator_test.go

@@ -7,11 +7,10 @@ import (
 	"testing"
 	"time"
 
-	tmdb "github.com/tendermint/tm-db"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
+	tmdb "github.com/tendermint/tm-db"
 )
 
 type MigratorTestSuite struct {
@@ -60,6 +59,7 @@ func (s *MigratorTestSuite) TestInitialize() {
 		m := &Migrator{
 			TargetDBType: "goleveldb",
 			HomePath:     tdir,
+			SourceDBType: "goleveldb",
 		}
 		err := m.Initialize()
 		require.NoError(t, err)
@@ -356,6 +356,23 @@ func (s *MigratorTestSuite) TestApplyDefaults() {
 			expected: "target type",
 		},
 
+		{
+			name: "source db type not set unchanged",
+			migrator: &Migrator{
+				SourceDBType: "",
+			},
+			getter:   func(m *Migrator) interface{} { return m.SourceDBType },
+			expected: "",
+		},
+		{
+			name: "source db type set unchanged",
+			migrator: &Migrator{
+				SourceDBType: "source type",
+			},
+			getter:   func(m *Migrator) interface{} { return m.SourceDBType },
+			expected: "source type",
+		},
+
 		{
 			name: "batch size not set unchanged",
 			migrator: &Migrator{
@@ -422,6 +439,7 @@ func (s *MigratorTestSuite) TestValidateBasic() {
 		rv := &Migrator{
 			HomePath:     "testing",
 			TargetDBType: "goleveldb",
+			SourceDBType: "goleveldb",
 		}
 		rv.ApplyDefaults()
 		return rv
@@ -456,6 +474,16 @@ func (s *MigratorTestSuite) TestValidateBasic() {
 			modifier:   func(m *Migrator) { m.TargetDBType = "not-possible" },
 			expInError: []string{"TargetDBType", "goleveldb", "\"not-possible\""},
 		},
+		{
+			name:       "SourceDBType empty",
+			modifier:   func(m *Migrator) { m.SourceDBType = "" },
+			expInError: nil,
+		},
+		{
+			name:       "SourceDBType not possible",
+			modifier:   func(m *Migrator) { m.SourceDBType = "not-possible" },
+			expInError: []string{"SourceDBType", "goleveldb", "\"not-possible\""},
+		},
 		{
 			name:       "SourceDataDir empty",
 			modifier:   func(m *Migrator) { m.SourceDataDir = "" },