From 4527d932e12b4fba7e0de84a8152dbe6474ae11b Mon Sep 17 00:00:00 2001 From: Akanksha kumari Date: Tue, 19 Oct 2021 03:46:22 +0530 Subject: [PATCH] Update RDS postgres restore command (#1118) * Update PostgresRestore command for Postgres version less than 13 Add function to fetch RDS DB EngineVersion * Update go.mod go.sum * Update pkg/function/utils.go Co-authored-by: Prasad Ghangal * Use const variable for PostgresDBEngine Version * Address Review comments Co-authored-by: Prasad Ghangal --- go.mod | 1 + go.sum | 1 + pkg/function/export_rds_snapshot_location.go | 16 ++- pkg/function/rds_functions_test.go | 111 +++++++++++-------- pkg/function/restore_rds_snapshot.go | 37 ++++++- pkg/function/utils.go | 14 +++ 6 files changed, 125 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 016122d533..be0cbdf774 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 github.com/graymeta/stow v0.0.0-00010101000000-000000000000 + github.com/hashicorp/go-version v1.2.0 github.com/huandu/xstrings v1.2.0 // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/jarcoal/httpmock v1.0.4 // indirect diff --git a/go.sum b/go.sum index bee15896ff..3698f15b7a 100644 --- a/go.sum +++ b/go.sum @@ -578,6 +578,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= diff --git a/pkg/function/export_rds_snapshot_location.go b/pkg/function/export_rds_snapshot_location.go index 04e78e6eb6..ceab2e4977 100644 --- a/pkg/function/export_rds_snapshot_location.go +++ b/pkg/function/export_rds_snapshot_location.go @@ -123,8 +123,14 @@ func exportRDSSnapshotToLoc(ctx context.Context, namespace, instanceID, snapshot // Create unique backupID backupID := fmt.Sprintf("backup-%s.tar.gz", rand.String(10)) + // get the engine version + dbEngineVersion, err := rdsDBEngineVersion(ctx, rdsCli, tmpInstanceID) + if err != nil { + return nil, errors.Wrapf(err, "Couldn't find DBInstance Version") + } + // Extract dump from DB - output, err := execDumpCommand(ctx, dbEngine, BackupAction, namespace, dbEndpoint, username, password, databases, backupPrefix, backupID, profile) + output, err := execDumpCommand(ctx, dbEngine, BackupAction, namespace, dbEndpoint, username, password, databases, backupPrefix, backupID, profile, dbEngineVersion) if err != nil { return nil, errors.Wrap(err, "Unable to extract and push db dump to location") } @@ -188,13 +194,13 @@ func (*exportRDSSnapshotToLocationFunc) RequiredArgs() []string { return []string{ExportRDSSnapshotToLocNamespaceArg, ExportRDSSnapshotToLocInstanceIDArg, ExportRDSSnapshotToLocSnapshotIDArg, ExportRDSSnapshotToLocDBEngineArg} } -func execDumpCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, namespace, dbEndpoint, username, password string, databases []string, backupPrefix, backupID string, profile *param.Profile) (map[string]interface{}, error) { +func execDumpCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, namespace, dbEndpoint, username, password string, databases []string, backupPrefix, backupID string, profile *param.Profile, dbEngineVersion string) (map[string]interface{}, error) { // Trim "\n" from creds username = strings.TrimSpace(username) password = strings.TrimSpace(password) // Prepare and execute command with kubetask - command, image, err := prepareCommand(ctx, dbEngine, action, dbEndpoint, username, password, databases, backupPrefix, backupID, profile) + command, image, err := prepareCommand(ctx, dbEngine, action, dbEndpoint, username, password, databases, backupPrefix, backupID, profile, dbEngineVersion) if err != nil { return nil, err } @@ -221,7 +227,7 @@ func execDumpCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction return kubeTask(ctx, cli, namespace, image, command, injectPostgresSecrets(secretName)) } -func prepareCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, dbEndpoint, username, password string, dbList []string, backupPrefix, backupID string, profile *param.Profile) ([]string, string, error) { +func prepareCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, dbEndpoint, username, password string, dbList []string, backupPrefix, backupID string, profile *param.Profile, dbEngineVersion string) ([]string, string, error) { // Convert profile object into json profileJson, err := json.Marshal(profile) if err != nil { @@ -256,7 +262,7 @@ func prepareCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, command, err := postgresBackupCommand(dbEndpoint, username, password, dbList, backupPrefix, backupID, profileJson) return command, postgresToolsImage, err case RestoreAction: - command, err := postgresRestoreCommand(dbEndpoint, username, password, dbList, backupPrefix, backupID, profileJson) + command, err := postgresRestoreCommand(dbEndpoint, username, password, dbList, backupPrefix, backupID, profileJson, dbEngineVersion) return command, postgresToolsImage, err } } diff --git a/pkg/function/rds_functions_test.go b/pkg/function/rds_functions_test.go index b85861e8f2..20a5c948c9 100644 --- a/pkg/function/rds_functions_test.go +++ b/pkg/function/rds_functions_test.go @@ -29,30 +29,51 @@ var _ = Suite(&RDSFunctionsTest{}) func (s *RDSFunctionsTest) TestPrepareCommand(c *C) { testCases := []struct { - name string - dbEngine RDSDBEngine - dbList []string - action RDSAction - dbEndpoint string - username string - password string - backupPrefix string - backupID string - errChecker Checker - tp param.TemplateParams - command []string + name string + dbEngine RDSDBEngine + dbList []string + action RDSAction + dbEndpoint string + username string + password string + backupPrefix string + backupID string + dbEngineVersion string + errChecker Checker + tp param.TemplateParams + command []string }{ { - name: "PostgreS restore command", - dbEngine: PostgrSQLEngine, - action: RestoreAction, - dbEndpoint: "db-endpoint", - username: "test-user", - password: "secret-pass", - backupPrefix: "/backup/postgres-backup", - backupID: "backup-id", - errChecker: IsNil, - dbList: []string{"template1"}, + name: "PostgreS restore command", + dbEngine: PostgrSQLEngine, + action: RestoreAction, + dbEndpoint: "db-endpoint", + username: "test-user", + password: "secret-pass", + backupPrefix: "/backup/postgres-backup", + backupID: "backup-id", + dbEngineVersion: "12.7", + errChecker: IsNil, + dbList: []string{"template1"}, + command: []string{"bash", "-o", "errexit", "-o", "pipefail", "-c", + fmt.Sprintf(` + export PGHOST=%s + kando location pull --profile '%s' --path "%s" - | gunzip -c -f | sed 's/LOCALE/LC_COLLATE/' | psql -q -U "${PGUSER}" %s + `, "db-endpoint", "null", fmt.Sprintf("%s/%s", "/backup/postgres-backup", "backup-id"), []string{"template1"}[0]), + }, + }, + { + name: "PostgreS restore command", + dbEngine: PostgrSQLEngine, + action: RestoreAction, + dbEndpoint: "db-endpoint", + username: "test-user", + password: "secret-pass", + backupPrefix: "/backup/postgres-backup", + backupID: "backup-id", + dbEngineVersion: "13.3", + errChecker: IsNil, + dbList: []string{"template1"}, command: []string{"bash", "-o", "errexit", "-o", "pipefail", "-c", fmt.Sprintf(` export PGHOST=%s @@ -61,16 +82,17 @@ func (s *RDSFunctionsTest) TestPrepareCommand(c *C) { }, }, { - name: "PostgreS backup command", - dbEngine: PostgrSQLEngine, - action: BackupAction, - dbEndpoint: "db-endpoint", - username: "test-user", - password: "secret-pass", - backupPrefix: "/backup/postgres-backup", - backupID: "backup-id", - errChecker: IsNil, - dbList: []string{"template1"}, + name: "PostgreS backup command", + dbEngine: PostgrSQLEngine, + action: BackupAction, + dbEndpoint: "db-endpoint", + username: "test-user", + password: "secret-pass", + backupPrefix: "/backup/postgres-backup", + backupID: "backup-id", + dbEngineVersion: "12.7", + errChecker: IsNil, + dbList: []string{"template1"}, command: []string{"bash", "-o", "errexit", "-o", "pipefail", "-c", fmt.Sprintf(` export PGHOST=%s @@ -88,22 +110,23 @@ func (s *RDSFunctionsTest) TestPrepareCommand(c *C) { }, }, { - name: "PostgreS backup command", - dbEngine: "MySQLDBEngine", - action: BackupAction, - dbEndpoint: "db-endpoint", - username: "test-user", - password: "secret-pass", - backupPrefix: "/backup/postgres-backup", - backupID: "backup-id", - errChecker: NotNil, - dbList: []string{"template1"}, - command: nil, + name: "PostgreS backup command", + dbEngine: "MySQLDBEngine", + action: BackupAction, + dbEndpoint: "db-endpoint", + username: "test-user", + password: "secret-pass", + backupPrefix: "/backup/postgres-backup", + backupID: "backup-id", + dbEngineVersion: "12.7", + errChecker: NotNil, + dbList: []string{"template1"}, + command: nil, }, } for _, tc := range testCases { - outCommand, _, err := prepareCommand(context.Background(), tc.dbEngine, tc.action, tc.dbEndpoint, tc.username, tc.password, tc.dbList, tc.backupPrefix, tc.backupID, tc.tp.Profile) + outCommand, _, err := prepareCommand(context.Background(), tc.dbEngine, tc.action, tc.dbEndpoint, tc.username, tc.password, tc.dbList, tc.backupPrefix, tc.backupID, tc.tp.Profile, tc.dbEngineVersion) c.Check(err, tc.errChecker, Commentf("Case %s failed", tc.name)) c.Assert(outCommand, DeepEquals, tc.command) diff --git a/pkg/function/restore_rds_snapshot.go b/pkg/function/restore_rds_snapshot.go index fe7d7967c1..93dd7407f9 100644 --- a/pkg/function/restore_rds_snapshot.go +++ b/pkg/function/restore_rds_snapshot.go @@ -20,6 +20,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" rdserr "github.com/aws/aws-sdk-go/service/rds" + "github.com/hashicorp/go-version" "github.com/pkg/errors" kanister "github.com/kanisterio/kanister/pkg" @@ -65,8 +66,9 @@ const ( // PostgreSQLEngine stores the postgres appname PostgreSQLEngine RDSDBEngine = "PostgreSQL" - restoredAuroraInstanceSuffix = "instance-1" - defaultAuroraInstanceClass = "db.r5.large" + restoredAuroraInstanceSuffix = "instance-1" + defaultAuroraInstanceClass = "db.r5.large" + RDSPostgresDBInstanceEngineVersion = "13.0" ) type restoreRDSSnapshotFunc struct{} @@ -167,7 +169,14 @@ func restoreRDSSnapshot(ctx context.Context, namespace, instanceID, snapshotID, } dbEndpoint := *descOp.DBInstances[0].Endpoint.Address - if _, err = execDumpCommand(ctx, dbEngine, RestoreAction, namespace, dbEndpoint, username, password, nil, backupArtifactPrefix, backupID, profile); err != nil { + + // get the engine version + dbEngineVersion, err := rdsDBEngineVersion(ctx, rdsCli, instanceID) + if err != nil { + return nil, errors.Wrapf(err, "Couldn't find DBInstance Version") + } + + if _, err = execDumpCommand(ctx, dbEngine, RestoreAction, namespace, dbEndpoint, username, password, nil, backupArtifactPrefix, backupID, profile, dbEngineVersion); err != nil { return nil, errors.Wrapf(err, "Failed to restore RDS from dump. InstanceID=%s", instanceID) } @@ -177,11 +186,27 @@ func restoreRDSSnapshot(ctx context.Context, namespace, instanceID, snapshotID, } // nolint:unparam -func postgresRestoreCommand(pgHost, username, password string, dbList []string, backupArtifactPrefix, backupID string, profile []byte) ([]string, error) { +func postgresRestoreCommand(pgHost, username, password string, dbList []string, backupArtifactPrefix, backupID string, profile []byte, dbEngineVersion string) ([]string, error) { + replaceCommand := "" if len(dbList) == 0 { return nil, errors.New("No database found. Atleast one db needed to connect") } + // check if PostgresDB version < 13 + v1, err := version.NewVersion(dbEngineVersion) + if err != nil { + return nil, errors.Wrapf(err, "Couldn't find DBInstance Version") + } + // Add Constraints + constraints, err := version.NewConstraint("< " + RDSPostgresDBInstanceEngineVersion) + if err != nil { + return nil, errors.Wrapf(err, "Couldn't add constraint to DBInstance Version") + } + // Verify Constraints + if constraints.Check(v1) { + replaceCommand = " sed 's/LOCALE/LC_COLLATE/' |" + } + return []string{ "bash", "-o", @@ -191,8 +216,8 @@ func postgresRestoreCommand(pgHost, username, password string, dbList []string, "-c", fmt.Sprintf(` export PGHOST=%s - kando location pull --profile '%s' --path "%s" - | gunzip -c -f | psql -q -U "${PGUSER}" %s - `, pgHost, profile, fmt.Sprintf("%s/%s", backupArtifactPrefix, backupID), dbList[0]), + kando location pull --profile '%s' --path "%s" - | gunzip -c -f |%s psql -q -U "${PGUSER}" %s + `, pgHost, profile, fmt.Sprintf("%s/%s", backupArtifactPrefix, backupID), replaceCommand, dbList[0]), }, nil } diff --git a/pkg/function/utils.go b/pkg/function/utils.go index 85e4e68e2a..9fc2a030f9 100644 --- a/pkg/function/utils.go +++ b/pkg/function/utils.go @@ -208,6 +208,20 @@ func findRDSEndpoint(ctx context.Context, rdsCli *rds.RDS, instanceID string) (s return *dbInstance.DBInstances[0].Endpoint.Address, nil } +// rdsDBEngineVersion returns the database engine version +func rdsDBEngineVersion(ctx context.Context, rdsCli *rds.RDS, instanceID string) (string, error) { + dbInstance, err := rdsCli.DescribeDBInstances(ctx, instanceID) + if err != nil { + return "", err + } + + if (len(dbInstance.DBInstances) == 0) || (dbInstance.DBInstances[0].EngineVersion == nil) { + return "", errors.Errorf("DB Instance's Engine version is nil") + } + + return *dbInstance.DBInstances[0].EngineVersion, nil +} + func createPostgresSecret(cli kubernetes.Interface, name, namespace, username, password string) error { secret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{