Skip to content

Commit

Permalink
Update RDS postgres restore command (#1118)
Browse files Browse the repository at this point in the history 
* Update PostgresRestore command for Postgres version less than 13
Add function to fetch RDS DB EngineVersion

* Update go.mod go.sum

* Update pkg/function/utils.go

Co-authored-by: Prasad Ghangal <prasad.ghangal@gmail.com>

* Use const variable for PostgresDBEngine Version

* Address Review comments

Co-authored-by: Prasad Ghangal <prasad.ghangal@gmail.com>
  • Loading branch information
@akankshakumari393 @PrasadG193
akankshakumari393 and PrasadG193 committed Oct 18, 2021
1 parent 357f0f9 commit 4527d93
Show file tree
Hide file tree
Showing 6 changed files with 125 additions and 55 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ require (
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0
github.com/graymeta/stow v0.0.0-00010101000000-000000000000
github.com/hashicorp/go-version v1.2.0
github.com/huandu/xstrings v1.2.0 // indirect
github.com/imdario/mergo v0.3.11 // indirect
github.com/jarcoal/httpmock v1.0.4 // indirect
Expand Down
1 change: 1 addition & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
Expand Down
16 changes: 11 additions & 5 deletions pkg/function/export_rds_snapshot_location.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,14 @@ func exportRDSSnapshotToLoc(ctx context.Context, namespace, instanceID, snapshot
// Create unique backupID
backupID := fmt.Sprintf("backup-%s.tar.gz", rand.String(10))

// get the engine version
dbEngineVersion, err := rdsDBEngineVersion(ctx, rdsCli, tmpInstanceID)
if err != nil {
return nil, errors.Wrapf(err, "Couldn't find DBInstance Version")
}

// Extract dump from DB
output, err := execDumpCommand(ctx, dbEngine, BackupAction, namespace, dbEndpoint, username, password, databases, backupPrefix, backupID, profile)
output, err := execDumpCommand(ctx, dbEngine, BackupAction, namespace, dbEndpoint, username, password, databases, backupPrefix, backupID, profile, dbEngineVersion)
if err != nil {
return nil, errors.Wrap(err, "Unable to extract and push db dump to location")
}
Expand Down Expand Up @@ -188,13 +194,13 @@ func (*exportRDSSnapshotToLocationFunc) RequiredArgs() []string {
return []string{ExportRDSSnapshotToLocNamespaceArg, ExportRDSSnapshotToLocInstanceIDArg, ExportRDSSnapshotToLocSnapshotIDArg, ExportRDSSnapshotToLocDBEngineArg}
}

func execDumpCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, namespace, dbEndpoint, username, password string, databases []string, backupPrefix, backupID string, profile *param.Profile) (map[string]interface{}, error) {
func execDumpCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, namespace, dbEndpoint, username, password string, databases []string, backupPrefix, backupID string, profile *param.Profile, dbEngineVersion string) (map[string]interface{}, error) {
// Trim "\n" from creds
username = strings.TrimSpace(username)
password = strings.TrimSpace(password)

// Prepare and execute command with kubetask
command, image, err := prepareCommand(ctx, dbEngine, action, dbEndpoint, username, password, databases, backupPrefix, backupID, profile)
command, image, err := prepareCommand(ctx, dbEngine, action, dbEndpoint, username, password, databases, backupPrefix, backupID, profile, dbEngineVersion)
if err != nil {
return nil, err
}
Expand All @@ -221,7 +227,7 @@ func execDumpCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction
return kubeTask(ctx, cli, namespace, image, command, injectPostgresSecrets(secretName))
}

func prepareCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, dbEndpoint, username, password string, dbList []string, backupPrefix, backupID string, profile *param.Profile) ([]string, string, error) {
func prepareCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, dbEndpoint, username, password string, dbList []string, backupPrefix, backupID string, profile *param.Profile, dbEngineVersion string) ([]string, string, error) {
// Convert profile object into json
profileJson, err := json.Marshal(profile)
if err != nil {
Expand Down Expand Up @@ -256,7 +262,7 @@ func prepareCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction,
command, err := postgresBackupCommand(dbEndpoint, username, password, dbList, backupPrefix, backupID, profileJson)
return command, postgresToolsImage, err
case RestoreAction:
command, err := postgresRestoreCommand(dbEndpoint, username, password, dbList, backupPrefix, backupID, profileJson)
command, err := postgresRestoreCommand(dbEndpoint, username, password, dbList, backupPrefix, backupID, profileJson, dbEngineVersion)
return command, postgresToolsImage, err
}
}
Expand Down
111 changes: 67 additions & 44 deletions pkg/function/rds_functions_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,30 +29,51 @@ var _ = Suite(&RDSFunctionsTest{})

func (s *RDSFunctionsTest) TestPrepareCommand(c *C) {
testCases := []struct {
name string
dbEngine RDSDBEngine
dbList []string
action RDSAction
dbEndpoint string
username string
password string
backupPrefix string
backupID string
errChecker Checker
tp param.TemplateParams
command []string
name string
dbEngine RDSDBEngine
dbList []string
action RDSAction
dbEndpoint string
username string
password string
backupPrefix string
backupID string
dbEngineVersion string
errChecker Checker
tp param.TemplateParams
command []string
}{
{
name: "PostgreS restore command",
dbEngine: PostgrSQLEngine,
action: RestoreAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
errChecker: IsNil,
dbList: []string{"template1"},
name: "PostgreS restore command",
dbEngine: PostgrSQLEngine,
action: RestoreAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
dbEngineVersion: "12.7",
errChecker: IsNil,
dbList: []string{"template1"},
command: []string{"bash", "-o", "errexit", "-o", "pipefail", "-c",
fmt.Sprintf(`
export PGHOST=%s
kando location pull --profile '%s' --path "%s" - | gunzip -c -f | sed 's/LOCALE/LC_COLLATE/' | psql -q -U "${PGUSER}" %s
`, "db-endpoint", "null", fmt.Sprintf("%s/%s", "/backup/postgres-backup", "backup-id"), []string{"template1"}[0]),
},
},
{
name: "PostgreS restore command",
dbEngine: PostgrSQLEngine,
action: RestoreAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
dbEngineVersion: "13.3",
errChecker: IsNil,
dbList: []string{"template1"},
command: []string{"bash", "-o", "errexit", "-o", "pipefail", "-c",
fmt.Sprintf(`
export PGHOST=%s
Expand All @@ -61,16 +82,17 @@ func (s *RDSFunctionsTest) TestPrepareCommand(c *C) {
},
},
{
name: "PostgreS backup command",
dbEngine: PostgrSQLEngine,
action: BackupAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
errChecker: IsNil,
dbList: []string{"template1"},
name: "PostgreS backup command",
dbEngine: PostgrSQLEngine,
action: BackupAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
dbEngineVersion: "12.7",
errChecker: IsNil,
dbList: []string{"template1"},
command: []string{"bash", "-o", "errexit", "-o", "pipefail", "-c",
fmt.Sprintf(`
export PGHOST=%s
Expand All @@ -88,22 +110,23 @@ func (s *RDSFunctionsTest) TestPrepareCommand(c *C) {
},
},
{
name: "PostgreS backup command",
dbEngine: "MySQLDBEngine",
action: BackupAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
errChecker: NotNil,
dbList: []string{"template1"},
command: nil,
name: "PostgreS backup command",
dbEngine: "MySQLDBEngine",
action: BackupAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
dbEngineVersion: "12.7",
errChecker: NotNil,
dbList: []string{"template1"},
command: nil,
},
}

for _, tc := range testCases {
outCommand, _, err := prepareCommand(context.Background(), tc.dbEngine, tc.action, tc.dbEndpoint, tc.username, tc.password, tc.dbList, tc.backupPrefix, tc.backupID, tc.tp.Profile)
outCommand, _, err := prepareCommand(context.Background(), tc.dbEngine, tc.action, tc.dbEndpoint, tc.username, tc.password, tc.dbList, tc.backupPrefix, tc.backupID, tc.tp.Profile, tc.dbEngineVersion)

c.Check(err, tc.errChecker, Commentf("Case %s failed", tc.name))
c.Assert(outCommand, DeepEquals, tc.command)
Expand Down
37 changes: 31 additions & 6 deletions pkg/function/restore_rds_snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (

"github.com/aws/aws-sdk-go/aws/awserr"
rdserr "github.com/aws/aws-sdk-go/service/rds"
"github.com/hashicorp/go-version"
"github.com/pkg/errors"

kanister "github.com/kanisterio/kanister/pkg"
Expand Down Expand Up @@ -65,8 +66,9 @@ const (
// PostgreSQLEngine stores the postgres appname
PostgreSQLEngine RDSDBEngine = "PostgreSQL"

restoredAuroraInstanceSuffix = "instance-1"
defaultAuroraInstanceClass = "db.r5.large"
restoredAuroraInstanceSuffix = "instance-1"
defaultAuroraInstanceClass = "db.r5.large"
RDSPostgresDBInstanceEngineVersion = "13.0"
)

type restoreRDSSnapshotFunc struct{}
Expand Down Expand Up @@ -167,7 +169,14 @@ func restoreRDSSnapshot(ctx context.Context, namespace, instanceID, snapshotID,
}

dbEndpoint := *descOp.DBInstances[0].Endpoint.Address
if _, err = execDumpCommand(ctx, dbEngine, RestoreAction, namespace, dbEndpoint, username, password, nil, backupArtifactPrefix, backupID, profile); err != nil {

// get the engine version
dbEngineVersion, err := rdsDBEngineVersion(ctx, rdsCli, instanceID)
if err != nil {
return nil, errors.Wrapf(err, "Couldn't find DBInstance Version")
}

if _, err = execDumpCommand(ctx, dbEngine, RestoreAction, namespace, dbEndpoint, username, password, nil, backupArtifactPrefix, backupID, profile, dbEngineVersion); err != nil {
return nil, errors.Wrapf(err, "Failed to restore RDS from dump. InstanceID=%s", instanceID)
}

Expand All @@ -177,11 +186,27 @@ func restoreRDSSnapshot(ctx context.Context, namespace, instanceID, snapshotID,
}

// nolint:unparam
func postgresRestoreCommand(pgHost, username, password string, dbList []string, backupArtifactPrefix, backupID string, profile []byte) ([]string, error) {
func postgresRestoreCommand(pgHost, username, password string, dbList []string, backupArtifactPrefix, backupID string, profile []byte, dbEngineVersion string) ([]string, error) {
replaceCommand := ""
if len(dbList) == 0 {
return nil, errors.New("No database found. Atleast one db needed to connect")
}

// check if PostgresDB version < 13
v1, err := version.NewVersion(dbEngineVersion)
if err != nil {
return nil, errors.Wrapf(err, "Couldn't find DBInstance Version")
}
// Add Constraints
constraints, err := version.NewConstraint("< " + RDSPostgresDBInstanceEngineVersion)
if err != nil {
return nil, errors.Wrapf(err, "Couldn't add constraint to DBInstance Version")
}
// Verify Constraints
if constraints.Check(v1) {
replaceCommand = " sed 's/LOCALE/LC_COLLATE/' |"
}

return []string{
"bash",
"-o",
Expand All @@ -191,8 +216,8 @@ func postgresRestoreCommand(pgHost, username, password string, dbList []string,
"-c",
fmt.Sprintf(`
export PGHOST=%s
kando location pull --profile '%s' --path "%s" - | gunzip -c -f | psql -q -U "${PGUSER}" %s
`, pgHost, profile, fmt.Sprintf("%s/%s", backupArtifactPrefix, backupID), dbList[0]),
kando location pull --profile '%s' --path "%s" - | gunzip -c -f |%s psql -q -U "${PGUSER}" %s
`, pgHost, profile, fmt.Sprintf("%s/%s", backupArtifactPrefix, backupID), replaceCommand, dbList[0]),
}, nil
}

Expand Down
14 changes: 14 additions & 0 deletions pkg/function/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,20 @@ func findRDSEndpoint(ctx context.Context, rdsCli *rds.RDS, instanceID string) (s
return *dbInstance.DBInstances[0].Endpoint.Address, nil
}

// rdsDBEngineVersion returns the database engine version
func rdsDBEngineVersion(ctx context.Context, rdsCli *rds.RDS, instanceID string) (string, error) {
dbInstance, err := rdsCli.DescribeDBInstances(ctx, instanceID)
if err != nil {
return "", err
}

if (len(dbInstance.DBInstances) == 0) || (dbInstance.DBInstances[0].EngineVersion == nil) {
return "", errors.Errorf("DB Instance's Engine version is nil")
}

return *dbInstance.DBInstances[0].EngineVersion, nil
}

func createPostgresSecret(cli kubernetes.Interface, name, namespace, username, password string) error {
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Expand Down

0 comments on commit 4527d93

Please sign in to comment.