Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update RDS postgres restore command #1118

Merged
merged 5 commits into from
Oct 18, 2021
Merged

Update RDS postgres restore command #1118

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
050a57e
Update PostgresRestore command for Postgres version less than 13
akankshakumari393 Oct 18, 2021
bfb3827
Update go.mod go.sum
akankshakumari393 Oct 18, 2021
527dcbc
Update pkg/function/utils.go
akankshakumari393 Oct 18, 2021
1a3b0c4
Use const variable for PostgresDBEngine Version
akankshakumari393 Oct 18, 2021
ad0e88a
Address Review comments
akankshakumari393 Oct 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ require (
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0
github.com/graymeta/stow v0.0.0-00010101000000-000000000000
github.com/hashicorp/go-version v1.2.0
github.com/huandu/xstrings v1.2.0 // indirect
github.com/imdario/mergo v0.3.11 // indirect
github.com/jarcoal/httpmock v1.0.4 // indirect
Expand Down
1 change: 1 addition & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
Expand Down
16 changes: 11 additions & 5 deletions pkg/function/export_rds_snapshot_location.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,14 @@ func exportRDSSnapshotToLoc(ctx context.Context, namespace, instanceID, snapshot
// Create unique backupID
backupID := fmt.Sprintf("backup-%s.tar.gz", rand.String(10))

// get the engine version
dbEngineVersion, err := findRDSDBEngineVersion(ctx, rdsCli, tmpInstanceID)
if err != nil {
return nil, errors.Wrapf(err, "Couldn't find DBInstance Version")
}

// Extract dump from DB
output, err := execDumpCommand(ctx, dbEngine, BackupAction, namespace, dbEndpoint, username, password, databases, backupPrefix, backupID, profile)
output, err := execDumpCommand(ctx, dbEngine, BackupAction, namespace, dbEndpoint, username, password, databases, backupPrefix, backupID, profile, dbEngineVersion)
if err != nil {
return nil, errors.Wrap(err, "Unable to extract and push db dump to location")
}
Expand Down Expand Up @@ -188,13 +194,13 @@ func (*exportRDSSnapshotToLocationFunc) RequiredArgs() []string {
return []string{ExportRDSSnapshotToLocNamespaceArg, ExportRDSSnapshotToLocInstanceIDArg, ExportRDSSnapshotToLocSnapshotIDArg, ExportRDSSnapshotToLocDBEngineArg}
}

func execDumpCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, namespace, dbEndpoint, username, password string, databases []string, backupPrefix, backupID string, profile *param.Profile) (map[string]interface{}, error) {
func execDumpCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, namespace, dbEndpoint, username, password string, databases []string, backupPrefix, backupID string, profile *param.Profile, dbEngineVersion string) (map[string]interface{}, error) {
// Trim "\n" from creds
username = strings.TrimSpace(username)
password = strings.TrimSpace(password)

// Prepare and execute command with kubetask
command, image, err := prepareCommand(ctx, dbEngine, action, dbEndpoint, username, password, databases, backupPrefix, backupID, profile)
command, image, err := prepareCommand(ctx, dbEngine, action, dbEndpoint, username, password, databases, backupPrefix, backupID, profile, dbEngineVersion)
if err != nil {
return nil, err
}
Expand All @@ -221,7 +227,7 @@ func execDumpCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction
return kubeTask(ctx, cli, namespace, image, command, injectPostgresSecrets(secretName))
}

func prepareCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, dbEndpoint, username, password string, dbList []string, backupPrefix, backupID string, profile *param.Profile) ([]string, string, error) {
func prepareCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction, dbEndpoint, username, password string, dbList []string, backupPrefix, backupID string, profile *param.Profile, dbEngineVersion string) ([]string, string, error) {
// Convert profile object into json
profileJson, err := json.Marshal(profile)
if err != nil {
Expand Down Expand Up @@ -256,7 +262,7 @@ func prepareCommand(ctx context.Context, dbEngine RDSDBEngine, action RDSAction,
command, err := postgresBackupCommand(dbEndpoint, username, password, dbList, backupPrefix, backupID, profileJson)
return command, postgresToolsImage, err
case RestoreAction:
command, err := postgresRestoreCommand(dbEndpoint, username, password, dbList, backupPrefix, backupID, profileJson)
command, err := postgresRestoreCommand(dbEndpoint, username, password, dbList, backupPrefix, backupID, profileJson, dbEngineVersion)
return command, postgresToolsImage, err
}
}
Expand Down
111 changes: 67 additions & 44 deletions pkg/function/rds_functions_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,30 +29,51 @@ var _ = Suite(&RDSFunctionsTest{})

func (s *RDSFunctionsTest) TestPrepareCommand(c *C) {
testCases := []struct {
name string
dbEngine RDSDBEngine
dbList []string
action RDSAction
dbEndpoint string
username string
password string
backupPrefix string
backupID string
errChecker Checker
tp param.TemplateParams
command []string
name string
dbEngine RDSDBEngine
dbList []string
action RDSAction
dbEndpoint string
username string
password string
backupPrefix string
backupID string
dbEngineVersion string
errChecker Checker
tp param.TemplateParams
command []string
}{
{
name: "PostgreS restore command",
dbEngine: PostgrSQLEngine,
action: RestoreAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
errChecker: IsNil,
dbList: []string{"template1"},
name: "PostgreS restore command",
dbEngine: PostgrSQLEngine,
action: RestoreAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
dbEngineVersion: "12.7",
errChecker: IsNil,
dbList: []string{"template1"},
command: []string{"bash", "-o", "errexit", "-o", "pipefail", "-c",
fmt.Sprintf(`
export PGHOST=%s
kando location pull --profile '%s' --path "%s" - | gunzip -c -f | sed 's/LOCALE/LC_COLLATE/' | psql -q -U "${PGUSER}" %s
`, "db-endpoint", "null", fmt.Sprintf("%s/%s", "/backup/postgres-backup", "backup-id"), []string{"template1"}[0]),
},
},
{
name: "PostgreS restore command",
dbEngine: PostgrSQLEngine,
action: RestoreAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
dbEngineVersion: "13.3",
errChecker: IsNil,
dbList: []string{"template1"},
command: []string{"bash", "-o", "errexit", "-o", "pipefail", "-c",
fmt.Sprintf(`
export PGHOST=%s
Expand All @@ -61,16 +82,17 @@ func (s *RDSFunctionsTest) TestPrepareCommand(c *C) {
},
},
{
name: "PostgreS backup command",
dbEngine: PostgrSQLEngine,
action: BackupAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
errChecker: IsNil,
dbList: []string{"template1"},
name: "PostgreS backup command",
dbEngine: PostgrSQLEngine,
action: BackupAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
dbEngineVersion: "12.7",
errChecker: IsNil,
dbList: []string{"template1"},
command: []string{"bash", "-o", "errexit", "-o", "pipefail", "-c",
fmt.Sprintf(`
export PGHOST=%s
Expand All @@ -88,22 +110,23 @@ func (s *RDSFunctionsTest) TestPrepareCommand(c *C) {
},
},
{
name: "PostgreS backup command",
dbEngine: "MySQLDBEngine",
action: BackupAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
errChecker: NotNil,
dbList: []string{"template1"},
command: nil,
name: "PostgreS backup command",
dbEngine: "MySQLDBEngine",
action: BackupAction,
dbEndpoint: "db-endpoint",
username: "test-user",
password: "secret-pass",
backupPrefix: "/backup/postgres-backup",
backupID: "backup-id",
dbEngineVersion: "12.7",
errChecker: NotNil,
dbList: []string{"template1"},
command: nil,
},
}

for _, tc := range testCases {
outCommand, _, err := prepareCommand(context.Background(), tc.dbEngine, tc.action, tc.dbEndpoint, tc.username, tc.password, tc.dbList, tc.backupPrefix, tc.backupID, tc.tp.Profile)
outCommand, _, err := prepareCommand(context.Background(), tc.dbEngine, tc.action, tc.dbEndpoint, tc.username, tc.password, tc.dbList, tc.backupPrefix, tc.backupID, tc.tp.Profile, tc.dbEngineVersion)

c.Check(err, tc.errChecker, Commentf("Case %s failed", tc.name))
c.Assert(outCommand, DeepEquals, tc.command)
Expand Down
32 changes: 28 additions & 4 deletions pkg/function/restore_rds_snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (

"github.com/aws/aws-sdk-go/aws/awserr"
rdserr "github.com/aws/aws-sdk-go/service/rds"
"github.com/hashicorp/go-version"
"github.com/pkg/errors"

kanister "github.com/kanisterio/kanister/pkg"
Expand Down Expand Up @@ -167,7 +168,14 @@ func restoreRDSSnapshot(ctx context.Context, namespace, instanceID, snapshotID,
}

dbEndpoint := *descOp.DBInstances[0].Endpoint.Address
if _, err = execDumpCommand(ctx, dbEngine, RestoreAction, namespace, dbEndpoint, username, password, nil, backupArtifactPrefix, backupID, profile); err != nil {

// get the engine version
dbEngineVersion, err := findRDSDBEngineVersion(ctx, rdsCli, instanceID)
if err != nil {
return nil, errors.Wrapf(err, "Couldn't find DBInstance Version")
}

if _, err = execDumpCommand(ctx, dbEngine, RestoreAction, namespace, dbEndpoint, username, password, nil, backupArtifactPrefix, backupID, profile, dbEngineVersion); err != nil {
return nil, errors.Wrapf(err, "Failed to restore RDS from dump. InstanceID=%s", instanceID)
}

Expand All @@ -177,11 +185,27 @@ func restoreRDSSnapshot(ctx context.Context, namespace, instanceID, snapshotID,
}

// nolint:unparam
func postgresRestoreCommand(pgHost, username, password string, dbList []string, backupArtifactPrefix, backupID string, profile []byte) ([]string, error) {
func postgresRestoreCommand(pgHost, username, password string, dbList []string, backupArtifactPrefix, backupID string, profile []byte, dbEngineVersion string) ([]string, error) {
replaceCommand := ""
if len(dbList) == 0 {
return nil, errors.New("No database found. Atleast one db needed to connect")
}

// check if PostgresDB version > 13
v1, err := version.NewVersion(dbEngineVersion)
if err != nil {
return nil, errors.Wrapf(err, "Couldn't find DBInstance Version")
}
// Add Constraints
constraints, err := version.NewConstraint(">= 13.0")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lets declare version as const

if err != nil {
return nil, errors.Wrapf(err, "Couldn't add constraint to DBInstance Version")
}
// Verify Constraints
if !constraints.Check(v1) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is it same as v1 >= RDSPostgresDBInstanceEngineVersion?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@PrasadG193 Since we have used !constraints.Check(v1) . It will be same as v1 < RDSPostgresDBInstanceEngineVersion which is if v1 < 13.0. Let me reverse the constraint used in line number 201 and update the If condition to make it more readable.

replaceCommand = " sed 's/LOCALE/LC_COLLATE/' |"
}

return []string{
"bash",
"-o",
Expand All @@ -191,8 +215,8 @@ func postgresRestoreCommand(pgHost, username, password string, dbList []string,
"-c",
fmt.Sprintf(`
export PGHOST=%s
kando location pull --profile '%s' --path "%s" - | gunzip -c -f | psql -q -U "${PGUSER}" %s
`, pgHost, profile, fmt.Sprintf("%s/%s", backupArtifactPrefix, backupID), dbList[0]),
kando location pull --profile '%s' --path "%s" - | gunzip -c -f |%s psql -q -U "${PGUSER}" %s
`, pgHost, profile, fmt.Sprintf("%s/%s", backupArtifactPrefix, backupID), replaceCommand, dbList[0]),
}, nil
}

Expand Down
14 changes: 14 additions & 0 deletions pkg/function/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,20 @@ func findRDSEndpoint(ctx context.Context, rdsCli *rds.RDS, instanceID string) (s
return *dbInstance.DBInstances[0].Endpoint.Address, nil
}

// findRDSEndpoint returns the database engine version
akankshakumari393 marked this conversation as resolved.
Show resolved Hide resolved
func findRDSDBEngineVersion(ctx context.Context, rdsCli *rds.RDS, instanceID string) (string, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
func findRDSDBEngineVersion(ctx context.Context, rdsCli *rds.RDS, instanceID string) (string, error) {
func rdsDBEngineVersion(ctx context.Context, rdsCli *rds.RDS, instanceID string) (string, error) {

dbInstance, err := rdsCli.DescribeDBInstances(ctx, instanceID)
if err != nil {
return "", err
}

if (len(dbInstance.DBInstances) == 0) || (dbInstance.DBInstances[0].EngineVersion == nil) {
return "", errors.Errorf("Received nil engine version")
Copy link
Contributor

@viveksinghggits viveksinghggits Oct 18, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we change this to
DB instance's Engine version is nil

}

return *dbInstance.DBInstances[0].EngineVersion, nil
}

func createPostgresSecret(cli kubernetes.Interface, name, namespace, username, password string) error {
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Expand Down