Add dbSubnetGroup to RestoreRDSSnapshot and ExportRDSLocation function (

#1953) * Update RDS postgres app implementation, execute postgres command from Deployment Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Fix Lint Remove unnecessary trailing new line Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Initialize testWorkloadName field Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Move BastionWorkload function to be used as utility * Minor refactor Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Remove export of BastionWorkload() utility function Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Add app name in error messages Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Minor refactor Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Minor refactor Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Update RDS Aurora app implementation, execute mysql query from Deployment Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Remove unused methods from rds aurora app Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Minor refactor Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Add helper methods to add DBSubnetGroup Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * add callers to create and delete dbsubnetGroups in rds postgres app test Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * add callers to create and delete dbsubnetGroups in rds aurora app test Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Refactor code * Refactor code Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * remove duplicate import Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * minor refactor * correct indentation * Fix error * Minor refactor Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Minor refactor Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Update RDS Aurora app implementation, execute mysql query from Deployment Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Remove unused methods from rds aurora app Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Minor refactor Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * minor refactor * Minor refactor * Modify Create RDS Snapshot function to add dbSubnetGroup as outputArtifact * Modify RestoreRDSSnapshot and ExportRDSLocation function to accept dbSubnetGroup as optional argument Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Pass VpcId while creating security group * Use securityGroup ID instead of security group name to authorizeSecurityGroupIngress Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Minor Fix * Modify Create RDS Snapshot function to add dbSubnetGroup as outputArtifact * Modify RestoreRDSSnapshot and ExportRDSLocation function to accept dbSubnetGroup as optional argument Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * [Refactor] Add function to get VPCId for RDS apps Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Remove unnecessary comment * Modify Create RDS Snapshot function to add dbSubnetGroup as outputArtifact * Refactor code Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Use switch case Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Refactor switch case Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Remove unnecessary comment Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com> * Refactor name --------- Signed-off-by: Akanksha Kumari <akankshakumari393@gmail.com>
kanisterio · Mar 29, 2023 · b1a4792 · b1a4792
1 parent 1049520
commit b1a4792
Show file tree

Hide file tree

Showing 6 changed files with 37 additions and 24 deletions.
diff --git a/pkg/app/rds_aurora_mysql.go b/pkg/app/rds_aurora_mysql.go
@@ -184,7 +184,7 @@ func (a *RDSAuroraMySQLDB) Install(ctx context.Context, namespace string) error
 	}
 
 	// create db instance in the cluster
-	_, err = rdsCli.CreateDBInstanceInCluster(ctx, a.id, fmt.Sprintf("%s-instance-1", a.id), AuroraDBInstanceClass, string(function.DBEngineAuroraMySQL))
+	_, err = rdsCli.CreateDBInstanceInCluster(ctx, a.id, fmt.Sprintf("%s-instance-1", a.id), AuroraDBInstanceClass, string(function.DBEngineAuroraMySQL), a.dbSubnetGroup)
 	if err != nil {
 		return errors.Wrap(err, "Error creating an instance in Aurora DB cluster")
 	}

diff --git a/pkg/aws/rds/rds.go b/pkg/aws/rds/rds.go
@@ -73,12 +73,13 @@ func (r RDS) CreateDBCluster(ctx context.Context, storage int64, instanceClass,
 	return r.CreateDBClusterWithContext(ctx, dbi)
 }
 
-func (r RDS) CreateDBInstanceInCluster(ctx context.Context, restoredClusterID, instanceID, instanceClass, dbEngine string) (*rds.CreateDBInstanceOutput, error) {
+func (r RDS) CreateDBInstanceInCluster(ctx context.Context, restoredClusterID, instanceID, instanceClass, dbEngine, dbSubnetGroup string) (*rds.CreateDBInstanceOutput, error) {
 	pa := true
 	dbi := &rds.CreateDBInstanceInput{
 		DBClusterIdentifier:  &restoredClusterID,
 		DBInstanceClass:      &instanceClass,
 		DBInstanceIdentifier: &instanceID,
+		DBSubnetGroupName:    &dbSubnetGroup,
 		Engine:               &dbEngine,
 		PubliclyAccessible:   &pa,
 	}
@@ -281,23 +282,25 @@ func (r RDS) WaitUntilDBClusterSnapshotDeleted(ctx context.Context, snapshotID s
 	return r.WaitUntilDBClusterSnapshotDeletedWithContext(ctx, sdi)
 }
 
-func (r RDS) RestoreDBInstanceFromDBSnapshot(ctx context.Context, instanceID, snapshotID string, sgIDs []string) (*rds.RestoreDBInstanceFromDBSnapshotOutput, error) {
+func (r RDS) RestoreDBInstanceFromDBSnapshot(ctx context.Context, instanceID, subnetGroupName, snapshotID string, sgIDs []string) (*rds.RestoreDBInstanceFromDBSnapshotOutput, error) {
 	rdbi := &rds.RestoreDBInstanceFromDBSnapshotInput{
 		DBInstanceIdentifier: &instanceID,
 		DBSnapshotIdentifier: &snapshotID,
+		DBSubnetGroupName:    &subnetGroupName,
 		VpcSecurityGroupIds:  convertSGIDs(sgIDs),
 	}
 	return r.RestoreDBInstanceFromDBSnapshotWithContext(ctx, rdbi)
 }
 
-func (r RDS) RestoreDBClusterFromDBSnapshot(ctx context.Context, instanceID, snapshotID, dbEngine, version string, sgIDs []string) (*rds.RestoreDBClusterFromSnapshotOutput, error) {
+func (r RDS) RestoreDBClusterFromDBSnapshot(ctx context.Context, instanceID, dbSubnetGroup, snapshotID, dbEngine, version string, sgIDs []string) (*rds.RestoreDBClusterFromSnapshotOutput, error) {
 	var rdi *rds.RestoreDBClusterFromSnapshotInput
 	if sgIDs == nil {
 		rdi = &rds.RestoreDBClusterFromSnapshotInput{
 			Engine:              &dbEngine,
 			EngineVersion:       &version,
 			DBClusterIdentifier: &instanceID,
 			SnapshotIdentifier:  &snapshotID,
+			DBSubnetGroupName:   &dbSubnetGroup,
 		}
 	} else {
 		rdi = &rds.RestoreDBClusterFromSnapshotInput{
@@ -306,6 +309,7 @@ func (r RDS) RestoreDBClusterFromDBSnapshot(ctx context.Context, instanceID, sna
 			DBClusterIdentifier: &instanceID,
 			SnapshotIdentifier:  &snapshotID,
 			VpcSecurityGroupIds: convertSGIDs(sgIDs),
+			DBSubnetGroupName:   &dbSubnetGroup,
 		}
 	}
 	return r.RestoreDBClusterFromSnapshotWithContext(ctx, rdi)

diff --git a/pkg/function/create_rds_snapshot.go b/pkg/function/create_rds_snapshot.go
@@ -136,9 +136,9 @@ func createRDSSnapshot(ctx context.Context, instanceID string, dbEngine RDSDBEng
 	var dbSubnetGroup *string
 	switch {
 	case isAuroraCluster(string(dbEngine)):
-		dbSubnetGroup, e = GetRDSAuroraDbSubnetGroup(ctx, rdsCli, instanceID)
+		dbSubnetGroup, e = GetRDSAuroraDBSubnetGroup(ctx, rdsCli, instanceID)
 	default:
-		dbSubnetGroup, e = GetRDSDbSubnetGroup(ctx, rdsCli, instanceID)
+		dbSubnetGroup, e = GetRDSDBSubnetGroup(ctx, rdsCli, instanceID)
 	}
 	if e != nil {
 		return nil, errors.Wrapf(e, "Failed to get dbSubnetGroup ids. InstanceID=%s", instanceID)

diff --git a/pkg/function/export_rds_snapshot_location.go b/pkg/function/export_rds_snapshot_location.go
@@ -54,6 +54,7 @@ const (
 	ExportRDSSnapshotToLocDatabasesArg       = "databases"
 	ExportRDSSnapshotToLocSecGrpIDArg        = "securityGroupID"
 	ExportRDSSnapshotToLocBackupID           = "backupID"
+	ExportRDSSnapshotToLocDBSubnetGroupArg   = "dbSubnetGroup"
 
 	PostgrSQLEngine RDSDBEngine = "PostgreSQL"
 
@@ -75,7 +76,7 @@ func (*exportRDSSnapshotToLocationFunc) Name() string {
 	return ExportRDSSnapshotToLocFuncName
 }
 
-func exportRDSSnapshotToLoc(ctx context.Context, namespace, instanceID, snapshotID, username, password string, databases []string, backupPrefix string, dbEngine RDSDBEngine, sgIDs []string, profile *param.Profile) (map[string]interface{}, error) {
+func exportRDSSnapshotToLoc(ctx context.Context, namespace, instanceID, snapshotID, username, password string, databases []string, dbSubnetGroup, backupPrefix string, dbEngine RDSDBEngine, sgIDs []string, profile *param.Profile) (map[string]interface{}, error) {
 	// Validate profilextractDumpFromDBe
 	if err := ValidateProfile(profile); err != nil {
 		return nil, errors.Wrap(err, "Profile Validation failed")
@@ -105,7 +106,7 @@ func exportRDSSnapshotToLoc(ctx context.Context, namespace, instanceID, snapshot
 
 	log.WithContext(ctx).Print("Spin up temporary RDS instance from the snapshot.", field.M{"SnapshotID": snapshotID, "InstanceID": tmpInstanceID})
 	// Create tmp instance from snapshot
-	if err := restoreFromSnapshot(ctx, rdsCli, tmpInstanceID, snapshotID, sgIDs); err != nil {
+	if err := restoreFromSnapshot(ctx, rdsCli, tmpInstanceID, dbSubnetGroup, snapshotID, sgIDs); err != nil {
 		return nil, errors.Wrapf(err, "Failed to restore snapshot. SnapshotID=%s", snapshotID)
 	}
 	defer func() {
@@ -150,7 +151,7 @@ func exportRDSSnapshotToLoc(ctx context.Context, namespace, instanceID, snapshot
 }
 
 func (crs *exportRDSSnapshotToLocationFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) {
-	var namespace, instanceID, snapshotID, username, password, backupArtifact string
+	var namespace, instanceID, snapshotID, username, password, dbSubnetGroup, backupArtifact string
 	var dbEngine RDSDBEngine
 
 	if err := Arg(args, ExportRDSSnapshotToLocNamespaceArg, &namespace); err != nil {
@@ -174,7 +175,9 @@ func (crs *exportRDSSnapshotToLocationFunc) Exec(ctx context.Context, tp param.T
 	if err := OptArg(args, ExportRDSSnapshotToLocBackupArtPrefixArg, &backupArtifact, instanceID); err != nil {
 		return nil, err
 	}
-
+	if err := OptArg(args, ExportRDSSnapshotToLocDBSubnetGroupArg, &dbSubnetGroup, "default"); err != nil {
+		return nil, err
+	}
 	// Find databases
 	databases, err := GetYamlList(args, ExportRDSSnapshotToLocDatabasesArg)
 	if err != nil {
@@ -187,7 +190,7 @@ func (crs *exportRDSSnapshotToLocationFunc) Exec(ctx context.Context, tp param.T
 		return nil, err
 	}
 
-	return exportRDSSnapshotToLoc(ctx, namespace, instanceID, snapshotID, username, password, databases, backupArtifact, dbEngine, sgIDs, tp.Profile)
+	return exportRDSSnapshotToLoc(ctx, namespace, instanceID, snapshotID, username, password, databases, dbSubnetGroup, backupArtifact, dbEngine, sgIDs, tp.Profile)
 }
 
 func (*exportRDSSnapshotToLocationFunc) RequiredArgs() []string {
@@ -210,6 +213,7 @@ func (*exportRDSSnapshotToLocationFunc) Arguments() []string {
 		ExportRDSSnapshotToLocBackupArtPrefixArg,
 		ExportRDSSnapshotToLocDatabasesArg,
 		ExportRDSSnapshotToLocSecGrpIDArg,
+		ExportRDSSnapshotToLocDBSubnetGroupArg,
 	}
 }
 

diff --git a/pkg/function/restore_rds_snapshot.go b/pkg/function/restore_rds_snapshot.go
@@ -57,7 +57,8 @@ const (
 	RestoreRDSSnapshotSecGrpID = "securityGroupID"
 	// RestoreRDSSnapshotEndpoint to set endpoint of restored rds instance
 	RestoreRDSSnapshotEndpoint = "endpoint"
-
+	// RestoreRDSSnapshotDBSubnetGroup is the dbSubnetGroup of the restored RDS instance
+	RestoreRDSSnapshotDBSubnetGroup = "dbSubnetGroup"
 	// RestoreRDSSnapshotUsername stores username of the database
 	RestoreRDSSnapshotUsername = "username"
 	// RestoreRDSSnapshotPassword stores the password of the database
@@ -92,11 +93,12 @@ func (*restoreRDSSnapshotFunc) Arguments() []string {
 		RestoreRDSSnapshotPassword,
 		RestoreRDSSnapshotNamespace,
 		RestoreRDSSnapshotSecGrpID,
+		RestoreRDSSnapshotDBSubnetGroup,
 	}
 }
 
 func (*restoreRDSSnapshotFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) {
-	var namespace, instanceID, snapshotID, backupArtifactPrefix, backupID, username, password string
+	var namespace, instanceID, subnetGroup, snapshotID, backupArtifactPrefix, backupID, username, password string
 	var dbEngine RDSDBEngine
 
 	if err := Arg(args, RestoreRDSSnapshotInstanceID, &instanceID); err != nil {
@@ -110,6 +112,9 @@ func (*restoreRDSSnapshotFunc) Exec(ctx context.Context, tp param.TemplateParams
 	if err := OptArg(args, RestoreRDSSnapshotDBEngine, &dbEngine, ""); err != nil {
 		return nil, err
 	}
+	if err := OptArg(args, RestoreRDSSnapshotDBSubnetGroup, &subnetGroup, "default"); err != nil {
+		return nil, err
+	}
 	// Find security groups
 	sgIDs, err := GetYamlList(args, RestoreRDSSnapshotSecGrpID)
 	if err != nil {
@@ -136,10 +141,10 @@ func (*restoreRDSSnapshotFunc) Exec(ctx context.Context, tp param.TemplateParams
 		}
 	}
 
-	return restoreRDSSnapshot(ctx, namespace, instanceID, snapshotID, backupArtifactPrefix, backupID, username, password, dbEngine, sgIDs, tp.Profile)
+	return restoreRDSSnapshot(ctx, namespace, instanceID, subnetGroup, snapshotID, backupArtifactPrefix, backupID, username, password, dbEngine, sgIDs, tp.Profile)
 }
 
-func restoreRDSSnapshot(ctx context.Context, namespace, instanceID, snapshotID, backupArtifactPrefix, backupID, username, password string, dbEngine RDSDBEngine, sgIDs []string, profile *param.Profile) (map[string]interface{}, error) {
+func restoreRDSSnapshot(ctx context.Context, namespace, instanceID, subnetGroup, snapshotID, backupArtifactPrefix, backupID, username, password string, dbEngine RDSDBEngine, sgIDs []string, profile *param.Profile) (map[string]interface{}, error) {
 	// Validate profile
 	if err := ValidateProfile(profile); err != nil {
 		return nil, errors.Wrap(err, "Error validating profile")
@@ -171,9 +176,9 @@ func restoreRDSSnapshot(ctx context.Context, namespace, instanceID, snapshotID,
 			}
 		}
 		if !isAuroraCluster(string(dbEngine)) {
-			return nil, restoreFromSnapshot(ctx, rdsCli, instanceID, snapshotID, sgIDs)
+			return nil, restoreFromSnapshot(ctx, rdsCli, instanceID, subnetGroup, snapshotID, sgIDs)
 		}
-		return nil, restoreAuroraFromSnapshot(ctx, rdsCli, instanceID, snapshotID, string(dbEngine), sgIDs)
+		return nil, restoreAuroraFromSnapshot(ctx, rdsCli, instanceID, subnetGroup, snapshotID, string(dbEngine), sgIDs)
 	}
 
 	// Restore from dump
@@ -235,7 +240,7 @@ func postgresRestoreCommand(pgHost, username, password string, dbList []string,
 	}, nil
 }
 
-func restoreFromSnapshot(ctx context.Context, rdsCli *rds.RDS, instanceID, snapshotID string, securityGrpIDs []string) error {
+func restoreFromSnapshot(ctx context.Context, rdsCli *rds.RDS, instanceID, subnetGroup, snapshotID string, securityGrpIDs []string) error {
 	log.WithContext(ctx).Print("Deleting existing RDS DB instance.", field.M{"instanceID": instanceID})
 	if _, err := rdsCli.DeleteDBInstance(ctx, instanceID); err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
@@ -254,7 +259,7 @@ func restoreFromSnapshot(ctx context.Context, rdsCli *rds.RDS, instanceID, snaps
 
 	log.WithContext(ctx).Print("Restoring RDS DB instance from snapshot.", field.M{"instanceID": instanceID, "snapshotID": snapshotID})
 	// Restore from snapshot
-	if _, err := rdsCli.RestoreDBInstanceFromDBSnapshot(ctx, instanceID, snapshotID, securityGrpIDs); err != nil {
+	if _, err := rdsCli.RestoreDBInstanceFromDBSnapshot(ctx, instanceID, subnetGroup, snapshotID, securityGrpIDs); err != nil {
 		return errors.Wrapf(err, "Error restoring RDS DB instance from snapshot")
 	}
 
@@ -264,7 +269,7 @@ func restoreFromSnapshot(ctx context.Context, rdsCli *rds.RDS, instanceID, snaps
 	return errors.Wrap(err, "Error while waiting for new rds instance to be ready.")
 }
 
-func restoreAuroraFromSnapshot(ctx context.Context, rdsCli *rds.RDS, instanceID, snapshotID, dbEngine string, securityGroupIDs []string) error {
+func restoreAuroraFromSnapshot(ctx context.Context, rdsCli *rds.RDS, instanceID, subnetGroup, snapshotID, dbEngine string, securityGroupIDs []string) error {
 	// To delete an Aurora RDS instance we will have to delete all the instance that are running through it
 	// Once all those instances are deleted, Aurora cluster will be deleted automatically
 	descOp, err := rdsCli.DescribeDBClusters(ctx, instanceID)
@@ -288,7 +293,7 @@ func restoreAuroraFromSnapshot(ctx context.Context, rdsCli *rds.RDS, instanceID,
 	}
 
 	log.WithContext(ctx).Print("Restoring RDS Aurora DB Cluster from snapshot.", field.M{"instanceID": instanceID, "snapshotID": snapshotID})
-	op, err := rdsCli.RestoreDBClusterFromDBSnapshot(ctx, instanceID, snapshotID, dbEngine, version, securityGroupIDs)
+	op, err := rdsCli.RestoreDBClusterFromDBSnapshot(ctx, instanceID, subnetGroup, snapshotID, dbEngine, version, securityGroupIDs)
 	if err != nil {
 		return errors.Wrap(err, "Error restorig aurora db cluster from snapshot")
 	}
@@ -302,7 +307,7 @@ func restoreAuroraFromSnapshot(ctx context.Context, rdsCli *rds.RDS, instanceID,
 
 	log.WithContext(ctx).Print("Creating DB instance in the cluster")
 	// After Aurora cluster is created, we will have to explictly create the DB instance
-	dbInsOp, err := rdsCli.CreateDBInstanceInCluster(ctx, *op.DBCluster.DBClusterIdentifier, fmt.Sprintf("%s-%s", *op.DBCluster.DBClusterIdentifier, restoredAuroraInstanceSuffix), defaultAuroraInstanceClass, dbEngine)
+	dbInsOp, err := rdsCli.CreateDBInstanceInCluster(ctx, *op.DBCluster.DBClusterIdentifier, fmt.Sprintf("%s-%s", *op.DBCluster.DBClusterIdentifier, restoredAuroraInstanceSuffix), defaultAuroraInstanceClass, dbEngine, subnetGroup)
 	if err != nil {
 		return errors.Wrap(err, "Error while creating Aurora DB instance in the cluster.")
 	}

diff --git a/pkg/function/utils.go b/pkg/function/utils.go
@@ -249,7 +249,7 @@ func isAuroraCluster(engine string) bool {
 	return false
 }
 
-func GetRDSDbSubnetGroup(ctx context.Context, rdsCli *rds.RDS, instanceID string) (*string, error) {
+func GetRDSDBSubnetGroup(ctx context.Context, rdsCli *rds.RDS, instanceID string) (*string, error) {
 	result, err := rdsCli.DescribeDBInstances(ctx, instanceID)
 	if err != nil {
 		return nil, err
@@ -260,7 +260,7 @@ func GetRDSDbSubnetGroup(ctx context.Context, rdsCli *rds.RDS, instanceID string
 	return result.DBInstances[0].DBSubnetGroup.DBSubnetGroupName, nil
 }
 
-func GetRDSAuroraDbSubnetGroup(ctx context.Context, rdsCli *rds.RDS, instanceID string) (*string, error) {
+func GetRDSAuroraDBSubnetGroup(ctx context.Context, rdsCli *rds.RDS, instanceID string) (*string, error) {
 	desc, err := rdsCli.DescribeDBClusters(ctx, instanceID)
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {