Add option to take a snapshot of a managed service source cluster

deployment/cdk/opensearch-service-migration/cdk.context.json

@@ -7,7 +7,7 @@
         "type": "none | basic | sigv4",
         "// basic auth documentation": "The next two lines are releavant for basic auth only",
         "username": "<USERNAME>",
-        "password_from_secret_arn": "<ARN_OF_SECRET_CONTAINING_PASSWORD>",
+        "passwordFromSecretArn": "<ARN_OF_SECRET_CONTAINING_PASSWORD>",
         "// sigv4 documentation": "The next two lines are releavant for sigv4 only",
         "region": "<REGION>",
         "serviceSigningName": "es | aoss"
@@ -19,7 +19,7 @@
         "type": "none | basic | sigv4",
         "// basic auth documentation": "The next two lines are releavant for basic auth only",
         "username": "<USERNAME>",
-        "password_from_secret_arn": "<ARN_OF_SECRET_CONTAINING_PASSWORD>",
+        "passwordFromSecretArn": "<ARN_OF_SECRET_CONTAINING_PASSWORD>",
         "// sigv4 documentation": "The next two lines are releavant for sigv4 only",
         "region": "<REGION>",
         "serviceSigningName": "es | aoss"

deployment/cdk/opensearch-service-migration/lib/common-utilities.ts

@@ -187,6 +187,33 @@ export function createDefaultECSTaskRole(scope: Construct, serviceName: string):
     return serviceTaskRole
 }
 
+export function createSnapshotOnAOSRole(scope: Construct, artifactS3Arn: string, migrationConsoleTaskRoleArn: string,
+                                        region: string, stage: string, defaultDeployId: string): Role {
+    const snapshotRole = new Role(scope, `SnapshotRole`, {
+        assumedBy: new ServicePrincipal('es.amazonaws.com'),  // Note that snapshots are not currently possible on AOSS
+        description: 'Role that grants OpenSearch Service permissions to access S3 to create snapshots',
+        roleName: `OSMigrations-${stage}-${region}-${defaultDeployId}-SnapshotRole`
+    });
+    snapshotRole.addToPolicy(new PolicyStatement({
+        effect: Effect.ALLOW,
+        actions: ['s3:ListBucket'],
+        resources: [artifactS3Arn],
+    }));
+
+    snapshotRole.addToPolicy(new PolicyStatement({
+        effect: Effect.ALLOW,
+        actions: ['s3:GetObject', 's3:PutObject', 's3:DeleteObject'],
+        resources: [`${artifactS3Arn}/*`],
+    }));
+
+    // The Migration Console Role needs to be able to pass the snapshot role
+    const requestingRole = Role.fromRoleArn(scope, 'RequestingRole', migrationConsoleTaskRoleArn);
+    snapshotRole.grantPassRole(requestingRole);
+
+    return snapshotRole
+}
+
+
 export function validateFargateCpuArch(cpuArch?: string): CpuArchitecture {
     const desiredArch = cpuArch ?? process.arch
     const desiredArchUpper = desiredArch.toUpperCase()

deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts

@@ -11,7 +11,8 @@ import {
     getSecretAccessPolicy,
     getMigrationStringParameterValue,
     hashStringSHA256,
-    MigrationSSMParameter
+    MigrationSSMParameter,
+    createSnapshotOnAOSRole
 } from "../common-utilities";
 import {StreamingSourceType} from "../streaming-source-type";
 import {LogGroup, RetentionDays} from "aws-cdk-lib/aws-logs";
@@ -33,6 +34,7 @@ export interface MigrationConsoleProps extends StackPropsExt {
     readonly servicesYaml: ServicesYaml,
     readonly otelCollectorEnabled?: boolean,
     readonly sourceCluster?: ClusterYaml,
+    readonly managedServiceSourceSnapshotEnabled?: boolean
 }
 
 export class MigrationConsoleStack extends MigrationServiceCore {
@@ -308,6 +310,7 @@ export class MigrationConsoleStack extends MigrationServiceCore {
                 parameter: MigrationSSMParameter.OSI_PIPELINE_LOG_GROUP_NAME,
             });
         }
+
         this.createService({
             serviceName: "migration-console",
             dockerDirectoryPath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/src/main/docker/migrationConsole"),
@@ -324,6 +327,9 @@ export class MigrationConsoleStack extends MigrationServiceCore {
             ...props
         });
 
+        if (props.managedServiceSourceSnapshotEnabled) {
+            const snapshotRole = createSnapshotOnAOSRole(this, artifactS3Arn, this.serviceTaskRole.roleArn, this.region, props.stage, props.defaultDeployId);
+        }
     }
 
 }

deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts

@@ -17,7 +17,7 @@ import {
 import {DockerImageAsset} from "aws-cdk-lib/aws-ecr-assets";
 import {Duration, RemovalPolicy, Stack} from "aws-cdk-lib";
 import {LogGroup, RetentionDays} from "aws-cdk-lib/aws-logs";
-import {PolicyStatement} from "aws-cdk-lib/aws-iam";
+import {PolicyStatement, Role} from "aws-cdk-lib/aws-iam";
 import {createDefaultECSTaskRole} from "../common-utilities";
 import {OtelCollectorSidecar} from "./migration-otel-collector-sidecar";
 import { IApplicationTargetGroup, INetworkTargetGroup } from "aws-cdk-lib/aws-elasticloadbalancingv2";
@@ -55,6 +55,7 @@ export interface MigrationServiceCoreProps extends StackPropsExt {
 export type ELBTargetGroup = IApplicationTargetGroup | INetworkTargetGroup;
 
 export class MigrationServiceCore extends Stack {
+    serviceTaskRole: Role;
 
     createService(props: MigrationServiceCoreProps) {
         if ((!props.dockerDirectoryPath && !props.dockerImageRegistryName) || (props.dockerDirectoryPath && props.dockerImageRegistryName)) {
@@ -66,8 +67,8 @@ export class MigrationServiceCore extends Stack {
             vpc: props.vpc
         })
 
-        const serviceTaskRole = createDefaultECSTaskRole(this, props.serviceName)
-        props.taskRolePolicies?.forEach(policy => serviceTaskRole.addToPolicy(policy))
+        this.serviceTaskRole = createDefaultECSTaskRole(this, props.serviceName)
+        props.taskRolePolicies?.forEach(policy => this.serviceTaskRole.addToPolicy(policy))
 
         const serviceTaskDef = new FargateTaskDefinition(this, "ServiceTaskDef", {
             ephemeralStorageGiB: props.ephemeralStorageGiB ? props.ephemeralStorageGiB : 75,
@@ -78,7 +79,7 @@ export class MigrationServiceCore extends Stack {
             family: `migration-${props.stage}-${props.serviceName}`,
             memoryLimitMiB: props.taskMemoryLimitMiB ? props.taskMemoryLimitMiB : 1024,
             cpu: props.taskCpuUnits ? props.taskCpuUnits : 256,
-            taskRole: serviceTaskRole
+            taskRole: this.serviceTaskRole
         });
         if (props.volumes) {
             props.volumes.forEach(vol => serviceTaskDef.addVolume(vol))

deployment/cdk/opensearch-service-migration/lib/stack-composer.ts

@@ -211,6 +211,7 @@ export class StackComposer {
         const reindexFromSnapshotServiceEnabled = this.getContextForType('reindexFromSnapshotServiceEnabled', 'boolean', defaultValues, contextJSON)
         const reindexFromSnapshotExtraArgs = this.getContextForType('reindexFromSnapshotExtraArgs', 'string', defaultValues, contextJSON)
         const albAcmCertArn = this.getContextForType('albAcmCertArn', 'string', defaultValues, contextJSON);
+        const managedServiceSourceSnapshotEnabled = this.getContextForType('managedServiceSourceSnapshotEnabled', 'boolean', defaultValues, contextJSON)
 
         // We're in a transition state from an older model with limited, individually defined fields and heading towards objects
         // that fully define the source and target cluster configurations. For the time being, we're supporting both.
@@ -231,6 +232,11 @@ export class StackComposer {
         const sourceCluster = (sourceClusterDefinition && !sourceClusterDisabled) ? parseClusterDefinition(sourceClusterDefinition) : undefined
         const sourceClusterEndpoint = sourceCluster?.endpoint
 
+        if (managedServiceSourceSnapshotEnabled && !sourceCluster?.auth.sigv4) {
+            throw new Error("A managed service source snapshot is only compatible with sigv4 authentication. If you would like to proceed" +
+                " please disable `managedServiceSourceSnapshotEnabled` and provide your own snapshot of the source cluster.")
+        }
+
         const targetClusterEndpointField = this.getContextForType('targetClusterEndpoint', 'string', defaultValues, contextJSON)
         let targetClusterDefinition = this.getContextForType('targetCluster', 'object', defaultValues, contextJSON)
         const usePreexistingTargetCluster = !!(targetClusterEndpointField || targetClusterDefinition)
@@ -266,7 +272,8 @@ export class StackComposer {
         }
 
         const targetClusterAuth = targetCluster?.auth
-        const targetVersion = this.getEngineVersion(targetCluster?.version ?? engineVersion)
+        const targetVersion = targetCluster?.version ? this.getEngineVersion(targetCluster?.version) : null
+        const engineVersionValue = engineVersion ? this.getEngineVersion(engineVersion) : this.getEngineVersion('OS_2.15')
 
         const requiredFields: { [key: string]: any; } = {"stage":stage}
         for (let key in requiredFields) {
@@ -275,7 +282,7 @@ export class StackComposer {
             }
         }
         if (addOnMigrationDeployId && vpcId) {
-            console.warn("Addon deployments will use the original deployment 'vpcId' regardless of passed 'vpcId' values")
+            console.warn("Add-on deployments will use the original deployment 'vpcId' regardless of passed 'vpcId' values")
         }
         if (stage.length > 15) {
             throw new Error(`Maximum allowed stage name length is 15 characters but received ${stage}`)
@@ -359,7 +366,7 @@ export class StackComposer {
         let openSearchStack
         if (!preexistingOrContainerTargetEndpoint) {
             openSearchStack = new OpenSearchDomainStack(scope, `openSearchDomainStack-${deployId}`, {
-                version: targetVersion,
+                version: engineVersionValue,
                 domainName: clusterDomainName,
                 dataNodeInstanceType: dataNodeType,
                 dataNodes: dataNodeCount,
@@ -609,6 +616,7 @@ export class StackComposer {
                 defaultDeployId: defaultDeployId,
                 fargateCpuArch: fargateCpuArch,
                 otelCollectorEnabled,
+                managedServiceSourceSnapshotEnabled,
                 env: props.env
             })
             // To enable the Migration Console to make requests to other service endpoints with services,

deployment/cdk/opensearch-service-migration/options.md

@@ -54,11 +54,12 @@ The optional component is:
 
 ### Reindex from Snapshot (RFS) Service Options
 
-| Name                              | Type    | Example                                                              | Description                                                                                                                                                                |
-| --------------------------------- | ------- | -------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| reindexFromSnapshotServiceEnabled | boolean | true                                                                 | Create resources for deploying and configuring the RFS ECS service                                                                                                         |
-| reindexFromSnapshotExtraArgs      | string  | "--target-aws-region us-east-1 --target-aws-service-signing-name es" | Extra arguments to provide to the Document Migration command with space separation. See [RFS Arguments](../../../DocumentsFromSnapshotMigration/README.md#Arguments). [^1] |
-| sourceClusterEndpoint             | string  | `"https://source-cluster.elb.us-east-1.endpoint.com"`                | The endpoint for the source cluster from which RFS will take a snapshot                                                                                                    |
+| Name                                | Type    | Example                                                              | Description                                                                                                                                                                |
+| ----------------------------------- | ------- | -------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| reindexFromSnapshotServiceEnabled   | boolean | true                                                                 | Create resources for deploying and configuring the RFS ECS service                                                                                                         |
+| reindexFromSnapshotExtraArgs        | string  | "--target-aws-region us-east-1 --target-aws-service-signing-name es" | Extra arguments to provide to the Document Migration command with space separation. See [RFS Arguments](../../../DocumentsFromSnapshotMigration/README.md#Arguments). [^1] |
+| sourceClusterEndpoint               | string  | `"https://source-cluster.elb.us-east-1.endpoint.com"`                | The endpoint for the source cluster from which RFS will take a snapshot                                                                                                    |
+| managedServiceSourceSnapshotEnabled | boolean | true                                                                 | Create the necessary roles and trust relationships to take a snapshot of a managed service source cluster. This is only compatible with SigV4 auth.                        |
 
 ### VPC Options