fix(deadline): configure identity registration settings using RenderQ…

…ueue backend security group (#633)

Fixes #632

packages/aws-rfdk/lib/core/lib/deployment-instance.ts

@@ -9,12 +9,14 @@ import {
   UpdatePolicy,
 } from '@aws-cdk/aws-autoscaling';
 import {
+  AmazonLinuxGeneration,
   Connections,
   IConnectable,
   IMachineImage,
   InstanceClass,
   InstanceSize,
   InstanceType,
+  ISecurityGroup,
   IVpc,
   MachineImage,
   SubnetSelection,
@@ -92,6 +94,13 @@ export interface DeploymentInstanceProps {
    */
   readonly machineImage?: IMachineImage;
 
+  /**
+   * A security group to associate with the DeploymentInstance
+   *
+   * @default A new security group is created for the DeploymentInstance
+   */
+  readonly securityGroup?: ISecurityGroup;
+
   /**
    * Whether the instance should self-terminate after the deployment succeeds
    *
@@ -164,17 +173,18 @@ export class DeploymentInstance extends Construct implements IScriptHost, IConne
     this.asg = new AutoScalingGroup(this, 'ASG', {
       instanceType: props.instanceType ?? InstanceType.of(InstanceClass.T3, InstanceSize.SMALL),
       keyName: props.keyName,
-      machineImage: props.machineImage ?? MachineImage.latestAmazonLinux(),
-      vpc: props.vpc,
-      vpcSubnets: props.vpcSubnets ?? {
-        subnetType: SubnetType.PRIVATE,
-      },
+      machineImage: props.machineImage ?? MachineImage.latestAmazonLinux({ generation: AmazonLinuxGeneration.AMAZON_LINUX_2 }),
       minCapacity: 1,
       maxCapacity: 1,
+      securityGroup: props.securityGroup,
       signals: Signals.waitForAll({
         timeout: props.executionTimeout ?? DeploymentInstance.DEFAULT_EXECUTION_TIMEOUT,
       }),
       updatePolicy: UpdatePolicy.replacingUpdate(),
+      vpc: props.vpc,
+      vpcSubnets: props.vpcSubnets ?? {
+        subnetType: SubnetType.PRIVATE,
+      },
     });
     this.node.defaultChild = this.asg;
 

packages/aws-rfdk/lib/core/test/deployment-instance.test.ts

@@ -13,6 +13,7 @@ import {
 } from '@aws-cdk/assert';
 import { CfnLaunchConfiguration } from '@aws-cdk/aws-autoscaling';
 import {
+  AmazonLinuxGeneration,
   AmazonLinuxImage,
   ExecuteFileOptions,
   InstanceType,
@@ -167,7 +168,7 @@ describe('DeploymentInstance', () => {
 
       test('uses latest Amazon Linux machine image', () => {
         // GIVEN
-        const amazonLinux = MachineImage.latestAmazonLinux();
+        const amazonLinux = MachineImage.latestAmazonLinux({ generation: AmazonLinuxGeneration.AMAZON_LINUX_2 });
         const imageId: { Ref: string } = stack.resolve(amazonLinux.getImage(stack)).imageId;
 
         // THEN
@@ -659,6 +660,26 @@ describe('DeploymentInstance', () => {
     }));
   });
 
+  test('uses specified security group', () => {
+    // GIVEN
+    const securityGroupId = 'securityGroupId';
+    const securityGroup = SecurityGroup.fromSecurityGroupId(depStack, 'SecurityGroup', securityGroupId);
+    stack = new cdk.Stack(app, 'SecurityGroupStack');
+
+    // WHEN
+    new DeploymentInstance(stack, DEFAULT_CONSTRUCT_ID, {
+      vpc,
+      securityGroup,
+    });
+
+    // THEN
+    expectCDK(stack).to(haveResourceLike('AWS::AutoScaling::LaunchConfiguration', {
+      SecurityGroups: arrayWith(
+        securityGroupId,
+      ),
+    }));
+  });
+
   describe('.selfTermination = false', () => {
     beforeAll(() => {
       // GIVEN

packages/aws-rfdk/lib/deadline/lib/render-queue.ts

@@ -378,7 +378,7 @@ export class RenderQueue extends RenderQueueBase implements IGrantable {
   /**
    * Depend on this to ensure that ECS Service is stable.
    */
-  private ecsServiceStabilized: WaitForStableService;
+  private readonly ecsServiceStabilized: WaitForStableService;
 
   constructor(scope: Construct, id: string, private readonly props: RenderQueueProps) {
     super(scope, id);
@@ -973,6 +973,7 @@ export class RenderQueue extends RenderQueueBase implements IGrantable {
     const deploymentInstanceNode = this.node.tryFindChild(CONFIGURE_REPOSITORY_CONSTRUCT_ID);
     if (deploymentInstanceNode === undefined) {
       return new DeploymentInstance(this, CONFIGURE_REPOSITORY_CONSTRUCT_ID, {
+        securityGroup: this.backendConnections.securityGroups[0],
         vpc: this.props.vpc,
         vpcSubnets: this.props.vpcSubnets ?? RenderQueue.DEFAULT_VPC_SUBNETS_OTHER,
       });

packages/aws-rfdk/lib/deadline/lib/secrets-management.ts

@@ -134,7 +134,6 @@ export class SecretsManagementIdentityRegistration extends Construct {
     });
 
     // Install python dependencies
-    this.preparePythonEnvironment(props);
     const localScriptFile = this.preparePythonScript(props);
     this.runPythonScript(props, localScriptFile);
 
@@ -286,12 +285,4 @@ export class SecretsManagementIdentityRegistration extends Construct {
       ].join(' '),
     );
   }
-
-  private preparePythonEnvironment(props: SecretsManagementIdentityRegistrationProps) {
-    // The script must run as ec2-user because Repository.configureClientInstance(...) used above will store the
-    // credentials in a per-user credential store that is only available to "ec2-user".
-    props.deploymentInstance.userData.addCommands(
-      'sudo -u ec2-user python3 -m pip install --user boto3',
-    );
-  }
 }

packages/aws-rfdk/lib/deadline/scripts/python/configure_identity_registration_settings.py

@@ -7,7 +7,6 @@
 """
 
 import argparse
-import base64
 import io
 import ipaddress
 import json
@@ -19,8 +18,6 @@
 
 from typing import Dict, Iterable, List, NamedTuple, Match
 
-import boto3
-
 # Regex's for validating and splitting arguments
 SECRET_ARN_RE = re.compile(r'''
     ^
@@ -60,7 +57,7 @@
 RE_CAMEL_HUMP_SUB = re.compile(r'[A-Z]?[a-z]+|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)|\d+|[A-Z]{2,}|[A-Z]$')
 
 # Constants for the naming convention of RFDK-managed identity registration settings
-RFDK_IDENTITY_REGISTRATION_SETTING_NAME_PREFIX = f'RfdkSubnet'
+RFDK_IDENTITY_REGISTRATION_SETTING_NAME_PREFIX = 'RfdkSubnet'
 RFDK_IDENTITY_REGISTRATION_SETTING_NAME_SEP = '|'
 RFDK_IDENTITY_REGISTRATION_SETTING_NAME_RE = re.compile(rf"""
     ^
@@ -223,29 +220,82 @@ def validate_config(config):
 ####################################
 
 
-def fetch_secret(secret, binary=False):
+def aws_cli(args: List[str]):
+    """
+    Run an AWS CLI command and return JSON parsed output
+    """
+
+    try:
+        text_output = subprocess.check_output(['aws'] + args, text=True)
+    except subprocess.CalledProcessError as e:
+        raise Exception(f"failed to call AWS CLI ({e.returncode}): \n{e.stdout}\n\n{e.stderr}") from e
+
+    try:
+        json_obj = json.loads(text_output)
+    except json.JSONDecodeError as e:
+        raise Exception(f"AWS CLI did not output JSON as expected ({e.msg}). Output was:\n{text_output}") from e
+
+    return json_obj
+
+
+def fetch_secret(secret):
     """
     Fetch a secret from AWS
 
     :return: returns the contents of the secret
     """
     if isinstance(secret, AwsSecret):
-        secrets_client = boto3.client('secretsmanager', region_name=secret.region)
-        secret_value = secrets_client.get_secret_value(SecretId=secret.arn)
-        if binary:
-            return base64.b64decode(secret_value.get('SecretBinary'))
-        else:
-            return secret_value.get('SecretString')
+        result = aws_cli([
+            'secretsmanager',
+            '--region', secret.region,
+            'get-secret-value',
+            '--secret-id', secret.arn
+        ])
+
+        """
+        Result will be of the form:
+        {
+            ...
+            "SecretString": "{\n  \"username\":\"david\",\n  \"password\":\"BnQw&XDWgaEeT9XGTT29\"\n}\n",
+            ...
+        }
+
+        See https://docs.aws.amazon.com/cli/latest/reference/secretsmanager/get-secret-value.html
+        """
+        return result['SecretString']
     else:
         raise TypeError('Unknown Secret type.')
 
 
 def get_subnet_cidrs(region: str, subnet_ids: Iterable[str]) -> Dict[str, str]:
-    ec2 = boto3.resource('ec2', region_name=region)
+    filters = f'Name=subnet-id,Values={",".join(subnet_ids)}'
 
+    result = aws_cli([
+        '--region', region,
+        'ec2',
+        'describe-subnets',
+        '--filters', filters
+    ])
+
+    """
+    Result will be of the form:
+    {
+        "Subnets": [
+            {
+                ...,
+                "CidrBlock": "172.31.80.0/20",
+                "SubnetId": "subnet-0bb1c79de3EXAMPLE",
+                ...
+            },
+            ...
+        ]
+    }
+
+    See https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-subnets.html
+    """
     return {
-        subnet.subnet_id: subnet.cidr_block
-        for subnet in ec2.subnets.filter(SubnetIds=list(subnet_ids))
+        subnet["SubnetId"]: subnet["CidrBlock"]
+        for subnet in result["Subnets"]
     }