From 48baa185b274030cab29a235469536585822313f Mon Sep 17 00:00:00 2001 From: Jericho Tolentino <68654047+jericht@users.noreply.github.com> Date: Thu, 19 Aug 2021 16:57:22 -0500 Subject: [PATCH] feat(deadline): add Deadline Secrets Management integration in the Render Queue (#528) --- .../aws-rfdk/lib/deadline/lib/render-queue.ts | 63 ++++++- .../aws-rfdk/lib/deadline/lib/repository.ts | 7 +- .../scripts/bash/installDeadlineRepository.sh | 25 ++- .../test/configure-spot-event-plugin.test.ts | 4 + .../lib/deadline/test/render-queue.test.ts | 175 +++++++++++++++++- .../test/spot-event-plugin-fleet.test.ts | 1 + .../test/usage-based-licensing.test.ts | 1 + .../test/worker-configuration.test.ts | 1 + .../lib/deadline/test/worker-fleet.test.ts | 1 + 9 files changed, 260 insertions(+), 18 deletions(-) diff --git a/packages/aws-rfdk/lib/deadline/lib/render-queue.ts b/packages/aws-rfdk/lib/deadline/lib/render-queue.ts index dea2c9830..5922d457e 100644 --- a/packages/aws-rfdk/lib/deadline/lib/render-queue.ts +++ b/packages/aws-rfdk/lib/deadline/lib/render-queue.ts @@ -29,6 +29,7 @@ import { Ec2TaskDefinition, LogDriver, PlacementConstraint, + Scope, UlimitName, } from '@aws-cdk/aws-ecs'; import { @@ -214,15 +215,23 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { */ private static readonly MINIMUM_LOAD_BALANCING_VERSION = new Version([10, 1, 10, 0]); + // TODO: Update this with the version of Deadline that includes the changes for RFDK Secrets Management. + // This is a temporary minimum version until this feature branch is merged + /** + * The minimum Deadline version required to enable Deadline Secrets Management on the Render Queue. + */ + private static readonly MINIMUM_SECRETS_MANAGEMENT_VERSION = new Version([10, 1, 15, 0]); + /** * Regular expression that validates a hostname (portion in front of the subdomain). */ private static readonly RE_VALID_HOSTNAME = /^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$/i; /** - * UID/GID for the RCS user. + * Information for the RCS user. */ - private static readonly RCS_USER = { uid: 1000, gid: 1000 }; + + private static readonly RCS_USER = { uid: 1000, gid: 1000, username: 'ec2-user' }; /** * The principal to grant permissions to. @@ -412,12 +421,34 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { }); this.logGroup.grantWrite(this.asg); + if (props.repository.secretsManagementSettings.enabled) { + const errors = []; + if (props.version.isLessThan(RenderQueue.MINIMUM_SECRETS_MANAGEMENT_VERSION)) { + errors.push(`The supplied Deadline version (${props.version.versionString}) is lower than the minimum required version: ${RenderQueue.MINIMUM_SECRETS_MANAGEMENT_VERSION.toString()}`); + } + if (props.repository.secretsManagementSettings.credentials === undefined) { + errors.push('The Repository does not have Secrets Management credentials'); + } + if (internalProtocol !== ApplicationProtocol.HTTPS) { + errors.push('The internal protocol on the Render Queue is not HTTPS.'); + } + if (externalProtocol !== ApplicationProtocol.HTTPS) { + errors.push('External TLS on the Render Queue is not enabled.'); + } + if (errors.length > 0) { + throw new Error(`Deadline Secrets Management is enabled on the supplied Repository but cannot be enabled on the Render Queue for the following reasons:\n${errors.join('\n')}`); + } + } const taskDefinition = this.createTaskDefinition({ image: props.images.remoteConnectionServer, portNumber: internalPortNumber, protocol: internalProtocol, repository: props.repository, runAsUser: RenderQueue.RCS_USER, + secretsManagementOptions: props.repository.secretsManagementSettings.enabled ? { + credentials: props.repository.secretsManagementSettings.credentials!, + posixUsername: RenderQueue.RCS_USER.username, + } : undefined, }); this.taskDefinition = taskDefinition; @@ -530,6 +561,8 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { }); } + props.repository.secretsManagementSettings.credentials?.grantRead(this); + this.ecsServiceStabilized = new WaitForStableService(this, 'WaitForStableService', { service: this.pattern.service, }); @@ -648,6 +681,7 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { protocol: ApplicationProtocol, repository: IRepository, runAsUser?: { uid: number, gid?: number }, + secretsManagementOptions?: { credentials: ISecret, posixUsername: string }, }) { const { image, portNumber, protocol, repository } = props; @@ -685,6 +719,10 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { environment.RCS_TLS_REQUIRE_CLIENT_CERT = 'no'; } + if (props.secretsManagementOptions !== undefined) { + environment.RCS_SM_CREDENTIALS_URI = props.secretsManagementOptions.credentials.secretArn; + } + // We can ignore this in test coverage because we always use RenderQueue.RCS_USER /* istanbul ignore next */ const user = props.runAsUser ? `${props.runAsUser.uid}:${props.runAsUser.gid}` : undefined; @@ -701,6 +739,27 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { containerDefinition.addMountPoints(connection.readWriteMountPoint); + if (props.secretsManagementOptions !== undefined) { + // Create volume to persist the RSA keypairs generated by Deadline between ECS tasks + // This makes it so subsequent ECS tasks use the same initial Secrets Management identity + const volumeName = 'deadline-user-keypairs'; + taskDefinition.addVolume({ + name: volumeName, + dockerVolumeConfiguration: { + scope: Scope.SHARED, + autoprovision: true, + driver: 'local', + }, + }); + + // Mount the volume into the container at the location where Deadline expects it + containerDefinition.addMountPoints({ + readOnly: false, + sourceVolume: volumeName, + containerPath: `/home/${props.secretsManagementOptions.posixUsername}/.config/.mono/keypairs`, + }); + } + // Increase ulimits containerDefinition.addUlimits( { diff --git a/packages/aws-rfdk/lib/deadline/lib/repository.ts b/packages/aws-rfdk/lib/deadline/lib/repository.ts index 438c6e34f..5c3c9cd34 100644 --- a/packages/aws-rfdk/lib/deadline/lib/repository.ts +++ b/packages/aws-rfdk/lib/deadline/lib/repository.ts @@ -190,6 +190,11 @@ export interface IRepository extends IConstruct { */ readonly version: IVersion; + /** + * Deadline Secrets Management settings. + */ + readonly secretsManagementSettings: SecretsManagementProps; + /** * Configures an ECS Container Instance and Task Definition for deploying a Deadline Client that directly connects to * this repository. @@ -557,7 +562,7 @@ export class Repository extends Construct implements IRepository { private readonly installerGroup: AutoScalingGroup; /** - * Deadline Secrets Management settings. + * @inheritdoc */ public readonly secretsManagementSettings: SecretsManagementProps; diff --git a/packages/aws-rfdk/lib/deadline/scripts/bash/installDeadlineRepository.sh b/packages/aws-rfdk/lib/deadline/scripts/bash/installDeadlineRepository.sh index bef513bca..f4deb1349 100644 --- a/packages/aws-rfdk/lib/deadline/scripts/bash/installDeadlineRepository.sh +++ b/packages/aws-rfdk/lib/deadline/scripts/bash/installDeadlineRepository.sh @@ -18,7 +18,7 @@ Required arguments: Optional arguments -s Deadline Repository settings file to import. -o The UID[:GID] that this script will chown the Repository files for. If GID is not specified, it defults to be the same as UID. - -c Secret management admin credentials ARN. If this parameter is specified, secrets management will be enabled. + -c Secrets management admin credentials ARN. If this parameter is specified, secrets management will be enabled. -r Region where stacks are deployed. Required to get secret management credentials." while getopts "i:p:v:s:o:c:r:" opt; do @@ -102,20 +102,21 @@ chmod +x $REPO_INSTALLER set +x -INSTALLER_DB_ARGS_STRING='' -for key in "${!INSTALLER_DB_ARGS[@]}"; do INSTALLER_DB_ARGS_STRING=$INSTALLER_DB_ARGS_STRING"${key} ${INSTALLER_DB_ARGS[$key]} "; done +REPO_ARGS=() + +for key in "${!INSTALLER_DB_ARGS[@]}"; do + REPO_ARGS+=("${key}" "${INSTALLER_DB_ARGS[$key]}") +done -REPOSITORY_SETTINGS_ARG_STRING='' if [ ! -z "${DEADLINE_REPOSITORY_SETTINGS_FILE+x}" ]; then if [ ! -f "$DEADLINE_REPOSITORY_SETTINGS_FILE" ]; then echo "ERROR: Repository settings file was specified but is not a file: $DEADLINE_REPOSITORY_SETTINGS_FILE." exit 1 else - REPOSITORY_SETTINGS_ARG_STRING="--importrepositorysettings true --repositorysettingsimportoperation append --repositorysettingsimportfile \"$DEADLINE_REPOSITORY_SETTINGS_FILE\"" + REPO_ARGS+=("--importrepositorysettings" "true" "--repositorysettingsimportoperation" "append" "--repositorysettingsimportfile" "$DEADLINE_REPOSITORY_SETTINGS_FILE") fi fi -SECRET_MANAGEMENT_ARG='' if [ ! -z "${SECRET_MANAGEMENT_ARN+x}" ]; then sudo yum install -y jq SM_SECRET_VALUE=$(aws secretsmanager get-secret-value --secret-id=$SECRET_MANAGEMENT_ARN --region=$AWS_REGION) @@ -131,8 +132,10 @@ if [ ! -z "${SECRET_MANAGEMENT_ARN+x}" ]; then echo "ERROR: Admin password is too weak. It must be at least 8 characters long and contain at least one lowercase letter, one uppercase letter, one symbol and one digit." exit 1 fi - echo "Secret management is enabled. Credentials are stored in secret: $SECRET_MANAGEMENT_ARN" - SECRET_MANAGEMENT_ARG="--installSecretsManagement true --secretsAdminName \"$SECRET_MANAGEMENT_USER\" --secretsAdminPassword \"$SECRET_MANAGEMENT_PASSWORD\"" + echo "Secrets management is enabled. Credentials are stored in secret: $SECRET_MANAGEMENT_ARN" + REPO_ARGS+=("--installSecretsManagement" "true" "--secretsAdminName" "$SECRET_MANAGEMENT_USER" "--secretsAdminPassword" "$SECRET_MANAGEMENT_PASSWORD") +else + echo "Secrets management is not enabled." fi if [[ -n "${DEADLINE_REPOSITORY_OWNER+x}" ]]; then @@ -164,7 +167,11 @@ if [[ -n "${DEADLINE_REPOSITORY_OWNER+x}" ]]; then fi fi -$REPO_INSTALLER --mode unattended --setpermissions false --prefix "$PREFIX" --installmongodb false --backuprepo false ${INSTALLER_DB_ARGS_STRING} $REPOSITORY_SETTINGS_ARG_STRING $SECRET_MANAGEMENT_ARG +# The syntax ${array[@]+"${array[@]}"} is a way to get around the expansion of an empty array raising an unbound variable error since this script +# sets the "u" shell option above. This is a use of the ${parameter+word} shell expansion. If the value of "parameter" is unset, nothing will be +# substituted in its place. If "parameter" is set, then the value of "word" is used, which is the expansion of the populated array. +# Since bash treats the expansion of an empty array as an unset variable, we can use this pattern expand the array only if it is populated. +$REPO_INSTALLER --mode unattended --setpermissions false --prefix "$PREFIX" --installmongodb false --backuprepo false ${REPO_ARGS[@]+"${REPO_ARGS[@]}"} if [[ -n "${REPOSITORY_OWNER_UID+x}" ]]; then echo "Changing ownership of Deadline Repository files to UID=$REPOSITORY_OWNER_UID GID=$REPOSITORY_OWNER_GID" diff --git a/packages/aws-rfdk/lib/deadline/test/configure-spot-event-plugin.test.ts b/packages/aws-rfdk/lib/deadline/test/configure-spot-event-plugin.test.ts index 7566635ee..bd7a6b536 100644 --- a/packages/aws-rfdk/lib/deadline/test/configure-spot-event-plugin.test.ts +++ b/packages/aws-rfdk/lib/deadline/test/configure-spot-event-plugin.test.ts @@ -85,6 +85,7 @@ describe('ConfigureSpotEventPlugin', () => { repository: new Repository(stack, 'Repository', { vpc, version, + secretsManagementSettings: { enabled: false }, }), trafficEncryption: { externalTLS: { enabled: false } }, version, @@ -688,6 +689,7 @@ describe('ConfigureSpotEventPlugin', () => { repository: new Repository(stack, 'Repository2', { vpc, version, + secretsManagementSettings: { enabled: false }, }), trafficEncryption: { externalTLS: { enabled: false } }, version, @@ -910,6 +912,7 @@ describe('ConfigureSpotEventPlugin', () => { repository: new Repository(newStack, 'Repository', { vpc, version, + secretsManagementSettings: { enabled: false }, }), trafficEncryption: { externalTLS: { enabled: false } }, version, @@ -945,6 +948,7 @@ describe('ConfigureSpotEventPlugin', () => { repository: new Repository(newStack, 'Repository', { vpc, version, + secretsManagementSettings: { enabled: false }, }), trafficEncryption: { externalTLS: { enabled: false } }, version, diff --git a/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts b/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts index 2e8566b4b..893637e2c 100644 --- a/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts +++ b/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts @@ -35,6 +35,7 @@ import { } from '@aws-cdk/aws-ec2'; import { ContainerImage, + Ec2TaskDefinition, TaskDefinition, } from '@aws-cdk/aws-ecs'; import { @@ -50,7 +51,10 @@ import { import { Bucket, } from '@aws-cdk/aws-s3'; -import { Secret } from '@aws-cdk/aws-secretsmanager'; +import { + CfnSecret, + Secret, +} from '@aws-cdk/aws-secretsmanager'; import { App, CfnElement, @@ -638,9 +642,14 @@ describe('RenderQueue', () => { beforeEach(() => { // GIVEN isolatedStack = new Stack(app, 'IsolatedStack'); + const nonSmRepository = new Repository(dependencyStack, 'NonSMRepository', { + vpc, + version, + secretsManagementSettings: { enabled: false }, + }); const props: RenderQueueProps = { images, - repository, + repository: nonSmRepository, version: new VersionQuery(isolatedStack, 'Version'), vpc, trafficEncryption: { @@ -802,10 +811,15 @@ describe('RenderQueue', () => { }, signingCertificate: caCert, }); + const nonSmRepository = new Repository(dependencyStack, 'NonSMRepository', { + vpc, + version, + secretsManagementSettings: { enabled: false }, + }); const props: RenderQueueProps = { images, - repository, + repository: nonSmRepository, version: new VersionQuery(isolatedStack, 'Version'), vpc, trafficEncryption: { @@ -1069,9 +1083,14 @@ describe('RenderQueue', () => { vpc, zoneName: ZONE_NAME, }); + const nonSmRepository = new Repository(dependencyStack, 'NonSMRepository', { + vpc, + version, + secretsManagementSettings: { enabled: false }, + }); const props: RenderQueueProps = { images, - repository, + repository: nonSmRepository, version: new VersionQuery(isolatedStack, 'Version'), vpc, hostname: { @@ -1998,9 +2017,14 @@ describe('RenderQueue', () => { beforeEach(() => { // GIVEN isolatedStack = new Stack(app, 'IsolatedStack'); + const nonSmRepository = new Repository(dependencyStack, 'NonSMRepository', { + vpc, + version, + secretsManagementSettings: { enabled: false }, + }); const props: RenderQueueProps = { images, - repository, + repository: nonSmRepository, trafficEncryption: { externalTLS: { enabled: false } }, version: new VersionQuery(isolatedStack, 'Version'), vpc, @@ -2099,9 +2123,14 @@ describe('RenderQueue', () => { }); const hostname = 'testrq'; const isolatedStack = new Stack(app, 'IsolatedStack'); + const nonSmRepository = new Repository(dependencyStack, 'NonSMRepository', { + vpc, + version, + secretsManagementSettings: { enabled: false }, + }); const props: RenderQueueProps = { images, - repository, + repository: nonSmRepository, version: new VersionQuery(isolatedStack, 'Version'), vpc, hostname: { @@ -2731,4 +2760,138 @@ describe('RenderQueue', () => { ), })); }); + + describe('Secrets Management', () => { + let rqSecretsManagementProps: RenderQueueProps; + + beforeEach(() => { + rqSecretsManagementProps = { + vpc, + images, + repository, + version: renderQueueVersion, + trafficEncryption: { + internalProtocol: ApplicationProtocol.HTTPS, + externalTLS: { enabled: true }, + }, + }; + }); + + test('throws if internal protocol is not HTTPS', () => { + // WHEN + expect(() => new RenderQueue(stack, 'SecretsManagementRenderQueue', { + ...rqSecretsManagementProps, + trafficEncryption: { + internalProtocol: ApplicationProtocol.HTTP, + }, + })) + + // THEN + .toThrowError(/The internal protocol on the Render Queue is not HTTPS./); + }); + + test('throws if external TLS is not enabled', () => { + // WHEN + expect(() => new RenderQueue(stack, 'SecretsManagementRenderQueue', { + ...rqSecretsManagementProps, + trafficEncryption: { + externalTLS: { enabled: false }, + }, + })) + + // THEN + .toThrowError(/External TLS on the Render Queue is not enabled./); + }); + + test('throws if repository does not have SM credentials', () => { + // WHEN + expect(() => new RenderQueue(stack, 'SecretsManagementRenderQueue', { + ...rqSecretsManagementProps, + repository: { + ...repository, + secretsManagementSettings: { + ...repository.secretsManagementSettings, + credentials: undefined, + }, + } as Repository, + })) + + // THEN + .toThrowError(/The Repository does not have Secrets Management credentials/); + }); + + test('throws if deadline version is too low', () => { + // GIVEN + const oldVersion = new VersionQuery(new Stack(app, 'OldDeadlineVersionStack'), 'OldDeadlineVersion', { version: '10.0.0.0' }); + + // WHEN + expect(() => new RenderQueue(stack, 'SecretsManagementRenderQueue', { + ...rqSecretsManagementProps, + version: oldVersion, + })) + + // THEN + /* eslint-disable-next-line dot-notation */ + .toThrowError(`The supplied Deadline version (${oldVersion.versionString}) is lower than the minimum required version: ${RenderQueue['MINIMUM_SECRETS_MANAGEMENT_VERSION'].toString()}`); + }); + + test('grants read permissions to secrets management credentials', () => { + // WHEN + const rq = new RenderQueue(stack, 'SecretsManagementRenderQueue', rqSecretsManagementProps); + + // THEN + expectCDK(stack).to(haveResourceLike('AWS::IAM::Policy', { + PolicyDocument: objectLike({ + Statement: arrayWith({ + Action: [ + 'secretsmanager:GetSecretValue', + 'secretsmanager:DescribeSecret', + ], + Effect: 'Allow', + Resource: stack.resolve((repository.secretsManagementSettings.credentials!.node.defaultChild as CfnSecret).ref), + }), + }), + Roles: [stack.resolve((rq.node.tryFindChild('RCSTask') as Ec2TaskDefinition).taskRole.roleName)], + })); + }); + + test('defines secrets management credentials environment variable', () => { + // WHEN + new RenderQueue(stack, 'SecretsManagementRenderQueue', rqSecretsManagementProps); + + // THEN + expectCDK(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { + ContainerDefinitions: arrayWith(objectLike({ + Environment: arrayWith({ + Name: 'RCS_SM_CREDENTIALS_URI', + Value: stack.resolve((repository.secretsManagementSettings.credentials!.node.defaultChild as CfnSecret).ref), + }), + })), + })); + }); + + test('creates and mounts docker volume for deadline key pairs', () => { + // WHEN + new RenderQueue(stack, 'SecretsManagementRenderQueue', rqSecretsManagementProps); + + // THEN + expectCDK(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { + ContainerDefinitions: arrayWith(objectLike({ + MountPoints: arrayWith({ + ContainerPath: '/home/ec2-user/.config/.mono/keypairs', + ReadOnly: false, + SourceVolume: 'deadline-user-keypairs', + }), + })), + Volumes: arrayWith({ + DockerVolumeConfiguration: { + Autoprovision: true, + Driver: 'local', + Scope: 'shared', + }, + Name: 'deadline-user-keypairs', + }), + })); + }); + }); }); diff --git a/packages/aws-rfdk/lib/deadline/test/spot-event-plugin-fleet.test.ts b/packages/aws-rfdk/lib/deadline/test/spot-event-plugin-fleet.test.ts index d41860546..78c52989a 100644 --- a/packages/aws-rfdk/lib/deadline/test/spot-event-plugin-fleet.test.ts +++ b/packages/aws-rfdk/lib/deadline/test/spot-event-plugin-fleet.test.ts @@ -99,6 +99,7 @@ describe('SpotEventPluginFleet', () => { repository: new Repository(stack, 'Repository', { vpc, version, + secretsManagementSettings: { enabled: false }, }), trafficEncryption: { externalTLS: { enabled: false } }, version, diff --git a/packages/aws-rfdk/lib/deadline/test/usage-based-licensing.test.ts b/packages/aws-rfdk/lib/deadline/test/usage-based-licensing.test.ts index fec8b95ba..46319afd4 100644 --- a/packages/aws-rfdk/lib/deadline/test/usage-based-licensing.test.ts +++ b/packages/aws-rfdk/lib/deadline/test/usage-based-licensing.test.ts @@ -86,6 +86,7 @@ describe('UsageBasedLicensing', () => { repository: new Repository(dependencyStack, 'RepositoryNonDefault', { vpc, version: versionedInstallers, + secretsManagementSettings: { enabled: false }, }), trafficEncryption: { externalTLS: { enabled: false } }, version: versionedInstallers, diff --git a/packages/aws-rfdk/lib/deadline/test/worker-configuration.test.ts b/packages/aws-rfdk/lib/deadline/test/worker-configuration.test.ts index 96140c6d0..20a0d75f9 100644 --- a/packages/aws-rfdk/lib/deadline/test/worker-configuration.test.ts +++ b/packages/aws-rfdk/lib/deadline/test/worker-configuration.test.ts @@ -328,6 +328,7 @@ describe('Test WorkerInstanceConfiguration connect to RenderQueue', () => { repository: new Repository(stack, 'Repository', { vpc, version, + secretsManagementSettings: { enabled: false }, }), trafficEncryption: { externalTLS: { enabled: false } }, }); diff --git a/packages/aws-rfdk/lib/deadline/test/worker-fleet.test.ts b/packages/aws-rfdk/lib/deadline/test/worker-fleet.test.ts index d7914d27a..fe74e17d5 100644 --- a/packages/aws-rfdk/lib/deadline/test/worker-fleet.test.ts +++ b/packages/aws-rfdk/lib/deadline/test/worker-fleet.test.ts @@ -89,6 +89,7 @@ beforeEach(() => { repository: new Repository(stack, 'Repository', { vpc, version, + secretsManagementSettings: { enabled: false }, }), trafficEncryption: { externalTLS: { enabled: false } }, version,