From af3b471ca7e7340c1afd17e760dc42790ea9a271 Mon Sep 17 00:00:00 2001 From: Jericho Tolentino Date: Mon, 9 Aug 2021 22:11:40 +0000 Subject: [PATCH] feat(deadline): add Deadline Secrets Management integration in the Render Queue --- .../lib/deadline/lib/render-queue-ref.ts | 18 +++ .../aws-rfdk/lib/deadline/lib/render-queue.ts | 52 ++++++- .../aws-rfdk/lib/deadline/lib/repository.ts | 7 +- .../scripts/bash/installDeadlineRepository.sh | 2 +- .../lib/deadline/test/render-queue.test.ts | 130 +++++++++++++++++- 5 files changed, 204 insertions(+), 5 deletions(-) diff --git a/packages/aws-rfdk/lib/deadline/lib/render-queue-ref.ts b/packages/aws-rfdk/lib/deadline/lib/render-queue-ref.ts index 7059ac176..76927fb42 100644 --- a/packages/aws-rfdk/lib/deadline/lib/render-queue-ref.ts +++ b/packages/aws-rfdk/lib/deadline/lib/render-queue-ref.ts @@ -377,6 +377,24 @@ export interface RenderQueueProps { * @default false */ readonly enableLocalFileCaching?: boolean; + + /** + * The credentials for configuring Deadline Secrets Management on the Render Queue. Providing this value will enable Secrets Management + * on the Render Queue. + * + * The secret must be in the following JSON format: + * + * ``` + * { + * username: string, + * password: string, + * } + * ``` + * + * @see https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/secrets-management/deadline-secrets-management.html + * @default Deadline Secrets Management is not enabled + */ + readonly secretsManagementCredentials?: ISecret; } /** diff --git a/packages/aws-rfdk/lib/deadline/lib/render-queue.ts b/packages/aws-rfdk/lib/deadline/lib/render-queue.ts index dea2c9830..9241d3296 100644 --- a/packages/aws-rfdk/lib/deadline/lib/render-queue.ts +++ b/packages/aws-rfdk/lib/deadline/lib/render-queue.ts @@ -29,6 +29,7 @@ import { Ec2TaskDefinition, LogDriver, PlacementConstraint, + Scope, UlimitName, } from '@aws-cdk/aws-ecs'; import { @@ -220,9 +221,9 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { private static readonly RE_VALID_HOSTNAME = /^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$/i; /** - * UID/GID for the RCS user. + * Information for the RCS user. */ - private static readonly RCS_USER = { uid: 1000, gid: 1000 }; + private static readonly RCS_USER = { uid: 1000, gid: 1000, username: 'ec2-user' }; /** * The principal to grant permissions to. @@ -412,12 +413,31 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { }); this.logGroup.grantWrite(this.asg); + if (props.secretsManagementCredentials !== undefined) { + const errors = []; + if (props.repository.secretsManagementSettings.enabled !== true) { + errors.push('Secrets Management is not enabled on the Repository'); + } + if (props.trafficEncryption?.internalProtocol !== ApplicationProtocol.HTTPS) { + errors.push('The internal protocol on the Render Queue is not HTTPS.'); + } + if (externalProtocol !== ApplicationProtocol.HTTPS) { + errors.push('External TLS on the Render Queue is not enabled.'); + } + if (errors.length > 0) { + throw new Error(`Secrets Management cannot be enabled on the Render Queue for the following reasons:\n${errors.join('\n')}`); + } + } const taskDefinition = this.createTaskDefinition({ image: props.images.remoteConnectionServer, portNumber: internalPortNumber, protocol: internalProtocol, repository: props.repository, runAsUser: RenderQueue.RCS_USER, + secretsManagementOptions: props.secretsManagementCredentials ? { + credentials: props.secretsManagementCredentials, + posixUsername: RenderQueue.RCS_USER.username, + } : undefined, }); this.taskDefinition = taskDefinition; @@ -530,6 +550,8 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { }); } + props.secretsManagementCredentials?.grantRead(this); + this.ecsServiceStabilized = new WaitForStableService(this, 'WaitForStableService', { service: this.pattern.service, }); @@ -648,6 +670,7 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { protocol: ApplicationProtocol, repository: IRepository, runAsUser?: { uid: number, gid?: number }, + secretsManagementOptions?: { credentials: ISecret, posixUsername: string }, }) { const { image, portNumber, protocol, repository } = props; @@ -685,6 +708,10 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { environment.RCS_TLS_REQUIRE_CLIENT_CERT = 'no'; } + if (props.secretsManagementOptions !== undefined) { + environment.RCS_SM_CREDENTIALS_URI = props.secretsManagementOptions.credentials.secretArn; + } + // We can ignore this in test coverage because we always use RenderQueue.RCS_USER /* istanbul ignore next */ const user = props.runAsUser ? `${props.runAsUser.uid}:${props.runAsUser.gid}` : undefined; @@ -701,6 +728,27 @@ export class RenderQueue extends RenderQueueBase implements IGrantable { containerDefinition.addMountPoints(connection.readWriteMountPoint); + if (props.secretsManagementOptions !== undefined) { + // Create volume to persist the RSA keypairs generated by Deadline between ECS tasks + // This makes it so subsequent ECS tasks use the same initial Secrets Management identity + const volumeName = 'deadline-user-keypairs'; + taskDefinition.addVolume({ + name: volumeName, + dockerVolumeConfiguration: { + scope: Scope.SHARED, + autoprovision: true, + driver: 'local', + }, + }); + + // Mount the volume into the container at the location where Deadline expects it + containerDefinition.addMountPoints({ + readOnly: false, + sourceVolume: volumeName, + containerPath: `/home/${props.secretsManagementOptions.posixUsername}/.config/.mono/keypairs`, + }); + } + // Increase ulimits containerDefinition.addUlimits( { diff --git a/packages/aws-rfdk/lib/deadline/lib/repository.ts b/packages/aws-rfdk/lib/deadline/lib/repository.ts index 438c6e34f..5c3c9cd34 100644 --- a/packages/aws-rfdk/lib/deadline/lib/repository.ts +++ b/packages/aws-rfdk/lib/deadline/lib/repository.ts @@ -190,6 +190,11 @@ export interface IRepository extends IConstruct { */ readonly version: IVersion; + /** + * Deadline Secrets Management settings. + */ + readonly secretsManagementSettings: SecretsManagementProps; + /** * Configures an ECS Container Instance and Task Definition for deploying a Deadline Client that directly connects to * this repository. @@ -557,7 +562,7 @@ export class Repository extends Construct implements IRepository { private readonly installerGroup: AutoScalingGroup; /** - * Deadline Secrets Management settings. + * @inheritdoc */ public readonly secretsManagementSettings: SecretsManagementProps; diff --git a/packages/aws-rfdk/lib/deadline/scripts/bash/installDeadlineRepository.sh b/packages/aws-rfdk/lib/deadline/scripts/bash/installDeadlineRepository.sh index bef513bca..579953742 100644 --- a/packages/aws-rfdk/lib/deadline/scripts/bash/installDeadlineRepository.sh +++ b/packages/aws-rfdk/lib/deadline/scripts/bash/installDeadlineRepository.sh @@ -132,7 +132,7 @@ if [ ! -z "${SECRET_MANAGEMENT_ARN+x}" ]; then exit 1 fi echo "Secret management is enabled. Credentials are stored in secret: $SECRET_MANAGEMENT_ARN" - SECRET_MANAGEMENT_ARG="--installSecretsManagement true --secretsAdminName \"$SECRET_MANAGEMENT_USER\" --secretsAdminPassword \"$SECRET_MANAGEMENT_PASSWORD\"" + SECRET_MANAGEMENT_ARG="--installSecretsManagement true --secretsAdminName $SECRET_MANAGEMENT_USER --secretsAdminPassword $SECRET_MANAGEMENT_PASSWORD" fi if [[ -n "${DEADLINE_REPOSITORY_OWNER+x}" ]]; then diff --git a/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts b/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts index 2e8566b4b..f1cf67265 100644 --- a/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts +++ b/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts @@ -35,6 +35,7 @@ import { } from '@aws-cdk/aws-ec2'; import { ContainerImage, + Ec2TaskDefinition, TaskDefinition, } from '@aws-cdk/aws-ecs'; import { @@ -50,7 +51,7 @@ import { import { Bucket, } from '@aws-cdk/aws-s3'; -import { Secret } from '@aws-cdk/aws-secretsmanager'; +import { CfnSecret, ISecret, Secret } from '@aws-cdk/aws-secretsmanager'; import { App, CfnElement, @@ -2731,4 +2732,131 @@ describe('RenderQueue', () => { ), })); }); + + describe('Secrets Management', () => { + let secretsManagementCredentials: ISecret; + let rqSecretsManagementProps: RenderQueueProps; + + beforeEach(() => { + secretsManagementCredentials = new Secret(stack, 'SecretsManagementCredentials'); + rqSecretsManagementProps = { + vpc, + images, + repository, + version: renderQueueVersion, + secretsManagementCredentials, + trafficEncryption: { + internalProtocol: ApplicationProtocol.HTTPS, + externalTLS: { enabled: true }, + }, + }; + }); + + test('throws if secrets management not enabled on repository', () => { + // GIVEN + const secret = new Secret(dependencyStack, 'DeadlineSecretsManagementCredentials'); + const smRepository = new Repository(dependencyStack, 'SecretsManagementRepository', { + vpc, + version, + secretsManagementSettings: { + enabled: false, + credentials: secret, + }, + }); + + // WHEN + expect(() => new RenderQueue(stack, 'SecretsManagementRenderQueue', { + ...rqSecretsManagementProps, + repository: smRepository, + })) + + // THEN + .toThrowError(/Secrets Management is not enabled on the Repository/); + }); + + test('throws if internal protocol is not HTTPS', () => { + // WHEN + expect(() => new RenderQueue(stack, 'SecretsManagementRenderQueue', { + ...rqSecretsManagementProps, + trafficEncryption: { + internalProtocol: ApplicationProtocol.HTTP, + }, + })) + + // THEN + .toThrowError(/The internal protocol on the Render Queue is not HTTPS./); + }); + + test('throws if external TLS is not enabled', () => { + // WHEN + expect(() => new RenderQueue(stack, 'SecretsManagementRenderQueue', { + ...rqSecretsManagementProps, + trafficEncryption: { + externalTLS: { enabled: false }, + }, + })) + + // THEN + .toThrowError(/External TLS on the Render Queue is not enabled./); + }); + + test('grants read permissions to secrets management credentials', () => { + // WHEN + const rq = new RenderQueue(stack, 'SecretsManagementRenderQueue', rqSecretsManagementProps); + + // THEN + expectCDK(stack).to(haveResourceLike('AWS::IAM::Policy', { + PolicyDocument: objectLike({ + Statement: arrayWith({ + Action: [ + 'secretsmanager:GetSecretValue', + 'secretsmanager:DescribeSecret', + ], + Effect: 'Allow', + Resource: stack.resolve((secretsManagementCredentials.node.defaultChild as CfnSecret).ref), + }), + }), + Roles: [stack.resolve((rq.node.tryFindChild('RCSTask') as Ec2TaskDefinition).taskRole.roleName)], + })); + }); + + test('defines secrets management credentials environment variable', () => { + // WHEN + new RenderQueue(stack, 'SecretsManagementRenderQueue', rqSecretsManagementProps); + + // THEN + expectCDK(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { + ContainerDefinitions: arrayWith(objectLike({ + Environment: arrayWith({ + Name: 'RCS_SM_CREDENTIALS_URI', + Value: stack.resolve((secretsManagementCredentials.node.defaultChild as CfnSecret).ref), + }), + })), + })); + }); + + test('creates and mounts docker volume for deadline key pairs', () => { + // WHEN + new RenderQueue(stack, 'SecretsManagementRenderQueue', rqSecretsManagementProps); + + // THEN + expectCDK(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { + ContainerDefinitions: arrayWith(objectLike({ + MountPoints: arrayWith({ + ContainerPath: '/home/ec2-user/.config/.mono/keypairs', + ReadOnly: false, + SourceVolume: 'deadline-user-keypairs', + }), + })), + Volumes: arrayWith({ + DockerVolumeConfiguration: { + Autoprovision: true, + Driver: 'local', + Scope: 'shared', + }, + Name: 'deadline-user-keypairs', + }), + })); + }); + }); });