From fd842b710aed904f309eb14c4f354de0d51dcb9c Mon Sep 17 00:00:00 2001 From: xuetinp <113963935+xuetinp@users.noreply.github.com> Date: Wed, 28 Sep 2022 17:37:26 +0000 Subject: [PATCH] fix(integ): fix rate limit errors when deploying cloudwatch log groups in integration tests (#827) --- .../bin/deadline_01_repository.ts | 3 ++ .../bin/deadline_02_renderQueue.ts | 3 ++ .../bin/deadline_03_workerFleetHttp.ts | 3 ++ .../bin/deadline_04_workerFleetHttps.ts | 3 ++ .../bin/deadline_05_secretsManagement.ts | 3 ++ integ/lib/log-retention-retry-aspect.ts | 32 +++++++++++++++++++ 6 files changed, 47 insertions(+) create mode 100644 integ/lib/log-retention-retry-aspect.ts diff --git a/integ/components/deadline/deadline_01_repository/bin/deadline_01_repository.ts b/integ/components/deadline/deadline_01_repository/bin/deadline_01_repository.ts index 2da3cc926..3356f9364 100644 --- a/integ/components/deadline/deadline_01_repository/bin/deadline_01_repository.ts +++ b/integ/components/deadline/deadline_01_repository/bin/deadline_01_repository.ts @@ -8,6 +8,7 @@ import { Stage, ThinkboxDockerRecipes, } from 'aws-rfdk/deadline'; +import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect'; import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect'; import { DatabaseType, StorageStruct } from '../../../../lib/storage-struct'; @@ -50,3 +51,5 @@ new RepositoryTestingTier(app, 'RFDKInteg-DL-TestingTier' + integStackTag, { env // Adds IAM Policy to Instance and ASG Roles Aspects.of(app).add(new SSMInstancePolicyAspect()); +// Adds log retention retry to all functions +Aspects.of(app).add(new LogRetentionRetryAspect()); diff --git a/integ/components/deadline/deadline_02_renderQueue/bin/deadline_02_renderQueue.ts b/integ/components/deadline/deadline_02_renderQueue/bin/deadline_02_renderQueue.ts index 11cf1f63d..ec6d6735d 100644 --- a/integ/components/deadline/deadline_02_renderQueue/bin/deadline_02_renderQueue.ts +++ b/integ/components/deadline/deadline_02_renderQueue/bin/deadline_02_renderQueue.ts @@ -8,6 +8,7 @@ import { Stage, ThinkboxDockerRecipes, } from 'aws-rfdk/deadline'; +import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect'; import { RenderStruct } from '../../../../lib/render-struct'; import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect'; @@ -61,3 +62,5 @@ new RenderQueueTestingTier(app, 'RFDKInteg-RQ-TestingTier' + integStackTag, { en // Adds IAM Policy to Instance and ASG Roles Aspects.of(app).add(new SSMInstancePolicyAspect()); +// Adds log retention retry to all functions +Aspects.of(app).add(new LogRetentionRetryAspect()); diff --git a/integ/components/deadline/deadline_03_workerFleetHttp/bin/deadline_03_workerFleetHttp.ts b/integ/components/deadline/deadline_03_workerFleetHttp/bin/deadline_03_workerFleetHttp.ts index 2090e355b..50468fb37 100644 --- a/integ/components/deadline/deadline_03_workerFleetHttp/bin/deadline_03_workerFleetHttp.ts +++ b/integ/components/deadline/deadline_03_workerFleetHttp/bin/deadline_03_workerFleetHttp.ts @@ -8,6 +8,7 @@ import { Stage, ThinkboxDockerRecipes, } from 'aws-rfdk/deadline'; +import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect'; import { RenderStruct } from '../../../../lib/render-struct'; import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect'; @@ -65,3 +66,5 @@ new WorkerFleetTestingTier(app, 'RFDKInteg-WF-TestingTier' + integStackTag, {env // Adds IAM Policy to Instance and ASG Roles Aspects.of(app).add(new SSMInstancePolicyAspect()); +// Adds log retention retry to all functions +Aspects.of(app).add(new LogRetentionRetryAspect()); diff --git a/integ/components/deadline/deadline_04_workerFleetHttps/bin/deadline_04_workerFleetHttps.ts b/integ/components/deadline/deadline_04_workerFleetHttps/bin/deadline_04_workerFleetHttps.ts index 3642b96bd..b6626a3bb 100644 --- a/integ/components/deadline/deadline_04_workerFleetHttps/bin/deadline_04_workerFleetHttps.ts +++ b/integ/components/deadline/deadline_04_workerFleetHttps/bin/deadline_04_workerFleetHttps.ts @@ -8,6 +8,7 @@ import { Stage, ThinkboxDockerRecipes, } from 'aws-rfdk/deadline'; +import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect'; import { RenderStruct } from '../../../../lib/render-struct'; import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect'; @@ -65,3 +66,5 @@ new WorkerFleetTestingTier(app, 'RFDKInteg-WFS-TestingTier' + integStackTag, {en // Adds IAM Policy to Instance and ASG Roles Aspects.of(app).add(new SSMInstancePolicyAspect()); +// Adds log retention retry to all functions +Aspects.of(app).add(new LogRetentionRetryAspect()); diff --git a/integ/components/deadline/deadline_05_secretsManagement/bin/deadline_05_secretsManagement.ts b/integ/components/deadline/deadline_05_secretsManagement/bin/deadline_05_secretsManagement.ts index 6b44de71b..c95d255d6 100644 --- a/integ/components/deadline/deadline_05_secretsManagement/bin/deadline_05_secretsManagement.ts +++ b/integ/components/deadline/deadline_05_secretsManagement/bin/deadline_05_secretsManagement.ts @@ -12,6 +12,7 @@ import { ThinkboxDockerRecipes, UsageBasedLicense, } from 'aws-rfdk/deadline'; +import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect'; import { RenderStruct, RenderStructUsageBasedLicensingProps, @@ -79,6 +80,8 @@ async function main() { // Adds IAM Policy to Instance and ASG Roles Aspects.of(app).add(new SSMInstancePolicyAspect()); + // Adds log retention retry to all functions + Aspects.of(app).add(new LogRetentionRetryAspect()); } /** diff --git a/integ/lib/log-retention-retry-aspect.ts b/integ/lib/log-retention-retry-aspect.ts new file mode 100644 index 000000000..19e6f45ac --- /dev/null +++ b/integ/lib/log-retention-retry-aspect.ts @@ -0,0 +1,32 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CfnResource, IAspect, Stack } from 'aws-cdk-lib'; +import { CfnFunction } from 'aws-cdk-lib/aws-lambda'; +import { LogRetention } from 'aws-cdk-lib/aws-logs'; +import { IConstruct } from 'constructs'; + +export class LogRetentionRetryAspect implements IAspect { + public visit(node: IConstruct): void { + // Define log retention retries to reduce the risk of the rate exceed error + // as the default create log group TPS is only 5. Make sure to set the timeout of log retention function + // to be greater than total retry time. That's because if the function that is used for a custom resource + // doesn't exit properly, it'd end up in retries and may take cloud formation an hour to realize that + // the custom resource failed. + if (node instanceof LogRetention) { + const logRetentionResource = node.node.defaultChild as CfnResource; + logRetentionResource.addPropertyOverride('SdkRetry', { + maxRetries: 7, + base: 200, + }); + + // referenced from cdk code: https://github.com/aws/aws-cdk/blob/v2.33.0/packages/@aws-cdk/aws-logs/lib/log-retention.ts#L116 + const logRetentionFunctionConstructId = 'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a'; + const logRetentionFunction = Stack.of(node).node.findChild(logRetentionFunctionConstructId); + const cfnFunction = logRetentionFunction.node.defaultChild as CfnFunction; + cfnFunction.addPropertyOverride('Timeout', 30); + } + } +}