fix(integ): fix rate limit errors when deploying cloudwatch log group…

…s in integration tests (#827)
aws · Sep 28, 2022 · fd842b7 · fd842b7
1 parent c7e6cea
commit fd842b7
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 0 deletions.
diff --git a/integ/components/deadline/deadline_01_repository/bin/deadline_01_repository.ts b/integ/components/deadline/deadline_01_repository/bin/deadline_01_repository.ts
@@ -8,6 +8,7 @@ import {
   Stage,
   ThinkboxDockerRecipes,
 } from 'aws-rfdk/deadline';
+import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect';
 
 import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect';
 import { DatabaseType, StorageStruct } from '../../../../lib/storage-struct';
@@ -50,3 +51,5 @@ new RepositoryTestingTier(app, 'RFDKInteg-DL-TestingTier' + integStackTag, { env
 
 // Adds IAM Policy to Instance and ASG Roles
 Aspects.of(app).add(new SSMInstancePolicyAspect());
+// Adds log retention retry to all functions
+Aspects.of(app).add(new LogRetentionRetryAspect());
diff --git a/integ/components/deadline/deadline_02_renderQueue/bin/deadline_02_renderQueue.ts b/integ/components/deadline/deadline_02_renderQueue/bin/deadline_02_renderQueue.ts
@@ -8,6 +8,7 @@ import {
   Stage,
   ThinkboxDockerRecipes,
 } from 'aws-rfdk/deadline';
+import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect';
 
 import { RenderStruct } from '../../../../lib/render-struct';
 import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect';
@@ -61,3 +62,5 @@ new RenderQueueTestingTier(app, 'RFDKInteg-RQ-TestingTier' + integStackTag, { en
 
 // Adds IAM Policy to Instance and ASG Roles
 Aspects.of(app).add(new SSMInstancePolicyAspect());
+// Adds log retention retry to all functions
+Aspects.of(app).add(new LogRetentionRetryAspect());
diff --git a/integ/components/deadline/deadline_03_workerFleetHttp/bin/deadline_03_workerFleetHttp.ts b/integ/components/deadline/deadline_03_workerFleetHttp/bin/deadline_03_workerFleetHttp.ts
@@ -8,6 +8,7 @@ import {
   Stage,
   ThinkboxDockerRecipes,
 } from 'aws-rfdk/deadline';
+import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect';
 
 import { RenderStruct } from '../../../../lib/render-struct';
 import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect';
@@ -65,3 +66,5 @@ new WorkerFleetTestingTier(app, 'RFDKInteg-WF-TestingTier' + integStackTag, {env
 
 // Adds IAM Policy to Instance and ASG Roles
 Aspects.of(app).add(new SSMInstancePolicyAspect());
+// Adds log retention retry to all functions
+Aspects.of(app).add(new LogRetentionRetryAspect());
diff --git a/integ/components/deadline/deadline_04_workerFleetHttps/bin/deadline_04_workerFleetHttps.ts b/integ/components/deadline/deadline_04_workerFleetHttps/bin/deadline_04_workerFleetHttps.ts
@@ -8,6 +8,7 @@ import {
   Stage,
   ThinkboxDockerRecipes,
 } from 'aws-rfdk/deadline';
+import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect';
 
 import { RenderStruct } from '../../../../lib/render-struct';
 import { SSMInstancePolicyAspect } from '../../../../lib/ssm-policy-aspect';
@@ -65,3 +66,5 @@ new WorkerFleetTestingTier(app, 'RFDKInteg-WFS-TestingTier' + integStackTag, {en
 
 // Adds IAM Policy to Instance and ASG Roles
 Aspects.of(app).add(new SSMInstancePolicyAspect());
+// Adds log retention retry to all functions
+Aspects.of(app).add(new LogRetentionRetryAspect());
diff --git a/integ/components/deadline/deadline_05_secretsManagement/bin/deadline_05_secretsManagement.ts b/integ/components/deadline/deadline_05_secretsManagement/bin/deadline_05_secretsManagement.ts
@@ -12,6 +12,7 @@ import {
   ThinkboxDockerRecipes,
   UsageBasedLicense,
 } from 'aws-rfdk/deadline';
+import { LogRetentionRetryAspect } from '../../../../lib/log-retention-retry-aspect';
 import {
   RenderStruct,
   RenderStructUsageBasedLicensingProps,
@@ -79,6 +80,8 @@ async function main() {
 
   // Adds IAM Policy to Instance and ASG Roles
   Aspects.of(app).add(new SSMInstancePolicyAspect());
+  // Adds log retention retry to all functions
+  Aspects.of(app).add(new LogRetentionRetryAspect());
 }
 
 /**

diff --git a/integ/lib/log-retention-retry-aspect.ts b/integ/lib/log-retention-retry-aspect.ts
@@ -0,0 +1,32 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { CfnResource, IAspect, Stack } from 'aws-cdk-lib';
+import { CfnFunction } from 'aws-cdk-lib/aws-lambda';
+import { LogRetention } from 'aws-cdk-lib/aws-logs';
+import { IConstruct } from 'constructs';
+
+export class LogRetentionRetryAspect implements IAspect {
+  public visit(node: IConstruct): void {
+    // Define log retention retries to reduce the risk of the rate exceed error
+    // as the default create log group TPS is only 5. Make sure to set the timeout of log retention function
+    // to be greater than total retry time. That's because if the function that is used for a custom resource
+    // doesn't exit properly, it'd end up in retries and may take cloud formation an hour to realize that
+    // the custom resource failed.
+    if (node instanceof LogRetention) {
+      const logRetentionResource = node.node.defaultChild as CfnResource;
+      logRetentionResource.addPropertyOverride('SdkRetry', {
+        maxRetries: 7,
+        base: 200,
+      });
+
+      // referenced from cdk code: https://github.com/aws/aws-cdk/blob/v2.33.0/packages/@aws-cdk/aws-logs/lib/log-retention.ts#L116
+      const logRetentionFunctionConstructId = 'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a';
+      const logRetentionFunction = Stack.of(node).node.findChild(logRetentionFunctionConstructId);
+      const cfnFunction = logRetentionFunction.node.defaultChild as CfnFunction;
+      cfnFunction.addPropertyOverride('Timeout', 30);
+    }
+  }
+}