Skip to content

Commit

Permalink
fix(core): fix rate limit errors when deploying cloudwatch log groups (…
Browse files Browse the repository at this point in the history 
…#822)
  • Loading branch information
@xuetinp
xuetinp committed Sep 21, 2022
1 parent c3f25fa commit 38df77f
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 1 deletion.
16 changes: 16 additions & 0 deletions packages/aws-rfdk/lib/core/lib/exporting-log-group.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@ import {
Code,
Runtime,
SingletonFunction,
CfnFunction,
} from 'aws-cdk-lib/aws-lambda';
import {
ILogGroup,
LogGroup,
LogRetention,
RetentionDays,
} from 'aws-cdk-lib/aws-logs';
import { Stack } from 'aws-cdk-lib/core';
import { Construct } from 'constructs';

/**
Expand Down Expand Up @@ -111,10 +113,24 @@ export class ExportingLogGroup extends Construct {
threshold: 1,
});

// Define log retention retry options to reduce the risk of the rate exceed error
// as the default create log group TPS is only 5. Make sure to set the timeout of log retention function
// to be greater than total retry time. That's because if the function that is used for a custom resource
// doesn't exit properly, it'd end up in retries and may take cloud formation an hour to realize that
// the custom resource failed.
const logRetention = new LogRetention(this, 'LogRetention', {
logGroupName: props.logGroupName,
retention: retentionInDays,
logRetentionRetryOptions: {
base: Duration.millis(200),
maxRetries: 7,
},
});
// referenced from cdk code: https://github.com/aws/aws-cdk/blob/v2.33.0/packages/@aws-cdk/aws-logs/lib/log-retention.ts#L116
const logRetentionFunctionConstructId = 'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a';
const logRetentionFunction = Stack.of(this).node.findChild(logRetentionFunctionConstructId);
const cfnFunction = logRetentionFunction.node.defaultChild as CfnFunction;
cfnFunction.addPropertyOverride('Timeout', 30);

this.logGroup = LogGroup.fromLogGroupArn(scope, `${props.logGroupName}LogGroup`, logRetention.logGroupArn);
this.logGroup.grant(exportLogsFunction, 'logs:CreateExportTask');
Expand Down
20 changes: 19 additions & 1 deletion packages/aws-rfdk/lib/core/lib/log-group-factory.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
* SPDX-License-Identifier: Apache-2.0
*/

import { Duration, Stack } from 'aws-cdk-lib';
import { CfnFunction } from 'aws-cdk-lib/aws-lambda';
import {
ILogGroup,
LogGroup,
Expand Down Expand Up @@ -52,7 +54,12 @@ export class LogGroupFactory {
const fullLogGroupName = props?.logGroupPrefix ? `${props.logGroupPrefix}${logGroupName}` : logGroupName;
const retention = props?.retention ? props.retention : LogGroupFactory.DEFAULT_LOG_RETENTION;

return props?.bucketName
// Define log retention retry options to reduce the risk of the rate exceed error
// as the default create log group TPS is only 5. Make sure to set the timeout of log retention function
// to be greater than total retry time. That's because if the function that is used for a custom resource
// doesn't exit properly, it'd end up in retries and may take cloud formation an hour to realize that
// the custom resource failed.
const logGroup = props?.bucketName
? new ExportingLogGroup(scope, logWrapperId, {
bucketName: props.bucketName,
logGroupName: fullLogGroupName,
Expand All @@ -64,7 +71,18 @@ export class LogGroupFactory {
new LogRetention(scope, logWrapperId, {
logGroupName: fullLogGroupName,
retention,
logRetentionRetryOptions: {
base: Duration.millis(200),
maxRetries: 7,
},
}).logGroupArn);
// referenced from cdk code: https://github.com/aws/aws-cdk/blob/v2.33.0/packages/@aws-cdk/aws-logs/lib/log-retention.ts#L116
const logRetentionFunctionConstructId = 'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a';
const logRetentionFunction = Stack.of(scope).node.findChild(logRetentionFunctionConstructId);
const cfnFunction = logRetentionFunction.node.defaultChild as CfnFunction;
cfnFunction.addPropertyOverride('Timeout', 30);

return logGroup;
}

/**
Expand Down
8 changes: 8 additions & 0 deletions packages/aws-rfdk/lib/core/test/exporting-log-group.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ test('default exporting log group is created correctly', () => {
],
},
LogGroupName: 'logGroup',
SdkRetry: {
maxRetries: 7,
base: 200,
},
RetentionInDays: 3,
});
Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', {
Expand Down Expand Up @@ -147,6 +151,10 @@ test('custom set retention is created correctly', () => {
],
},
LogGroupName: 'logGroup',
SdkRetry: {
maxRetries: 7,
base: 200,
},
RetentionInDays: 7,
});
Template.fromStack(stack).resourceCountIs('AWS::Lambda::Function', 2);
Expand Down
98 changes: 98 additions & 0 deletions packages/aws-rfdk/lib/core/test/log-group-factory.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,24 @@ describe('log group', () => {
// THEN
Template.fromStack(stack).hasResourceProperties('Custom::LogRetention', {
LogGroupName: 'testLogGroup',
SdkRetry: {
maxRetries: 7,
base: 200,
},
RetentionInDays: 3,
});

expect(Object.keys(Template.fromStack(stack).findResources('AWS::Lambda::Function', {
Properties: {
Role: {
'Fn::GetAtt': [
'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8aServiceRole9741ECFB',
'Arn',
],
},
Timeout: 30,
},
})).length).toEqual(1);
Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Function', Match.not({
Role: {
'Fn::GetAtt': [
Expand All @@ -44,8 +60,25 @@ describe('log group', () => {
// THEN
Template.fromStack(stack).hasResourceProperties('Custom::LogRetention', {
LogGroupName: 'prefix-testLogGroup',
SdkRetry: {
maxRetries: 7,
base: 200,
},
RetentionInDays: 3,
});

expect(Object.keys(Template.fromStack(stack).findResources('AWS::Lambda::Function', {
Properties: {
Role: {
'Fn::GetAtt': [
'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8aServiceRole9741ECFB',
'Arn',
],
},
Timeout: 30,
},
})).length).toEqual(1);

Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Function', Match.not({
Role: {
'Fn::GetAtt': [
Expand All @@ -67,8 +100,25 @@ describe('log group', () => {
// THEN
Template.fromStack(stack).hasResourceProperties('Custom::LogRetention', {
LogGroupName: 'testLogGroup',
SdkRetry: {
maxRetries: 7,
base: 200,
},
RetentionInDays: 7,
});

expect(Object.keys(Template.fromStack(stack).findResources('AWS::Lambda::Function', {
Properties: {
Role: {
'Fn::GetAtt': [
'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8aServiceRole9741ECFB',
'Arn',
],
},
Timeout: 30,
},
})).length).toEqual(1);

Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Function', Match.not({
Role: {
'Fn::GetAtt': [
Expand All @@ -92,9 +142,25 @@ describe('exporting log group', () => {
// THEN
Template.fromStack(stack).hasResourceProperties('Custom::LogRetention', {
LogGroupName: 'testLogGroup',
SdkRetry: {
maxRetries: 7,
base: 200,
},
RetentionInDays: 3,
});

expect(Object.keys(Template.fromStack(stack).findResources('AWS::Lambda::Function', {
Properties: {
Role: {
'Fn::GetAtt': [
'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8aServiceRole9741ECFB',
'Arn',
],
},
Timeout: 30,
},
})).length).toEqual(1);

Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Function', {
Role: {
'Fn::GetAtt': [
Expand Down Expand Up @@ -124,9 +190,25 @@ describe('exporting log group', () => {
// THEN
Template.fromStack(stack).hasResourceProperties('Custom::LogRetention', {
LogGroupName: 'prefix-testLogGroup',
SdkRetry: {
maxRetries: 7,
base: 200,
},
RetentionInDays: 3,
});

expect(Object.keys(Template.fromStack(stack).findResources('AWS::Lambda::Function', {
Properties: {
Role: {
'Fn::GetAtt': [
'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8aServiceRole9741ECFB',
'Arn',
],
},
Timeout: 30,
},
})).length).toEqual(1);

Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Function', {
Role: {
'Fn::GetAtt': [
Expand Down Expand Up @@ -157,9 +239,25 @@ describe('exporting log group', () => {
// THEN
Template.fromStack(stack).hasResourceProperties('Custom::LogRetention', {
LogGroupName: 'testLogGroup',
SdkRetry: {
maxRetries: 7,
base: 200,
},
RetentionInDays: 7,
});

expect(Object.keys(Template.fromStack(stack).findResources('AWS::Lambda::Function', {
Properties: {
Role: {
'Fn::GetAtt': [
'LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8aServiceRole9741ECFB',
'Arn',
],
},
Timeout: 30,
},
})).length).toEqual(1);

Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Function', {
Role: {
'Fn::GetAtt': [
Expand Down

0 comments on commit 38df77f

Please sign in to comment.