feat(app-staging-synthesizer): clean up staging resources on deletion

lerna.json

@@ -4,6 +4,7 @@
   "packages": [
     "packages/aws-cdk-lib",
     "packages/cdk-assets",
+    "packages/custom-resource-handlers",
     "packages/aws-cdk",
     "packages/cdk",
     "packages/@aws-cdk/*",

package.json

@@ -72,6 +72,7 @@
       "packages/aws-cdk",
       "packages/cdk",
       "packages/cdk-assets",
+      "packages/custom-resource-handlers",
       "packages/@aws-cdk/*",
       "packages/awslint",
       "packages/@aws-cdk-containers/*",
@@ -139,6 +140,8 @@
       "@aws-cdk/pipelines/aws-sdk/**",
       "@aws-cdk/yaml-cfn/yaml",
       "@aws-cdk/yaml-cfn/yaml/**",
+      "aws-cdk-lib/@aws-cdk/custom-resource-handlers",
+      "aws-cdk-lib/@aws-cdk/custom-resource-handlers/**",
       "aws-cdk-lib/@balena/dockerignore",
       "aws-cdk-lib/@balena/dockerignore/**",
       "aws-cdk-lib/case",

packages/@aws-cdk/app-staging-synthesizer-alpha/lib/default-staging-stack.ts

@@ -11,6 +11,7 @@ import {
   RemovalPolicy,
   Stack,
   StackProps,
+  INLINE_CUSTOM_RESOURCE_CONTEXT,
 } from 'aws-cdk-lib';
 import * as ecr from 'aws-cdk-lib/aws-ecr';
 import * as iam from 'aws-cdk-lib/aws-iam';
@@ -86,6 +87,14 @@ export interface DefaultStagingStackOptions {
    * @default - up to 3 versions stored
    */
   readonly imageAssetVersionCount?: number;
+
+  /**
+   * Auto deletes objects in the staging S3 bucket and images in the
+   * staging ECR repositories.
+   *
+   * @default false
+   */
+  readonly autoDeleteStagingAssets?: boolean;
 }
 
 /**
@@ -198,6 +207,7 @@ export class DefaultStagingStack extends Stack implements IStagingResources {
   private imageRole?: iam.IRole;
   private didImageRole = false;
   private imageRoleManifestArn?: string;
+  private autoDeleteStagingAssets: boolean;
 
   private readonly deployRoleArn?: string;
 
@@ -207,6 +217,12 @@ export class DefaultStagingStack extends Stack implements IStagingResources {
       synthesizer: new BootstraplessSynthesizer(),
     });
 
+    // For all resources under the default staging stack, we want to inline custom
+    // resources because the staging bucket necessary for custom resource assets
+    // does not exist yet.
+    this.node.setContext(INLINE_CUSTOM_RESOURCE_CONTEXT, true);
+    this.autoDeleteStagingAssets = props.autoDeleteStagingAssets ?? false;
+
     this.appId = this.validateAppId(props.appId);
     this.dependencyStack = this;
 
@@ -316,7 +332,12 @@ export class DefaultStagingStack extends Stack implements IStagingResources {
     // Create the bucket once the dependencies have been created
     const bucket = new s3.Bucket(this, bucketId, {
       bucketName: stagingBucketName,
-      removalPolicy: RemovalPolicy.RETAIN,
+      ...(this.autoDeleteStagingAssets ? {
+        removalPolicy: RemovalPolicy.DESTROY,
+        autoDeleteObjects: true,
+      } : {
+        removalPolicy: RemovalPolicy.RETAIN,
+      }),
       encryption: s3.BucketEncryption.KMS,
       encryptionKey: key,
 
@@ -375,7 +396,14 @@ export class DefaultStagingStack extends Stack implements IStagingResources {
           description: 'Garbage collect old image versions and keep the specified number of latest versions',
           maxImageCount: this.props.imageAssetVersionCount ?? 3,
         }],
+        ...(this.autoDeleteStagingAssets ? {
+          removalPolicy: RemovalPolicy.DESTROY,
+          autoDeleteImages: true,
+        } : {
+          removalPolicy: RemovalPolicy.RETAIN,
+        }),
       });
+
       if (this.imageRole) {
         this.stagingRepos[asset.assetName].grantPullPush(this.imageRole);
         this.stagingRepos[asset.assetName].grantRead(this.imageRole);

packages/@aws-cdk/app-staging-synthesizer-alpha/package.json

@@ -79,6 +79,9 @@
     "announce": false
   },
   "cdk-build": {
+    "pre": [
+      "./scripts/airlift-custom-resource-handlers.sh"
+    ],
     "env": {
       "AWSLINT_BASE_CONSTRUCT": true
     }
@@ -93,7 +96,8 @@
     "@aws-cdk/integ-tests-alpha": "0.0.0",
     "constructs": "^10.0.0",
     "@aws-cdk/cdk-build-tools": "0.0.0",
-    "@aws-cdk/pkglint": "0.0.0"
+    "@aws-cdk/pkglint": "0.0.0",
+    "@aws-cdk/custom-resource-handlers": "0.0.0"
   },
   "peerDependencies": {
     "aws-cdk-lib": "0.0.0",

packages/@aws-cdk/app-staging-synthesizer-alpha/scripts/airlift-custom-resource-handlers.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+scriptdir=$(cd $(dirname $0) && pwd)
+customresourcedir=${scriptdir}/../../../custom-resource-handlers
+packagedir=${scriptdir}/..
+
+cd ${packagedir}
+
+mkdir -p custom-resource-handlers/aws-s3/auto-delete-objects-handler
+mkdir -p custom-resource-handlers/aws-ecr/auto-delete-images-handler
+
+cp ${customresourcedir}/lib/aws-s3/auto-delete-objects-handler/index.js ${packagedir}/custom-resource-handlers/aws-s3/auto-delete-objects-handler
+cp ${customresourcedir}/lib/aws-ecr/auto-delete-images-handler/index.js ${packagedir}/custom-resource-handlers/aws-ecr/auto-delete-images-handler

packages/@aws-cdk/app-staging-synthesizer-alpha/test/integ.synth-default-resources.ts

@@ -1,5 +1,5 @@
 import * as path from 'path';
-import * as integ from '@aws-cdk/integ-tests-alpha';
+// import * as integ from '@aws-cdk/integ-tests-alpha';
 import { App, Stack } from 'aws-cdk-lib';
 import * as lambda from 'aws-cdk-lib/aws-lambda';
 import { AppStagingSynthesizer } from '../lib';
@@ -9,6 +9,7 @@ const app = new App();
 const stack = new Stack(app, 'synthesize-default-resources', {
   synthesizer: AppStagingSynthesizer.defaultResources({
     appId: 'default-resources',
+    autoDeleteStagingAssets: true,
   }),
 });
 
@@ -44,8 +45,8 @@ new lambda.Function(stack, 'lambda-ecr-2', {
   runtime: lambda.Runtime.FROM_IMAGE,
 });
 
-new integ.IntegTest(app, 'integ-tests', {
-  testCases: [stack],
-});
+// new integ.IntegTest(app, 'integ-tests', {
+//   testCases: [stack],
+// });
 
 app.synth();

packages/aws-cdk-lib/aws-ecr/lib/repository.ts

@@ -24,6 +24,7 @@ import {
 
 const AUTO_DELETE_IMAGES_RESOURCE_TYPE = 'Custom::ECRAutoDeleteImages';
 const AUTO_DELETE_IMAGES_TAG = 'aws-cdk:auto-delete-images';
+const REPO_ARN_SYMBOL = Symbol.for('@aws-cdk/aws-ecr.RepoArns');
 
 /**
  * Represents an ECR repository.
@@ -816,26 +817,35 @@ export class Repository extends RepositoryBase {
   }
 
   private enableAutoDeleteImages() {
-    // Use a iam policy to allow the custom resource to list & delete
-    // images in the repository and the ability to get all repositories to find the arn needed on delete.
+    const firstTime = Stack.of(this).node.tryFindChild(`${AUTO_DELETE_IMAGES_RESOURCE_TYPE}CustomResourceProvider`) === undefined;
     const provider = CustomResourceProvider.getOrCreateProvider(this, AUTO_DELETE_IMAGES_RESOURCE_TYPE, {
-      codeDirectory: path.join(__dirname, 'auto-delete-images-handler'),
+      codeDirectory: path.join(__dirname, '..', '..', 'custom-resource-handlers', 'lib', 'aws-ecr', 'auto-delete-images-handler'),
+      useCfnResponseWrapper: false,
       runtime: builtInCustomResourceProviderNodeRuntime(this),
       description: `Lambda function for auto-deleting images in ${this.repositoryName} repository.`,
-      policyStatements: [
-        {
-          Effect: 'Allow',
-          Action: [
-            'ecr:BatchDeleteImage',
-            'ecr:DescribeRepositories',
-            'ecr:ListImages',
-            'ecr:ListTagsForResource',
-          ],
-          Resource: [this._resource.attrArn],
-        },
-      ],
     });
 
+    if (firstTime) {
+      const repoArns = [this._resource. attrArn];
+      (provider as any)[REPO_ARN_SYMBOL] = repoArns;
+
+      // Use a iam policy to allow the custom resource to list & delete
+      // images in the repository and the ability to get all repositories to find the arn needed on delete.
+      // We lazily produce a list of repositories associated with this custom resource provider.
+      provider.addToRolePolicy({
+        Effect: 'Allow',
+        Action: [
+          'ecr:BatchDeleteImage',
+          'ecr:DescribeRepositories',
+          'ecr:ListImages',
+          'ecr:ListTagsForResource',
+        ],
+        Resource: Lazy.list({ produce: () => repoArns }),
+      });
+    } else {
+      (provider as any)[REPO_ARN_SYMBOL].push(this._resource.attrArn);
+    }
+
     const customResource = new CustomResource(this, 'AutoDeleteImagesCustomResource', {
       resourceType: AUTO_DELETE_IMAGES_RESOURCE_TYPE,
       serviceToken: provider.serviceToken,

packages/aws-cdk-lib/aws-s3/lib/bucket.ts

@@ -2384,7 +2384,8 @@ export class Bucket extends BucketBase {
 
   private enableAutoDeleteObjects() {
     const provider = CustomResourceProvider.getOrCreateProvider(this, AUTO_DELETE_OBJECTS_RESOURCE_TYPE, {
-      codeDirectory: path.join(__dirname, 'auto-delete-objects-handler'),
+      codeDirectory: path.join(__dirname, '..', '..', 'custom-resource-handlers', 'lib', 'aws-s3', 'auto-delete-objects-handler'),
+      useCfnResponseWrapper: false,
       runtime: builtInCustomResourceProviderNodeRuntime(this),
       description: `Lambda function for auto-deleting objects in ${this.bucketName} S3 bucket.`,
     });

packages/aws-cdk-lib/core/lib/custom-resource-provider/custom-resource-provider.ts

@@ -1,7 +1,6 @@
-import * as fs from 'fs';
 import * as path from 'path';
 import { Construct } from 'constructs';
-import * as fse from 'fs-extra';
+import * as fs from 'fs-extra';
 import * as cxapi from '../../../cx-api';
 import { FactName } from '../../../region-info';
 import { AssetStaging } from '../asset-staging';
@@ -17,6 +16,7 @@ import { Token } from '../token';
 
 const ENTRYPOINT_FILENAME = '__entrypoint__';
 const ENTRYPOINT_NODEJS_SOURCE = path.join(__dirname, 'nodejs-entrypoint.js');
+export const INLINE_CUSTOM_RESOURCE_CONTEXT = '@aws-cdk/core:inlineCustomResourceIfPossible';
 
 /**
  * The lambda runtime used by default for aws-cdk vended custom resources. Can change
@@ -32,6 +32,16 @@ export function builtInCustomResourceProviderNodeRuntime(scope: Construct): Cust
  *
  */
 export interface CustomResourceProviderProps {
+  /**
+   * Whether or not the cloudformation response wrapper (`nodejs-entrypoint.ts`) is used.
+   * If set to `true`, `nodejs-entrypoint.js` is bundled in the same asset as the custom resource
+   * and set as the entrypoint. If set to `false`, the custom resource provided is the
+   * entrypoint.
+   *
+   * @default - `true` if `inlineCode: false` and `false` otherwise.
+   */
+  readonly useCfnResponseWrapper?: boolean;
+
   /**
    * A local file system directory with the provider's code. The code will be
    * bundled into a zip asset and wired to the provider's AWS Lambda function.
@@ -216,7 +226,14 @@ export class CustomResourceProvider extends Construct {
    * The hash of the lambda code backing this provider. Can be used to trigger updates
    * on code changes, even when the properties of a custom resource remain unchanged.
    */
-  public readonly codeHash: string;
+  public get codeHash(): string {
+    if (!this._codeHash) {
+      throw new Error('This custom resource uses inlineCode: true and does not have a codeHash');
+    }
+    return this._codeHash;
+  }
+
+  private _codeHash?: string;
 
   private policyStatements?: any[];
   private _role?: CfnResource;
@@ -231,21 +248,7 @@ export class CustomResourceProvider extends Construct {
       throw new Error(`cannot find ${props.codeDirectory}/index.js`);
     }
 
-    const stagingDirectory = FileSystem.mkdtemp('cdk-custom-resource');
-    fse.copySync(props.codeDirectory, stagingDirectory, { filter: (src, _dest) => !src.endsWith('.ts') });
-    fs.copyFileSync(ENTRYPOINT_NODEJS_SOURCE, path.join(stagingDirectory, `${ENTRYPOINT_FILENAME}.js`));
-
-    const staging = new AssetStaging(this, 'Staging', {
-      sourcePath: stagingDirectory,
-    });
-
-    const assetFileName = staging.relativeStagedPath(stack);
-
-    const asset = stack.synthesizer.addFileAsset({
-      fileName: assetFileName,
-      sourceHash: staging.assetHash,
-      packaging: FileAssetPackaging.ZIP_DIRECTORY,
-    });
+    const { code, codeHandler, metadata } = this.createCodePropAndMetadata(props, stack);
 
     if (props.policyStatements) {
       for (const statement of props.policyStatements) {
@@ -304,13 +307,10 @@ export class CustomResourceProvider extends Construct {
     const handler = new CfnResource(this, 'Handler', {
       type: 'AWS::Lambda::Function',
       properties: {
-        Code: {
-          S3Bucket: asset.bucketName,
-          S3Key: asset.objectKey,
-        },
+        Code: code,
         Timeout: timeout.toSeconds(),
         MemorySize: memory.toMebibytes(),
-        Handler: `${ENTRYPOINT_FILENAME}.handler`,
+        Handler: codeHandler,
         Role: this.roleArn,
         Runtime: customResourceProviderRuntimeToString(props.runtime),
         Environment: this.renderEnvironmentVariables(props.environment),
@@ -322,13 +322,66 @@ export class CustomResourceProvider extends Construct {
       handler.addDependency(this._role);
     }
 
-    if (this.node.tryGetContext(cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT)) {
-      handler.addMetadata(cxapi.ASSET_RESOURCE_METADATA_PATH_KEY, assetFileName);
-      handler.addMetadata(cxapi.ASSET_RESOURCE_METADATA_PROPERTY_KEY, 'Code');
+    if (metadata) {
+      Object.entries(metadata).forEach(([k, v]) => handler.addMetadata(k, v));
     }
 
     this.serviceToken = Token.asString(handler.getAtt('Arn'));
-    this.codeHash = staging.assetHash;
+  }
+
+  /**
+   * Returns the code property for the custom resource as well as any metadata.
+   * If the code is to be uploaded as an asset, the asset gets created in this function.
+   */
+  private createCodePropAndMetadata(props: CustomResourceProviderProps, stack: Stack): {
+    code: Code,
+    codeHandler: string,
+    metadata?: {[key: string]: string},
+  } {
+    let codeHandler = 'index.handler';
+    const inlineCode = this.node.tryGetContext(INLINE_CUSTOM_RESOURCE_CONTEXT);
+    if (!inlineCode) {
+      const stagingDirectory = FileSystem.mkdtemp('cdk-custom-resource');
+      fs.copySync(props.codeDirectory, stagingDirectory, { filter: (src, _dest) => !src.endsWith('.ts') });
+
+      if (props.useCfnResponseWrapper ?? true) {
+        fs.copyFileSync(ENTRYPOINT_NODEJS_SOURCE, path.join(stagingDirectory, `${ENTRYPOINT_FILENAME}.js`));
+        codeHandler = `${ENTRYPOINT_FILENAME}.handler`;
+      }
+
+      const staging = new AssetStaging(this, 'Staging', {
+        sourcePath: stagingDirectory,
+      });
+
+      const assetFileName = staging.relativeStagedPath(stack);
+
+      const asset = stack.synthesizer.addFileAsset({
+        fileName: assetFileName,
+        sourceHash: staging.assetHash,
+        packaging: FileAssetPackaging.ZIP_DIRECTORY,
+      });
+
+      this._codeHash = staging.assetHash;
+
+      return {
+        code: {
+          S3Bucket: asset.bucketName,
+          S3Key: asset.objectKey,
+        },
+        codeHandler,
+        metadata: this.node.tryGetContext(cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT) ? {
+          [cxapi.ASSET_RESOURCE_METADATA_PATH_KEY]: assetFileName,
+          [cxapi.ASSET_RESOURCE_METADATA_PROPERTY_KEY]: 'Code',
+        } : undefined,
+      };
+    }
+
+    return {
+      code: {
+        ZipFile: fs.readFileSync(path.join(props.codeDirectory, 'index.js'), 'utf-8'),
+      },
+      codeHandler,
+    };
   }
 
   /**
@@ -408,3 +461,10 @@ function customResourceProviderRuntimeToString(x: CustomResourceProviderRuntime)
       return 'nodejs18.x';
   }
 }
+
+type Code = {
+  ZipFile: string,
+} | {
+  S3Bucket: string,
+  S3Key: string,
+};