From 537cb021b62d85a6e249d050855fdadbc98dfcc2 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Wed, 11 Sep 2024 10:42:52 +0200 Subject: [PATCH 01/11] feat(cli): `cdk rollback` Add a CLI feature to roll a stuck change back. This is mostly useful for deployments performed using `--no-rollback`: if a failure occurs, the stack gets stuck in an `UPDATE_FAILED` state from which there are 2 options: - Try again using a new template - Roll back to the last stable state There used to be no way to perform the second operation using the CDK CLI, but there now is. `cdk rollback` works in 2 situations: - A paused fail state; it will initiating a fresh rollback. - A paused rollback state; it will retry the rollback, optionally skipping some resources. `cdk rollback --force` will look up all failed resources and continue skipping them until the rollback has finished. --- packages/@aws-cdk-testing/cli-integ/README.md | 2 +- .../cli-integ/lib/with-cdk-app.ts | 19 +- .../cdk-apps/rollback-test-app/app.js | 95 +++++++++ .../cdk-apps/rollback-test-app/cdk.json | 7 + .../tests/cli-integ-tests/cli.integtest.ts | 47 +++++ packages/aws-cdk/README.md | 59 ++++-- .../lib/api/bootstrap/bootstrap-template.yaml | 4 +- packages/aws-cdk/lib/api/cxapp/exec.ts | 3 + packages/aws-cdk/lib/api/deployments.ts | 191 +++++++++++++++++- .../cloudformation/stack-activity-monitor.ts | 90 +++------ .../util/cloudformation/stack-event-poller.ts | 172 ++++++++++++++++ .../api/util/cloudformation/stack-status.ts | 30 +++ packages/aws-cdk/lib/cdk-toolkit.ts | 62 ++++++ packages/aws-cdk/lib/cli.ts | 25 +++ .../test/api/stack-activity-monitor.test.ts | 12 ++ 15 files changed, 725 insertions(+), 93 deletions(-) create mode 100644 packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js create mode 100644 packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/cdk.json create mode 100644 packages/aws-cdk/lib/api/util/cloudformation/stack-event-poller.ts diff --git a/packages/@aws-cdk-testing/cli-integ/README.md b/packages/@aws-cdk-testing/cli-integ/README.md index 2dc2e9c70d8cc..d1dd485660151 100644 --- a/packages/@aws-cdk-testing/cli-integ/README.md +++ b/packages/@aws-cdk-testing/cli-integ/README.md @@ -37,7 +37,7 @@ Test suites are written as a collection of Jest tests, and they are run using Je ### Setup -Building the @aws-cdk-testing package is not very different from building the rest of the CDK. However, If you are having issues with the tests, you can ensure your enviornment is built properly by following the steps below: +Building the @aws-cdk-testing package is not very different from building the rest of the CDK. However, If you are having issues with the tests, you can ensure your environment is built properly by following the steps below: ```shell yarn install # Install dependencies diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 619ca8b96175c..d80865647086e 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -24,7 +24,8 @@ export const EXTENDED_TEST_TIMEOUT_S = 30 * 60; * For backwards compatibility with existing tests (so we don't have to change * too much) the inner block is expected to take a `TestFixture` object. */ -export function withCdkApp( +export function withSpecificCdkApp( + appName: string, block: (context: TestFixture) => Promise, ): (context: TestContext & AwsContext & DisableBootstrapContext) => Promise { return async (context: TestContext & AwsContext & DisableBootstrapContext) => { @@ -36,7 +37,7 @@ export function withCdkApp( context.output.write(` Test directory: ${integTestDir}\n`); context.output.write(` Region: ${context.aws.region}\n`); - await cloneDirectory(path.join(RESOURCES_DIR, 'cdk-apps', 'app'), integTestDir, context.output); + await cloneDirectory(path.join(RESOURCES_DIR, 'cdk-apps', appName), integTestDir, context.output); const fixture = new TestFixture( integTestDir, stackNamePrefix, @@ -87,6 +88,16 @@ export function withCdkApp( }; } +/** + * Like `withSpecificCdkApp`, but uses the default integration testing app with a million stacks in it + */ +export function withCdkApp( + block: (context: TestFixture) => Promise, +): (context: TestContext & AwsContext & DisableBootstrapContext) => Promise { + // 'app' is the name of the default integration app in the `cdk-apps` directory + return withSpecificCdkApp('app', block); +} + export function withCdkMigrateApp(language: string, block: (context: TestFixture) => Promise) { return async (context: A) => { const stackName = `cdk-migrate-${language}-integ-${context.randomString}`; @@ -188,6 +199,10 @@ export function withDefaultFixture(block: (context: TestFixture) => Promise Promise) { + return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withSpecificCdkApp(appName, block))); +} + export function withExtendedTimeoutFixture(block: (context: TestFixture) => Promise) { return withAws(withTimeout(EXTENDED_TEST_TIMEOUT_S, withCdkApp(block))); } diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js new file mode 100644 index 0000000000000..c2d772ec3ff47 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js @@ -0,0 +1,95 @@ +const cdk = require('aws-cdk-lib'); +const lambda = require('aws-cdk-lib/aws-lambda'); +const cr = require('aws-cdk-lib/custom-resources'); + +/** + * This stack will be deployed in multiple phases, to achieve a very specific effect + * + * It contains resources r1 and r2, where r1 gets deployed first. + * + * - PHASE = 1: both resources deploy regularly. + * - PHASE = 2: r1 gets updated, r2 will fail to update, and r1 will fail its rollback. + * + * To exercise this app: + * + * ``` + * env PHASE=1 npx cdk deploy + * env PHASE=2 npx cdk deploy --no-rollback + * # This will leave the stack in UPDATE_FAILED + * + * env PHASE=2 npx cdk rollback + * # This will start a rollback that will fail because r1 fails its rollabck + * + * env PHASE=2 npx cdk rollback --force + * # This will retry the rollabck and skip r1 + * ``` + */ +class RollbacktestStack extends cdk.Stack { + constructor(scope, id, props) { + super(scope, id, props); + + let r1props = {}; + let r2props = {}; + + const phase = process.env.PHASE; + switch (phase) { + case '1': + // Normal deployment + break; + case '2': + // r1 updates normally, r2 fails updating, r1 fails rollback + r1props.FailRollback = true; + r2props.FailUpdate = true; + break; + } + + const fn = new lambda.Function(this, 'Fun', { + runtime: lambda.Runtime.NODEJS_LATEST, + code: lambda.Code.fromInline(`exports.handler = async function(event, ctx) { + const key = \`Fail\${event.RequestType}\`; + if (event.ResourceProperties[key]) { + throw new Error(\`\${event.RequestType} fails!\`); + } + if (event.OldResourceProperties?.FailRollback) { + throw new Error('Failing rollback!'); + } + return {}; + }`), + handler: 'index.handler', + timeout: cdk.Duration.minutes(1), + }); + const provider = new cr.Provider(this, "MyProvider", { + onEventHandler: fn, + }); + + const r1 = new cdk.CustomResource(this, 'r1', { + serviceToken: provider.serviceToken, + properties: r1props, + }); + const r2 = new cdk.CustomResource(this, 'r2', { + serviceToken: provider.serviceToken, + properties: r2props, + }); + r2.node.addDependency(r1); + } +} + +const app = new cdk.App({ + context: { + '@aws-cdk/core:assetHashSalt': process.env.CODEBUILD_BUILD_ID, // Force all assets to be unique, but consistent in one build + }, +}); + +const defaultEnv = { + account: process.env.CDK_DEFAULT_ACCOUNT, + region: process.env.CDK_DEFAULT_REGION +}; + +const stackPrefix = process.env.STACK_NAME_PREFIX; +if (!stackPrefix) { + throw new Error(`the STACK_NAME_PREFIX environment variable is required`); +} + +// Sometimes we don't want to synthesize all stacks because it will impact the results +new RollbacktestStack(app, `${stackPrefix}-test-rollback`, { env: defaultEnv }); +app.synth(); \ No newline at end of file diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/cdk.json b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/cdk.json new file mode 100644 index 0000000000000..44809158dbdac --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/cdk.json @@ -0,0 +1,7 @@ +{ + "app": "node app.js", + "versionReporting": false, + "context": { + "aws-cdk:enableDiffNoFail": "true" + } +} diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts index 8ebfcc0752716..78ab95e257541 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts @@ -33,6 +33,7 @@ import { withCDKMigrateFixture, withExtendedTimeoutFixture, randomString, + withSpecificFixture, } from '../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime @@ -2200,3 +2201,49 @@ integTest( expect(noticesUnacknowledged).toEqual(noticesUnacknowledgedAlias); }), ); + +integTest( + 'test cdk rollback', + withSpecificFixture('rollback-test-app', async (fixture) => { + let phase = '1'; + + // Should succeed + await fixture.cdkDeploy('test-rollback', { + options: ['--no-rollback'], + modEnv: { PHASE: phase }, + verbose: false, + }); + try { + phase = '2'; + + // Should fail + const deployOutput = await fixture.cdkDeploy('test-rollback', { + options: ['--no-rollback'], + modEnv: { PHASE: phase }, + verbose: false, + allowErrExit: true, + }); + if (!deployOutput.includes('UPDATE_FAILED')) { + throw new Error(`Expected output to contain UPDATE_FAILED, got: ${deployOutput}`); + } + + // Should still fail + const rollbackOutput = await fixture.cdk(['rollback'], { + modEnv: { PHASE: phase }, + verbose: false, + allowErrExit: true, + }); + if (!rollbackOutput.includes('Failing rollback')) { + throw new Error(`Expected output to contain "Failing rollback", got: ${rollbackOutput}`); + } + + // Rollback and force cleanup + await fixture.cdk(['rollback', '--force'], { + modEnv: { PHASE: phase }, + verbose: false, + }); + } finally { + await fixture.cdkDestroy('test-rollback'); + } + }), +); \ No newline at end of file diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index 268f0989c6a5e..2f246ef820482 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -19,6 +19,7 @@ The AWS CDK Toolkit provides the `cdk` command-line interface that can be used t | [`cdk synth`](#cdk-synthesize) | Synthesize a CDK app to CloudFormation template(s) | | [`cdk diff`](#cdk-diff) | Diff stacks against current state | | [`cdk deploy`](#cdk-deploy) | Deploy a stack into an AWS account | +| [`cdk rollback`](#cdk-rollback) | Roll back a failed deployment | | [`cdk import`](#cdk-import) | Import existing AWS resources into a CDK stack | | [`cdk migrate`](#cdk-migrate) | Migrate AWS resources, CloudFormation stacks, and CloudFormation templates to CDK | | [`cdk watch`](#cdk-watch) | Watches a CDK app for deployable and hotswappable changes | @@ -202,6 +203,10 @@ $ cdk deploy --no-rollback $ cdk deploy -R ``` +If a deployment fails you can update your code and immediately retry the +deployment from the point of failure. If you would like to explicitly roll back a failed, paused deployment, +use `cdk rollback`. + NOTE: you cannot use `--no-rollback` for any updates that would cause a resource replacement, only for updates and creations of new resources. @@ -395,7 +400,7 @@ development, your prod app may not have any resources or the resources are comme out. In this scenario, you will receive an error message stating that the app has no stacks. -To bypass this error messages, you can pass the `--ignore-no-stacks` flag to the +To bypass this error messages, you can pass the `--ignore-no-stacks` flag to the `deploy` command: ```console @@ -466,6 +471,20 @@ and might have breaking changes in the future. > *: `Fn::GetAtt` is only partially supported. Refer to [this implementation](https://github.com/aws/aws-cdk/blob/main/packages/aws-cdk/lib/api/evaluate-cloudformation-template.ts#L477-L492) for supported resources and attributes. +### `cdk rollback` + +If a deployment performed using `cdk deploy --no-rollback` fails, your +deployment will be left in a failed, paused state. From this state you can +update your code and try the deployment again, or roll the deployment back to +the last stable state. + +To roll the deployment back, use `cdk rollback`. This will initiate a rollback +to the last stable state of your stack. + +Some resources may fail to roll back. If they do, you can try again by calling +`cdk rollback --orphan `. Or, run `cdk rollback --force` to have +the CDK CLI automatically orphan all failing resources. + ### `cdk watch` The `watch` command is similar to `deploy`, @@ -596,9 +615,9 @@ This feature currently has the following limitations: ### `cdk migrate` -⚠️**CAUTION**⚠️: CDK Migrate is currently experimental and may have breaking changes in the future. +⚠️**CAUTION**⚠️: CDK Migrate is currently experimental and may have breaking changes in the future. -CDK Migrate generates a CDK app from deployed AWS resources using `--from-scan`, deployed AWS CloudFormation stacks using `--from-stack`, and local AWS CloudFormation templates using `--from-path`. +CDK Migrate generates a CDK app from deployed AWS resources using `--from-scan`, deployed AWS CloudFormation stacks using `--from-stack`, and local AWS CloudFormation templates using `--from-path`. To learn more about the CDK Migrate feature, see [Migrate to AWS CDK](https://docs.aws.amazon.com/cdk/v2/guide/migrate.html). For more information on `cdk migrate` command options, see [cdk migrate command reference](https://docs.aws.amazon.com/cdk/v2/guide/ref-cli-cdk-migrate.html). @@ -630,7 +649,7 @@ Account and Region information are retrieved from default CDK CLI sources. Use ` $ cdk migrate --language typescript --from-scan --stack-name "myCloudFormationStack" ``` -Since CDK Migrate relies on the IaC generator service, any limitations of IaC generator will apply to CDK Migrate. For general limitations, see [Considerations](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/generate-IaC.html#generate-template-considerations). +Since CDK Migrate relies on the IaC generator service, any limitations of IaC generator will apply to CDK Migrate. For general limitations, see [Considerations](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/generate-IaC.html#generate-template-considerations). IaC generator limitations with discovering resource and property values will also apply here. As a result, CDK Migrate will only migrate resources supported by IaC generator. Some of your resources may not be supported and some property values may not be accessible. For more information, see [Iac generator and write-only properties](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/generate-IaC-write-only-properties.html) and [Supported resource types](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/generate-IaC-supported-resources.html). @@ -647,8 +666,8 @@ $ # template.json is a valid cloudformation template in the local directory $ cdk migrate --stack-name MyAwesomeApplication --language typescript --from-path MyTemplate.json ``` -This command generates a new directory named `MyAwesomeApplication` within your current working directory, and -then initializes a new CDK application within that directory. The CDK app contains a `MyAwesomeApplication` stack with resources configured to match those in your local CloudFormation template. +This command generates a new directory named `MyAwesomeApplication` within your current working directory, and +then initializes a new CDK application within that directory. The CDK app contains a `MyAwesomeApplication` stack with resources configured to match those in your local CloudFormation template. This results in a CDK application with the following structure, where the lib directory contains a stack definition with the same resource configuration as the provided template.json. @@ -678,13 +697,13 @@ This will generate a Python CDK app which will synthesize the same configuration ##### Generate a TypeScript CDK app from deployed AWS resources that are not associated with a stack -If you have resources in your account that were provisioned outside AWS IaC tools and would like to manage them with the CDK, you can use the `--from-scan` option to generate the application. +If you have resources in your account that were provisioned outside AWS IaC tools and would like to manage them with the CDK, you can use the `--from-scan` option to generate the application. In this example, we use the `--filter` option to specify which resources to migrate. You can filter resources to limit the number of resources migrated to only those specified by the `--filter` option, including any resources they depend on, or resources that depend on them (for example A filter which specifies a single Lambda Function, will find that specific table and any alarms that may monitor it). The `--filter` argument offers both AND as well as OR filtering. OR filtering can be specified by passing multiple `--filter` options, and AND filtering can be specified by passing a single `--filter` option with multiple comma separated key/value pairs as seen below (see below for examples). It is recommended to use the `--filter` option to limit the number of resources returned as some resource types provide sample resources by default in all accounts which can add to the resource limits. -`--from-scan` takes 3 potential arguments: `--new`, `most-recent`, and undefined. If `--new` is passed, CDK Migrate will initiate a new scan of the account and use that new scan to discover resources. If `--most-recent` is passed, CDK Migrate will use the most recent scan of the account to discover resources. If neither `--new` nor `--most-recent` are passed, CDK Migrate will take the most recent scan of the account to discover resources, unless there is no recent scan, in which case it will initiate a new scan. +`--from-scan` takes 3 potential arguments: `--new`, `most-recent`, and undefined. If `--new` is passed, CDK Migrate will initiate a new scan of the account and use that new scan to discover resources. If `--most-recent` is passed, CDK Migrate will use the most recent scan of the account to discover resources. If neither `--new` nor `--most-recent` are passed, CDK Migrate will take the most recent scan of the account to discover resources, unless there is no recent scan, in which case it will initiate a new scan. ```console # Filtering options @@ -717,14 +736,14 @@ $ cdk migrate --stack-name MyAwesomeApplication --language typescript --from-sca - CDK Migrate will only generate L1 constructs and does not currently support any higher level abstractions. - CDK Migrate successfully generating an application does *not* guarantee the application is immediately deployable. -It simply generates a CDK application which will synthesize a template that has identical resource configurations -to the provided template. +It simply generates a CDK application which will synthesize a template that has identical resource configurations +to the provided template. - - CDK Migrate does not interact with the CloudFormation service to verify the template -provided can deploy on its own. Although by default any CDK app generated using the `--from-scan` option exclude -CloudFormation managed resources, CDK Migrate will not verify prior to deployment that any resources scanned, or in the provided + - CDK Migrate does not interact with the CloudFormation service to verify the template +provided can deploy on its own. Although by default any CDK app generated using the `--from-scan` option exclude +CloudFormation managed resources, CDK Migrate will not verify prior to deployment that any resources scanned, or in the provided template are already managed in other CloudFormation templates, nor will it verify that the resources in the provided -template are available in the desired regions, which may impact ADC or Opt-In regions. +template are available in the desired regions, which may impact ADC or Opt-In regions. - If the provided template has parameters without default values, those will need to be provided before deploying the generated application. @@ -741,13 +760,13 @@ In practice this is how CDK Migrate generated applications will operate in the f ##### **The provided template is already deployed to CloudFormation in the account/region** -If the provided template came directly from a deployed CloudFormation stack, and that stack has not experienced any drift, +If the provided template came directly from a deployed CloudFormation stack, and that stack has not experienced any drift, then the generated application will be immediately deployable, and will not cause any changes to the deployed resources. Drift might occur if a resource in your template was modified outside of CloudFormation, namely via the AWS Console or AWS CLI. ##### **The provided template is not deployed to CloudFormation in the account/region, and there *is not* overlap with existing resources in the account/region** -If the provided template represents a set of resources that have no overlap with resources already deployed in the account/region, +If the provided template represents a set of resources that have no overlap with resources already deployed in the account/region, then the generated application will be immediately deployable. This could be because the stack has never been deployed, or the application was generated from a stack deployed in another account/region. @@ -764,16 +783,16 @@ In practice this means for any resource in the provided template, for example, } ``` -There must not exist a resource of that type with the same identifier in the desired region. In this example that identfier +There must not exist a resource of that type with the same identifier in the desired region. In this example that identfier would be "MyBucket" ##### **The provided template is not deployed to CloudFormation in the account/region, and there *is* overlap with existing resources in the account/region** -If the provided template represents a set of resources that overlap with resources already deployed in the account/region, -then the generated application will not be immediately deployable. If those overlapped resources are already managed by +If the provided template represents a set of resources that overlap with resources already deployed in the account/region, +then the generated application will not be immediately deployable. If those overlapped resources are already managed by another CloudFormation stack in that account/region, then those resources will need to be manually removed from the provided template. Otherwise, if the overlapped resources are not managed by another CloudFormation stack, then first remove those -resources from your CDK Application Stack, deploy the cdk application successfully, then re-add them and run `cdk import` +resources from your CDK Application Stack, deploy the cdk application successfully, then re-add them and run `cdk import` to import them into your deployed stack. ### `cdk destroy` diff --git a/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml b/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml index dace6e7977a4a..fe0f8f0d1a073 100644 --- a/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml +++ b/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml @@ -457,6 +457,8 @@ Resources: - cloudformation:ExecuteChangeSet - cloudformation:CreateStack - cloudformation:UpdateStack + - cloudformation:RollbackStack + - cloudformation:ContinueUpdateRollback Resource: "*" - Sid: PipelineCrossAccountArtifactsBucket # Read/write buckets in different accounts. Permissions to buckets in @@ -623,7 +625,7 @@ Resources: Type: String Name: Fn::Sub: '/cdk-bootstrap/${Qualifier}/version' - Value: '21' + Value: '22' Outputs: BucketName: Description: The name of the S3 bucket owned by the CDK toolkit stack diff --git a/packages/aws-cdk/lib/api/cxapp/exec.ts b/packages/aws-cdk/lib/api/cxapp/exec.ts index 4069d7a660143..4f608c14fb108 100644 --- a/packages/aws-cdk/lib/api/cxapp/exec.ts +++ b/packages/aws-cdk/lib/api/cxapp/exec.ts @@ -49,6 +49,9 @@ export async function execProgram(aws: SdkProvider, config: Configuration): Prom if (!outdir) { throw new Error('unexpected: --output is required'); } + if (typeof outdir !== 'string') { + throw new Error(`--output takes a string, got ${JSON.stringify(outdir)}`); + } try { await fs.mkdirp(outdir); } catch (error: any) { diff --git a/packages/aws-cdk/lib/api/deployments.ts b/packages/aws-cdk/lib/api/deployments.ts index ee357e01b525c..44c31be1bb8a6 100644 --- a/packages/aws-cdk/lib/api/deployments.ts +++ b/packages/aws-cdk/lib/api/deployments.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'crypto'; import * as cxapi from '@aws-cdk/cx-api'; import * as cdk_assets from 'cdk-assets'; import { AssetManifest, IManifestEntry } from 'cdk-assets'; @@ -8,14 +9,18 @@ import { deployStack, DeployStackResult, destroyStack, DeploymentMethod } from ' import { EnvironmentResources, EnvironmentResourcesRegistry } from './environment-resources'; import { HotswapMode } from './hotswap/common'; import { loadCurrentTemplateWithNestedStacks, loadCurrentTemplate, RootTemplateWithNestedStacks } from './nested-stack-helpers'; -import { CloudFormationStack, Template, ResourcesToImport, ResourceIdentifierSummaries } from './util/cloudformation'; -import { StackActivityProgress } from './util/cloudformation/stack-activity-monitor'; +import { CloudFormationStack, Template, ResourcesToImport, ResourceIdentifierSummaries, stabilizeStack } from './util/cloudformation'; +import { StackActivityMonitor, StackActivityProgress } from './util/cloudformation/stack-activity-monitor'; +import { StackEventPoller } from './util/cloudformation/stack-event-poller'; +import { RollbackChoice } from './util/cloudformation/stack-status'; import { replaceEnvPlaceholders } from './util/placeholders'; import { makeBodyParameterAndUpload } from './util/template-body-parameter'; import { Tag } from '../cdk-toolkit'; import { debug, warning, error } from '../logging'; import { buildAssets, publishAssets, BuildAssetsOptions, PublishAssetsOptions, PublishingAws, EVENT_TO_LOGGER } from '../util/asset-publishing'; +const BOOTSTRAP_STACK_VERSION_FOR_ROLLBACK = 22; + /** * SDK obtained by assuming the lookup role * for a given environment @@ -208,6 +213,70 @@ export interface DeployStackOptions { ignoreNoStacks?: boolean; } +export interface RollbackStackOptions { + /** + * Stack to roll back + */ + readonly stack: cxapi.CloudFormationStackArtifact; + + /** + * Execution role for the deployment (pass through to CloudFormation) + * + * @default - Current role + */ + readonly roleArn?: string; + + /** + * Don't show stack deployment events, just wait + * + * @default false + */ + readonly quiet?: boolean; + + /** + * Whether we are on a CI system + * + * @default false + */ + readonly ci?: boolean; + + /** + * Name of the toolkit stack, if not the default name + * + * @default 'CDKToolkit' + */ + readonly toolkitStackName?: string; + + /** + * Whether to force a rollback or not + * + * Forcing a rollback will orphan all undeletable resources. + * + * @default false + */ + readonly force?: boolean; + + /** + * Orphan the resources with the given logical IDs + * + * @default - No orphaning + */ + readonly orphanLogicalIds?: string[]; + + /** + * Display mode for stack deployment progress. + * + * @default - StackActivityProgress.Bar - stack events will be displayed for + * the resource currently being deployed. + */ + readonly progress?: StackActivityProgress; +} + +export interface RollbackStackResult { + readonly notInRollbackableState?: boolean; + readonly success?: boolean; +} + interface AssetOptions { /** * Stack with assets to build. @@ -417,6 +486,118 @@ export class Deployments { }); } + public async rollbackStack(options: RollbackStackOptions): Promise { + const { + stackSdk, + resolvedEnvironment: _, + cloudFormationRoleArn, + envResources, + } = await this.prepareSdkFor(options.stack, options.roleArn, Mode.ForWriting); + + // Do a verification of the bootstrap stack version + await this.validateBootstrapStackVersion( + options.stack.stackName, + BOOTSTRAP_STACK_VERSION_FOR_ROLLBACK, + options.stack.bootstrapStackVersionSsmParameter, + envResources); + + const cfn = stackSdk.cloudFormation(); + const deployName = options.stack.stackName; + + while (true) { + let cloudFormationStack = await CloudFormationStack.lookup(cfn, deployName); + + // TODO: currently, we only roll back UPDATE_FAILED and CREATE_FAILED stacks. Conceivably, we could also + // handle UPDATE_ROLLBACK_FAILED by offering some option like "force rollback". + switch (cloudFormationStack.stackStatus.rollbackChoice) { + case RollbackChoice.NONE: + warning(`Stack ${deployName} does not need a rollback: ${cloudFormationStack.stackStatus}`); + return { notInRollbackableState: true }; + + case RollbackChoice.START_ROLLBACK: + debug(`Initiating rollback of stack ${deployName}`); + await cfn.rollbackStack({ + StackName: deployName, + RoleARN: cloudFormationRoleArn, + ClientRequestToken: randomUUID(), + // Enabling this is just the better overall default, the only reason it isn't the upstream default is backwards compatibility + RetainExceptOnCreate: true, + }).promise(); + break; + + case RollbackChoice.CONTINUE_UPDATE_ROLLBACK: + let resourcesToSkip: string[] = options.orphanLogicalIds ?? []; + + if (options.force && resourcesToSkip.length > 0) { + throw new Error('Cannot combine --force with --orphan'); + } + if (options.force) { + // Find the failed resources from the deployment and automatically skip them + // (Using deployment log because we definitely have `DescribeStackEvents` permissions, and we might not have + // `DescribeStackResources` permissions). + const poller = new StackEventPoller(cfn, { + stackName: deployName, + stackStatuses: ['ROLLBACK_IN_PROGRESS', 'UPDATE_ROLLBACK_IN_PROGRESS'], + }); + await poller.poll(); + resourcesToSkip = poller.resourceErrors + .filter(r => !r.isStackEvent && r.parentStackLogicalIds.length === 0) + .map(r => r.event.LogicalResourceId ?? ''); + } + + const skipDescription = resourcesToSkip.length > 0 + ? ` (orphaning: ${resourcesToSkip.join(', ')})` + : ''; + warning(`Continuing rollback of stack ${deployName}${skipDescription}`); + await cfn.continueUpdateRollback({ + StackName: deployName, + ClientRequestToken: randomUUID(), + RoleARN: cloudFormationRoleArn, + ResourcesToSkip: resourcesToSkip, + }).promise(); + break; + + default: + throw new Error(`Unexpected rollback choice: ${cloudFormationStack.stackStatus.rollbackChoice}`); + } + + const monitor = options.quiet ? undefined : StackActivityMonitor.withDefaultPrinter(cfn, deployName, options.stack, { + ci: options.ci, + }).start(); + + let stackErrorMessage: string | undefined = undefined; + let finalStackState = cloudFormationStack; + try { + const successStack = await stabilizeStack(cfn, deployName); + + // This shouldn't really happen, but catch it anyway. You never know. + if (!successStack) { throw new Error('Stack deploy failed (the stack disappeared while we were rolling it back)'); } + finalStackState = successStack; + + const errors = monitor?.errors?.join(', '); + if (errors) { + stackErrorMessage = errors; + } + } catch (e: any) { + stackErrorMessage = suffixWithErrors(e.message, monitor?.errors); + } finally { + await monitor?.stop(); + } + + if (finalStackState.stackStatus.isRollbackSuccess || !stackErrorMessage) { + return { success: true }; + } + + // Either we need to ignore some resources to continue the rollback, or something went wrong + if (finalStackState.stackStatus.rollbackChoice === RollbackChoice.CONTINUE_UPDATE_ROLLBACK && options.force) { + // Do another loop-de-loop + continue; + } + + throw new Error(`${stackErrorMessage} (fix problem and retry, or orphan these resources using --orphan or --force)`);; + } + } + public async destroyStack(options: DestroyStackOptions): Promise { const { stackSdk, cloudFormationRoleArn: roleArn } = await this.prepareSdkFor(options.stack, options.roleArn, Mode.ForWriting); @@ -720,3 +901,9 @@ class ParallelSafeAssetProgress implements cdk_assets.IPublishProgressListener { */ export class CloudFormationDeployments extends Deployments { } + +function suffixWithErrors(msg: string, errors?: string[]) { + return errors && errors.length > 0 + ? `${msg}: ${errors.join(', ')}` + : msg; +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/util/cloudformation/stack-activity-monitor.ts b/packages/aws-cdk/lib/api/util/cloudformation/stack-activity-monitor.ts index 1b2422a219168..6db3b7f67941c 100644 --- a/packages/aws-cdk/lib/api/util/cloudformation/stack-activity-monitor.ts +++ b/packages/aws-cdk/lib/api/util/cloudformation/stack-activity-monitor.ts @@ -3,11 +3,11 @@ import * as cxschema from '@aws-cdk/cloud-assembly-schema'; import * as cxapi from '@aws-cdk/cx-api'; import * as aws from 'aws-sdk'; import * as chalk from 'chalk'; +import { ResourceEvent, StackEventPoller } from './stack-event-poller'; import { error, logLevel, LogLevel, setLogLevel } from '../../../logging'; import { RewritableBlock } from '../display'; -export interface StackActivity { - readonly event: aws.CloudFormation.StackEvent; +export interface StackActivity extends ResourceEvent { readonly metadata?: ResourceMetadata; } @@ -116,17 +116,13 @@ export class StackActivityMonitor { } /** - * Resource errors found while monitoring the deployment + * The poller used to read stack events */ - public readonly errors = new Array(); + public readonly poller: StackEventPoller; - private active = false; - private activity: { [eventId: string]: StackActivity } = { }; + public readonly errors: string[] = []; - /** - * Determines which events not to display - */ - private readonly startTime: number; + private active = false; /** * Current tick timer @@ -139,13 +135,16 @@ export class StackActivityMonitor { private readPromise?: Promise; constructor( - private readonly cfn: aws.CloudFormation, + cfn: aws.CloudFormation, private readonly stackName: string, private readonly printer: IActivityPrinter, private readonly stack?: cxapi.CloudFormationStackArtifact, changeSetCreationTime?: Date, ) { - this.startTime = changeSetCreationTime?.getTime() ?? Date.now(); + this.poller = new StackEventPoller(cfn, { + stackName, + startTime: changeSetCreationTime?.getTime() ?? Date.now(), + }); } public start() { @@ -221,61 +220,17 @@ export class StackActivityMonitor { * see a next page and the last event in the page is new to us (and within the time window). * haven't seen the final event */ - private async readNewEvents(stackName?: string): Promise { - const stackToPollForEvents = stackName ?? this.stackName; - const events: StackActivity[] = []; - const CFN_SUCCESS_STATUS = ['UPDATE_COMPLETE', 'CREATE_COMPLETE', 'DELETE_COMPLETE', 'DELETE_SKIPPED']; - try { - let nextToken: string | undefined; - let finished = false; - while (!finished) { - const response = await this.cfn.describeStackEvents({ StackName: stackToPollForEvents, NextToken: nextToken }).promise(); - const eventPage = response?.StackEvents ?? []; - - for (const event of eventPage) { - // Event from before we were interested in 'em - if (event.Timestamp.valueOf() < this.startTime) { - finished = true; - break; - } - - // Already seen this one - if (event.EventId in this.activity) { - finished = true; - break; - } - - // Fresh event - events.push(this.activity[event.EventId] = { - event: event, - metadata: this.findMetadataFor(event.LogicalResourceId), - }); - - if (event.ResourceType === 'AWS::CloudFormation::Stack' && !CFN_SUCCESS_STATUS.includes(event.ResourceStatus ?? '')) { - // If the event is not for `this` stack and has a physical resource Id, recursively call for events in the nested stack - if (event.PhysicalResourceId && event.PhysicalResourceId !== stackToPollForEvents) { - await this.readNewEvents(event.PhysicalResourceId); - } - } - } + private async readNewEvents(): Promise { + const pollEvents = await this.poller.poll(); - // We're also done if there's nothing left to read - nextToken = response?.NextToken; - if (nextToken === undefined) { - finished = true; - } - } - } catch (e: any) { - if (e.code === 'ValidationError' && e.message === `Stack [${stackToPollForEvents}] does not exist`) { - return; - } - throw e; - } + const activities: StackActivity[] = pollEvents.map(event => ({ + ...event, + metadata: this.findMetadataFor(event.event.LogicalResourceId), + })); - events.reverse(); - for (const event of events) { - this.checkForErrors(event); - this.printer.addActivity(event); + for (const activity of activities) { + this.checkForErrors(activity); + this.printer.addActivity(activity ); } } @@ -298,6 +253,7 @@ export class StackActivityMonitor { } private checkForErrors(activity: StackActivity) { + if (hasErrorMessage(activity.event.ResourceStatus ?? '')) { const isCancelled = (activity.event.ResourceStatusReason ?? '').indexOf('cancelled') > -1; @@ -550,7 +506,7 @@ export class HistoryActivityPrinter extends ActivityPrinterBase { this.stream.write('\nFailed resources:\n'); for (const failure of this.failures) { // Root stack failures are not interesting - if (failure.event.StackName === failure.event.LogicalResourceId) { + if (failure.isStackEvent) { continue; } @@ -707,7 +663,7 @@ export class CurrentActivityPrinter extends ActivityPrinterBase { const lines = new Array(); for (const failure of this.failures) { // Root stack failures are not interesting - if (failure.event.StackName === failure.event.LogicalResourceId) { + if (failure.isStackEvent) { continue; } diff --git a/packages/aws-cdk/lib/api/util/cloudformation/stack-event-poller.ts b/packages/aws-cdk/lib/api/util/cloudformation/stack-event-poller.ts new file mode 100644 index 0000000000000..8bc218a568ac3 --- /dev/null +++ b/packages/aws-cdk/lib/api/util/cloudformation/stack-event-poller.ts @@ -0,0 +1,172 @@ +import * as aws from 'aws-sdk'; + +export interface StackEventPollerProps { + /** + * The stack to poll + */ + readonly stackName: string; + + /** + * IDs of parent stacks of this resource, in case of resources in nested stacks + */ + readonly parentStackLogicalIds?: string[]; + + /** + * Timestamp for the oldest event we're interested in + * + * @default - Read all events + */ + readonly startTime?: number; + + /** + * Stop reading when we see the stack entering this status + * + * Should be something like `CREATE_IN_PROGRESS`, `UPDATE_IN_PROGRESS`, + * `DELETE_IN_PROGRESS, `ROLLBACK_IN_PROGRESS`. + * + * @default - Read all events + */ + readonly stackStatuses?: string[]; +} + +export interface ResourceEvent { + readonly event: aws.CloudFormation.StackEvent; + readonly parentStackLogicalIds: string[]; + + /** + * Whether this event regards the root stack + * + * @default false + */ + readonly isStackEvent?: boolean; +} + +export class StackEventPoller { + public readonly events: ResourceEvent[] = []; + public complete: boolean = false; + + private readonly eventIds = new Set(); + private readonly nestedStackPollers: Record = {}; + + constructor(private readonly cfn: aws.CloudFormation, private readonly props: StackEventPollerProps) { + } + + /** + * From all accumulated events, return only the errors + */ + public get resourceErrors(): ResourceEvent[] { + return this.events.filter(e => e.event.ResourceStatus?.endsWith('_FAILED') && !e.isStackEvent); + } + + /** + * Poll for new stack events + * + * Will not return events older than events indicated by the constructor filters. + * + * Recurses into nested stacks, and returns events old-to-new. + */ + public async poll(): Promise { + const events: ResourceEvent[] = []; + try { + let nextToken: string | undefined; + let finished = false; + while (!finished) { + const response = await this.cfn.describeStackEvents({ StackName: this.props.stackName, NextToken: nextToken }).promise(); + const eventPage = response?.StackEvents ?? []; + + for (const event of eventPage) { + // Event from before we were interested in 'em + if (this.props.startTime !== undefined && event.Timestamp.valueOf() < this.props.startTime) { + finished = true; + break; + } + + // Already seen this one + if (this.eventIds.has(event.EventId)) { + finished = true; + break; + } + this.eventIds.add(event.EventId); + + // The events for the stack itself are also included next to events about resources; we can test for them in this way. + const isParentStackEvent = event.PhysicalResourceId === event.StackId; + + if (isParentStackEvent && this.props.stackStatuses?.includes(event.ResourceStatus ?? '')) { + finished = true; + break; + } + + // Fresh event + const resEvent: ResourceEvent = { + event: event, + parentStackLogicalIds: this.props.parentStackLogicalIds ?? [], + isStackEvent: isParentStackEvent, + }; + events.push(resEvent); + + if (!isParentStackEvent && event.ResourceType === 'AWS::CloudFormation::Stack' && isStackBeginOperationState(event.ResourceStatus)) { + // If the event is not for `this` stack and has a physical resource Id, recursively call for events in the nested stack + this.trackNestedStack(event, [...this.props.parentStackLogicalIds ?? [], event.LogicalResourceId ?? '']); + } + + if (isParentStackEvent && isStackTerminalState(event.ResourceStatus)) { + this.complete = true; + } + } + + // We're also done if there's nothing left to read + nextToken = response?.NextToken; + if (nextToken === undefined) { + finished = true; + } + } + } catch (e: any) { + if (e.code === 'ValidationError' && e.message === `Stack [${this.props.stackName}] does not exist`) { + // Ignore + } else { + throw e; + } + } + + // Also poll all nested stacks we're currently tracking + for (const [logicalId, poller] of Object.entries(this.nestedStackPollers)) { + events.push(...await poller.poll()); + if (poller.complete) { + delete this.nestedStackPollers[logicalId]; + } + } + + // Return what we have so far + events.sort((a, b) => a.event.Timestamp.valueOf() - b.event.Timestamp.valueOf()); + this.events.push(...events); + return events; + } + + /** + * On the CREATE_IN_PROGRESS, UPDATE_IN_PROGRESS, DELETE_IN_PROGRESS event of a nested stack, poll the nested stack updates + */ + private trackNestedStack(event: aws.CloudFormation.StackEvent, parentStackLogicalIds: string[]) { + const logicalId = event.LogicalResourceId ?? ''; + if (!this.nestedStackPollers[logicalId]) { + this.nestedStackPollers[logicalId] = new StackEventPoller(this.cfn, { + stackName: event.PhysicalResourceId ?? '', + parentStackLogicalIds: parentStackLogicalIds, + startTime: event.Timestamp.valueOf(), + }); + } + } +} + +function isStackBeginOperationState(state: string | undefined) { + return [ + 'CREATE_IN_PROGRESS', + 'UPDATE_IN_PROGRESS', + 'DELETE_IN_PROGRESS', + 'UPDATE_ROLLBACK_IN_PROGRESS', + 'ROLLBACK_IN_PROGRESS', + ].includes(state ?? ''); +} + +function isStackTerminalState(state: string | undefined) { + return !(state ?? '').endsWith('_IN_PROGRESS'); +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts b/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts index 473858b4bac18..f2fdff66c3360 100644 --- a/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts +++ b/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts @@ -46,7 +46,37 @@ export class StackStatus { || this.name === 'UPDATE_ROLLBACK_COMPLETE'; } + /** + * Whether the stack is in a paused state due to `--no-rollback`. + * + * The possible actions here are retrying a new `--no-rollback` deployment, or initiating a rollback. + */ + get rollbackChoice(): RollbackChoice { + switch (this.name) { + case 'CREATE_FAILED': + case 'UPDATE_FAILED': + return RollbackChoice.START_ROLLBACK; + case 'UPDATE_ROLLBACK_FAILED': + return RollbackChoice.CONTINUE_UPDATE_ROLLBACK; + case 'ROLLBACK_FAILED': + // Unfortunately there is no option to continue a failed rollback without + // a stable target state. + return RollbackChoice.NONE; + default: + return RollbackChoice.NONE; + } + } + public toString(): string { return this.name + (this.reason ? ` (${this.reason})` : ''); } } + +/** + * Describe the current rollback options for this state + */ +export enum RollbackChoice { + START_ROLLBACK, + CONTINUE_UPDATE_ROLLBACK, + NONE, +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index af64056e2fc29..ad6d05b6f8447 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -433,6 +433,40 @@ export class CdkToolkit { } } + /** + * Roll back the given stack or stacks. + */ + public async rollback(options: RollbackOptions) { + const startSynthTime = new Date().getTime(); + const stackCollection = await this.selectStacksForDeploy(options.selector, true); + const elapsedSynthTime = new Date().getTime() - startSynthTime; + print('\n✨ Synthesis time: %ss\n', formatTime(elapsedSynthTime)); + + if (stackCollection.stackCount === 0) { + // eslint-disable-next-line no-console + console.error('No stacks selected'); + return; + } + + for (const stack of stackCollection.stackArtifacts) { + print('Rolling back %s', chalk.bold(stack.displayName)); + const startRollbackTime = new Date().getTime(); + try { + await this.props.deployments.rollbackStack({ + stack, + roleArn: options.roleArn, + toolkitStackName: options.toolkitStackName, + force: options.force, + }); + const elapsedRollbackTime = new Date().getTime() - startRollbackTime; + print('\n✨ Rollback time: %ss\n', formatTime(elapsedRollbackTime)); + } catch (e: any) { + error('\n ❌ %s failed: %s', chalk.bold(stack.displayName), e.message); + throw new Error('Rollback failed (use --force to orphan failing resources)'); + } + } + } + public async watch(options: WatchOptions) { const rootDir = path.dirname(path.resolve(PROJECT_CONFIG)); debug("root directory used for 'watch' is: %s", rootDir); @@ -1348,6 +1382,34 @@ export interface DeployOptions extends CfnDeployOptions, WatchOptions { readonly ignoreNoStacks?: boolean; } +export interface RollbackOptions { + /** + * Criteria for selecting stacks to deploy + */ + readonly selector: StackSelector; + + /** + * Name of the toolkit stack to use/deploy + * + * @default CDKToolkit + */ + readonly toolkitStackName?: string; + + /** + * Role to pass to CloudFormation for deployment + * + * @default - Default stack role + */ + readonly roleArn?: string; + + /** + * Whether to force the rollback or not + * + * @default false + */ + readonly force?: boolean; +} + export interface ImportOptions extends CfnDeployOptions { /** * Build a physical resource mapping and write it to the given file, without performing the actual import operation diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index ac7be73f2f019..5d7de4912d2fb 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -176,6 +176,23 @@ async function parseCommandLineArguments(args: string[]) { .option('asset-prebuild', { type: 'boolean', desc: 'Whether to build all assets before deploying the first stack (useful for failing Docker builds)', default: true }) .option('ignore-no-stacks', { type: 'boolean', desc: 'Whether to deploy if the app contains no stacks', default: false }), ) + .command('rollback [STACKS..]', 'Rolls back the stack(s) named STACKS to their last stable state', (yargs: Argv) => yargs + .option('all', { type: 'boolean', default: false, desc: 'Roll back all available stacks' }) + .option('toolkit-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack to create', requiresArg: true }) + .option('force', { + alias: 'f', + type: 'boolean', + desc: 'Orphan all resources for which the rollback operation fails.', + }) + .option('orphan', { + // alias: 'o' conflicts with --output + type: 'array', + nargs: 1, + requiresArg: true, + desc: 'Orphan the given resources, identified by their logical ID (can be specified multiple times)', + default: [], + }), + ) .command('import [STACK]', 'Import existing resource(s) into the given STACK', (yargs: Argv) => yargs .option('execute', { type: 'boolean', desc: 'Whether to execute ChangeSet (--no-execute will NOT execute the ChangeSet)', default: true }) .option('change-set-name', { type: 'string', desc: 'Name of the CloudFormation change set to create' }) @@ -615,6 +632,14 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise { HookType: 'hook1', HookStatusReason: 'stack1 must obey certain rules', }, + parentStackLogicalIds: [], }); historyActivityPrinter.addActivity({ event: { @@ -254,6 +265,7 @@ test('print failed resources because of hook failures', () => { StackName: 'stack-name', ResourceStatusReason: 'The following hook(s) failed: hook1', }, + parentStackLogicalIds: [], }); historyActivityPrinter.stop(); }); From c16c7a86b3e36bb772e8300ada2bb3cfa58db0af Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 12 Sep 2024 11:54:59 +0200 Subject: [PATCH 02/11] Fix tests --- .../aws-cdk/test/util/stack-monitor.test.ts | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/aws-cdk/test/util/stack-monitor.test.ts b/packages/aws-cdk/test/util/stack-monitor.test.ts index e32098a03a9ee..e0d9bb673fbd7 100644 --- a/packages/aws-cdk/test/util/stack-monitor.test.ts +++ b/packages/aws-cdk/test/util/stack-monitor.test.ts @@ -124,11 +124,20 @@ describe('stack monitor, collecting errors from events', () => { return { StackEvents: [ addErrorToStackEvent( - event(100), { + event(102), { logicalResourceId: 'nestedStackLogicalResourceId', physicalResourceId: 'nestedStackPhysicalResourceId', resourceType: 'AWS::CloudFormation::Stack', resourceStatusReason: 'nested stack failed', + resourceStatus: 'UPDATE_FAILED', + }, + ), + addErrorToStackEvent( + event(100), { + logicalResourceId: 'nestedStackLogicalResourceId', + physicalResourceId: 'nestedStackPhysicalResourceId', + resourceType: 'AWS::CloudFormation::Stack', + resourceStatus: 'UPDATE_IN_PROGRESS', }, ), ], @@ -253,18 +262,28 @@ async function testMonitorWithEventCalls( let describeStackEvents = (jest.fn() as jest.Mock); let finished = false; + let error: Error | undefined = undefined; for (const invocation of beforeStopInvocations) { const invocation_ = invocation; // Capture loop variable in local because of closure semantics const isLast = invocation === beforeStopInvocations[beforeStopInvocations.length - 1]; describeStackEvents = describeStackEvents.mockImplementationOnce(request => { - const ret = invocation_(request); - if (isLast) { + try { + const ret = invocation_(request); + if (isLast) { + finished = true; + } + return ret; + } catch (e: any) { finished = true; + error = e; + throw e; } - return ret; }); } + if (error) { + throw error; + } for (const invocation of afterStopInvocations) { describeStackEvents = describeStackEvents.mockImplementationOnce(invocation); } From 4245ff5eadf292814ddc2d776cde3d2280ed35b6 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Wed, 25 Sep 2024 14:37:30 +0200 Subject: [PATCH 03/11] Unit tests --- packages/aws-cdk/lib/api/deployments.ts | 21 +- .../api/cloudformation-deployments.test.ts | 334 ++++++++++++------ .../test/api/fake-cloudformation-stack.ts | 13 +- packages/aws-cdk/test/cdk-toolkit.test.ts | 35 +- packages/aws-cdk/test/util/mock-sdk.ts | 14 +- 5 files changed, 289 insertions(+), 128 deletions(-) diff --git a/packages/aws-cdk/lib/api/deployments.ts b/packages/aws-cdk/lib/api/deployments.ts index 1ae3317910340..1003238694b2c 100644 --- a/packages/aws-cdk/lib/api/deployments.ts +++ b/packages/aws-cdk/lib/api/deployments.ts @@ -270,6 +270,13 @@ export interface RollbackStackOptions { * the resource currently being deployed. */ readonly progress?: StackActivityProgress; + + /** + * Whether to validate the version of the bootstrap stack permissions + * + * @default true + */ + readonly validateBootstrapStackVersion?: boolean; } export interface RollbackStackResult { @@ -494,12 +501,14 @@ export class Deployments { envResources, } = await this.prepareSdkFor(options.stack, options.roleArn, Mode.ForWriting); - // Do a verification of the bootstrap stack version - await this.validateBootstrapStackVersion( - options.stack.stackName, - BOOTSTRAP_STACK_VERSION_FOR_ROLLBACK, - options.stack.bootstrapStackVersionSsmParameter, - envResources); + if (options.validateBootstrapStackVersion ?? true) { + // Do a verification of the bootstrap stack version + await this.validateBootstrapStackVersion( + options.stack.stackName, + BOOTSTRAP_STACK_VERSION_FOR_ROLLBACK, + options.stack.bootstrapStackVersionSsmParameter, + envResources); + } const cfn = stackSdk.cloudFormation(); const deployName = options.stack.stackName; diff --git a/packages/aws-cdk/test/api/cloudformation-deployments.test.ts b/packages/aws-cdk/test/api/cloudformation-deployments.test.ts index cbaf7c3d8746c..423293922de00 100644 --- a/packages/aws-cdk/test/api/cloudformation-deployments.test.ts +++ b/packages/aws-cdk/test/api/cloudformation-deployments.test.ts @@ -10,13 +10,16 @@ import { HotswapMode } from '../../lib/api/hotswap/common'; import { ToolkitInfo } from '../../lib/api/toolkit-info'; import { CloudFormationStack } from '../../lib/api/util/cloudformation'; import { testStack } from '../util'; -import { mockBootstrapStack, MockSdkProvider } from '../util/mock-sdk'; +import { mockBootstrapStack, MockedHandlerType, MockSdkProvider } from '../util/mock-sdk'; let sdkProvider: MockSdkProvider; let deployments: Deployments; let mockToolkitInfoLookup: jest.Mock; let currentCfnStackResources: { [key: string]: CloudFormation.StackResourceSummary[] }; let numberOfTimesListStackResourcesWasCalled: number; +let mockRollbackStack: MockedHandlerType = jest.fn(); +let mockContinueUpdateRollback: MockedHandlerType = jest.fn(); +let mockDescribeStackEvents: MockedHandlerType = jest.fn(); beforeEach(() => { jest.resetAllMocks(); sdkProvider = new MockSdkProvider(); @@ -35,6 +38,9 @@ beforeEach(() => { StackResourceSummaries: stackResources, }; }, + rollbackStack: mockRollbackStack, + continueUpdateRollback: mockContinueUpdateRollback, + describeStackEvents: mockDescribeStackEvents, }); ToolkitInfo.lookup = mockToolkitInfoLookup = jest.fn().mockResolvedValue(ToolkitInfo.bootstrapStackNotFoundInfo('TestBootstrapStack')); @@ -331,90 +337,76 @@ test('readCurrentTemplateWithNestedStacks() can handle non-Resources in the temp }); test('readCurrentTemplateWithNestedStacks() with a 3-level nested + sibling structure works', async () => { - const cfnStack = new FakeCloudformationStack({ - stackName: 'MultiLevelRoot', - stackId: 'StackId', - }); - CloudFormationStack.lookup = (async (_, stackName: string) => { - switch (stackName) { - case 'MultiLevelRoot': - cfnStack.template = async () => ({ - Resources: { - NestedStack: { - Type: 'AWS::CloudFormation::Stack', - Properties: { - TemplateURL: 'https://www.magic-url.com', - }, - Metadata: { - 'aws:asset:path': 'one-resource-two-stacks-stack.nested.template.json', - }, + givenStacks({ + MultiLevelRoot: { + template: { + Resources: { + NestedStack: { + Type: 'AWS::CloudFormation::Stack', + Properties: { + TemplateURL: 'https://www.magic-url.com', + }, + Metadata: { + 'aws:asset:path': 'one-resource-two-stacks-stack.nested.template.json', }, }, - }); - break; - - case 'NestedStack': - cfnStack.template = async () => ({ - Resources: { - SomeResource: { - Type: 'AWS::Something', - Properties: { - Property: 'old-value', - }, + }, + }, + }, + NestedStack: { + template: { + Resources: { + SomeResource: { + Type: 'AWS::Something', + Properties: { + Property: 'old-value', }, - GrandChildStackA: { - Type: 'AWS::CloudFormation::Stack', - Properties: { - TemplateURL: 'https://www.magic-url.com', - }, - Metadata: { - 'aws:asset:path': 'one-resource-stack.nested.template.json', - }, + }, + GrandChildStackA: { + Type: 'AWS::CloudFormation::Stack', + Properties: { + TemplateURL: 'https://www.magic-url.com', }, - GrandChildStackB: { - Type: 'AWS::CloudFormation::Stack', - Properties: { - TemplateURL: 'https://www.magic-url.com', - }, - Metadata: { - 'aws:asset:path': 'one-resource-stack.nested.template.json', - }, + Metadata: { + 'aws:asset:path': 'one-resource-stack.nested.template.json', }, }, - }); - break; - - case 'GrandChildStackA': - cfnStack.template = async () => ({ - Resources: { - SomeResource: { - Type: 'AWS::Something', - Properties: { - Property: 'old-value', - }, + GrandChildStackB: { + Type: 'AWS::CloudFormation::Stack', + Properties: { + TemplateURL: 'https://www.magic-url.com', + }, + Metadata: { + 'aws:asset:path': 'one-resource-stack.nested.template.json', }, }, - }); - break; - - case 'GrandChildStackB': - cfnStack.template = async () => ({ - Resources: { - SomeResource: { - Type: 'AWS::Something', - Properties: { - Property: 'old-value', - }, + }, + }, + }, + GrandChildStackA: { + template: { + Resources: { + SomeResource: { + Type: 'AWS::Something', + Properties: { + Property: 'old-value', }, }, - }); - break; - - default: - throw new Error('unknown stack name ' + stackName + ' found in deployments.test.ts'); - } - - return cfnStack; + }, + }, + }, + GrandChildStackB: { + template: { + Resources: { + SomeResource: { + Type: 'AWS::Something', + Properties: { + Property: 'old-value', + }, + }, + }, + }, + }, }); const rootStack = testStack({ @@ -682,36 +674,31 @@ test('readCurrentTemplateWithNestedStacks() on an undeployed parent stack with a test('readCurrentTemplateWithNestedStacks() caches calls to listStackResources()', async () => { // GIVEN - const cfnStack = new FakeCloudformationStack({ - stackName: 'CachingRoot', - stackId: 'StackId', - }); - CloudFormationStack.lookup = (async (_cfn, _stackName: string) => { - cfnStack.template = async () => ({ - Resources: - { - NestedStackA: { - Type: 'AWS::CloudFormation::Stack', - Properties: { - TemplateURL: 'https://www.magic-url.com', - }, - Metadata: { - 'aws:asset:path': 'one-resource-stack.nested.template.json', - }, - }, - NestedStackB: { - Type: 'AWS::CloudFormation::Stack', - Properties: { - TemplateURL: 'https://www.magic-url.com', + givenStacks({ + '*': { + template: { + Resources: { + NestedStackA: { + Type: 'AWS::CloudFormation::Stack', + Properties: { + TemplateURL: 'https://www.magic-url.com', + }, + Metadata: { + 'aws:asset:path': 'one-resource-stack.nested.template.json', + }, }, - Metadata: { - 'aws:asset:path': 'one-resource-stack.nested.template.json', + NestedStackB: { + Type: 'AWS::CloudFormation::Stack', + Properties: { + TemplateURL: 'https://www.magic-url.com', + }, + Metadata: { + 'aws:asset:path': 'one-resource-stack.nested.template.json', + }, }, }, }, - }); - - return cfnStack; + }, }); const rootStack = testStack({ @@ -756,15 +743,112 @@ test('readCurrentTemplateWithNestedStacks() caches calls to listStackResources() expect(numberOfTimesListStackResourcesWasCalled).toEqual(1); }); -test('readCurrentTemplateWithNestedStacks() succesfully ignores stacks without metadata', async () => { +test('rollback stack assumes role if necessary', async() => { + const mockForEnvironment = jest.fn().mockImplementation(() => { return { sdk: sdkProvider.sdk }; }); + sdkProvider.forEnvironment = mockForEnvironment; + givenStacks({ + '*': { template: {} }, + }); + + await deployments.rollbackStack({ + stack: testStack({ + stackName: 'boop', + properties: { + assumeRoleArn: 'bloop:${AWS::Region}:${AWS::AccountId}', + }, + }), + validateBootstrapStackVersion: false, + }); + + expect(mockForEnvironment).toHaveBeenCalledWith(expect.anything(), expect.anything(), expect.objectContaining({ + assumeRoleArn: 'bloop:here:123456789012', + })); +}); + +test('rollback stack allows rolling back from UPDATE_FAILED', async() => { // GIVEN - const cfnStack = new FakeCloudformationStack({ - stackName: 'MetadataRoot', - stackId: 'StackId', + givenStacks({ + '*': { template: {}, stackStatus: 'UPDATE_FAILED' }, }); - CloudFormationStack.lookup = (async (_, stackName: string) => { - if (stackName === 'MetadataRoot') { - cfnStack.template = async () => ({ + + // WHEN + await deployments.rollbackStack({ + stack: testStack({ stackName: 'boop' }), + validateBootstrapStackVersion: false, + }); + + // THEN + expect(mockRollbackStack).toHaveBeenCalled(); +}); + +test('rollback stack allows continue rollback from UPDATE_ROLLBACK_FAILED', async() => { + // GIVEN + givenStacks({ + '*': { template: {}, stackStatus: 'UPDATE_ROLLBACK_FAILED' }, + }); + + // WHEN + await deployments.rollbackStack({ + stack: testStack({ stackName: 'boop' }), + validateBootstrapStackVersion: false, + }); + + // THEN + expect(mockContinueUpdateRollback).toHaveBeenCalled(); +}); + +test('rollback stack fails in UPDATE_COMPLETE state', async() => { + // GIVEN + givenStacks({ + '*': { template: {}, stackStatus: 'UPDATE_COMPLETE' }, + }); + + // WHEN + const response = await deployments.rollbackStack({ + stack: testStack({ stackName: 'boop' }), + validateBootstrapStackVersion: false, + }); + + // THEN + expect(response.notInRollbackableState).toBe(true); +}); + +test('continue rollback stack with force ignores any failed resources', async() => { + // GIVEN + givenStacks({ + '*': { template: {}, stackStatus: 'UPDATE_ROLLBACK_FAILED' }, + }); + mockDescribeStackEvents.mockReturnValue({ + StackEvents: [ + { + EventId: 'asdf', + StackId: 'stack/MyStack', + StackName: 'MyStack', + Timestamp: new Date(), + LogicalResourceId: 'Xyz', + ResourceStatus: 'UPDATE_FAILED', + }, + ], + }); + + // WHEN + await deployments.rollbackStack({ + stack: testStack({ stackName: 'boop' }), + validateBootstrapStackVersion: false, + force: true, + }); + + // THEN + expect(mockContinueUpdateRollback).toHaveBeenCalledWith(expect.objectContaining({ + ResourcesToSkip: ['Xyz'], + })); +}); + +test('readCurrentTemplateWithNestedStacks() succesfully ignores stacks without metadata', async () => { + // GIVEN + givenStacks({ + 'MetadataRoot': { + template: { Resources: { WithMetadata: { Type: 'AWS::CloudFormation::Stack', @@ -776,10 +860,10 @@ test('readCurrentTemplateWithNestedStacks() succesfully ignores stacks without m }, }, }, - }); - - } else { - cfnStack.template = async () => ({ + }, + }, + '*': { + template: { Resources: { SomeResource: { Type: 'AWS::Something', @@ -788,10 +872,8 @@ test('readCurrentTemplateWithNestedStacks() succesfully ignores stacks without m }, }, }, - }); - } - - return cfnStack; + }, + }, }); const rootStack = testStack({ @@ -918,3 +1000,23 @@ function stackSummaryOf(logicalId: string, resourceType: string, physicalResourc LastUpdatedTimestamp: new Date(), }; } + +function givenStacks(stacks: Record) { + jest.spyOn(CloudFormationStack, 'lookup').mockImplementation(async (_, stackName) => { + let stack = stacks[stackName]; + if (!stack) { + stack = stacks['*']; + } + if (stack) { + const cfnStack = new FakeCloudformationStack({ + stackName, + stackId: `stack/${stackName}`, + stackStatus: stack.stackStatus, + }); + cfnStack.setTemplate(stack.template); + return cfnStack; + } else { + return new FakeCloudformationStack({ stackName }); + } + }); +} \ No newline at end of file diff --git a/packages/aws-cdk/test/api/fake-cloudformation-stack.ts b/packages/aws-cdk/test/api/fake-cloudformation-stack.ts index 1668ea0b55d33..2590b914dcdd3 100644 --- a/packages/aws-cdk/test/api/fake-cloudformation-stack.ts +++ b/packages/aws-cdk/test/api/fake-cloudformation-stack.ts @@ -2,10 +2,12 @@ import { CloudFormation } from 'aws-sdk'; import { CloudFormationStack, Template } from '../../lib/api/util/cloudformation'; import { instanceMockFrom } from '../util'; +import { StackStatus } from '../../lib/api/util/cloudformation/stack-status'; export interface FakeCloudFormationStackProps { readonly stackName: string; - readonly stackId: string; + readonly stackId?: string; + readonly stackStatus?: string; } export class FakeCloudformationStack extends CloudFormationStack { @@ -29,7 +31,12 @@ export class FakeCloudformationStack extends CloudFormationStack { return Promise.resolve(this.__template); } - public get stackId(): string { - return this.props.stackId; + public get exists() { + return this.props.stackId !== undefined; + } + + public get stackStatus() { + const status = this.props.stackStatus ?? 'UPDATE_COMPLETE'; + return new StackStatus(status, 'The test said so'); } } diff --git a/packages/aws-cdk/test/cdk-toolkit.test.ts b/packages/aws-cdk/test/cdk-toolkit.test.ts index f67c35ad8dae7..dcbb555812397 100644 --- a/packages/aws-cdk/test/cdk-toolkit.test.ts +++ b/packages/aws-cdk/test/cdk-toolkit.test.ts @@ -64,13 +64,15 @@ import { instanceMockFrom, MockCloudExecutable, TestStackArtifact } from './util import { MockSdkProvider } from './util/mock-sdk'; import { Bootstrapper } from '../lib/api/bootstrap'; import { DeployStackResult } from '../lib/api/deploy-stack'; -import { Deployments, DeployStackOptions, DestroyStackOptions } from '../lib/api/deployments'; +import { Deployments, DeployStackOptions, DestroyStackOptions, RollbackStackOptions, RollbackStackResult } from '../lib/api/deployments'; import { HotswapMode } from '../lib/api/hotswap/common'; import { Template } from '../lib/api/util/cloudformation'; import { CdkToolkit, Tag } from '../lib/cdk-toolkit'; import { RequireApproval } from '../lib/diff'; import { Configuration } from '../lib/settings'; import { flatten } from '../lib/util'; +import { mocked } from 'jest-mock'; +import { SdkProvider } from '../lib'; process.env.CXAPI_DISABLE_SELECT_BY_ID = '1'; @@ -1224,6 +1226,31 @@ describe('synth', () => { expect(mockData.mock.calls.length).toEqual(1); expect(mockData.mock.calls[0][0]).toBeDefined(); }); + + test('rollback uses deployment role', async () => { + cloudExecutable = new MockCloudExecutable({ + stacks: [ + MockStack.MOCK_STACK_C, + ], + }); + + const mockedRollback = jest.spyOn(Deployments.prototype, 'rollbackStack').mockResolvedValue({ + success: true, + }); + + const toolkit = new CdkToolkit({ + cloudExecutable, + configuration: cloudExecutable.configuration, + sdkProvider: cloudExecutable.sdkProvider, + deployments: new Deployments({ sdkProvider: new MockSdkProvider() }), + }); + + await toolkit.rollback({ + selector: { patterns: [] }, + }); + + expect(mockedRollback).toHaveBeenCalled(); + }); }); class MockStack { @@ -1400,6 +1427,12 @@ class FakeCloudFormation extends Deployments { }); } + public rollbackStack(_options: RollbackStackOptions): Promise { + return Promise.resolve({ + success: true, + }); + } + public destroyStack(options: DestroyStackOptions): Promise { expect(options.stack).toBeDefined(); return Promise.resolve(); diff --git a/packages/aws-cdk/test/util/mock-sdk.ts b/packages/aws-cdk/test/util/mock-sdk.ts index 0d943fadb3dea..d280ef9f02942 100644 --- a/packages/aws-cdk/test/util/mock-sdk.ts +++ b/packages/aws-cdk/test/util/mock-sdk.ts @@ -269,11 +269,21 @@ type AwsCallInputOutput = // Determine the type of the mock handler from the type of the Input/Output type pair. // Don't need to worry about the 'never', TypeScript will propagate it upwards making it // impossible to specify the field that has 'never' anywhere in its type. -type MockHandlerType = +type HandlerType = AI extends [any, any] ? (input: AI[0]) => AI[1] : AI; // Any subset of the full type that synchronously returns the output structure is okay -export type SyncHandlerSubsetOf = {[K in keyof S]?: MockHandlerType>}; +export type SyncHandlerSubsetOf = {[K in keyof S]?: HandlerType>}; + +/** + * A jest Mock function we can pass into SdkProvider.stubXXX + * + * Use as follows: + * + * ```ts + * const mockDescribeStackEvents: MockedHandlerType = jest.fn(); + */ +export type MockedHandlerType = AwsCallInputOutput extends [infer IN, infer OUT] ? jest.Mock : never; /** * Fake AWS response. From be2231931410a2b3f442b6ca54074ec7f017e5a3 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Wed, 25 Sep 2024 14:53:30 +0200 Subject: [PATCH 04/11] Remove unused imports --- packages/aws-cdk/test/cdk-toolkit.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/aws-cdk/test/cdk-toolkit.test.ts b/packages/aws-cdk/test/cdk-toolkit.test.ts index dcbb555812397..d2c46cc15698d 100644 --- a/packages/aws-cdk/test/cdk-toolkit.test.ts +++ b/packages/aws-cdk/test/cdk-toolkit.test.ts @@ -71,8 +71,6 @@ import { CdkToolkit, Tag } from '../lib/cdk-toolkit'; import { RequireApproval } from '../lib/diff'; import { Configuration } from '../lib/settings'; import { flatten } from '../lib/util'; -import { mocked } from 'jest-mock'; -import { SdkProvider } from '../lib'; process.env.CXAPI_DISABLE_SELECT_BY_ID = '1'; From c6d9b19737a62d2f9d36e9454a87b5b4a555fb06 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 26 Sep 2024 14:41:12 +0200 Subject: [PATCH 05/11] Fix fake class incompatibilities --- .../aws-cdk/test/api/fake-cloudformation-stack.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/aws-cdk/test/api/fake-cloudformation-stack.ts b/packages/aws-cdk/test/api/fake-cloudformation-stack.ts index 2590b914dcdd3..918d6c4d5bb37 100644 --- a/packages/aws-cdk/test/api/fake-cloudformation-stack.ts +++ b/packages/aws-cdk/test/api/fake-cloudformation-stack.ts @@ -39,4 +39,15 @@ export class FakeCloudformationStack extends CloudFormationStack { const status = this.props.stackStatus ?? 'UPDATE_COMPLETE'; return new StackStatus(status, 'The test said so'); } + + public get stackId() { + if (!this.props.stackId) { + throw new Error('Cannot retrieve stackId from a non-existent stack'); + } + return this.props.stackId; + } + + public get outputs(): Record { + return {}; + } } From bf7bef3aae1f5d953c5e6364373d3ce38182020f Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 1 Oct 2024 15:06:18 +0200 Subject: [PATCH 06/11] Initial set of review comments --- .../tests/cli-integ-tests/cli.integtest.ts | 10 +++---- packages/aws-cdk/README.md | 8 ++++-- .../lib/api/bootstrap/bootstrap-template.yaml | 2 +- packages/aws-cdk/lib/api/deployments.ts | 15 +++++------ packages/aws-cdk/lib/cdk-toolkit.ts | 26 ++++++++++++++++++- packages/aws-cdk/lib/cli.ts | 8 +++++- 6 files changed, 50 insertions(+), 19 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts index dbe457bb75f43..ae3ec9584a6e8 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts @@ -2284,9 +2284,8 @@ integTest( verbose: false, allowErrExit: true, }); - if (!deployOutput.includes('UPDATE_FAILED')) { - throw new Error(`Expected output to contain UPDATE_FAILED, got: ${deployOutput}`); - } + + expect(deployOutput).toContain('UPDATE_FAILED'); // Should still fail const rollbackOutput = await fixture.cdk(['rollback'], { @@ -2294,9 +2293,8 @@ integTest( verbose: false, allowErrExit: true, }); - if (!rollbackOutput.includes('Failing rollback')) { - throw new Error(`Expected output to contain "Failing rollback", got: ${rollbackOutput}`); - } + + expect(rollbackOutput).toContain('Failing rollback'); // Rollback and force cleanup await fixture.cdk(['rollback', '--force'], { diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index 2f246ef820482..77425f7d06e3f 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -482,8 +482,12 @@ To roll the deployment back, use `cdk rollback`. This will initiate a rollback to the last stable state of your stack. Some resources may fail to roll back. If they do, you can try again by calling -`cdk rollback --orphan `. Or, run `cdk rollback --force` to have -the CDK CLI automatically orphan all failing resources. +`cdk rollback --orphan ` (can be specified multiple times). Or, run +`cdk rollback --force` to have the CDK CLI automatically orphan all failing +resources. + +(`cdk rollback` requires version 23 of the bootstrap stack, since it depends on +new permissions necessary to call the appropriate CloudFormation APIs) ### `cdk watch` diff --git a/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml b/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml index d48f850d52e99..ad71c39535426 100644 --- a/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml +++ b/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml @@ -653,7 +653,7 @@ Resources: Type: String Name: Fn::Sub: '/cdk-bootstrap/${Qualifier}/version' - Value: '22' + Value: '23' Outputs: BucketName: Description: The name of the S3 bucket owned by the CDK toolkit stack diff --git a/packages/aws-cdk/lib/api/deployments.ts b/packages/aws-cdk/lib/api/deployments.ts index 1003238694b2c..db60548cb20b4 100644 --- a/packages/aws-cdk/lib/api/deployments.ts +++ b/packages/aws-cdk/lib/api/deployments.ts @@ -19,7 +19,7 @@ import { replaceEnvPlaceholders } from './util/placeholders'; import { makeBodyParameterAndUpload } from './util/template-body-parameter'; import { buildAssets, publishAssets, BuildAssetsOptions, PublishAssetsOptions, PublishingAws, EVENT_TO_LOGGER } from '../util/asset-publishing'; -const BOOTSTRAP_STACK_VERSION_FOR_ROLLBACK = 22; +const BOOTSTRAP_STACK_VERSION_FOR_ROLLBACK = 23; /** * SDK obtained by assuming the lookup role @@ -494,6 +494,11 @@ export class Deployments { } public async rollbackStack(options: RollbackStackOptions): Promise { + let resourcesToSkip: string[] = options.orphanLogicalIds ?? []; + if (options.force && resourcesToSkip.length > 0) { + throw new Error('Cannot combine --force with --orphan'); + } + const { stackSdk, resolvedEnvironment: _, @@ -513,11 +518,10 @@ export class Deployments { const cfn = stackSdk.cloudFormation(); const deployName = options.stack.stackName; + // We loop in case of `--force` and the stack ends up in `CONTINUE_UPDATE_ROLLBACK`. while (true) { let cloudFormationStack = await CloudFormationStack.lookup(cfn, deployName); - // TODO: currently, we only roll back UPDATE_FAILED and CREATE_FAILED stacks. Conceivably, we could also - // handle UPDATE_ROLLBACK_FAILED by offering some option like "force rollback". switch (cloudFormationStack.stackStatus.rollbackChoice) { case RollbackChoice.NONE: warning(`Stack ${deployName} does not need a rollback: ${cloudFormationStack.stackStatus}`); @@ -535,11 +539,6 @@ export class Deployments { break; case RollbackChoice.CONTINUE_UPDATE_ROLLBACK: - let resourcesToSkip: string[] = options.orphanLogicalIds ?? []; - - if (options.force && resourcesToSkip.length > 0) { - throw new Error('Cannot combine --force with --orphan'); - } if (options.force) { // Find the failed resources from the deployment and automatically skip them // (Using deployment log because we definitely have `DescribeStackEvents` permissions, and we might not have diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 3de04a24644c7..10f861238b7c0 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -449,16 +449,23 @@ export class CdkToolkit { return; } + let anyRollbackable = false; + for (const stack of stackCollection.stackArtifacts) { print('Rolling back %s', chalk.bold(stack.displayName)); const startRollbackTime = new Date().getTime(); try { - await this.props.deployments.rollbackStack({ + const result = await this.props.deployments.rollbackStack({ stack, roleArn: options.roleArn, toolkitStackName: options.toolkitStackName, force: options.force, + validateBootstrapStackVersion: options.validateBootstrapStackVersion, + orphanLogicalIds: options.orphanLogicalIds, }); + if (!result.notInRollbackableState) { + anyRollbackable = true; + } const elapsedRollbackTime = new Date().getTime() - startRollbackTime; print('\n✨ Rollback time: %ss\n', formatTime(elapsedRollbackTime)); } catch (e: any) { @@ -466,6 +473,9 @@ export class CdkToolkit { throw new Error('Rollback failed (use --force to orphan failing resources)'); } } + if (!anyRollbackable) { + throw new Error('No stacks were in a state that could be rolled back'); + } } public async watch(options: WatchOptions) { @@ -1409,6 +1419,20 @@ export interface RollbackOptions { * @default false */ readonly force?: boolean; + + /** + * Logical IDs of resources to orphan + * + * @default - No orphaning + */ + readonly orphanLogicalIds?: string[]; + + /** + * Whether to validate the version of the bootstrap stack permissions + * + * @default true + */ + readonly validateBootstrapStackVersion?: boolean; } export interface ImportOptions extends CfnDeployOptions { diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index 56d6fb2c546f1..b5d79dfd1fa62 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -178,12 +178,16 @@ async function parseCommandLineArguments(args: string[]) { ) .command('rollback [STACKS..]', 'Rolls back the stack(s) named STACKS to their last stable state', (yargs: Argv) => yargs .option('all', { type: 'boolean', default: false, desc: 'Roll back all available stacks' }) - .option('toolkit-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack to create', requiresArg: true }) + .option('toolkit-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack the environment is bootstrapped with', requiresArg: true }) .option('force', { alias: 'f', type: 'boolean', desc: 'Orphan all resources for which the rollback operation fails.', }) + .option('validate-bootstrap-version', { + type: 'boolean', + desc: 'Whether to validate the bootstrap stack version. Defaults to \'true\', disable with --no-validate-bootstrap-version.', + }) .option('orphan', { // alias: 'o' conflicts with --output type: 'array', @@ -638,6 +642,8 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise Date: Tue, 1 Oct 2024 15:15:24 +0200 Subject: [PATCH 07/11] Add integ test without --force --- .../cdk-apps/rollback-test-app/app.js | 6 +++- .../tests/cli-integ-tests/cli.integtest.ts | 36 ++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js index c2d772ec3ff47..35ba19dd5ca38 100644 --- a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js @@ -36,7 +36,11 @@ class RollbacktestStack extends cdk.Stack { case '1': // Normal deployment break; - case '2': + case '2a': + // r1 updates normally, r2 fails updating + r2props.FailUpdate = true; + break; + case '2b': // r1 updates normally, r2 fails updating, r1 fails rollback r1props.FailRollback = true; r2props.FailUpdate = true; diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts index 4c312c923e586..4bf11212617d2 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts @@ -2273,7 +2273,41 @@ integTest( verbose: false, }); try { - phase = '2'; + phase = '2a'; + + // Should fail + const deployOutput = await fixture.cdkDeploy('test-rollback', { + options: ['--no-rollback'], + modEnv: { PHASE: phase }, + verbose: false, + allowErrExit: true, + }); + expect(deployOutput).toContain('UPDATE_FAILED'); + + // Rollback + await fixture.cdk(['rollback'], { + modEnv: { PHASE: phase }, + verbose: false, + }); + } finally { + await fixture.cdkDestroy('test-rollback'); + } + }), +); + +integTest( + 'test cdk rollback --force', + withSpecificFixture('rollback-test-app', async (fixture) => { + let phase = '1'; + + // Should succeed + await fixture.cdkDeploy('test-rollback', { + options: ['--no-rollback'], + modEnv: { PHASE: phase }, + verbose: false, + }); + try { + phase = '2b'; // Fail update and also fail rollback // Should fail const deployOutput = await fixture.cdkDeploy('test-rollback', { From 453bb8914067c25a010e2f423110950d16c2f881 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 1 Oct 2024 15:17:37 +0200 Subject: [PATCH 08/11] Prevent infinite loop --- packages/aws-cdk/lib/api/deployments.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/aws-cdk/lib/api/deployments.ts b/packages/aws-cdk/lib/api/deployments.ts index 59a07d097cefc..fe9cdf37d8809 100644 --- a/packages/aws-cdk/lib/api/deployments.ts +++ b/packages/aws-cdk/lib/api/deployments.ts @@ -520,7 +520,8 @@ export class Deployments { const deployName = options.stack.stackName; // We loop in case of `--force` and the stack ends up in `CONTINUE_UPDATE_ROLLBACK`. - while (true) { + let maxLoops = 10; + while (maxLoops--) { let cloudFormationStack = await CloudFormationStack.lookup(cfn, deployName); switch (cloudFormationStack.stackStatus.rollbackChoice) { @@ -605,6 +606,7 @@ export class Deployments { throw new Error(`${stackErrorMessage} (fix problem and retry, or orphan these resources using --orphan or --force)`);; } + throw new Error('Rollback did not finish after a large number of iterations; stopping because it looks like we\'re not making progress anymore. You can retry if rollback was progressing as expected.'); } public async destroyStack(options: DestroyStackOptions): Promise { From ae05b6ff946600ae5e099a476266f751a71671e5 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 1 Oct 2024 16:35:42 +0200 Subject: [PATCH 09/11] Make ROLLBACK_FAILED an error --- packages/aws-cdk/lib/api/deployments.ts | 3 +++ .../aws-cdk/lib/api/util/cloudformation/stack-status.ts | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/aws-cdk/lib/api/deployments.ts b/packages/aws-cdk/lib/api/deployments.ts index fe9cdf37d8809..8322d82232f60 100644 --- a/packages/aws-cdk/lib/api/deployments.ts +++ b/packages/aws-cdk/lib/api/deployments.ts @@ -567,6 +567,9 @@ export class Deployments { }).promise(); break; + case RollbackChoice.ROLLBACK_FAILED: + throw new Error(`Stack ${deployName} failed creation and rollback. This state cannot be rolled back. You can recreate this stack by running 'cdk deploy'.`); + default: throw new Error(`Unexpected rollback choice: ${cloudFormationStack.stackStatus.rollbackChoice}`); } diff --git a/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts b/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts index f2fdff66c3360..4dd113aaa30db 100644 --- a/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts +++ b/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts @@ -61,7 +61,7 @@ export class StackStatus { case 'ROLLBACK_FAILED': // Unfortunately there is no option to continue a failed rollback without // a stable target state. - return RollbackChoice.NONE; + return RollbackChoice.ROLLBACK_FAILED; default: return RollbackChoice.NONE; } @@ -78,5 +78,11 @@ export class StackStatus { export enum RollbackChoice { START_ROLLBACK, CONTINUE_UPDATE_ROLLBACK, + /** + * A sign that stack creation AND its rollback have failed. + * + * There is no way to recover from this, other than recreating the stack. + */ + ROLLBACK_FAILED, NONE, } \ No newline at end of file From 3d3e97d4c055adaa46a6cdd5a7fe80d8790f4a3f Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Wed, 2 Oct 2024 11:18:12 +0200 Subject: [PATCH 10/11] No error --- packages/aws-cdk/lib/api/deployments.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/aws-cdk/lib/api/deployments.ts b/packages/aws-cdk/lib/api/deployments.ts index 8322d82232f60..f3aae0bec571a 100644 --- a/packages/aws-cdk/lib/api/deployments.ts +++ b/packages/aws-cdk/lib/api/deployments.ts @@ -568,7 +568,8 @@ export class Deployments { break; case RollbackChoice.ROLLBACK_FAILED: - throw new Error(`Stack ${deployName} failed creation and rollback. This state cannot be rolled back. You can recreate this stack by running 'cdk deploy'.`); + warning(`Stack ${deployName} failed creation and rollback. This state cannot be rolled back. You can recreate this stack by running 'cdk deploy'.`); + return { notInRollbackableState: true }; default: throw new Error(`Unexpected rollback choice: ${cloudFormationStack.stackStatus.rollbackChoice}`); From 18c7ea615373d3908b2dc46b147ef7e901cab4da Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Wed, 2 Oct 2024 11:39:19 +0200 Subject: [PATCH 11/11] Update docs --- .../resources/cdk-apps/rollback-test-app/app.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js index 35ba19dd5ca38..419e30898c9bf 100644 --- a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js @@ -8,19 +8,20 @@ const cr = require('aws-cdk-lib/custom-resources'); * It contains resources r1 and r2, where r1 gets deployed first. * * - PHASE = 1: both resources deploy regularly. - * - PHASE = 2: r1 gets updated, r2 will fail to update, and r1 will fail its rollback. + * - PHASE = 2a: r1 gets updated, r2 will fail to update + * - PHASE = 2b: r1 gets updated, r2 will fail to update, and r1 will fail its rollback. * * To exercise this app: * * ``` * env PHASE=1 npx cdk deploy - * env PHASE=2 npx cdk deploy --no-rollback + * env PHASE=2b npx cdk deploy --no-rollback * # This will leave the stack in UPDATE_FAILED * - * env PHASE=2 npx cdk rollback + * env PHASE=2b npx cdk rollback * # This will start a rollback that will fail because r1 fails its rollabck * - * env PHASE=2 npx cdk rollback --force + * env PHASE=2b npx cdk rollback --force * # This will retry the rollabck and skip r1 * ``` */