feat(cli): add ability to configure hotswap properties for ECS (#30511)

### Issue #29618

### Reason for this change

We aim to speed up deployment times in our development environment by using the hotswap feature. However, our services have dependencies on each other, and the current hotswap behavior is too disruptive. 

### Description of changes

We modified the hotswap implementation for ECS services to pass the `minimumHealthyPercent` and `maximumHealthyPercent` configurable parameters. These parameters are exposed to the cli and can be passed as `--hotswap-ecs-minimum-healthy-percent <number>` and `--hotswap-ecs-maximum-healthy-percent <number>`

The implementation is careful to maintain the existing behaviour. That is, if none of the new flags is used, the current `minimumHealthyPercent = 0` and  `maximumHealthyPercent = undefined` values are used.

### Description of how you validated changes

We added a unit test validating that the correct values are passed to the task definition. We also executed using the locally built version of cdk validating that the behavior is as expected: the parameters are respected during hotswap deployments, and the existing API is maintained.

### Checklist
- [x] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md)

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts

@@ -2301,6 +2301,58 @@ integTest('hotswap deployment supports AppSync APIs with many functions',
   }),
 );
 
+integTest('hotswap ECS deployment respects properties override', withDefaultFixture(async (fixture) => {
+  // Update the CDK context with the new ECS properties
+  let ecsMinimumHealthyPercent = 100;
+  let ecsMaximumHealthyPercent = 200;
+  let cdkJson = JSON.parse(await fs.readFile(path.join(fixture.integTestDir, 'cdk.json'), 'utf8'));
+  cdkJson = {
+    ...cdkJson,
+    hotswap: {
+      ecs: {
+        minimumHealthyPercent: ecsMinimumHealthyPercent,
+        maximumHealthyPercent: ecsMaximumHealthyPercent,
+      },
+    },
+  };
+
+  await fs.writeFile(path.join(fixture.integTestDir, 'cdk.json'), JSON.stringify(cdkJson));
+
+  // GIVEN
+  const stackArn = await fixture.cdkDeploy('ecs-hotswap', {
+    captureStderr: false,
+  });
+
+  // WHEN
+  await fixture.cdkDeploy('ecs-hotswap', {
+    options: [
+      '--hotswap',
+    ],
+    modEnv: {
+      DYNAMIC_ECS_PROPERTY_VALUE: 'new value',
+    },
+  });
+
+  const describeStacksResponse = await fixture.aws.cloudFormation.send(
+    new DescribeStacksCommand({
+      StackName: stackArn,
+    }),
+  );
+
+  const clusterName = describeStacksResponse.Stacks?.[0].Outputs?.find(output => output.OutputKey == 'ClusterName')?.OutputValue!;
+  const serviceName = describeStacksResponse.Stacks?.[0].Outputs?.find(output => output.OutputKey == 'ServiceName')?.OutputValue!;
+
+  // THEN
+  const describeServicesResponse = await fixture.aws.ecs.send(
+    new DescribeServicesCommand({
+      cluster: clusterName,
+      services: [serviceName],
+    }),
+  );
+  expect(describeServicesResponse.services?.[0].deploymentConfiguration?.minimumHealthyPercent).toEqual(ecsMinimumHealthyPercent);
+  expect(describeServicesResponse.services?.[0].deploymentConfiguration?.maximumPercent).toEqual(ecsMaximumHealthyPercent);
+}));
+
 async function listChildren(parent: string, pred: (x: string) => Promise<boolean>) {
   const ret = new Array<string>();
   for (const child of await fs.readdir(parent, { encoding: 'utf-8' })) {

packages/aws-cdk/README.md

@@ -451,6 +451,19 @@ Hotswapping is currently supported for the following changes
 - VTL mapping template changes for AppSync Resolvers and Functions.
 - Schema changes for AppSync GraphQL Apis.
 
+You can optionally configure the behavior of your hotswap deployments in `cdk.json`. Currently you can only configure ECS hotswap behavior:
+
+```json
+{
+"hotswap": {
+    "ecs": {
+      "minimumHealthyPercent": 100,
+      "maximumHealthyPercent": 250
+    }
+  }
+}
+```
+
 **⚠ Note #1**: This command deliberately introduces drift in CloudFormation stacks in order to speed up deployments.
 For this reason, only use it for development purposes.
 **Never use this flag for your production deployments**!

packages/aws-cdk/lib/api/deploy-stack.ts

@@ -5,7 +5,7 @@ import * as uuid from 'uuid';
 import { ISDK, SdkProvider } from './aws-auth';
 import { EnvironmentResources } from './environment-resources';
 import { CfnEvaluationException } from './evaluate-cloudformation-template';
-import { HotswapMode, ICON } from './hotswap/common';
+import { HotswapMode, HotswapPropertyOverrides, ICON } from './hotswap/common';
 import { tryHotswapDeployment } from './hotswap-deployments';
 import { addMetadataAssetsToManifest } from '../assets';
 import { Tag } from '../cdk-toolkit';
@@ -173,6 +173,11 @@ export interface DeployStackOptions {
    */
   readonly hotswap?: HotswapMode;
 
+  /**
+   * Extra properties that configure hotswap behavior
+   */
+  readonly hotswapPropertyOverrides?: HotswapPropertyOverrides;
+
   /**
    * The extra string to append to the User-Agent header when performing AWS SDK calls.
    *
@@ -264,6 +269,7 @@ export async function deployStack(options: DeployStackOptions): Promise<DeploySt
     : templateParams.supplyAll(finalParameterValues);
 
   const hotswapMode = options.hotswap ?? HotswapMode.FULL_DEPLOYMENT;
+  const hotswapPropertyOverrides = options.hotswapPropertyOverrides ?? new HotswapPropertyOverrides();
 
   if (await canSkipDeploy(options, cloudFormationStack, stackParams.hasChanges(cloudFormationStack.parameters))) {
     debug(`${deployName}: skipping deployment (use --force to override)`);
@@ -303,7 +309,7 @@ export async function deployStack(options: DeployStackOptions): Promise<DeploySt
     // attempt to short-circuit the deployment if possible
     try {
       const hotswapDeploymentResult = await tryHotswapDeployment(
-        options.sdkProvider, stackParams.values, cloudFormationStack, stackArtifact, hotswapMode,
+        options.sdkProvider, stackParams.values, cloudFormationStack, stackArtifact, hotswapMode, hotswapPropertyOverrides,
       );
       if (hotswapDeploymentResult) {
         return hotswapDeploymentResult;

packages/aws-cdk/lib/api/deployments.ts

@@ -10,7 +10,7 @@ import { ISDK } from './aws-auth/sdk';
 import { CredentialsOptions, SdkForEnvironment, SdkProvider } from './aws-auth/sdk-provider';
 import { deployStack, DeployStackResult, destroyStack, DeploymentMethod } from './deploy-stack';
 import { EnvironmentResources, EnvironmentResourcesRegistry } from './environment-resources';
-import { HotswapMode } from './hotswap/common';
+import { HotswapMode, HotswapPropertyOverrides } from './hotswap/common';
 import { loadCurrentTemplateWithNestedStacks, loadCurrentTemplate, RootTemplateWithNestedStacks } from './nested-stack-helpers';
 import { determineAllowCrossAccountAssetPublishing } from './util/checks';
 import { CloudFormationStack, Template, ResourcesToImport, ResourceIdentifierSummaries, stabilizeStack, uploadStackTemplateAssets } from './util/cloudformation';
@@ -182,6 +182,11 @@ export interface DeployStackOptions {
    */
   readonly hotswap?: HotswapMode;
 
+  /**
+  * Properties that configure hotswap behavior
+  */
+  readonly hotswapPropertyOverrides?: HotswapPropertyOverrides;
+
   /**
    * The extra string to append to the User-Agent header when performing AWS SDK calls.
    *
@@ -498,6 +503,7 @@ export class Deployments {
       ci: options.ci,
       rollback: options.rollback,
       hotswap: options.hotswap,
+      hotswapPropertyOverrides: options.hotswapPropertyOverrides,
       extraUserAgent: options.extraUserAgent,
       resourcesToImport: options.resourcesToImport,
       overrideTemplate: options.overrideTemplate,

packages/aws-cdk/lib/api/hotswap-deployments.ts

@@ -7,7 +7,7 @@ import { EvaluateCloudFormationTemplate } from './evaluate-cloudformation-templa
 import { print } from '../logging';
 import { isHotswappableAppSyncChange } from './hotswap/appsync-mapping-templates';
 import { isHotswappableCodeBuildProjectChange } from './hotswap/code-build-projects';
-import { ICON, ChangeHotswapResult, HotswapMode, HotswappableChange, NonHotswappableChange, HotswappableChangeCandidate, ClassifiedResourceChanges, reportNonHotswappableChange, reportNonHotswappableResource } from './hotswap/common';
+import { ICON, ChangeHotswapResult, HotswapMode, HotswappableChange, NonHotswappableChange, HotswappableChangeCandidate, HotswapPropertyOverrides, ClassifiedResourceChanges, reportNonHotswappableChange, reportNonHotswappableResource } from './hotswap/common';
 import { isHotswappableEcsServiceChange } from './hotswap/ecs-services';
 import { isHotswappableLambdaFunctionChange } from './hotswap/lambda-functions';
 import { skipChangeForS3DeployCustomResourcePolicy, isHotswappableS3BucketDeploymentChange } from './hotswap/s3-bucket-deployments';
@@ -20,7 +20,10 @@ import { CloudFormationStack } from './util/cloudformation';
 const pLimit: typeof import('p-limit') = require('p-limit');
 
 type HotswapDetector = (
-  logicalId: string, change: HotswappableChangeCandidate, evaluateCfnTemplate: EvaluateCloudFormationTemplate
+  logicalId: string,
+  change: HotswappableChangeCandidate,
+  evaluateCfnTemplate: EvaluateCloudFormationTemplate,
+  hotswapPropertyOverrides: HotswapPropertyOverrides,
 ) => Promise<ChangeHotswapResult>;
 
 const RESOURCE_DETECTORS: { [key: string]: HotswapDetector } = {
@@ -62,7 +65,7 @@ const RESOURCE_DETECTORS: { [key: string]: HotswapDetector } = {
 export async function tryHotswapDeployment(
   sdkProvider: SdkProvider, assetParams: { [key: string]: string },
   cloudFormationStack: CloudFormationStack, stackArtifact: cxapi.CloudFormationStackArtifact,
-  hotswapMode: HotswapMode,
+  hotswapMode: HotswapMode, hotswapPropertyOverrides: HotswapPropertyOverrides,
 ): Promise<DeployStackResult | undefined> {
   // resolve the environment, so we can substitute things like AWS::Region in CFN expressions
   const resolvedEnv = await sdkProvider.resolveEnvironment(stackArtifact.environment);
@@ -86,7 +89,7 @@ export async function tryHotswapDeployment(
 
   const stackChanges = cfn_diff.fullDiff(currentTemplate.deployedRootTemplate, stackArtifact.template);
   const { hotswappableChanges, nonHotswappableChanges } = await classifyResourceChanges(
-    stackChanges, evaluateCfnTemplate, sdk, currentTemplate.nestedStacks,
+    stackChanges, evaluateCfnTemplate, sdk, currentTemplate.nestedStacks, hotswapPropertyOverrides,
   );
 
   logNonHotswappableChanges(nonHotswappableChanges, hotswapMode);
@@ -113,6 +116,7 @@ async function classifyResourceChanges(
   evaluateCfnTemplate: EvaluateCloudFormationTemplate,
   sdk: ISDK,
   nestedStackNames: { [nestedStackName: string]: NestedStackTemplates },
+  hotswapPropertyOverrides: HotswapPropertyOverrides,
 ): Promise<ClassifiedResourceChanges> {
   const resourceDifferences = getStackResourceDifferences(stackChanges);
 
@@ -131,7 +135,14 @@ async function classifyResourceChanges(
   // gather the results of the detector functions
   for (const [logicalId, change] of Object.entries(resourceDifferences)) {
     if (change.newValue?.Type === 'AWS::CloudFormation::Stack' && change.oldValue?.Type === 'AWS::CloudFormation::Stack') {
-      const nestedHotswappableResources = await findNestedHotswappableChanges(logicalId, change, nestedStackNames, evaluateCfnTemplate, sdk);
+      const nestedHotswappableResources = await findNestedHotswappableChanges(
+        logicalId,
+        change,
+        nestedStackNames,
+        evaluateCfnTemplate,
+        sdk,
+        hotswapPropertyOverrides,
+      );
       hotswappableResources.push(...nestedHotswappableResources.hotswappableChanges);
       nonHotswappableResources.push(...nestedHotswappableResources.nonHotswappableChanges);
 
@@ -151,7 +162,7 @@ async function classifyResourceChanges(
     const resourceType: string = hotswappableChangeCandidate.newValue.Type;
     if (resourceType in RESOURCE_DETECTORS) {
       // run detector functions lazily to prevent unhandled promise rejections
-      promises.push(() => RESOURCE_DETECTORS[resourceType](logicalId, hotswappableChangeCandidate, evaluateCfnTemplate));
+      promises.push(() => RESOURCE_DETECTORS[resourceType](logicalId, hotswappableChangeCandidate, evaluateCfnTemplate, hotswapPropertyOverrides));
     } else {
       reportNonHotswappableChange(nonHotswappableResources, hotswappableChangeCandidate, undefined, 'This resource type is not supported for hotswap deployments');
     }
@@ -233,6 +244,7 @@ async function findNestedHotswappableChanges(
   nestedStackTemplates: { [nestedStackName: string]: NestedStackTemplates },
   evaluateCfnTemplate: EvaluateCloudFormationTemplate,
   sdk: ISDK,
+  hotswapPropertyOverrides: HotswapPropertyOverrides,
 ): Promise<ClassifiedResourceChanges> {
   const nestedStack = nestedStackTemplates[logicalId];
   if (!nestedStack.physicalName) {
@@ -256,7 +268,12 @@ async function findNestedHotswappableChanges(
     nestedStackTemplates[logicalId].deployedTemplate, nestedStackTemplates[logicalId].generatedTemplate,
   );
 
-  return classifyResourceChanges(nestedDiff, evaluateNestedCfnTemplate, sdk, nestedStackTemplates[logicalId].nestedStackTemplates);
+  return classifyResourceChanges(
+    nestedDiff,
+    evaluateNestedCfnTemplate,
+    sdk,
+    nestedStackTemplates[logicalId].nestedStackTemplates,
+    hotswapPropertyOverrides);
 }
 
 /** Returns 'true' if a pair of changes is for the same resource. */

packages/aws-cdk/lib/api/hotswap/common.ts

@@ -98,6 +98,52 @@ export class HotswappableChangeCandidate {
 
 type Exclude = { [key: string]: Exclude | true }
 
+/**
+ * Represents configuration property overrides for hotswap deployments
+ */
+export class HotswapPropertyOverrides {
+  // Each supported resource type will have its own properties. Currently this is ECS
+  ecsHotswapProperties?: EcsHotswapProperties;
+
+  public constructor (ecsHotswapProperties?: EcsHotswapProperties) {
+    this.ecsHotswapProperties = ecsHotswapProperties;
+  }
+}
+
+/**
+ * Represents configuration properties for ECS hotswap deployments
+ */
+export class EcsHotswapProperties {
+  // The lower limit on the number of your service's tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount
+  readonly minimumHealthyPercent?: number;
+  // The upper limit on the number of your service's tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount
+  readonly maximumHealthyPercent?: number;
+
+  public constructor (minimumHealthyPercent?: number, maximumHealthyPercent?: number) {
+    if (minimumHealthyPercent !== undefined && minimumHealthyPercent < 0 ) {
+      throw new Error('hotswap-ecs-minimum-healthy-percent can\'t be a negative number');
+    }
+    if (maximumHealthyPercent !== undefined && maximumHealthyPercent < 0 ) {
+      throw new Error('hotswap-ecs-maximum-healthy-percent can\'t be a negative number');
+    }
+    // In order to preserve the current behaviour, when minimumHealthyPercent is not defined, it will be set to the currently default value of 0
+    if (minimumHealthyPercent == undefined) {
+      this.minimumHealthyPercent = 0;
+    } else {
+      this.minimumHealthyPercent = minimumHealthyPercent;
+    }
+    this.maximumHealthyPercent = maximumHealthyPercent;
+  }
+
+  /**
+   * Check if any hotswap properties are defined
+   * @returns true if all properties are undefined, false otherwise
+  */
+  public isEmpty(): boolean {
+    return this.minimumHealthyPercent === 0 && this.maximumHealthyPercent === undefined;
+  }
+}
+
 /**
  * This function transforms all keys (recursively) in the provided `val` object.
  *

packages/aws-cdk/lib/api/hotswap/ecs-services.ts

@@ -1,10 +1,13 @@
 import * as AWS from 'aws-sdk';
-import { ChangeHotswapResult, classifyChanges, HotswappableChangeCandidate, lowerCaseFirstCharacter, reportNonHotswappableChange, transformObjectKeys } from './common';
+import { ChangeHotswapResult, classifyChanges, HotswappableChangeCandidate, HotswapPropertyOverrides, lowerCaseFirstCharacter, reportNonHotswappableChange, transformObjectKeys } from './common';
 import { ISDK } from '../aws-auth';
 import { EvaluateCloudFormationTemplate } from '../evaluate-cloudformation-template';
 
 export async function isHotswappableEcsServiceChange(
-  logicalId: string, change: HotswappableChangeCandidate, evaluateCfnTemplate: EvaluateCloudFormationTemplate,
+  logicalId: string,
+  change: HotswappableChangeCandidate,
+  evaluateCfnTemplate: EvaluateCloudFormationTemplate,
+  hotswapPropertyOverrides: HotswapPropertyOverrides,
 ): Promise<ChangeHotswapResult> {
   // the only resource change we can evaluate here is an ECS TaskDefinition
   if (change.newValue.Type !== 'AWS::ECS::TaskDefinition') {
@@ -83,6 +86,10 @@ export async function isHotswappableEcsServiceChange(
         const registerTaskDefResponse = await sdk.ecs().registerTaskDefinition(lowercasedTaskDef).promise();
         const taskDefRevArn = registerTaskDefResponse.taskDefinition?.taskDefinitionArn;
 
+        let ecsHotswapProperties = hotswapPropertyOverrides.ecsHotswapProperties;
+        let minimumHealthyPercent = ecsHotswapProperties?.minimumHealthyPercent;
+        let maximumHealthyPercent = ecsHotswapProperties?.maximumHealthyPercent;
+
         // Step 2 - update the services using that TaskDefinition to point to the new TaskDefinition Revision
         const servicePerClusterUpdates: { [cluster: string]: Array<{ promise: Promise<any>; ecsService: EcsService }> } = {};
         for (const ecsService of ecsServicesReferencingTaskDef) {
@@ -105,7 +112,8 @@ export async function isHotswappableEcsServiceChange(
               cluster: clusterName,
               forceNewDeployment: true,
               deploymentConfiguration: {
-                minimumHealthyPercent: 0,
+                minimumHealthyPercent: minimumHealthyPercent !== undefined ? minimumHealthyPercent : 0,
+                maximumPercent: maximumHealthyPercent !== undefined ? maximumHealthyPercent : undefined,
               },
             }).promise(),
             ecsService: ecsService,

packages/aws-cdk/lib/cdk-toolkit.ts

@@ -13,7 +13,7 @@ import { CloudAssembly, DefaultSelection, ExtendedStackSelection, StackCollectio
 import { CloudExecutable } from './api/cxapp/cloud-executable';
 import { Deployments } from './api/deployments';
 import { GarbageCollector } from './api/garbage-collection/garbage-collector';
-import { HotswapMode } from './api/hotswap/common';
+import { HotswapMode, HotswapPropertyOverrides, EcsHotswapProperties } from './api/hotswap/common';
 import { findCloudWatchLogGroups } from './api/logs/find-cloudwatch-logs';
 import { CloudWatchLogEventMonitor } from './api/logs/logs-monitor';
 import { createDiffChangeSet, ResourcesToImport } from './api/util/cloudformation';
@@ -237,6 +237,14 @@ export class CdkToolkit {
       warning('⚠️ They should only be used for development - never use them for your production Stacks!\n');
     }
 
+    let hotswapPropertiesFromSettings = this.props.configuration.settings.get(['hotswap']) || {};
+
+    let hotswapPropertyOverrides = new HotswapPropertyOverrides();
+    hotswapPropertyOverrides.ecsHotswapProperties = new EcsHotswapProperties(
+      hotswapPropertiesFromSettings.ecs?.minimumHealthyPercent,
+      hotswapPropertiesFromSettings.ecs?.maximumHealthyPercent,
+    );
+
     const stacks = stackCollection.stackArtifacts;
 
     const stackOutputs: { [key: string]: any } = { };
@@ -347,6 +355,7 @@ export class CdkToolkit {
           ci: options.ci,
           rollback: options.rollback,
           hotswap: options.hotswap,
+          hotswapPropertyOverrides: hotswapPropertyOverrides,
           extraUserAgent: options.extraUserAgent,
           assetParallelism: options.assetParallelism,
           ignoreNoStacks: options.ignoreNoStacks,

packages/aws-cdk/lib/settings.ts

@@ -292,6 +292,12 @@ export class Settings {
       assetParallelism: argv['asset-parallelism'],
       assetPrebuild: argv['asset-prebuild'],
       ignoreNoStacks: argv['ignore-no-stacks'],
+      hotswap: {
+        ecs: {
+          minimumEcsHealthyPercent: argv.minimumEcsHealthyPercent,
+          maximumEcsHealthyPercent: argv.maximumEcsHealthyPercent,
+        },
+      },
       unstable: argv.unstable,
     });
   }

packages/aws-cdk/test/api/deploy-stack.test.ts

@@ -151,7 +151,7 @@ test('correctly passes CFN parameters when hotswapping', async () => {
   });
 
   // THEN
-  expect(tryHotswapDeployment).toHaveBeenCalledWith(expect.anything(), { A: 'A-value', B: 'B=value' }, expect.anything(), expect.anything(), HotswapMode.FALL_BACK);
+  expect(tryHotswapDeployment).toHaveBeenCalledWith(expect.anything(), { A: 'A-value', B: 'B=value' }, expect.anything(), expect.anything(), HotswapMode.FALL_BACK, expect.anything());
 });
 
 test('correctly passes SSM parameters when hotswapping', async () => {
@@ -181,7 +181,7 @@ test('correctly passes SSM parameters when hotswapping', async () => {
   });
 
   // THEN
-  expect(tryHotswapDeployment).toHaveBeenCalledWith(expect.anything(), { SomeParameter: 'SomeValue' }, expect.anything(), expect.anything(), HotswapMode.FALL_BACK);
+  expect(tryHotswapDeployment).toHaveBeenCalledWith(expect.anything(), { SomeParameter: 'SomeValue' }, expect.anything(), expect.anything(), HotswapMode.FALL_BACK, expect.anything());
 });
 
 test('call CreateStack when method=direct and the stack doesnt exist yet', async () => {