From 4582ac5c9e2cd9c7de289be06682ecd5b813b26a Mon Sep 17 00:00:00 2001 From: Jimmy Gaussen Date: Wed, 13 Mar 2024 18:50:28 +0100 Subject: [PATCH 1/3] chore(msk-alpha): update KafkaVersion (#29440) ### Issue # (if applicable) Could not find any in the backlog ### Reason for this change Update the CDK listed Kafka versions to match the current availability, as well as add missing deprecated versions ### Description of changes * Added latest version * `3.6.0` also supports tiered storage, see [docs](https://docs.aws.amazon.com/msk/latest/developerguide/msk-tiered-storage.html#msk-tiered-storage-requirements). I replaced the prefix check with a list of supported versions, as I'm not sure if say every reason after 3.6.0 will support it, and the `.tiered` prefix isn't consistently applied anymore * Added two unlisted, deprecated versions, as they are still returned by the SDK. The goal is to remove any differences between the SDK and the CDK to ease future automation ### Description of how you validated changes I compared the current CDK versions to live SDK data, using the `kafka:ListKafkaVersions` API results. ### Checklist - [x] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md) ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --------- Co-authored-by: GZ --- packages/@aws-cdk/aws-msk-alpha/README.md | 5 +-- .../aws-msk-alpha/lib/cluster-version.ts | 35 ++++++++++++++++++- .../aws-msk-alpha/test/cluster.test.ts | 8 ++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-msk-alpha/README.md b/packages/@aws-cdk/aws-msk-alpha/README.md index 81b74be3aa279..8155f8829a756 100644 --- a/packages/@aws-cdk/aws-msk-alpha/README.md +++ b/packages/@aws-cdk/aws-msk-alpha/README.md @@ -217,7 +217,8 @@ You can configure an MSK cluster storage mode using the `storageMode` property. Tiered storage is a low-cost storage tier for Amazon MSK that scales to virtually unlimited storage, making it cost-effective to build streaming data applications. -> Visit [Tiered storage](https://docs.aws.amazon.com/msk/latest/developerguide/msk-tiered-storage.html) for more details. +> Visit [Tiered storage](https://docs.aws.amazon.com/msk/latest/developerguide/msk-tiered-storage.html) +to see the list of compatible Kafka versions and for more details. ```ts declare const vpc: ec2.Vpc; @@ -225,7 +226,7 @@ declare const bucket: s3.IBucket; const cluster = new msk.Cluster(this, 'cluster', { clusterName: 'myCluster', - kafkaVersion: msk.KafkaVersion.V2_8_2_TIERED, + kafkaVersion: msk.KafkaVersion.V3_6_0, vpc, storageMode: msk.StorageMode.TIERED, }); diff --git a/packages/@aws-cdk/aws-msk-alpha/lib/cluster-version.ts b/packages/@aws-cdk/aws-msk-alpha/lib/cluster-version.ts index 405c0c6ee0ecc..b3c5277018b1c 100644 --- a/packages/@aws-cdk/aws-msk-alpha/lib/cluster-version.ts +++ b/packages/@aws-cdk/aws-msk-alpha/lib/cluster-version.ts @@ -11,6 +11,15 @@ export class KafkaVersion { */ public static readonly V1_1_1 = KafkaVersion.of('1.1.1'); + /** + * **Deprecated by Amazon MSK. You can't create a Kafka cluster with a deprecated version.** + * + * Kafka version 2.1.0 + * + * @deprecated use the latest runtime instead + */ + public static readonly V2_1_0 = KafkaVersion.of('2.1.0'); + /** * Kafka version 2.2.1 */ @@ -21,6 +30,15 @@ export class KafkaVersion { */ public static readonly V2_3_1 = KafkaVersion.of('2.3.1'); + /** + * **Deprecated by Amazon MSK. You can't create a Kafka cluster with a deprecated version.** + * + * Kafka version 2.4.1 + * + * @deprecated use the latest runtime instead + */ + public static readonly V2_4_1 = KafkaVersion.of('2.4.1'); + /** * Kafka version 2.4.1 */ @@ -111,6 +129,11 @@ export class KafkaVersion { */ public static readonly V3_5_1 = KafkaVersion.of('3.5.1'); + /** + * Kafka version 3.6.0 + */ + public static readonly V3_6_0 = KafkaVersion.of('3.6.0'); + /** * Custom cluster version * @param version custom version number @@ -119,6 +142,16 @@ export class KafkaVersion { return new KafkaVersion(version); } + /** + * List of Kafka versions that support tiered storage + * + * @see https://docs.aws.amazon.com/msk/latest/developerguide/msk-tiered-storage.html#msk-tiered-storage-requirements + */ + private static readonly TIERED_STORAGE_COMPATIBLE_VERSIONS = [ + KafkaVersion.V2_8_2_TIERED, + KafkaVersion.V3_6_0, + ].map(({ version }) => version); + /** * * @param version cluster version number @@ -129,6 +162,6 @@ export class KafkaVersion { * Checks if the cluster version supports tiered storage mode. */ public isTieredStorageCompatible() { - return this.version.endsWith('.tiered'); + return KafkaVersion.TIERED_STORAGE_COMPATIBLE_VERSIONS.includes(this.version); }; } diff --git a/packages/@aws-cdk/aws-msk-alpha/test/cluster.test.ts b/packages/@aws-cdk/aws-msk-alpha/test/cluster.test.ts index babbb235b40a3..7c4bd6a10f4ec 100644 --- a/packages/@aws-cdk/aws-msk-alpha/test/cluster.test.ts +++ b/packages/@aws-cdk/aws-msk-alpha/test/cluster.test.ts @@ -790,6 +790,12 @@ describe('MSK Cluster', () => { describe('created with storage mode', () => { describe('with tiered storage mode', () => { + test('version.isTieredStorageCompatible', () => { + expect(msk.KafkaVersion.V2_8_2_TIERED.isTieredStorageCompatible()).toBeTruthy(); + expect(msk.KafkaVersion.V3_5_1.isTieredStorageCompatible()).toBeFalsy(); + expect(msk.KafkaVersion.V3_6_0.isTieredStorageCompatible()).toBeTruthy(); + }); + test('create a cluster with tiered storage mode', () => { new msk.Cluster(stack, 'Cluster', { clusterName: 'cluster', @@ -797,7 +803,7 @@ describe('MSK Cluster', () => { kafkaVersion: msk.KafkaVersion.V2_8_2_TIERED, vpc, storageMode: msk.StorageMode.TIERED, - }), + }); Template.fromStack(stack).hasResourceProperties('AWS::MSK::Cluster', { StorageMode: 'TIERED', }); From 135b5208dce849f49b6f540f7199ece057a7ef22 Mon Sep 17 00:00:00 2001 From: Calvin Combs <66279577+comcalvi@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:20:12 -0700 Subject: [PATCH 2/3] feat(CLI): improved nested stack diff (#29172) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Issue # (if applicable) ### Reason for this change The existing nested stack diff places a fake property, `NestedTemplate`, in the templates to be diffed. This prevents displaying resource replacement information in the diff, like we do for top level stacks. This PR does *not* add changeset replacement information from changesets, but it does add replacement information from the spec. ### Description of changes Reworked nested stack diff to treat nested stacks as top level stacks. This improves the visual UX and sets us up for using changesets with nested stacks. #### Before Screenshot 2024-02-19 at 1 47 59 PM #### After Screenshot 2024-02-19 at 1 48 48 PM ### Description of how you validated changes Unit tests + manual tests. ### Checklist - [x] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md) ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../cloudformation-diff/lib/diff-template.ts | 6 +- .../cloudformation-diff/lib/format.ts | 8 +- packages/aws-cdk/lib/api/deployments.ts | 10 +- .../api/evaluate-cloudformation-template.ts | 34 +- .../aws-cdk/lib/api/hotswap-deployments.ts | 22 +- .../aws-cdk/lib/api/nested-stack-helpers.ts | 85 ++-- packages/aws-cdk/lib/cdk-toolkit.ts | 16 +- packages/aws-cdk/lib/diff.ts | 35 +- .../api/cloudformation-deployments.test.ts | 381 ++++++++++-------- .../api/hotswap/nested-stacks-hotswap.test.ts | 251 ++++++------ packages/aws-cdk/test/diff.test.ts | 357 +++++++++++++--- ...with-two-nested-stacks-stack.template.json | 7 +- ...mbda-two-stacks-stack.nested.template.json | 4 +- packages/aws-cdk/test/util.ts | 4 +- 14 files changed, 757 insertions(+), 463 deletions(-) diff --git a/packages/@aws-cdk/cloudformation-diff/lib/diff-template.ts b/packages/@aws-cdk/cloudformation-diff/lib/diff-template.ts index 1902d757f486d..284bb3a8d5f46 100644 --- a/packages/@aws-cdk/cloudformation-diff/lib/diff-template.ts +++ b/packages/@aws-cdk/cloudformation-diff/lib/diff-template.ts @@ -54,7 +54,7 @@ export function fullDiff( normalize(newTemplate); const theDiff = diffTemplate(currentTemplate, newTemplate); if (changeSet) { - filterFalsePositivies(theDiff, changeSet); + filterFalsePositives(theDiff, changeSet); addImportInformation(theDiff, changeSet); } if (isImport) { @@ -64,7 +64,7 @@ export function fullDiff( return theDiff; } -function diffTemplate( +export function diffTemplate( currentTemplate: { [key: string]: any }, newTemplate: { [key: string]: any }, ): types.TemplateDiff { @@ -235,7 +235,7 @@ function addImportInformation(diff: types.TemplateDiff, changeSet?: CloudFormati } } -function filterFalsePositivies(diff: types.TemplateDiff, changeSet: CloudFormation.DescribeChangeSetOutput) { +function filterFalsePositives(diff: types.TemplateDiff, changeSet: CloudFormation.DescribeChangeSetOutput) { const replacements = findResourceReplacements(changeSet); diff.resources.forEachDifference((logicalId: string, change: types.ResourceDifference) => { if (change.resourceType.includes('AWS::Serverless')) { diff --git a/packages/@aws-cdk/cloudformation-diff/lib/format.ts b/packages/@aws-cdk/cloudformation-diff/lib/format.ts index 7935f774fd468..724af468c2f45 100644 --- a/packages/@aws-cdk/cloudformation-diff/lib/format.ts +++ b/packages/@aws-cdk/cloudformation-diff/lib/format.ts @@ -30,7 +30,7 @@ export interface FormatStream extends NodeJS.WritableStream { export function formatDifferences( stream: FormatStream, templateDiff: TemplateDiff, - logicalToPathMap: { [logicalId: string]: string } = { }, + logicalToPathMap: { [logicalId: string]: string } = {}, context: number = 3) { const formatter = new Formatter(stream, logicalToPathMap, templateDiff, context); @@ -59,7 +59,7 @@ export function formatDifferences( export function formatSecurityChanges( stream: NodeJS.WritableStream, templateDiff: TemplateDiff, - logicalToPathMap: {[logicalId: string]: string} = {}, + logicalToPathMap: { [logicalId: string]: string } = {}, context?: number) { const formatter = new Formatter(stream, logicalToPathMap, templateDiff, context); @@ -254,7 +254,7 @@ class Formatter { const oldStr = JSON.stringify(oldObject, null, 2); const newStr = JSON.stringify(newObject, null, 2); const diff = _diffStrings(oldStr, newStr, this.context); - for (let i = 0 ; i < diff.length ; i++) { + for (let i = 0; i < diff.length; i++) { this.print('%s %s %s', linePrefix, i === 0 ? '└─' : ' ', diff[i]); } } else { @@ -466,7 +466,7 @@ function _diffStrings(oldStr: string, newStr: string, context: number): string[] function _findIndent(lines: string[]): number { let indent = Number.MAX_SAFE_INTEGER; for (const line of lines) { - for (let i = 1 ; i < line.length ; i++) { + for (let i = 1; i < line.length; i++) { if (line.charAt(i) !== ' ') { indent = indent > i - 1 ? i - 1 : indent; break; diff --git a/packages/aws-cdk/lib/api/deployments.ts b/packages/aws-cdk/lib/api/deployments.ts index 925cdebd45e15..b7bea90c56c28 100644 --- a/packages/aws-cdk/lib/api/deployments.ts +++ b/packages/aws-cdk/lib/api/deployments.ts @@ -7,7 +7,7 @@ import { CredentialsOptions, SdkForEnvironment, SdkProvider } from './aws-auth/s import { deployStack, DeployStackResult, destroyStack, DeploymentMethod } from './deploy-stack'; import { EnvironmentResources, EnvironmentResourcesRegistry } from './environment-resources'; import { HotswapMode } from './hotswap/common'; -import { loadCurrentTemplateWithNestedStacks, loadCurrentTemplate, flattenNestedStackNames, TemplateWithNestedStackCount } from './nested-stack-helpers'; +import { loadCurrentTemplateWithNestedStacks, loadCurrentTemplate, RootTemplateWithNestedStacks } from './nested-stack-helpers'; import { CloudFormationStack, Template, ResourcesToImport, ResourceIdentifierSummaries } from './util/cloudformation'; import { StackActivityProgress } from './util/cloudformation/stack-activity-monitor'; import { replaceEnvPlaceholders } from './util/placeholders'; @@ -327,13 +327,9 @@ export class Deployments { public async readCurrentTemplateWithNestedStacks( rootStackArtifact: cxapi.CloudFormationStackArtifact, retrieveProcessedTemplate: boolean = false, - ): Promise { + ): Promise { const sdk = (await this.prepareSdkWithLookupOrDeployRole(rootStackArtifact)).stackSdk; - const templateWithNestedStacks = await loadCurrentTemplateWithNestedStacks(rootStackArtifact, sdk, retrieveProcessedTemplate); - return { - deployedTemplate: templateWithNestedStacks.deployedTemplate, - nestedStackCount: flattenNestedStackNames(templateWithNestedStacks.nestedStackNames).length, - }; + return loadCurrentTemplateWithNestedStacks(rootStackArtifact, sdk, retrieveProcessedTemplate); } public async readCurrentTemplate(stackArtifact: cxapi.CloudFormationStackArtifact): Promise