From a86a710bd3d84c2f2bb1fb83903541acbdcb656a Mon Sep 17 00:00:00 2001 From: Hogan Bobertz Date: Fri, 26 Jan 2024 17:24:19 -0500 Subject: [PATCH 1/6] feat(migrate): Add template generator integration to CDK Migrate (#204) * initial jazz integration * cdk migrate with jazz integration * delete unused tests * move comment to appropriate test * refactor + comments to make it more readable * add filter ts doc * fix the test * Update packages/aws-cdk/lib/commands/migrate.ts Co-authored-by: Rico Hermans * Update packages/aws-cdk/lib/commands/migrate.ts Co-authored-by: Rico Hermans * Update packages/aws-cdk/lib/commands/migrate.ts Co-authored-by: Rico Hermans * Update packages/aws-cdk/lib/commands/migrate.ts Co-authored-by: Rico Hermans * Update packages/aws-cdk/lib/commands/migrate.ts Co-authored-by: Rico Hermans * minor refactors * make the progress bar safer * fix test * changes to managed flag pruning * Update packages/aws-cdk/lib/cdk-toolkit.ts Co-authored-by: Madeline Kusters <80541297+madeline-k@users.noreply.github.com> * Update packages/aws-cdk/lib/commands/migrate.ts Co-authored-by: Madeline Kusters <80541297+madeline-k@users.noreply.github.com> * Update packages/aws-cdk/lib/commands/migrate.ts Co-authored-by: Rico Hermans * Update packages/aws-cdk/lib/commands/migrate.ts Co-authored-by: Madeline Kusters <80541297+madeline-k@users.noreply.github.com> * Update packages/aws-cdk/lib/commands/migrate.ts Co-authored-by: Rico Hermans * resolve conflicts * or/and functionality * refactor user experience * display time since last scan * add user agent to cfn calls * make resources more statically typed * update and fix flitering * refactor template generator experience * refactor code * fix cli options * adjust apis to reflect template generator changes * update model * update tests * updated tests * fixes for migrate tests * fix generate template test mock * add failure test for most-recent * add filter failure test * feat(migrate): add migrate.json file creation to migrate workflow for imports (#212) * write file migrate.json to output * simplified source options * fixing tests * fix tests * fix test conflict and refactor tests * remove commented out test * add filter fail test * add default start test * add test for successful filters * remove duplicate test * update testing for import * Update packages/aws-cdk/lib/commands/migrate.ts Co-authored-by: Kendra Neil <53584728+TheRealAmazonKendra@users.noreply.github.com> * add test for migrate.json * add tests for migrate.json creation * remove dumb comment I put in --------- Co-authored-by: Hogan Bobertz Co-authored-by: Kendra Neil <53584728+TheRealAmazonKendra@users.noreply.github.com> * readd comment * add additional tests and address feedback * update error message * update test to no longer clean up generated templates * comment out test for now * address feedback * address feedback and refactor * remove bangs --------- Co-authored-by: Hogan Bobertz Co-authored-by: Rico Hermans Co-authored-by: Madeline Kusters <80541297+madeline-k@users.noreply.github.com> Co-authored-by: Kendra Neil <53584728+TheRealAmazonKendra@users.noreply.github.com> --- .../cli-integ/resources/cdk-apps/app/app.js | 15 +- .../tests/cli-integ-tests/cli.integtest.ts | 88 ++- packages/aws-cdk/lib/cdk-toolkit.ts | 69 +- packages/aws-cdk/lib/cli.ts | 19 +- packages/aws-cdk/lib/commands/migrate.ts | 698 +++++++++++++++++- packages/aws-cdk/test/cdk-toolkit.test.ts | 8 +- .../aws-cdk/test/commands/migrate.test.ts | 270 ++++++- .../templates/empty-template.yml | 0 8 files changed, 1126 insertions(+), 41 deletions(-) create mode 100644 packages/aws-cdk/test/commands/test-resources/templates/empty-template.yml diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js index f8e94bc2920df..746017ceee30d 100755 --- a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js @@ -86,7 +86,20 @@ class MigrateStack extends cdk.Stack { value: queue.node.defaultChild.logicalId, }); } - + if (process.env.SAMPLE_RESOURCES) { + const myTopic = new sns.Topic(this, 'migratetopic1', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + cdk.Tags.of(myTopic).add('tag1', 'value1'); + const myTopic2 = new sns.Topic(this, 'migratetopic2', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + cdk.Tags.of(myTopic2).add('tag2', 'value2'); + const myQueue = new sqs.Queue(this, 'migratequeue1', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + cdk.Tags.of(myQueue).add('tag3', 'value3'); + } } } diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts index a6a92b0a7cde2..134b303348475 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts @@ -1,7 +1,7 @@ import { promises as fs, existsSync } from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { integTest, cloneDirectory, shell, withDefaultFixture, retry, sleep, randomInteger, withSamIntegrationFixture, RESOURCES_DIR, withCDKMigrateFixture } from '../../lib'; +import { AwsClients, TestFixture, integTest, cloneDirectory, shell, withDefaultFixture, retry, sleep, randomInteger, withSamIntegrationFixture, RESOURCES_DIR, withCDKMigrateFixture } from '../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime @@ -571,9 +571,9 @@ integTest('deploy with role', withDefaultFixture(async (fixture) => { } })); -// TODO add go back in when template synths properly +// TODO add more testing that ensures the symmetry of the generated constructs to the resources. ['typescript', 'python', 'csharp', 'java'].forEach(language => { - integTest(`cdk migrate ${language}`, withCDKMigrateFixture(language, async (fixture) => { + integTest(`cdk migrate ${language} deploys successfully`, withCDKMigrateFixture(language, async (fixture) => { if (language === 'python') { await fixture.shell(['pip', 'install', '-r', 'requirements.txt']); } @@ -588,6 +588,88 @@ integTest('deploy with role', withDefaultFixture(async (fixture) => { })); }); +['typescript', 'python', 'csharp', 'java'].forEach(language => { + integTest(`cdk migrate generates migrate.json in ${language}`, withCDKMigrateFixture(language, async (fixture) => { + + const migrateFile = await fs.readFile(path.join(fixture.integTestDir, 'migrate.json'), 'utf8'); + const expectedFile = `{ + \"//\": \"This file is generated by cdk migrate. It will be automatically deleted after the first successful deployment of this app to the environment of the original resources.\", + \"Source\": \"localfile\" + }`; + expect(JSON.parse(migrateFile)).toEqual(JSON.parse(expectedFile)); + await fixture.cdkDestroy(fixture.stackNamePrefix); + })); +}); + +// TODO: Uncomment this out before launch. Commenting this for now so pipelines won't complain. +// integTest('cdk migrate --from-scan with AND/OR filters correctly filters resources', withDefaultFixture(async (fixture) => { +// const migrateStackJr = 'migrate-stack-jr'; + +// await fixture.cdkDeploy('migrate-stack', { +// modEnv: { SAMPLE_RESOURCES: '1' }, +// }); +// await fixture.cdk( +// ['migrate', '--stack-name', migrateStackJr, '--from-scan', 'new', '--filter', 'type=AWS::SNS::Topic,tag-key=tag1', 'type=AWS::SQS::Queue,tag-key=tag3'], +// { modEnv: { MIGRATE_INTEG_TEST: '1' }, neverRequireApproval: true, verbose: true, captureStderr: false }, +// ); + +// try { + +// const response = await fixture.aws.cloudFormation('describeGeneratedTemplate', { +// GeneratedTemplateName: migrateStackJr, +// }); +// const resourceNames = []; +// for (const resource of response.Resources || []) { +// if (resource.LogicalResourceId) { +// resourceNames.push(resource.LogicalResourceId); +// } +// } +// fixture.log(`Resources: ${resourceNames}`); +// expect(resourceNames.some(ele => ele && ele.includes('migratetopic1'))).toBeTruthy(); +// expect(resourceNames.some(ele => ele && ele.includes('migratequeue1'))).toBeTruthy(); +// expect(response.Resources?.length).toEqual(2); +// } finally { +// await fixture.cdkDestroy('migrate-stack'); +// } +// })); + +['typescript', 'python', 'csharp', 'java'].forEach(language => { + integTest(`cdk migrate --from-stack creates deployable ${language} app`, withDefaultFixture(async (fixture) => { + const migrateStackName = fixture.fullStackName('migrate-stack'); + await fixture.aws.cloudFormation('createStack', { + StackName: migrateStackName, + TemplateBody: await fs.readFile(path.join(__dirname, '..', '..', 'resources', 'templates', 'sqs-template.json'), 'utf8').toString(), + }); + try { + let stackStatus = 'CREATE_IN_PROGRESS'; + while (stackStatus === 'CREATE_IN_PROGRESS') { + stackStatus = await (await (fixture.aws.cloudFormation('describeStacks', { StackName: migrateStackName }))).Stacks?.[0].StackStatus!; + await sleep(1000); + } + await fixture.cdk( + ['migrate', '--stack-name', migrateStackName, '--from-stack'], + { modEnv: { MIGRATE_INTEG_TEST: '1' }, neverRequireApproval: true, verbose: true, captureStderr: false }, + ); + const awsClients = await AwsClients.default(fixture.output); + const fixtureJr = new TestFixture( + path.join(fixture.integTestDir, migrateStackName), + fixture.stackNamePrefix, + fixture.output, + awsClients, + '', + ); + await fixtureJr.cdkDeploy('migrate-stack', { neverRequireApproval: true, verbose: true, captureStderr: false }); + const response = await fixture.aws.cloudFormation('describeStacks', { + StackName: migrateStackName, + }); + + expect(response.Stacks?.[0].StackStatus).toEqual('UPDATE_COMPLETE'); + } finally { + await fixture.cdkDestroy(migrateStackName); + } + })); +}); + integTest('cdk diff', withDefaultFixture(async (fixture) => { const diff1 = await fixture.cdk(['diff', fixture.fullStackName('test-1')]); expect(diff1).toContain('AWS::SNS::Topic'); diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 79d11364f1605..4e769134753a1 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -17,7 +17,7 @@ import { findCloudWatchLogGroups } from './api/logs/find-cloudwatch-logs'; import { CloudWatchLogEventMonitor } from './api/logs/logs-monitor'; import { createDiffChangeSet, ResourcesToImport } from './api/util/cloudformation'; import { StackActivityProgress } from './api/util/cloudformation/stack-activity-monitor'; -import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, validateSourceOptions } from './commands/migrate'; +import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, parseSourceOptions, generateTemplate, FromScan, TemplateSourceOptions, GenerateTemplateOutput, CfnTemplateGeneratorProvider, writeMigrateJsonFile, buildGenertedTemplateOutput, buildCfnClient } from './commands/migrate'; import { printSecurityDiff, printStackDiff, RequireApproval } from './diff'; import { ResourceImporter } from './import'; import { data, debug, error, highlight, print, success, warning, withCorkedLogging } from './logging'; @@ -721,17 +721,58 @@ export class CdkToolkit { const language = options.language?.toLowerCase() ?? 'typescript'; try { - validateSourceOptions(options.fromPath, options.fromStack); - const template = readFromPath(options.fromPath) || - await readFromStack(options.stackName, this.props.sdkProvider, setEnvironment(options.account, options.region)); - const stack = generateStack(template!, options.stackName, language); + const environment = setEnvironment(options.account, options.region); + let generateTemplateOutput: GenerateTemplateOutput | undefined; + // if neither fromPath nor fromStack is provided, generate a template using cloudformation + const scanType = parseSourceOptions(options.fromPath, options.fromStack, options.stackName).source; + if (scanType == TemplateSourceOptions.SCAN) { + generateTemplateOutput = await generateTemplate({ + stackName: options.stackName, + filters: options.filter, + fromScan: options.fromScan, + sdkProvider: this.props.sdkProvider, + environment: environment, + }); + } else if (scanType == TemplateSourceOptions.PATH) { + const templateBody = readFromPath(options.fromPath!); + + const parsedTemplate = deserializeStructure(templateBody); + const templateId = parsedTemplate.Metadata?.TemplateId?.toString(); + if (templateId) { + // if we have a template id, we can call describe generated template to get the resource identifiers + // resource metadata, and template source to generate the template + const cfn = new CfnTemplateGeneratorProvider(await buildCfnClient(this.props.sdkProvider, environment)); + const generatedTemplateSummary = await cfn.describeGeneratedTemplate(templateId); + generateTemplateOutput = buildGenertedTemplateOutput(generatedTemplateSummary, templateBody, generatedTemplateSummary.GeneratedTemplateId!); + } else { + generateTemplateOutput = { + templateBody: templateBody, + source: 'localfile', + }; + } + } else if (scanType == TemplateSourceOptions.STACK) { + const template = await readFromStack(options.stackName, this.props.sdkProvider, environment); + if (!template) { + throw new Error(`No template found for stack-name: ${options.stackName}`); + } + generateTemplateOutput = { + templateBody: template, + source: options.stackName, + }; + } else { + // We shouldn't ever get here, but just in case. + throw new Error(`Invalid source option provided: ${scanType}`); + } + const stack = generateStack(generateTemplateOutput!.templateBody, options.stackName, language); success(' ⏳ Generating CDK app for %s...', chalk.blue(options.stackName)); await generateCdkApp(options.stackName, stack!, language, options.outputPath, options.compress); + if (generateTemplateOutput) { + writeMigrateJsonFile(options.outputPath, options.stackName, generateTemplateOutput); + } } catch (e) { - error(' ❌ Migrate failed for `%s`: %s', chalk.blue(options.stackName), (e as Error).message); + error(' ❌ Migrate failed for `%s`: %s', options.stackName, (e as Error).message); throw e; } - } private async selectStacksForList(patterns: string[]) { @@ -1336,6 +1377,20 @@ export interface MigrateOptions { */ readonly region?: string; + /** + * Filtering criteria used to select the resources to be included in the generated CDK app. + * + * @default - Include all resources + */ + readonly filter?: string[]; + + /** + * Whether to initiate a new account scan for generating the CDK app. + * + * @default false + */ + readonly fromScan?: FromScan; + /** * Whether to zip the generated cdk app folder. * diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index 0871e2a42814c..d7465e78693bf 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -22,7 +22,7 @@ import { CdkToolkit, AssetBuildTime } from '../lib/cdk-toolkit'; import { realHandler as context } from '../lib/commands/context'; import { realHandler as docs } from '../lib/commands/docs'; import { realHandler as doctor } from '../lib/commands/doctor'; -import { MIGRATE_SUPPORTED_LANGUAGES } from '../lib/commands/migrate'; +import { MIGRATE_SUPPORTED_LANGUAGES, getMigrateScanType } from '../lib/commands/migrate'; import { RequireApproval } from '../lib/diff'; import { availableInitLanguages, cliInit, printAvailableTemplates } from '../lib/init'; import { data, debug, error, print, setLogLevel, setCI } from '../lib/logging'; @@ -281,6 +281,21 @@ async function parseCommandLineArguments(args: string[]) { .option('from-path', { type: 'string', desc: 'The path to the CloudFormation template to migrate. Use this for locally stored templates' }) .option('from-stack', { type: 'boolean', desc: 'Use this flag to retrieve the template for an existing CloudFormation stack' }) .option('output-path', { type: 'string', desc: 'The output path for the migrated CDK app' }) + .option('from-scan', { + type: 'string', + desc: 'Determines if a new scan should be created, or the last successful existing scan should be used ' + + '\n options are "new" or "most-recent"', + }) + .option('filter', { + type: 'array', + desc: 'Filters the resource scan based on the provided criteria in the following format: "key1=value1,key2=value2"' + + '\n This field can be passed multiple times for OR style filtering: ' + + '\n filtering options: ' + + '\n resource-identifier: A key-value pair that identifies the target resource. i.e. {"ClusterName", "myCluster"}' + + '\n resource-type-prefix: A string that represents a type-name prefix. i.e. "AWS::DynamoDB::"' + + '\n tag-key: a string that matches resources with at least one tag with the provided key. i.e. "myTagKey"' + + '\n tag-value: a string that matches resources with at least one tag with the provided value. i.e. "myTagValue"', + }) .option('compress', { type: 'boolean', desc: 'Use this flag to zip the generated CDK app' }), ) .command('context', 'Manage cached context values', (yargs: Argv) => yargs @@ -679,6 +694,8 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise { const resolvedOutputPath = path.join(outputPath ?? process.cwd(), stackName); const formattedStackName = decamelize(stackName); @@ -93,13 +95,17 @@ export function generateStack(template: string, stackName: string, language: str * @param inputPath The location of the template * @returns A string representation of the template if present, otherwise undefined */ -export function readFromPath(inputPath?: string): string | undefined { +export function readFromPath(inputPath: string): string { + let readFile: string; try { - return inputPath ? fs.readFileSync(inputPath, 'utf8') : undefined; + readFile = fs.readFileSync(inputPath, 'utf8'); } catch (e) { throw new Error(`'${inputPath}' is not a valid path.`); } - + if (readFile == '') { + throw new Error(`Cloudformation template filepath: '${inputPath}' is an empty file.`); + } + return readFile; } /** @@ -122,6 +128,148 @@ export async function readFromStack(stackName: string, sdkProvider: SdkProvider, return; } +/** + * Takes in a stack name and account and region and returns a generated cloudformation template using the cloudformation + * template generator. + * + * @param GenerateTemplateOptions An object containing the stack name, filters, sdkProvider, environment, and newScan flag + * @returns a generated cloudformation template + */ +export async function generateTemplate( + options: GenerateTemplateOptions, +): Promise { + const cfn = new CfnTemplateGeneratorProvider(await buildCfnClient(options.sdkProvider, options.environment)); + + const scanId = await findLastSuccessfulScan(cfn, options); + + // if a customer accidentally ctrl-c's out of the command and runs it again, this will continue the progress bar where it left off + const curScan = await cfn.describeResourceScan(scanId); + if ((curScan.Status == ScanStatus.IN_PROGRESS)) { + print('Resource scan in progress. Please wait, this can take 10 minutes or longer.'); + await scanProgressBar(scanId, cfn); + } + + displayTimeDiff(new Date(), new Date(curScan.StartTime!)); + + let resources: CloudFormation.ScannedResources = await cfn.listResourceScanResources(scanId!, options.filters); + + print('finding related resources.'); + let relatedResources = await cfn.getResourceScanRelatedResources(scanId!, resources); + + print(`Found ${relatedResources.length} resources.`); + + print('Generating CFN template from scanned resources.'); + const templateArn = (await cfn.createGeneratedTemplate(options.stackName, relatedResources)).GeneratedTemplateId!; + + let generatedTemplate = await cfn.describeGeneratedTemplate(templateArn); + + print('Please wait, template creation in progress. This may take a couple minutes.'); + while (generatedTemplate.Status !== ScanStatus.COMPLETE && generatedTemplate.Status !== ScanStatus.FAILED) { + await printDots(`[${generatedTemplate.Status}] Template Creation in Progress`, 400); + generatedTemplate = await cfn.describeGeneratedTemplate(templateArn); + } + print(''); + print('Template successfully generated!'); + return buildGenertedTemplateOutput(generatedTemplate, (await cfn.getGeneratedTemplate(templateArn)).TemplateBody!, templateArn); +} + +async function findLastSuccessfulScan(cfn: CfnTemplateGeneratorProvider, options: GenerateTemplateOptions): Promise { + let resourceScanSummaries: CloudFormation.ResourceScanSummaries | undefined = []; + const clientRequestToken = `cdk-migrate-${options.environment.account}-${options.environment.region}`; + if (options.fromScan === FromScan.NEW) { + print(`Starting new scan for account ${options.environment.account} in region ${options.environment.region}`); + try { + await cfn.startResourceScan(clientRequestToken); + resourceScanSummaries = (await cfn.listResourceScans()).ResourceScanSummaries; + } catch (e) { + // continuing here because if the scan fails on a new-scan it is very likely because there is either already a scan in progress + // or the customer hit a rate limit. In either case we want to continue with the most recent scan. + // If this happens to fail for a credential error then that will be caught immediately after anyway. + print(`Scan failed to start due to error '${(e as Error).message}', defaulting to latest scan.`); + } + } else { + resourceScanSummaries = (await cfn.listResourceScans()).ResourceScanSummaries; + await cfn.checkForResourceScan(resourceScanSummaries, options, clientRequestToken); + } + // get the latest scan, which we know will exist + resourceScanSummaries = (await cfn.listResourceScans()).ResourceScanSummaries; + let scanId: string | undefined = resourceScanSummaries![0].ResourceScanId; + + // find the most recent scan that isn't in a failed state in case we didn't start a new one + for (const summary of resourceScanSummaries!) { + if (summary.Status !== ScanStatus.FAILED) { + scanId = summary.ResourceScanId!; + break; + } + } + + return scanId!; +} + +/** + * Takes a string of filters in the format of key1=value1,key2=value2 and returns a map of the filters. + * + * @param filters a string of filters in the format of key1=value1,key2=value2 + * @returns a map of the filters + */ +function parseFilters(filters: string): {[key in FilterType]: string | undefined } { + if (!filters) { + return { + 'resource-identifier': undefined, + 'resource-type-prefix': undefined, + 'tag-key': undefined, + 'tag-value': undefined, + }; + }; + + const filterShorthands: { [key: string]: FilterType } = { + 'identifier': FilterType.RESOURCE_IDENTIFIER, + 'id': FilterType.RESOURCE_IDENTIFIER, + 'type': FilterType.RESOURCE_TYPE_PREFIX, + 'type-prefix': FilterType.RESOURCE_TYPE_PREFIX, + }; + + const filterList = filters.split(','); + + let filterMap: { [key in FilterType]: string | undefined} = { + [FilterType.RESOURCE_IDENTIFIER]: undefined, + [FilterType.RESOURCE_TYPE_PREFIX]: undefined, + [FilterType.TAG_KEY]: undefined, + [FilterType.TAG_VALUE]: undefined, + }; + + for (const fil of filterList) { + const filter = fil.split('='); + let filterKey = filter[0]; + const filterValue = filter[1]; + // if the key is a shorthand, replace it with the full name + if (filterKey in filterShorthands) { + filterKey = filterShorthands[filterKey]; + } + if (Object.values(FilterType).includes(filterKey as any)) { + filterMap[filterKey as keyof typeof filterMap] = filterValue; + } else { + throw new Error(`Invalid filter: ${filterKey}`); + } + } + return filterMap; +} + +/** + * Takes a list of any type and breaks it up into chunks of a specified size. + * + * @param list The list to break up + * @param chunkSize The size of each chunk + * @returns A list of lists of the specified size + */ +export function chunks(list: any[], chunkSize: number): any[][] { + const chunkedList: any[][] = []; + for (let i = 0; i < list.length; i += chunkSize) { + chunkedList.push(list.slice(i, i + chunkSize)); + } + return chunkedList; +} + /** * Sets the account and region for making CloudFormation calls. * @param account The account to use @@ -132,17 +280,547 @@ export function setEnvironment(account?: string, region?: string): Environment { return { account: account ?? UNKNOWN_ACCOUNT, region: region ?? UNKNOWN_REGION, name: 'cdk-migrate-env' }; } +/** + * Enum for the source options for the template + */ +export enum TemplateSourceOptions { + PATH = 'path', + STACK = 'stack', + SCAN = 'scan', +} + +/** + * An object representing the source of a template. + */ +type TemplateSource = + | { source: TemplateSourceOptions.SCAN } + | { source: TemplateSourceOptions.PATH, templatePath: string } + | { source: TemplateSourceOptions.STACK, stackName: string } + ; + +/** + * Enum for the status of a resource scan + */ +export enum ScanStatus { + IN_PROGRESS = 'IN_PROGRESS', + COMPLETE = 'COMPLETE', + FAILED = 'FAILED', +} + +export enum FilterType { + RESOURCE_IDENTIFIER = 'resource-identifier', + RESOURCE_TYPE_PREFIX = 'resource-type-prefix', + TAG_KEY = 'tag-key', + TAG_VALUE = 'tag-value', +} + /** * Validates that exactly one source option has been provided. * @param fromPath The content of the flag `--from-path` * @param fromStack the content of the flag `--from-stack` */ -export function validateSourceOptions(fromPath?: string, fromStack?: boolean) { +export function parseSourceOptions(fromPath?: string, fromStack?: boolean, stackName?: string): TemplateSource { if (fromPath && fromStack) { throw new Error('Only one of `--from-path` or `--from-stack` may be provided.'); } - + if (!stackName) { + throw new Error('`--stack-name` is a required field.'); + } if (!fromPath && !fromStack) { - throw new Error('Either `--from-path` or `--from-stack` must be used to provide the source of the CloudFormation template.'); + return { source: TemplateSourceOptions.SCAN }; + } + if (fromPath) { + return { source: TemplateSourceOptions.PATH, templatePath: fromPath }; + } + return { source: TemplateSourceOptions.STACK, stackName: stackName! }; +} + +/** + * Takes a set of resources and removes any with the managedbystack flag set to true. + * + * @param resourceList the list of resources provided by the list scanned resources calls + * @returns a list of resources not managed by cfn stacks + */ +function excludeManaged(resourceList: CloudFormation.ScannedResources): CloudFormation.ScannedResourceIdentifiers { + return resourceList.filter((r) => !r.ManagedByStack).map((r) => ({ + ResourceType: r.ResourceType!, + ResourceIdentifier: r.ResourceIdentifier!, + })); +} + +/** + * Transforms a list of resources into a list of resource identifiers by removing the ManagedByStack flag. + * Setting the value of the field to undefined effectively removes it from the object. + * + * @param resourceList the list of resources provided by the list scanned resources calls + * @returns a list of ScannedResourceIdentifiers + */ +function resourceIdentifiers(resourceList: CloudFormation.ScannedResources): CloudFormation.ScannedResourceIdentifiers { + const identifiers: CloudFormation.ScannedResourceIdentifiers = []; + resourceList.forEach((r) => { + const identifier: CloudFormation.ScannedResourceIdentifier = { + ResourceType: r.ResourceType!, + ResourceIdentifier: r.ResourceIdentifier!, + }; + identifiers.push(identifier); + }); + return identifiers; +} + +/** + * Takes a scan id and maintains a progress bar to display the progress of a scan to the user. + * + * @param scanId A string representing the scan id + * @param cloudFormation The CloudFormation sdk client to use + */ +export async function scanProgressBar(scanId: string, cfn: CfnTemplateGeneratorProvider) { + let curProgress = 0.5; + // we know it's in progress initially since we wouldn't have gotten here if it wasn't + let curScan: CloudFormation.DescribeResourceScanOutput = { Status: ScanStatus.IN_PROGRESS }; + while (curScan.Status == ScanStatus.IN_PROGRESS) { + curScan = await cfn.describeResourceScan(scanId); + curProgress = curScan.PercentageCompleted ?? curProgress; + printBar(30, curProgress); + await new Promise(resolve => setTimeout(resolve, 2000)); + } + print(''); + print('✅ Scan Complete!'); +} + +/** + * Prints a progress bar to the console. To be used in a while loop to show progress of a long running task. + * The progress bar deletes the current line on the console and rewrites it with the progress amount. + * + * @param width The width of the progress bar + * @param progress The current progress to display as a percentage of 100 + */ +export function printBar(width: number, progress: number) { + if (!process.env.MIGRATE_INTEG_TEST) { + const FULL_BLOCK = '█'; + const PARTIAL_BLOCK = ['', '▏', '▎', '▍', '▌', '▋', '▊', '▉']; + const fraction = Math.min(progress / 100, 1); + const innerWidth = Math.max(1, width - 2); + const chars = innerWidth * fraction; + const remainder = chars - Math.floor(chars); + + const fullChars = FULL_BLOCK.repeat(Math.floor(chars)); + const partialChar = PARTIAL_BLOCK[Math.floor(remainder * PARTIAL_BLOCK.length)]; + const filler = '·'.repeat(innerWidth - Math.floor(chars) - (partialChar ? 1 : 0)); + + const color = chalk.green; + + rewriteLine('[' + color(fullChars + partialChar) + filler + `] (${progress}%)`); + } +} + +/** + * Prints a message to the console with a series periods appended to it. To be used in a while loop to show progress of a long running task. + * The message deletes the current line and rewrites it several times to display 1-3 periods to show the user that the task is still running. + * + * @param message The message to display + * @param timeoutx4 The amount of time to wait before printing the next period + */ +export async function printDots(message: string, timeoutx4: number) { + + if (!process.env.MIGRATE_INTEG_TEST) { + rewriteLine(message + ' .'); + await new Promise(resolve => setTimeout(resolve, timeoutx4)); + + rewriteLine(message + ' ..'); + await new Promise(resolve => setTimeout(resolve, timeoutx4)); + + rewriteLine(message + ' ...'); + await new Promise(resolve => setTimeout(resolve, timeoutx4)); + + rewriteLine(message); + await new Promise(resolve => setTimeout(resolve, timeoutx4)); + } + +} + +/** + * Rewrites the current line on the console and writes a new message to it. + * This is a helper funciton for printDots and printBar. + * + * @param message The message to display + */ +export function rewriteLine(message: string) { + process.stdout.clearLine(0); + process.stdout.cursorTo(0); + process.stdout.write(message); +} + +/** + * Prints the time difference between two dates in days, hours, and minutes. + * + * @param time1 The first date to compare + * @param time2 The second date to compare + */ +export function displayTimeDiff(time1: Date, time2: Date): void { + const diff = Math.abs(time1.getTime() - time2.getTime()); + + const days = Math.floor(diff / (1000 * 60 * 60 * 24)); + const hours = Math.floor((diff % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60)); + const minutes = Math.floor((diff % (1000 * 60 * 60)) / (1000 * 60)); + + print(`Using the latest successful scan which is ${days} days, ${hours} hours, and ${minutes} minutes old.`); +} + +/** + * Writes a migrate.json file to the output directory. + * + * @param outputPath The path to write the migrate.json file to + * @param stackName The name of the stack + * @param generatedOutput The output of the template generator + */ +export function writeMigrateJsonFile(outputPath: string | undefined, stackName: string, generatedOutput: GenerateTemplateOutput) { + const outputToJson = { + '//': 'This file is generated by cdk migrate. It will be automatically deleted after the first successful deployment of this app to the environment of the original resources.', + 'Source': generatedOutput.source, + 'Resources': generatedOutput.resources, + }; + fs.writeFileSync(`${path.join(outputPath ?? process.cwd(), stackName)}/migrate.json`, JSON.stringify(outputToJson, null, 2)); +} + +/** + * Takes a string representing the from-scan flag and returns a FromScan enum value. + * + * @param scanType A string representing the from-scan flag + * @returns A FromScan enum value + */ +export function getMigrateScanType(scanType: string) { + switch (scanType) { + case 'new': + return FromScan.NEW; + case 'most-recent': + return FromScan.MOST_RECENT; + case '': + return FromScan.DEFAULT; + case undefined: + return FromScan.DEFAULT; + default: + throw new Error(`Unknown scan type: ${scanType}`); + } +} + +/** + * Builds the GenerateTemplateOutput object from the DescribeGeneratedTemplateOutput and the template body. + * + * @param generatedTemplateSummary The output of the describe generated template call + * @param templateBody The body of the generated template + * @returns A GenerateTemplateOutput object + */ +export function buildGenertedTemplateOutput(generatedTemplateSummary: CloudFormation.DescribeGeneratedTemplateOutput, + templateBody: string, source: string): GenerateTemplateOutput { + return { + templateBody: templateBody, + source: source, + resources: generatedTemplateSummary.Resources!.map((r) => ({ + resourceType: r.ResourceType!, + logicalResourceId: r.LogicalResourceId!, + resourceIdentifier: r.ResourceIdentifier!, + })), + }; +} + +export async function buildCfnClient(sdkProvider: SdkProvider, environment: Environment) { + const cloudFormationClient = (await sdkProvider.forEnvironment(environment, Mode.ForReading)).sdk.cloudFormation(); + + cloudFormationClient.config.customUserAgent = 'cdk-migrate'; + + return cloudFormationClient; +} + +/** + * Class for making CloudFormation template generator calls + */ +export class CfnTemplateGeneratorProvider { + private cfn: CloudFormation; + constructor(cfn: CloudFormation) { + this.cfn = cfn; + } + + async checkForResourceScan(resourceScanSummaries: CloudFormation.ResourceScanSummaries | undefined, + options: GenerateTemplateOptions, clientRequestToken: string) { + + if (!resourceScanSummaries || resourceScanSummaries.length === 0) { + if (options.fromScan === FromScan.MOST_RECENT) { + throw new Error('No scans found. Please either start a new scan with the `--from-scan` new or do not specify a `--from-scan` option.'); + } else { + print('No scans found. Initiating a new resource scan.'); + await this.startResourceScan(clientRequestToken); + } + } + } + + /** + * Retrieves a tokenized list of resources and their associated scan. If a token is present the function + * will loop through all pages and combine them into a single list of ScannedRelatedResources + * + * @param scanId scan id for the to list resources for + * @param resources A list of resources to find related resources for + */ + async getResourceScanRelatedResources(scanId: string, resources: CloudFormation.ScannedResources ) + : Promise { + let relatedResourceList = resources; + + // break the list of resources into chunks of 100 to avoid hitting the 100 resource limit + for (const chunk of chunks(resources, 100)) { + // get the first page of related resources + const res = await this.cfn.listResourceScanRelatedResources({ + ResourceScanId: scanId, + Resources: chunk, + }).promise(); + + // add the first page to the list + relatedResourceList.push(...(res.RelatedResources ?? [])); + let nextToken = res.NextToken; + + // if there are more pages, cycle through them and add them to the list before moving on to the next chunk + while (nextToken) { + const nextRelatedResources = await this.cfn.listResourceScanRelatedResources({ + ResourceScanId: scanId, + Resources: resourceIdentifiers(resources), + NextToken: nextToken, + }).promise(); + nextToken = nextRelatedResources.NextToken; + relatedResourceList.push(...(nextRelatedResources.RelatedResources ?? [])); + } + } + + let uniqueResources: {[key: string]: CloudFormation.ScannedResource} = {}; + + // de-duplicating the list of related resources based on the resource identifier value + // The resource identifier key is not known at compile time so we need to get it using Object.keys. There will only + // ever be a single key so [0] retrives the first and only key + for (const resource of relatedResourceList) { + if (!resource.ResourceIdentifier) continue; + const key = Object.keys(resource.ResourceIdentifier!)[0]; + + // Creating our unique identifier using the resource type, the key, and the value of the resource identifier + // The resource identifier is a combination of a key value pair defined by a resource's schema, and the resource type of the resource. + const uniqueIdentifer = `${resource.ResourceType}:${key}:${resource.ResourceIdentifier![key]}`; + uniqueResources[uniqueIdentifer] = resource; + } + + // convert the dictionary back into a list + relatedResourceList = Object.values(uniqueResources); + + // prune the managedbystack flag off of them again. + return process.env.MIGRATE_INTEG_TEST ? resourceIdentifiers(relatedResourceList) : resourceIdentifiers(excludeManaged(relatedResourceList)) ; + } + + /** + * Kicks off a scan of a customers account, returning the scan id. A scan can take + * 10 minutes or longer to complete. However this will return a scan id as soon as + * the scan has begun. + * + * @returns A string representing the scan id + */ + async startResourceScan(requestToken: string ) { + return (await this.cfn.startResourceScan({ + ClientRequestToken: requestToken, + }).promise()).ResourceScanId; + } + + /** + * Gets the most recent scans a customer has completed + * + * @returns a list of resource scan summaries + */ + async listResourceScans() { + + return this.cfn.listResourceScans().promise(); + } + + /** + * Retrieves a tokenized list of resources from a resource scan. If a token is present, this function + * will loop through all pages and combine them into a single list of ScannedResources. + * Additionally will apply any filters provided by the customer. + * + * @param scanId scan id for the to list resources for + * @param filters a string of filters in the format of key1=value1,key2=value2 + * @returns a combined list of all resources from the scan + */ + async listResourceScanResources(scanId: string, filters: string[] = [] ) + : Promise { + + let resourceList: CloudFormation.ScannedResources = []; + let resourceScanInputs: CloudFormation.ListResourceScanResourcesInput; + + if (filters.length > 0) { + print('Applying filters to resource scan.'); + for (const filter of filters) { + const filterList = parseFilters(filter); + resourceScanInputs = { + ResourceScanId: scanId, + ResourceIdentifier: filterList[FilterType.RESOURCE_IDENTIFIER], + ResourceTypePrefix: filterList[FilterType.RESOURCE_TYPE_PREFIX], + TagKey: filterList[FilterType.TAG_KEY], + TagValue: filterList[FilterType.TAG_VALUE], + }; + const resources = await this.cfn.listResourceScanResources(resourceScanInputs).promise(); + resourceList = resourceList.concat(resources.Resources ?? []); + let nextToken = resources.NextToken; + + // cycle through the pages adding all resources to the list until we run out of pages + while (nextToken) { + resourceScanInputs.NextToken = nextToken; + const nextResources = await this.cfn.listResourceScanResources(resourceScanInputs).promise(); + nextToken = nextResources.NextToken; + resourceList = resourceList!.concat(nextResources.Resources ?? []); + } + } + } else { + print('No filters provided. Retrieving all resources from scan.'); + resourceScanInputs = { + ResourceScanId: scanId, + }; + const resources = await this.cfn.listResourceScanResources(resourceScanInputs).promise(); + resourceList = resourceList!.concat(resources.Resources ?? []); + let nextToken = resources.NextToken; + + // cycle through the pages adding all resources to the list until we run out of pages + while (nextToken) { + resourceScanInputs.NextToken = nextToken; + const nextResources = await this.cfn.listResourceScanResources(resourceScanInputs).promise(); + nextToken = nextResources.NextToken; + resourceList = resourceList!.concat(nextResources.Resources ?? []); + } + } + + return process.env.MIGRATE_INTEG_TEST ? resourceIdentifiers(resourceList) : resourceIdentifiers(excludeManaged(resourceList)); } + + /** + * Retrieves information about a resource scan. + * + * @param scanId scan id for the to list resources for + * @returns information about the scan + */ + async describeResourceScan(scanId: string ): + Promise { + + return this.cfn.describeResourceScan({ + ResourceScanId: scanId, + }).promise(); + } + + /** + * Describes the current status of the template being generated. + * + * @param templateId A string representing the template id + * @returns DescribeGeneratedTemplateOutput an object containing the template status and results + */ + async describeGeneratedTemplate(templateId: string ): + Promise { + + const generatedTemplate = await this.cfn.describeGeneratedTemplate({ + GeneratedTemplateName: templateId, + }).promise(); + + if (generatedTemplate.Status == ScanStatus.FAILED) { + throw new Error(generatedTemplate.StatusReason); + } + + return generatedTemplate; + } + + /** + * Retrieves a completed generated cloudformation template from the template generator. + * + * @param templateId A string representing the template id + * @param cloudFormation The CloudFormation sdk client to use + * @returns DescribeGeneratedTemplateOutput an object containing the template status and body + */ + async getGeneratedTemplate(templateId: string ): + Promise { + + return this.cfn.getGeneratedTemplate({ + GeneratedTemplateName: templateId, + }).promise(); + } + + /** + * Kicks off a template generation for a set of resources. + * + * @param stackName The name of the stack + * @param resources A list of resources to generate the template from + * @returns CreateGeneratedTemplateOutput an object containing the template arn to query on later + */ + async createGeneratedTemplate(stackName: string, resources: CloudFormation.ResourceDefinitions ) { + + const createTemplateOutput = await this.cfn.createGeneratedTemplate({ + Resources: resources, + GeneratedTemplateName: stackName, + }).promise(); + + if (createTemplateOutput.GeneratedTemplateId === undefined) { + throw new Error('CreateGeneratedTemplate failed to retrun an Arn.'); + } + return createTemplateOutput; + } +} + +/** + * The possible ways to choose a scan to generate a CDK application from + */ +export enum FromScan { + /** + * Initiate a new resource scan to build the CDK application from. + */ + NEW, + + /** + * Use the last successful scan to build the CDK application from. Will fail if no scan is found. + */ + MOST_RECENT, + + /** + * Starts a scan if none exists, otherwise uses the most recent successful scan to build the CDK application from. + */ + DEFAULT, +} + +/** + * Interface for the options object passed to the generateTemplate function + * + * @param stackName The name of the stack + * @param filters A list of filters to apply to the scan + * @param fromScan An enum value specifying whether a new scan should be started or the most recent successful scan should be used + * @param sdkProvider The sdk provider for making CloudFormation calls + * @param environment The account and region where the stack is deployed + */ +export interface GenerateTemplateOptions { + stackName: string; + filters?: string[]; + fromScan?: FromScan; + sdkProvider: SdkProvider; + environment: Environment; +} + +/** + * Interface for the output of the generateTemplate function + * + * @param TemplateBody The generated template + * @param Source The source of the template + * @param Resources A list of resources that were used to generate the template + */ +export interface GenerateTemplateOutput { + templateBody: string; + source: string; + resources?: GeneratedResourceImportIdentifier[]; +} + +/** + * Interface representing the format of a resource identifier required for resource import + * + * @param ResourceType The type of resource + * @param LogicalResourceId The logical id of the resource + * @param ResourceIdentifier The resource identifier of the resource + */ +export interface GeneratedResourceImportIdentifier { + resourceType: string; + logicalResourceId: string; + resourceIdentifier: CloudFormation.ResourceIdentifierProperties; } diff --git a/packages/aws-cdk/test/cdk-toolkit.test.ts b/packages/aws-cdk/test/cdk-toolkit.test.ts index 48501295947a0..d89dcf103035d 100644 --- a/packages/aws-cdk/test/cdk-toolkit.test.ts +++ b/packages/aws-cdk/test/cdk-toolkit.test.ts @@ -867,12 +867,6 @@ describe('synth', () => { const autoscalingTemplatePath = path.join(...templatePath, 'autoscaling-template.yml'); const s3TemplatePath = path.join(...templatePath, 's3-template.json'); - test('migrate fails when neither --from-path or --from-stack are provided', async () => { - const toolkit = defaultToolkitSetup(); - await expect(() => toolkit.migrate({ stackName: 'no-source' })).rejects.toThrow('Either `--from-path` or `--from-stack` must be used to provide the source of the CloudFormation template.'); - expect(stderrMock.mock.calls[1][0]).toContain(' ❌ Migrate failed for `no-source`: Either `--from-path` or `--from-stack` must be used to provide the source of the CloudFormation template.'); - }); - test('migrate fails when both --from-path and --from-stack are provided', async () => { const toolkit = defaultToolkitSetup(); await expect(() => toolkit.migrate({ @@ -925,7 +919,7 @@ describe('synth', () => { fromPath: path.join(__dirname, 'commands', 'test-resources', 'templates', 'sqs-template.json'), language: 'rust', })).rejects.toThrowError('stack generation failed due to error \'unreachable\''); - expect(stderrMock.mock.calls[1][0]).toContain(' ❌ Migrate failed for `cannot-generate-template`: stack generation failed due to error \'unreachable\''); + expect(stderrMock.mock.calls[1][0].trim()).toContain(' ❌ Migrate failed for `cannot-generate-template`: stack generation failed due to error \'unreachable\''); }); cliTest('migrate succeeds for valid template from local path when no lanugage is provided', async (workDir) => { diff --git a/packages/aws-cdk/test/commands/migrate.test.ts b/packages/aws-cdk/test/commands/migrate.test.ts index c26870bc85b7b..adcb65d438995 100644 --- a/packages/aws-cdk/test/commands/migrate.test.ts +++ b/packages/aws-cdk/test/commands/migrate.test.ts @@ -2,8 +2,9 @@ import { exec as _exec } from 'child_process'; import * as os from 'os'; import * as path from 'path'; import { promisify } from 'util'; +import { CloudFormation } from 'aws-sdk'; import * as fs from 'fs-extra'; -import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, validateSourceOptions } from '../../lib/commands/migrate'; +import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, parseSourceOptions, generateTemplate, TemplateSourceOptions, GenerateTemplateOptions, FromScan } from '../../lib/commands/migrate'; import { MockSdkProvider, MockedObject, SyncHandlerSubsetOf } from '../util/mock-sdk'; const exec = promisify(_exec); @@ -19,6 +20,7 @@ describe('Migrate Function Tests', () => { const stackPath = [...testResourcePath, 'stacks']; const validTemplatePath = path.join(...templatePath, 's3-template.json'); + const emptyTemplatePath = path.join(...templatePath, 'empty-template.yml'); const validTemplate = readFromPath(validTemplatePath)!; beforeEach(async () => { @@ -29,28 +31,28 @@ describe('Migrate Function Tests', () => { sdkProvider.stubCloudFormation(cfnMocks as any); }); - test('validateSourceOptions throws if both --from-path and --from-stack is provided', () => { - expect(() => validateSourceOptions('any-value', true)).toThrowError('Only one of `--from-path` or `--from-stack` may be provided.'); + test('parseSourceOptions throws if both --from-path and --from-stack is provided', () => { + expect(() => parseSourceOptions('any-value', true, 'my-awesome-stack')).toThrowError('Only one of `--from-path` or `--from-stack` may be provided.'); }); - test('validateSourceOptions throws if neither --from-path or --from-stack is provided', () => { - expect(() => validateSourceOptions(undefined, undefined)).toThrowError('Either `--from-path` or `--from-stack` must be used to provide the source of the CloudFormation template.'); + test('parseSourceOptions returns from-scan when neither --from-path or --from-stack are provided', () => { + expect(parseSourceOptions(undefined, undefined, 'my-stack-name')).toStrictEqual({ source: TemplateSourceOptions.SCAN }); }); - test('validateSourceOptions does not throw when only --from-path is supplied', () => { - expect(() => validateSourceOptions('any-value', false)).not.toThrow(); + test('parseSourceOptions does not throw when only --from-path is supplied', () => { + expect(parseSourceOptions('my-file-path', undefined, 'my-stack-name')).toStrictEqual({ source: TemplateSourceOptions.PATH, templatePath: 'my-file-path' }); }); - test('validateSourceOptions does now throw when only --from-stack is provided', () => { - expect(() => validateSourceOptions(undefined, true)).not.toThrow(); + test('parseSourceOptions does now throw when only --from-stack is provided', () => { + expect(parseSourceOptions(undefined, true, 'my-stack-name')).toStrictEqual({ source: TemplateSourceOptions.STACK, stackName: 'my-stack-name' }); }); test('readFromPath produces a string representation of the template at a given path', () => { expect(readFromPath(validTemplatePath)).toEqual(fs.readFileSync(validTemplatePath, 'utf8')); }); - test('readFromPath returns undefined when template file is not provided', () => { - expect(readFromPath()).toEqual(undefined); + test('readFromPath throws error when template file is empty', () => { + expect(() => readFromPath(emptyTemplatePath)).toThrow(`\'${emptyTemplatePath}\' is an empty file.`); }); test('readFromPath throws error when template file does not exist at a given path', () => { @@ -252,4 +254,248 @@ async function withTempDir(cb: (dir: string) => void | Promise) { } finally { await fs.remove(tmpDir); } -} \ No newline at end of file +} + +describe('generateTemplate', () => { + let sdkProvider: MockSdkProvider; + let cloudFormationMocks: MockedObject>; + const sampleResource = { + ResourceType: 'AWS::S3::Bucket', + ManagedByStack: true, + ResourceIdentifier: { 'my-key': 'my-bucket' }, + LogicalResourceId: 'my-bucket', + }; + const sampleResource2 = { + ResourceType: 'AWS::EC2::Instance', + ResourceIdentifier: { + instanceId: 'i-1234567890abcdef0', + }, + LogicalResourceId: 'my-ec2-instance', + ManagedByStack: true, + }; + const stackName = 'my-stack'; + const environment = setEnvironment('123456789012', 'us-east-1'); + const scanId = 'fake-scan-id'; + + beforeEach(() => { + sdkProvider = new MockSdkProvider(); + cloudFormationMocks = { + startResourceScan: jest.fn(), + listResourceScans: jest.fn(), + describeResourceScan: jest.fn(), + listResourceScanResources: jest.fn(), + createGeneratedTemplate: jest.fn(), + describeGeneratedTemplate: jest.fn(), + getGeneratedTemplate: jest.fn(), + listResourceScanRelatedResources: jest.fn(), + config: { + getCredentials: jest.fn(), + getToken: jest.fn(), + loadFromPath: jest.fn(), + update: jest.fn(), + getPromisesDependency: jest.fn(), + setPromisesDependency: jest.fn(), + customUserAgent: 'cdk-migrate', + }, + }; + + sdkProvider.stubCloudFormation(cloudFormationMocks as any); + }); + + test('generateTemplate successfully generates template with a new scan', async () => { + const resourceScanSummaries = [{ ResourceScanId: scanId, Status: 'COMPLETE', PercentageCompleted: 100 }]; + + cloudFormationMocks.startResourceScan!.mockReturnValue({ ResourceScanId: scanId }); + cloudFormationMocks.listResourceScans!.mockReturnValue({ ResourceScanSummaries: resourceScanSummaries }); + cloudFormationMocks.describeResourceScan!.mockReturnValue({ Status: 'COMPLETED' }); + cloudFormationMocks.listResourceScanResources!.mockReturnValue({ Resources: [sampleResource2] }); + cloudFormationMocks.createGeneratedTemplate!.mockReturnValue({ GeneratedTemplateId: 'template-arn' }); + cloudFormationMocks.describeGeneratedTemplate!.mockReturnValue({ Status: 'COMPLETE', Resources: [sampleResource, sampleResource2] }); + cloudFormationMocks.getGeneratedTemplate!.mockReturnValue({ TemplateBody: 'template-body' }); + cloudFormationMocks.listResourceScanRelatedResources!.mockReturnValue({ RelatedResources: [sampleResource] }); + + const opts: GenerateTemplateOptions = { + stackName: stackName, + filters: [], + fromScan: FromScan.NEW, + sdkProvider: sdkProvider, + environment: environment, + }; + const expectedResult = { + resources: [ + { + logicalResourceId: 'my-bucket', + resourceIdentifier: { 'my-key': 'my-bucket' }, + resourceType: 'AWS::S3::Bucket', + }, + { + logicalResourceId: 'my-ec2-instance', + resourceIdentifier: { instanceId: 'i-1234567890abcdef0' }, + resourceType: 'AWS::EC2::Instance', + }, + ], + source: 'template-arn', + templateBody: 'template-body', + }; + const template = await generateTemplate(opts); + expect(template).toEqual(expectedResult); + }); + + test('generateTemplate successfully defaults to latest scan instead of starting a new one', async () => { + const resourceScanSummaryComplete = [{ ResourceScanId: scanId, Status: 'COMPLETE', PercentageCompleted: 100 }]; + const resourceScanSummaryInProgress = [{ ResourceScanId: scanId, Status: 'IN_PROGRESS', PercentageCompleted: 50 }]; + + cloudFormationMocks.startResourceScan!.mockImplementation(() => { throw new Error('No >:('); }); + cloudFormationMocks.listResourceScans!.mockReturnValueOnce({ ResourceScanSummaries: resourceScanSummaryInProgress }); + cloudFormationMocks.listResourceScans!.mockReturnValueOnce({ ResourceScanSummaries: resourceScanSummaryComplete }); + cloudFormationMocks.describeResourceScan!.mockReturnValue({ Status: 'COMPLETED' }); + cloudFormationMocks.listResourceScanResources!.mockReturnValue({ Resources: [sampleResource2] }); + cloudFormationMocks.createGeneratedTemplate!.mockReturnValue({ GeneratedTemplateId: 'template-arn' }); + cloudFormationMocks.describeGeneratedTemplate!.mockReturnValue({ Status: 'COMPLETE', Resources: [sampleResource, sampleResource2] }); + cloudFormationMocks.getGeneratedTemplate!.mockReturnValue({ TemplateBody: 'template-body' }); + cloudFormationMocks.listResourceScanRelatedResources!.mockReturnValue({ RelatedResources: [sampleResource2] }); + + const opts = { + stackName: stackName, + filters: [], + newScan: true, + sdkProvider: sdkProvider, + environment: environment, + }; + const template = await generateTemplate(opts); + const expectedResult = { + resources: [ + { + logicalResourceId: 'my-bucket', + resourceIdentifier: { 'my-key': 'my-bucket' }, + resourceType: 'AWS::S3::Bucket', + }, + { + logicalResourceId: 'my-ec2-instance', + resourceIdentifier: { instanceId: 'i-1234567890abcdef0' }, + resourceType: 'AWS::EC2::Instance', + }, + ], + source: 'template-arn', + templateBody: 'template-body', + }; + expect(template).toEqual(expectedResult); + }); + + test('generateTemplate throws an error when from-scan most-recent is passed but no scans are found.', async () => { + const resourceScanSummaries: CloudFormation.ResourceScanSummary[] = []; + + cloudFormationMocks.listResourceScans!.mockReturnValue({ ResourceScanSummaries: resourceScanSummaries }); + + const opts: GenerateTemplateOptions = { + stackName: stackName, + filters: [], + fromScan: FromScan.MOST_RECENT, + sdkProvider: sdkProvider, + environment: environment, + }; + await expect(generateTemplate(opts)).rejects.toThrow('No scans found. Please either start a new scan with the `--from-scan` new or do not specify a `--from-scan` option.'); + }); + + test('generateTemplate throws an error when an invalid key is passed in the filters', async () => { + const resourceScanSummaries = [{ ResourceScanId: scanId, Status: 'COMPLETE', PercentageCompleted: 100 }]; + + cloudFormationMocks.startResourceScan!.mockReturnValue({ ResourceScanId: scanId }); + cloudFormationMocks.listResourceScans!.mockReturnValue({ ResourceScanSummaries: resourceScanSummaries }); + cloudFormationMocks.describeResourceScan!.mockReturnValue({ Status: 'COMPLETED' }); + cloudFormationMocks.listResourceScanResources!.mockReturnValue({ Resources: [sampleResource2] }); + cloudFormationMocks.createGeneratedTemplate!.mockReturnValue({ GeneratedTemplateId: 'template-arn' }); + cloudFormationMocks.describeGeneratedTemplate!.mockReturnValue({ Status: 'COMPLETE' }); + cloudFormationMocks.getGeneratedTemplate!.mockReturnValue({ TemplateBody: 'template-body' }); + cloudFormationMocks.listResourceScanRelatedResources!.mockReturnValue({ RelatedResources: [sampleResource] }); + + const opts: GenerateTemplateOptions = { + stackName: stackName, + filters: ['invalid-key=invalid-value'], + fromScan: FromScan.MOST_RECENT, + sdkProvider: sdkProvider, + environment: environment, + }; + await expect(generateTemplate(opts)).rejects.toThrow('Invalid filter: invalid-key'); + }); + + test('generateTemplate defaults to starting a new scan when no options are provided', async () => { + const resourceScanSummaryComplete = [{ ResourceScanId: scanId, Status: 'COMPLETE', PercentageCompleted: 100 }]; + const resourceScanSummaryInProgress = [{ ResourceScanId: scanId, Status: 'IN_PROGRESS', PercentageCompleted: 50 }]; + + cloudFormationMocks.startResourceScan!.mockReturnValue({ ResourceScanId: scanId }); + cloudFormationMocks.listResourceScans!.mockReturnValueOnce({ ResourceScanSummaries: undefined }); + cloudFormationMocks.listResourceScans!.mockReturnValueOnce({ ResourceScanSummaries: resourceScanSummaryInProgress }); + cloudFormationMocks.listResourceScans!.mockReturnValueOnce({ ResourceScanSummaries: resourceScanSummaryComplete }); + cloudFormationMocks.describeResourceScan!.mockReturnValue({ Status: 'COMPLETED' }); + cloudFormationMocks.listResourceScanResources!.mockReturnValue({ Resources: [sampleResource2] }); + cloudFormationMocks.createGeneratedTemplate!.mockReturnValue({ GeneratedTemplateId: 'template-arn' }); + cloudFormationMocks.describeGeneratedTemplate!.mockReturnValue({ Status: 'COMPLETE', Resources: [sampleResource, sampleResource2] }); + cloudFormationMocks.getGeneratedTemplate!.mockReturnValue({ TemplateBody: 'template-body' }); + cloudFormationMocks.listResourceScanRelatedResources!.mockReturnValue({ RelatedResources: [sampleResource] }); + + const opts: GenerateTemplateOptions = { + stackName: stackName, + sdkProvider: sdkProvider, + environment: environment, + }; + const template = await generateTemplate(opts); + const expectedResult = { + resources: [ + { + logicalResourceId: 'my-bucket', + resourceIdentifier: { 'my-key': 'my-bucket' }, + resourceType: 'AWS::S3::Bucket', + }, + { + logicalResourceId: 'my-ec2-instance', + resourceIdentifier: { instanceId: 'i-1234567890abcdef0' }, + resourceType: 'AWS::EC2::Instance', + }, + ], + source: 'template-arn', + templateBody: 'template-body', + }; + expect(template).toEqual(expectedResult); + expect(cloudFormationMocks.startResourceScan).toHaveBeenCalled(); + }); + + test('generateTemplate successfully generates templates with valid filter options', async () => { + const resourceScanSummaries = [{ ResourceScanId: scanId, Status: 'COMPLETE', PercentageCompleted: 100 }]; + + cloudFormationMocks.startResourceScan!.mockReturnValue({ ResourceScanId: scanId }); + cloudFormationMocks.listResourceScans!.mockReturnValue({ ResourceScanSummaries: resourceScanSummaries }); + cloudFormationMocks.describeResourceScan!.mockReturnValue({ Status: 'COMPLETED' }); + cloudFormationMocks.listResourceScanResources!.mockReturnValue({ Resources: [sampleResource2] }); + cloudFormationMocks.createGeneratedTemplate!.mockReturnValue({ GeneratedTemplateId: 'template-arn' }); + cloudFormationMocks.describeGeneratedTemplate!.mockReturnValue({ Status: 'COMPLETE', Resources: [sampleResource, sampleResource2] }); + cloudFormationMocks.getGeneratedTemplate!.mockReturnValue({ TemplateBody: 'template-body' }); + cloudFormationMocks.listResourceScanRelatedResources!.mockReturnValue({ RelatedResources: [sampleResource] }); + + const opts: GenerateTemplateOptions = { + stackName: stackName, + filters: ['type=AWS::S3::Bucket,identifier={"my-key":"my-bucket"}', 'type=AWS::EC2::Instance'], + sdkProvider: sdkProvider, + environment: environment, + }; + const template = await generateTemplate(opts); + const expectedResult = { + resources: [ + { + logicalResourceId: 'my-bucket', + resourceIdentifier: { 'my-key': 'my-bucket' }, + resourceType: 'AWS::S3::Bucket', + }, + { + logicalResourceId: 'my-ec2-instance', + resourceIdentifier: { instanceId: 'i-1234567890abcdef0' }, + resourceType: 'AWS::EC2::Instance', + }, + ], + source: 'template-arn', + templateBody: 'template-body', + }; + expect(template).toEqual(expectedResult); + }); + +}); \ No newline at end of file diff --git a/packages/aws-cdk/test/commands/test-resources/templates/empty-template.yml b/packages/aws-cdk/test/commands/test-resources/templates/empty-template.yml new file mode 100644 index 0000000000000..e69de29bb2d1d From a45e25511e60eaf78bdd28e2127c7e9282644ce6 Mon Sep 17 00:00:00 2001 From: Hogan Bobertz Date: Wed, 31 Jan 2024 17:45:20 -0500 Subject: [PATCH 2/6] chore: add readme generation into cdk migrate (#223) * add readme generation * update testing and add readme udpates * add test for readme generation * fix stupid bug I created * improve error message * uncomment tests * address feedback * update readme generation * update warnings: * update warnings: --------- Co-authored-by: Hogan Bobertz --- .../cli-integ/resources/cdk-apps/app/app.js | 15 +++ .../tests/cli-integ-tests/cli.integtest.ts | 121 +++++++++++------- packages/aws-cdk/lib/cdk-toolkit.ts | 22 +++- packages/aws-cdk/lib/commands/migrate.ts | 96 ++++++++++++-- .../aws-cdk/test/commands/migrate.test.ts | 109 ++++++---------- 5 files changed, 233 insertions(+), 130 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js index 746017ceee30d..ed1117dc9dc94 100755 --- a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js @@ -100,6 +100,21 @@ class MigrateStack extends cdk.Stack { }); cdk.Tags.of(myQueue).add('tag3', 'value3'); } + if (process.env.LAMBDA_RESOURCES) { + const myFunction = new lambda.Function(this, 'migratefunction1', { + code: lambda.Code.fromInline('console.log("hello world")'), + handler: 'index.handler', + runtime: lambda.Runtime.NODEJS_18_X, + }); + cdk.Tags.of(myFunction).add('lambda-tag', 'lambda-value'); + + const myFunction2 = new lambda.Function(this, 'migratefunction2', { + code: lambda.Code.fromInline('console.log("hello world2")'), + handler: 'index.handler', + runtime: lambda.Runtime.NODEJS_18_X, + }); + cdk.Tags.of(myFunction2).add('lambda-tag', 'lambda-value'); + } } } diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts index 134b303348475..b29e12b69a60e 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts @@ -588,57 +588,90 @@ integTest('deploy with role', withDefaultFixture(async (fixture) => { })); }); -['typescript', 'python', 'csharp', 'java'].forEach(language => { - integTest(`cdk migrate generates migrate.json in ${language}`, withCDKMigrateFixture(language, async (fixture) => { - - const migrateFile = await fs.readFile(path.join(fixture.integTestDir, 'migrate.json'), 'utf8'); - const expectedFile = `{ - \"//\": \"This file is generated by cdk migrate. It will be automatically deleted after the first successful deployment of this app to the environment of the original resources.\", - \"Source\": \"localfile\" - }`; - expect(JSON.parse(migrateFile)).toEqual(JSON.parse(expectedFile)); - await fixture.cdkDestroy(fixture.stackNamePrefix); - })); -}); +integTest('cdk migrate generates migrate.json', withCDKMigrateFixture('typescript', async (fixture) => { + + const migrateFile = await fs.readFile(path.join(fixture.integTestDir, 'migrate.json'), 'utf8'); + const expectedFile = `{ + \"//\": \"This file is generated by cdk migrate. It will be automatically deleted after the first successful deployment of this app to the environment of the original resources.\", + \"Source\": \"localfile\" + }`; + expect(JSON.parse(migrateFile)).toEqual(JSON.parse(expectedFile)); + await fixture.cdkDestroy(fixture.stackNamePrefix); +})); + +integTest('cdk migrate --from-scan with AND/OR filters correctly filters resources', withDefaultFixture(async (fixture) => { + const migrateStackJr = 'migrate-stack-jr'; + + await fixture.cdkDeploy('migrate-stack', { + modEnv: { SAMPLE_RESOURCES: '1' }, + }); + await fixture.cdk( + ['migrate', '--stack-name', migrateStackJr, '--from-scan', 'new', '--filter', 'type=AWS::SNS::Topic,tag-key=tag1', 'type=AWS::SQS::Queue,tag-key=tag3'], + { modEnv: { MIGRATE_INTEG_TEST: '1' }, neverRequireApproval: true, verbose: true, captureStderr: false }, + ); -// TODO: Uncomment this out before launch. Commenting this for now so pipelines won't complain. -// integTest('cdk migrate --from-scan with AND/OR filters correctly filters resources', withDefaultFixture(async (fixture) => { -// const migrateStackJr = 'migrate-stack-jr'; - -// await fixture.cdkDeploy('migrate-stack', { -// modEnv: { SAMPLE_RESOURCES: '1' }, -// }); -// await fixture.cdk( -// ['migrate', '--stack-name', migrateStackJr, '--from-scan', 'new', '--filter', 'type=AWS::SNS::Topic,tag-key=tag1', 'type=AWS::SQS::Queue,tag-key=tag3'], -// { modEnv: { MIGRATE_INTEG_TEST: '1' }, neverRequireApproval: true, verbose: true, captureStderr: false }, -// ); - -// try { - -// const response = await fixture.aws.cloudFormation('describeGeneratedTemplate', { -// GeneratedTemplateName: migrateStackJr, -// }); -// const resourceNames = []; -// for (const resource of response.Resources || []) { -// if (resource.LogicalResourceId) { -// resourceNames.push(resource.LogicalResourceId); -// } -// } -// fixture.log(`Resources: ${resourceNames}`); -// expect(resourceNames.some(ele => ele && ele.includes('migratetopic1'))).toBeTruthy(); -// expect(resourceNames.some(ele => ele && ele.includes('migratequeue1'))).toBeTruthy(); -// expect(response.Resources?.length).toEqual(2); -// } finally { -// await fixture.cdkDestroy('migrate-stack'); -// } -// })); + try { + + const response = await fixture.aws.cloudFormation('describeGeneratedTemplate', { + GeneratedTemplateName: migrateStackJr, + }); + const resourceNames = []; + for (const resource of response.Resources || []) { + if (resource.LogicalResourceId) { + resourceNames.push(resource.LogicalResourceId); + } + } + fixture.log(`Resources: ${resourceNames}`); + expect(resourceNames.some(ele => ele && ele.includes('migratetopic1'))).toBeTruthy(); + expect(resourceNames.some(ele => ele && ele.includes('migratequeue1'))).toBeTruthy(); + expect(response.Resources?.length).toEqual(2); + } finally { + await fixture.cdkDestroy('migrate-stack'); + } +})); + +integTest('cdk migrate --from-scan for resources with Write Only Properties generates warnings', withDefaultFixture(async (fixture) => { + const stackName = `cdk-migrate-integ-${fixture.randomString}`; + + await fixture.cdkDeploy('migrate-stack', { + modEnv: { + LAMBDA_RESOURCES: '1', + }, + }); + await fixture.cdk( + ['migrate', '--stack-name', stackName, '--from-scan', 'new', '--filter', 'type=AWS::Lambda::Function,tag-key=lambda-tag'], + { modEnv: { MIGRATE_INTEG_TEST: '1' }, neverRequireApproval: true, verbose: true, captureStderr: false }, + ); + + try { + + const response = await fixture.aws.cloudFormation('describeGeneratedTemplate', { + GeneratedTemplateName: stackName, + }); + const resourceNames = []; + for (const resource of response.Resources || []) { + if (resource.LogicalResourceId && resource.ResourceType === 'AWS::Lambda::Function') { + resourceNames.push(resource.LogicalResourceId); + } + } + fixture.log(`Resources: ${resourceNames}`); + const readmePath = path.join(fixture.integTestDir, stackName, 'README.md'); + const readme = await fs.readFile(readmePath, 'utf8'); + expect(readme).toContain('## Warnings'); + for (const resourceName of resourceNames) { + expect(readme).toContain(`### ${resourceName}`); + } + } finally { + await fixture.cdkDestroy('migrate-stack'); + } +})); ['typescript', 'python', 'csharp', 'java'].forEach(language => { integTest(`cdk migrate --from-stack creates deployable ${language} app`, withDefaultFixture(async (fixture) => { const migrateStackName = fixture.fullStackName('migrate-stack'); await fixture.aws.cloudFormation('createStack', { StackName: migrateStackName, - TemplateBody: await fs.readFile(path.join(__dirname, '..', '..', 'resources', 'templates', 'sqs-template.json'), 'utf8').toString(), + TemplateBody: await fs.readFile(path.join(__dirname, '..', '..', 'resources', 'templates', 'sqs-template.json'), 'utf8'), }); try { let stackStatus = 'CREATE_IN_PROGRESS'; diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 4e769134753a1..d4fc838562814 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -17,7 +17,7 @@ import { findCloudWatchLogGroups } from './api/logs/find-cloudwatch-logs'; import { CloudWatchLogEventMonitor } from './api/logs/logs-monitor'; import { createDiffChangeSet, ResourcesToImport } from './api/util/cloudformation'; import { StackActivityProgress } from './api/util/cloudformation/stack-activity-monitor'; -import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, parseSourceOptions, generateTemplate, FromScan, TemplateSourceOptions, GenerateTemplateOutput, CfnTemplateGeneratorProvider, writeMigrateJsonFile, buildGenertedTemplateOutput, buildCfnClient } from './commands/migrate'; +import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, parseSourceOptions, generateTemplate, FromScan, TemplateSourceOptions, GenerateTemplateOutput, CfnTemplateGeneratorProvider, writeMigrateJsonFile, buildGenertedTemplateOutput, buildCfnClient, appendWarningsToReadme, isThereAWarning } from './commands/migrate'; import { printSecurityDiff, printStackDiff, RequireApproval } from './diff'; import { ResourceImporter } from './import'; import { data, debug, error, highlight, print, success, warning, withCorkedLogging } from './logging'; @@ -746,8 +746,10 @@ export class CdkToolkit { generateTemplateOutput = buildGenertedTemplateOutput(generatedTemplateSummary, templateBody, generatedTemplateSummary.GeneratedTemplateId!); } else { generateTemplateOutput = { - templateBody: templateBody, - source: 'localfile', + migrateJson: { + templateBody: templateBody, + source: 'localfile', + }, }; } } else if (scanType == TemplateSourceOptions.STACK) { @@ -756,18 +758,24 @@ export class CdkToolkit { throw new Error(`No template found for stack-name: ${options.stackName}`); } generateTemplateOutput = { - templateBody: template, - source: options.stackName, + migrateJson: { + templateBody: template, + source: options.stackName, + }, }; } else { // We shouldn't ever get here, but just in case. throw new Error(`Invalid source option provided: ${scanType}`); } - const stack = generateStack(generateTemplateOutput!.templateBody, options.stackName, language); + const stack = generateStack(generateTemplateOutput.migrateJson.templateBody, options.stackName, language); success(' ⏳ Generating CDK app for %s...', chalk.blue(options.stackName)); await generateCdkApp(options.stackName, stack!, language, options.outputPath, options.compress); if (generateTemplateOutput) { - writeMigrateJsonFile(options.outputPath, options.stackName, generateTemplateOutput); + writeMigrateJsonFile(options.outputPath, options.stackName, generateTemplateOutput.migrateJson); + } + if (isThereAWarning(generateTemplateOutput)) { + warning(' ⚠️ Some resources could not be migrated completely. Please review the README.md file for more information.'); + appendWarningsToReadme(`${path.join(options.outputPath ?? process.cwd(), options.stackName)}/README.md`, generateTemplateOutput.resources!); } } catch (e) { error(' ❌ Migrate failed for `%s`: %s', options.stackName, (e as Error).message); diff --git a/packages/aws-cdk/lib/commands/migrate.ts b/packages/aws-cdk/lib/commands/migrate.ts index 391628705a953..04b44298781d6 100644 --- a/packages/aws-cdk/lib/commands/migrate.ts +++ b/packages/aws-cdk/lib/commands/migrate.ts @@ -473,11 +473,11 @@ export function displayTimeDiff(time1: Date, time2: Date): void { * @param stackName The name of the stack * @param generatedOutput The output of the template generator */ -export function writeMigrateJsonFile(outputPath: string | undefined, stackName: string, generatedOutput: GenerateTemplateOutput) { +export function writeMigrateJsonFile(outputPath: string | undefined, stackName: string, migrateJson: MigrateJsonFormat) { const outputToJson = { '//': 'This file is generated by cdk migrate. It will be automatically deleted after the first successful deployment of this app to the environment of the original resources.', - 'Source': generatedOutput.source, - 'Resources': generatedOutput.resources, + 'Source': migrateJson.source, + 'Resources': migrateJson.resources, }; fs.writeFileSync(`${path.join(outputPath ?? process.cwd(), stackName)}/migrate.json`, JSON.stringify(outputToJson, null, 2)); } @@ -503,6 +503,23 @@ export function getMigrateScanType(scanType: string) { } } +/** + * Takes a generatedTemplateOutput objct and returns a boolean representing whether there are any warnings on any rescources. + * + * @param generatedTemplateOutput A GenerateTemplateOutput object + * @returns A boolean representing whether there are any warnings on any rescources + */ +export function isThereAWarning(generatedTemplateOutput: GenerateTemplateOutput) { + if (generatedTemplateOutput.resources) { + for (const resource of generatedTemplateOutput.resources) { + if (resource.Warnings && resource.Warnings.length > 0) { + return true; + } + } + } + return false; +} + /** * Builds the GenerateTemplateOutput object from the DescribeGeneratedTemplateOutput and the template body. * @@ -512,17 +529,29 @@ export function getMigrateScanType(scanType: string) { */ export function buildGenertedTemplateOutput(generatedTemplateSummary: CloudFormation.DescribeGeneratedTemplateOutput, templateBody: string, source: string): GenerateTemplateOutput { - return { + const resources: CloudFormation.ResourceDetails | undefined = generatedTemplateSummary.Resources; + const migrateJson: MigrateJsonFormat = { templateBody: templateBody, source: source, resources: generatedTemplateSummary.Resources!.map((r) => ({ - resourceType: r.ResourceType!, - logicalResourceId: r.LogicalResourceId!, - resourceIdentifier: r.ResourceIdentifier!, + ResourceType: r.ResourceType!, + LogicalResourceId: r.LogicalResourceId!, + ResourceIdentifier: r.ResourceIdentifier!, })), }; + return { + migrateJson: migrateJson, + resources: resources, + }; } +/** + * Builds a CloudFormation sdk client for making requests with the CFN template generator. + * + * @param sdkProvider The sdk provider for making CloudFormation calls + * @param environment The account and region where the stack is deployed + * @returns A CloudFormation sdk client + */ export async function buildCfnClient(sdkProvider: SdkProvider, environment: Environment) { const cloudFormationClient = (await sdkProvider.forEnvironment(environment, Mode.ForReading)).sdk.cloudFormation(); @@ -531,6 +560,36 @@ export async function buildCfnClient(sdkProvider: SdkProvider, environment: Envi return cloudFormationClient; } +/** + * Appends a list of warnings to a readme file. + * + * @param filepath The path to the readme file + * @param resources A list of resources to append warnings for + */ +export function appendWarningsToReadme(filepath: string, resources: CloudFormation.ResourceDetails) { + const readme = fs.readFileSync(filepath, 'utf8'); + const lines = readme.split('\n'); + const index = lines.findIndex((line) => line.trim() === 'Enjoy!'); + let linesToAdd = ['\n## Warnings']; + linesToAdd.push('### Write-only properties'); + linesToAdd.push("Write-only properties are resource property values that can be written to but can't be read by AWS CloudFormation or CDK Migrate. For more information, see [IaC generator and write-only properties](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/generate-IaC-write-only-properties.html)."); + linesToAdd.push('\n'); + linesToAdd.push('Write-only properties discovered during migration are organized here by resource ID and categorized by write-only property type. Resolve write-only properties by providing property values in your CDK app. For guidance, see [Resolve write-only properties](https://docs.aws.amazon.com/cdk/v2/guide/migrate.html#migrate-resources-writeonly).'); + for (const resource of resources) { + if (resource.Warnings && resource.Warnings.length > 0) { + linesToAdd.push(`### ${resource.LogicalResourceId}`); + for (const warning of resource.Warnings) { + linesToAdd.push(`- **${warning.Type}**: `); + for (const property of warning.Properties!) { + linesToAdd.push(` - ${property.PropertyPath}: ${property.Description}`); + } + } + } + } + lines.splice(index, 0, ...linesToAdd); + fs.writeFileSync(filepath, lines.join('\n')); +} + /** * Class for making CloudFormation template generator calls */ @@ -688,6 +747,9 @@ export class CfnTemplateGeneratorProvider { resourceList = resourceList!.concat(nextResources.Resources ?? []); } } + if (resourceList.length === 0) { + throw new Error(`No resources found with filters ${filters.join(' ')}. Please try again with different filters.`); + } return process.env.MIGRATE_INTEG_TEST ? resourceIdentifiers(resourceList) : resourceIdentifiers(excludeManaged(resourceList)); } @@ -802,11 +864,22 @@ export interface GenerateTemplateOptions { /** * Interface for the output of the generateTemplate function * + * @param migrateJson The generated Migrate.json file + * @param resources The generated template + */ +export interface GenerateTemplateOutput { + migrateJson: MigrateJsonFormat; + resources?: CloudFormation.ResourceDetails; +} + +/** + * Interface defining the format of the generated Migrate.json file + * * @param TemplateBody The generated template * @param Source The source of the template * @param Resources A list of resources that were used to generate the template */ -export interface GenerateTemplateOutput { +export interface MigrateJsonFormat { templateBody: string; source: string; resources?: GeneratedResourceImportIdentifier[]; @@ -820,7 +893,8 @@ export interface GenerateTemplateOutput { * @param ResourceIdentifier The resource identifier of the resource */ export interface GeneratedResourceImportIdentifier { - resourceType: string; - logicalResourceId: string; - resourceIdentifier: CloudFormation.ResourceIdentifierProperties; + // cdk deploy expects the migrate.json resource identifiers to be PascalCase, not camelCase. + ResourceType: string; + LogicalResourceId: string; + ResourceIdentifier: CloudFormation.ResourceIdentifierProperties; } diff --git a/packages/aws-cdk/test/commands/migrate.test.ts b/packages/aws-cdk/test/commands/migrate.test.ts index adcb65d438995..a21b9e9d3e6a3 100644 --- a/packages/aws-cdk/test/commands/migrate.test.ts +++ b/packages/aws-cdk/test/commands/migrate.test.ts @@ -276,6 +276,42 @@ describe('generateTemplate', () => { const stackName = 'my-stack'; const environment = setEnvironment('123456789012', 'us-east-1'); const scanId = 'fake-scan-id'; + const defaultExpectedResult = { + migrateJson: { + resources: [ + { + LogicalResourceId: 'my-bucket', + ResourceIdentifier: { 'my-key': 'my-bucket' }, + ResourceType: 'AWS::S3::Bucket', + }, + { + LogicalResourceId: 'my-ec2-instance', + ResourceIdentifier: { instanceId: 'i-1234567890abcdef0' }, + ResourceType: 'AWS::EC2::Instance', + }, + ], + source: 'template-arn', + templateBody: 'template-body', + }, + resources: [ + { + LogicalResourceId: 'my-bucket', + ManagedByStack: true, + ResourceIdentifier: { + 'my-key': 'my-bucket', + }, + ResourceType: 'AWS::S3::Bucket', + }, + { + LogicalResourceId: 'my-ec2-instance', + ManagedByStack: true, + ResourceIdentifier: { + instanceId: 'i-1234567890abcdef0', + }, + ResourceType: 'AWS::EC2::Instance', + }, + ], + }; beforeEach(() => { sdkProvider = new MockSdkProvider(); @@ -321,24 +357,9 @@ describe('generateTemplate', () => { sdkProvider: sdkProvider, environment: environment, }; - const expectedResult = { - resources: [ - { - logicalResourceId: 'my-bucket', - resourceIdentifier: { 'my-key': 'my-bucket' }, - resourceType: 'AWS::S3::Bucket', - }, - { - logicalResourceId: 'my-ec2-instance', - resourceIdentifier: { instanceId: 'i-1234567890abcdef0' }, - resourceType: 'AWS::EC2::Instance', - }, - ], - source: 'template-arn', - templateBody: 'template-body', - }; + const template = await generateTemplate(opts); - expect(template).toEqual(expectedResult); + expect(template).toEqual(defaultExpectedResult); }); test('generateTemplate successfully defaults to latest scan instead of starting a new one', async () => { @@ -363,23 +384,7 @@ describe('generateTemplate', () => { environment: environment, }; const template = await generateTemplate(opts); - const expectedResult = { - resources: [ - { - logicalResourceId: 'my-bucket', - resourceIdentifier: { 'my-key': 'my-bucket' }, - resourceType: 'AWS::S3::Bucket', - }, - { - logicalResourceId: 'my-ec2-instance', - resourceIdentifier: { instanceId: 'i-1234567890abcdef0' }, - resourceType: 'AWS::EC2::Instance', - }, - ], - source: 'template-arn', - templateBody: 'template-body', - }; - expect(template).toEqual(expectedResult); + expect(template).toEqual(defaultExpectedResult); }); test('generateTemplate throws an error when from-scan most-recent is passed but no scans are found.', async () => { @@ -440,23 +445,7 @@ describe('generateTemplate', () => { environment: environment, }; const template = await generateTemplate(opts); - const expectedResult = { - resources: [ - { - logicalResourceId: 'my-bucket', - resourceIdentifier: { 'my-key': 'my-bucket' }, - resourceType: 'AWS::S3::Bucket', - }, - { - logicalResourceId: 'my-ec2-instance', - resourceIdentifier: { instanceId: 'i-1234567890abcdef0' }, - resourceType: 'AWS::EC2::Instance', - }, - ], - source: 'template-arn', - templateBody: 'template-body', - }; - expect(template).toEqual(expectedResult); + expect(template).toEqual(defaultExpectedResult); expect(cloudFormationMocks.startResourceScan).toHaveBeenCalled(); }); @@ -479,23 +468,7 @@ describe('generateTemplate', () => { environment: environment, }; const template = await generateTemplate(opts); - const expectedResult = { - resources: [ - { - logicalResourceId: 'my-bucket', - resourceIdentifier: { 'my-key': 'my-bucket' }, - resourceType: 'AWS::S3::Bucket', - }, - { - logicalResourceId: 'my-ec2-instance', - resourceIdentifier: { instanceId: 'i-1234567890abcdef0' }, - resourceType: 'AWS::EC2::Instance', - }, - ], - source: 'template-arn', - templateBody: 'template-body', - }; - expect(template).toEqual(expectedResult); + expect(template).toEqual(defaultExpectedResult); }); }); \ No newline at end of file From 3a8f92690c99729b91aba97f740be180852f84e1 Mon Sep 17 00:00:00 2001 From: Hogan Bobertz Date: Thu, 1 Feb 2024 12:34:37 -0500 Subject: [PATCH 3/6] chore: fix cdk migrate not deleting templates and lack of deduplication (#228) * fix bugs * only create cfnTemplateGenerator if we need it * fix integ tests --------- Co-authored-by: Hogan Bobertz --- .../tests/cli-integ-tests/cli.integtest.ts | 14 +++-- packages/aws-cdk/lib/cdk-toolkit.ts | 18 +++++- packages/aws-cdk/lib/commands/migrate.ts | 55 +++++++++++++------ 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts index b29e12b69a60e..ac3281cc07dba 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts @@ -600,20 +600,19 @@ integTest('cdk migrate generates migrate.json', withCDKMigrateFixture('typescrip })); integTest('cdk migrate --from-scan with AND/OR filters correctly filters resources', withDefaultFixture(async (fixture) => { - const migrateStackJr = 'migrate-stack-jr'; + const stackName = `cdk-migrate-integ-${fixture.randomString}`; await fixture.cdkDeploy('migrate-stack', { modEnv: { SAMPLE_RESOURCES: '1' }, }); await fixture.cdk( - ['migrate', '--stack-name', migrateStackJr, '--from-scan', 'new', '--filter', 'type=AWS::SNS::Topic,tag-key=tag1', 'type=AWS::SQS::Queue,tag-key=tag3'], + ['migrate', '--stack-name', stackName, '--from-scan', 'new', '--filter', 'type=AWS::SNS::Topic,tag-key=tag1', 'type=AWS::SQS::Queue,tag-key=tag3'], { modEnv: { MIGRATE_INTEG_TEST: '1' }, neverRequireApproval: true, verbose: true, captureStderr: false }, ); try { - const response = await fixture.aws.cloudFormation('describeGeneratedTemplate', { - GeneratedTemplateName: migrateStackJr, + GeneratedTemplateName: stackName, }); const resourceNames = []; for (const resource of response.Resources || []) { @@ -624,9 +623,11 @@ integTest('cdk migrate --from-scan with AND/OR filters correctly filters resourc fixture.log(`Resources: ${resourceNames}`); expect(resourceNames.some(ele => ele && ele.includes('migratetopic1'))).toBeTruthy(); expect(resourceNames.some(ele => ele && ele.includes('migratequeue1'))).toBeTruthy(); - expect(response.Resources?.length).toEqual(2); } finally { await fixture.cdkDestroy('migrate-stack'); + await fixture.aws.cloudFormation('deleteGeneratedTemplate', { + GeneratedTemplateName: stackName, + }); } })); @@ -663,6 +664,9 @@ integTest('cdk migrate --from-scan for resources with Write Only Properties gene } } finally { await fixture.cdkDestroy('migrate-stack'); + await fixture.aws.cloudFormation('deleteGeneratedTemplate', { + GeneratedTemplateName: stackName, + }); } })); diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index d4fc838562814..28128e333de27 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -719,10 +719,12 @@ export class CdkToolkit { public async migrate(options: MigrateOptions): Promise { warning('This is an experimental feature and development on it is still in progress. We make no guarantees about the outcome or stability of the functionality.'); const language = options.language?.toLowerCase() ?? 'typescript'; + const environment = setEnvironment(options.account, options.region); + let generateTemplateOutput: GenerateTemplateOutput | undefined; + let cfn: CfnTemplateGeneratorProvider | undefined; + let templateToDelete: string | undefined; try { - const environment = setEnvironment(options.account, options.region); - let generateTemplateOutput: GenerateTemplateOutput | undefined; // if neither fromPath nor fromStack is provided, generate a template using cloudformation const scanType = parseSourceOptions(options.fromPath, options.fromStack, options.stackName).source; if (scanType == TemplateSourceOptions.SCAN) { @@ -733,6 +735,7 @@ export class CdkToolkit { sdkProvider: this.props.sdkProvider, environment: environment, }); + templateToDelete = generateTemplateOutput.templateId; } else if (scanType == TemplateSourceOptions.PATH) { const templateBody = readFromPath(options.fromPath!); @@ -741,7 +744,7 @@ export class CdkToolkit { if (templateId) { // if we have a template id, we can call describe generated template to get the resource identifiers // resource metadata, and template source to generate the template - const cfn = new CfnTemplateGeneratorProvider(await buildCfnClient(this.props.sdkProvider, environment)); + cfn = new CfnTemplateGeneratorProvider(await buildCfnClient(this.props.sdkProvider, environment)); const generatedTemplateSummary = await cfn.describeGeneratedTemplate(templateId); generateTemplateOutput = buildGenertedTemplateOutput(generatedTemplateSummary, templateBody, generatedTemplateSummary.GeneratedTemplateId!); } else { @@ -780,6 +783,15 @@ export class CdkToolkit { } catch (e) { error(' ❌ Migrate failed for `%s`: %s', options.stackName, (e as Error).message); throw e; + } finally { + if (templateToDelete) { + if (!cfn) { + cfn = new CfnTemplateGeneratorProvider(await buildCfnClient(this.props.sdkProvider, environment)); + } + if (!process.env.MIGRATE_INTEG_TEST) { + await cfn.deleteGeneratedTemplate(templateToDelete); + } + } } } diff --git a/packages/aws-cdk/lib/commands/migrate.ts b/packages/aws-cdk/lib/commands/migrate.ts index 04b44298781d6..fd34a6f6184f8 100644 --- a/packages/aws-cdk/lib/commands/migrate.ts +++ b/packages/aws-cdk/lib/commands/migrate.ts @@ -539,9 +539,11 @@ export function buildGenertedTemplateOutput(generatedTemplateSummary: CloudForma ResourceIdentifier: r.ResourceIdentifier!, })), }; + const templateId = generatedTemplateSummary.GeneratedTemplateId!; return { migrateJson: migrateJson, resources: resources, + templateId: templateId, }; } @@ -590,6 +592,27 @@ export function appendWarningsToReadme(filepath: string, resources: CloudFormati fs.writeFileSync(filepath, lines.join('\n')); } +/** + * takes a list of resources and returns a list of unique resources based on the resource type and logical resource id. + * + * @param resources A list of resources to deduplicate + * @returns A list of unique resources + */ +function deduplicateResources(resources: CloudFormation.ResourceDetails) { + let uniqueResources: {[key: string]: CloudFormation.ResourceDetail} = {}; + + for (const resource of resources) { + const key = Object.keys(resource.ResourceIdentifier!)[0]; + + // Creating our unique identifier using the resource type, the key, and the value of the resource identifier + // The resource identifier is a combination of a key value pair defined by a resource's schema, and the resource type of the resource. + const uniqueIdentifer = `${resource.ResourceType}:${key}:${resource.ResourceIdentifier![key]}`; + uniqueResources[uniqueIdentifer] = resource; + } + + return Object.values(uniqueResources); +}; + /** * Class for making CloudFormation template generator calls */ @@ -647,23 +670,7 @@ export class CfnTemplateGeneratorProvider { } } - let uniqueResources: {[key: string]: CloudFormation.ScannedResource} = {}; - - // de-duplicating the list of related resources based on the resource identifier value - // The resource identifier key is not known at compile time so we need to get it using Object.keys. There will only - // ever be a single key so [0] retrives the first and only key - for (const resource of relatedResourceList) { - if (!resource.ResourceIdentifier) continue; - const key = Object.keys(resource.ResourceIdentifier!)[0]; - - // Creating our unique identifier using the resource type, the key, and the value of the resource identifier - // The resource identifier is a combination of a key value pair defined by a resource's schema, and the resource type of the resource. - const uniqueIdentifer = `${resource.ResourceType}:${key}:${resource.ResourceIdentifier![key]}`; - uniqueResources[uniqueIdentifer] = resource; - } - - // convert the dictionary back into a list - relatedResourceList = Object.values(uniqueResources); + relatedResourceList = deduplicateResources(relatedResourceList); // prune the managedbystack flag off of them again. return process.env.MIGRATE_INTEG_TEST ? resourceIdentifiers(relatedResourceList) : resourceIdentifiers(excludeManaged(relatedResourceList)) ; @@ -750,6 +757,7 @@ export class CfnTemplateGeneratorProvider { if (resourceList.length === 0) { throw new Error(`No resources found with filters ${filters.join(' ')}. Please try again with different filters.`); } + resourceList = deduplicateResources(resourceList); return process.env.MIGRATE_INTEG_TEST ? resourceIdentifiers(resourceList) : resourceIdentifiers(excludeManaged(resourceList)); } @@ -822,6 +830,18 @@ export class CfnTemplateGeneratorProvider { } return createTemplateOutput; } + + /** + * Deletes a generated template from the template generator. + * + * @param templateArn The arn of the template to delete + * @returns A promise that resolves when the template has been deleted + */ + async deleteGeneratedTemplate(templateArn: string): Promise { + await this.cfn.deleteGeneratedTemplate({ + GeneratedTemplateName: templateArn, + }).promise(); + } } /** @@ -870,6 +890,7 @@ export interface GenerateTemplateOptions { export interface GenerateTemplateOutput { migrateJson: MigrateJsonFormat; resources?: CloudFormation.ResourceDetails; + templateId?: string; } /** From 4ec6ab591b765d26bdfd29cf9fb243913db133c2 Mon Sep 17 00:00:00 2001 From: Hogan Bobertz Date: Thu, 1 Feb 2024 14:25:05 -0500 Subject: [PATCH 4/6] remove trim command that fails codebuild job --- packages/aws-cdk/test/cdk-toolkit.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-cdk/test/cdk-toolkit.test.ts b/packages/aws-cdk/test/cdk-toolkit.test.ts index d89dcf103035d..1a6f55ef63c52 100644 --- a/packages/aws-cdk/test/cdk-toolkit.test.ts +++ b/packages/aws-cdk/test/cdk-toolkit.test.ts @@ -919,7 +919,7 @@ describe('synth', () => { fromPath: path.join(__dirname, 'commands', 'test-resources', 'templates', 'sqs-template.json'), language: 'rust', })).rejects.toThrowError('stack generation failed due to error \'unreachable\''); - expect(stderrMock.mock.calls[1][0].trim()).toContain(' ❌ Migrate failed for `cannot-generate-template`: stack generation failed due to error \'unreachable\''); + expect(stderrMock.mock.calls[1][0]).toContain(' ❌ Migrate failed for `cannot-generate-template`: stack generation failed due to error \'unreachable\''); }); cliTest('migrate succeeds for valid template from local path when no lanugage is provided', async (workDir) => { From fc3650c42b393fabc6c70068c339ac76b0b69daf Mon Sep 17 00:00:00 2001 From: Hogan Bobertz Date: Thu, 1 Feb 2024 20:48:14 -0500 Subject: [PATCH 5/6] add extended timeout fixture --- packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts | 5 +++++ .../cli-integ/tests/cli-integ-tests/cli.integtest.ts | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 8124088e00634..16226c4cde259 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -12,6 +12,7 @@ import { AwsContext, withAws } from './with-aws'; import { withTimeout } from './with-timeout'; export const DEFAULT_TEST_TIMEOUT_S = 10 * 60; +export const EXTENDED_TEST_TIMEOUT_S = 30 * 60; /** * Higher order function to execute a block with a CDK app fixture @@ -185,6 +186,10 @@ export function withDefaultFixture(block: (context: TestFixture) => Promise Promise) { + return withAws(withTimeout(EXTENDED_TEST_TIMEOUT_S, withCdkApp(block))); +} + export function withCDKMigrateFixture(language: string, block: (content: TestFixture) => Promise) { return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withCdkMigrateApp(language, block))); } diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts index ac3281cc07dba..c6cae4d0bb0b8 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts @@ -1,7 +1,7 @@ import { promises as fs, existsSync } from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { AwsClients, TestFixture, integTest, cloneDirectory, shell, withDefaultFixture, retry, sleep, randomInteger, withSamIntegrationFixture, RESOURCES_DIR, withCDKMigrateFixture } from '../../lib'; +import { AwsClients, TestFixture, integTest, cloneDirectory, shell, withDefaultFixture, retry, sleep, randomInteger, withSamIntegrationFixture, RESOURCES_DIR, withCDKMigrateFixture, withExtendedTimeoutFixture } from '../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime @@ -599,7 +599,7 @@ integTest('cdk migrate generates migrate.json', withCDKMigrateFixture('typescrip await fixture.cdkDestroy(fixture.stackNamePrefix); })); -integTest('cdk migrate --from-scan with AND/OR filters correctly filters resources', withDefaultFixture(async (fixture) => { +integTest('cdk migrate --from-scan with AND/OR filters correctly filters resources', withExtendedTimeoutFixture(async (fixture) => { const stackName = `cdk-migrate-integ-${fixture.randomString}`; await fixture.cdkDeploy('migrate-stack', { @@ -631,7 +631,7 @@ integTest('cdk migrate --from-scan with AND/OR filters correctly filters resourc } })); -integTest('cdk migrate --from-scan for resources with Write Only Properties generates warnings', withDefaultFixture(async (fixture) => { +integTest('cdk migrate --from-scan for resources with Write Only Properties generates warnings', withExtendedTimeoutFixture(async (fixture) => { const stackName = `cdk-migrate-integ-${fixture.randomString}`; await fixture.cdkDeploy('migrate-stack', { @@ -671,7 +671,7 @@ integTest('cdk migrate --from-scan for resources with Write Only Properties gene })); ['typescript', 'python', 'csharp', 'java'].forEach(language => { - integTest(`cdk migrate --from-stack creates deployable ${language} app`, withDefaultFixture(async (fixture) => { + integTest(`cdk migrate --from-stack creates deployable ${language} app`, withExtendedTimeoutFixture(async (fixture) => { const migrateStackName = fixture.fullStackName('migrate-stack'); await fixture.aws.cloudFormation('createStack', { StackName: migrateStackName, From b2af1e4c5037a53a03cdaf988840073dd1682e34 Mon Sep 17 00:00:00 2001 From: Hogan Bobertz Date: Fri, 2 Feb 2024 00:30:15 -0500 Subject: [PATCH 6/6] bug fix for cli integ tests --- .../tests/cli-integ-tests/cli.integtest.ts | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts index c6cae4d0bb0b8..d152adb412fe2 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts @@ -1,7 +1,7 @@ import { promises as fs, existsSync } from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { AwsClients, TestFixture, integTest, cloneDirectory, shell, withDefaultFixture, retry, sleep, randomInteger, withSamIntegrationFixture, RESOURCES_DIR, withCDKMigrateFixture, withExtendedTimeoutFixture } from '../../lib'; +import { integTest, cloneDirectory, shell, withDefaultFixture, retry, sleep, randomInteger, withSamIntegrationFixture, RESOURCES_DIR, withCDKMigrateFixture, withExtendedTimeoutFixture } from '../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime @@ -687,22 +687,15 @@ integTest('cdk migrate --from-scan for resources with Write Only Properties gene ['migrate', '--stack-name', migrateStackName, '--from-stack'], { modEnv: { MIGRATE_INTEG_TEST: '1' }, neverRequireApproval: true, verbose: true, captureStderr: false }, ); - const awsClients = await AwsClients.default(fixture.output); - const fixtureJr = new TestFixture( - path.join(fixture.integTestDir, migrateStackName), - fixture.stackNamePrefix, - fixture.output, - awsClients, - '', - ); - await fixtureJr.cdkDeploy('migrate-stack', { neverRequireApproval: true, verbose: true, captureStderr: false }); + await fixture.shell(['cd', path.join(fixture.integTestDir, migrateStackName)]); + await fixture.cdk(['deploy', migrateStackName], { neverRequireApproval: true, verbose: true, captureStderr: false }); const response = await fixture.aws.cloudFormation('describeStacks', { StackName: migrateStackName, }); expect(response.Stacks?.[0].StackStatus).toEqual('UPDATE_COMPLETE'); } finally { - await fixture.cdkDestroy(migrateStackName); + await fixture.cdkDestroy('migrate-stack'); } })); });