From c2479f2828c65f3fd94856ec0a1be947c2426951 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 22 Oct 2019 12:02:50 +0200 Subject: [PATCH 1/2] fix(cli): add Cloud Assembly backwards compat tests Make sure that the CLI will continue to be able to load old cloud assemblies. We do this by using the CLI feature that it can directly load Cloud Assemblies through its `--app` argument, hard copying a couple of cloud assemblies into the test set and trying to synth those. In addition to adding a test for loading v0.36.0 assemblies, also add tests for 2 common context provider requests (AZs and VPC), to make sure new CLIs will continue to be able to handle these context lookup requests. Fixes #4475, pre-emptively adds a regression test for #4544. --- packages/aws-cdk/.gitignore | 1 + packages/aws-cdk/test/integ/cli/.gitignore | 1 + .../aws-cdk/test/integ/cli/app/.gitignore | 1 - .../0.36.0/InitStack.template.json | 1 + .../integ/cli/cloud-assemblies/0.36.0/cdk.out | 1 + .../cli/cloud-assemblies/0.36.0/manifest.json | 19 +++++++ .../InitStack.template.json | 2 + .../1.10.0-lookup-default-vpc/cdk.out | 1 + .../manifest.json.js | 37 ++++++++++++++ .../InitStack.template.json | 2 + .../1.10.0-request-azs/cdk.out | 1 + .../1.10.0-request-azs/manifest.json.js | 34 +++++++++++++ packages/aws-cdk/test/integ/cli/common.bash | 7 ++- .../test/integ/cli/test-backwards-compat.sh | 51 +++++++++++++++++++ .../test-cdk-ssm-parameter-provider-error.sh | 4 +- 15 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 packages/aws-cdk/test/integ/cli/.gitignore delete mode 100644 packages/aws-cdk/test/integ/cli/app/.gitignore create mode 100644 packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/InitStack.template.json create mode 100644 packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/cdk.out create mode 100644 packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/manifest.json create mode 100644 packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/InitStack.template.json create mode 100644 packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/cdk.out create mode 100644 packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/manifest.json.js create mode 100644 packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/InitStack.template.json create mode 100644 packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/cdk.out create mode 100644 packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/manifest.json.js create mode 100755 packages/aws-cdk/test/integ/cli/test-backwards-compat.sh diff --git a/packages/aws-cdk/.gitignore b/packages/aws-cdk/.gitignore index 75d51ebc9fd05..17e9b17659ffc 100644 --- a/packages/aws-cdk/.gitignore +++ b/packages/aws-cdk/.gitignore @@ -16,3 +16,4 @@ coverage *.snk !test/integ/run-wrappers/dist +!test/integ/cli/**/* diff --git a/packages/aws-cdk/test/integ/cli/.gitignore b/packages/aws-cdk/test/integ/cli/.gitignore new file mode 100644 index 0000000000000..e136cb6cd26fb --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/.gitignore @@ -0,0 +1 @@ +cdk.context.json diff --git a/packages/aws-cdk/test/integ/cli/app/.gitignore b/packages/aws-cdk/test/integ/cli/app/.gitignore deleted file mode 100644 index d4aa116a26c73..0000000000000 --- a/packages/aws-cdk/test/integ/cli/app/.gitignore +++ /dev/null @@ -1 +0,0 @@ -!*.js diff --git a/packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/InitStack.template.json b/packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/InitStack.template.json new file mode 100644 index 0000000000000..9e26dfeeb6e64 --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/InitStack.template.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/cdk.out b/packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/cdk.out new file mode 100644 index 0000000000000..bfe018f236569 --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/cdk.out @@ -0,0 +1 @@ +{"version":"0.36.0"} diff --git a/packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/manifest.json b/packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/manifest.json new file mode 100644 index 0000000000000..025294fe09aa7 --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/cloud-assemblies/0.36.0/manifest.json @@ -0,0 +1,19 @@ +{ + "version": "0.36.0", + "artifacts": { + "InitStack": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "InitStack.template.json" + } + } + }, + "runtime": { + "libraries": { + "@aws-cdk/core": "1.12.0", + "@aws-cdk/cx-api": "1.12.0", + "jsii-runtime": "node.js/v8.11.4" + } + } +} diff --git a/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/InitStack.template.json b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/InitStack.template.json new file mode 100644 index 0000000000000..2c63c0851048d --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/InitStack.template.json @@ -0,0 +1,2 @@ +{ +} diff --git a/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/cdk.out b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/cdk.out new file mode 100644 index 0000000000000..77925ad4488a6 --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/cdk.out @@ -0,0 +1 @@ +{"version":"1.10.0"} \ No newline at end of file diff --git a/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/manifest.json.js b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/manifest.json.js new file mode 100644 index 0000000000000..e328ea2e9d2fd --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-lookup-default-vpc/manifest.json.js @@ -0,0 +1,37 @@ +process.stdout.write(JSON.stringify({ + "version": "1.10.0", + "artifacts": { + "InitStack": { + "type": "aws:cloudformation:stack", + "environment": `aws://${process.env.TEST_ACCOUNT}/${process.env.TEST_REGION}`, + "properties": { + "templateFile": "InitStack.template.json" + } + } + }, + "runtime": { + "libraries": { + "@aws-cdk/core": "1.14.0", + "@aws-cdk/cx-api": "1.14.0", + "@aws-cdk/aws-ec2": "1.14.0", + "@aws-cdk/aws-iam": "1.14.0", + "@aws-cdk/region-info": "1.14.0", + "@aws-cdk/aws-ssm": "1.14.0", + "@aws-cdk/aws-cloudwatch": "1.14.0", + "jsii-runtime": "node.js/v8.11.4" + } + }, + "missing": [ + { + "key": `vpc-provider:account=${process.env.TEST_ACCOUNT}:filter.isDefault=true:region=${process.env.TEST_REGION}`, + "props": { + "account": process.env.TEST_ACCOUNT, + "region": process.env.TEST_REGION, + "filter": { + "isDefault": "true" + } + }, + "provider": "vpc-provider" + } + ] +}, undefined, 2)); diff --git a/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/InitStack.template.json b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/InitStack.template.json new file mode 100644 index 0000000000000..2c63c0851048d --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/InitStack.template.json @@ -0,0 +1,2 @@ +{ +} diff --git a/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/cdk.out b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/cdk.out new file mode 100644 index 0000000000000..77925ad4488a6 --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/cdk.out @@ -0,0 +1 @@ +{"version":"1.10.0"} \ No newline at end of file diff --git a/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/manifest.json.js b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/manifest.json.js new file mode 100644 index 0000000000000..9c88f8ad78941 --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/cloud-assemblies/1.10.0-request-azs/manifest.json.js @@ -0,0 +1,34 @@ +process.stdout.write(JSON.stringify({ + "version": "1.10.0", + "artifacts": { + "InitStack": { + "type": "aws:cloudformation:stack", + "environment": `aws://${process.env.TEST_ACCOUNT}/${process.env.TEST_REGION}`, + "properties": { + "templateFile": "InitStack.template.json" + } + } + }, + "runtime": { + "libraries": { + "@aws-cdk/core": "1.14.0", + "@aws-cdk/cx-api": "1.14.0", + "@aws-cdk/aws-ec2": "1.14.0", + "@aws-cdk/aws-iam": "1.14.0", + "@aws-cdk/region-info": "1.14.0", + "@aws-cdk/aws-ssm": "1.14.0", + "@aws-cdk/aws-cloudwatch": "1.14.0", + "jsii-runtime": "node.js/v8.11.4" + } + }, + "missing": [ + { + "key": `availability-zones:account=${process.env.TEST_ACCOUNT}:region=${process.env.TEST_REGION}`, + "props": { + "account": process.env.TEST_ACCOUNT, + "region": process.env.TEST_REGION, + }, + "provider": "availability-zones" + } + ] +}, undefined, 2)); diff --git a/packages/aws-cdk/test/integ/cli/common.bash b/packages/aws-cdk/test/integ/cli/common.bash index 95ee98f14ddf9..50a85c9889f0a 100644 --- a/packages/aws-cdk/test/integ/cli/common.bash +++ b/packages/aws-cdk/test/integ/cli/common.bash @@ -3,8 +3,11 @@ scriptdir=$(cd $(dirname $0) && pwd) cd ${scriptdir} if [[ -z "${CREDS_SET:-}" ]]; then - # Check that credentials are configured - aws sts get-caller-identity > /dev/null + # Check that credentials are configured (will error & abort if not) + creds=$(aws sts get-caller-identity) + + export TEST_ACCOUNT=$(node -p "($creds).Account") + export TEST_REGION=${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-1}} export CREDS_SET=1 fi diff --git a/packages/aws-cdk/test/integ/cli/test-backwards-compat.sh b/packages/aws-cdk/test/integ/cli/test-backwards-compat.sh new file mode 100755 index 0000000000000..a55442793dd8c --- /dev/null +++ b/packages/aws-cdk/test/integ/cli/test-backwards-compat.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -euo pipefail +scriptdir=$(cd $(dirname $0) && pwd) +source ${scriptdir}/common.bash +# ---------------------------------------------------------- + +tmpdir=$(dirname $(mktemp -u)) +casmdir=$tmpdir/cdk-integ-cx +stderrfile=$casmdir/cdk.err + +# Copy the files from a cloud assembly source dir into the +# tempdir. Evaluate .js files found their (interpret them +# as templates). +function prepare_cloud_assembly() { + echo "ASSEMBLY ${1}" + rm -rf $casmdir + mkdir -p $casmdir + cp -R $1/* $casmdir + + # Execute templates to produce file with the same name + # but without .js extension + shopt -s nullglob + for template in $casmdir/*.js; do + node $template > ${template%.js} + done +} + +# Assert that there was no providerError in CDK's stderr +# Because we rely on the app/framework to actually error in case the +# provider fails, we inspect the logs here. +function assert_no_error() { + if grep '$providerError' $stderrfile; then + cat $stderrfile >&2 + echo "There was an error executing the context provider!" >&2 + exit 1 + fi +} + +# Echo the TEST_ACCOUNT, TEST_REGION vars to make bash abort if they're not set. +echo "Running backwards compatibility test (account ${TEST_ACCOUNT}, region ${TEST_REGION})" + +for assembly in ${scriptdir}/cloud-assemblies/*; do + prepare_cloud_assembly $assembly + + rm -f cdk.context.json + cdk -a $casmdir -v synth > /dev/null 2> $stderrfile + + assert_no_error +done + +echo "✅ success" diff --git a/packages/aws-cdk/test/integ/cli/test-cdk-ssm-parameter-provider-error.sh b/packages/aws-cdk/test/integ/cli/test-cdk-ssm-parameter-provider-error.sh index 3b007f6e37b93..224e930658e0a 100755 --- a/packages/aws-cdk/test/integ/cli/test-cdk-ssm-parameter-provider-error.sh +++ b/packages/aws-cdk/test/integ/cli/test-cdk-ssm-parameter-provider-error.sh @@ -7,15 +7,13 @@ source ${scriptdir}/common.bash setup parameterName=/does/not/exist -account=$(node -p "($(aws sts get-caller-identity)).Account") -region=${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-1}} function cdk_synth() { (cdk synth $@ 2>&1 || true) | strip_color_codes } assert "cdk_synth ${STACK_NAME_PREFIX}-missing-ssm-parameter -c test:ssm-parameter-name=${parameterName}" < Date: Wed, 23 Oct 2019 10:08:54 +0200 Subject: [PATCH 2/2] Address review comments --- .../aws-cdk/test/integ/cli/test-backwards-compat.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/aws-cdk/test/integ/cli/test-backwards-compat.sh b/packages/aws-cdk/test/integ/cli/test-backwards-compat.sh index a55442793dd8c..eb14faf7010af 100755 --- a/packages/aws-cdk/test/integ/cli/test-backwards-compat.sh +++ b/packages/aws-cdk/test/integ/cli/test-backwards-compat.sh @@ -12,10 +12,11 @@ stderrfile=$casmdir/cdk.err # tempdir. Evaluate .js files found their (interpret them # as templates). function prepare_cloud_assembly() { - echo "ASSEMBLY ${1}" + local asmdir=$1 + echo "ASSEMBLY ${asmdir}" rm -rf $casmdir mkdir -p $casmdir - cp -R $1/* $casmdir + cp -R $asmdir/* $casmdir # Execute templates to produce file with the same name # but without .js extension @@ -29,9 +30,11 @@ function prepare_cloud_assembly() { # Because we rely on the app/framework to actually error in case the # provider fails, we inspect the logs here. function assert_no_error() { + local asmdir=$1 + if grep '$providerError' $stderrfile; then cat $stderrfile >&2 - echo "There was an error executing the context provider!" >&2 + echo "There was an error executing the context provider for assembly ${asmdir}!" >&2 exit 1 fi } @@ -45,7 +48,7 @@ for assembly in ${scriptdir}/cloud-assemblies/*; do rm -f cdk.context.json cdk -a $casmdir -v synth > /dev/null 2> $stderrfile - assert_no_error + assert_no_error $assembly done echo "✅ success"