From aaf89a741de88c842c629fb525c8857caa419962 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Thu, 30 Dec 2021 14:24:41 -0800 Subject: [PATCH] feat: allow invoking built CDK synthesized templates (#3549) * feat: allow invoking built CDK synthesized templates * run black * fixing testing issue * add unit testing to cover custom resource Id * run black * skip normalizing the resources that have been normalized before * apply pr comments * apply pr comments --- samcli/lib/build/app_builder.py | 5 +- .../samlib/resource_metadata_normalizer.py | 96 ++++- .../integration/buildcmd/build_integ_base.py | 79 ++-- tests/integration/buildcmd/test_build_cmd.py | 74 +++- .../Dockerfile | 15 + .../__init__.py | 0 .../main.py | 19 + .../requirements.txt | 1 + .../__init__.py | 0 .../main.py | 19 + .../requirements.txt | 1 + ...thesized_template_zip_image_functions.json | 400 ++++++++++++++++++ .../unit/lib/build_module/test_app_builder.py | 46 ++ .../test_resource_metadata_normalizer.py | 143 ++++++- 14 files changed, 827 insertions(+), 71 deletions(-) create mode 100644 tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/Dockerfile create mode 100644 tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/__init__.py create mode 100644 tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/main.py create mode 100644 tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/requirements.txt create mode 100644 tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/__init__.py create mode 100644 tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/main.py create mode 100644 tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/requirements.txt create mode 100644 tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_zip_image_functions.json diff --git a/samcli/lib/build/app_builder.py b/samcli/lib/build/app_builder.py index bb81ecc73518..fa45f0e1fcdd 100644 --- a/samcli/lib/build/app_builder.py +++ b/samcli/lib/build/app_builder.py @@ -33,6 +33,7 @@ AWS_SERVERLESS_FUNCTION, AWS_SERVERLESS_LAYERVERSION, ) +from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer from samcli.lib.docker.log_streamer import LogStreamer, LogStreamError from samcli.lib.providers.provider import ResourcesToBuildCollector, Function, get_full_path, Stack, LayerVersion from samcli.lib.utils.colors import Colored @@ -277,8 +278,8 @@ def update_template( template_dict = stack.template_dict for logical_id, resource in template_dict.get("Resources", {}).items(): - - full_path = get_full_path(stack.stack_path, logical_id) + resource_iac_id = ResourceMetadataNormalizer.get_resource_id(resource, logical_id) + full_path = get_full_path(stack.stack_path, resource_iac_id) has_build_artifact = full_path in built_artifacts is_stack = full_path in stack_output_template_path_by_stack_path diff --git a/samcli/lib/samlib/resource_metadata_normalizer.py b/samcli/lib/samlib/resource_metadata_normalizer.py index 6070e64aac36..42a88550a821 100644 --- a/samcli/lib/samlib/resource_metadata_normalizer.py +++ b/samcli/lib/samlib/resource_metadata_normalizer.py @@ -9,10 +9,15 @@ from samcli.lib.iac.cdk.utils import is_cdk_project +from samcli.lib.utils.resources import AWS_CLOUDFORMATION_STACK + +CDK_NESTED_STACK_RESOURCE_ID_SUFFIX = ".NestedStack" + RESOURCES_KEY = "Resources" PROPERTIES_KEY = "Properties" METADATA_KEY = "Metadata" +RESOURCE_CDK_PATH_METADATA_KEY = "aws:cdk:path" ASSET_PATH_METADATA_KEY = "aws:asset:path" ASSET_PROPERTY_METADATA_KEY = "aws:asset:property" @@ -20,6 +25,8 @@ ASSET_DOCKERFILE_PATH_KEY = "aws:asset:dockerfile-path" ASSET_DOCKERFILE_BUILD_ARGS_KEY = "aws:asset:docker-build-args" +SAM_RESOURCE_ID_KEY = "SamResourceId" +SAM_IS_NORMALIZED = "SamNormalized" SAM_METADATA_DOCKERFILE_KEY = "Dockerfile" SAM_METADATA_DOCKER_CONTEXT_KEY = "DockerContext" SAM_METADATA_DOCKER_BUILD_ARGS_KEY = "DockerBuildArgs" @@ -53,18 +60,21 @@ def normalize(template_dict, normalize_parameters=False): for logical_id, resource in resources.items(): resource_metadata = resource.get(METADATA_KEY, {}) - asset_property = resource_metadata.get(ASSET_PROPERTY_METADATA_KEY) - - if asset_property == IMAGE_ASSET_PROPERTY: - asset_metadata = ResourceMetadataNormalizer._extract_image_asset_metadata(resource_metadata) - ResourceMetadataNormalizer._update_resource_metadata(resource_metadata, asset_metadata) - # For image-type functions, the asset path is expected to be the name of the Docker image. - # When building, we set the name of the image to be the logical id of the function. - asset_path = logical_id.lower() - else: - asset_path = resource_metadata.get(ASSET_PATH_METADATA_KEY) - - ResourceMetadataNormalizer._replace_property(asset_property, asset_path, resource, logical_id) + is_normalized = resource_metadata.get(SAM_IS_NORMALIZED, False) + if not is_normalized: + asset_property = resource_metadata.get(ASSET_PROPERTY_METADATA_KEY) + if asset_property == IMAGE_ASSET_PROPERTY: + asset_metadata = ResourceMetadataNormalizer._extract_image_asset_metadata(resource_metadata) + ResourceMetadataNormalizer._update_resource_metadata(resource_metadata, asset_metadata) + # For image-type functions, the asset path is expected to be the name of the Docker image. + # When building, we set the name of the image to be the logical id of the function. + asset_path = logical_id.lower() + else: + asset_path = resource_metadata.get(ASSET_PATH_METADATA_KEY) + + ResourceMetadataNormalizer._replace_property(asset_property, asset_path, resource, logical_id) + if asset_path and asset_property: + resource_metadata[SAM_IS_NORMALIZED] = True # Set SkipBuild metadata iff is-bundled metadata exists, and value is True skip_build = resource_metadata.get(ASSET_BUNDLED_METADATA_KEY, False) @@ -182,3 +192,65 @@ def _update_resource_metadata(metadata, updated_values): """ for key, val in updated_values.items(): metadata[key] = val + + @staticmethod + def get_resource_id(resource_properties, logical_id): + """ + Get unique id for a resource. + for any resource, the resource id can be the customer defined id if exist, if not exist it can be the + cdk-defined resource id, or the logical id if the resource id is not found. + + Parameters + ---------- + resource_properties dict + Properties of this resource + logical_id str + LogicalID of the resource + + Returns + ------- + str + The unique function id + """ + resource_metadata = resource_properties.get("Metadata", {}) + customer_defined_id = resource_metadata.get(SAM_RESOURCE_ID_KEY) + + if isinstance(customer_defined_id, str) and customer_defined_id: + LOG.debug( + "Sam customer defined id is more priority than other IDs. Customer defined id for resource %s is %s", + logical_id, + customer_defined_id, + ) + return customer_defined_id + + resource_cdk_path = resource_metadata.get(RESOURCE_CDK_PATH_METADATA_KEY) + + if not isinstance(resource_cdk_path, str) or not resource_cdk_path: + LOG.debug( + "There is no customer defined id or cdk path defined for resource %s, so we will use the resource " + "logical id as the resource id", + logical_id, + ) + return logical_id + + # aws:cdk:path metadata format of functions: {stack_id}/{function_id}/Resource + # Design doc of CDK path: https://github.com/aws/aws-cdk/blob/master/design/construct-tree.md + cdk_path_partitions = resource_cdk_path.split("/") + + LOG.debug("CDK Path for resource %s is %s", logical_id, cdk_path_partitions) + + if len(cdk_path_partitions) < 2: + LOG.warning( + "Cannot detect function id from aws:cdk:path metadata '%s', using default logical id", resource_cdk_path + ) + return logical_id + + cdk_resource_id = cdk_path_partitions[-2] + + # Check if the Resource is nested Stack + if resource_properties.get("Type", "") == AWS_CLOUDFORMATION_STACK and cdk_resource_id.endswith( + CDK_NESTED_STACK_RESOURCE_ID_SUFFIX + ): + cdk_resource_id = cdk_resource_id[: -len(CDK_NESTED_STACK_RESOURCE_ID_SUFFIX)] + + return cdk_resource_id diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index 06485abc9e67..340ba9b4f9e2 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -189,10 +189,14 @@ def _verify_invoke_built_function(self, template_path, function_logical_id, over "-t", str(template_path), "--no-event", - "--parameter-overrides", - overrides, ] + if overrides: + cmdlist += [ + "--parameter-overrides", + overrides, + ] + LOG.info("Running invoke Command: {}".format(cmdlist)) process_execute = run_command(cmdlist) @@ -518,12 +522,21 @@ class BuildIntegPythonBase(BuildIntegBase): } FUNCTION_LOGICAL_ID = "Function" + prop = "CodeUri" - def _test_with_default_requirements(self, runtime, codeuri, use_container, relative_path, architecture=None): + def _test_with_default_requirements( + self, + runtime, + codeuri, + use_container, + relative_path, + do_override=True, + check_function_only=False, + architecture=None, + ): if use_container and (SKIP_DOCKER_TESTS or SKIP_DOCKER_BUILD): self.skipTest(SKIP_DOCKER_MESSAGE) - - overrides = self.get_override(runtime, codeuri, architecture, "main.handler") + overrides = self.get_override(runtime, codeuri, architecture, "main.handler") if do_override else None cmdlist = self.get_command_list(use_container=use_container, parameter_overrides=overrides) LOG.info("Running Command: {}".format(cmdlist)) @@ -533,37 +546,41 @@ def _test_with_default_requirements(self, runtime, codeuri, use_container, relat self.default_build_dir, self.FUNCTION_LOGICAL_ID, self.EXPECTED_FILES_PROJECT_MANIFEST ) - self._verify_resource_property( - str(self.built_template), - "OtherRelativePathResource", - "BodyS3Location", - os.path.relpath( - os.path.normpath(os.path.join(str(relative_path), "SomeRelativePath")), - str(self.default_build_dir), - ), - ) + if not check_function_only: + self._verify_resource_property( + str(self.built_template), + "OtherRelativePathResource", + "BodyS3Location", + os.path.relpath( + os.path.normpath(os.path.join(str(relative_path), "SomeRelativePath")), + str(self.default_build_dir), + ), + ) - self._verify_resource_property( - str(self.built_template), - "GlueResource", - "Command.ScriptLocation", - os.path.relpath( - os.path.normpath(os.path.join(str(relative_path), "SomeRelativePath")), - str(self.default_build_dir), - ), - ) + self._verify_resource_property( + str(self.built_template), + "GlueResource", + "Command.ScriptLocation", + os.path.relpath( + os.path.normpath(os.path.join(str(relative_path), "SomeRelativePath")), + str(self.default_build_dir), + ), + ) - self._verify_resource_property( - str(self.built_template), - "ExampleNestedStack", - "TemplateURL", - "https://s3.amazonaws.com/examplebucket/exampletemplate.yml", - ) + self._verify_resource_property( + str(self.built_template), + "ExampleNestedStack", + "TemplateURL", + "https://s3.amazonaws.com/examplebucket/exampletemplate.yml", + ) expected = {"pi": "3.14"} if not SKIP_DOCKER_TESTS: self._verify_invoke_built_function( - self.built_template, self.FUNCTION_LOGICAL_ID, self._make_parameter_override_arg(overrides), expected + self.built_template, + self.FUNCTION_LOGICAL_ID, + self._make_parameter_override_arg(overrides) if do_override else None, + expected, ) if use_container: self.verify_docker_container_cleanedup(runtime) @@ -580,7 +597,7 @@ def _verify_built_artifact(self, build_dir, function_logical_id, expected_files) resource_artifact_dir = build_dir.joinpath(function_logical_id) # Make sure the template has correct CodeUri for resource - self._verify_resource_property(str(template_path), function_logical_id, "CodeUri", function_logical_id) + self._verify_resource_property(str(template_path), function_logical_id, self.prop, function_logical_id) all_artifacts = set(os.listdir(str(resource_artifact_dir))) actual_files = all_artifacts.intersection(expected_files) diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index bff35c421b9c..f15dcebd3840 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -240,27 +240,59 @@ def _validate_skipped_built_function( ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), "Skip build tests on windows when running in CI unless overridden", ) +@parameterized_class( + ( + "template", + "FUNCTION_LOGICAL_ID", + "overrides", + "runtime", + "codeuri", + "use_container", + "check_function_only", + "prop", + ), + [ + ("template.yaml", "Function", True, "python2.7", "Python", False, False, "CodeUri"), + ("template.yaml", "Function", True, "python3.6", "Python", False, False, "CodeUri"), + ("template.yaml", "Function", True, "python3.7", "Python", False, False, "CodeUri"), + ("template.yaml", "Function", True, "python3.8", "Python", False, False, "CodeUri"), + ("template.yaml", "Function", True, "python3.9", "Python", False, False, "CodeUri"), + ("template.yaml", "Function", True, "python3.7", "PythonPEP600", False, False, "CodeUri"), + ("template.yaml", "Function", True, "python3.8", "PythonPEP600", False, False, "CodeUri"), + ("template.yaml", "Function", True, "python2.7", "Python", "use_container", False, "CodeUri"), + ("template.yaml", "Function", True, "python3.6", "Python", "use_container", False, "CodeUri"), + ("template.yaml", "Function", True, "python3.7", "Python", "use_container", False, "CodeUri"), + ("template.yaml", "Function", True, "python3.8", "Python", "use_container", False, "CodeUri"), + ("template.yaml", "Function", True, "python3.9", "Python", "use_container", False, "CodeUri"), + ( + "cdk_v1_synthesized_template_zip_image_functions.json", + "RandomCitiesFunction5C47A2B8", + False, + None, + None, + False, + True, + "Code", + ), + ], +) class TestBuildCommand_PythonFunctions(BuildIntegPythonBase): - @parameterized.expand( - [ - ("python2.7", "Python", False), - ("python3.6", "Python", False), - ("python3.7", "Python", False), - ("python3.8", "Python", False), - ("python3.9", "Python", False), - # numpy 1.20.3 (in PythonPEP600/requirements.txt) only support python 3.7+ - ("python3.7", "PythonPEP600", False), - ("python3.8", "PythonPEP600", False), - ("python2.7", "Python", "use_container"), - ("python3.6", "Python", "use_container"), - ("python3.7", "Python", "use_container"), - ("python3.8", "Python", "use_container"), - ("python3.9", "Python", "use_container"), - ] - ) + overrides = True + runtime = "python2.7" + codeuri = "Python" + use_container = False + check_function_only = False + @pytest.mark.flaky(reruns=3) - def test_with_default_requirements(self, runtime, codeuri, use_container): - self._test_with_default_requirements(runtime, codeuri, use_container, self.test_data_path) + def test_with_default_requirements(self): + self._test_with_default_requirements( + self.runtime, + self.codeuri, + self.use_container, + self.test_data_path, + do_override=self.overrides, + check_function_only=self.check_function_only, + ) @skipIf( @@ -290,7 +322,9 @@ class TestBuildCommand_PythonFunctions_With_Specified_Architecture(BuildIntegPyt ) @pytest.mark.flaky(reruns=3) def test_with_default_requirements(self, runtime, codeuri, use_container, architecture): - self._test_with_default_requirements(runtime, codeuri, use_container, self.test_data_path, architecture) + self._test_with_default_requirements( + runtime, codeuri, use_container, self.test_data_path, architecture=architecture + ) @skipIf( diff --git a/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/Dockerfile b/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/Dockerfile new file mode 100644 index 000000000000..170b1f1cca3e --- /dev/null +++ b/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/Dockerfile @@ -0,0 +1,15 @@ +ARG BASE_RUNTIME + +FROM public.ecr.aws/lambda/python:$BASE_RUNTIME + +ARG FUNCTION_DIR="/var/task" + +RUN mkdir -p $FUNCTION_DIR + +COPY main.py $FUNCTION_DIR + +COPY __init__.py $FUNCTION_DIR + +COPY requirements.txt $FUNCTION_DIR + +RUN python -m pip install -r $FUNCTION_DIR/requirements.txt -t $FUNCTION_DIR \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/__init__.py b/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/main.py b/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/main.py new file mode 100644 index 000000000000..0311bf9f49b4 --- /dev/null +++ b/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/main.py @@ -0,0 +1,19 @@ +import numpy + +# from cryptography.fernet import Fernet + + +def handler(event, context): + + # Try using some of the modules to make sure they work & don't crash the process + # print(Fernet.generate_key()) + + return {"pi": "{0:.2f}".format(numpy.pi)} + + +def first_function_handler(event, context): + return "Hello World" + + +def second_function_handler(event, context): + return "Hello Mars" diff --git a/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/requirements.txt b/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/requirements.txt new file mode 100644 index 000000000000..141927094f94 --- /dev/null +++ b/tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/requirements.txt @@ -0,0 +1 @@ +numpy<1.20.4 diff --git a/tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/__init__.py b/tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/main.py b/tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/main.py new file mode 100644 index 000000000000..0311bf9f49b4 --- /dev/null +++ b/tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/main.py @@ -0,0 +1,19 @@ +import numpy + +# from cryptography.fernet import Fernet + + +def handler(event, context): + + # Try using some of the modules to make sure they work & don't crash the process + # print(Fernet.generate_key()) + + return {"pi": "{0:.2f}".format(numpy.pi)} + + +def first_function_handler(event, context): + return "Hello World" + + +def second_function_handler(event, context): + return "Hello Mars" diff --git a/tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/requirements.txt b/tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/requirements.txt new file mode 100644 index 000000000000..141927094f94 --- /dev/null +++ b/tests/integration/testdata/buildcmd/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837/requirements.txt @@ -0,0 +1 @@ +numpy<1.20.4 diff --git a/tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_zip_image_functions.json b/tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_zip_image_functions.json new file mode 100644 index 000000000000..b2e39b3bd2b6 --- /dev/null +++ b/tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_zip_image_functions.json @@ -0,0 +1,400 @@ +{ + "Resources": { + "RandomCitiesFunctionServiceRole4EFB1CF5": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + }, + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/RandomCitiesFunction/ServiceRole/Resource" + } + }, + "RandomCitiesFunction5C47A2B8": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParametersb998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837S3Bucket9F6483DC" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersb998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837S3VersionKey61C1B485" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersb998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837S3VersionKey61C1B485" + } + ] + } + ] + } + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "RandomCitiesFunctionServiceRole4EFB1CF5", + "Arn" + ] + }, + "Handler": "main.handler", + "Runtime": "python3.7" + }, + "DependsOn": [ + "RandomCitiesFunctionServiceRole4EFB1CF5" + ], + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/RandomCitiesFunction/Resource", + "aws:asset:path": "asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837", + "aws:asset:is-bundled": false, + "aws:asset:property": "Code" + } + }, + "ColorsRandomFunctionServiceRole1DE8E7EB": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + }, + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/ColorsRandomFunction/ServiceRole/Resource" + } + }, + "ColorsRandomFunctionF61B9209": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "ImageUri": { + "Fn::Join": [ + "", + [ + { + "Ref": "AWS::AccountId" + }, + ".dkr.ecr.", + { + "Ref": "AWS::Region" + }, + ".", + { + "Ref": "AWS::URLSuffix" + }, + "/aws-cdk/assets:6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456" + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "ColorsRandomFunctionServiceRole1DE8E7EB", + "Arn" + ] + }, + "ImageConfig": { + "Command": [ + "main.handler" + ] + }, + "PackageType": "Image" + }, + "DependsOn": [ + "ColorsRandomFunctionServiceRole1DE8E7EB" + ], + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/ColorsRandomFunction/Resource", + "aws:asset:path": "asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456", + "aws:asset:dockerfile-path": "Dockerfile", + "aws:asset:docker-build-args": { + "BASE_RUNTIME": "3.7" + }, + "aws:asset:property": "Code.ImageUri" + } + }, + "CDKMetadata": { + "Type": "AWS::CDK::Metadata", + "Properties": { + "Analytics": "v2:deflate64:H4sIAAAAAAAA/01OS24CMQw9C/uM6QjYF6iQuquGE7gZdxSGJJXtFKEodycBgbp6Xz+5h361gbfFO16ks+O8zDYyQT4q2tlsRUgrnVyYzD4GUU5Wzf4nfCGjJyVuYiCJiS01XlujUxdDMW0yn9F/jwj5kIJtduu8+Ee0M/Gnx4meXjEOPeQhnu9zDYuRVYftFYH7R1XDLtVT3aGQIcvP+N/go1kzGOg3itPI11YvxYQ4Epxk+devod/AenES5zpOQZ0nGB54A2I5QvsaAQAA" + }, + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/CDKMetadata/Default" + }, + "Condition": "CDKMetadataAvailable" + } + }, + "Parameters": { + "AssetParametersb998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837S3Bucket9F6483DC": { + "Type": "String", + "Description": "S3 bucket for asset \"b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837\"" + }, + "AssetParametersb998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837S3VersionKey61C1B485": { + "Type": "String", + "Description": "S3 key for asset version \"b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837\"" + }, + "AssetParametersb998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837ArtifactHash13EC3BFB": { + "Type": "String", + "Description": "Artifact hash for asset \"b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe5837\"" + } + }, + "Conditions": { + "CDKMetadataAvailable": { + "Fn::Or": [ + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "af-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ca-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-northwest-1" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-3" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "me-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "sa-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-2" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-2" + ] + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/tests/unit/lib/build_module/test_app_builder.py b/tests/unit/lib/build_module/test_app_builder.py index f2eb7787ae54..7d8866417aca 100644 --- a/tests/unit/lib/build_module/test_app_builder.py +++ b/tests/unit/lib/build_module/test_app_builder.py @@ -619,6 +619,20 @@ def setUp(self): "Resources": { "MyFunction1": {"Type": "AWS::Serverless::Function", "Properties": {"CodeUri": "oldvalue"}}, "MyFunction2": {"Type": "AWS::Lambda::Function", "Properties": {"Code": "oldvalue"}}, + "MyCDKFunction": { + "Type": "AWS::Lambda::Function", + "Properties": {"Code": "oldvalue"}, + "Metadata": { + "aws:cdk:path": "Stack/CDKFunc/Resource", + }, + }, + "MyCustomIdFunction": { + "Type": "AWS::Lambda::Function", + "Properties": {"Code": "oldvalue"}, + "Metadata": { + "SamResourceId": "CustomIdFunc", + }, + }, "GlueResource": {"Type": "AWS::Glue::Job", "Properties": {"Command": {"ScriptLocation": "something"}}}, "OtherResource": {"Type": "AWS::Lambda::Version", "Properties": {"CodeUri": "something"}}, "MyImageFunction1": { @@ -645,6 +659,8 @@ def test_must_update_resources_with_build_artifacts(self): built_artifacts = { "MyFunction1": "/path/to/build/MyFunction1", "MyFunction2": "/path/to/build/MyFunction2", + "CDKFunc": "/path/to/build/MyCDKFunction", + "CustomIdFunc": "/path/to/build/MyCustomIdFunction", "MyServerlessLayer": "/path/to/build/ServerlessLayer", "MyLambdaLayer": "/path/to/build/LambdaLayer", "MyImageFunction1": "myimagefunction1:Tag", @@ -661,6 +677,20 @@ def test_must_update_resources_with_build_artifacts(self): "Type": "AWS::Lambda::Function", "Properties": {"Code": os.path.join("build", "MyFunction2")}, }, + "MyCDKFunction": { + "Type": "AWS::Lambda::Function", + "Properties": {"Code": os.path.join("build", "MyCDKFunction")}, + "Metadata": { + "aws:cdk:path": "Stack/CDKFunc/Resource", + }, + }, + "MyCustomIdFunction": { + "Type": "AWS::Lambda::Function", + "Properties": {"Code": os.path.join("build", "MyCustomIdFunction")}, + "Metadata": { + "SamResourceId": "CustomIdFunc", + }, + }, "GlueResource": {"Type": "AWS::Glue::Job", "Properties": {"Command": {"ScriptLocation": "something"}}}, "OtherResource": {"Type": "AWS::Lambda::Version", "Properties": {"CodeUri": "something"}}, "MyImageFunction1": { @@ -698,6 +728,8 @@ def test_must_update_resources_with_build_artifacts_and_template_paths_in_multi_ "ChildStackXXX/MyLambdaLayer": "/path/to/build/ChildStackXXX/LambdaLayer", "ChildStackXXX/MyFunction1": "/path/to/build/ChildStackXXX/MyFunction1", "ChildStackXXX/MyFunction2": "/path/to/build/ChildStackXXX/MyFunction2", + "ChildStackXXX/CDKFunc": "/path/to/build/ChildStackXXX/MyCDKFunction", + "ChildStackXXX/CustomIdFunc": "/path/to/build/ChildStackXXX/MyCustomIdFunction", "ChildStackXXX/MyImageFunction1": "myimagefunction1:Tag", } stack_output_paths = { @@ -715,6 +747,20 @@ def test_must_update_resources_with_build_artifacts_and_template_paths_in_multi_ "Type": "AWS::Lambda::Function", "Properties": {"Code": os.path.join("build", "ChildStackXXX", "MyFunction2")}, }, + "MyCDKFunction": { + "Type": "AWS::Lambda::Function", + "Properties": {"Code": os.path.join("build", "ChildStackXXX", "MyCDKFunction")}, + "Metadata": { + "aws:cdk:path": "Stack/CDKFunc/Resource", + }, + }, + "MyCustomIdFunction": { + "Type": "AWS::Lambda::Function", + "Properties": {"Code": os.path.join("build", "ChildStackXXX", "MyCustomIdFunction")}, + "Metadata": { + "SamResourceId": "CustomIdFunc", + }, + }, "GlueResource": {"Type": "AWS::Glue::Job", "Properties": {"Command": {"ScriptLocation": "something"}}}, "OtherResource": {"Type": "AWS::Lambda::Version", "Properties": {"CodeUri": "something"}}, "MyImageFunction1": { diff --git a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py index f6cf20ff97ab..5694d07f353e 100644 --- a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py +++ b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py @@ -4,12 +4,17 @@ from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer -class TestResourceMeatadataNormalizer(TestCase): +class TestResourceMetadataNormalizer(TestCase): def test_replace_property_with_path(self): template_data = { "Resources": { "Function1": { - "Properties": {"Code": "some value"}, + "Properties": { + "Code": { + "S3Bucket": {"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"}, + "S3Key": "00c88ea957f8f667f083d6073f00c49dd2ed7ddd87bb7a3b6f01d014243a3b22.zip", + } + }, "Metadata": {"aws:asset:path": "new path", "aws:asset:property": "Code"}, } } @@ -18,16 +23,22 @@ def test_replace_property_with_path(self): ResourceMetadataNormalizer.normalize(template_data) self.assertEqual("new path", template_data["Resources"]["Function1"]["Properties"]["Code"]) + self.assertEqual(True, template_data["Resources"]["Function1"]["Metadata"]["SamNormalized"]) def test_replace_all_resources_that_contain_metadata(self): template_data = { "Resources": { "Function1": { - "Properties": {"Code": "some value"}, + "Properties": { + "Code": { + "S3Bucket": {"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"}, + "S3Key": "00c88ea957f8f667f083d6073f00c49dd2ed7ddd87bb7a3b6f01d014243a3b22.zip", + } + }, "Metadata": {"aws:asset:path": "new path", "aws:asset:property": "Code"}, }, "Resource2": { - "Properties": {"SomeRandomProperty": "some value"}, + "Properties": {"SomeRandomProperty": {"Fn::Sub": "${AWS::AccountId}/some_value"}}, "Metadata": {"aws:asset:path": "super cool path", "aws:asset:property": "SomeRandomProperty"}, }, } @@ -47,7 +58,9 @@ def test_replace_all_resources_that_contain_image_metadata(self): "Function1": { "Properties": { "Code": { - "ImageUri": "Some Value", + "ImageUri": { + "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:b5d75370ccc2882b90f701c8f98952aae957e85e1928adac8860222960209056" + } } }, "Metadata": { @@ -79,7 +92,9 @@ def test_replace_all_resources_that_contain_image_metadata_windows_paths(self): "Function1": { "Properties": { "Code": { - "ImageUri": "Some Value", + "ImageUri": { + "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:b5d75370ccc2882b90f701c8f98952aae957e85e1928adac8860222960209056" + } } }, "Metadata": { @@ -316,3 +331,119 @@ def test_cdk_template_parameters_should_be_normalized(self): "AssetParametersb9866fd422d32492c62394e8c406ab4004f0c80364bab4957e67e31cf1130481ArtifactHash0A652345123" ].get("Default") ) + + def test_skip_normalizing_already_normalized_resource(self): + template_data = { + "Resources": { + "Function1": { + "Properties": { + "Code": { + "S3Bucket": {"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"}, + "S3Key": "00c88ea957f8f667f083d6073f00c49dd2ed7ddd87bb7a3b6f01d014243a3b22.zip", + } + }, + "Metadata": {"aws:asset:path": "new path", "aws:asset:property": "Code"}, + } + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertEqual("new path", template_data["Resources"]["Function1"]["Properties"]["Code"]) + self.assertEqual(True, template_data["Resources"]["Function1"]["Metadata"]["SamNormalized"]) + + # Normalized resource will not be normalized again + template_data["Resources"]["Function1"]["Metadata"]["aws:asset:path"] = "updated path" + ResourceMetadataNormalizer.normalize(template_data) + self.assertEqual("new path", template_data["Resources"]["Function1"]["Properties"]["Code"]) + + +class TestResourceMetadataNormalizerGetResourceId(TestCase): + def test_use_cdk_id_as_resource_id(self): + resource_id = ResourceMetadataNormalizer.get_resource_id( + { + "Type": "any:value", + "Properties": {"key": "value"}, + "Metadata": {"aws:cdk:path": "stack_id/func_cdk_id/Resource"}, + }, + "logical_id", + ) + + self.assertEquals("func_cdk_id", resource_id) + + def test_use_logical_id_as_resource_id_incase_of_invalid_cdk_path(self): + resource_id = ResourceMetadataNormalizer.get_resource_id( + {"Type": "any:value", "Properties": {"key": "value"}, "Metadata": {"aws:cdk:path": "func_cdk_id"}}, + "logical_id", + ) + + self.assertEquals("logical_id", resource_id) + + def test_use_cdk_id_as_resource_id_for_nested_stack(self): + resource_id = ResourceMetadataNormalizer.get_resource_id( + { + "Type": "AWS::CloudFormation::Stack", + "Properties": {"key": "value"}, + "Metadata": { + "aws:cdk:path": "parent_stack_id/nested_stack_id.NestedStack/nested_stack_id.NestedStackResource" + }, + }, + "logical_id", + ) + + self.assertEquals("nested_stack_id", resource_id) + + def test_use_logical_id_as_resource_id_for_invalid_nested_stack_path(self): + resource_id = ResourceMetadataNormalizer.get_resource_id( + { + "Type": "AWS::CloudFormation::Stack", + "Properties": {"key": "value"}, + "Metadata": { + "aws:cdk:path": "parent_stack_id/nested_stack_idNestedStack/nested_stack_id.NestedStackResource" + }, + }, + "logical_id", + ) + + self.assertEquals("nested_stack_idNestedStack", resource_id) + + def test_use_provided_customer_defined_id(self): + resource_id = ResourceMetadataNormalizer.get_resource_id( + { + "Type": "any:value", + "Properties": {"key": "value"}, + "Metadata": {"SamResourceId": "custom_id", "aws:cdk:path": "stack_id/func_cdk_id/Resource"}, + }, + "logical_id", + ) + + self.assertEquals("custom_id", resource_id) + + def test_use_provided_customer_defined_id_for_nested_stack(self): + resource_id = ResourceMetadataNormalizer.get_resource_id( + { + "Type": "AWS::CloudFormation::Stack", + "Properties": {"key": "value"}, + "Metadata": { + "SamResourceId": "custom_nested_stack_id", + "aws:cdk:path": "parent_stack_id/nested_stack_id.NestedStack/nested_stack_id.NestedStackResource", + }, + }, + "logical_id", + ) + + self.assertEquals("custom_nested_stack_id", resource_id) + + def test_use_logical_id_if_metadata_is_not_therer(self): + resource_id = ResourceMetadataNormalizer.get_resource_id( + {"Type": "any:value", "Properties": {"key": "value"}}, "logical_id" + ) + + self.assertEquals("logical_id", resource_id) + + def test_use_logical_id_if_cdk_path_not_exist(self): + resource_id = ResourceMetadataNormalizer.get_resource_id( + {"Type": "any:value", "Properties": {"key": "value"}, "Metadata": {}}, "logical_id" + ) + + self.assertEquals("logical_id", resource_id)