feat: allow invoking built CDK synthesized templates (aws#3549)

* feat: allow invoking built CDK synthesized templates

* run black

* fixing testing issue

* add unit testing to cover custom resource Id

* run black

* skip normalizing the resources that have been normalized before

* apply pr comments

* apply pr comments

samcli/lib/build/app_builder.py

@@ -33,6 +33,7 @@
     AWS_SERVERLESS_FUNCTION,
     AWS_SERVERLESS_LAYERVERSION,
 )
+from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer
 from samcli.lib.docker.log_streamer import LogStreamer, LogStreamError
 from samcli.lib.providers.provider import ResourcesToBuildCollector, Function, get_full_path, Stack, LayerVersion
 from samcli.lib.utils.colors import Colored
@@ -277,8 +278,8 @@ def update_template(
         template_dict = stack.template_dict
 
         for logical_id, resource in template_dict.get("Resources", {}).items():
-
-            full_path = get_full_path(stack.stack_path, logical_id)
+            resource_iac_id = ResourceMetadataNormalizer.get_resource_id(resource, logical_id)
+            full_path = get_full_path(stack.stack_path, resource_iac_id)
             has_build_artifact = full_path in built_artifacts
             is_stack = full_path in stack_output_template_path_by_stack_path
 

samcli/lib/samlib/resource_metadata_normalizer.py

@@ -9,17 +9,24 @@
 
 from samcli.lib.iac.cdk.utils import is_cdk_project
 
+from samcli.lib.utils.resources import AWS_CLOUDFORMATION_STACK
+
+CDK_NESTED_STACK_RESOURCE_ID_SUFFIX = ".NestedStack"
+
 RESOURCES_KEY = "Resources"
 PROPERTIES_KEY = "Properties"
 METADATA_KEY = "Metadata"
 
+RESOURCE_CDK_PATH_METADATA_KEY = "aws:cdk:path"
 ASSET_PATH_METADATA_KEY = "aws:asset:path"
 ASSET_PROPERTY_METADATA_KEY = "aws:asset:property"
 
 IMAGE_ASSET_PROPERTY = "Code.ImageUri"
 ASSET_DOCKERFILE_PATH_KEY = "aws:asset:dockerfile-path"
 ASSET_DOCKERFILE_BUILD_ARGS_KEY = "aws:asset:docker-build-args"
 
+SAM_RESOURCE_ID_KEY = "SamResourceId"
+SAM_IS_NORMALIZED = "SamNormalized"
 SAM_METADATA_DOCKERFILE_KEY = "Dockerfile"
 SAM_METADATA_DOCKER_CONTEXT_KEY = "DockerContext"
 SAM_METADATA_DOCKER_BUILD_ARGS_KEY = "DockerBuildArgs"
@@ -53,18 +60,21 @@ def normalize(template_dict, normalize_parameters=False):
 
         for logical_id, resource in resources.items():
             resource_metadata = resource.get(METADATA_KEY, {})
-            asset_property = resource_metadata.get(ASSET_PROPERTY_METADATA_KEY)
-
-            if asset_property == IMAGE_ASSET_PROPERTY:
-                asset_metadata = ResourceMetadataNormalizer._extract_image_asset_metadata(resource_metadata)
-                ResourceMetadataNormalizer._update_resource_metadata(resource_metadata, asset_metadata)
-                # For image-type functions, the asset path is expected to be the name of the Docker image.
-                # When building, we set the name of the image to be the logical id of the function.
-                asset_path = logical_id.lower()
-            else:
-                asset_path = resource_metadata.get(ASSET_PATH_METADATA_KEY)
-
-            ResourceMetadataNormalizer._replace_property(asset_property, asset_path, resource, logical_id)
+            is_normalized = resource_metadata.get(SAM_IS_NORMALIZED, False)
+            if not is_normalized:
+                asset_property = resource_metadata.get(ASSET_PROPERTY_METADATA_KEY)
+                if asset_property == IMAGE_ASSET_PROPERTY:
+                    asset_metadata = ResourceMetadataNormalizer._extract_image_asset_metadata(resource_metadata)
+                    ResourceMetadataNormalizer._update_resource_metadata(resource_metadata, asset_metadata)
+                    # For image-type functions, the asset path is expected to be the name of the Docker image.
+                    # When building, we set the name of the image to be the logical id of the function.
+                    asset_path = logical_id.lower()
+                else:
+                    asset_path = resource_metadata.get(ASSET_PATH_METADATA_KEY)
+
+                ResourceMetadataNormalizer._replace_property(asset_property, asset_path, resource, logical_id)
+                if asset_path and asset_property:
+                    resource_metadata[SAM_IS_NORMALIZED] = True
 
             # Set SkipBuild metadata iff is-bundled metadata exists, and value is True
             skip_build = resource_metadata.get(ASSET_BUNDLED_METADATA_KEY, False)
@@ -182,3 +192,65 @@ def _update_resource_metadata(metadata, updated_values):
         """
         for key, val in updated_values.items():
             metadata[key] = val
+
+    @staticmethod
+    def get_resource_id(resource_properties, logical_id):
+        """
+        Get unique id for a resource.
+        for any resource, the resource id can be the customer defined id if exist, if not exist it can be the
+        cdk-defined resource id, or the logical id if the resource id is not found.
+
+        Parameters
+        ----------
+        resource_properties dict
+            Properties of this resource
+        logical_id str
+            LogicalID of the resource
+
+        Returns
+        -------
+        str
+            The unique function id
+        """
+        resource_metadata = resource_properties.get("Metadata", {})
+        customer_defined_id = resource_metadata.get(SAM_RESOURCE_ID_KEY)
+
+        if isinstance(customer_defined_id, str) and customer_defined_id:
+            LOG.debug(
+                "Sam customer defined id is more priority than other IDs. Customer defined id for resource %s is %s",
+                logical_id,
+                customer_defined_id,
+            )
+            return customer_defined_id
+
+        resource_cdk_path = resource_metadata.get(RESOURCE_CDK_PATH_METADATA_KEY)
+
+        if not isinstance(resource_cdk_path, str) or not resource_cdk_path:
+            LOG.debug(
+                "There is no customer defined id or cdk path defined for resource %s, so we will use the resource "
+                "logical id as the resource id",
+                logical_id,
+            )
+            return logical_id
+
+        # aws:cdk:path metadata format of functions: {stack_id}/{function_id}/Resource
+        # Design doc of CDK path: https://github.com/aws/aws-cdk/blob/master/design/construct-tree.md
+        cdk_path_partitions = resource_cdk_path.split("/")
+
+        LOG.debug("CDK Path for resource %s is %s", logical_id, cdk_path_partitions)
+
+        if len(cdk_path_partitions) < 2:
+            LOG.warning(
+                "Cannot detect function id from aws:cdk:path metadata '%s', using default logical id", resource_cdk_path
+            )
+            return logical_id
+
+        cdk_resource_id = cdk_path_partitions[-2]
+
+        # Check if the Resource is nested Stack
+        if resource_properties.get("Type", "") == AWS_CLOUDFORMATION_STACK and cdk_resource_id.endswith(
+            CDK_NESTED_STACK_RESOURCE_ID_SUFFIX
+        ):
+            cdk_resource_id = cdk_resource_id[: -len(CDK_NESTED_STACK_RESOURCE_ID_SUFFIX)]
+
+        return cdk_resource_id

tests/integration/buildcmd/build_integ_base.py

@@ -189,10 +189,14 @@ def _verify_invoke_built_function(self, template_path, function_logical_id, over
             "-t",
             str(template_path),
             "--no-event",
-            "--parameter-overrides",
-            overrides,
         ]
 
+        if overrides:
+            cmdlist += [
+                "--parameter-overrides",
+                overrides,
+            ]
+
         LOG.info("Running invoke Command: {}".format(cmdlist))
 
         process_execute = run_command(cmdlist)
@@ -518,12 +522,21 @@ class BuildIntegPythonBase(BuildIntegBase):
     }
 
     FUNCTION_LOGICAL_ID = "Function"
+    prop = "CodeUri"
 
-    def _test_with_default_requirements(self, runtime, codeuri, use_container, relative_path, architecture=None):
+    def _test_with_default_requirements(
+        self,
+        runtime,
+        codeuri,
+        use_container,
+        relative_path,
+        do_override=True,
+        check_function_only=False,
+        architecture=None,
+    ):
         if use_container and (SKIP_DOCKER_TESTS or SKIP_DOCKER_BUILD):
             self.skipTest(SKIP_DOCKER_MESSAGE)
-
-        overrides = self.get_override(runtime, codeuri, architecture, "main.handler")
+        overrides = self.get_override(runtime, codeuri, architecture, "main.handler") if do_override else None
         cmdlist = self.get_command_list(use_container=use_container, parameter_overrides=overrides)
 
         LOG.info("Running Command: {}".format(cmdlist))
@@ -533,37 +546,41 @@ def _test_with_default_requirements(self, runtime, codeuri, use_container, relat
             self.default_build_dir, self.FUNCTION_LOGICAL_ID, self.EXPECTED_FILES_PROJECT_MANIFEST
         )
 
-        self._verify_resource_property(
-            str(self.built_template),
-            "OtherRelativePathResource",
-            "BodyS3Location",
-            os.path.relpath(
-                os.path.normpath(os.path.join(str(relative_path), "SomeRelativePath")),
-                str(self.default_build_dir),
-            ),
-        )
+        if not check_function_only:
+            self._verify_resource_property(
+                str(self.built_template),
+                "OtherRelativePathResource",
+                "BodyS3Location",
+                os.path.relpath(
+                    os.path.normpath(os.path.join(str(relative_path), "SomeRelativePath")),
+                    str(self.default_build_dir),
+                ),
+            )
 
-        self._verify_resource_property(
-            str(self.built_template),
-            "GlueResource",
-            "Command.ScriptLocation",
-            os.path.relpath(
-                os.path.normpath(os.path.join(str(relative_path), "SomeRelativePath")),
-                str(self.default_build_dir),
-            ),
-        )
+            self._verify_resource_property(
+                str(self.built_template),
+                "GlueResource",
+                "Command.ScriptLocation",
+                os.path.relpath(
+                    os.path.normpath(os.path.join(str(relative_path), "SomeRelativePath")),
+                    str(self.default_build_dir),
+                ),
+            )
 
-        self._verify_resource_property(
-            str(self.built_template),
-            "ExampleNestedStack",
-            "TemplateURL",
-            "https://s3.amazonaws.com/examplebucket/exampletemplate.yml",
-        )
+            self._verify_resource_property(
+                str(self.built_template),
+                "ExampleNestedStack",
+                "TemplateURL",
+                "https://s3.amazonaws.com/examplebucket/exampletemplate.yml",
+            )
 
         expected = {"pi": "3.14"}
         if not SKIP_DOCKER_TESTS:
             self._verify_invoke_built_function(
-                self.built_template, self.FUNCTION_LOGICAL_ID, self._make_parameter_override_arg(overrides), expected
+                self.built_template,
+                self.FUNCTION_LOGICAL_ID,
+                self._make_parameter_override_arg(overrides) if do_override else None,
+                expected,
             )
         if use_container:
             self.verify_docker_container_cleanedup(runtime)
@@ -580,7 +597,7 @@ def _verify_built_artifact(self, build_dir, function_logical_id, expected_files)
         resource_artifact_dir = build_dir.joinpath(function_logical_id)
 
         # Make sure the template has correct CodeUri for resource
-        self._verify_resource_property(str(template_path), function_logical_id, "CodeUri", function_logical_id)
+        self._verify_resource_property(str(template_path), function_logical_id, self.prop, function_logical_id)
 
         all_artifacts = set(os.listdir(str(resource_artifact_dir)))
         actual_files = all_artifacts.intersection(expected_files)

tests/integration/buildcmd/test_build_cmd.py

@@ -240,27 +240,59 @@ def _validate_skipped_built_function(
     ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE),
     "Skip build tests on windows when running in CI unless overridden",
 )
+@parameterized_class(
+    (
+        "template",
+        "FUNCTION_LOGICAL_ID",
+        "overrides",
+        "runtime",
+        "codeuri",
+        "use_container",
+        "check_function_only",
+        "prop",
+    ),
+    [
+        ("template.yaml", "Function", True, "python2.7", "Python", False, False, "CodeUri"),
+        ("template.yaml", "Function", True, "python3.6", "Python", False, False, "CodeUri"),
+        ("template.yaml", "Function", True, "python3.7", "Python", False, False, "CodeUri"),
+        ("template.yaml", "Function", True, "python3.8", "Python", False, False, "CodeUri"),
+        ("template.yaml", "Function", True, "python3.9", "Python", False, False, "CodeUri"),
+        ("template.yaml", "Function", True, "python3.7", "PythonPEP600", False, False, "CodeUri"),
+        ("template.yaml", "Function", True, "python3.8", "PythonPEP600", False, False, "CodeUri"),
+        ("template.yaml", "Function", True, "python2.7", "Python", "use_container", False, "CodeUri"),
+        ("template.yaml", "Function", True, "python3.6", "Python", "use_container", False, "CodeUri"),
+        ("template.yaml", "Function", True, "python3.7", "Python", "use_container", False, "CodeUri"),
+        ("template.yaml", "Function", True, "python3.8", "Python", "use_container", False, "CodeUri"),
+        ("template.yaml", "Function", True, "python3.9", "Python", "use_container", False, "CodeUri"),
+        (
+            "cdk_v1_synthesized_template_zip_image_functions.json",
+            "RandomCitiesFunction5C47A2B8",
+            False,
+            None,
+            None,
+            False,
+            True,
+            "Code",
+        ),
+    ],
+)
 class TestBuildCommand_PythonFunctions(BuildIntegPythonBase):
-    @parameterized.expand(
-        [
-            ("python2.7", "Python", False),
-            ("python3.6", "Python", False),
-            ("python3.7", "Python", False),
-            ("python3.8", "Python", False),
-            ("python3.9", "Python", False),
-            # numpy 1.20.3 (in PythonPEP600/requirements.txt) only support python 3.7+
-            ("python3.7", "PythonPEP600", False),
-            ("python3.8", "PythonPEP600", False),
-            ("python2.7", "Python", "use_container"),
-            ("python3.6", "Python", "use_container"),
-            ("python3.7", "Python", "use_container"),
-            ("python3.8", "Python", "use_container"),
-            ("python3.9", "Python", "use_container"),
-        ]
-    )
+    overrides = True
+    runtime = "python2.7"
+    codeuri = "Python"
+    use_container = False
+    check_function_only = False
+
     @pytest.mark.flaky(reruns=3)
-    def test_with_default_requirements(self, runtime, codeuri, use_container):
-        self._test_with_default_requirements(runtime, codeuri, use_container, self.test_data_path)
+    def test_with_default_requirements(self):
+        self._test_with_default_requirements(
+            self.runtime,
+            self.codeuri,
+            self.use_container,
+            self.test_data_path,
+            do_override=self.overrides,
+            check_function_only=self.check_function_only,
+        )
 
 
 @skipIf(
@@ -290,7 +322,9 @@ class TestBuildCommand_PythonFunctions_With_Specified_Architecture(BuildIntegPyt
     )
     @pytest.mark.flaky(reruns=3)
     def test_with_default_requirements(self, runtime, codeuri, use_container, architecture):
-        self._test_with_default_requirements(runtime, codeuri, use_container, self.test_data_path, architecture)
+        self._test_with_default_requirements(
+            runtime, codeuri, use_container, self.test_data_path, architecture=architecture
+        )
 
 
 @skipIf(

tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/Dockerfile

@@ -0,0 +1,15 @@
+ARG BASE_RUNTIME
+
+FROM public.ecr.aws/lambda/python:$BASE_RUNTIME
+
+ARG FUNCTION_DIR="/var/task"
+
+RUN mkdir -p $FUNCTION_DIR
+
+COPY main.py $FUNCTION_DIR
+
+COPY __init__.py $FUNCTION_DIR
+
+COPY requirements.txt $FUNCTION_DIR
+
+RUN python -m pip install -r $FUNCTION_DIR/requirements.txt -t $FUNCTION_DIR

tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/main.py

@@ -0,0 +1,19 @@
+import numpy
+
+# from cryptography.fernet import Fernet
+
+
+def handler(event, context):
+
+    # Try using some of the modules to make sure they work & don't crash the process
+    # print(Fernet.generate_key())
+
+    return {"pi": "{0:.2f}".format(numpy.pi)}
+
+
+def first_function_handler(event, context):
+    return "Hello World"
+
+
+def second_function_handler(event, context):
+    return "Hello Mars"

tests/integration/testdata/buildcmd/asset.6598609927b272b36fdf01072092f9851ddcd1b41ba294f736ce77091f5cc456/requirements.txt

@@ -0,0 +1 @@
+numpy<1.20.4