feat: add CDK synthesized templates support to sam deploy and package…

… commands (aws#3554)

* support package and deploy CDK synthesized templates

* add unit testing, and integration testing

* run black

* add deploy integration testing

* fix failed integration test cases

* apply pr comments

* fix package image, check if ecr_repository is not empty

* black and use Asset parameters pattern

* apply pr comments

samcli/commands/_utils/template.py

@@ -10,7 +10,7 @@
 from botocore.utils import set_value_from_jmespath
 
 from samcli.commands.exceptions import UserException
-from samcli.lib.samlib.resource_metadata_normalizer import ASSET_PATH_METADATA_KEY
+from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer, ASSET_PATH_METADATA_KEY
 from samcli.lib.utils.packagetype import ZIP, IMAGE
 from samcli.yamlhelper import yaml_parse, yaml_dump
 from samcli.lib.utils.resources import (
@@ -265,6 +265,7 @@ def get_template_parameters(template_file):
     Template Parameters as a dictionary
     """
     template_dict = get_template_data(template_file=template_file)
+    ResourceMetadataNormalizer.normalize(template_dict, True)
     return template_dict.get("Parameters", dict())
 
 

samcli/commands/package/package_context.py

@@ -14,7 +14,6 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-
 import json
 import logging
 import os
@@ -135,7 +134,14 @@ def run(self):
             raise PackageFailedError(template_file=self.template_file, ex=str(ex)) from ex
 
     def _export(self, template_path, use_json):
-        template = Template(template_path, os.getcwd(), self.uploaders, self.code_signer)
+        template = Template(
+            template_path,
+            os.getcwd(),
+            self.uploaders,
+            self.code_signer,
+            normalize_template=True,
+            normalize_parameters=True,
+        )
         exported_template = template.export()
 
         if use_json:

samcli/lib/package/artifact_exporter.py

@@ -20,6 +20,7 @@
 
 from botocore.utils import set_value_from_jmespath
 
+from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer
 from samcli.lib.utils.resources import (
     AWS_SERVERLESS_FUNCTION,
     AWS_CLOUDFORMATION_STACK,
@@ -78,7 +79,9 @@ def do_export(self, resource_id, resource_dict, parent_dir):
                 property_name=self.PROPERTY_NAME, resource_id=resource_id, template_path=abs_template_path
             )
 
-        exported_template_dict = Template(template_path, parent_dir, self.uploaders, self.code_signer).export()
+        exported_template_dict = Template(
+            template_path, parent_dir, self.uploaders, self.code_signer, normalize_template=True
+        ).export()
 
         exported_template_str = yaml_dump(exported_template_dict)
 
@@ -127,6 +130,8 @@ def __init__(
         ),
         metadata_to_export=frozenset(METADATA_EXPORT_LIST),
         template_str: Optional[str] = None,
+        normalize_template: bool = False,
+        normalize_parameters: bool = False,
     ):
         """
         Reads the template and makes it ready for export
@@ -144,6 +149,8 @@ def __init__(
             self.template_dir = template_dir
             self.code_signer = code_signer
         self.template_dict = yaml_parse(template_str)
+        if normalize_template:
+            ResourceMetadataNormalizer.normalize(self.template_dict, normalize_parameters)
         self.resources_to_export = resources_to_export
         self.metadata_to_export = metadata_to_export
         self.uploaders = uploaders

samcli/lib/package/ecr_uploader.py

@@ -79,7 +79,9 @@ def upload(self, image, resource_name):
 
             _tag = tag_translation(image, docker_image_id=docker_img.id, gen_tag=self.tag)
             repository = (
-                self.ecr_repo if not isinstance(self.ecr_repo_multi, dict) else self.ecr_repo_multi.get(resource_name)
+                self.ecr_repo
+                if not self.ecr_repo_multi or not isinstance(self.ecr_repo_multi, dict)
+                else self.ecr_repo_multi.get(resource_name)
             )
 
             docker_img.tag(repository=repository, tag=_tag)

samcli/lib/package/image_utils.py

@@ -44,6 +44,10 @@ def tag_translation(image, docker_image_id=None, gen_tag="latest"):
 
     # NOTE(sriram-mv): Checksum truncation Length is set to 12
     _id = docker_image_id.split(":")[1][:SHA_CHECKSUM_TRUNCATION_LENGTH]
-    name, tag = image.split(":")
+    if ":" in image:
+        name, tag = image.split(":")
+    else:
+        name = image
+        tag = None
     _tag = tag if tag else gen_tag
     return f"{name}-{_id}-{_tag}"

samcli/lib/samlib/resource_metadata_normalizer.py

@@ -4,6 +4,10 @@
 
 import logging
 from pathlib import Path
+import json
+import re
+
+from samcli.lib.iac.cdk.utils import is_cdk_project
 
 RESOURCES_KEY = "Resources"
 PROPERTIES_KEY = "Properties"
@@ -23,12 +27,17 @@
 ASSET_BUNDLED_METADATA_KEY = "aws:asset:is-bundled"
 SAM_METADATA_SKIP_BUILD_KEY = "SkipBuild"
 
+# https://github.com/aws/aws-cdk/blob/b1ecd3d49d7ebf97a54a80d06779ef0f0b113c16/packages/%40aws-cdk/assert-internal/lib/canonicalize-assets.ts#L19
+CDK_ASSET_PARAMETER_PATTERN = re.compile(
+    "^AssetParameters[0-9a-fA-F]{64}(?:S3Bucket|S3VersionKey|ArtifactHash)[0-9a-fA-F]{8}$"
+)
+
 LOG = logging.getLogger(__name__)
 
 
 class ResourceMetadataNormalizer:
     @staticmethod
-    def normalize(template_dict):
+    def normalize(template_dict, normalize_parameters=False):
         """
         Normalize all Resources in the template with the Metadata Key on the resource.
 
@@ -67,6 +76,35 @@ def normalize(template_dict):
                     },
                 )
 
+        # This is a work around to allow the customer to use sam deploy or package commands without the need to provide
+        # values for the CDK auto generated asset parameters. The suggested solution is to let CDK add some metadata to
+        # the autogenerated parameters, so sam can skip them safely.
+
+        # Normalizing CDK auto generated parameters. Search for parameters that meet these conditions:
+        # 1- parameter name matches pattern
+        #    `AssetParameters[0-9a-f]{64}(?:S3Bucket|S3VersionKey|ArtifactHash)[0-9A-F]{8}`
+        # 2- parameter type is string
+        # 3- there is no reference to this parameter any where in the template resources
+        # 4- there is no default value for this parameter
+        # We set an empty string as default value for the matching parameters, so the customer can use sam deploy or
+        # package commands without providing values for the auto generated parameters, as these parameters are not used
+        # in SAM (sam set the resources paths directly, and does not depend on template parameters)
+        if normalize_parameters and is_cdk_project(template_dict):
+            resources_as_string = json.dumps(resources)
+            parameters = template_dict.get("Parameters", {})
+
+            default_value = " "
+            for parameter_name, parameter_value in parameters.items():
+                parameter_name_match = CDK_ASSET_PARAMETER_PATTERN.match(parameter_name)
+                if (
+                    parameter_name_match
+                    and "Default" not in parameter_value
+                    and parameter_value.get("Type", "") == "String"
+                    and f'"Ref": "{parameter_name}"' not in resources_as_string
+                ):
+                    LOG.debug("set default value for parameter %s to '%s'", parameter_name, default_value)
+                    parameter_value["Default"] = default_value
+
     @staticmethod
     def _replace_property(property_key, property_value, resource, logical_id):
         """

tests/integration/deploy/test_deploy_command.py

@@ -1,5 +1,4 @@
 import os
-from samcli.lib.bootstrap.companion_stack.data_types import CompanionStack
 import shutil
 import tempfile
 import time
@@ -40,6 +39,7 @@ def setUpClass(cls):
             cls.docker_client.api.pull(repository=repo, tag=tag)
             cls.docker_client.api.tag(f"{repo}:{tag}", "emulation-python3.8", tag="latest")
             cls.docker_client.api.tag(f"{repo}:{tag}", "emulation-python3.8-2", tag="latest")
+            cls.docker_client.api.tag(f"{repo}:{tag}", "colorsrandomfunctionf61b9209", tag="latest")
 
         # setup signing profile arn & name
         cls.signing_profile_name = os.environ.get("AWS_SIGNING_PROFILE_NAME")
@@ -70,7 +70,7 @@ def tearDown(self):
                 cfn_client.delete_stack(StackName=stack_name)
         super().tearDown()
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_package_and_deploy_no_s3_bucket_all_args(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
         with tempfile.NamedTemporaryFile(delete=False) as output_template_file:
@@ -119,7 +119,7 @@ def test_package_and_deploy_no_s3_bucket_all_args(self, template_file):
             deploy_process = run_command(deploy_command_list_execute)
             self.assertEqual(deploy_process.process.returncode, 0)
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_no_package_and_deploy_with_s3_bucket_all_args(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -145,7 +145,13 @@ def test_no_package_and_deploy_with_s3_bucket_all_args(self, template_file):
         deploy_process_execute = run_command(deploy_command_list)
         self.assertEqual(deploy_process_execute.process.returncode, 0)
 
-    @parameterized.expand(["aws-serverless-function-image.yaml", "aws-lambda-function-image.yaml"])
+    @parameterized.expand(
+        [
+            "aws-serverless-function-image.yaml",
+            "aws-lambda-function-image.yaml",
+            "cdk_v1_synthesized_template_image_functions.json",
+        ]
+    )
     def test_no_package_and_deploy_with_s3_bucket_all_args_image_repository(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -173,7 +179,11 @@ def test_no_package_and_deploy_with_s3_bucket_all_args_image_repository(self, te
         self.assertEqual(deploy_process_execute.process.returncode, 0)
 
     @parameterized.expand(
-        [("Hello", "aws-serverless-function-image.yaml"), ("MyLambdaFunction", "aws-lambda-function-image.yaml")]
+        [
+            ("Hello", "aws-serverless-function-image.yaml"),
+            ("MyLambdaFunction", "aws-lambda-function-image.yaml"),
+            ("ColorsRandomFunctionF61B9209", "cdk_v1_synthesized_template_image_functions.json"),
+        ]
     )
     def test_no_package_and_deploy_with_s3_bucket_all_args_image_repositories(self, resource_id, template_file):
         template_path = self.test_data_path.joinpath(template_file)
@@ -201,7 +211,13 @@ def test_no_package_and_deploy_with_s3_bucket_all_args_image_repositories(self,
         deploy_process_execute = run_command(deploy_command_list)
         self.assertEqual(deploy_process_execute.process.returncode, 0)
 
-    @parameterized.expand(["aws-serverless-function-image.yaml", "aws-lambda-function-image.yaml"])
+    @parameterized.expand(
+        [
+            "aws-serverless-function-image.yaml",
+            "aws-lambda-function-image.yaml",
+            "cdk_v1_synthesized_template_image_functions.json",
+        ]
+    )
     def test_no_package_and_deploy_with_s3_bucket_all_args_resolve_image_repos(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -228,7 +244,7 @@ def test_no_package_and_deploy_with_s3_bucket_all_args_resolve_image_repos(self,
         deploy_process_execute = run_command(deploy_command_list)
         self.assertEqual(deploy_process_execute.process.returncode, 0)
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_no_package_and_deploy_with_s3_bucket_and_no_confirm_changeset(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -256,7 +272,7 @@ def test_no_package_and_deploy_with_s3_bucket_and_no_confirm_changeset(self, tem
         deploy_process_execute = run_command(deploy_command_list)
         self.assertEqual(deploy_process_execute.process.returncode, 0)
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_deploy_no_redeploy_on_same_built_artifacts(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
         # Build project
@@ -293,7 +309,7 @@ def test_deploy_no_redeploy_on_same_built_artifacts(self, template_file):
         # Does not cause a re-deploy
         self.assertEqual(deploy_process_execute.process.returncode, 1)
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_no_package_and_deploy_with_s3_bucket_all_args_confirm_changeset(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -319,7 +335,7 @@ def test_no_package_and_deploy_with_s3_bucket_all_args_confirm_changeset(self, t
         deploy_process_execute = run_command_with_input(deploy_command_list, "Y".encode())
         self.assertEqual(deploy_process_execute.process.returncode, 0)
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_deploy_without_s3_bucket(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -352,7 +368,7 @@ def test_deploy_without_s3_bucket(self, template_file):
             deploy_process_execute.stderr,
         )
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_deploy_without_stack_name(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -373,7 +389,7 @@ def test_deploy_without_stack_name(self, template_file):
         deploy_process_execute = run_command(deploy_command_list)
         self.assertEqual(deploy_process_execute.process.returncode, 2)
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_deploy_without_capabilities(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -396,8 +412,7 @@ def test_deploy_without_capabilities(self, template_file):
         deploy_process_execute = run_command(deploy_command_list)
         self.assertEqual(deploy_process_execute.process.returncode, 1)
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
-    def test_deploy_without_template_file(self, template_file):
+    def test_deploy_without_template_file(self):
         stack_name = self._method_to_stack_name(self.id())
 
         # Package and Deploy in one go without confirming change set.
@@ -417,7 +432,7 @@ def test_deploy_without_template_file(self, template_file):
         # Error template file not specified
         self.assertEqual(deploy_process_execute.process.returncode, 1)
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_deploy_with_s3_bucket_switch_region(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -474,7 +489,7 @@ def test_deploy_with_s3_bucket_switch_region(self, template_file):
             stderr,
         )
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_deploy_twice_with_no_fail_on_empty_changeset(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -513,7 +528,7 @@ def test_deploy_twice_with_no_fail_on_empty_changeset(self, template_file):
         stdout = deploy_process_execute.stdout.strip()
         self.assertIn(bytes(f"No changes to deploy. Stack {stack_name} is up to date", encoding="utf-8"), stdout)
 
-    @parameterized.expand(["aws-serverless-function.yaml"])
+    @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
     def test_deploy_twice_with_fail_on_empty_changeset(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
@@ -1171,8 +1186,8 @@ def test_deploy_update_failed_disable_rollback(self, template_file):
         deploy_process_execute = run_command(deploy_command_list)
         self.assertEqual(deploy_process_execute.process.returncode, 0)
 
-    def test_deploy_logs_warning_with_cdk_project(self):
-        template_file = "aws-serverless-function-cdk.yaml"
+    @parameterized.expand(["aws-serverless-function-cdk.yaml", "cdk_v1_synthesized_template_zip_functions.json"])
+    def test_deploy_logs_warning_with_cdk_project(self, template_file):
         template_path = self.test_data_path.joinpath(template_file)
 
         stack_name = self._method_to_stack_name(self.id())