Merge pull request #3454 from ekcrisp/feature/use-s3-uploader-for-CF-…

…package-with-include-transforms

Feature/use s3 uploader for cf package with include transforms

awscli/customizations/cloudformation/artifact_exporter.py

@@ -432,6 +432,19 @@ def do_export(self, resource_id, resource_dict, parent_dir):
     CloudFormationStackResource
 ]
 
+def include_transform_export_handler(template_dict, uploader):
+    if template_dict.get("Name", None) != "AWS::Include":
+        return template_dict
+    include_location = template_dict.get("Parameters", {}).get("Location", {})
+    if (is_local_file(include_location)):
+        template_dict["Parameters"]["Location"] = uploader.upload_with_dedup(include_location)
+    return template_dict
+
+GLOBAL_EXPORT_DICT = {
+    "Fn::Transform": include_transform_export_handler
+}
+
+
 class Template(object):
     """
     Class to export a CloudFormation template
@@ -459,6 +472,25 @@ def __init__(self, template_path, parent_dir, uploader,
         self.resources_to_export = resources_to_export
         self.uploader = uploader
 
+    def export_global_artifacts(self, template_dict):
+        """
+        Template params such as AWS::Include transforms are not specific to 
+        any resource type but contain artifacts that should be exported,
+        here we iterate through the template dict and export params with a 
+        handler defined in GLOBAL_EXPORT_DICT
+        """
+        for key, val in template_dict.items():
+            if key in GLOBAL_EXPORT_DICT:
+                template_dict[key] = GLOBAL_EXPORT_DICT[key](val, self.uploader)
+            elif isinstance(val, dict):
+                self.export_global_artifacts(val)
+            elif isinstance(val, list):
+                for item in val:
+                    if isinstance(item, dict):
+                        self.export_global_artifacts(item)
+        return template_dict
+
+
     def export(self):
         """
         Exports the local artifacts referenced by the given template to an
@@ -470,6 +502,8 @@ def export(self):
         if "Resources" not in self.template_dict:
             return self.template_dict
 
+        self.template_dict = self.export_global_artifacts(self.template_dict)
+
         for resource_id, resource in self.template_dict["Resources"].items():
 
             resource_type = resource.get("Type", None)

awscli/examples/cloudformation/_package_description.rst

@@ -8,7 +8,7 @@ Use this command to quickly upload local artifacts that might be required by
 your template. After you package your template's artifacts, run the deploy
 command to ``deploy`` the returned template.
 
-This command can upload local artifacts specified by following properties of a resource:
+This command can upload local artifacts referenced in the following places:
 
 
     - ``BodyS3Location`` property for the ``AWS::ApiGateway::RestApi`` resource
@@ -18,6 +18,7 @@ This command can upload local artifacts specified by following properties of a r
     - ``RequestMappingTemplateS3Location`` property for the ``AWS::AppSync::Resolver`` resource
     - ``ResponseMappingTemplateS3Location`` property for the ``AWS::AppSync::Resolver`` resource
     - ``DefinitionUri`` property for the ``AWS::Serverless::Api`` resource
+    - ``Location`` parameter for the ``AWS::Include`` transform
     - ``SourceBundle`` property for the ``AWS::ElasticBeanstalk::ApplicationVersion`` resource
     - ``TemplateURL`` property for the ``AWS::CloudFormation::Stack`` resource
 

tests/unit/customizations/cloudformation/test_artifact_exporter.py

@@ -7,7 +7,7 @@
 import zipfile
 
 from nose.tools import assert_true, assert_false, assert_equal
-from contextlib import contextmanager,closing
+from contextlib import contextmanager, closing
 from mock import patch, Mock, MagicMock
 from botocore.stub import Stubber
 from awscli.testutils import unittest, FileCreator
@@ -19,7 +19,7 @@
     ServerlessFunctionResource, GraphQLSchemaResource, \
     LambdaFunctionResource, ApiGatewayRestApiResource, \
     ElasticBeanstalkApplicationVersion, CloudFormationStackResource, \
-    copy_to_temp_dir
+    copy_to_temp_dir, include_transform_export_handler, GLOBAL_EXPORT_DICT
 
 
 def test_is_s3_url():
@@ -772,6 +772,92 @@ def test_template_export(self, yaml_parse_mock):
             resource_type2_instance.export.assert_called_once_with(
                 "Resource2", mock.ANY, template_dir)
 
+    @patch("awscli.customizations.cloudformation.artifact_exporter.yaml_parse")
+    def test_template_global_export(self, yaml_parse_mock):
+        parent_dir = os.path.sep
+        template_dir = os.path.join(parent_dir, 'foo', 'bar')
+        template_path = os.path.join(template_dir, 'path')
+        template_str = self.example_yaml_template()
+
+        resource_type1_class = Mock()
+        resource_type1_instance = Mock()
+        resource_type1_class.return_value = resource_type1_instance
+        resource_type2_class = Mock()
+        resource_type2_instance = Mock()
+        resource_type2_class.return_value = resource_type2_instance
+
+        resources_to_export = {
+            "resource_type1": resource_type1_class,
+            "resource_type2": resource_type2_class
+        }
+        properties1 = {"foo": "bar", "Fn::Transform": {"Name": "AWS::Include", "Parameters": {"Location": "foo.yaml"}}}
+        properties2 = {"foo": "bar", "Fn::Transform": {"Name": "AWS::OtherTransform"}}
+        properties_in_list = {"Fn::Transform": {"Name": "AWS::Include", "Parameters": {"Location": "bar.yaml"}}}
+        template_dict = {
+            "Resources": {
+                "Resource1": {
+                    "Type": "resource_type1",
+                    "Properties": properties1
+                },
+                "Resource2": {
+                    "Type": "resource_type2",
+                    "Properties": properties2,
+                }
+            },
+            "List": ["foo", properties_in_list]
+        }
+        open_mock = mock.mock_open()
+        include_transform_export_handler_mock = Mock()
+        include_transform_export_handler_mock.return_value = {"Name": "AWS::Include", "Parameters": {"Location": "s3://foo"}}
+        yaml_parse_mock.return_value = template_dict
+
+        with patch(
+                "awscli.customizations.cloudformation.artifact_exporter.open",
+                open_mock(read_data=template_str)) as open_mock:
+            with patch.dict(GLOBAL_EXPORT_DICT, {"Fn::Transform": include_transform_export_handler_mock}):
+                template_exporter = Template(
+                    template_path, parent_dir, self.s3_uploader_mock,
+                    resources_to_export)
+
+                exported_template = template_exporter.export_global_artifacts(template_exporter.template_dict)
+
+                first_call_args, kwargs = include_transform_export_handler_mock.call_args_list[0]
+                second_call_args, kwargs = include_transform_export_handler_mock.call_args_list[1]
+                third_call_args, kwargs = include_transform_export_handler_mock.call_args_list[2]
+                call_args = [first_call_args[0], second_call_args[0], third_call_args[0]]
+                self.assertTrue({"Name": "AWS::Include", "Parameters": {"Location": "foo.yaml"}} in call_args)
+                self.assertTrue({"Name": "AWS::OtherTransform"} in call_args)
+                self.assertTrue({"Name": "AWS::Include", "Parameters": {"Location": "bar.yaml"}} in call_args)
+                self.assertEquals(include_transform_export_handler_mock.call_count, 3)
+                #new s3 url is added to include location
+                self.assertEquals(exported_template["Resources"]["Resource1"]["Properties"]["Fn::Transform"], {"Name": "AWS::Include", "Parameters": {"Location": "s3://foo"}})
+                self.assertEquals(exported_template["List"][1]["Fn::Transform"], {"Name": "AWS::Include", "Parameters": {"Location": "s3://foo"}})
+
+    @patch("awscli.customizations.cloudformation.artifact_exporter.is_local_file")
+    def test_include_transform_export_handler(self, is_local_file_mock):
+        #exports transform
+        self.s3_uploader_mock.upload_with_dedup.return_value = "s3://foo"
+        is_local_file_mock.return_value = True
+        handler_output = include_transform_export_handler({"Name": "AWS::Include", "Parameters": {"Location": "foo.yaml"}}, self.s3_uploader_mock)
+        self.s3_uploader_mock.upload_with_dedup.assert_called_once_with("foo.yaml")
+        self.assertEquals(handler_output, {'Name': 'AWS::Include', 'Parameters': {'Location': 's3://foo'}})
+
+    @patch("awscli.customizations.cloudformation.artifact_exporter.is_local_file")
+    def test_include_transform_export_handler_non_local_file(self, is_local_file_mock):
+        #returns unchanged template dict if transform not a local file
+        is_local_file_mock.return_value = False
+        handler_output = include_transform_export_handler({"Name": "AWS::Include", "Parameters": {"Location": "http://foo.yaml"}}, self.s3_uploader_mock)
+        self.s3_uploader_mock.upload_with_dedup.assert_not_called()
+        self.assertEquals(handler_output, {"Name": "AWS::Include", "Parameters": {"Location": "http://foo.yaml"}})
+
+    @patch("awscli.customizations.cloudformation.artifact_exporter.is_local_file")
+    def test_include_transform_export_handler_non_include_transform(self, is_local_file_mock):
+        #ignores transform that is not aws::include
+        handler_output = include_transform_export_handler({"Name": "AWS::OtherTransform", "Parameters": {"Location": "foo.yaml"}}, self.s3_uploader_mock)
+        self.s3_uploader_mock.upload_with_dedup.assert_not_called()
+        self.assertEquals(handler_output, {"Name": "AWS::OtherTransform", "Parameters": {"Location": "foo.yaml"}})
+
+
     def test_template_export_path_be_folder(self):
 
         template_path = "/path/foo"