From c1dfa13cac1e98aac4d5972f85767aeaa6e24b99 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Sat, 25 May 2019 07:32:14 -0400 Subject: [PATCH] cli package terraform support --- chalice/cli/__init__.py | 8 +- chalice/cli/factory.py | 6 +- chalice/deploy/swagger.py | 15 ++ chalice/package.py | 377 ++++++++++++++++++++++++++++++++----- docs/source/index.rst | 1 + docs/source/topics/tf.rst | 128 +++++++++++++ tests/unit/test_package.py | 315 +++++++++++++++++++++++++++++-- 7 files changed, 786 insertions(+), 64 deletions(-) create mode 100644 docs/source/topics/tf.rst diff --git a/chalice/cli/__init__.py b/chalice/cli/__init__.py index b43038e0ea..d0f463759b 100644 --- a/chalice/cli/__init__.py +++ b/chalice/cli/__init__.py @@ -374,13 +374,15 @@ def generate_sdk(ctx, sdk_type, stage, outdir): "this argument is specified, a single " "zip file will be created instead.")) @click.option('--stage', default=DEFAULT_STAGE_NAME) +@click.option('--pkg-format', default='cloudformation', + type=click.Choice(['cloudformation', 'terraform'])) @click.argument('out') @click.pass_context -def package(ctx, single_file, stage, out): - # type: (click.Context, bool, str, str) -> None +def package(ctx, single_file, stage, out, pkg_format='cloudformation'): + # type: (click.Context, bool, str, str, str) -> None factory = ctx.obj['factory'] # type: CLIFactory config = factory.create_config_obj(stage) - packager = factory.create_app_packager(config) + packager = factory.create_app_packager(config, pkg_format) if single_file: dirname = tempfile.mkdtemp() try: diff --git a/chalice/cli/factory.py b/chalice/cli/factory.py index 2b6d736baa..590d4cc3a5 100644 --- a/chalice/cli/factory.py +++ b/chalice/cli/factory.py @@ -177,9 +177,9 @@ def _validate_config_from_disk(self, config): except ValueError: raise UnknownConfigFileVersion(string_version) - def create_app_packager(self, config): - # type: (Config) -> AppPackager - return create_app_packager(config) + def create_app_packager(self, config, package_format): + # type: (Config, str) -> AppPackager + return create_app_packager(config, package_format) def create_log_retriever(self, session, lambda_arn): # type: (Session, str) -> LogRetriever diff --git a/chalice/deploy/swagger.py b/chalice/deploy/swagger.py index ea27bf7427..725ab7721c 100644 --- a/chalice/deploy/swagger.py +++ b/chalice/deploy/swagger.py @@ -279,3 +279,18 @@ def _auth_uri(self, authorizer): '/functions/{%s}/invocations' % varname, ['region_name', varname], ) + + +class TerraformSwaggerGenerator(SwaggerGenerator): + + def __init__(self): + # type: () -> None + pass + + def _uri(self, lambda_arn=None): + # type: (Optional[str]) -> Any + return '${aws_lambda_function.api_handler.invoke_arn}' + + def _auth_uri(self, authorizer): + # type: (ChaliceAuthorizer) -> Any + return '${aws_lambda_function.%s.invoke_arn}' % (authorizer.name) diff --git a/chalice/package.py b/chalice/package.py index 0012794e3f..8e394040f2 100644 --- a/chalice/package.py +++ b/chalice/package.py @@ -1,10 +1,12 @@ +import json import os import copy from typing import Any, Dict, List, Set, Union # noqa from typing import cast -from chalice.deploy.swagger import CFNSwaggerGenerator +from chalice.deploy.swagger import ( + CFNSwaggerGenerator, TerraformSwaggerGenerator) from chalice.utils import OSUtils, UI, serialize_to_json, to_cfn_resource_name from chalice.config import Config # noqa from chalice.deploy import models @@ -14,23 +16,34 @@ from chalice.deploy.deployer import create_build_stage -def create_app_packager(config): - # type: (Config) -> AppPackager +def create_app_packager(config, package_format='cloudformation'): + # type: (Config, str) -> AppPackager osutils = OSUtils() ui = UI() application_builder = ApplicationGraphBuilder() deps_builder = DependencyBuilder() - build_stage = create_build_stage( - osutils, ui, CFNSwaggerGenerator() - ) - resource_builder = ResourceBuilder(application_builder, - deps_builder, build_stage) + post_processor = None # type: Union[None, TemplatePostProcessor] + generator = None # type: Union[None, TemplateGenerator] + + if package_format == 'cloudformation': + build_stage = create_build_stage( + osutils, ui, CFNSwaggerGenerator()) + post_processor = SAMPostProcessor(osutils=osutils) + generator = SAMTemplateGenerator() + else: + build_stage = create_build_stage( + osutils, ui, TerraformSwaggerGenerator()) + generator = TerraformGenerator() + post_processor = TerraformPostProcessor(osutils=osutils) + + resource_builder = ResourceBuilder( + application_builder, deps_builder, build_stage) + return AppPackager( - SAMTemplateGenerator(), + generator, resource_builder, - TemplatePostProcessor(osutils=osutils), - osutils, - ) + post_processor, + osutils) class UnsupportedFeatureError(Exception): @@ -61,7 +74,43 @@ def construct_resources(self, config, chalice_stage_name): return resources -class SAMTemplateGenerator(object): +class TemplateGenerator(object): + + template_file = None # type: str + + def dispatch(self, resource, template): + # type: (models.Model, Dict[str, Any]) -> None + name = '_generate_%s' % resource.__class__.__name__.lower() + handler = getattr(self, name, self._default) + handler(resource, template) + + def generate(self, resources): + # type: (List[models.Model]) -> Dict[str, Any] + raise NotImplementedError() + + def _generate_filebasediampolicy(self, resource, template): + # type: (models.FileBasedIAMPolicy, Dict[str, Any]) -> None + pass + + def _generate_autogeniampolicy(self, resource, template): + # type: (models.AutoGenIAMPolicy, Dict[str, Any]) -> None + pass + + def _generate_deploymentpackage(self, resource, template): + # type: (models.DeploymentPackage, Dict[str, Any]) -> None + pass + + def _generate_precreatediamrole(self, resource, template): + # type: (models.PreCreatedIAMRole, Dict[str, Any]) -> None + pass + + def _default(self, resource, template): + # type: (models.Model, Dict[str, Any]) -> None + raise UnsupportedFeatureError(resource) + + +class SAMTemplateGenerator(TemplateGenerator): + _BASE_TEMPLATE = { 'AWSTemplateFormatVersion': '2010-09-09', 'Transform': 'AWS::Serverless-2016-10-31', @@ -69,18 +118,18 @@ class SAMTemplateGenerator(object): 'Resources': {}, } + template_file = "sam.json" + def __init__(self): # type: () -> None self._seen_names = set([]) # type: Set[str] - def generate_sam_template(self, resources): + def generate(self, resources): # type: (List[models.Model]) -> Dict[str, Any] template = copy.deepcopy(self._BASE_TEMPLATE) self._seen_names.clear() for resource in resources: - name = '_generate_%s' % resource.__class__.__name__.lower() - handler = getattr(self, name, self._default) - handler(resource, template) + self.dispatch(resource, template) return template def _generate_scheduledevent(self, resource, template): @@ -256,22 +305,6 @@ def _generate_managediamrole(self, resource, template): } } - def _generate_filebasediampolicy(self, resource, template): - # type: (models.FileBasedIAMPolicy, Dict[str, Any]) -> None - pass - - def _generate_autogeniampolicy(self, resource, template): - # type: (models.AutoGenIAMPolicy, Dict[str, Any]) -> None - pass - - def _generate_deploymentpackage(self, resource, template): - # type: (models.DeploymentPackage, Dict[str, Any]) -> None - pass - - def _generate_precreatediamrole(self, resource, template): - # type: (models.PreCreatedIAMRole, Dict[str, Any]) -> None - pass - def _generate_s3bucketnotification(self, resource, template): # type: (models.S3BucketNotification, Dict[str, Any]) -> None message = ( @@ -330,10 +363,6 @@ def _generate_sqseventsource(self, resource, template): } } - def _default(self, resource, template): - # type: (models.Model, Dict[str, Any]) -> None - raise NotImplementedError(resource) - def _register_cfn_resource_name(self, name): # type: (str) -> str cfn_name = to_cfn_resource_name(name) @@ -346,15 +375,253 @@ def _register_cfn_resource_name(self, name): return cfn_name +class TerraformGenerator(TemplateGenerator): + + template_file = "chalice.tf.json" + + def generate(self, resources): + # type: (List[models.Model]) -> Dict[str, Any] + template = { + 'resource': {}, + 'terraform': { + 'required_version': '> 0.11.0, < 0.12' + }, + 'data': { + 'aws_caller_identity': {'chalice': {}}, + 'aws_region': {'chalice': {}} + } + } + for resource in resources: + self.dispatch(resource, template) + return template + + def _fref(self, lambda_function, attr='arn'): + # type: (models.ManagedModel, str) -> str + return '${aws_lambda_function.%s.%s}' % ( + lambda_function.resource_name, attr) + + def _arnref(self, arn_template, **kw): + # type: (str, str) -> str + d = dict( + region='${aws_region.chalice.name}', + account_id='${aws_caller_identity.chalice.account_id}') + d.update(kw) + return arn_template % d + + def _generate_managediamrole(self, resource, template): + # type: (models.ManagedIAMRole, Dict[str, Any]) -> None + template['resource'].setdefault('aws_iam_role', {})[ + resource.resource_name] = { + 'name': resource.role_name, + 'assume_role_policy': json.dumps(resource.trust_policy) + } + + template['resource'].setdefault('aws_iam_role_policy', {})[ + resource.resource_name] = { + 'name': resource.resource_name + 'Policy', + 'policy': json.dumps(resource.policy.document), + 'role': '${aws_iam_role.%s.id}' % resource.resource_name, + } + + def _generate_s3bucketnotification(self, resource, template): + # type: (models.S3BucketNotification, Dict[str, Any]) -> None + + bnotify = { + 'events': resource.events, + 'lambda_function_arn': self._fref(resource.lambda_function) + } + + if resource.prefix: + bnotify['filter_prefix'] = resource.prefix + if resource.suffix: + bnotify['filter_suffix'] = resource.suffix + + template['resource'].setdefault('aws_s3_bucket_notification', {})[ + resource.resource_name] = { + 'bucket': resource.bucket, + 'lambda_function': bnotify + } + + template['resource'].setdefault('aws_lambda_permission', {})[ + resource.resource_name] = { + 'statement_id': resource.resource_name, + 'action': 'lambda:InvokeFunction', + 'function_name': self._fref(resource.lambda_function), + 'principal': 's3.amazonaws.com', + 'source_arn': 'arn:aws:s3:::%s' % resource.bucket + } + + def _generate_sqseventsource(self, resource, template): + # type: (models.SQSEventSource, Dict[str, Any]) -> None + template['resource'].setdefault('aws_lambda_event_source_mapping', {})[ + resource.resource_name] = { + 'event_source_arn': self._arnref( + "arn:aws:sqs:%(region)s:%(account_id)s:%(queue)s", + queue=resource.queue), + 'batch_size': resource.batch_size, + 'function_name': self._fref(resource.lambda_function), + } + + def _generate_snslambdasubscription(self, resource, template): + # type: (models.SNSLambdaSubscription, Dict[str, Any]) -> None + + if resource.topic.startswith('arn:aws'): + topic_arn = resource.topic + else: + topic_arn = self._arnref( + 'arn:aws:sns:%(region)s:%(account_id)s:%(topic)s', + topic=resource.topic) + + template['resource'].setdefault('aws_sns_topic_subscription', {})[ + resource.resource_name] = { + 'topic_arn': topic_arn, + 'protocol': 'lambda', + 'endpoint': self._fref(resource.lambda_function) + } + template['resource'].setdefault('aws_lambda_permission', {})[ + resource.resource_name] = { + 'function_name': self._fref( + resource.lambda_function), + 'action': 'lambda:InvokeFunction', + 'principal': 'sns.amazonaws.com', + 'source_arn': topic_arn + } + + def _generate_scheduledevent(self, resource, template): + # type: (models.ScheduledEvent, Dict[str, Any]) -> None + + template['resource'].setdefault( + 'aws_cloudwatch_event_rule', {})[ + resource.resource_name] = { + 'name': resource.resource_name, + 'schedule_expression': resource.schedule_expression + } + template['resource'].setdefault( + 'aws_cloudwatch_event_target', {})[ + resource.resource_name] = { + 'rule': '${aws_cloudwatch_event_rule.%s.name}' % ( + resource.resource_name), + 'target_id': resource.resource_name, + 'arn': self._fref(resource.lambda_function) + } + template['resource'].setdefault( + 'aws_lambda_permission', {})[ + resource.resource_name] = { + 'function_name': self._fref(resource.lambda_function), + 'action': 'lambda:InvokeFunction', + 'principal': 'events.amazonaws.com', + 'source_arn': "${aws_cloudwatch_event_rule.%s.arn}" % ( + resource.resource_name) + } + + def _generate_lambdafunction(self, resource, template): + # type: (models.LambdaFunction, Dict[str, Any]) -> None + + func_definition = { + 'function_name': resource.function_name, + 'runtime': resource.runtime, + 'handler': resource.handler, + 'memory_size': resource.memory_size, + 'tags': resource.tags, + 'timeout': resource.timeout, + 'source_code_hash': '${filebase64sha256("%s")}' % ( + resource.deployment_package.filename), + 'filename': resource.deployment_package.filename} + + if resource.security_group_ids and resource.subnet_ids: + func_definition['vpc_config'] = { + 'subnet_ids': resource.subnet_ids, + 'security_group_ids': resource.security_group_ids + } + if resource.reserved_concurrency is not None: + func_definition['reserved_concurrent_executions'] = ( + resource.reserved_concurrency + ) + if resource.environment_variables: + func_definition['environment'] = { + 'variables': resource.environment_variables + } + + if isinstance(resource.role, models.ManagedIAMRole): + func_definition['role'] = '${aws_iam_role.%s.arn}' % ( + resource.role.resource_name) + else: + # resource is a PreCreatedIAMRole. + role = cast(models.PreCreatedIAMRole, resource.role) + func_definition['role'] = role.role_arn + + template['resource'].setdefault('aws_lambda_function', {})[ + resource.resource_name] = func_definition + + def _generate_restapi(self, resource, template): + # type: (models.RestAPI, Dict[str, Any]) -> None + + # typechecker happiness + swagger_doc = cast(Dict, resource.swagger_doc) + template['resource'].setdefault('aws_api_gateway_rest_api', {})[ + resource.resource_name] = { + 'body': json.dumps(swagger_doc), + # Terraform will diff explicitly configured attributes + # to the current state of the resource. Attributes configured + # via swagger on the REST api need to be duplicated here, else + # terraform will set them back to empty. + 'name': swagger_doc['info']['title'], + 'binary_media_types': swagger_doc[ + 'x-amazon-apigateway-binary-media-types'] + } + + template['resource'].setdefault('aws_api_gateway_stage', {})[ + resource.resource_name] = { + 'rest_api_id': '${aws_api_gateway_rest_api.%s.id}' % ( + resource.resource_name), + 'stage_name': resource.api_gateway_stage, + 'deployment_id': '${aws_api_gateway_deployment.%s.id}' % ( + resource.resource_name) + } + + template['resource'].setdefault('aws_api_gateway_deployment', {})[ + resource.resource_name] = { + 'rest_api_id': '${aws_api_gateway_rest_api.%s.id}' % ( + resource.resource_name), + } + + template['resource'].setdefault('aws_lambda_permission', {})[ + resource.resource_name + '_invoke'] = { + 'function_name': self._fref(resource.lambda_function), + 'action': 'lambda:InvokeFunction', + 'principal': 'apigateway.amazonaws.com', + 'source_arn': + "${aws_api_gateway_rest_api.%s.execution_arn}/*/*/*" % ( + resource.resource_name) + } + + template.setdefault('output', {})[ + 'EndpointURL'] = { + 'value': '${aws_api_gateway_stage.%s.invoke_url}' % ( + resource.resource_name) + } + + for auth in resource.authorizers: + template['resource']['aws_lambda_permission'][ + auth.resource_name + '_invoke'] = { + 'function_name': self._fref(auth), + 'action': 'lambda:InvokeFunction', + 'principal': 'apigateway.amazonaws.com', + 'source_arn': ( + "${aws_api_gateway_rest_api.%s.execution_arn}" % ( + auth.resource_name) + "/*/*/*") + } + + class AppPackager(object): def __init__(self, - sam_templater, # type: SAMTemplateGenerator + templater, # type: TemplateGenerator resource_builder, # type: ResourceBuilder post_processor, # type: TemplatePostProcessor osutils, # type: OSUtils ): # type: (...) -> None - self._sam_templater = sam_templater + self._templater = templater self._resource_builder = resource_builder self._template_post_processor = post_processor self._osutils = osutils @@ -369,16 +636,14 @@ def package_app(self, config, outdir, chalice_stage_name): resources = self._resource_builder.construct_resources( config, chalice_stage_name) - # SAM template - sam_template = self._sam_templater.generate_sam_template( - resources) + template = self._templater.generate(resources) if not self._osutils.directory_exists(outdir): self._osutils.makedirs(outdir) self._template_post_processor.process( - sam_template, config, outdir, chalice_stage_name) + template, config, outdir, chalice_stage_name) self._osutils.set_file_contents( - filename=os.path.join(outdir, 'sam.json'), - contents=self._to_json(sam_template), + filename=os.path.join(outdir, self._templater.template_file), + contents=self._to_json(template), binary=False ) @@ -388,6 +653,13 @@ def __init__(self, osutils): # type: (OSUtils) -> None self._osutils = osutils + def process(self, template, config, outdir, chalice_stage_name): + # type: (Dict[str, Any], Config, str, str) -> None + raise NotImplementedError() + + +class SAMPostProcessor(TemplatePostProcessor): + def process(self, template, config, outdir, chalice_stage_name): # type: (Dict[str, Any], Config, str, str) -> None self._fixup_deployment_package(template, outdir) @@ -410,3 +682,18 @@ def _fixup_deployment_package(self, template, outdir): self._osutils.copy(original_location, new_location) copied = True resource['Properties']['CodeUri'] = './deployment.zip' + + +class TerraformPostProcessor(TemplatePostProcessor): + + def process(self, template, config, outdir, chalice_stage_name): + # type: (Dict[str, Any], Config, str, str) -> None + + copied = False + for r in template['resource'].get('aws_lambda_function', {}).values(): + if not copied: + asset_path = os.path.join(outdir, 'deployment.zip') + self._osutils.copy(r['filename'], asset_path) + copied = True + r['filename'] = "./deployment.zip" + r['source_code_hash'] = '${filebase64sha256("./deployment.zip")}' diff --git a/docs/source/index.rst b/docs/source/index.rst index 5c484f722a..a9d86dd81c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -60,6 +60,7 @@ Topics topics/packaging topics/pyversion topics/cfn + topics/tf topics/authorizers topics/events topics/purelambda diff --git a/docs/source/topics/tf.rst b/docs/source/topics/tf.rst new file mode 100644 index 0000000000..34070c87d5 --- /dev/null +++ b/docs/source/topics/tf.rst @@ -0,0 +1,128 @@ +Terraform Support +================= + +When you run ``chalice deploy``, chalice will deploy your application using the +`AWS SDK for Python `__). Chalice also +provides functionality that allows you to manage deployments yourself using +terraform. This is provided via the ``chalice package --pkg-format terraform`` +command. + +When you run this command, chalice will generate the AWS Lambda deployment +package that contains your application as well as a `Terraform `__ +template. You can then use a terraform to deploy your chalice application. + +Considerations +-------------- + +Using the ``chalice package`` command is useful when you don't want to +use ``chalice deploy`` to manage your deployments. There's several reasons +why you might want to do this: + +* You have pre-existing infrastructure and tooling set up to manage + Terraform deployments. +* You want to integrate with other Terraform resources to manage all + your application resources, including resources outside of your + chalice app. +* You'd like to integrate with `AWS CodePipeline + `__ to automatically deploy + changes when you push to a git repo. + +Keep in mind that you can't switch between ``chalice deploy`` and +``chalice package`` + Terraform for deploying your app. + +If you choose to use ``chalice package`` and Terraform to deploy +your app, you won't be able to switch back to ``chalice deploy``. +Running ``chalice deploy`` would create an entirely new set of AWS +resources (API Gateway Rest API, AWS Lambda function, etc). + +Example +------- + +In this example, we'll create a chalice app and deploy it using +the AWS CLI. + +First install the necessary packages:: + + $ virtualenv /tmp/venv + $ . /tmp/venv/bin/activate + $ pip install chalice awscli + $ chalice new-project test-cfn-deploy + $ cd test-cfn-deploy + +At this point we've installed chalice and the AWS CLI and we have +a basic app created locally. Next we'll run the ``package`` command +and look at its contents:: + + $ $ chalice package --pkg-format terraform /tmp/packaged-app/ + Creating deployment package. + $ ls -la /tmp/packaged-app/ + -rw-r--r-- 1 j wheel 3355270 May 25 14:20 deployment.zip + -rw-r--r-- 1 j wheel 3068 May 25 14:20 sam.json + + $ unzip -l /tmp/packaged-app/deployment.zip | tail -n 5 + 17292 05-25-17 14:19 chalice/app.py + 283 05-25-17 14:19 chalice/__init__.py + 796 05-25-17 14:20 app.py + -------- ------- + 9826899 723 files + + $ head < /tmp/packaged-app/chalice.tf.json + { + "resource": { + "aws_lambda_function": { + "function_name": "radical-dev-sfn_origin", + "runtime": "python3.7", + + +As you can see in the above example, the ``package --pkg-format`` +command created a directory that contained two files, a +``deployment.zip`` file, which is the Lambda deployment package, and a +``chalice.tf.json`` file, which is the Terraform template that can be +deployed using Terraform. Next we're going to use the Terraform CLI +to deploy our app. + +First let's run Terraform init to install the AWS Terraform Provider:: + + $ cd /tmp-packaged-app + $ terraform init + +Now we can deploy our app using the ``terraform apply`` command:: + + $ terraform apply + data.aws_region.chalice: Refreshing state... + data.aws_caller_identity.chalice: Refreshing state... + + An execution plan has been generated and is shown below. + Resource actions are indicated with the following symbols: + + create + + ... (omit plan output) + + Plan: 14 to add, 0 to change, 0 to destroy. + + Do you want to perform these actions? + Terraform will perform the actions described above. + Only 'yes' will be accepted to approve. + Enter a value: yes + + ... (omit apply output) + + Apply complete! Resources: 14 added, 0 changed, 0 destroyed. + + Outputs: + + EndpointURL = https://7bnxriulj5.execute-api.us-east-1.amazonaws.com/dev + +This will take a minute to complete, but once it's done, the endpoint url +will be available as an output which we can then curl:: + + $ http "https://7bnxriulj5.execute-api.us-east-1.amazonaws.com/dev" + HTTP/1.1 200 OK + Connection: keep-alive + Content-Length: 18 + Content-Type: application/json + ... + + { + "hello": "world" + } diff --git a/tests/unit/test_package.py b/tests/unit/test_package.py index 6f18776bd5..53bfa1350a 100644 --- a/tests/unit/test_package.py +++ b/tests/unit/test_package.py @@ -24,9 +24,15 @@ def test_can_create_app_packager(): assert isinstance(packager, package.AppPackager) +def test_can_create_terraform_app_packager(): + config = Config() + packager = package.create_app_packager(config, 'terraform') + assert isinstance(packager, package.AppPackager) + + def test_template_post_processor_moves_files_once(): mock_osutils = mock.Mock(spec=OSUtils) - p = package.TemplatePostProcessor(mock_osutils) + p = package.SAMPostProcessor(mock_osutils) template = { 'Resources': { 'foo': { @@ -56,19 +62,45 @@ def test_template_post_processor_moves_files_once(): ) -class TestSAMTemplate(object): +def test_terraform_post_processor_moves_files_once(): + mock_osutils = mock.Mock(spec=OSUtils) + p = package.TerraformPostProcessor(mock_osutils) + template = { + 'resource': { + 'aws_lambda_function': { + 'foo': {'filename': 'old-dir.zip'}, + 'bar': {'filename': 'old-dir.zip'}, + } + } + } + + p.process(template, config=None, + outdir='outdir', chalice_stage_name='dev') + mock_osutils.copy.assert_called_with( + 'old-dir.zip', os.path.join('outdir', 'deployment.zip')) + assert mock_osutils.copy.call_count == 1 + assert template['resource']['aws_lambda_function'][ + 'foo']['filename'] == ('./deployment.zip') + assert template['resource']['aws_lambda_function'][ + 'bar']['filename'] == ('./deployment.zip') + + +class TemplateTestBase(object): + + template_gen_factory = None + def setup_method(self): self.resource_builder = package.ResourceBuilder( application_builder=ApplicationGraphBuilder(), deps_builder=DependencyBuilder(), build_stage=mock.Mock(spec=BuildStage) ) - self.template_gen = package.SAMTemplateGenerator() + self.template_gen = self.template_gen_factory() def generate_template(self, config, chalice_stage_name): resources = self.resource_builder.construct_resources( config, chalice_stage_name) - return self.template_gen.generate_sam_template(resources) + return self.template_gen.generate(resources) def lambda_function(self): return models.LambdaFunction( @@ -88,6 +120,263 @@ def lambda_function(self): reserved_concurrency=None, ) + +class TestTerraformTemplate(TemplateTestBase): + + template_gen_factory = package.TerraformGenerator + + EmptyPolicy = { + 'Version': '2012-10-18', + 'Statement': { + 'Sid': '', + 'Effect': 'Allow', + 'Action': 'lambda:*' + } + } + + def generate_template(self, config, chalice_stage_name): + resources = self.resource_builder.construct_resources( + config, 'dev') + + # Patch up resources that have mocks (due to build stage) + # that we need to serialize to json. + for r in resources: + # For terraform rest api construction, we need a swagger + # doc on the api resource as we'll be serializing it to + # json. + if isinstance(r, models.RestAPI): + r.swagger_doc = { + 'info': {'title': 'some-app'}, + 'x-amazon-apigateway-binary-media-types': [] + } + # Same for iam policies on roles + elif isinstance(r, models.FileBasedIAMPolicy): + r.document = self.EmptyPolicy + + return self.template_gen.generate(resources) + + def get_function(self, template): + functions = list(template['resource'][ + 'aws_lambda_function'].values()) + assert len(functions) == 1 + return functions[0] + + def test_supports_precreated_role(self): + builder = DependencyBuilder() + resources = builder.build_dependencies( + models.Application( + stage='dev', + resources=[self.lambda_function()], + ) + ) + template = self.template_gen.generate(resources) + assert template['resource'][ + 'aws_lambda_function']['foo']['role'] == 'role:arn' + + def test_adds_env_vars_when_provided(self, sample_app): + function = self.lambda_function() + function.environment_variables = {'foo': 'bar'} + template = self.template_gen.generate([function]) + tf_resource = self.get_function(template) + assert tf_resource['environment'] == { + 'variables': { + 'foo': 'bar' + } + } + + def test_adds_vpc_config_when_provided(self): + function = self.lambda_function() + function.security_group_ids = ['sg1', 'sg2'] + function.subnet_ids = ['sn1', 'sn2'] + template = self.template_gen.generate([function]) + tf_resource = self.get_function(template) + assert tf_resource['vpc_config'] == { + 'subnet_ids': ['sn1', 'sn2'], + 'security_group_ids': ['sg1', 'sg2']} + + def test_adds_reserved_concurrency_when_provided(self, sample_app): + function = self.lambda_function() + function.reserved_concurrency = 5 + template = self.template_gen.generate([function]) + tf_resource = self.get_function(template) + assert tf_resource['reserved_concurrent_executions'] == 5 + + def test_can_generate_scheduled_event(self): + function = self.lambda_function() + event = models.ScheduledEvent( + resource_name='foo-event', + rule_name='myrule', + schedule_expression='rate(5 minutes)', + lambda_function=function, + ) + template = self.template_gen.generate( + [function, event] + ) + rule = template['resource'][ + 'aws_cloudwatch_event_rule'][event.resource_name] + + assert rule == { + 'name': event.resource_name, + 'schedule_expression': 'rate(5 minutes)'} + + def test_can_generate_rest_api(self, sample_app_with_auth): + config = Config.create(chalice_app=sample_app_with_auth, + project_dir='.', + api_gateway_stage='api') + template = self.generate_template(config, 'dev') + resources = template['resource'] + # Lambda function should be created. + assert resources['aws_lambda_function'] + # Along with permission to invoke from API Gateway. + assert list(resources['aws_lambda_permission'].values())[0] == { + 'function_name': '${aws_lambda_function.api_handler.arn}', + 'action': 'lambda:InvokeFunction', + 'principal': 'apigateway.amazonaws.com', + 'source_arn': ( + '${aws_api_gateway_rest_api.rest_api.execution_arn}/*/*/*') + } + assert resources['aws_api_gateway_rest_api'] + # We should also create the auth lambda function. + assert 'myauth' in resources['aws_lambda_function'] + + # Along with permission to invoke from API Gateway. + assert resources['aws_lambda_permission']['myauth_invoke'] == { + 'action': 'lambda:InvokeFunction', + 'function_name': '${aws_lambda_function.myauth.arn}', + 'principal': 'apigateway.amazonaws.com', + 'source_arn': ( + '${aws_api_gateway_rest_api.myauth.execution_arn}/*/*/*') + } + + # Also verify we add the expected outputs when using + # a Rest API. + assert template['output'] == { + 'EndpointURL': { + 'value': '${aws_api_gateway_stage.rest_api.invoke_url}'} + } + + def test_can_package_s3_event_handler_sans_filters(self, sample_app): + @sample_app.on_s3_event(bucket='foo') + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + api_gateway_stage='api') + + template = self.generate_template(config, 'dev') + assert template['resource']['aws_s3_bucket_notification'][ + 'handler-s3event'] == { + 'bucket': 'foo', + 'lambda_function': { + 'events': ['s3:ObjectCreated:*'], + 'lambda_function_arn': ( + '${aws_lambda_function.handler.arn}') + } + } + + def test_can_package_s3_event_handler(self, sample_app): + @sample_app.on_s3_event( + bucket='foo', prefix='incoming', suffix='.csv') + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + api_gateway_stage='api') + + template = self.generate_template(config, 'dev') + assert template['resource']['aws_lambda_permission'][ + 'handler-s3event'] == { + 'action': 'lambda:InvokeFunction', + 'function_name': '${aws_lambda_function.handler.arn}', + 'principal': 's3.amazonaws.com', + 'source_arn': 'arn:aws:s3:::foo', + 'statement_id': 'handler-s3event' + } + + assert template['resource']['aws_s3_bucket_notification'][ + 'handler-s3event'] == { + 'bucket': 'foo', + 'lambda_function': { + 'events': ['s3:ObjectCreated:*'], + 'filter_prefix': 'incoming', + 'filter_suffix': '.csv', + 'lambda_function_arn': ( + '${aws_lambda_function.handler.arn}') + } + } + + def test_can_package_sns_handler(self, sample_app): + @sample_app.on_sns_message(topic='foo') + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + api_gateway_stage='api') + template = self.generate_template(config, 'dev') + + assert template['resource']['aws_sns_topic_subscription'][ + 'handler-sns-subscription'] == { + 'topic_arn': ('arn:aws:sns:${aws_region.chalice.name}:' + '${aws_caller_identity.chalice.account_id}:foo'), + 'protocol': 'lambda', + 'endpoint': '${aws_lambda_function.handler.arn}' + } + + def test_can_package_sns_arn_handler(self, sample_app): + arn = 'arn:aws:sns:space-leo-1:1234567890:foo' + + @sample_app.on_sns_message(topic=arn) + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + api_gateway_stage='api') + template = self.generate_template(config, 'dev') + + assert template['resource']['aws_sns_topic_subscription'][ + 'handler-sns-subscription'] == { + 'topic_arn': arn, + 'protocol': 'lambda', + 'endpoint': '${aws_lambda_function.handler.arn}' + } + + assert template['resource']['aws_lambda_permission'][ + 'handler-sns-subscription'] == { + 'function_name': '${aws_lambda_function.handler.arn}', + 'action': 'lambda:InvokeFunction', + 'principal': 'sns.amazonaws.com', + 'source_arn': 'arn:aws:sns:space-leo-1:1234567890:foo' + } + + def test_can_package_sqs_handler(self, sample_app): + @sample_app.on_sqs_message(queue='foo', batch_size=5) + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + api_gateway_stage='api') + template = self.generate_template(config, 'dev') + + assert template['resource'][ + 'aws_lambda_event_source_mapping'][ + 'handler-sqs-event-source'] == { + 'event_source_arn': ( + 'arn:aws:sqs:${aws_region.chalice.name}:' + '${aws_caller_identity.chalice.account_id}:foo'), + 'function_name': '${aws_lambda_function.handler.arn}', + 'batch_size': 5 + } + + +class TestSAMTemplate(TemplateTestBase): + + template_gen_factory = package.SAMTemplateGenerator + def test_sam_generates_sam_template_basic(self, sample_app): config = Config.create(chalice_app=sample_app, project_dir='.', @@ -115,7 +404,7 @@ def test_supports_precreated_role(self): resources=[self.lambda_function()], ) ) - template = self.template_gen.generate_sam_template(resources) + template = self.template_gen.generate(resources) assert template['Resources']['Foo']['Properties']['Role'] == 'role:arn' def test_sam_injects_policy(self, sample_app): @@ -140,7 +429,7 @@ def test_sam_injects_policy(self, sample_app): layers=[], reserved_concurrency=None, ) - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource == { 'Type': 'AWS::Serverless::Function', @@ -158,7 +447,7 @@ def test_sam_injects_policy(self, sample_app): def test_adds_env_vars_when_provided(self, sample_app): function = self.lambda_function() function.environment_variables = {'foo': 'bar'} - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource['Properties']['Environment'] == { 'Variables': { @@ -170,7 +459,7 @@ def test_adds_vpc_config_when_provided(self): function = self.lambda_function() function.security_group_ids = ['sg1', 'sg2'] function.subnet_ids = ['sn1', 'sn2'] - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource['Properties']['VpcConfig'] == { 'SecurityGroupIds': ['sg1', 'sg2'], @@ -180,7 +469,7 @@ def test_adds_vpc_config_when_provided(self): def test_adds_reserved_concurrency_when_provided(self, sample_app): function = self.lambda_function() function.reserved_concurrency = 5 - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource['Properties']['ReservedConcurrentExecutions'] == 5 @@ -190,7 +479,7 @@ def test_duplicate_resource_name_raises_error(self): one.resource_name = 'foo_bar' two.resource_name = 'foo__bar' with pytest.raises(package.DuplicateResourceNameError): - self.template_gen.generate_sam_template([one, two]) + self.template_gen.generate([one, two]) def test_role_arn_inserted_when_necessary(self): function = models.LambdaFunction( @@ -209,7 +498,7 @@ def test_role_arn_inserted_when_necessary(self): layers=[], reserved_concurrency=None, ) - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource == { 'Type': 'AWS::Serverless::Function', @@ -232,7 +521,7 @@ def test_can_generate_scheduled_event(self): schedule_expression='rate(5 minutes)', lambda_function=function, ) - template = self.template_gen.generate_sam_template( + template = self.template_gen.generate( [function, event] ) resources = template['Resources'] @@ -311,7 +600,7 @@ def test_managed_iam_role(self): trust_policy=LAMBDA_TRUST_POLICY, policy=models.AutoGenIAMPolicy(document={'iam': 'policy'}), ) - template = self.template_gen.generate_sam_template([role]) + template = self.template_gen.generate([role]) resources = template['Resources'] assert len(resources) == 1 cfn_role = resources['DefaultRole']