diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7da61d841..aaaa81c79 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,8 @@ Next Release (TBD) (`#1160 https://github.com/aws/chalice/pull/1160`__) * Add --merge-template option to package command (`#1195 https://github.com/aws/chalice/pull/1195`__) +* Add support for packaging via terraform + (`#1129 https://github.com/aws/chalice/pull/1129`__) 1.9.1 diff --git a/chalice/cli/__init__.py b/chalice/cli/__init__.py index bd3dca241..a49764c05 100644 --- a/chalice/cli/__init__.py +++ b/chalice/cli/__init__.py @@ -368,6 +368,14 @@ def generate_sdk(ctx, sdk_type, stage, outdir): @cli.command('package') +@click.option('--pkg-format', default='cloudformation', + help=('Specify the provisioning engine to use for ' + 'template output. Chalice supports both ' + 'CloudFormation and Terraform. Default ' + 'is CloudFormation.'), + type=click.Choice(['cloudformation', 'terraform'])) +@click.option('--stage', default=DEFAULT_STAGE_NAME, + help="Chalice Stage to package.") @click.option('--single-file', is_flag=True, default=False, help=("Create a single packaged file. " @@ -375,22 +383,26 @@ def generate_sdk(ctx, sdk_type, stage, outdir): "specifies a directory in which the " "package assets will be placed. If " "this argument is specified, a single " - "zip file will be created instead.")) -@click.option('--stage', default=DEFAULT_STAGE_NAME) + "zip file will be created instead. CloudFormation Only.")) @click.option('--merge-template', help=('Specify a JSON template to be merged ' 'into the generated template. This is useful ' 'for adding resources to a Chalice template or ' - 'modify values in the template.')) + 'modify values in the template. CloudFormation Only.')) @click.argument('out') @click.pass_context -def package(ctx, single_file, stage, merge_template, out): - # type: (click.Context, bool, str, str, str) -> None +def package(ctx, single_file, stage, merge_template, + out, pkg_format): + # type: (click.Context, bool, str, str, str, str) -> None factory = ctx.obj['factory'] # type: CLIFactory - config = factory.create_config_obj( - chalice_stage_name=stage, - ) - packager = factory.create_app_packager(config, merge_template) + config = factory.create_config_obj(stage) + packager = factory.create_app_packager(config, pkg_format, merge_template) + if pkg_format == 'terraform' and (merge_template or single_file): + click.echo(( + "Terraform format does not support " + "merge-template or single-file options")) + raise click.Abort() + if single_file: dirname = tempfile.mkdtemp() try: diff --git a/chalice/cli/factory.py b/chalice/cli/factory.py index 02ab0e0e3..1f2766510 100644 --- a/chalice/cli/factory.py +++ b/chalice/cli/factory.py @@ -179,9 +179,10 @@ def _validate_config_from_disk(self, config): except ValueError: raise UnknownConfigFileVersion(string_version) - def create_app_packager(self, config, merge_template=None): - # type: (Config, OptStr) -> AppPackager - return create_app_packager(config, merge_template=merge_template) + def create_app_packager(self, config, package_format, merge_template=None): + # type: (Config, str, OptStr) -> AppPackager + return create_app_packager( + config, package_format, merge_template=merge_template) def create_log_retriever(self, session, lambda_arn): # type: (Session, str) -> LogRetriever diff --git a/chalice/deploy/swagger.py b/chalice/deploy/swagger.py index c961ecc44..588188fa3 100644 --- a/chalice/deploy/swagger.py +++ b/chalice/deploy/swagger.py @@ -286,3 +286,18 @@ def _auth_uri(self, authorizer): '/functions/{%s}/invocations' % varname, ['region_name', varname], ) + + +class TerraformSwaggerGenerator(SwaggerGenerator): + + def __init__(self): + # type: () -> None + pass + + def _uri(self, lambda_arn=None): + # type: (Optional[str]) -> Any + return '${aws_lambda_function.api_handler.invoke_arn}' + + def _auth_uri(self, authorizer): + # type: (ChaliceAuthorizer) -> Any + return '${aws_lambda_function.%s.invoke_arn}' % (authorizer.name) diff --git a/chalice/package.py b/chalice/package.py index 28db579e9..3ae3918b2 100644 --- a/chalice/package.py +++ b/chalice/package.py @@ -1,11 +1,13 @@ +# pylint: disable=too-many-lines +import json import os import copy -import json from typing import Any, Optional, Dict, List, Set, Union # noqa from typing import cast -from chalice.deploy.swagger import CFNSwaggerGenerator +from chalice.deploy.swagger import ( + CFNSwaggerGenerator, TerraformSwaggerGenerator) from chalice.utils import OSUtils, UI, serialize_to_json, to_cfn_resource_name from chalice.config import Config # noqa from chalice.deploy import models @@ -15,31 +17,41 @@ from chalice.deploy.deployer import create_build_stage -def create_app_packager(config, merge_template=None): - # type: (Config, Optional[str]) -> AppPackager +def create_app_packager( + config, package_format='cloudformation', merge_template=None): + # type: (Config, str, Optional[str]) -> AppPackager osutils = OSUtils() ui = UI() application_builder = ApplicationGraphBuilder() deps_builder = DependencyBuilder() - build_stage = create_build_stage( - osutils, ui, CFNSwaggerGenerator() - ) - resource_builder = ResourceBuilder(application_builder, - deps_builder, build_stage) - processors = [ - ReplaceCodeLocationPostProcessor(osutils=osutils), - TemplateMergePostProcessor( - osutils=osutils, - merger=TemplateDeepMerger(), - merge_template=merge_template, - ), - ] + post_processors = [] # type: List[TemplatePostProcessor] + generator = None # type: Union[None, TemplateGenerator] + + if package_format == 'cloudformation': + build_stage = create_build_stage( + osutils, ui, CFNSwaggerGenerator()) + post_processors.extend([ + SAMCodeLocationPostProcessor(osutils=osutils), + TemplateMergePostProcessor( + osutils=osutils, + merger=TemplateDeepMerger(), + merge_template=merge_template)]) + generator = SAMTemplateGenerator(config) + else: + build_stage = create_build_stage( + osutils, ui, TerraformSwaggerGenerator()) + generator = TerraformGenerator(config) + post_processors.append( + TerraformCodeLocationPostProcessor(osutils=osutils)) + + resource_builder = ResourceBuilder( + application_builder, deps_builder, build_stage) + return AppPackager( - SAMTemplateGenerator(), + generator, resource_builder, - CompositePostProcessor(processors), - osutils, - ) + CompositePostProcessor(post_processors), + osutils) class UnsupportedFeatureError(Exception): @@ -70,7 +82,47 @@ def construct_resources(self, config, chalice_stage_name): return resources -class SAMTemplateGenerator(object): +class TemplateGenerator(object): + + template_file = None # type: str + + def __init__(self, config): + # type: (Config) -> None + self._config = config + + def dispatch(self, resource, template): + # type: (models.Model, Dict[str, Any]) -> None + name = '_generate_%s' % resource.__class__.__name__.lower() + handler = getattr(self, name, self._default) + handler(resource, template) + + def generate(self, resources): + # type: (List[models.Model]) -> Dict[str, Any] + raise NotImplementedError() + + def _generate_filebasediampolicy(self, resource, template): + # type: (models.FileBasedIAMPolicy, Dict[str, Any]) -> None + pass + + def _generate_autogeniampolicy(self, resource, template): + # type: (models.AutoGenIAMPolicy, Dict[str, Any]) -> None + pass + + def _generate_deploymentpackage(self, resource, template): + # type: (models.DeploymentPackage, Dict[str, Any]) -> None + pass + + def _generate_precreatediamrole(self, resource, template): + # type: (models.PreCreatedIAMRole, Dict[str, Any]) -> None + pass + + def _default(self, resource, template): + # type: (models.Model, Dict[str, Any]) -> None + raise UnsupportedFeatureError(resource) + + +class SAMTemplateGenerator(TemplateGenerator): + _BASE_TEMPLATE = { 'AWSTemplateFormatVersion': '2010-09-09', 'Transform': 'AWS::Serverless-2016-10-31', @@ -78,18 +130,19 @@ class SAMTemplateGenerator(object): 'Resources': {}, } - def __init__(self): - # type: () -> None + template_file = "sam.json" + + def __init__(self, config): + # type: (Config) -> None + super(SAMTemplateGenerator, self).__init__(config) self._seen_names = set([]) # type: Set[str] - def generate_sam_template(self, resources): + def generate(self, resources): # type: (List[models.Model]) -> Dict[str, Any] template = copy.deepcopy(self._BASE_TEMPLATE) self._seen_names.clear() for resource in resources: - name = '_generate_%s' % resource.__class__.__name__.lower() - handler = getattr(self, name, self._default) - handler(resource, template) + self.dispatch(resource, template) return template def _generate_scheduledevent(self, resource, template): @@ -295,7 +348,7 @@ def _add_websocket_lambda_invoke_permission( ('arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}' ':${WebsocketAPIId}/*'), {'WebsocketAPIId': api_ref}, - ] + ], }, } } @@ -449,22 +502,6 @@ def _generate_managediamrole(self, resource, template): } } - def _generate_filebasediampolicy(self, resource, template): - # type: (models.FileBasedIAMPolicy, Dict[str, Any]) -> None - pass - - def _generate_autogeniampolicy(self, resource, template): - # type: (models.AutoGenIAMPolicy, Dict[str, Any]) -> None - pass - - def _generate_deploymentpackage(self, resource, template): - # type: (models.DeploymentPackage, Dict[str, Any]) -> None - pass - - def _generate_precreatediamrole(self, resource, template): - # type: (models.PreCreatedIAMRole, Dict[str, Any]) -> None - pass - def _generate_s3bucketnotification(self, resource, template): # type: (models.S3BucketNotification, Dict[str, Any]) -> None message = ( @@ -523,10 +560,6 @@ def _generate_sqseventsource(self, resource, template): } } - def _default(self, resource, template): - # type: (models.Model, Dict[str, Any]) -> None - raise NotImplementedError(resource) - def _register_cfn_resource_name(self, name): # type: (str) -> str cfn_name = to_cfn_resource_name(name) @@ -539,15 +572,303 @@ def _register_cfn_resource_name(self, name): return cfn_name +class TerraformGenerator(TemplateGenerator): + + template_file = "chalice.tf.json" + + def generate(self, resources): + # type: (List[models.Model]) -> Dict[str, Any] + template = { + 'resource': {}, + 'terraform': { + 'required_version': '> 0.11.0, < 0.13.0' + }, + 'provider': { + 'template': {'version': '~> 2'}, + 'aws': {'version': '~> 2'}, + 'null': {'version': '~> 2'}, + }, + 'data': { + 'aws_caller_identity': {'chalice': {}}, + 'aws_region': {'chalice': {}}, + 'null_data_provider': { + 'chalice': { + 'inputs': { + 'app': self._config.app_name, + 'stage': self._config.chalice_stage + } + } + } + } + } + + for resource in resources: + self.dispatch(resource, template) + return template + + def _fref(self, lambda_function, attr='arn'): + # type: (models.ManagedModel, str) -> str + return '${aws_lambda_function.%s.%s}' % ( + lambda_function.resource_name, attr) + + def _arnref(self, arn_template, **kw): + # type: (str, str) -> str + d = dict( + region='${data.aws_region.chalice.name}', + account_id='${data.aws_caller_identity.chalice.account_id}') + d.update(kw) + return arn_template % d + + def _generate_managediamrole(self, resource, template): + # type: (models.ManagedIAMRole, Dict[str, Any]) -> None + template['resource'].setdefault('aws_iam_role', {})[ + resource.resource_name] = { + 'name': resource.role_name, + 'assume_role_policy': json.dumps(resource.trust_policy) + } + + template['resource'].setdefault('aws_iam_role_policy', {})[ + resource.resource_name] = { + 'name': resource.resource_name + 'Policy', + 'policy': json.dumps(resource.policy.document), + 'role': '${aws_iam_role.%s.id}' % resource.resource_name, + } + + def _generate_websocketapi(self, resource, template): + # type: (models.WebsocketAPI, Dict[str, Any]) -> None + + message = ( + "Unable to package chalice apps that use experimental " + "Websocket decorators. Terraform AWS Provider " + "support for websocket is pending see " + "https://git.io/fj9X8 for details and progress. " + "You can deploy this app using `chalice deploy`." + ) + raise NotImplementedError(message) + + def _generate_s3bucketnotification(self, resource, template): + # type: (models.S3BucketNotification, Dict[str, Any]) -> None + + bnotify = { + 'events': resource.events, + 'lambda_function_arn': self._fref(resource.lambda_function) + } + + if resource.prefix: + bnotify['filter_prefix'] = resource.prefix + if resource.suffix: + bnotify['filter_suffix'] = resource.suffix + + # we use the bucket name here because we need to aggregate + # all the notifications subscribers for a bucket. + # Due to cyclic references to buckets created in terraform + # we also try to detect and resolve. + if '{aws_s3_bucket.' in resource.bucket: + bucket_name = resource.bucket.split('.')[1] + else: + bucket_name = resource.bucket + template['resource'].setdefault( + 'aws_s3_bucket_notification', {}).setdefault( + bucket_name + '_notify', + {'bucket': resource.bucket}).setdefault( + 'lambda_function', []).append(bnotify) + + template['resource'].setdefault('aws_lambda_permission', {})[ + resource.resource_name] = { + 'statement_id': resource.resource_name, + 'action': 'lambda:InvokeFunction', + 'function_name': resource.lambda_function.function_name, + 'principal': 's3.amazonaws.com', + 'source_arn': 'arn:aws:s3:::%s' % resource.bucket + } + + def _generate_sqseventsource(self, resource, template): + # type: (models.SQSEventSource, Dict[str, Any]) -> None + template['resource'].setdefault('aws_lambda_event_source_mapping', {})[ + resource.resource_name] = { + 'event_source_arn': self._arnref( + "arn:aws:sqs:%(region)s:%(account_id)s:%(queue)s", + queue=resource.queue), + 'batch_size': resource.batch_size, + 'function_name': resource.lambda_function.function_name, + } + + def _generate_snslambdasubscription(self, resource, template): + # type: (models.SNSLambdaSubscription, Dict[str, Any]) -> None + + if resource.topic.startswith('arn:aws'): + topic_arn = resource.topic + else: + topic_arn = self._arnref( + 'arn:aws:sns:%(region)s:%(account_id)s:%(topic)s', + topic=resource.topic) + + template['resource'].setdefault('aws_sns_topic_subscription', {})[ + resource.resource_name] = { + 'topic_arn': topic_arn, + 'protocol': 'lambda', + 'endpoint': self._fref(resource.lambda_function) + } + template['resource'].setdefault('aws_lambda_permission', {})[ + resource.resource_name] = { + 'function_name': resource.lambda_function.function_name, + 'action': 'lambda:InvokeFunction', + 'principal': 'sns.amazonaws.com', + 'source_arn': topic_arn + } + + def _generate_scheduledevent(self, resource, template): + # type: (models.ScheduledEvent, Dict[str, Any]) -> None + + template['resource'].setdefault( + 'aws_cloudwatch_event_rule', {})[ + resource.resource_name] = { + 'name': resource.resource_name, + 'schedule_expression': resource.schedule_expression + } + template['resource'].setdefault( + 'aws_cloudwatch_event_target', {})[ + resource.resource_name] = { + 'rule': '${aws_cloudwatch_event_rule.%s.name}' % ( + resource.resource_name), + 'target_id': resource.resource_name, + 'arn': self._fref(resource.lambda_function) + } + template['resource'].setdefault( + 'aws_lambda_permission', {})[ + resource.resource_name] = { + 'function_name': resource.lambda_function.function_name, + 'action': 'lambda:InvokeFunction', + 'principal': 'events.amazonaws.com', + 'source_arn': "${aws_cloudwatch_event_rule.%s.arn}" % ( + resource.resource_name) + } + + def _generate_lambdafunction(self, resource, template): + # type: (models.LambdaFunction, Dict[str, Any]) -> None + + func_definition = { + 'function_name': resource.function_name, + 'runtime': resource.runtime, + 'handler': resource.handler, + 'memory_size': resource.memory_size, + 'tags': resource.tags, + 'timeout': resource.timeout, + 'source_code_hash': '${filebase64sha256("%s")}' % ( + resource.deployment_package.filename), + 'filename': resource.deployment_package.filename} + + if resource.security_group_ids and resource.subnet_ids: + func_definition['vpc_config'] = { + 'subnet_ids': resource.subnet_ids, + 'security_group_ids': resource.security_group_ids + } + if resource.reserved_concurrency is not None: + func_definition['reserved_concurrent_executions'] = ( + resource.reserved_concurrency + ) + if resource.environment_variables: + func_definition['environment'] = { + 'variables': resource.environment_variables + } + if resource.layers: + func_definition['layers'] = list(resource.layers) + + if isinstance(resource.role, models.ManagedIAMRole): + func_definition['role'] = '${aws_iam_role.%s.arn}' % ( + resource.role.resource_name) + else: + # resource is a PreCreatedIAMRole. + role = cast(models.PreCreatedIAMRole, resource.role) + func_definition['role'] = role.role_arn + + template['resource'].setdefault('aws_lambda_function', {})[ + resource.resource_name] = func_definition + + def _generate_restapi(self, resource, template): + # type: (models.RestAPI, Dict[str, Any]) -> None + + # typechecker happiness + swagger_doc = cast(Dict, resource.swagger_doc) + template['data'].setdefault( + 'template_file', {}).setdefault( + 'chalice_api_swagger', {})['template'] = json.dumps( + swagger_doc) + + template['resource'].setdefault('aws_api_gateway_rest_api', {})[ + resource.resource_name] = { + 'body': '${data.template_file.chalice_api_swagger.rendered}', + # Terraform will diff explicitly configured attributes + # to the current state of the resource. Attributes configured + # via swagger on the REST api need to be duplicated here, else + # terraform will set them back to empty. + 'name': swagger_doc['info']['title'], + 'binary_media_types': swagger_doc[ + 'x-amazon-apigateway-binary-media-types'], + 'endpoint_configuration': {'types': [resource.endpoint_type]} + } + + if 'x-amazon-apigateway-policy' in swagger_doc: + template['resource'][ + 'aws_api_gateway_rest_api'][ + resource.resource_name]['policy'] = swagger_doc[ + 'x-amazon-apigateway-policy'] + if resource.minimum_compression.isdigit(): + template['resource'][ + 'aws_api_gateway_rest_api'][ + resource.resource_name][ + 'minimum_compression_size'] = int( + resource.minimum_compression) + + template['resource'].setdefault('aws_api_gateway_deployment', {})[ + resource.resource_name] = { + 'stage_name': resource.api_gateway_stage, + # Ensure that the deployment gets redeployed if we update + # the swagger description for the api by using its checksum + # in the stage description. + 'stage_description': ( + "${md5(data.template_file.chalice_api_swagger.rendered)}"), + 'rest_api_id': '${aws_api_gateway_rest_api.%s.id}' % ( + resource.resource_name), + } + + template['resource'].setdefault('aws_lambda_permission', {})[ + resource.resource_name + '_invoke'] = { + 'function_name': resource.lambda_function.function_name, + 'action': 'lambda:InvokeFunction', + 'principal': 'apigateway.amazonaws.com', + 'source_arn': + "${aws_api_gateway_rest_api.%s.execution_arn}/*" % ( + resource.resource_name) + } + + template.setdefault('output', {})[ + 'EndpointURL'] = { + 'value': '${aws_api_gateway_deployment.%s.invoke_url}' % ( + resource.resource_name) + } + + for auth in resource.authorizers: + template['resource']['aws_lambda_permission'][ + auth.resource_name + '_invoke'] = { + 'function_name': auth.function_name, + 'action': 'lambda:InvokeFunction', + 'principal': 'apigateway.amazonaws.com', + 'source_arn': ( + "${aws_api_gateway_rest_api.%s.execution_arn}" % ( + resource.resource_name) + "/*") + } + + class AppPackager(object): def __init__(self, - sam_templater, # type: SAMTemplateGenerator + templater, # type: TemplateGenerator resource_builder, # type: ResourceBuilder post_processor, # type: TemplatePostProcessor osutils, # type: OSUtils ): # type: (...) -> None - self._sam_templater = sam_templater + self._templater = templater self._resource_builder = resource_builder self._template_post_processor = post_processor self._osutils = osutils @@ -562,16 +883,14 @@ def package_app(self, config, outdir, chalice_stage_name): resources = self._resource_builder.construct_resources( config, chalice_stage_name) - # SAM template - sam_template = self._sam_templater.generate_sam_template( - resources) + template = self._templater.generate(resources) if not self._osutils.directory_exists(outdir): self._osutils.makedirs(outdir) self._template_post_processor.process( - sam_template, config, outdir, chalice_stage_name) + template, config, outdir, chalice_stage_name) self._osutils.set_file_contents( - filename=os.path.join(outdir, 'sam.json'), - contents=self._to_json(sam_template), + filename=os.path.join(outdir, self._templater.template_file), + contents=self._to_json(template), binary=False ) @@ -583,10 +902,11 @@ def __init__(self, osutils): def process(self, template, config, outdir, chalice_stage_name): # type: (Dict[str, Any], Config, str, str) -> None - raise NotImplementedError('process') + raise NotImplementedError() + +class SAMCodeLocationPostProcessor(TemplatePostProcessor): -class ReplaceCodeLocationPostProcessor(TemplatePostProcessor): def process(self, template, config, outdir, chalice_stage_name): # type: (Dict[str, Any], Config, str, str) -> None self._fixup_deployment_package(template, outdir) @@ -611,6 +931,21 @@ def _fixup_deployment_package(self, template, outdir): resource['Properties']['CodeUri'] = './deployment.zip' +class TerraformCodeLocationPostProcessor(TemplatePostProcessor): + + def process(self, template, config, outdir, chalice_stage_name): + # type: (Dict[str, Any], Config, str, str) -> None + + copied = False + for r in template['resource'].get('aws_lambda_function', {}).values(): + if not copied: + asset_path = os.path.join(outdir, 'deployment.zip') + self._osutils.copy(r['filename'], asset_path) + copied = True + r['filename'] = "./deployment.zip" + r['source_code_hash'] = '${filebase64sha256("./deployment.zip")}' + + class TemplateMergePostProcessor(TemplatePostProcessor): def __init__(self, osutils, merger, merge_template=None): # type: (OSUtils, TemplateMerger, Optional[str]) -> None diff --git a/docs/source/index.rst b/docs/source/index.rst index 45e5f9a87..47e1aae57 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -60,6 +60,7 @@ Topics topics/packaging topics/pyversion topics/cfn + topics/tf topics/authorizers topics/events topics/purelambda diff --git a/docs/source/topics/tf.rst b/docs/source/topics/tf.rst new file mode 100644 index 000000000..445bf6808 --- /dev/null +++ b/docs/source/topics/tf.rst @@ -0,0 +1,128 @@ +Terraform Support +================= + +When you run ``chalice deploy``, chalice will deploy your application using the +`AWS SDK for Python `__). Chalice also +provides functionality that allows you to manage deployments yourself using +terraform. This is provided via the ``chalice package --pkg-format terraform`` +command. + +When you run this command, chalice will generate the AWS Lambda +deployment package that contains your application and a `Terraform +`__ configuration file. You can then use the +terraform cli to deploy your chalice application. + +Considerations +-------------- + +Using the ``chalice package`` command is useful when you don't want to +use ``chalice deploy`` to manage your deployments. There's several reasons +why you might want to do this: + +* You have pre-existing infrastructure and tooling set up to manage + Terraform deployments. +* You want to integrate with other Terraform resources to manage all + your application resources, including resources outside of your + chalice app. +* You'd like to integrate with `AWS CodePipeline + `__ to automatically deploy + changes when you push to a git repo. + +Keep in mind that you can't switch between ``chalice deploy`` and +``chalice package`` + Terraform for deploying your app. + +If you choose to use ``chalice package`` and Terraform to deploy +your app, you won't be able to switch back to ``chalice deploy``. +Running ``chalice deploy`` would create an entirely new set of AWS +resources (API Gateway Rest API, AWS Lambda function, etc). + +Example +------- + +In this example, we'll create a chalice app and deploy it using +the AWS CLI. + +First install the necessary packages:: + + $ virtualenv /tmp/venv + $ . /tmp/venv/bin/activate + $ pip install chalice awscli + $ chalice new-project test-tf-deploy + $ cd test-tf-deploy + +At this point we've installed chalice and the AWS CLI and we have +a basic app created locally. Next we'll run the ``package`` command:: + + $ chalice package --pkg-format terraform /tmp/packaged-app/ + Creating deployment package. + $ ls -la /tmp/packaged-app/ + -rw-r--r-- 1 j wheel 3355270 May 25 14:20 deployment.zip + -rw-r--r-- 1 j wheel 3068 May 25 14:20 chalice.tf.json + + $ unzip -l /tmp/packaged-app/deployment.zip | tail -n 5 + 17292 05-25-17 14:19 chalice/app.py + 283 05-25-17 14:19 chalice/__init__.py + 796 05-25-17 14:20 app.py + -------- ------- + 9826899 723 files + + +As you can see in the above example, the ``package --pkg-format`` +command created a directory that contained two files, a +``deployment.zip`` file, which is the Lambda deployment package, and a +``chalice.tf.json`` file, which is the Terraform template that can be +deployed using Terraform. Next we're going to use the Terraform CLI +to deploy our app. + +Note terraform will deploy run against all terraform files in this +directory, so we can add additional resources for our application by +adding terraform additional files here. The Chalice terraform template +includes two static data values (`app` and `stage` names) that we can +optionally use when constructing these additional resources, +ie. `${data.null_data_source.chalice.outputs.app}` + +First let's run Terraform init to install the AWS Terraform Provider:: + + $ cd /tmp/packaged-app + $ terraform init + +Now we can deploy our app using the ``terraform apply`` command:: + + $ terraform apply + data.aws_region.chalice: Refreshing state... + data.aws_caller_identity.chalice: Refreshing state... + + An execution plan has been generated and is shown below. + Resource actions are indicated with the following symbols: + + create + + ... (omit plan output) + + Plan: 14 to add, 0 to change, 0 to destroy. + + Do you want to perform these actions? + Terraform will perform the actions described above. + Only 'yes' will be accepted to approve. + Enter a value: yes + + ... (omit apply output) + + Apply complete! Resources: 14 added, 0 changed, 0 destroyed. + + Outputs: + + EndpointURL = https://7bnxriulj5.execute-api.us-east-1.amazonaws.com/dev + +This will take a minute to complete, but once it's done, the endpoint url +will be available as an output which we can then curl:: + + $ http "$(terraform output EndpointURL)" + HTTP/1.1 200 OK + Connection: keep-alive + Content-Length: 18 + Content-Type: application/json + ... + + { + "hello": "world" + } diff --git a/tests/functional/cli/test_cli.py b/tests/functional/cli/test_cli.py index 54e04d27b..41e6ebcf7 100644 --- a/tests/functional/cli/test_cli.py +++ b/tests/functional/cli/test_cli.py @@ -152,6 +152,23 @@ def test_can_package_with_single_file(runner): assert sorted(f.namelist()) == ['deployment.zip', 'sam.json'] +def test_package_terraform_err_with_single_file_or_merge(runner): + with runner.isolated_filesystem(): + cli.create_new_project_skeleton('testproject') + os.chdir('testproject') + result = _run_cli_command( + runner, cli.package, ['--pkg-format', 'terraform', + '--single-file', 'module']) + assert result.exit_code == 1, result.output + assert "Terraform format does not support" in result.output + + result = _run_cli_command( + runner, cli.package, ['--pkg-format', 'terraform', + '--merge-template', 'foo.json', 'module']) + assert result.exit_code == 1, result.output + assert "Terraform format does not support" in result.output + + def test_debug_flag_enables_logging(runner): with runner.isolated_filesystem(): cli.create_new_project_skeleton('testproject') diff --git a/tests/unit/deploy/test_swagger.py b/tests/unit/deploy/test_swagger.py index 99e4e86d4..0c39b8691 100644 --- a/tests/unit/deploy/test_swagger.py +++ b/tests/unit/deploy/test_swagger.py @@ -1,4 +1,5 @@ -from chalice.deploy.swagger import SwaggerGenerator, CFNSwaggerGenerator +from chalice.deploy.swagger import ( + SwaggerGenerator, CFNSwaggerGenerator, TerraformSwaggerGenerator) from chalice import CORSConfig from chalice.app import CustomAuthorizer, CognitoUserPoolAuthorizer from chalice.app import IAMAuthorizer, Chalice @@ -565,8 +566,7 @@ def foo(): } -def test_can_custom_resource_policy_with_cfn(sample_app): - swagger_gen = CFNSwaggerGenerator() +def test_can_custom_resource_policy(sample_app, swagger_gen): rest_api = RestAPI( resource_name='dev', swagger_doc={}, @@ -676,3 +676,32 @@ def foo(): } } } + + +def test_custom_auth_with_tf(sample_app): + swagger_gen = TerraformSwaggerGenerator() + + # No "name=" kwarg provided should default + # to a name of "auth". + @sample_app.authorizer(ttl_seconds=10, execution_role='arn:role') + def auth(auth_request): + pass + + @sample_app.route('/auth', authorizer=auth) + def foo(): + pass + + doc = swagger_gen.generate_swagger(sample_app) + assert 'securityDefinitions' in doc + assert doc['securityDefinitions']['auth'] == { + 'in': 'header', + 'name': 'Authorization', + 'type': 'apiKey', + 'x-amazon-apigateway-authtype': 'custom', + 'x-amazon-apigateway-authorizer': { + 'type': 'token', + 'authorizerCredentials': 'arn:role', + 'authorizerResultTtlInSeconds': 10, + 'authorizerUri': '${aws_lambda_function.auth.invoke_arn}' + } + } diff --git a/tests/unit/test_package.py b/tests/unit/test_package.py index 69f7c9f24..a09734b5e 100644 --- a/tests/unit/test_package.py +++ b/tests/unit/test_package.py @@ -25,9 +25,15 @@ def test_can_create_app_packager(): assert isinstance(packager, package.AppPackager) +def test_can_create_terraform_app_packager(): + config = Config() + packager = package.create_app_packager(config, 'terraform') + assert isinstance(packager, package.AppPackager) + + def test_template_post_processor_moves_files_once(): mock_osutils = mock.Mock(spec=OSUtils) - p = package.ReplaceCodeLocationPostProcessor(mock_osutils) + p = package.SAMCodeLocationPostProcessor(mock_osutils) template = { 'Resources': { 'foo': { @@ -57,6 +63,36 @@ def test_template_post_processor_moves_files_once(): ) +def test_terraform_post_processor_moves_files_once(): + mock_osutils = mock.Mock(spec=OSUtils) + p = package.TerraformCodeLocationPostProcessor(mock_osutils) + template = { + 'resource': { + 'aws_lambda_function': { + 'foo': {'filename': 'old-dir.zip'}, + 'bar': {'filename': 'old-dir.zip'}, + } + } + } + + p.process(template, config=None, + outdir='outdir', chalice_stage_name='dev') + mock_osutils.copy.assert_called_with( + 'old-dir.zip', os.path.join('outdir', 'deployment.zip')) + assert mock_osutils.copy.call_count == 1 + assert template['resource']['aws_lambda_function'][ + 'foo']['filename'] == ('./deployment.zip') + assert template['resource']['aws_lambda_function'][ + 'bar']['filename'] == ('./deployment.zip') + + +def test_template_generator_default(): + tgen = package.TemplateGenerator(Config()) + + with pytest.raises(package.UnsupportedFeatureError): + tgen.dispatch(models.Model(), {}) + + class TestTemplateMergePostProcessor(object): def test_can_call_merge(self): mock_osutils = mock.Mock(spec=OSUtils) @@ -174,19 +210,22 @@ def test_does_call_processors_once(self): template, config, 'out', 'dev') -class TestSAMTemplate(object): +class TemplateTestBase(object): + + template_gen_factory = None + def setup_method(self): self.resource_builder = package.ResourceBuilder( application_builder=ApplicationGraphBuilder(), deps_builder=DependencyBuilder(), build_stage=mock.Mock(spec=BuildStage) ) - self.template_gen = package.SAMTemplateGenerator() + self.template_gen = self.template_gen_factory(Config()) def generate_template(self, config, chalice_stage_name): resources = self.resource_builder.construct_resources( config, chalice_stage_name) - return self.template_gen.generate_sam_template(resources) + return self.template_gen_factory(config).generate(resources) def lambda_function(self): return models.LambdaFunction( @@ -206,6 +245,344 @@ def lambda_function(self): reserved_concurrency=None, ) + +class TestTerraformTemplate(TemplateTestBase): + + template_gen_factory = package.TerraformGenerator + + EmptyPolicy = { + 'Version': '2012-10-18', + 'Statement': { + 'Sid': '', + 'Effect': 'Allow', + 'Action': 'lambda:*' + } + } + + def generate_template(self, config, chalice_stage_name): + resources = self.resource_builder.construct_resources( + config, 'dev') + + # Patch up resources that have mocks (due to build stage) + # that we need to serialize to json. + for r in resources: + # For terraform rest api construction, we need a swagger + # doc on the api resource as we'll be serializing it to + # json. + if isinstance(r, models.RestAPI): + r.swagger_doc = { + 'info': {'title': 'some-app'}, + 'x-amazon-apigateway-binary-media-types': [] + } + if (isinstance(r, models.RestAPI) and + config.api_gateway_endpoint_type == 'PRIVATE'): + r.swagger_doc['x-amazon-apigateway-policy'] = ( + r.policy.document) + + # Same for iam policies on roles + elif isinstance(r, models.FileBasedIAMPolicy): + r.document = self.EmptyPolicy + + return self.template_gen_factory(config).generate(resources) + + def get_function(self, template): + functions = list(template['resource'][ + 'aws_lambda_function'].values()) + assert len(functions) == 1 + return functions[0] + + def test_supports_precreated_role(self): + builder = DependencyBuilder() + resources = builder.build_dependencies( + models.Application( + stage='dev', + resources=[self.lambda_function()], + ) + ) + template = self.template_gen.generate(resources) + assert template['resource'][ + 'aws_lambda_function']['foo']['role'] == 'role:arn' + + def test_adds_env_vars_when_provided(self, sample_app): + function = self.lambda_function() + function.environment_variables = {'foo': 'bar'} + template = self.template_gen.generate([function]) + tf_resource = self.get_function(template) + assert tf_resource['environment'] == { + 'variables': { + 'foo': 'bar' + } + } + + def test_adds_vpc_config_when_provided(self): + function = self.lambda_function() + function.security_group_ids = ['sg1', 'sg2'] + function.subnet_ids = ['sn1', 'sn2'] + template = self.template_gen.generate([function]) + tf_resource = self.get_function(template) + assert tf_resource['vpc_config'] == { + 'subnet_ids': ['sn1', 'sn2'], + 'security_group_ids': ['sg1', 'sg2']} + + def test_adds_layers_when_provided(self): + function = self.lambda_function() + function.layers = layers = ['arn://layer1', 'arn://layer2'] + template = self.template_gen.generate([function]) + tf_resource = self.get_function(template) + assert tf_resource['layers'] == layers + + def test_adds_reserved_concurrency_when_provided(self, sample_app): + function = self.lambda_function() + function.reserved_concurrency = 5 + template = self.template_gen.generate([function]) + tf_resource = self.get_function(template) + assert tf_resource['reserved_concurrent_executions'] == 5 + + def test_can_generate_scheduled_event(self): + function = self.lambda_function() + event = models.ScheduledEvent( + resource_name='foo-event', + rule_name='myrule', + schedule_expression='rate(5 minutes)', + lambda_function=function, + ) + template = self.template_gen.generate( + [function, event] + ) + rule = template['resource'][ + 'aws_cloudwatch_event_rule'][event.resource_name] + + assert rule == { + 'name': event.resource_name, + 'schedule_expression': 'rate(5 minutes)'} + + def test_can_generate_rest_api(self, sample_app_with_auth): + config = Config.create(chalice_app=sample_app_with_auth, + project_dir='.', + minimum_compression_size=8192, + api_gateway_endpoint_type='PRIVATE', + api_gateway_endpoint_vpce='vpce-abc123', + app_name='sample_app', + api_gateway_stage='api') + template = self.generate_template(config, 'dev') + resources = template['resource'] + # Lambda function should be created. + assert resources['aws_lambda_function'] + # Along with permission to invoke from API Gateway. + assert list(resources['aws_lambda_permission'].values())[0] == { + 'function_name': 'sample_app-dev', + 'action': 'lambda:InvokeFunction', + 'principal': 'apigateway.amazonaws.com', + 'source_arn': ( + '${aws_api_gateway_rest_api.rest_api.execution_arn}/*') + } + assert resources['aws_api_gateway_rest_api'] + assert resources['aws_api_gateway_rest_api'][ + 'rest_api']['policy'] + assert resources['aws_api_gateway_rest_api'][ + 'rest_api']['minimum_compression_size'] == 8192 + assert resources['aws_api_gateway_rest_api'][ + 'rest_api']['endpoint_configuration'] == {'types': ['PRIVATE']} + + assert 'aws_api_gateway_stage' not in resources + assert resources['aws_api_gateway_deployment']['rest_api'] == { + 'rest_api_id': '${aws_api_gateway_rest_api.rest_api.id}', + 'stage_description': ( + '${md5(data.template_file.chalice_api_swagger.rendered)}'), + 'stage_name': 'api' + } + + # We should also create the auth lambda function. + assert 'myauth' in resources['aws_lambda_function'] + + # Along with permission to invoke from API Gateway. + assert resources['aws_lambda_permission']['myauth_invoke'] == { + 'action': 'lambda:InvokeFunction', + 'function_name': 'sample_app-dev-myauth', + 'principal': 'apigateway.amazonaws.com', + 'source_arn': ( + '${aws_api_gateway_rest_api.rest_api.execution_arn}/*') + } + + # Also verify we add the expected outputs when using + # a Rest API. + assert template['output'] == { + 'EndpointURL': { + 'value': '${aws_api_gateway_deployment.rest_api.invoke_url}'} + } + + def test_can_package_s3_event_handler_with_tf_ref(self, sample_app): + @sample_app.on_s3_event( + bucket='${aws_s3_bucket.my_data_bucket.id}') + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + api_gateway_stage='api') + + template = self.generate_template(config, 'dev') + assert template['resource']['aws_s3_bucket_notification'][ + 'my_data_bucket_notify'] == { + 'bucket': '${aws_s3_bucket.my_data_bucket.id}', + 'lambda_function': [{ + 'events': ['s3:ObjectCreated:*'], + 'lambda_function_arn': ( + '${aws_lambda_function.handler.arn}') + }] + } + + def test_can_generate_chalice_terraform_static_data(self, sample_app): + config = Config.create(chalice_app=sample_app, + project_dir='.', + app_name='myfoo', + api_gateway_stage='dev') + + template = self.generate_template(config, 'dev') + assert template['data']['null_data_provider']['chalice']['inputs'] == { + 'app': 'myfoo', + 'stage': 'dev' + } + + def test_can_package_s3_event_handler_sans_filters(self, sample_app): + @sample_app.on_s3_event(bucket='foo') + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + api_gateway_stage='api') + + template = self.generate_template(config, 'dev') + assert template['resource']['aws_s3_bucket_notification'][ + 'foo_notify'] == { + 'bucket': 'foo', + 'lambda_function': [{ + 'events': ['s3:ObjectCreated:*'], + 'lambda_function_arn': ( + '${aws_lambda_function.handler.arn}') + }] + } + + def test_can_package_s3_event_handler(self, sample_app): + @sample_app.on_s3_event( + bucket='foo', prefix='incoming', suffix='.csv') + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + app_name='sample_app', + api_gateway_stage='api') + + template = self.generate_template(config, 'dev') + assert template['resource']['aws_lambda_permission'][ + 'handler-s3event'] == { + 'action': 'lambda:InvokeFunction', + 'function_name': 'sample_app-dev-handler', + 'principal': 's3.amazonaws.com', + 'source_arn': 'arn:aws:s3:::foo', + 'statement_id': 'handler-s3event' + } + + assert template['resource']['aws_s3_bucket_notification'][ + 'foo_notify'] == { + 'bucket': 'foo', + 'lambda_function': [{ + 'events': ['s3:ObjectCreated:*'], + 'filter_prefix': 'incoming', + 'filter_suffix': '.csv', + 'lambda_function_arn': ( + '${aws_lambda_function.handler.arn}') + }] + } + + def test_can_package_sns_handler(self, sample_app): + @sample_app.on_sns_message(topic='foo') + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + api_gateway_stage='api') + template = self.generate_template(config, 'dev') + + assert template['resource']['aws_sns_topic_subscription'][ + 'handler-sns-subscription'] == { + 'topic_arn': ( + 'arn:aws:sns:${data.aws_region.chalice.name}:' + '${data.aws_caller_identity.chalice.account_id}:foo'), + 'protocol': 'lambda', + 'endpoint': '${aws_lambda_function.handler.arn}' + } + + def test_can_package_sns_arn_handler(self, sample_app): + arn = 'arn:aws:sns:space-leo-1:1234567890:foo' + + @sample_app.on_sns_message(topic=arn) + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + app_name='sample_app', + api_gateway_stage='api') + template = self.generate_template(config, 'dev') + + assert template['resource']['aws_sns_topic_subscription'][ + 'handler-sns-subscription'] == { + 'topic_arn': arn, + 'protocol': 'lambda', + 'endpoint': '${aws_lambda_function.handler.arn}' + } + + assert template['resource']['aws_lambda_permission'][ + 'handler-sns-subscription'] == { + 'function_name': 'sample_app-dev-handler', + 'action': 'lambda:InvokeFunction', + 'principal': 'sns.amazonaws.com', + 'source_arn': 'arn:aws:sns:space-leo-1:1234567890:foo' + } + + def test_can_package_sqs_handler(self, sample_app): + @sample_app.on_sqs_message(queue='foo', batch_size=5) + def handler(event): + pass + + config = Config.create(chalice_app=sample_app, + project_dir='.', + app_name='sample_app', + api_gateway_stage='api') + template = self.generate_template(config, 'dev') + + assert template['resource'][ + 'aws_lambda_event_source_mapping'][ + 'handler-sqs-event-source'] == { + 'event_source_arn': ( + 'arn:aws:sqs:${data.aws_region.chalice.name}:' + '${data.aws_caller_identity.chalice.account_id}:foo'), + 'function_name': 'sample_app-dev-handler', + 'batch_size': 5 + } + + def test_package_websocket_with_error_message(self, sample_websocket_app): + config = Config.create(chalice_app=sample_websocket_app, + project_dir='.', + app_name='sample_app', + api_gateway_stage='api') + with pytest.raises(NotImplementedError) as excinfo: + self.generate_template(config, 'dev') + + # Should mention the decorator name. + assert 'Websocket decorators' in str(excinfo.value) + # Should mention you can use `chalice deploy`. + assert 'chalice deploy' in str(excinfo.value) + + +class TestSAMTemplate(TemplateTestBase): + + template_gen_factory = package.SAMTemplateGenerator + def test_sam_generates_sam_template_basic(self, sample_app): config = Config.create(chalice_app=sample_app, project_dir='.', @@ -233,7 +610,7 @@ def test_supports_precreated_role(self): resources=[self.lambda_function()], ) ) - template = self.template_gen.generate_sam_template(resources) + template = self.template_gen.generate(resources) assert template['Resources']['Foo']['Properties']['Role'] == 'role:arn' def test_sam_injects_policy(self, sample_app): @@ -258,7 +635,7 @@ def test_sam_injects_policy(self, sample_app): layers=[], reserved_concurrency=None, ) - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource == { 'Type': 'AWS::Serverless::Function', @@ -276,7 +653,7 @@ def test_sam_injects_policy(self, sample_app): def test_adds_env_vars_when_provided(self, sample_app): function = self.lambda_function() function.environment_variables = {'foo': 'bar'} - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource['Properties']['Environment'] == { 'Variables': { @@ -288,7 +665,7 @@ def test_adds_vpc_config_when_provided(self): function = self.lambda_function() function.security_group_ids = ['sg1', 'sg2'] function.subnet_ids = ['sn1', 'sn2'] - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource['Properties']['VpcConfig'] == { 'SecurityGroupIds': ['sg1', 'sg2'], @@ -298,14 +675,14 @@ def test_adds_vpc_config_when_provided(self): def test_adds_reserved_concurrency_when_provided(self, sample_app): function = self.lambda_function() function.reserved_concurrency = 5 - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource['Properties']['ReservedConcurrentExecutions'] == 5 def test_adds_layers_when_provided(self, sample_app): function = self.lambda_function() function.layers = ['arn:aws:layer1', 'arn:aws:layer2'] - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource['Properties']['Layers'] == [ 'arn:aws:layer1', @@ -318,7 +695,7 @@ def test_duplicate_resource_name_raises_error(self): one.resource_name = 'foo_bar' two.resource_name = 'foo__bar' with pytest.raises(package.DuplicateResourceNameError): - self.template_gen.generate_sam_template([one, two]) + self.template_gen.generate([one, two]) def test_role_arn_inserted_when_necessary(self): function = models.LambdaFunction( @@ -337,7 +714,7 @@ def test_role_arn_inserted_when_necessary(self): layers=[], reserved_concurrency=None, ) - template = self.template_gen.generate_sam_template([function]) + template = self.template_gen.generate([function]) cfn_resource = list(template['Resources'].values())[0] assert cfn_resource == { 'Type': 'AWS::Serverless::Function', @@ -360,7 +737,7 @@ def test_can_generate_scheduled_event(self): schedule_expression='rate(5 minutes)', lambda_function=function, ) - template = self.template_gen.generate_sam_template( + template = self.template_gen.generate( [function, event] ) resources = template['Resources'] @@ -616,7 +993,7 @@ def test_managed_iam_role(self): trust_policy=LAMBDA_TRUST_POLICY, policy=models.AutoGenIAMPolicy(document={'iam': 'policy'}), ) - template = self.template_gen.generate_sam_template([role]) + template = self.template_gen.generate([role]) resources = template['Resources'] assert len(resources) == 1 cfn_role = resources['DefaultRole']