diff --git a/README.md b/README.md index 74cc028..e719744 100644 --- a/README.md +++ b/README.md @@ -9,15 +9,21 @@ Cloud Development Kit (CDK) and more like Terraform. Helicopyter uses [CDKTF](https://github.com/hashicorp/terraform-cdk) and is inspired by [Configerator](https://research.facebook.com/file/877841159827226/holistic-configuration-management-at-facebook.pdf), [Terraformpy](https://github.com/NerdWalletOSS/terraformpy), and [Terraform JSON configuration syntax](https://developer.hashicorp.com/terraform/language/syntax/json). ## What Helicopyter does (goals) -- Fix the CDKTF naming mess. Meaningful names make review easy. Terraform's resource prefix style results in meaningful - names and aligns with "Namespaces are one honking great idea -- let's do more of those!" The AWS CDK style of - suffixing hashes generates difficult-to-review `terraform plan` output and ignores the existing namespaces. -- Provide a directory structure that groups primarily by "codename" (could be called application, service) and secondarily by tool. For now it assumes f'deploys/{codename}/terraform'. +- Fix the CDK style naming mess. Meaningful names make review easy. Terraform's resource prefix + style results in meaningful names and aligns with "Namespaces are one honking great idea -- let's do + more of those!" The AWS CDK style of suffixing hashes generates difficult-to-review `terraform plan` + output and ignores the existing namespaces. +- Provide a `f'deploys/{cona}/terraform'` directory structure, grouping + * Primarily by "codename" (cona), probably synonymous with application, deployment, and service + * Secondarily by tool, such as `ansible`, `docker`, `terraform`, `python` +- Enable hand-written Hashicorp Configuration Language (HCL) files and auto-generated HCL/JSON to + co-exist, allowing incremental adoption. +- Golang Terraform has a pretty good command line interface. The `ht[aip]` functions in + `includes.sh` try to wrap it very lightly. ## What Helicopyter will probably never do (non-goals) -- Terraform has a pretty good command line interface. Helicopyter focuses on generating JSON for it. Helicopyter does - not try to wrap the `terraform` command line interface itself and using CDKTF's wrapper is untested and not - recommended. +- Support languages other than Python +- Make use of the CDKTF's command line interface. Integration with it is untested and not recommended. ## What Helicopyter might do in the future - Support multiple backend configurations per codename @@ -25,3 +31,4 @@ Helicopyter uses [CDKTF](https://github.com/hashicorp/terraform-cdk) and is insp - `__str__()` for `to_string()`, etc. - Why do we need a Node.js server? Can we build dataclasses or Pydantic models out of the type annotations already being generated? +- Provide helper classes or functions for useful but annoyingly verbose patterns such as local-exec provisioner command diff --git a/deploys/buddies/terraform/main.tf.json b/deploys/buddies/terraform/main.tf.json index 07f82d3..836149b 100644 --- a/deploys/buddies/terraform/main.tf.json +++ b/deploys/buddies/terraform/main.tf.json @@ -18,7 +18,7 @@ "christopher_covington": { "//": { "metadata": { - "path": "buddies/cdktf_cdktf_provider_github.membership/christopher_covington", + "path": "buddies/github_membership/christopher_covington", "uniqueId": "christopher_covington" } }, @@ -28,7 +28,7 @@ "darren_pham": { "//": { "metadata": { - "path": "buddies/cdktf_cdktf_provider_github.membership/darren_pham", + "path": "buddies/github_membership/darren_pham", "uniqueId": "darren_pham" } }, @@ -38,7 +38,7 @@ "duncan_tormey": { "//": { "metadata": { - "path": "buddies/cdktf_cdktf_provider_github.membership/duncan_tormey", + "path": "buddies/github_membership/duncan_tormey", "uniqueId": "duncan_tormey" } }, @@ -48,7 +48,7 @@ "james_braza": { "//": { "metadata": { - "path": "buddies/cdktf_cdktf_provider_github.membership/james_braza", + "path": "buddies/github_membership/james_braza", "uniqueId": "james_braza" } }, @@ -58,7 +58,7 @@ "matt_fowler": { "//": { "metadata": { - "path": "buddies/cdktf_cdktf_provider_github.membership/matt_fowler", + "path": "buddies/github_membership/matt_fowler", "uniqueId": "matt_fowler" } }, diff --git a/deploys/demo/terraform/main.py b/deploys/demo/terraform/main.py index fce72e3..b5e207e 100644 --- a/deploys/demo/terraform/main.py +++ b/deploys/demo/terraform/main.py @@ -1,6 +1,7 @@ """Demonstrate a simple HeliStack synth function using CDKTF constructs.""" -from cdktf import LocalExecProvisioner +from cdktf import LocalExecProvisioner, TerraformLocal, TerraformOutput, TerraformVariable +from cdktf_cdktf_provider_null.resource import Resource as NullResource from helicopyter import HeliStack @@ -11,18 +12,17 @@ def synth(stack: HeliStack) -> None: Also infer the ENVIronment (ENVI) from the workspace and echo it to standard output. """ - NullResource = stack.load('null_resource') # noqa: N806 + stack.push(TerraformLocal, 'cona', stack.cona) + stack.push(TerraformLocal, 'envi', '${terraform.workspace}') - stack.Local('cona', stack.cona) - stack.Local('envi', '${terraform.workspace}') - - NullResource( - 'main', + stack.push( + NullResource, + 'this', provisioners=[ LocalExecProvisioner( command='echo $envi', environment={'envi': '${local.envi}'}, type='local-exec' ) ], ) - gash = stack.Variable('gash', type='string') - stack.Output('gash', value=gash.to_string()) + gash = stack.push(TerraformVariable, 'gash', type='string') + stack.push(TerraformOutput, 'gash', value=gash.to_string()) diff --git a/deploys/demo/terraform/main.tf.json b/deploys/demo/terraform/main.tf.json index f9ce909..40696a3 100644 --- a/deploys/demo/terraform/main.tf.json +++ b/deploys/demo/terraform/main.tf.json @@ -25,11 +25,11 @@ }, "resource": { "null_resource": { - "main": { + "this": { "//": { "metadata": { - "path": "demo/cdktf_cdktf_provider_null.resource/main", - "uniqueId": "main" + "path": "demo/null_resource/this", + "uniqueId": "this" } }, "provisioner": [ diff --git a/deploys/foundation/terraform/main.tf.json b/deploys/foundation/terraform/main.tf.json index d834880..a512491 100644 --- a/deploys/foundation/terraform/main.tf.json +++ b/deploys/foundation/terraform/main.tf.json @@ -32,7 +32,7 @@ "airdjang": { "//": { "metadata": { - "path": "foundation/cdktf_cdktf_provider_github.repository/airdjang", + "path": "foundation/github_repository/airdjang", "uniqueId": "airdjang" } }, @@ -61,7 +61,7 @@ "allowedflare": { "//": { "metadata": { - "path": "foundation/cdktf_cdktf_provider_github.repository/allowedflare", + "path": "foundation/github_repository/allowedflare", "uniqueId": "allowedflare" } }, @@ -89,7 +89,7 @@ "helicopyter": { "//": { "metadata": { - "path": "foundation/cdktf_cdktf_provider_github.repository/helicopyter", + "path": "foundation/github_repository/helicopyter", "uniqueId": "helicopyter" } }, diff --git a/helicopyter.py b/helicopyter.py index ace30e9..3e2b6df 100644 --- a/helicopyter.py +++ b/helicopyter.py @@ -1,21 +1,13 @@ """Generate JSON which Terraform can use from Python.""" -from collections.abc import Callable, Iterable -from functools import partial +from collections.abc import Iterable from importlib import import_module from json import dump from pathlib import Path from subprocess import check_output from typing import Any, TypeVar -from cdktf import ( - App, - TerraformElement, - TerraformLocal, - TerraformOutput, - TerraformStack, - TerraformVariable, -) +from cdktf import App, TerraformElement, TerraformStack from constructs import Construct, Node from tap import Tap @@ -25,11 +17,8 @@ def __init__(self, cona: str) -> None: # Something is automatically creating outdir, which is cdktf.out by default super().__init__(App(outdir='.'), cona) - self.Local = partial(TerraformLocal, Construct(self, 'local')) - self.Output = partial(TerraformOutput, Construct(self, 'output')) - self.Variable = partial(TerraformVariable, Construct(self, 'variable')) self.cona = cona - self.imports: dict[str, str] = {} # {to: id} + self.imports: dict[str, str] = {} # {to_0: id_0, to_1, id_1, ...} self._scopes: dict[str, Construct] = {} def _allocate_logical_id(self, element: Node | TerraformElement) -> str: @@ -38,33 +27,6 @@ def _allocate_logical_id(self, element: Node | TerraformElement) -> str: raise TypeError('AWS CDK unsupported; please use CDKTF') return element.node.id - def scopes(self, module_name: str) -> Construct: - """Get or create a Construct for module_name for scoping purposes.""" - if module_name not in self._scopes: - self._scopes[module_name] = Construct(self, module_name) - return self._scopes[module_name] - - def load(self, label: str) -> Callable[..., type[TerraformElement]]: - """ - Return a Data or Resource class given a substring of the module/package name. - - Example usage: - AccessApplication = stack.load('cloudflare_access_application') - - In contrast to HeliStack.push, this method is more concise but obscures type annotations. - """ - provider, _, snake_case_element = label.partition('_') - module = import_module(f'cdktf_cdktf_provider_{provider}.{snake_case_element}') - if snake_case_element == 'provider': - camel_case_element = f'{provider.title()}Provider' - else: - camel_case_element = ''.join(part.title() for part in snake_case_element.split('_')) - element_class = getattr(module, camel_case_element) - if issubclass(element_class, TerraformElement): - print(f'Loading {module.__name__}') - return partial(element_class, self.scopes(module.__name__)) - raise Exception(f'{camel_case_element} is not a TerraformElement') - def provide(self, name: str) -> type[TerraformElement]: """ Return a Provider class instance given its short name. @@ -91,12 +53,16 @@ def push( Example usage: from cdktf_cdktf_provider_cloudflare.access_application import AccessApplication stack.push(AccessApplication, 'mydomain-wildcard', domain='*.mydomain.com') - - In contrast to HeliStack.load, this method preserves type annotations at the cost of - verbose imports. """ - print(f'Pushing {id_} to {Element.__module__}') - element = Element(self.scopes(Element.__module__), id_, *args, **kwargs) + if Element.__module__ == 'cdktf': + scope_name = Element.__name__.lower().replace('terraform', '') + else: + scope_name = Element.__module__.replace('cdktf_cdktf_provider_', '').replace('.', '_') + if scope_name not in self._scopes: + self._scopes[scope_name] = Construct(self, scope_name) + + print(f'Pushing {scope_name}.{id_}') + element = Element(self._scopes[scope_name], id_, *args, **kwargs) if import_id: self.imports[ f"{Element.__module__.replace('cdktf_cdktf_provider_', '').replace('.', '_')}.{id_}" diff --git a/pyproject.toml b/pyproject.toml index e7690fd..45bfdbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ ignore = [ 'EM102', 'INP001', 'ISC001', + 'PT013', # `import pytest` would be inconsistent with other imports 'Q000', # ruff format will single quote 'Q001', # ruff format will single quote 'Q003', # ruff format will single quote diff --git a/test_helicopyter.py b/test_helicopyter.py index f68867f..02a4529 100644 --- a/test_helicopyter.py +++ b/test_helicopyter.py @@ -1,41 +1,38 @@ """Test the helicopyter module.""" -from unittest import TestCase - -import pytest -from cdktf_cdktf_provider_null.resource import Resource +from cdktf import TerraformLocal, TerraformOutput, TerraformVariable +from cdktf_cdktf_provider_null.resource import Resource as NullResource +from pytest import raises from helicopyter import HeliStack -class TestHeliStack(TestCase): - def test_load(self) -> None: - """Multiple calls should work if and only if the id_ string is unique.""" - stack = HeliStack('foo') - NullResource = stack.load('null_resource') # noqa: N806 - my_first_null = NullResource('bar') - assert isinstance(my_first_null, Resource) - - my_second_null = NullResource('baz') - assert isinstance(my_second_null, Resource) +def test_helistack() -> None: + """The class must instantiate and provide the cona attribute and provide and push methods.""" + stack = HeliStack('foo') + assert stack.cona == 'foo' + assert callable(stack.provide) + assert callable(stack.push) - with pytest.raises(RuntimeError): - NullResource('bar') - with pytest.raises(RuntimeError): - stack.push(Resource, 'bar') +def test_push_id() -> None: + """Within a given Element such as the NullResource, the id_ must be unique.""" + stack = HeliStack('foo') + my_first_null = stack.push(NullResource, 'bar') + assert isinstance(my_first_null, NullResource) - def test_push(self) -> None: - """Multiple calls should work if and only if the id_ string is unique.""" - stack = HeliStack('foo') - my_first_null = stack.push(Resource, 'bar') - assert isinstance(my_first_null, Resource) + my_second_null = stack.push(NullResource, 'baz') + assert isinstance(my_second_null, NullResource) - my_second_null = stack.push(Resource, 'baz') - assert isinstance(my_second_null, Resource) + with raises(RuntimeError): + stack.push(NullResource, 'bar') - with pytest.raises(RuntimeError): - stack.push(Resource, 'bar') - with pytest.raises(RuntimeError): - stack.load('null_resource')('bar') +def test_push_provider() -> None: + """The same id_ must be allowed for different Elements.""" + stack = HeliStack('foo') + stack.push(NullResource, 'bar') + stack.push(TerraformLocal, 'bar', 'bar') + stack.push(TerraformOutput, 'bar', value='bar') + stack.push(TerraformVariable, 'bar') + stack.to_terraform()