From 0a15dce0f622f6e6129f986cc8822c4578cdd7f2 Mon Sep 17 00:00:00 2001 From: Christopher Covington Date: Mon, 23 Sep 2024 06:36:39 -0400 Subject: [PATCH] Generate Hashicorp Configuration Language (HCL) by default (#31) ### Background and Links * Generating HCL is better than generating JSON; debugging and switching to manual writing are easier ### Changes and Testing * Flip the HCL default * Allow formatting with `tofu` * Bump versions ### Questions and Followup * How to migrate to OpenTofu generally? Support both `terraform` and `tofu` on the command line for a time? (Does an environment variable or detection function choose?) Or cut over hard? * How to adopt `asdf`? (Or is there a better alternative?) Can we get to a place where GitHub Actions continuously tests the same developer environment setup steps we expect people to run on their local development machines? --- .github/workflows/check.yaml | 4 + .nvmrc | 2 +- .python-version | 2 +- .../allowedflare/terraform/.opentofu-version | 1 + .../terraform/.terraform.lock.hcl | 5 +- deploys/allowedflare/terraform/main.py | 4 +- deploys/buddies/terraform/.terraform.lock.hcl | 25 ++++ deploys/buddies/terraform/main.tf | 38 ++++++ deploys/buddies/terraform/main.tf.json | 78 ----------- deploys/demo/terraform/main.tf | 26 ++++ deploys/demo/terraform/main.tf.json | 53 ------- deploys/foundation/terraform/main.tf | 85 ++++++++++++ deploys/foundation/terraform/main.tf.json | 129 ------------------ helicopyter.py | 17 ++- includes.sh | 22 ++- requirements.in | 1 + requirements.txt | 8 +- stacks/__init__.py | 0 stacks/base.py | 32 +++++ 19 files changed, 252 insertions(+), 280 deletions(-) create mode 100644 deploys/allowedflare/terraform/.opentofu-version create mode 100644 deploys/buddies/terraform/.terraform.lock.hcl create mode 100644 deploys/buddies/terraform/main.tf delete mode 100644 deploys/buddies/terraform/main.tf.json create mode 100644 deploys/demo/terraform/main.tf delete mode 100644 deploys/demo/terraform/main.tf.json create mode 100644 deploys/foundation/terraform/main.tf delete mode 100644 deploys/foundation/terraform/main.tf.json create mode 100644 stacks/__init__.py create mode 100644 stacks/base.py diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 0616080..2b03962 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -9,6 +9,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + path: asdf + repository: asdf-vm/asdf - run: source includes.sh summarize - uses: actions/setup-node@v4 diff --git a/.nvmrc b/.nvmrc index 8ddbc0c..805b5a4 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v18.16.0 +v20.9.0 diff --git a/.python-version b/.python-version index 171a6a9..871f80a 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.12.1 +3.12.3 diff --git a/deploys/allowedflare/terraform/.opentofu-version b/deploys/allowedflare/terraform/.opentofu-version new file mode 100644 index 0000000..53adb84 --- /dev/null +++ b/deploys/allowedflare/terraform/.opentofu-version @@ -0,0 +1 @@ +1.8.2 diff --git a/deploys/allowedflare/terraform/.terraform.lock.hcl b/deploys/allowedflare/terraform/.terraform.lock.hcl index 7f177d5..cb3eebc 100644 --- a/deploys/allowedflare/terraform/.terraform.lock.hcl +++ b/deploys/allowedflare/terraform/.terraform.lock.hcl @@ -1,12 +1,11 @@ -# This file is maintained automatically by "terraform init". +# This file is maintained automatically by "tofu init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/cloudflare/cloudflare" { +provider "registry.opentofu.org/cloudflare/cloudflare" { version = "4.30.0" constraints = "4.30.0" hashes = [ "h1:FhhTF09/BBk37akGLFx9/uWkGUGwSNRub8vP80TaF7Q=", - "h1:W/q4chfazm9sEz6PVB7K2Uow+RXjLGwZ5F+jTRPfu/k=", "zh:218d1948b59e3d2e3af082724a0d057bcca5a5643c5e7c3b85eefc02430edd6b", "zh:24eb677bc1b205565efb5c0d1c464f63d1e240aac61f5b2ef15165fe842cb7e2", "zh:27896ed2a4f05f6a46ef25e674e445e89bd4bfba8cddbe95940109c6dc3179cc", diff --git a/deploys/allowedflare/terraform/main.py b/deploys/allowedflare/terraform/main.py index ee91fd3..3dbdf0c 100644 --- a/deploys/allowedflare/terraform/main.py +++ b/deploys/allowedflare/terraform/main.py @@ -8,10 +8,10 @@ from cdktf_cdktf_provider_cloudflare.access_policy import AccessPolicy, AccessPolicyInclude from cdktf_cdktf_provider_cloudflare.worker_route import WorkerRoute -from helicopyter import HeliStack +from stacks.base import BaseStack -def synth(stack: HeliStack) -> None: +def synth(stack: BaseStack) -> None: stack.provide('cloudflare') # If this was a private repository, I'd probably set these variables using string literals diff --git a/deploys/buddies/terraform/.terraform.lock.hcl b/deploys/buddies/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..ab75b64 --- /dev/null +++ b/deploys/buddies/terraform/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/integrations/github" { + version = "6.2.2" + constraints = "6.2.2" + hashes = [ + "h1:DnqW/NW0VJAluU5CNXXxNpOdBbAFKjGn2+L/pC2SO9M=", + "zh:43d7e5f1e11d67e38ca717016d209d6d9a6fa03321b489f91984351bfb143b69", + "zh:46e788395034b410bf59dfa43eb748a3d81ecfd23fc442349990fd7d92bd856a", + "zh:5234b7d5c5817ff7ebec29756050708372a071a701e2c8236e714a0bd29ef160", + "zh:74c485a241cc8e8cb99f988d38116fb14e51de896761fc9ca35a34ca5c999a7e", + "zh:7606789521c50937913ea13f851150828b5f9b8804ba80c5b2538c0b019339d8", + "zh:760fb0e74590459689c7159456b6e76f165634f7d0f89f5572d56b57d387f645", + "zh:7979d9085d809bb7d0db2c67e6c3443d1c18d12e51b72220dcb4cc5e883cd64a", + "zh:8bed25d8199bf8b2e7ccf67edc1a4a2fc041bd490b2c11565c669b80be43896c", + "zh:9ff82a6279fb7ae0cd9e44f1e73b64dd2aeca43d4d3096f3f2866b1ebbcb9431", + "zh:a886055ecd63ccb9b880e3c3301c0eca9acb108580d12519617554ae2be9a393", + "zh:c1f20386704919c7964a95daffcb29f494efb061abc28469840df4532833cecf", + "zh:cb6e9c4e33d6a57770073867e174c09c0eed401ee70473a688d20cb1cf0394f7", + "zh:f89ca130cc90b87dc25d036fe8f8cadb6fb53dc33368a032c5cee6275f3bcddc", + "zh:f94a2d1174091f04ed361192cdda9503baa3d161849d4f218c55a96bfb1ea33d", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", + ] +} diff --git a/deploys/buddies/terraform/main.tf b/deploys/buddies/terraform/main.tf new file mode 100644 index 0000000..845e3cb --- /dev/null +++ b/deploys/buddies/terraform/main.tf @@ -0,0 +1,38 @@ +# AUTOGENERATED by helicopyter + +terraform { + required_providers { + github = { + version = "6.2.2" + source = "integrations/github" + } + } +} + +provider "github" { +} + +resource "github_membership" "christopher_covington" { + role = "admin" + username = "covracer" +} + +resource "github_membership" "darren_pham" { + role = "admin" + username = "darpham" +} + +resource "github_membership" "duncan_tormey" { + role = "admin" + username = "DuncanTormey" +} + +resource "github_membership" "james_braza" { + role = "admin" + username = "jamesbraza" +} + +resource "github_membership" "matt_fowler" { + role = "admin" + username = "mattefowler" +} diff --git a/deploys/buddies/terraform/main.tf.json b/deploys/buddies/terraform/main.tf.json deleted file mode 100644 index 836149b..0000000 --- a/deploys/buddies/terraform/main.tf.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "//": { - "AUTOGENERATED": "by helicopyter", - "metadata": { - "backend": "local", - "stackName": "buddies", - "version": "0.20.0" - }, - "outputs": {} - }, - "provider": { - "github": [ - {} - ] - }, - "resource": { - "github_membership": { - "christopher_covington": { - "//": { - "metadata": { - "path": "buddies/github_membership/christopher_covington", - "uniqueId": "christopher_covington" - } - }, - "role": "admin", - "username": "covracer" - }, - "darren_pham": { - "//": { - "metadata": { - "path": "buddies/github_membership/darren_pham", - "uniqueId": "darren_pham" - } - }, - "role": "admin", - "username": "darpham" - }, - "duncan_tormey": { - "//": { - "metadata": { - "path": "buddies/github_membership/duncan_tormey", - "uniqueId": "duncan_tormey" - } - }, - "role": "admin", - "username": "DuncanTormey" - }, - "james_braza": { - "//": { - "metadata": { - "path": "buddies/github_membership/james_braza", - "uniqueId": "james_braza" - } - }, - "role": "admin", - "username": "jamesbraza" - }, - "matt_fowler": { - "//": { - "metadata": { - "path": "buddies/github_membership/matt_fowler", - "uniqueId": "matt_fowler" - } - }, - "role": "admin", - "username": "mattefowler" - } - } - }, - "terraform": { - "required_providers": { - "github": { - "source": "integrations/github", - "version": "6.2.2" - } - } - } -} diff --git a/deploys/demo/terraform/main.tf b/deploys/demo/terraform/main.tf new file mode 100644 index 0000000..b6835f9 --- /dev/null +++ b/deploys/demo/terraform/main.tf @@ -0,0 +1,26 @@ +# AUTOGENERATED by helicopyter + +locals { + cona = "demo" +} + +locals { + envi = terraform.workspace +} + +resource "null_resource" "this" { + provisioner "local-exec" { + command = "echo $envi" + environment = { + envi = "${local.envi}" + } + } +} + +variable "gash" { + type = string +} + +output "gash" { + value = var.gash +} diff --git a/deploys/demo/terraform/main.tf.json b/deploys/demo/terraform/main.tf.json deleted file mode 100644 index 40696a3..0000000 --- a/deploys/demo/terraform/main.tf.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "//": { - "AUTOGENERATED": "by helicopyter", - "metadata": { - "backend": "local", - "stackName": "demo", - "version": "0.20.0" - }, - "outputs": { - "demo": { - "output": { - "gash": "gash" - } - } - } - }, - "locals": { - "cona": "demo", - "envi": "${terraform.workspace}" - }, - "output": { - "gash": { - "value": "${var.gash}" - } - }, - "resource": { - "null_resource": { - "this": { - "//": { - "metadata": { - "path": "demo/null_resource/this", - "uniqueId": "this" - } - }, - "provisioner": [ - { - "local-exec": { - "command": "echo $envi", - "environment": { - "envi": "${local.envi}" - } - } - } - ] - } - } - }, - "variable": { - "gash": { - "type": "string" - } - } -} diff --git a/deploys/foundation/terraform/main.tf b/deploys/foundation/terraform/main.tf new file mode 100644 index 0000000..80de50e --- /dev/null +++ b/deploys/foundation/terraform/main.tf @@ -0,0 +1,85 @@ +# AUTOGENERATED by helicopyter + +terraform { + required_providers { + github = { + version = "6.2.2" + source = "integrations/github" + } + } +} + +provider "github" { +} + +resource "github_repository" "airdjang" { + allow_auto_merge = true + allow_merge_commit = false + allow_rebase_merge = true + allow_squash_merge = true + allow_update_branch = true + delete_branch_on_merge = true + description = "Airflow + Django" + has_downloads = false + has_issues = true + has_projects = false + has_wiki = false + merge_commit_message = "PR_BODY" + merge_commit_title = "PR_TITLE" + name = "airdjang" + squash_merge_commit_message = "PR_BODY" + squash_merge_commit_title = "PR_TITLE" + topics = [ + "airflow", + "django", + "python" + ] +} + +resource "github_repository" "allowedflare" { + allow_auto_merge = true + allow_merge_commit = false + allow_rebase_merge = true + allow_squash_merge = true + allow_update_branch = true + delete_branch_on_merge = true + description = "Intranet connectivity for Django and more" + has_downloads = false + has_issues = true + has_projects = false + has_wiki = false + merge_commit_message = "PR_BODY" + merge_commit_title = "PR_TITLE" + name = "allowedflare" + squash_merge_commit_message = "PR_BODY" + squash_merge_commit_title = "PR_TITLE" + topics = [ + "django", + "python" + ] +} + +resource "github_repository" "helicopyter" { + allow_auto_merge = true + allow_merge_commit = false + allow_rebase_merge = true + allow_squash_merge = true + allow_update_branch = true + delete_branch_on_merge = true + description = "Python-defined infrastructure" + has_downloads = false + has_issues = true + has_projects = false + has_wiki = false + merge_commit_message = "PR_BODY" + merge_commit_title = "PR_TITLE" + name = "helicopyter" + squash_merge_commit_message = "PR_BODY" + squash_merge_commit_title = "PR_TITLE" + topics = [ + "ansible", + "cdktf", + "python", + "terraform" + ] +} diff --git a/deploys/foundation/terraform/main.tf.json b/deploys/foundation/terraform/main.tf.json deleted file mode 100644 index a512491..0000000 --- a/deploys/foundation/terraform/main.tf.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "//": { - "AUTOGENERATED": "by helicopyter", - "metadata": { - "backend": "local", - "stackName": "foundation", - "version": "0.20.0" - }, - "outputs": {} - }, - "import": [ - { - "id": "airdjang", - "to": "github_repository.airdjang" - }, - { - "id": "allowedflare", - "to": "github_repository.allowedflare" - }, - { - "id": "helicopyter", - "to": "github_repository.helicopyter" - } - ], - "provider": { - "github": [ - {} - ] - }, - "resource": { - "github_repository": { - "airdjang": { - "//": { - "metadata": { - "path": "foundation/github_repository/airdjang", - "uniqueId": "airdjang" - } - }, - "allow_auto_merge": true, - "allow_merge_commit": false, - "allow_rebase_merge": true, - "allow_squash_merge": true, - "allow_update_branch": true, - "delete_branch_on_merge": true, - "description": "Airflow + Django", - "has_downloads": false, - "has_issues": true, - "has_projects": false, - "has_wiki": false, - "merge_commit_message": "PR_BODY", - "merge_commit_title": "PR_TITLE", - "name": "airdjang", - "squash_merge_commit_message": "PR_BODY", - "squash_merge_commit_title": "PR_TITLE", - "topics": [ - "airflow", - "django", - "python" - ] - }, - "allowedflare": { - "//": { - "metadata": { - "path": "foundation/github_repository/allowedflare", - "uniqueId": "allowedflare" - } - }, - "allow_auto_merge": true, - "allow_merge_commit": false, - "allow_rebase_merge": true, - "allow_squash_merge": true, - "allow_update_branch": true, - "delete_branch_on_merge": true, - "description": "Intranet connectivity for Django and more", - "has_downloads": false, - "has_issues": true, - "has_projects": false, - "has_wiki": false, - "merge_commit_message": "PR_BODY", - "merge_commit_title": "PR_TITLE", - "name": "allowedflare", - "squash_merge_commit_message": "PR_BODY", - "squash_merge_commit_title": "PR_TITLE", - "topics": [ - "django", - "python" - ] - }, - "helicopyter": { - "//": { - "metadata": { - "path": "foundation/github_repository/helicopyter", - "uniqueId": "helicopyter" - } - }, - "allow_auto_merge": true, - "allow_merge_commit": false, - "allow_rebase_merge": true, - "allow_squash_merge": true, - "allow_update_branch": true, - "delete_branch_on_merge": true, - "description": "Python-defined infrastructure", - "has_downloads": false, - "has_issues": true, - "has_projects": false, - "has_wiki": false, - "merge_commit_message": "PR_BODY", - "merge_commit_title": "PR_TITLE", - "name": "helicopyter", - "squash_merge_commit_message": "PR_BODY", - "squash_merge_commit_title": "PR_TITLE", - "topics": [ - "ansible", - "cdktf", - "python", - "terraform" - ] - } - } - }, - "terraform": { - "required_providers": { - "github": { - "source": "integrations/github", - "version": "6.2.2" - } - } - } -} diff --git a/helicopyter.py b/helicopyter.py index 27c9892..16bbb83 100644 --- a/helicopyter.py +++ b/helicopyter.py @@ -78,7 +78,8 @@ def multisynth( all_or_conas_or_paths: Iterable[str], *, change_directory: Path | None = None, - hashicorp_configuration_language: bool = False, + hashicorp_configuration_language: bool = True, + format_with: str = 'terraform', ) -> None: if not all_or_conas_or_paths: print('No codenames specified. Doing nothing.') @@ -118,10 +119,16 @@ def multisynth( print(f'`def synth(stack: HeliStack):` appears to be missing from {python_file}') raise if hashicorp_configuration_language: - unformatted = stack.to_hcl_terraform()['hcl'].encode() - formatted = check_output(['terraform', 'fmt', '-'], input=unformatted).decode() # noqa: S603 + unformatted = stack.to_hcl_terraform()['hcl'] + autoformatted = check_output( + [format_with, 'fmt', '-'], # noqa: S603 + input=unformatted.encode(), + ).decode() + formatted = autoformatted.replace('}\n\n\n}', '}\n}').replace( + '}\nresource', '}\n\nresource' + ) (top_directory / relative_path).write_text( - '# AUTOGENERATED by helicopyter\n\n' + formatted + '# AUTOGENERATED by helicopyter\n\n' + formatted.strip() + '\n' ) else: dictionary = stack.to_terraform() @@ -139,7 +146,7 @@ def multisynth( class Parameters(Tap): conas: list[str] # space-separated COdeNAmes directory: Path | None = None - hashicorp_configuration_language: bool = False + hashicorp_configuration_language: bool = True def configure(self) -> None: # noqa: D102 self.add_argument('conas') # Positional argument diff --git a/includes.sh b/includes.sh index 54d8181..5716fee 100644 --- a/includes.sh +++ b/includes.sh @@ -31,20 +31,28 @@ case $OS in ;; esac -if [[ -f $HOME/code/asdf/asdf.sh ]]; then - # shellcheck disable=SC1091 - source "$HOME/code/asdf/asdf.sh" \ - && source "$HOME/code/asdf/completions/asdf.bash" -else - echo "Please run +# Accept any kind of installation, such as Homebrew +if ! [[ $(command -v asdf) ]]; then + if [[ -f $HOME/code/asdf/asdf.sh ]]; then + # Automate git installation configuration + # shellcheck disable=SC1091 + source "$HOME/code/asdf/asdf.sh" \ + && source "$HOME/code/asdf/completions/asdf.bash" + else + # Recommend git installation + echo "Please run git clone https://github.com/asdf-vm/asdf.git ~/code/asdf" + fi fi +CLICOLOR_FORCE=1 # For `tree`; might also color `ls` on FreeBSD and Darwin +export CLICOLOR_FORCE + # F: no-op for single page, R: color, X: keep text when exiting, i: case insensitive searching LESS='-FRXi' export LESS -PACKAGES="bind9-host curl fping git less tmux" +PACKAGES="bind9-host curl fping git less tmux tree" export PACKAGES # Aliases only work in interactive shells diff --git a/requirements.in b/requirements.in index 799d8dd..a7c148e 100644 --- a/requirements.in +++ b/requirements.in @@ -12,6 +12,7 @@ yamllint # Build-Time (Not Part of pre-commit) build +cdktf-cdktf-provider-aws cdktf-cdktf-provider-cloudflare cdktf-cdktf-provider-docker cdktf-cdktf-provider-github diff --git a/requirements.txt b/requirements.txt index 8f1311c..91b96f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,10 +11,12 @@ cattrs==23.2.3 # via jsii cdktf==0.20.0 # via + # cdktf-cdktf-provider-aws # cdktf-cdktf-provider-cloudflare # cdktf-cdktf-provider-docker # cdktf-cdktf-provider-github # cdktf-cdktf-provider-null +cdktf-cdktf-provider-aws==19.33.0 cdktf-cdktf-provider-cloudflare==11.8.0 cdktf-cdktf-provider-docker==11.0.0 cdktf-cdktf-provider-github==14.2.2 @@ -31,6 +33,7 @@ codespell==2.3.0 constructs==10.3.0 # via # cdktf + # cdktf-cdktf-provider-aws # cdktf-cdktf-provider-cloudflare # cdktf-cdktf-provider-docker # cdktf-cdktf-provider-github @@ -71,9 +74,10 @@ jeepney==0.8.0 # via # keyring # secretstorage -jsii==1.101.0 +jsii==1.103.1 # via # cdktf + # cdktf-cdktf-provider-aws # cdktf-cdktf-provider-cloudflare # cdktf-cdktf-provider-docker # cdktf-cdktf-provider-github @@ -123,6 +127,7 @@ ptyprocess==0.7.0 publication==0.0.3 # via # cdktf + # cdktf-cdktf-provider-aws # cdktf-cdktf-provider-cloudflare # cdktf-cdktf-provider-docker # cdktf-cdktf-provider-github @@ -183,6 +188,7 @@ typed-argument-parser==1.9.0 typeguard==2.13.3 # via # cdktf + # cdktf-cdktf-provider-aws # cdktf-cdktf-provider-cloudflare # cdktf-cdktf-provider-docker # cdktf-cdktf-provider-github diff --git a/stacks/__init__.py b/stacks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/stacks/base.py b/stacks/base.py new file mode 100644 index 0000000..197d736 --- /dev/null +++ b/stacks/base.py @@ -0,0 +1,32 @@ +"""Base stack with R2 backend.""" + +from cdktf import S3Backend + +from helicopyter import HeliStack + + +class BaseStack(HeliStack): + """ + R2 backend requires the following environment variables. + + AWS_ACCESS_KEY_ID - R2 token + AWS_SECRET_ACCESS_KEY - R2 secret + #=AWS_ENDPOINT_URL_S3 - R2 location: https://ACCOUNT_ID.r2.cloudflarestorage.com + """ + + def __init__(self, cona: str) -> None: + super().__init__(cona) + S3Backend( + self, + bucket='terraform', + # Just in case somebody uses the default environment for two different codenames + key=f'{cona}.tfstate', + region='auto', + skip_credentials_validation=True, + skip_metadata_api_check=True, + skip_region_validation=True, + skip_requesting_account_id=True, + skip_s3_checksum=True, + use_path_style=True, + workspace_key_prefix=cona, + )