diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..4166d1d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,25 @@ +name: Test +on: [push] +jobs: + terraform: + name: Test Terraform + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Terraform lint + run: ./pleasew run sequential --include lint //terraform/... + env: + TF_CLEAN_OUTPUT: "true" + + - name: Terraform plan + run: ./pleasew query alltargets //terraform/... | grep "_plan$" | ./pleasew run sequential - + env: + TF_CLEAN_OUTPUT: "true" + + - name: Terraform apply + run: ./pleasew query alltargets //terraform/... | grep "_apply$" | ./pleasew run sequential - + env: + TF_CLI_ARGS_apply: "-auto-approve" + TF_CLEAN_OUTPUT: "true" diff --git a/.plzconfig b/.plzconfig index 1217e33..dd1cb83 100644 --- a/.plzconfig +++ b/.plzconfig @@ -1,5 +1,5 @@ [please] -version = >=14.6.0 +version = >=15.15.0 [build] path = /usr/local/bin:/usr/bin:/bin @@ -23,6 +23,10 @@ default-docker-repo = repo.please.build ; yarn yarn-workspace = //js/yarn_workspace_example:workspace yarn-offline-mirror = js/yarn_workspace_example/third_party +; terraform +terraform-module-builder-src = //terraform/scripts:module_builder +terraform-runner-src = //terraform/scripts:runner +terraform-workspace-builder-src = //terraform/scripts:workspace_builder [proto] protoctool = //third_party/proto:protoc diff --git a/terraform/BUILD b/terraform/BUILD new file mode 100644 index 0000000..9316cbb --- /dev/null +++ b/terraform/BUILD @@ -0,0 +1,5 @@ +export_file( + name = "terraform", + src = "terraform.build_defs", + visibility = ["PUBLIC"], +) diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..5ce0243 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,86 @@ +Terraform build rules +===================== + +These build defs contain a set of rules for using Terraform configuration with plz. + +This includes support for the following: + * `terraform_provider`: Terraform Providers + * `terraform_module`: Terraform Remote Modules + * `terraform_module`: Terraform Local Modules + * `terraform_toolchain`: Multiple versions of Terraform + * Terraform fmt/validate + + +## `terraform_toolchain` + +This build rule allows you to specify a Terraform version to download and re-use in `terraform_root` rules. You can repeat this for multiple versions if you like, see `//third_party/terraform/BUILD` for examples. + +## `terraform_provider` + +This build rule allows you to specify a [Terraform provider](https://www.terraform.io/docs/providers/index.html) to re-use in your `terraform_root` rules. See `//third_party/terraform/provider/BUILD` for examples. + +## `terraform_module` + +This build rule allows you to specify a [Terraform module](https://www.terraform.io/docs/language/modules/index.html) to re-use in your `terraform_root` rules or as dependencies in other `terraform_module` rules. Terraform modules can be sourced remotely or exist locally on the filesystem. + +See `//third_party/terraform/module/BUILD` for examples of remote Terraform modules. +See `//terraform/examples//my_module/BUILD` for examples of local terraform modules. + +In your Terraform source code, you should refer to your modules by their canonical build label. e.g.: + +``` +module "remote_module" { + source = "//third_party/terraform/module:a_module" +} + +module "my_module" { + source = "//terraform/examples/0.12/my_module:my_module" +} +``` + +## `terraform_root` + +This build rule allows to specify a [Terraform root module](https://www.terraform.io/docs/language/modules/index.html#the-root-module) which is the root configuration where Terraform will be executed. In this build rule, you reference the `srcs` for the root module as well as optionally (but recommended) the providers and modules those `srcs` use. This is optional as we cannot disable the pulling of providers and modules in Terraform 0.13+, so we only pre-populate the Terraform cache. However, it is advisable to use these parameters to reduce network load so that providers and modules are only downloaded once. + +We support substitution of the following please build environment variables into your source terraform files: + - `PKG` + - `PKG_DIR` + - `NAME` + - `ARCH` + - `OS` +This allows you to template Terraform code to keep your code DRY. for example: A terraform remote state configuration can that can be re-used in all `terraform_root`s: +``` +terraform { + backend "s3" { + region = "eu-west-1" + bucket = "my-terraform-state" + key = "$PKG/$NAME.tfstate" + dynamodb_table = "my-terraform-state-lock" + encrypt = true + } +} +``` +The above will result in a terraform state tree consistent with the structure of your repository. + +This build rule generates the following subrules which perform the Terraform workflows: + * `_plan` + * `_apply` + * `_destroy` + * `_bin` for all other workflows e.g. `plz run //my_infrastructure_tf_bin -- init && plz run //my_infrastructure_tf_bin -- console` + +For all of these workflows, we support passing in flags via please as expected, e.g.: +``` +$ plz run //my_tf:my_tf_plan -- -lock=false +$ plz run //my_tf:my_tf_import -- resource_type.my_resource resource_id +``` + +We also add an environment variable `TF_CLEAN_OUTPUT` which strips noisy Terraform output on a best effort basis. This is incompatible with interactive commands, so we only advise setting this in automation. + + +It additionally adds linters under the `lint` label for: +* `terraform fmt -check` +* `terraform validate` + +See `//terraform/examples//BUILD` for examples of `terraform_root`. + +**NOTE**: This build rule utilises a [Terraform working directory](https://www.terraform.io/docs/cli/init/index.html) in `plz-out`, so whilst this is okay for demonstrations, you must use [Terraform Remote State](https://www.terraform.io/docs/language/state/remote.html) for your regular work. This can be added either simply through your `srcs` or through a `pre_binaries` binary. diff --git a/terraform/examples/0.11/BUILD b/terraform/examples/0.11/BUILD new file mode 100644 index 0000000..d358cec --- /dev/null +++ b/terraform/examples/0.11/BUILD @@ -0,0 +1,14 @@ +subinclude("//terraform") + +terraform_root( + name = "example", + srcs = ["main.tf", "data.tf"], + toolchain = "//third_party/terraform:terraform_0_11", + providers = [ + "//third_party/terraform/provider:null", + ], + modules = [ + "//third_party/terraform/module:cloudposse_null_label_0_11", + "//terraform/examples/0.11/my_module:my_module", + ] +) diff --git a/terraform/examples/0.11/data.tf b/terraform/examples/0.11/data.tf new file mode 100644 index 0000000..3c81ad7 --- /dev/null +++ b/terraform/examples/0.11/data.tf @@ -0,0 +1 @@ +resource "null_resource" "empty" {} diff --git a/terraform/examples/0.11/main.tf b/terraform/examples/0.11/main.tf new file mode 100644 index 0000000..e592ab7 --- /dev/null +++ b/terraform/examples/0.11/main.tf @@ -0,0 +1,27 @@ +provider "null" { + version = "~> 2.1" +} + +resource "null_resource" "version" { + provisioner "local-exec" { + command = "terraform version" + } +} + +module "label" { + source = "//third_party/terraform/module:cloudposse_null_label_0_11" + namespace = "eg" + stage = "prod" + name = "bastion" + attributes = ["public"] + delimiter = "-" + + tags = { + "BusinessUnit" = "XYZ" + "Snapshot" = "true" + } +} + +module "my_label" { + source = "//terraform/examples/0.11/my_module:my_module" +} diff --git a/terraform/examples/0.11/my_module/BUILD b/terraform/examples/0.11/my_module/BUILD new file mode 100644 index 0000000..f3448f1 --- /dev/null +++ b/terraform/examples/0.11/my_module/BUILD @@ -0,0 +1,10 @@ +subinclude("//terraform") + +terraform_module( + name = "my_module", + srcs = ["main.tf", "data.tf"], + deps = [ + "//third_party/terraform/module:cloudposse_null_label_0_11", + ], + visibility = ["//terraform/examples/0.11/..."], +) diff --git a/terraform/examples/0.11/my_module/data.tf b/terraform/examples/0.11/my_module/data.tf new file mode 100644 index 0000000..3c81ad7 --- /dev/null +++ b/terraform/examples/0.11/my_module/data.tf @@ -0,0 +1 @@ +resource "null_resource" "empty" {} diff --git a/terraform/examples/0.11/my_module/main.tf b/terraform/examples/0.11/my_module/main.tf new file mode 100644 index 0000000..03d25a8 --- /dev/null +++ b/terraform/examples/0.11/my_module/main.tf @@ -0,0 +1,19 @@ +module "label" { + source = "//third_party/terraform/module:cloudposse_null_label_0_11" + namespace = "eg" + stage = "prod" + name = "bastion" + attributes = ["public"] + delimiter = "-" + + tags = { + "BusinessUnit" = "XYZ" + "Snapshot" = "true" + } +} + +resource "null_resource" "version" { + provisioner "local-exec" { + command = "terraform version" + } +} diff --git a/terraform/examples/0.12/BUILD b/terraform/examples/0.12/BUILD new file mode 100644 index 0000000..72aa3c9 --- /dev/null +++ b/terraform/examples/0.12/BUILD @@ -0,0 +1,14 @@ +subinclude("//terraform") + +terraform_root( + name = "example", + srcs = ["main.tf"], + toolchain = "//third_party/terraform:terraform_0_12", + providers = [ + "//third_party/terraform/provider:null", + ], + modules = [ + "//third_party/terraform/module:cloudposse_null_label_0_12", + "//terraform/examples/0.12/my_module:my_module", + ] +) diff --git a/terraform/examples/0.12/main.tf b/terraform/examples/0.12/main.tf new file mode 100644 index 0000000..d4ea8cc --- /dev/null +++ b/terraform/examples/0.12/main.tf @@ -0,0 +1,27 @@ +provider "null" { + version = "~> 2.1" +} + +resource "null_resource" "version" { + provisioner "local-exec" { + command = "terraform version" + } +} + +module "label" { + source = "//third_party/terraform/module:cloudposse_null_label_0_12" + namespace = "eg" + stage = "prod" + name = "bastion" + attributes = ["public"] + delimiter = "-" + + tags = { + "BusinessUnit" = "XYZ", + "Snapshot" = "true" + } +} + +module "my_label" { + source = "//terraform/examples/0.12/my_module:my_module" +} diff --git a/terraform/examples/0.12/my_module/BUILD b/terraform/examples/0.12/my_module/BUILD new file mode 100644 index 0000000..690819c --- /dev/null +++ b/terraform/examples/0.12/my_module/BUILD @@ -0,0 +1,10 @@ +subinclude("//terraform") + +terraform_module( + name = "my_module", + srcs = ["main.tf"], + deps = [ + "//third_party/terraform/module:cloudposse_null_label_0_12", + ], + visibility = ["//terraform/examples/0.12/..."], +) diff --git a/terraform/examples/0.12/my_module/main.tf b/terraform/examples/0.12/my_module/main.tf new file mode 100644 index 0000000..3f0e4c0 --- /dev/null +++ b/terraform/examples/0.12/my_module/main.tf @@ -0,0 +1,19 @@ +module "label" { + source = "//third_party/terraform/module:cloudposse_null_label_0_12" + namespace = "eg" + stage = "prod" + name = "bastion" + attributes = ["public"] + delimiter = "-" + + tags = { + "BusinessUnit" = "XYZ", + "Snapshot" = "true" + } +} + +resource "null_resource" "version" { + provisioner "local-exec" { + command = "terraform version" + } +} diff --git a/terraform/examples/0.13/BUILD b/terraform/examples/0.13/BUILD new file mode 100644 index 0000000..c3522a2 --- /dev/null +++ b/terraform/examples/0.13/BUILD @@ -0,0 +1,14 @@ +subinclude("//terraform") + +terraform_root( + name = "example", + srcs = ["main.tf"], + toolchain = "//third_party/terraform:terraform_0_13", + providers = [ + "//third_party/terraform/provider:null", + ], + modules = [ + "//third_party/terraform/module:cloudposse_null_label_0_12", + "//terraform/examples/0.13/my_module:my_module", + ] +) diff --git a/terraform/examples/0.13/main.tf b/terraform/examples/0.13/main.tf new file mode 100644 index 0000000..60efc69 --- /dev/null +++ b/terraform/examples/0.13/main.tf @@ -0,0 +1,27 @@ +provider "null" { + version = "~> 2.1" +} + +resource "null_resource" "version" { + provisioner "local-exec" { + command = "terraform version" + } +} + +module "label" { + source = "//third_party/terraform/module:cloudposse_null_label_0_12" + namespace = "eg" + stage = "prod" + name = "bastion" + attributes = ["public"] + delimiter = "-" + + tags = { + "BusinessUnit" = "XYZ", + "Snapshot" = "true" + } +} + +module "my_label" { + source = "//terraform/examples/0.13/my_module:my_module" +} diff --git a/terraform/examples/0.13/my_module/BUILD b/terraform/examples/0.13/my_module/BUILD new file mode 100644 index 0000000..ff6d3fd --- /dev/null +++ b/terraform/examples/0.13/my_module/BUILD @@ -0,0 +1,10 @@ +subinclude("//terraform") + +terraform_module( + name = "my_module", + srcs = ["main.tf"], + deps = [ + "//third_party/terraform/module:cloudposse_null_label_0_12", + ], + visibility = ["//terraform/examples/0.13/..."], +) diff --git a/terraform/examples/0.13/my_module/main.tf b/terraform/examples/0.13/my_module/main.tf new file mode 100644 index 0000000..3f0e4c0 --- /dev/null +++ b/terraform/examples/0.13/my_module/main.tf @@ -0,0 +1,19 @@ +module "label" { + source = "//third_party/terraform/module:cloudposse_null_label_0_12" + namespace = "eg" + stage = "prod" + name = "bastion" + attributes = ["public"] + delimiter = "-" + + tags = { + "BusinessUnit" = "XYZ", + "Snapshot" = "true" + } +} + +resource "null_resource" "version" { + provisioner "local-exec" { + command = "terraform version" + } +} diff --git a/terraform/examples/0.14/BUILD b/terraform/examples/0.14/BUILD new file mode 100644 index 0000000..9c6ad4e --- /dev/null +++ b/terraform/examples/0.14/BUILD @@ -0,0 +1,14 @@ +subinclude("//terraform") + +terraform_root( + name = "example", + srcs = ["main.tf"], + toolchain = "//third_party/terraform:terraform_0_14", + providers = [ + "//third_party/terraform/provider:null", + ], + modules = [ + "//third_party/terraform/module:cloudposse_null_label_0_12", + "//terraform/examples/0.14/my_module:my_module", + ] +) diff --git a/terraform/examples/0.14/main.tf b/terraform/examples/0.14/main.tf new file mode 100644 index 0000000..4fdc34c --- /dev/null +++ b/terraform/examples/0.14/main.tf @@ -0,0 +1,36 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/null" + version = "~> 2.1" + } + } +} + +provider "null" { + +} + +resource "null_resource" "version" { + provisioner "local-exec" { + command = "terraform version" + } +} + +module "label" { + source = "//third_party/terraform/module:cloudposse_null_label_0_12" + namespace = "eg" + stage = "prod" + name = "bastion" + attributes = ["public"] + delimiter = "-" + + tags = { + "BusinessUnit" = "XYZ", + "Snapshot" = "true" + } +} + +module "my_label" { + source = "//terraform/examples/0.14/my_module:my_module" +} diff --git a/terraform/examples/0.14/my_module/BUILD b/terraform/examples/0.14/my_module/BUILD new file mode 100644 index 0000000..553b8ef --- /dev/null +++ b/terraform/examples/0.14/my_module/BUILD @@ -0,0 +1,10 @@ +subinclude("//terraform") + +terraform_module( + name = "my_module", + srcs = ["main.tf"], + deps = [ + "//third_party/terraform/module:cloudposse_null_label_0_12", + ], + visibility = ["//terraform/examples/0.14/..."], +) diff --git a/terraform/examples/0.14/my_module/main.tf b/terraform/examples/0.14/my_module/main.tf new file mode 100644 index 0000000..3f0e4c0 --- /dev/null +++ b/terraform/examples/0.14/my_module/main.tf @@ -0,0 +1,19 @@ +module "label" { + source = "//third_party/terraform/module:cloudposse_null_label_0_12" + namespace = "eg" + stage = "prod" + name = "bastion" + attributes = ["public"] + delimiter = "-" + + tags = { + "BusinessUnit" = "XYZ", + "Snapshot" = "true" + } +} + +resource "null_resource" "version" { + provisioner "local-exec" { + command = "terraform version" + } +} diff --git a/terraform/scripts/BUILD b/terraform/scripts/BUILD new file mode 100644 index 0000000..7ae8112 --- /dev/null +++ b/terraform/scripts/BUILD @@ -0,0 +1,6 @@ +for f in glob(["*.sh"]): + export_file( + name=splitext(f)[0], + src = f, + visibility = ["PUBLIC"], + ) diff --git a/terraform/scripts/module_builder.sh b/terraform/scripts/module_builder.sh new file mode 100644 index 0000000..d0a1e19 --- /dev/null +++ b/terraform/scripts/module_builder.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# This script prepares a Terraform module for use with Please by: +# * Replacing sub-modules (deps) with local references. +# * Ensuring all sub-modules have local references. +set -euo pipefail + +# dependencies prepares a module dependencies for a module +# A Terraform module can have dependencies and can be depended on. +# To accomodate this, we add +function dependencies { + mkdir "${OUTS}/modules/" + for m in $SRCS_DEPS; do + replace=$(basename "$m") + mapfile -t searches <"${m}/.module_source_searches" + for search in "${searches[@]}"; do + find . -name "*.tf" -exec sed -i "s#[^\"]*${search}[^\"]*#./modules/${replace}#g" {} + + done + cp -r "$m" "${OUTS}/modules/" + done +} + +# dependants prepares the module for having dependants +function dependants { + # add a replace-me search for an interesting part of the URL + echo "${URL}" | cut -f3-5 -d/ > "${OUTS}/.module_source_searches" + # add a replace-me search for the canonical Please build rule + echo "${PKG}:${NAME}" >> "${OUTS}/.module_source_searches" +} + +# strip removes the given files/directories from the module +function strip { + for s in "${STRIP[@]}"; do + rm -rf "${OUTS:?}/${s}" + done +} + +# validate_module_sources validates that the module has no remaining modules that are not declared in deps +function validate_module_sources { + if grep -r --include \*.tf -A3 "module \"" "${OUTS}" | grep -E "source\s*=\s*\"[^\/\.]+.*"; then + echo "found module source not declared in deps" + exit 1 + fi +} + +mv "${OG_MODULE_DIR}" "${OUTS}" + +if [[ -v SRCS_DEPS ]]; then + dependencies +fi + +strip + +dependants + +validate_module_sources diff --git a/terraform/scripts/runner.sh b/terraform/scripts/runner.sh new file mode 100644 index 0000000..be7fed6 --- /dev/null +++ b/terraform/scripts/runner.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# This script runs Terraform in the target's working directory with the following features: +# - Plugin cache directory pointing to our prepared plugins directory. +# - Strips out various noisy output (https://github.com/hashicorp/terraform/issues/20960) +set -euo pipefail + +ABS="${PWD}" + +TERRAFORM_BIN="${ABS}/${TERRAFORM_BIN}" +PATH="$(dirname "${TERRAFORM_BIN}"):$PATH" +export PATH +export TF_PLUGIN_CACHE_DIR="${ABS}/${TERRAFORM_WORKSPACE}/_plugins" + +TF_CLEAN_OUTPUT="${TF_CLEAN_OUTPUT:-false}" + +# tf_clean_output strips the Terraform output down. +# This is useful in CI/CD where Terraform logs are usually noisy by default. +function tf_clean_output { + local cmds extra_args is_last + IFS=" " read -r -a cmds <<< "$1" + shift + is_last="$1" + shift + extra_args=("${@}") + + args=("${cmds[@]}") + if [ "${is_last}" == "true" ]; then + args=("${args[@]}" "${extra_args[@]}") + fi + echo "..> terraform ${args[*]}" + if [ "${TF_CLEAN_OUTPUT}" == "false" ]; then + "${TERRAFORM_BIN}" "${args[@]}" + else + "${TERRAFORM_BIN}" "${args[@]}" \ + | sed '/successfully initialized/,$d' \ + | sed "/You didn't specify an \"-out\"/,\$d" \ + | sed '/.terraform.lock.hcl/,$d' \ + | sed '/Refreshing state/d' \ + | sed '/The refreshed state will be used to calculate this plan/d' \ + | sed '/persisted to local or remote state storage/d' \ + | sed '/^[[:space:]]*$/d' + fi +} + +cd "${TERRAFORM_WORKSPACE}" + +for bin in "${PRE_BINARIES[@]}"; do + "${ABS}/${bin}" +done + +for i in "${!TERRAFORM_CMDS[@]}"; do + cmd="${TERRAFORM_CMDS[i]}" + if [ $((i+1)) == "${#TERRAFORM_CMDS[@]}" ]; then + tf_clean_output "${cmd}" "true" "$@" + else + tf_clean_output "${cmd}" "false" "$@" + fi + + echo "" +done + +# if there's no TERRAFORM_CMDS given, we assume that we just want to run Terraform directly with the given args. +if [ "${#TERRAFORM_CMDS[@]}" == "0" ]; then + "${TERRAFORM_BIN}" "${@}" +fi + +for bin in "${POST_BINARIES[@]}"; do + "${ABS}/${bin}" +done diff --git a/terraform/scripts/workspace_builder.sh b/terraform/scripts/workspace_builder.sh new file mode 100644 index 0000000..872209d --- /dev/null +++ b/terraform/scripts/workspace_builder.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# This script prepares a Terraform Workspace with: +# * Terraform Plugins +# * Terraform Modules +set -euo pipefail + +TERRAFORM_MINOR_VERSION="$(head -n1 < <($TERRAFORM_BIN version) | awk '{ print $2 }' | cut -f1-2 -d\.)" + +PLUGIN_DIR="${OUTS}/_plugins" +MODULE_DIR="${OUTS}/_modules" + +mkdir -p "${OUTS}" + +# plugins_v0.11+ configures plugins for Terraform 0.11+ +# Terraform v0.11+ store plugins in the following structure: +# `./${os}_{arch}/${binary}` +# e.g. ``./linux_amd64/terraform-provider-null_v2.1.2_x4` +function plugins_v0.11+ { + local plugin_dir + local plugin_bin + plugin_dir="${PLUGIN_DIR}/${CONFIG_OS}_${CONFIG_ARCH}" + mkdir -p "${plugin_dir}" + for plugin in $SRCS_PLUGINS; do + plugin_bin="$(find "$plugin" -not -path '*/\.*' -type f | head -n1)" + cp "$plugin_bin" "${plugin_dir}/" + done +} + +# plugins_v0.13+ configures plugins for Terraform 0.13+ +# Terraform v0.13+ store plugins in the following structure: +# `./${registry}/${namespace}/${type}/${version}/${os}_{arch}/${binary}` +# e.g. `./registry.terraform.io/hashicorp/null/2.1.2/linux_amd64/terraform-provider-null_v2.1.2_x4` +function plugins_v0.13+ { + local registry namespace provider_name version plugin_dir plugin_bin + for plugin in $SRCS_PLUGINS; do + registry=$(<"${plugin}/.registry") + namespace=$(<"${plugin}/.namespace") + provider_name=$(<"${plugin}/.provider_name") + version=$(<"${plugin}/.version") + plugin_dir="${PLUGIN_DIR}/${registry}/${namespace}/${provider_name}/${version}/${CONFIG_OS}_${CONFIG_ARCH}" + plugin_bin="$(find "$plugin" -not -path '*/\.*' -type f | head -n1)" + mkdir -p "${plugin_dir}" + cp "$plugin_bin" "${plugin_dir}/" + done +} + +# copy plugins (providers) +if [[ -v SRCS_PLUGINS ]]; then + case "${TERRAFORM_MINOR_VERSION}" in + "v0.11") plugins_v0.11+ ;; + "v0.12") plugins_v0.11+ ;; + "v0.13") plugins_v0.13+ ;; + *) plugins_v0.13+ ;; + esac +fi + +# modules configures modules for Terraform +# Terraform modules via Please work by copying the module's source to +# a relative sub-directory of the workspace and updating the reference to +# that sub-directory. +function modules { + local rel_module_dir + + mkdir -p "${MODULE_DIR}" + rel_module_dir="${MODULE_DIR//$OUTS/\.}" + + for module in $SRCS_MODULES; do + cp -r "${module}" "${MODULE_DIR}/" + done + + for module in "${!MODULE_PATHS[@]}"; do + path="${MODULE_PATHS[$module]}" + find "${PKG_DIR}" -maxdepth 1 -name "*.tf" -exec sed -i "s#${module}#${rel_module_dir}/$(basename "${path}")#g" {} + + done +} + + +# build_env_to_tf_srcs replaces various BUILD-time +# environment variables in the Terraform source files. +# This is useful for re-using source file in multiple workspaces, +# such as templating a Terraform remote state configuration. +function build_env_to_tf_srcs { + find "${PKG_DIR}" -maxdepth 1 -name "*.tf" -exec sed -i "s#\$PKG#${PKG}#g" {} + + find "${PKG_DIR}" -maxdepth 1 -name "*.tf" -exec sed -i "s#\$PKG_DIR#${PKG_DIR}#g" {} + + NAME="$(echo "${NAME}" | sed 's/^_\(.*\)_wd$/\1/')" + find "${PKG_DIR}" -maxdepth 1 -name "*.tf" -exec sed -i "s#\$NAME#${NAME}#g" {} + + find "${PKG_DIR}" -maxdepth 1 -name "*.tf" -exec sed -i "s#\$ARCH#${ARCH}#g" {} + + find "${PKG_DIR}" -maxdepth 1 -name "*.tf" -exec sed -i "s#\$OS#${OS}#g" {} + +} + +# copy modules +if [[ -v SRCS_MODULES ]]; then + modules +fi + +# substitute build env vars to srcs +build_env_to_tf_srcs + +# shift srcs into outs +for src in $SRCS_SRCS; do + cp "${src}" "${OUTS}/" +done diff --git a/terraform/terraform.build_defs b/terraform/terraform.build_defs new file mode 100644 index 0000000..b553acf --- /dev/null +++ b/terraform/terraform.build_defs @@ -0,0 +1,312 @@ +"""Build rules for working with Hashicorp Terraform (https://terraform.io) +""" + +TERRAFORM_DEFAULT_TOOLCHAIN = CONFIG.get('TERRAFORM_DEFAULT_TOOLCHAIN') or "//third_party/binary:terraform" + +MODULE_BUILDER_SRC = CONFIG.get('TERRAFORM_MODULE_BUILDER_SRC') or "//third_party/terraform:module_builder" +RUNNER_SRC = CONFIG.get('TERRAFORM_RUNNER_SRC') or "//third_party/terraform:runner" +WORKSPACE_BUILDER_SRC = CONFIG.get('TERRAFORM_WORKSPACE_BUILDER_SRC') or "//third_party/terraform:workspace_builder" + +def terraform_toolchain( + name:str, + version:str, + hashes:list = [], + labels: list = [], + visibility:list = [], +): + """Build rule for obtaining a version of the Terraform CLI. + + Args: + name: The name of the build rule. + version: The version of Terraform to download in MAJOR.MINOR.PATCH format. e.g. "0.12.3". + hashes: The hashes to verify the downloaded archive against. + labels: The additonal labels to add to the build rule. + visibility: The targets to make the toolchain visible to. + """ + download=remote_file( + name = f"_{name}_download", + out = f"_{name}_download", + url = f"https://releases.hashicorp.com/terraform/{version}/terraform_{version}_{CONFIG.OS}_{CONFIG.ARCH}.zip", + hashes = hashes, + extract = True, + ) + return genrule( + name=name, + srcs=[download], + # We output into a directory so we can add `terraform` to the PATH at runtime. + outs=[f"_{name}_download/terraform"], + cmd="mkdir -p $(dirname $OUTS) && mv $SRCS/terraform $OUTS", + visibility = visibility, + binary = True, + ) + +def terraform_provider( + name: str, + version: str = None, + registry: str = "registry.terraform.io", + namespace: str = "hashicorp", + provider_name: str = None, + url: str = None, + hashes: list = [], + labels: list = [], + visibility: list = [], +): + """Build rule for obtaining a Terraform Provider. + + Args: + name: The name of the build rule. + version: The version of Terraform the Terraform Provider MAJOR.MINOR.PATCH format. e.g. "2.1.2". + registry: Terraform 0.13+ - The Terraform registy hostname this provider is from. + namespace: Terraform 0.13+ - The Terraform registy namespace this provider is in. + provider_name: Terraform 0.13+ - The Terraform registy type this provider is e.g. "null", defaults to name. + url: The url to download and extract the Terraform provider from. + hashes: The hashes to verify the downloaded archive against. + labels: The additonal labels to add to the build rule. + visibility: The targets to make the toolchain visible to. + """ + provider_name = provider_name or name + url = url if url else f"https://releases.hashicorp.com/terraform-provider-{provider_name}/{version}/terraform-provider-{provider_name}_{version}_{CONFIG.OS}_{CONFIG.ARCH}.zip" + provider_download=remote_file( + name = f"_{name}_download", + url = url, + hashes = hashes, + ) + provider=genrule( + name = name, + srcs = [provider_download], + # We extract the binary into a directory here + # to preserve the filename which is usually + # terraform-provider-aws_v2.70.0_x4 + cmd = f""" +unzip $SRCS -d tmp +mkdir $OUTS +mv tmp/* $OUTS/ +echo "{registry}" > $OUTS/.registry +echo "{namespace}" > $OUTS/.namespace +echo "{provider_name}" > $OUTS/.provider_name +echo "{version}" > $OUTS/.version +""", + outs = [name], + visibility = visibility, + ) + + +def terraform_module( + name: str, + srcs: list = None, + url: str = None, + strip: list = [], + hashes: list = [], + deps: list = [], + licences: list = [], + labels: list = [], + visibility: list = [], +): + """Build rule for obtaining a remote Terraform Module or defining a local Terraform module. + + Args: + name: The name of the build rule. + srcs: The source Terraform files for the Terraform module. + url: The url to download and extract the Terraform module from. + strip: The files/directories to strip from the module. + hashes: The hashes to verify the downloaded archive against. + deps: The modules that this module depends on. + licences: The licences associated with the module. + labels: The additonal labels to add to the build rule. + visibility: The targets to make the toolchain visible to. + """ + og_module=None + if url: + og_module=remote_file( + name = f"_{name}_download", + url = url, + hashes = hashes, + licences = licences, + extract = True, + ) + else: + og_module=genrule( + name = f"_{name}_srcs", + srcs = srcs, + outs = [f"_{name}_srcs"], + # srcs in other directories should be modules + cmd = "mkdir $OUTS && for src in $SRCS; do cp $src $OUTS/; done", + ) + deps=[canonicalise(dep) for dep in deps] + + strip_bash_array = _to_bash_array("STRIP", strip) + + genrule( + name = name, + srcs = { + "og" : [og_module], + "deps" : deps, + }, + outs = [name], + exported_deps=deps, + deps=deps, + visibility=visibility, + tools=[MODULE_BUILDER_SRC], + cmd = f""" +set -euo pipefail +{_bash_version_check_cmd} + +URL="{url}" +OG_MODULE_DIR="$(location {og_module})" +{strip_bash_array} + +source "$(out_location {MODULE_BUILDER_SRC})" + """, + ) + +def terraform_root( + name: str, + srcs: list, + modules: list = [], + providers: list = [], + toolchain: str = None, + pre_binaries: list = [], + post_binaries: list = [], + labels: list = [], + visibility: list = [], +): + """Build rule for running Terraform against Terraform configuration. + + Args: + name: The name of the build rule. + srcs: The source Terraform files for the root module. + modules: The Terraform modules that the srcs use. + providers: The Terraform providers that the srcs use. + toolchain: The Terraform toolchain to use with against the srcs. + pre_binares: A list of binaries to run before performing Terraform commands. This is useful for preparing authentication. + post_binares: A list of binaries to run after performing Terraform commands. This is useful for cleaning up authentication, or alternate resource lifecycles. + labels: The additonal labels to add to the build rule. + visibility: The targets to make the toolchain visible to. + """ + # determine the terraform binary to use + toolchain = toolchain or TERRAFORM_DEFAULT_TOOLCHAIN + + # create a workspace for terraform to use + workspace = _terraform_workspace(name, srcs, modules, providers, toolchain) + + pre_binary_bash_array = _to_bash_array("PRE_BINARIES", [f"$(out_exe {b})" for b in pre_binaries]) + post_binary_bash_array = _to_bash_array("POST_BINARIES", [f"$(out_exe {b})" for b in post_binaries]) + + cmds = { + "plan": ["init", "plan"], + "apply": ["init", "apply"], + "destroy": ["init", "destroy"], + "bin": [], + } + for k in cmds.keys(): + commands = cmds[k] + cmd_bash_array = _to_bash_array("TERRAFORM_CMDS", commands) + + sh_cmd( + name = f"{name}_tf_{k}", + shell = "/bin/bash", + cmd = f""" +set -euo pipefail +{_bash_version_check_cmd} + +TERRAFORM_BIN="$(out_exe {toolchain})" +TERRAFORM_WORKSPACE="$(out_location {workspace})" +{pre_binary_bash_array} +{cmd_bash_array} +{post_binary_bash_array} + +source "$(out_location {RUNNER_SRC})" + """, + data = [workspace, toolchain, RUNNER_SRC] + pre_binaries + post_binaries, + labels = [f"terraform_{k}"] + labels, + visibility = visibility, + ) + + _linters(name, toolchain, workspace, labels, visibility) + +_bash_version_check_cmd = """ +if [ -z "${BASH_VERSINFO}" ] || [ -z "${BASH_VERSINFO[0]}" ] || [ ${BASH_VERSINFO[0]} -lt 4 ]; then + echo "This script requires Bash version >= 4" + exit 1 +fi +""" + +def _to_bash_array(var_name:str, items:list): + bash_array=[f"{var_name}=()"] + bash_array+=[f"{var_name}+=({i})" for i in items] + return "\n".join(bash_array) + +def _to_bash_map(var_name:str, items:dict): + bash_map=[f"declare -A {var_name}"] + for k in items.keys(): + v = items[k] + bash_map+=[f'{var_name}["{k}"]="{v}"'] + return "\n".join(bash_map) + +def _terraform_workspace( + name: str, + srcs: list, + modules: list = [], + providers: list = [], + toolchain: str = None, +): + modules = [canonicalise(module) for module in modules] + module_paths = {m : f"$(out_location {m})" for m in modules} + module_paths_bash_map = _to_bash_map("MODULE_PATHS", module_paths) + + return genrule( + name = f"_{name}_wd", + outs = [f"_{name}_wd"], + tools = [toolchain, WORKSPACE_BUILDER_SRC], + srcs = { + # srcs in other directories should be modules + "srcs": srcs, + "modules": modules, + "plugins": providers, + }, + cmd = f""" +set -euo pipefail +{_bash_version_check_cmd} + +CONFIG_OS="{CONFIG.OS}" +CONFIG_ARCH="{CONFIG.ARCH}" +TERRAFORM_BIN="$(out_exe {toolchain})" + +{module_paths_bash_map} + +source "$(out_location {WORKSPACE_BUILDER_SRC})" + """, + ) + +def _linters( + name:str, + toolchain:str, + workspace:str, + labels:list, + visibility:list, +): + linters = { + "validate": ["\"init -backend=false\"", "validate"], + "fmt": ["\"fmt -check -diff\""], + } + for k in linters.keys(): + commands = linters[k] + cmd_bash_array = _to_bash_array("TERRAFORM_CMDS", commands) + + sh_cmd( + name = f"{name}_tf_{k}", + shell = "/bin/bash", + cmd = f""" +set -euo pipefail +{_bash_version_check_cmd} + +TERRAFORM_BIN="$(out_exe {toolchain})" +TERRAFORM_WORKSPACE="$(out_location {workspace})" +{cmd_bash_array} + +source "$(out_location {RUNNER_SRC})" + """, + data = [workspace, toolchain, RUNNER_SRC], + labels = [f"terraform_{k}", "lint"] + labels, + visibility = visibility, + ) diff --git a/third_party/terraform/BUILD b/third_party/terraform/BUILD new file mode 100644 index 0000000..04e323e --- /dev/null +++ b/third_party/terraform/BUILD @@ -0,0 +1,29 @@ +subinclude("//terraform") + +terraform_toolchain( + name = "terraform_0_11", + version = "0.11.14", + hashes = ["9b9a4492738c69077b079e595f5b2a9ef1bc4e8fb5596610f69a6f322a8af8dd"], + visibility = ["//terraform/examples/0.11/..."], +) + +terraform_toolchain( + name = "terraform_0_12", + version = "0.12.30", + hashes = ["a646b61232ac0c400ec8cc2c062f4e36b5a9e8515f11f7f5f61eb03fe058f18d"], + visibility = ["//terraform/examples/0.12/..."], +) + +terraform_toolchain( + name = "terraform_0_13", + version = "0.13.6", + hashes = ["55f2db00b05675026be9c898bdd3e8230ff0c5c78dd12d743ca38032092abfc9"], + visibility = ["//terraform/examples/0.13/..."], +) + +terraform_toolchain( + name = "terraform_0_14", + version = "0.14.6", + hashes = ["63a5a45edde435fa3f278c86ce96346ee7f6b204ea949734f26f963b7dbc1074"], + visibility = ["//terraform/examples/0.14/..."], +) diff --git a/third_party/terraform/module/BUILD b/third_party/terraform/module/BUILD new file mode 100644 index 0000000..498bdc9 --- /dev/null +++ b/third_party/terraform/module/BUILD @@ -0,0 +1,19 @@ +subinclude("//terraform") + +terraform_module( + name = "cloudposse_null_label_0_11", + url = "https://github.com/cloudposse/terraform-null-label/archive/0.11.1.tar.gz", + hashes = ["7c17d3a9fba885c660b1757bf79079d1f0af2bf654fbefc1a376531e1ef9dbe9"], + licences = ["Apache-2.0"], + visibility = ["PUBLIC"], + strip = ["examples"], +) + +terraform_module( + name = "cloudposse_null_label_0_12", + url = "https://github.com/cloudposse/terraform-null-label/archive/0.22.1.tar.gz", + hashes = ["fe0e24ab7d161c582cd575cd34202e5ce3213f292d1b329a9523dbd5a085388c"], + licences = ["Apache-2.0"], + visibility = ["PUBLIC"], + strip = ["examples", "exports"], +) diff --git a/third_party/terraform/provider/BUILD b/third_party/terraform/provider/BUILD new file mode 100644 index 0000000..2bcf40e --- /dev/null +++ b/third_party/terraform/provider/BUILD @@ -0,0 +1,7 @@ +subinclude("//terraform") + +terraform_provider( + name = "null", + version = "2.1.2", + visibility = ["PUBLIC"], +)