From d81cd4fbf77e46ede8ba2981e4e67453940366e7 Mon Sep 17 00:00:00 2001 From: Tyler Hendrickson <1851017+TylerHendrickson@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:39:00 -0500 Subject: [PATCH] Developer experience enhancements (#894) * Add recommended VS Code settings * Add recommended VS Code extensions * Add Taskfile commands for localstack workflows * Update README with localstack shortcuts * Track .vscode/ IDE settings * YAML niceties --- .gitignore | 1 - .taskfiles/local.yml | 154 ++++++++++++++++++++++++++++++++++++++++ .vscode/extensions.json | 13 ++++ .vscode/settings.json | 10 +++ README.md | 19 +++++ Taskfile.yml | 4 ++ openapi/openapi.yaml | 2 + 7 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 .taskfiles/local.yml create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index e32cce91..10bfafcd 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ # IDE settings .idea/ -.vscode/ # Python stuff __pycache__/ diff --git a/.taskfiles/local.yml b/.taskfiles/local.yml new file mode 100644 index 00000000..6f261767 --- /dev/null +++ b/.taskfiles/local.yml @@ -0,0 +1,154 @@ +# yaml-language-server: $schema=https://taskfile.dev/schema.json +# https://taskfile.dev + +version: '3' + +env: + AWS_REGION: us-west-2 + AWS_DEFAULT_REGION: us-west-2 + AWS_SDK_LOAD_CONFIG: "true" + AWS_ACCESS_KEY_ID: test + AWS_SECRETE_ACCESS_KEY: test + TERRAFORM_STATE_BUCKET_NAME: local-terraform + +tasks: + check-dependencies: + desc: Checks dependencies for LocalStack development + silent: true + cmds: + - cmd: | + if ! command -v {{ .ITEM }} &> /dev/null; then + echo "❌ missing dependency: {{ .ITEM }}" + exit 1 + else + echo "✅ {{ .ITEM }}" + fi + vars: + COMMAND: '{{ .ITEM }}' + for: + - terraform + - tflocal + - aws + - awslocal + - docker + - task: check-tf-version + vars: + PREFIX: '{{ .TASK }}' + - task: check-aws-version + vars: + PREFIX: '{{ .TASK }}' + + check-command-exists: + prefix: '{{ .PREFIX }}' + internal: true + silent: true + cmd: | + if ! command -v {{ .COMMAND }} &> /dev/null; then + echo "❌ missing dependency: {{ .COMMAND }}" + exit 1 + else + echo "✅ {{ .COMMAND }}" + fi + + check-tf-version: + silent: true + internal: true + dir: terraform + prefix: '{{ .PREFIX }}' + vars: + VERSION_FILE_CONTENTS: + sh: cat .terraform-version + TARGET_VERSION: '{{ catLines .VERSION_FILE_CONTENTS | trim }}' + FOUND_VERSION_RAW: + sh: terraform version | head -n1 + FOUND_VERSION: '{{ .FOUND_VERSION_RAW | replace "Terraform v" "" }}' + cmd: | + if [ "{{ .TARGET_VERSION }}" != "{{ .FOUND_VERSION }}" ]; then + echo "❌ Incorrect terraform version (want v{{ .TARGET_VERSION }})" + exit 1 + else + echo "✅ Terraform v{{ .FOUND_VERSION }} is installed" + fi + + check-aws-version: + silent: true + internal: true + prefix: '{{ .PREFIX }}' + vars: + FOUND_VERSION_RAW: + sh: aws --version + FOUND_VERSION: '{{ mustRegexFind "[0-9]" .FOUND_VERSION_RAW }}' + TARGET_VERSION: '2' + cmd: | + if [ "{{ .TARGET_VERSION }}" != "{{ .FOUND_VERSION }}" ]; then + echo "❌ Incorrect AWS CLI version (want v{{ .TARGET_VERSION }}.x.x)" + exit 1 + else + echo "✅ AWS CLI v{{ .FOUND_VERSION }} is installed" + fi + + ensure-tfstate-bucket-exists: + desc: Creates an S3 state bucket for Terraform if one does not already exist + status: + - awslocal s3api head-bucket --bucket "$TERRAFORM_STATE_BUCKET_NAME" + cmds: + - awslocal s3 mb "s3://$TERRAFORM_STATE_BUCKET_NAME" + + tf-init: + desc: Initializes the Terraform project in the running LocalStack instance + run: always + dir: terraform + cmds: + - task: ensure-tfstate-bucket-exists + - cmd: tflocal init -backend-config="local.s3.tfbackend" -reconfigure + + tf-apply: + desc: Applies the Terraform project to the running LocalStack instance + run: always + dir: terraform + cmds: + - tflocal apply -var-file="local.tfvars" -auto-approve + + from-scratch: + desc: Initializes and deploys the service to the running LocalStack instance + cmds: + - task: tf-init + - task: :prebuild-lambda + - task: deploy + + deploy: + desc: Builds Lambda handlers and applies the Terraform project to the running LocalStack instance + run: always + cmds: + - task: :build + - task: tf-apply + + invoke-DownloadGrantsGovDB: + desc: Invokes DownloadGrantsGovDB Lambda on LocalStack + summary: | + Triggers the DownloadGrantsGovDB Lambda pipeline step. + + By default, the invocation payload will specify a timestamp that will cause the Lambda function + to target a download for yesterday's date, which may be overriden by setting an INVOCATION_DATE + environment variable (in YYYY-MM-DD format) when running this task. + + Set an INVOCATION_EVENT environment variable to provide an arbitrary (JSON) invocation event. + + This task assumes AWS CLI v2 is installed, which invokes Lambda functions with a payload + that is the base64-encoded result of the invocation event. To provide your own payload + (which overrides all invocation defaults), set a PAYLOAD environment variable. + + The invocation response is printed to stdout by default. Set an OUTFILE environment variable + to log to a different file. + cmds: + - '# Invocation event: {{ .INVOCATION_EVENT }}' + - cmd: awslocal lambda invoke --function-name '{{ .FUNCTION_NAME }}' --payload '{{ .PAYLOAD }}' '{{ .OUTFILE }}' + vars: + FUNCTION_NAME: grants-ingest-DownloadGrantsGovDB + YESTERDAY_DATE: '{{ now | mustDateModify "-24h" | date "2006-01-02" }}' + INVOCATION_DATE: '{{ default .YESTERDAY_DATE ( env "INVOCATION_DATE" ) }}' + INVOCATION_TIME: '{{ default "05:00:00" ( env "INVOCATION_TIME" ) }}' + INVOCATION_TIMESTAMP: '{{ .INVOCATION_DATE }}T{{ .INVOCATION_TIME }}-04:00' + INVOCATION_EVENT: '{"timestamp": {{ quote .INVOCATION_TIMESTAMP }}}' + PAYLOAD: '{{ default ( b64enc .INVOCATION_EVENT ) ( env "PAYLOAD" ) }}' + OUTFILE: '{{ default "/dev/null" ( env "OUTFILE" ) }}' diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..6b6a1a04 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,13 @@ +{ + "recommendations": [ + "mikestead.dotenv", + "leighlondon.eml", + "ms-azuretools.vscode-docker", + "golang.go", + "hashicorp.terraform", + "redhat.vscode-xml", + "redhat.vscode-yaml", + "task.vscode-task", + "github.vscode-github-actions" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..afa10d8b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "diffEditor.ignoreTrimWhitespace": false, + "files.insertFinalNewline": true, + "[yaml]": { + "editor.insertSpaces": true, + "editor.detectIndentation": true, + "editor.tabSize": 2, + "diffEditor.ignoreTrimWhitespace": false, + } +} diff --git a/README.md b/README.md index fe5ac4d0..455264de 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,10 @@ To begin, make sure the following tools are available in your development worksp However, you can choose whatever LocalStack installation option best suits your environment. +> [!TIP] +> Check that you have the necessary dependencies for LocalStack development by running `task local:check-dependencies` + + #### Environment Variables We recommend setting the following environment variables in your development workspace when working @@ -111,6 +115,20 @@ with LocalStack: #### Provisioning Infrastructure +#### Quickstart + +Certain steps described in the "Manual Provisioning" section below may be achieved using the following +shortcut commands provided by this repository's Taskfile: +- **Prerequisite:** Ensure LocalStack is started by running `docker compose up -d` +- **Deploy to LocalStack for the first time:** `task local:from-scratch` +- **Subsequent deployments to LocalStack:** `task local:deploy` +- **Invoke the `DownloadGrantsGovDB` Lambda function:** `task local:invoke-DownloadGrantsGovDB` + +See `.taskfile/local.yml` for more information and other shortcuts. + + +#### Manual Provisioning + After starting LocalStack, create a terraform state deployment bucket in the localstack environment. This guide, as well as the provided Terraform backend file for local development, assumes that this bucket is named `local-terraform` and has a region of `us-west-2`. @@ -220,6 +238,7 @@ The following items can be referred to as a quick "cheat-sheet" for development: - Compile binary Lambda function handlers: `task build` - Compile the CLI tool: `task build-cli` - Run all QA checks normally executed during CI: `task check` +- Initialize and deploy a test environment after starting LocalStack: `task local:from-scratch` ## Contributing diff --git a/Taskfile.yml b/Taskfile.yml index 8f91b145..3a9ecc97 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,3 +1,4 @@ +--- # yaml-language-server: $schema=https://taskfile.dev/schema.json # https://taskfile.dev @@ -5,6 +6,9 @@ version: '3' output: prefixed +includes: + local: .taskfiles/local.yml + tasks: default: silent: true diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 7c89b8ca..17739f3c 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -1,3 +1,5 @@ +--- +# yaml-language-server: https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/schemas/v3.1/schema.json openapi: 3.1.0 info: title: USDR standard representation for federal grant data