diff --git a/.github/.github.env b/.github/.github.env index 97289f3..ad8c623 100644 --- a/.github/.github.env +++ b/.github/.github.env @@ -2,3 +2,5 @@ COGNITO_USER_POOL_ID=rapid-pool RESOURCE_PREFIX=rapid ALLOWED_EMAIL_DOMAINS=example1.com,example2.com LAYERS=raw,layer +DOMAIN_NAME=example.com +DATA_BUCKET=the-bucket diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 1278154..3835c6e 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -45,8 +45,6 @@ jobs: - name: Populate .env with additional vars run: | cp ./.github/.github.env .env - echo DOMAIN_NAME=${{ secrets.DOMAIN_NAME }} >> .env - echo DATA_BUCKET=${{ secrets.DATA_BUCKET }} >> .env echo AWS_ACCOUNT=${{ secrets.AWS_ACCOUNT }} >> .env echo AWS_REGION=${{ secrets.AWS_REGION }} >> .env echo AWS_DEFAULT_REGION=${{ secrets.AWS_REGION }} >> .env @@ -76,7 +74,7 @@ jobs: run: | echo "TWINE_USERNAME=${{ secrets.TWINE_USERNAME_TEST }}" >> .env echo "TWINE_PASSWORD=${{ secrets.TWINE_PASSWORD_TEST }}" >> .env - echo TWINE_NON_INTERACTIVE=${{ secrets.TWINE_NON_INTERACTIVE }} >> .env + echo "TWINE_NON_INTERACTIVE=true" >> .env - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1f5bfe3..65a0807 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,8 +44,6 @@ jobs: - name: Populate .env with additional vars run: | cp ./.github/.github.env .env - echo DOMAIN_NAME=${{ secrets.DOMAIN_NAME }} >> .env - echo DATA_BUCKET=${{ secrets.DATA_BUCKET }} >> .env echo AWS_ACCOUNT=${{ secrets.AWS_ACCOUNT }} >> .env echo AWS_REGION=${{ secrets.AWS_REGION }} >> .env echo AWS_DEFAULT_REGION=${{ secrets.AWS_REGION }} >> .env @@ -69,7 +67,7 @@ jobs: run: make api-tag-prod-candidate - name: API Deploy Image to Prod - run: make api-tag-live-in-prod + run: make api-app-live-in-prod - name: API Allow for Application to Start run: sleep 120 @@ -81,6 +79,8 @@ jobs: - name: API E2E Tests id: e2e-tests env: + DOMAIN_NAME: ${{ secrets.DOMAIN_NAME }} + DATA_BUCKET: ${{ secrets.DATA_BUCKET }} COGNITO_USER_POOL_ID: ${{ secrets.COGNITO_USER_POOL_ID }} RESOURCE_PREFIX: ${{ secrets.RESOURCE_PREFIX }} ALLOWED_EMAIL_DOMAINS: ${{ secrets.ALLOWED_EMAIL_DOMAINS }} diff --git a/Makefile b/Makefile index 8b2d897..6e11259 100644 --- a/Makefile +++ b/Makefile @@ -88,8 +88,8 @@ api-tag-and-upload-release-image:## Tag and upload the api release image api-tag-prod-candidate: ## Tag the uploaded api image as a candidate for PROD deployment @cd api/; $(MAKE) tag-prod-candidate -api-tag-live-in-prod: ## Deploy the latest version of the api - @cd api/; $(MAKE) tag-live-in-prod +api-app-live-in-prod: ## Deploy the latest version of the api + @cd api/; $(MAKE) app-live-in-prod api-check-app-is-running: @cd api/; $(MAKE) check-app-is-running diff --git a/api/Makefile b/api/Makefile index 4aa6398..7b03249 100644 --- a/api/Makefile +++ b/api/Makefile @@ -1,5 +1,5 @@ -ECS_SERVICE=rapid-ecs-service -ECS_CLUSTER=rapid-cluster +ECS_SERVICE=rapid-preprod-ecs-service +ECS_CLUSTER=rapid-preprod-cluster LATEST_COMMIT_HASH=$(shell git rev-parse --short HEAD) ACCOUNT_ECR_URI=$(AWS_ACCOUNT).dkr.ecr.$(AWS_REGION).amazonaws.com IMAGE_NAME=data-f1-registry diff --git a/docs/infrastructure/deployment.md b/docs/infrastructure/deployment.md index 0266808..c2c67d7 100644 --- a/docs/infrastructure/deployment.md +++ b/docs/infrastructure/deployment.md @@ -74,9 +74,6 @@ Our infrastructure is built using AWS, so you'll need an AWS account, and access Follow these steps to set up the AWS profile: - [Install/Update AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) -- [Set up a named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) if you already have the AWS cli. - -After setting up the named profile, the current session can be checked by running ```aws sts get-caller-identity```. We have a file (`scripts/env_setup.sh) with the required exports to use the 'gov' profile. These exports have to be run when starting a new session. We use `jq` in our scripts to help the `make` targets work correctly, please [Install jq](https://stedolan.github.io/jq/download/) before running any make command. @@ -184,8 +181,6 @@ In order to gain the admin privileges necessary for infrastructure changes one n enabled only for user's defined in `input-params.tfvars`, only after logging into the AWS console for the first time as an IAM user and enabling MFA. -Then, to assume the role, set up the profile (`scripts/env_setup.sh`), run ```make infra-assume-role``` and follow the prompts. - ### Deploying remaining infra-blocks Once the state backend has been configured, provide/change the following inputs in `input-params.tfvars`. diff --git a/infrastructure/blocks/pipeline-ami/data.tf b/infrastructure/blocks/pipeline-ami/data.tf new file mode 100644 index 0000000..603ec83 --- /dev/null +++ b/infrastructure/blocks/pipeline-ami/data.tf @@ -0,0 +1,9 @@ +data "terraform_remote_state" "vpc-state" { + backend = "s3" + workspace = "prod" + + config = { + key = "vpc/terraform.tfstate" + bucket = var.state_bucket + } +} diff --git a/infrastructure/scripts/initialisation-script.sh.tpl b/infrastructure/blocks/pipeline-ami/install.sh similarity index 53% rename from infrastructure/scripts/initialisation-script.sh.tpl rename to infrastructure/blocks/pipeline-ami/install.sh index 73a1d86..020b8ed 100644 --- a/infrastructure/scripts/initialisation-script.sh.tpl +++ b/infrastructure/blocks/pipeline-ami/install.sh @@ -1,5 +1,3 @@ -#!/usr/bin/env bash - # Enable SSM sudo snap install amazon-ssm-agent --classic sudo snap start amazon-ssm-agent @@ -38,23 +36,3 @@ sudo apt install gh -y curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip sudo ./aws/install - -# ---- Start docker service -sudo service docker start - -# ---- Allow ubuntu user to manage Docker service -sudo usermod -a -G docker ubuntu - -# Install GitHub Actions Runner -# Need to run these commands as the ubuntu user for correct permissions -sudo -u ubuntu mkdir /home/ubuntu/actions-runner -cd /home/ubuntu/actions-runner -sudo -u ubuntu curl -o actions-runner-linux-x64-2.307.1.tar.gz -L https://github.com/actions/runner/releases/download/v2.307.1/actions-runner-linux-x64-2.307.1.tar.gz -sudo -u ubuntu tar xzf ./actions-runner-linux-x64-2.307.1.tar.gz -sudo -u ubuntu ./config.sh --url https://github.com/no10ds --token "${runner-registration-token}" --name Data-F1-Pipeline-Runner --unattended --replace - -# Run the GitHub Actions Runner -sudo -u ubuntu ./run.sh & - -# # Configure the GitHub Actions Runner to start on reboot -sudo crontab -l -u ubuntu | echo "@reboot sudo -u ubuntu /home/ubuntu/actions-runner/run.sh &" | sudo crontab -u ubuntu - \ No newline at end of file diff --git a/infrastructure/blocks/pipeline-ami/packer.tf b/infrastructure/blocks/pipeline-ami/packer.tf new file mode 100644 index 0000000..3b24632 --- /dev/null +++ b/infrastructure/blocks/pipeline-ami/packer.tf @@ -0,0 +1,25 @@ +resource "null_resource" "packer_build" { + triggers = { + sha256_ami_config = filesha256("${path.module}/template.json") + sha256_ami_install = filesha256("${path.module}/install.sh") + version = var.pipeline_ami_version + } + + provisioner "local-exec" { + command = <