From a450297e5a7486ea8f62bacf170c9e08398a6d5c Mon Sep 17 00:00:00 2001 From: Liam Lloyd Date: Wed, 6 Nov 2024 16:43:17 -0800 Subject: [PATCH] Test deploy access copy lambda to dev --- .github/workflows/build.yml | 20 +++ .github/workflows/dev_deploy.yml | 7 +- .github/workflows/generate_image_tags.yml | 6 + .github/workflows/test.yml | 88 +++++++------- .../test_cluster/access_copy_dev_lambda.tf | 114 ++++++++++++++++++ terraform/test_cluster/variables.tf | 10 ++ 6 files changed, 200 insertions(+), 45 deletions(-) create mode 100644 terraform/test_cluster/access_copy_dev_lambda.tf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 043e74f..065fc76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,3 +84,23 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - name: Publish Image to ECR run: docker push $THUMBNAIL_REFRESH_IMAGE_TAG + build_access_copy_lambda: + needs: + - generate_image_tags + runs-on: ubuntu-20.04 + env: + ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }} + steps: + - uses: actions/checkout@v3 + - name: Build Image + run: docker build -t $ACCESS_COPY_LAMBDA_IMAGE_TAG --build-arg="AWS_RDS_CERT_BUNDLE=$AWS_RDS_CERT_BUNDLE" -f Dockerfile.access_copy_attacher . + env: + AWS_RDS_CERT_BUNDLE: ${{ secrets.AWS_RDS_CERT_BUNDLE }} + - name: AWS Login + run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Publish Image to ECR + run: docker push $ACCESS_COPY_LAMBDA_IMAGE_TAG diff --git a/.github/workflows/dev_deploy.yml b/.github/workflows/dev_deploy.yml index 3b5d902..d62c18a 100644 --- a/.github/workflows/dev_deploy.yml +++ b/.github/workflows/dev_deploy.yml @@ -28,6 +28,7 @@ jobs: AM_CLEANUP_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.AM_CLEANUP_IMAGE_TAG }} RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }} THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }} + ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }} defaults: run: working-directory: ./terraform/test_cluster @@ -60,7 +61,8 @@ jobs: -target=kubernetes_deployment.stela_dev \ -target=kubernetes_cron_job_v1.archivematica_cleanup_dev \ -target=aws_lambda_function.record_thumbnail_lambda \ - -target=kubernetes_cron_job_v1.thumbnail_refresh_dev + -target=kubernetes_cron_job_v1.thumbnail_refresh_dev \ + -target=aws_lambda_function.access_copy_dev_lambda - name: Terraform Apply run: | terraform apply -auto-approve -input=false \ @@ -75,4 +77,5 @@ jobs: -target=kubernetes_deployment.stela_dev \ -target=kubernetes_cron_job_v1.archivematica_cleanup_dev \ -target=aws_lambda_function.record_thumbnail_lambda \ - -target=kubernetes_cron_job_v1.thumbnail_refresh_dev + -target=kubernetes_cron_job_v1.thumbnail_refresh_dev \ + -target=aws_lambda_function.access_copy_dev_lambda diff --git a/.github/workflows/generate_image_tags.yml b/.github/workflows/generate_image_tags.yml index 07583e5..ddb897b 100644 --- a/.github/workflows/generate_image_tags.yml +++ b/.github/workflows/generate_image_tags.yml @@ -10,6 +10,8 @@ on: value: ${{ jobs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }} THUMBNAIL_REFRESH_IMAGE_TAG: value: ${{ jobs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }} + ACCESS_COPY_LAMBDA_IMAGE_TAG: + values: ${{ jobes.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }} jobs: generate_image_tags: runs-on: ubuntu-20.04 @@ -18,6 +20,7 @@ jobs: AM_CLEANUP_IMAGE_TAG: ${{ steps.generate_am_cleanup_image_tag.outputs.AM_CLEANUP_IMAGE_TAG }} RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ steps.generate_record_thumbnail_lambda_image_tag.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }} THUMBNAIL_REFRESH_IMAGE_TAG: ${{ steps.generate_thumbnail_refresh_image_tag.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }} + ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ steps.generate_access_copy_lambda_image_tag.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }} steps: - uses: actions/checkout@v3 - name: Set ECR domain env var @@ -43,3 +46,6 @@ jobs: - name: Generate Thumbnail Refresh Image Tag id: generate_thumbnail_refresh_image_tag run: echo "THUMBNAIL_REFRESH_IMAGE_TAG=$ECR_DOMAIN/stela:thumbnail_refresh-$BRANCH_TYPE-$ABBREVIATED_COMMIT_HASH" >> "$GITHUB_OUTPUT" + - name: Generate Access Copy Lambda Image Tag + id: generate_access_copy_lambda_image_tag + run: echo "ACCESS_COPY_LAMBDA_IMAGE_TAG=$ECR_DOMAIN/stela:access_copy_lambda-$BRANCH_TYPE-$ABBREVIATED_COMMIT_HASH" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46f8428..0587bcf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,46 +1,48 @@ name: Unit tests on: - push: - branches-ignore: - - main - workflow_dispatch: - workflow_call: + push: + branches-ignore: + - main + workflow_dispatch: + workflow_call: jobs: - run_tests: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - with: - path: ./stela - - uses: actions/setup-node@v1 - with: - node-version: "18" - - name: Checkout back-end - uses: actions/checkout@v3 - with: - ssh-key: ${{ secrets.BACKEND_ACCESS_PRIVATE_SSH_KEY }} - repository: PermanentOrg/back-end - ref: main - path: ./back-end - - name: Checkout devenv - uses: actions/checkout@v3 - with: - ssh-key: ${{ secrets.DEVENV_ACCESS_PRIVATE_SSH_KEY }} - repository: PermanentOrg/devenv - ref: main - path: ./devenv - - run: (cd stela; npm install --production=false) - - run : (cd stela; npm run build -ws) - - run: touch stela/.env - - run: touch devenv/.env - - run: (cd devenv; docker compose up database_setup -d; docker logs devenv-database_setup-1) - - run: (cd stela/packages/api; npm run start-containers) - - run: (cd stela/packages/api; docker compose run stela npm run test-ci) - - run: (cd stela; npm run test -w @stela/account_space_updater) - - uses: codecov/codecov-action@v2 - - run: (cd stela; npm run test -w @stela/record_thumbnail_attacher) - - uses: codecov/codecov-action@v2 - - run: (cd stela; npm run test -w @stela/archivematica_cleanup) - - uses: codecov/codecov-action@v2 - - run: (cd stela; npm run test -w @stela/thumbnail_refresh) - - uses: codecov/codecov-action@v2 + run_tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + with: + path: ./stela + - uses: actions/setup-node@v1 + with: + node-version: "18" + - name: Checkout back-end + uses: actions/checkout@v3 + with: + ssh-key: ${{ secrets.BACKEND_ACCESS_PRIVATE_SSH_KEY }} + repository: PermanentOrg/back-end + ref: main + path: ./back-end + - name: Checkout devenv + uses: actions/checkout@v3 + with: + ssh-key: ${{ secrets.DEVENV_ACCESS_PRIVATE_SSH_KEY }} + repository: PermanentOrg/devenv + ref: main + path: ./devenv + - run: (cd stela; npm install --production=false) + - run: (cd stela; npm run build -ws) + - run: touch stela/.env + - run: touch devenv/.env + - run: (cd devenv; docker compose up database_setup -d; docker logs devenv-database_setup-1) + - run: (cd stela/packages/api; npm run start-containers) + - run: (cd stela/packages/api; docker compose run stela npm run test-ci) + - run: (cd stela; npm run test -w @stela/account_space_updater) + - uses: codecov/codecov-action@v2 + - run: (cd stela; npm run test -w @stela/record_thumbnail_attacher) + - uses: codecov/codecov-action@v2 + - run: (cd stela; npm run test -w @stela/archivematica_cleanup) + - uses: codecov/codecov-action@v2 + - run: (cd stela; npm run test -w @stela/thumbnail_refresh) + - uses: codecov/codecov-action@v2 + - run: (cd stela; npm run test -w @stela/access_copy_attacher) + - uses: codecov/codecov-action@v2 diff --git a/terraform/test_cluster/access_copy_dev_lambda.tf b/terraform/test_cluster/access_copy_dev_lambda.tf new file mode 100644 index 0000000..78b2b26 --- /dev/null +++ b/terraform/test_cluster/access_copy_dev_lambda.tf @@ -0,0 +1,114 @@ +resource "aws_sqs_queue" "access_copy_dev_deadletter_queue" { + name = "access-copy-dev-deadletter-queue" +} + +resource "aws_sqs_queue" "access_copy_dev_queue" { + name = "access-copy-dev-queue" + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.access_copy_dev_deadletter_queue.arn + maxReceiveCount = 3 + }) +} + +resource "aws_sqs_queue_policy" "access_copy_dev_queue_policy" { + queue_url = aws_sqs_queue.access_copy_dev_queue.id + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Principal = { + Service = "sns.amazonaws.com" + }, + Action = "sqs:SendMessage", + Resource = aws_sqs_queue.access_copy_dev_queue.arn, + Condition = { + ArnEquals = { + "aws:SourceArn" = aws_sns_topic.record_thumbnail_dev_topic.arn + } + } + } + ] + }) +} + +resource "aws_sns_topic_subscription" "access_copy_dev_subscription" { + topic_arn = aws_sns_topic.record_thumbnail_dev_topic.arn + protocol = "sqs" + endpoint = aws_sqs_queue.access_copy_dev_queue.arn +} + +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "access_copy_dev_lambda_role" { + name = "access-copy-dev-lambda-role" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_role_policy" "access_copy_dev_lambda_policy" { + name = "access-copy-lambda-policy" + role = aws_iam_role.access_copy_dev_lambda_role.name + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ] + Effect = "Allow" + Resource = ["*", aws_sqs_queue.access_copy_dev_queue.arn] + }, + ] + }) +} + +resource "aws_lambda_function" "access_copy_dev_lambda" { + package_type = "Image" + image_uri = var.access_copy_dev_lambda_image + function_name = "access-copy-dev-lambda" + role = aws_iam_role.access_copy_dev_lambda_role.arn + timeout = 30 + + vpc_config { + security_group_ids = [var.dev_security_group_id] + subnet_ids = var.subnet_ids + } + + environment { + variables = { + ENV = var.dev_env + SENTRY_DSN = var.sentry_dsn + DATABASE_URL = var.dev_database_url + CLOUDFRONT_URL = var.dev_cloudfront_url + CLOUDFRONT_KEY_PAIR_ID = var.cloudfront_key_pair_id + CLOUDFRONT_PRIVATE_KEY = var.cloudfront_private_key + } + } +} + +resource "aws_lambda_event_source_mapping" "record_thumbnail_dev_event_source_mapping" { + event_source_arn = aws_sqs_queue.record_thumbnail_dev_queue.arn + function_name = aws_lambda_function.record_thumbnail_dev_lambda.arn + batch_size = 10 + maximum_batching_window_in_seconds = 0 +} diff --git a/terraform/test_cluster/variables.tf b/terraform/test_cluster/variables.tf index bdf2f6a..325aeb9 100644 --- a/terraform/test_cluster/variables.tf +++ b/terraform/test_cluster/variables.tf @@ -78,6 +78,16 @@ variable "thumbnail_refresh_staging_image" { type = string } +variable "access_copy_dev_lambda_image" { + description = "Tag of the access copy lambda image to deploy to dev" + type = string +} + +variable "access_copy_staging_lambda_image" { + description = "Tag of the access copy lambda image to deploy to staging" + type = string +} + variable "dev_security_group_id" { description = "ID of the Development security group" type = string