Skip to content

Commit

Permalink
Add terraform and Github Actions config to deploy account space updat…
Browse files Browse the repository at this point in the history
…e lambda

Now that we have a working handler for the account space update lambda,
this commit adds the terraform and Github Actions configuration
necessary to include this lambda in deploys.
  • Loading branch information
Liam Lloyd committed Nov 27, 2024
1 parent 2ed02e8 commit 2050786
Show file tree
Hide file tree
Showing 12 changed files with 422 additions and 8 deletions.
20 changes: 20 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -104,3 +104,23 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- name: Publish Image to ECR
run: docker push $ACCESS_COPY_LAMBDA_IMAGE_TAG
build_account_space_updater:
needs:
- generate_image_tags
runs-on: ubuntu-20.04
env:
ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
steps:
- uses: actions/checkout@v3
- name: Build Image
run: docker build -t $ACCOUNT_SPACE_UPDATER_IMAGE_TAG --build-arg="AWS_RDS_CERT_BUNDLE=$AWS_RDS_CERT_BUNDLE" -f Dockerfile.account_space_updater .
env:
AWS_RDS_CERT_BUNDLE: ${{ secrets.AWS_RDS_CERT_BUNDLE }}
- name: AWS Login
run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- name: Publish Image to ECR
run: docker push $ACCOUNT_SPACE_UPDATER_IMAGE_TAG
11 changes: 9 additions & 2 deletions .github/workflows/dev_deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ jobs:
RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
defaults:
run:
working-directory: ./terraform/test_cluster
Expand Down Expand Up @@ -60,11 +61,14 @@ jobs:
-var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
-var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
-target=kubernetes_deployment.stela_dev \
-target=kubernetes_cron_job_v1.archivematica_cleanup_dev \
-target=aws_lambda_function.record_thumbnail_lambda \
-target=kubernetes_cron_job_v1.thumbnail_refresh_dev \
-target=aws_lambda_function.access_copy_dev_lambda
-target=aws_lambda_function.access_copy_dev_lambda \
-target=aws_lambda_function.account_space_update_dev_lambda
- name: Terraform Apply
run: |
terraform apply -auto-approve -input=false \
Expand All @@ -78,8 +82,11 @@ jobs:
-var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
-var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
-target=kubernetes_deployment.stela_dev \
-target=kubernetes_cron_job_v1.archivematica_cleanup_dev \
-target=aws_lambda_function.record_thumbnail_lambda \
-target=kubernetes_cron_job_v1.thumbnail_refresh_dev \
-target=aws_lambda_function.access_copy_dev_lambda
-target=aws_lambda_function.access_copy_dev_lambda \
-target=aws_lambda_function.account_space_update_dev_lambda
9 changes: 7 additions & 2 deletions .github/workflows/full_test_deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ jobs:
RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
defaults:
run:
working-directory: ./terraform/test_cluster
Expand Down Expand Up @@ -54,7 +55,9 @@ jobs:
-var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
-var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
-var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG"
- name: Terraform Apply
run: |
terraform apply -auto-approve -input=false \
Expand All @@ -67,4 +70,6 @@ jobs:
-var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
-var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
-var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG"
6 changes: 6 additions & 0 deletions .github/workflows/generate_image_tags.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ on:
value: ${{ jobs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
ACCESS_COPY_LAMBDA_IMAGE_TAG:
value: ${{ jobs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
ACCOUNT_SPACE_UPDATER_IMAGE_TAG:
value: ${{ jobs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
jobs:
generate_image_tags:
runs-on: ubuntu-20.04
Expand All @@ -21,6 +23,7 @@ jobs:
RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ steps.generate_record_thumbnail_lambda_image_tag.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
THUMBNAIL_REFRESH_IMAGE_TAG: ${{ steps.generate_thumbnail_refresh_image_tag.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ steps.generate_access_copy_lambda_image_tag.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ steps.generate_account_space_updater_image_tag.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
steps:
- uses: actions/checkout@v3
- name: Set ECR domain env var
Expand Down Expand Up @@ -49,3 +52,6 @@ jobs:
- name: Generate Access Copy Lambda Image Tag
id: generate_access_copy_lambda_image_tag
run: echo "ACCESS_COPY_LAMBDA_IMAGE_TAG=$ECR_DOMAIN/stela:access_copy_lambda-$BRANCH_TYPE-$ABBREVIATED_COMMIT_HASH" >> "$GITHUB_OUTPUT"
- name: Generate Account Space Updater Image Tag
id: generate_account_space_updater_image_tag
run: echo "ACCOUNT_SPACE_UPDATER_IMAGE_TAG=$ECR_DOMAIN/stela:account_space_updater-$BRANCH_TYPE-$ABBREVIATED_COMMIT_HASH" >> "$GITHUB_OUTPUT"
7 changes: 5 additions & 2 deletions .github/workflows/prod_deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ jobs:
RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
defaults:
run:
working-directory: ./terraform/prod_cluster
Expand All @@ -47,12 +48,14 @@ jobs:
-var="archivematica_cleanup_image=$AM_CLEANUP_IMAGE_TAG" \
-var="record_thumbnail_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
-var="thumbnail_refresh_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="access_copy_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
-var="access_copy_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="account_space_updater_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG"
- name: Terraform Apply
run: |
terraform apply -auto-approve -input=false \
-var="stela_image=$API_IMAGE_TAG" \
-var="archivematica_cleanup_image=$AM_CLEANUP_IMAGE_TAG" \
-var="record_thumbnail_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
-var="thumbnail_refresh_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="access_copy_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
-var="access_copy_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="account_space_updater_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG"
11 changes: 9 additions & 2 deletions .github/workflows/staging_deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ jobs:
RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
defaults:
run:
working-directory: ./terraform/test_cluster
Expand Down Expand Up @@ -58,11 +59,14 @@ jobs:
-var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
-var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
-target=kubernetes_deployment.stela_staging \
-target=kubernetes_cron_job_v1.archivematica_cleanup_staging \
-target=aws_lambda_function.record_thumbnail_lambda_staging \
-target=kubernetes_cron_job_v1.thumbnail_refresh_staging \
-target=aws_lambda_function.access_copy_lambda_staging
-target=aws_lambda_function.access_copy_lambda_staging \
-target=aws_lambda_function.account_space_update_staging_lambda
- name: Terraform Apply
run: |
terraform apply -auto-approve -input=false \
Expand All @@ -74,8 +78,11 @@ jobs:
-var="record_thumbnail_staging_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
-var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
-var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
-target=kubernetes_deployment.stela_staging \
-target=kubernetes_cron_job_v1.archivematica_cleanup_staging \
-target=aws_lambda_function.record_thumbnail_lambda_staging \
-target=kubernetes_cron_job_v1.thumbnail_refresh_staging \
-target=aws_lambda_function.access_copy_lambda_staging
-target=aws_lambda_function.access_copy_lambda_staging \
-target=aws_lambda_function.account_space_update_staging_lambda
6 changes: 6 additions & 0 deletions Dockerfile.account_space_updater
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,14 @@ RUN npm run build -ws


FROM public.ecr.aws/lambda/nodejs:18 as final

ARG AWS_RDS_CERT_BUNDLE

WORKDIR ${LAMBDA_TASK_ROOT}

RUN mkdir /etc/ca-certificates
RUN echo -e $AWS_RDS_CERT_BUNDLE > /etc/ca-certificates/rds-us-west-2-ca-bundle.pem

COPY --from=builder /usr/local/apps/stela/packages/account_space_updater/dist ./packages/account_space_updater/dist
COPY --from=builder /usr/local/apps/stela/packages/account_space_updater/package.json ./packages/account_space_updater/package.json
COPY --from=builder /usr/local/apps/stela/packages/logger/dist ./packages/logger/dist
Expand Down
115 changes: 115 additions & 0 deletions terraform/prod_cluster/account_space_prod_lambda.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
resource "aws_sqs_queue" "account_space_update_prod_deadletter_queue" {
name = "account-space-update-prod-deadletter-queue"
}

resource "aws_sqs_queue" "account_space_update_prod_queue" {
name = "account-space-update-prod-queue"

redrive_policy = jsonencode({
deadLetterTargetArn = aws_sqs_queue.account_space_update_prod_deadletter_queue.arn
maxReceiveCount = 3
})
}

resource "aws_sqs_queue_policy" "account_space_update_prod_queue_policy" {
queue_url = aws_sqs_queue.account_space_update_prod_queue.id
policy = jsonencode({
Version = "2012-10-17",
Statement = [
{
Effect = "Allow",
Principal = {
Service = "sns.amazonaws.com"
},
Action = "sqs:SendMessage",
Resource = aws_sqs_queue.account_space_update_prod_queue.arn,
Condition = {
ArnEquals = {
"aws:SourceArn" = var.event_topic_arn
}
}
}
]
})
}

resource "aws_sns_topic_subscription" "account_space_update_prod_subscription" {
topic_arn = var.event_topic_arn
protocol = "sqs"
endpoint = aws_sqs_queue.account_space_update_prod_queue.arn
filter_policy = jsonencode({
Entity = ["record"],
Action = ["create", "copy"]
})
}

data "aws_iam_policy_document" "assume_prod_account_space_update_role" {
statement {
effect = "Allow"
principals {
type = "Service"
identifiers = ["lambda.amazonaws.com"]
}
actions = ["sts:AssumeRole"]
}
}

resource "aws_iam_role" "account_space_update_prod_lambda_role" {
name = "account-space-update-prod-lambda-role"
assume_role_policy = data.aws_iam_policy_document.assume_prod_account_space_update_role.json
}

resource "aws_iam_role_policy" "account_space_update_prod_lambda_policy" {
name = "account-space-update-lambda-policy"
role = aws_iam_role.account_space_update_prod_lambda_role.name
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents",
"ec2:CreateNetworkInterface",
"ec2:DescribeNetworkInterfaces",
"ec2:DeleteNetworkInterface",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSubnets",
"sqs:ReceiveMessage",
"sqs:DeleteMessage",
"sqs:GetQueueAttributes",
]
Effect = "Allow"
Resource = ["*", aws_sqs_queue.account_space_update_prod_queue.arn]
},
]
})
}

resource "aws_lambda_function" "account_space_update_prod_lambda" {
package_type = "Image"
image_uri = var.account_space_updater_lambda_image
function_name = "account-space-update-prod-lambda"
role = aws_iam_role.account_space_update_prod_lambda_role.arn
timeout = 30

vpc_config {
security_group_ids = [var.prod_security_group_id]
subnet_ids = var.subnet_ids
}

environment {
variables = {
ENV = var.prod_env
SENTRY_DSN = var.sentry_dsn
DATABASE_URL = var.prod_database_url
}
}
}

resource "aws_lambda_event_source_mapping" "account_space_update_prod_event_source_mapping" {
event_source_arn = aws_sqs_queue.account_space_update_prod_queue.arn
function_name = aws_lambda_function.account_space_update_prod_lambda.arn
batch_size = 10
maximum_batching_window_in_seconds = 0
}
5 changes: 5 additions & 0 deletions terraform/prod_cluster/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ variable "access_copy_lambda_image" {
type = string
}

variable "account_space_updater_lambda_image" {
description = "Tag of the account space updater lambda image to deploy"
type = string
}

variable "prod_security_group_id" {
description = "ID of the Production security group"
type = string
Expand Down
Loading

0 comments on commit 2050786

Please sign in to comment.