Add infrastructure for access copy lambda

The previous commit added a lambda handler to attach Archivematica-generated access copies to records in Permanent. This commit adds terraform and Github Actions code to deploy this lambda.
PermanentOrg · Nov 9, 2024 · f020f12 · f020f12
1 parent 5514c75
commit f020f12
Show file tree

Hide file tree

Showing 14 changed files with 503 additions and 51 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -84,3 +84,23 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Publish Image to ECR
         run: docker push $THUMBNAIL_REFRESH_IMAGE_TAG
+  build_access_copy_lambda:
+    needs:
+      - generate_image_tags
+    runs-on: ubuntu-20.04
+    env:
+      ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build Image
+        run: docker build -t $ACCESS_COPY_LAMBDA_IMAGE_TAG --build-arg="AWS_RDS_CERT_BUNDLE=$AWS_RDS_CERT_BUNDLE" -f Dockerfile.access_copy_attacher .
+        env:
+          AWS_RDS_CERT_BUNDLE: ${{ secrets.AWS_RDS_CERT_BUNDLE }}
+      - name: AWS Login
+        run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Publish Image to ECR
+        run: docker push $ACCESS_COPY_LAMBDA_IMAGE_TAG
diff --git a/.github/workflows/dev_deploy.yml b/.github/workflows/dev_deploy.yml
@@ -28,6 +28,7 @@ jobs:
       AM_CLEANUP_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.AM_CLEANUP_IMAGE_TAG }}
       RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
+      ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
     defaults:
       run:
         working-directory: ./terraform/test_cluster
@@ -57,10 +58,13 @@ jobs:
           -var="record_thumbnail_staging_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
           -var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
+          -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
           -target=kubernetes_deployment.stela_dev \
           -target=kubernetes_cron_job_v1.archivematica_cleanup_dev \
           -target=aws_lambda_function.record_thumbnail_lambda \
-          -target=kubernetes_cron_job_v1.thumbnail_refresh_dev
+          -target=kubernetes_cron_job_v1.thumbnail_refresh_dev \
+          -target=aws_lambda_function.access_copy_dev_lambda
       - name: Terraform Apply
         run: |
           terraform apply -auto-approve -input=false \
@@ -72,7 +76,10 @@ jobs:
           -var="record_thumbnail_staging_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
           -var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
+          -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
           -target=kubernetes_deployment.stela_dev \
           -target=kubernetes_cron_job_v1.archivematica_cleanup_dev \
           -target=aws_lambda_function.record_thumbnail_lambda \
-          -target=kubernetes_cron_job_v1.thumbnail_refresh_dev
+          -target=kubernetes_cron_job_v1.thumbnail_refresh_dev \
+          -target=aws_lambda_function.access_copy_dev_lambda
diff --git a/.github/workflows/full_test_deploy.yml b/.github/workflows/full_test_deploy.yml
@@ -25,6 +25,7 @@ jobs:
       AM_CLEANUP_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.AM_CLEANUP_IMAGE_TAG }}
       RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
+      ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
     defaults:
       run:
         working-directory: ./terraform/test_cluster
@@ -51,7 +52,9 @@ jobs:
           -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
           -var="record_thumbnail_staging_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
           -var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-          -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG"
+          -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
+          -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
       - name: Terraform Apply
         run: |
           terraform apply -auto-approve -input=false \
@@ -62,4 +65,6 @@ jobs:
           -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
           -var="record_thumbnail_staging_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
           -var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-          -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG"
+          -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
+          -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
diff --git a/.github/workflows/generate_image_tags.yml b/.github/workflows/generate_image_tags.yml
@@ -10,6 +10,8 @@ on:
         value: ${{ jobs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG:
         value: ${{ jobs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
+      ACCESS_COPY_LAMBDA_IMAGE_TAG:
+        value: ${{ jobs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
 jobs:
   generate_image_tags:
     runs-on: ubuntu-20.04
@@ -18,6 +20,7 @@ jobs:
       AM_CLEANUP_IMAGE_TAG: ${{ steps.generate_am_cleanup_image_tag.outputs.AM_CLEANUP_IMAGE_TAG }}
       RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ steps.generate_record_thumbnail_lambda_image_tag.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG: ${{ steps.generate_thumbnail_refresh_image_tag.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
+      ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ steps.generate_access_copy_lambda_image_tag.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
     steps:
       - uses: actions/checkout@v3
       - name: Set ECR domain env var
@@ -43,3 +46,6 @@ jobs:
       - name: Generate Thumbnail Refresh Image Tag
         id: generate_thumbnail_refresh_image_tag
         run: echo "THUMBNAIL_REFRESH_IMAGE_TAG=$ECR_DOMAIN/stela:thumbnail_refresh-$BRANCH_TYPE-$ABBREVIATED_COMMIT_HASH" >> "$GITHUB_OUTPUT"
+      - name: Generate Access Copy Lambda Image Tag
+        id: generate_access_copy_lambda_image_tag
+        run: echo "ACCESS_COPY_LAMBDA_IMAGE_TAG=$ECR_DOMAIN/stela:access_copy_lambda-$BRANCH_TYPE-$ABBREVIATED_COMMIT_HASH" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/prod_deploy.yml b/.github/workflows/prod_deploy.yml
@@ -23,6 +23,7 @@ jobs:
       AM_CLEANUP_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.AM_CLEANUP_IMAGE_TAG }}
       RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
+      ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
     defaults:
       run:
         working-directory: ./terraform/prod_cluster
@@ -45,11 +46,13 @@ jobs:
           -var="stela_image=$API_IMAGE_TAG" \
           -var="archivematica_cleanup_image=$AM_CLEANUP_IMAGE_TAG" \
           -var="record_thumbnail_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
-          -var="thumbnail_refresh_image=$THUMBNAIL_REFRESH_IMAGE_TAG"
+          -var="thumbnail_refresh_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
+          -var="access_copy_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
       - name: Terraform Apply
         run: |
           terraform apply -auto-approve -input=false \
           -var="stela_image=$API_IMAGE_TAG" \
           -var="archivematica_cleanup_image=$AM_CLEANUP_IMAGE_TAG" \
           -var="record_thumbnail_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
-          -var="thumbnail_refresh_image=$THUMBNAIL_REFRESH_IMAGE_TAG"
+          -var="thumbnail_refresh_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
+          -var="access_copy_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
diff --git a/.github/workflows/staging_deploy.yml b/.github/workflows/staging_deploy.yml
@@ -26,6 +26,7 @@ jobs:
       AM_CLEANUP_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.AM_CLEANUP_IMAGE_TAG }}
       RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
+      ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
     defaults:
       run:
         working-directory: ./terraform/test_cluster
@@ -55,10 +56,13 @@ jobs:
           -var="record_thumbnail_staging_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
           -var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
+          -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
           -target=kubernetes_deployment.stela_staging \
           -target=kubernetes_cron_job_v1.archivematica_cleanup_staging \
           -target=aws_lambda_function.record_thumbnail_lambda_staging \
-          -target=kubernetes_cron_job_v1.thumbnail_refresh_staging
+          -target=kubernetes_cron_job_v1.thumbnail_refresh_staging \
+          -target=aws_lambda_function.access_copy_lambda_staging
       - name: Terraform Apply
         run: |
           terraform apply -auto-approve -input=false \
@@ -73,4 +77,5 @@ jobs:
           -target=kubernetes_deployment.stela_staging \
           -target=kubernetes_cron_job_v1.archivematica_cleanup_staging \
           -target=aws_lambda_function.record_thumbnail_lambda_staging \
-          -target=kubernetes_cron_job_v1.thumbnail_refresh_staging
+          -target=kubernetes_cron_job_v1.thumbnail_refresh_staging \
+          -target=aws_lambda_function.access_copy_lambda_staging
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,46 +1,48 @@
 name: Unit tests
 on:
-    push:
-        branches-ignore:
-            - main
-    workflow_dispatch:
-    workflow_call:
+  push:
+    branches-ignore:
+      - main
+  workflow_dispatch:
+  workflow_call:
 jobs:
-    run_tests:
-        runs-on: ubuntu-20.04
-        steps:
-            - uses: actions/checkout@v3
-              with:
-                  path: ./stela
-            - uses: actions/setup-node@v1
-              with:
-                  node-version: "18"
-            - name: Checkout back-end
-              uses: actions/checkout@v3
-              with:
-                  ssh-key: ${{ secrets.BACKEND_ACCESS_PRIVATE_SSH_KEY }}
-                  repository: PermanentOrg/back-end
-                  ref: main
-                  path: ./back-end
-            - name: Checkout devenv
-              uses: actions/checkout@v3
-              with:
-                  ssh-key: ${{ secrets.DEVENV_ACCESS_PRIVATE_SSH_KEY }}
-                  repository: PermanentOrg/devenv
-                  ref: main
-                  path: ./devenv
-            - run: (cd stela; npm install --production=false)
-            - run : (cd stela; npm run build -ws)
-            - run: touch stela/.env
-            - run: touch devenv/.env
-            - run: (cd devenv; docker compose up database_setup -d; docker logs devenv-database_setup-1)
-            - run: (cd stela/packages/api; npm run start-containers)
-            - run: (cd stela/packages/api; docker compose run stela npm run test-ci)
-            - run: (cd stela; npm run test -w @stela/account_space_updater)
-            - uses: codecov/codecov-action@v2
-            - run: (cd stela; npm run test -w @stela/record_thumbnail_attacher)
-            - uses: codecov/codecov-action@v2
-            - run: (cd stela; npm run test -w @stela/archivematica_cleanup)
-            - uses: codecov/codecov-action@v2
-            - run: (cd stela; npm run test -w @stela/thumbnail_refresh)
-            - uses: codecov/codecov-action@v2
+  run_tests:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: ./stela
+      - uses: actions/setup-node@v1
+        with:
+          node-version: "18"
+      - name: Checkout back-end
+        uses: actions/checkout@v3
+        with:
+          ssh-key: ${{ secrets.BACKEND_ACCESS_PRIVATE_SSH_KEY }}
+          repository: PermanentOrg/back-end
+          ref: main
+          path: ./back-end
+      - name: Checkout devenv
+        uses: actions/checkout@v3
+        with:
+          ssh-key: ${{ secrets.DEVENV_ACCESS_PRIVATE_SSH_KEY }}
+          repository: PermanentOrg/devenv
+          ref: main
+          path: ./devenv
+      - run: (cd stela; npm install --production=false)
+      - run: (cd stela; npm run build -ws)
+      - run: touch stela/.env
+      - run: touch devenv/.env
+      - run: (cd devenv; docker compose up database_setup -d; docker logs devenv-database_setup-1)
+      - run: (cd stela/packages/api; npm run start-containers)
+      - run: (cd stela/packages/api; docker compose run stela npm run test-ci)
+      - run: (cd stela; npm run test -w @stela/account_space_updater)
+      - uses: codecov/codecov-action@v2
+      - run: (cd stela; npm run test -w @stela/record_thumbnail_attacher)
+      - uses: codecov/codecov-action@v2
+      - run: (cd stela; npm run test -w @stela/archivematica_cleanup)
+      - uses: codecov/codecov-action@v2
+      - run: (cd stela; npm run test -w @stela/thumbnail_refresh)
+      - uses: codecov/codecov-action@v2
+      - run: (cd stela; npm run test -w @stela/access_copy_attacher)
+      - uses: codecov/codecov-action@v2
diff --git a/Dockerfile.access_copy_attacher b/Dockerfile.access_copy_attacher
@@ -26,9 +26,16 @@ COPY --from=builder /usr/local/apps/stela/packages/access_copy_attacher/dist ./p
 COPY --from=builder /usr/local/apps/stela/packages/access_copy_attacher/package.json ./packages/access_copy_attacher/package.json
 COPY --from=builder /usr/local/apps/stela/packages/logger/dist ./packages/logger/dist
 COPY --from=builder /usr/local/apps/stela/packages/logger/package.json ./packages/logger/package.json
+COPY --from=builder /usr/local/apps/stela/packages/s3-utils/dist ./packages/s3-utils/dist
+COPY --from=builder /usr/local/apps/stela/packages/s3-utils/package.json ./packages/s3-utils/package.json
+COPY --from=builder /usr/local/apps/stela/packages/archivematica-utils/dist ./packages/archivematica-utils/dist
+COPY --from=builder /usr/local/apps/stela/packages/archivematica-utils/package.json ./packages/archivematica-utils/package.json
+COPY --from=builder /usr/local/apps/stela/packages/file-utils/dist ./packages/file-utils/dist
+COPY --from=builder /usr/local/apps/stela/packages/file-utils/package.json ./packages/file-utils/package.json
 COPY --from=builder /usr/local/apps/stela/package.json ./package.json
 COPY --from=builder /usr/local/apps/stela/package-lock.json ./package-lock.json
 
+RUN npm cache clean --force
 RUN npm install -g npm@8.19.3
 RUN npm install --workspace @stela/access_copy_attacher
 

diff --git a/Dockerfile.record_thumbnail_attacher b/Dockerfile.record_thumbnail_attacher
@@ -26,6 +26,10 @@ COPY --from=builder /usr/local/apps/stela/packages/record_thumbnail_attacher/dis
 COPY --from=builder /usr/local/apps/stela/packages/record_thumbnail_attacher/package.json ./packages/record_thumbnail_attacher/package.json
 COPY --from=builder /usr/local/apps/stela/packages/logger/dist ./packages/logger/dist
 COPY --from=builder /usr/local/apps/stela/packages/logger/package.json ./packages/logger/package.json
+COPY --from=builder /usr/local/apps/stela/packages/s3-utils/dist ./packages/s3-utils/dist
+COPY --from=builder /usr/local/apps/stela/packages/s3-utils/package.json ./packages/s3-utils/package.json
+COPY --from=builder /usr/local/apps/stela/packages/archivematica-utils/dist ./packages/archivematica-utils/dist
+COPY --from=builder /usr/local/apps/stela/packages/archivematica-utils/package.json ./packages/archivematica-utils/package.json
 COPY --from=builder /usr/local/apps/stela/package.json ./package.json
 COPY --from=builder /usr/local/apps/stela/package-lock.json ./package-lock.json