feat(lambda-layer): add pipeline to build Lambda layer in v3 (#4826)

* Pipeline for v3

* Pipeline for v3

* Pipeline for v3

* Adding v3 stack

* Changing working directory

* Addressing Andrea's feedback

* Refactoring publish SAR

* Bumping version

* Renaming SAR name

* Addressing Heitor's feedback

* Addressing Heitor's feedback

* Addressing Heitor's feedback

* Addressing Heitor's feedback

.github/workflows/publish_v3_layer.yml

@@ -0,0 +1,314 @@
+name: Deploy v3 layer to all regions
+
+# PROCESS
+#
+# 1. Compile Layer using cdk-aws-lambda-powertools-layer CDK construct for Python3.8-3.12 and x86/ARM architectures (uses custom runner as it's CPU heavy)
+# 2. Kick off pipeline for beta, prod, and canary releases
+# 3. Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged
+# 4. Builds and publishes docs with latest Layer ARN using given version (generally coming from release)
+
+# USAGE
+#
+# NOTE: meant to be used with ./.github/workflows/release-v3.yml
+#
+# publish_layer:
+#   needs: [seal, release, create_tag]
+#   secrets: inherit
+#   permissions:
+#     id-token: write
+#     contents: write
+#     pages: write
+#     pull-requests: write
+#   uses: ./.github/workflows/publish_v2_layer.yml
+#   with:
+#     latest_published_version: ${{ needs.seal.outputs.RELEASE_VERSION }}
+#     pre_release: ${{ inputs.pre_release }}
+#     source_code_artifact_name: ${{ needs.seal.outputs.artifact_name }}
+#     source_code_integrity_hash: ${{ needs.seal.outputs.integrity_hash }}
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      latest_published_version:
+        description: "Latest PyPi published version to rebuild latest docs for, e.g. 3.0.0, 3.0.0a1 (pre-release)"
+        required: true
+      source_code_artifact_name:
+        description: "Artifact name to restore sealed source code"
+        type: string
+        required: true
+      source_code_integrity_hash:
+        description: "Sealed source code integrity hash"
+        type: string
+        required: true
+      pre_release:
+        description: "Publishes documentation using a pre-release tag (3.0.0a1)."
+        default: false
+        type: boolean
+        required: false
+  workflow_call:
+    inputs:
+      latest_published_version:
+        type: string
+        description: "Latest PyPi published version to rebuild latest docs for, e.g. 3.0.0, 3.0.0a1 (pre-release)"
+        required: true
+      pre_release:
+        description: "Publishes documentation using a pre-release tag (3.0.0a1)."
+        default: false
+        type: boolean
+        required: false
+      source_code_artifact_name:
+        description: "Artifact name to restore sealed source code"
+        type: string
+        required: true
+      source_code_integrity_hash:
+        description: "Sealed source code integrity hash"
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+
+env:
+  RELEASE_COMMIT: ${{ github.sha }}
+
+jobs:
+  build-layer:
+    permissions:
+      # lower privilege propagated from parent workflow (release.yml)
+      contents: read
+      id-token: write
+      pages: none
+      pull-requests: none
+    runs-on: aws-powertools_ubuntu-latest_8-core
+    strategy:
+      max-parallel: 5
+      matrix:
+        python-version: ["3.8","3.9","3.10","3.11","3.12"]
+    defaults:
+      run:
+        working-directory: ./layer_v3
+    steps:
+      - name: checkout
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29  # v4.1.6
+        with:
+          ref: ${{ env.RELEASE_COMMIT }}
+
+      - name: Restore sealed source code
+        uses: ./.github/actions/seal-restore
+        with:
+          integrity_hash: ${{ inputs.source_code_integrity_hash }}
+          artifact_name: ${{ inputs.source_code_artifact_name }}
+
+      - name: Install poetry
+        run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0
+      - name: Setup Node.js
+        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+        with:
+          node-version: "18.20.4"
+      - name: Setup python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Resolve and install project dependencies
+        # CDK spawns system python when compiling stack
+        # therefore it ignores both activated virtual env and cached interpreter by GH
+        run: |
+          poetry export --format requirements.txt --output requirements.txt
+          pip install --require-hashes -r requirements.txt
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v2.0.0
+        with:
+          platforms: arm64
+        # NOTE: we need QEMU to build Layer against a different architecture (e.g., ARM)
+
+      - name: Set up Docker Buildx
+        id: builder
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+        with:
+          install: true
+          driver: docker
+          platforms: linux/amd64,linux/arm64
+
+      - name: Install CDK
+        working-directory: ./
+        run: |
+          npm ci
+          npx cdk --version
+
+      # Baking time for PyPi eventual consistency; 60s seemed more than enough
+      # https://github.com/aws-powertools/powertools-lambda-python/issues/2491
+      - name: Baking time (PyPi)
+        run: sleep 60
+
+      - name: CDK build
+        run: npx cdk synth --verbose --context version="${{ inputs.latest_published_version }}" --context pythonVersion="${{ matrix.python-version }}" -o cdk.out
+      - name: zip output
+        run: zip -r cdk.py${{ matrix.python-version }}.out.zip cdk.out
+      - name: Archive CDK artifacts
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: cdk-layer-artifact-py${{ matrix.python-version }}
+          path: layer/cdk.py${{ matrix.python-version }}.out.zip
+
+  beta:
+    needs: build-layer
+    # lower privilege propagated from parent workflow (release.yml)
+    permissions:
+      id-token: write
+      contents: read
+      pages: write             # docs will be updated with latest Layer ARNs
+      pull-requests: write     # creation-action will create a PR with Layer ARN updates
+    uses: ./.github/workflows/reusable_deploy_v3_layer_stack.yml
+    secrets: inherit
+    with:
+      stage: "BETA"
+      environment: "layer-beta"
+      source_code_artifact_name: ${{ inputs.source_code_artifact_name }}
+      source_code_integrity_hash: ${{ inputs.source_code_integrity_hash }}
+
+  prod:
+    needs: beta
+    # lower privilege propagated from parent workflow (release.yml)
+    permissions:
+      id-token: write
+      contents: read
+      pages: write             # docs will be updated with latest Layer ARNs
+      pull-requests: write     # creation-action will create a PR with Layer ARN updates
+    uses: ./.github/workflows/reusable_deploy_v3_layer_stack.yml
+    secrets: inherit
+    with:
+      stage: "PROD"
+      environment: "layer-prod"
+      source_code_artifact_name: ${{ inputs.source_code_artifact_name }}
+      source_code_integrity_hash: ${{ inputs.source_code_integrity_hash }}
+
+  sar-beta:
+    needs: beta  # canaries run on Layer Beta env
+    permissions:
+      # lower privilege propagated from parent workflow (release.yml)
+      id-token: write
+      contents: read
+      pull-requests: none
+      pages: none
+    uses: ./.github/workflows/reusable_deploy_v3_sar.yml
+    secrets: inherit
+    with:
+      stage: "BETA"
+      environment: "layer-beta"
+      package-version: ${{ inputs.latest_published_version }}
+      source_code_artifact_name: ${{ inputs.source_code_artifact_name }}
+      source_code_integrity_hash: ${{ inputs.source_code_integrity_hash }}
+
+
+  sar-prod:
+    needs: sar-beta
+    permissions:
+      # lower privilege propagated from parent workflow (release.yml)
+      id-token: write
+      contents: read
+      pull-requests: none
+      pages: none
+    uses: ./.github/workflows/reusable_deploy_v3_sar.yml
+    secrets: inherit
+    with:
+      stage: "PROD"
+      environment: "layer-prod"
+      package-version: ${{ inputs.latest_published_version }}
+      source_code_artifact_name: ${{ inputs.source_code_artifact_name }}
+      source_code_integrity_hash: ${{ inputs.source_code_integrity_hash }}
+
+
+  # Updating the documentation with the latest Layer ARNs is a two-phase process
+  #
+  # 1. Update layer ARNs with latest deployed locally and create a PR with these changes
+  # 2. Pull from temporary branch with these changes and update the docs we're releasing
+  #
+  # This keeps our permissions tight and we don't run into a conflict,
+  # where a new release creates a new doc (2.16.0) while layers are still pointing to 2.15
+  # because the PR has to be merged while release process is running
+
+  update_v3_layer_arn_docs:
+    needs: prod
+    outputs:
+      temp_branch: ${{ steps.create-pr.outputs.temp_branch }}
+    runs-on: ubuntu-latest
+    permissions:
+      # lower privilege propagated from parent workflow (release.yml)
+      contents: write
+      pull-requests: write
+      id-token: none
+      pages: none
+    steps:
+      - name: Checkout repository # reusable workflows start clean, so we need to checkout again
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29  # v4.1.6
+        with:
+          ref: ${{ env.RELEASE_COMMIT }}
+
+      - name: Restore sealed source code
+        uses: ./.github/actions/seal-restore
+        with:
+          integrity_hash: ${{ inputs.source_code_integrity_hash }}
+          artifact_name: ${{ inputs.source_code_artifact_name }}
+
+      - name: Download CDK layer artifacts
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+        with:
+          path: cdk-layer-stack
+          pattern: cdk-layer-stack-*  # merge all Layer artifacts created per region earlier (reusable_deploy_v2_layer_stack.yml; step "Save Layer ARN artifact")
+          merge-multiple: true
+      - name: Replace layer versions in documentation
+        run: |
+          ls -la cdk-layer-stack/
+          ./layer/scripts/update_layer_arn.sh cdk-layer-stack
+      # NOTE: It felt unnecessary creating yet another PR to update changelog w/ latest tag
+      # since this is the only step in the release where we update docs from a temp branch
+      - name: Update changelog with latest tag
+        run: make changelog
+      - name: Create PR
+        id: create-pr
+        uses: ./.github/actions/create-pr
+        with:
+          files: "docs/index.md examples CHANGELOG.md"
+          temp_branch_prefix: "ci-layer-docs"
+          pull_request_title: "chore(ci): layer docs update"
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+
+  prepare_docs_alias:
+    runs-on: ubuntu-latest
+    permissions:
+      # lower privilege propagated from parent workflow (release.yml)
+      contents: read
+      pages: none
+      id-token: none
+      pull-requests: none
+    outputs:
+      DOCS_ALIAS: ${{ steps.set-alias.outputs.DOCS_ALIAS }}
+    steps:
+      - name: Set docs alias
+        id: set-alias
+        run: |
+          DOCS_ALIAS=latest
+          if [[ "${{ inputs.pre_release }}" == true ]] ; then
+            DOCS_ALIAS=alpha
+          fi
+          echo DOCS_ALIAS="$DOCS_ALIAS" >> "$GITHUB_OUTPUT"
+
+  release_docs:
+    needs: [update_v3_layer_arn_docs, prepare_docs_alias]
+    permissions:
+      # lower privilege propagated from parent workflow (release.yml)
+      contents: write
+      pages: write
+      pull-requests: none
+      id-token: write
+    secrets: inherit
+    uses: ./.github/workflows/reusable_publish_docs.yml
+    with:
+      version: ${{ inputs.latest_published_version }}
+      alias: ${{ needs.prepare_docs_alias.outputs.DOCS_ALIAS }}
+      git_ref: ${{ needs.update_v3_layer_arn_docs.outputs.temp_branch }}