From d2a091eca97c4ca4b2b6efe8c4a69866e56e7efd Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 14:32:06 -0400
Subject: [PATCH 01/27] Streamlines Chainlink Image Builds

---
 .github/workflows/run-e2e-tests.yml | 281 ++++++++++++----------------
 1 file changed, 118 insertions(+), 163 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index d7303ff9..2ff79376 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -19,6 +19,19 @@ on:
           develop or commit sha"
         required: false
         type: string
+      chainlink_image_name:
+        description:
+          "Enter Chainlink image name to use for the tests. Example: 'chainlink'
+          or 'ccip'"
+        required: false
+        type: string
+        default: "chainlink"
+      test_chainlink_image_type:
+        description:
+          "Run tests by type of Chainlink image used. Example: 'amd64' or
+          'plugins-arm64'"
+        required: false
+        type: string
       chainlink_upgrade_version:
         description:
           "Enter Chainlink version to use for the upgrade tests. Example:
@@ -218,12 +231,15 @@ on:
         required: false
 
 env:
+  CHAINLINK_IMAGE_BASE:
+    ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION
+    }}.amazonaws.com
   CHAINLINK_IMAGE:
     ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION
-    }}.amazonaws.com/chainlink
+    }}.amazonaws.com/${{ inputs.chainlink_image_name || 'chainlink' }}
   QA_CHAINLINK_IMAGE:
     ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION
-    }}.amazonaws.com/chainlink
+    }}.amazonaws.com/${{ inputs.chainlink_image_name || 'chainlink' }}
   DEFAULT_CHAINLINK_VERSION: ${{ inputs.chainlink_version }}
   DEFAULT_CHAINLINK_PLUGINS_VERSION:
     ${{ inputs.chainlink_version != '' && format('{0}-plugins',
@@ -244,56 +260,20 @@ jobs:
   validate-inputs:
     name: Validate workflow inputs
     runs-on: ubuntu-latest
-    outputs:
-      require_chainlink_image_versions_in_qa_ecr_matrix:
-        ${{ steps.set-required-chainlink-image-versions-matrix.outputs.versions
-        }}
-      require_chainlink_plugin_versions_in_qa_ecr_matrix:
-        ${{ steps.set-required-chainlink-plugin-versions-matrix.outputs.versions
-        }}
     steps:
       - name: Check input conditions
         run: |
-          if [[ "${{ inputs.test_ids }}" != "" && "${{ inputs.test_trigger }}" != "" ]]; then
-            echo "::error::Error: Both 'test_ids' and 'test_trigger' are provided. Please specify only one."
+          if [[ "${{ inputs.test_ids }}" != "" && "${{ inputs.test_workflow }}" != "" ]]; then
+            echo "::error::Error: Both 'test_ids' and 'test_workflow' are provided. Please specify only one."
+            exit 1
+          fi
+          if [[ "${{ inputs.test_chainlink_image_type }}" != "" && "${{ inputs.chainlink_version }}" != "" ]]; then
+            echo "::error::Error: Both 'test_chainlink_image_type' and 'chainlink_version' are provided. Please specify only one."
             exit 1
           fi
           if [[ "${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}" != "" ]]; then
             echo "Will run tests with custom test secrets"
           fi
-      - name: Install jq
-        run: sudo apt-get install jq
-
-      - name: Create matrix for required Chainlink image versions
-        id: set-required-chainlink-image-versions-matrix
-        shell: bash
-        run: |
-          image_versions="${{ inputs.require_chainlink_image_versions_in_qa_ecr }}"
-          default_version="${{ env.DEFAULT_CHAINLINK_VERSION }}"
-          current_sha="${{ github.sha }}"
-
-          if [[ "$default_version" == "$current_sha" ]]; then
-            # Append the current SHA to required image versions
-            if [[ -z "$image_versions" ]]; then
-              image_versions="${{ github.sha }}"
-            else
-              image_versions+=",${{ github.sha }}"
-            fi            
-          fi
-
-          # Convert the comma-separated string to a JSON array
-          image_versions=$(echo "$image_versions" | jq -Rc 'if . == "" then "" else split(",") | if . == [""] then "" else . end end')
-
-          echo "Required Chainlink image versions: $image_versions"   
-          echo "versions=$image_versions" >> "$GITHUB_OUTPUT"
-
-      - name: Create matrix for required Chainlink plugin versions
-        id: set-required-chainlink-plugin-versions-matrix
-        shell: bash
-        run: |
-          image_versions=$(echo "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" | jq -Rc 'if . == "" then "" else split(",") | if . == [""] then "" else . end end')
-          echo "Required Chainlink plugin image versions: $image_versions"   
-          echo "versions=$image_versions" >> "$GITHUB_OUTPUT"
 
   check-test-configurations:
     name: Check test configurations
@@ -306,7 +286,7 @@ jobs:
       - name: Install citool
         shell: bash
         run: go install
-          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@83100a879006dde55ace09a5dfd99b37e62f5a3f # v1.34.4
+          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@95212946b92c2bacb36dc9382b6db62d4db3a24d # DEBUG: Testing version
       - name: Run Check Tests Command
         run: |
           if ! citool check-tests ${{ github.workspace }}/${{ inputs.check_test_path }} ${{ github.workspace }}/${{ inputs.test_path }}; then
@@ -334,8 +314,8 @@ jobs:
           exit 1
           fi
 
-  load-test-configurations:
-    name: Load test configurations
+  build-test-configurations:
+    name: Build Test Configurations
     needs: [validate-inputs]
     runs-on: ubuntu-latest
     outputs:
@@ -344,6 +324,8 @@ jobs:
       docker-matrix: ${{ steps.set-docker-matrix.outputs.matrix }}
       k8s-runner-matrix: ${{ steps.set-k8s-runner-matrix.outputs.matrix }}
       workflow_id: ${{ steps.gen_id.outputs.workflow_id }}
+      chainlink_image_tag:
+        ${{ steps.determine-chainlink-image-tag.outputs.chainlink_image_tag }}
     steps:
       - name: Checkout code
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
@@ -355,10 +337,30 @@ jobs:
       - name: Install citool
         shell: bash
         run: go install
-          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@83100a879006dde55ace09a5dfd99b37e62f5a3f # v1.34.4
+          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@95212946b92c2bacb36dc9382b6db62d4db3a24d # DEBUG: Testing version
       - name: Install jq
         run: sudo apt-get install jq
 
+      - name: Get Chainlink Image Tag
+        id: get-chainlink-image-tag
+        uses: ./.github/actions/get-chainlink-image-tag
+      - name: Determine Chainlink Image Tag
+        id: determine-chainlink-image-tag
+        run: |
+          chainlink_image_tag=""
+          if [[ "${{ inputs.chainlink_version}}" != "" ]]; then
+            echo "'chainlink_version' provided"
+            chainlink_image_tag=${{ inputs.chainlink_version }}
+          else
+            echo "determining chainlink image tag based on workflow event"
+            chainlink_image_tag=${{ steps.get-chainlink-image-tag.outputs.base-image-tag }}-${{ inputs.test_chainlink_image_type }}
+          fi
+
+          echo "chainlink_image_tag=$chainlink_image_tag"
+          echo "chainlink_image_tag=$chainlink_image_tag" >> $GITHUB_OUTPUT
+          echo "## Chainlink Image" >> $GITHUB_STEP_SUMMARY
+          echo "${{ env.CHAINLINK_IMAGE }}:$chainlink_image_tag" >> $GITHUB_STEP_SUMMARY
+
       - name: Generate Docker Tests Matrix
         id: set-docker-matrix
         shell: bash
@@ -369,7 +371,7 @@ jobs:
             MATRIX_JSON=$(echo '${{ inputs.custom_test_list_json }}' | jq -c '{tests: [.tests[] | select(.test_env_type == "docker")]}')
           else
             echo "Using default test list"
-            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'docker' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_trigger }}')
+            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'docker' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_trigger }}' --chainlink-image-type '${{ inputs.test_chainlink_image_type }})
           fi
 
           echo "Docker tests:"
@@ -386,7 +388,7 @@ jobs:
             MATRIX_JSON=$(echo '${{ inputs.custom_test_list_json }}' | jq -c '{tests: [.tests[] | select(.test_env_type == "k8s-remote-runner")]}')
           else
             echo "Using default test list"
-            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'k8s-remote-runner' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_trigger }}')
+            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'k8s-remote-runner' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_trigger }}' --chainlink-image-type '${{ inputs.test_chainlink_image_type }})
           fi
 
           echo "K8s tests:"
@@ -458,98 +460,62 @@ jobs:
         shell: bash
         run: echo "workflow_id=$(uuidgen)" >> "$GITHUB_OUTPUT"
 
-  # Check if Chainlink images required for the tests exist. If not, build and push the images to QA ECR
-  require-chainlink-image-versions-in-qa-ecr:
-    name: Get Chainlink image
-    needs: [validate-inputs, load-test-configurations]
-    if:
-      ${{
-      fromJson(needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix)
-      != '' }}
-    runs-on: ubuntu-latest
+  wait-for-chainlink-image:
+    name: Wait for Chainlink Image
     environment: integration
-    permissions:
-      id-token: write
-      contents: read
-    env:
-      CHAINLINK_IMAGE:
-        ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION
-        }}.amazonaws.com/chainlink
-    strategy:
-      matrix:
-        version:
-          ${{
-          fromJson(needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix)
-          }}
-    steps:
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-
-      - name: Get Chainlink image
-        uses: ./.github/actions/build-chainlink-image
-        with:
-          dockerfile: core/chainlink.Dockerfile
-          git_commit_sha: ${{ matrix.version }}
-          tag_suffix: ""
-          check_image_exists: "true"
-          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-
-  # Check if Chainlink plugins required for the tests exist. If not, build and push the images to QA ECR
-  require-chainlink-plugin-versions-in-qa-ecr:
-    name: Get Chainlink plugins image
-    needs: [validate-inputs, load-test-configurations]
-    if:
-      ${{
-      fromJson(needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix)
-      != '' }}
     runs-on: ubuntu-latest
-    environment: integration
-    permissions:
-      id-token: write
-      contents: read
+    needs: [build-test-configurations]
     env:
-      CHAINLINK_IMAGE:
-        ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION
-        }}.amazonaws.com/chainlink
-    strategy:
-      matrix:
-        version:
-          ${{
-          fromJson(needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix)
-          }}
+      POLL_INTERVAL: 10
+      TIMEOUT: 600
     steps:
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-
-      - name: Get Chainlink plugins image
-        uses: ./.github/actions/build-chainlink-image
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
-          dockerfile: plugins/chainlink.Dockerfile
-          git_commit_sha: ${{ matrix.version }}
-          tag_suffix: "-plugins"
-          check_image_exists: "true"
-          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          role-to-assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.QA_AWS_REGION }}
+          mask-aws-account-id: true
+          role-session-name: e2e-tests-wait-for-chainlink-image
+      - name: Wait for Chainlink Image
+        run: |
+          START_TIME=$(date +%s)
+          echo "Checking for '${{ env.CHAINLINK_IMAGE }}' in ECR..."
+          while true; do
+            IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?contains(imageTags, `${{ needs.build-test-configurations.outputs.chainlink_image_tag }}`)].imageTags[]'' --output text)
+            if [[ "$IMAGE" == "$IMAGE_TAG" ]]; then
+                echo "Found tag!"
+                exit 0
+            else
+                echo "Not found, waiting $POLL_INTERVAL seconds..."
+            fi
+            
+            CURRENT_TIME=$(date +%s)
+            ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+            if [[ "$ELAPSED_TIME" -ge "$TIMEOUT" ]]; then
+                echo "Timeout reached after $TIMEOUT seconds."
+                exit 1
+            fi
+            
+            sleep "$POLL_INTERVAL"
+          done
 
   # Run Docker tests
   run-docker-tests:
-    name: ${{ matrix.tests.id }}
+    name: ${{ matrix.tests.name || matrix.tests.id }}
     needs:
       [
-        load-test-configurations,
-        require-chainlink-image-versions-in-qa-ecr,
-        require-chainlink-plugin-versions-in-qa-ecr,
+        build-test-configurations,
         get_latest_chainlink_release_version,
+        wait-for-chainlink-image,
       ]
     # Run when none of the needed jobs fail or are cancelled (skipped or successful jobs are ok)
     if:
-      ${{ needs.load-test-configurations.outputs.run-docker-tests == 'true' &&
+      ${{ needs.build-test-configurations.outputs.run-docker-tests == 'true' &&
       always() && !failure() && !cancelled() }}
     runs-on: ${{ matrix.tests.runs_on }}
     strategy:
       fail-fast: false
-      matrix: ${{fromJson(needs.load-test-configurations.outputs.docker-matrix)}}
+      matrix: ${{fromJson(needs.build-test-configurations.outputs.docker-matrix)}}
     environment: integration
     permissions:
       actions: read
@@ -565,7 +531,9 @@ jobs:
       TEST_CONFIG_OVERRIDE_PATH:
         ${{ matrix.tests.test_config_override_path ||
         inputs.test_config_override_path }}
-      TEST_ID: ${{ matrix.tests.id_sanitized || matrix.tests.id }}
+      TEST_ID:
+        ${{ matrix.tests.id_sanitized || matrix.tests.id }}${{
+        inputs.test_chainlink_image_type }}
     steps:
       - name: Collect Metrics
         if: always()
@@ -585,24 +553,11 @@ jobs:
       - name: Install jq
         run: sudo apt-get install -y jq
 
-      - name: Show test config override path in summary
-        if: ${{ env.TEST_CONFIG_OVERRIDE_PATH }}
-        shell: bash
-        run: |
-          echo "### Test config override path" >> "$GITHUB_STEP_SUMMARY"
-          echo "[${{ env.TEST_CONFIG_OVERRIDE_PATH }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ env.TEST_CONFIG_OVERRIDE_PATH }})" >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Show chainlink version in summary
-        if:
-          ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION ||
-          env.DEFAULT_CHAINLINK_VERSION }}
-        shell: bash
+      - name: Show Test Configuration
         run: |
-          echo "### Chainlink version" >> "$GITHUB_STEP_SUMMARY"
-          echo "${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}" >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Show test configuration in logs
-        run: echo '${{ toJson(matrix.tests) }}' | jq .
+          echo "Test Confgi Override Path: ${{ env.TEST_CONFIG_OVERRIDE_PATH }}"
+          echo "Test Configuration"  
+          echo '${{ toJson(matrix.tests) }}' | jq .
 
       - name: Setup GAP for Grafana
         uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
@@ -647,12 +602,11 @@ jobs:
         id: run_tests
         uses: smartcontractkit/.github/actions/ctf-run-tests@b8731364b119e88983e94b0c4da87fc27ddb41b8 # ctf-run-tests@0.0.0
         env:
-          DETACH_RUNNER: true
           E2E_TEST_CHAINLINK_VERSION:
-            ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION ||
-            env.DEFAULT_CHAINLINK_VERSION }}
+            ${{ needs.build-test-configurations.outputs.chainlink_image_tag }}
           E2E_TEST_CHAINLINK_UPGRADE_VERSION:
-            ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION }}
+            ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION ||
+            needs.build-test-configurations.outputs.chainlink_image_tag }}
           E2E_TEST_CHAINLINK_POSTGRES_VERSION:
             ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_POSTGRES_VERSION
             }}
@@ -691,7 +645,8 @@ jobs:
             ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_IMAGE ||
             env.CHAINLINK_IMAGE }}
           default_e2e_test_chainlink_upgrade_image:
-            ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_IMAGE }}
+            ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_IMAGE ||
+            env.CHAINLINK_IMAGE}}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: ${{ env.TEST_ID }}-test-logs
           artifacts_location: |
@@ -766,7 +721,7 @@ jobs:
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name:
-            test_result_${{ needs.load-test-configurations.outputs.workflow_id
+            test_result_${{ needs.build-test-configurations.outputs.workflow_id
             }}_${{ env.TEST_ID }}
           path: test_result.json
           retention-days: 1
@@ -776,8 +731,8 @@ jobs:
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name:
-            custom_test_artifacts_${{ env.TEST_ID }}_${{
-            needs.load-test-configurations.outputs.workflow_id }}
+            custom_test_artifacts_${{ env.TEST_ID }}${{
+            needs.build-test-configurations.outputs.workflow_id }}
           path: ${{ matrix.tests.test_artifacts_on_failure }}
           retention-days: 1
 
@@ -787,12 +742,12 @@ jobs:
 
   # Run K8s tests using old remote runner
 
-  get-remote-runner-test-image:
-    needs: [load-test-configurations]
+  prepare-remote-runner-test-image:
+    needs: [build-test-configurations]
     if:
-      ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' &&
+      ${{ needs.build-test-configurations.outputs.run-k8s-tests == 'true' &&
       always() && !failure() && !cancelled() }}
-    name: Get remote runner test image
+    name: Prepare remote runner test image
     runs-on: ubuntu-latest
     environment: integration
     permissions:
@@ -835,20 +790,20 @@ jobs:
   run-k8s-runner-tests:
     needs:
       [
-        load-test-configurations,
-        get-remote-runner-test-image,
+        build-test-configurations,
+        prepare-remote-runner-test-image,
         require-chainlink-image-versions-in-qa-ecr,
         require-chainlink-plugin-versions-in-qa-ecr,
         get_latest_chainlink_release_version,
       ]
     if:
-      ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' &&
+      ${{ needs.build-test-configurations.outputs.run-k8s-tests == 'true' &&
       always() && !failure() && !cancelled() }}
     name: ${{ matrix.tests.id }}
     runs-on: ${{ matrix.tests.runs_on }}
     strategy:
       fail-fast: false
-      matrix: ${{fromJson(needs.load-test-configurations.outputs.k8s-runner-matrix)}}
+      matrix: ${{fromJson(needs.build-test-configurations.outputs.k8s-runner-matrix)}}
     environment: integration
     permissions:
       actions: read
@@ -899,9 +854,9 @@ jobs:
 
       - name: Show remote runner version in summary
         run: |
-          echo "Remote Runner Version: ${{ needs.get-remote-runner-test-image.outputs.remote-runner-version }}"
+          echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
           echo "### Remote Runner Version" >> "$GITHUB_STEP_SUMMARY"
-          echo "${{ needs.get-remote-runner-test-image.outputs.remote-runner-version }}" >> "$GITHUB_STEP_SUMMARY"
+          echo "${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Show test configuration in logs
         run: echo '${{ toJson(matrix.tests) }}' | jq .
@@ -995,7 +950,7 @@ jobs:
         with:
           name:
             custom_test_artifacts_${{ env.TEST_ID }}_${{
-            needs.load-test-configurations.outputs.workflow_id }}
+            needs.build-test-configurations.outputs.workflow_id }}
           path: ${{ matrix.tests.test_artifacts_on_failure }}
           retention-days: 1
 
@@ -1004,7 +959,7 @@ jobs:
         uses: smartcontractkit/.github/actions/ctf-show-grafana-in-test-summary@b6e37806737eef87e8c9137ceeb23ef0bff8b1db # ctf-show-grafana-in-test-summary@0.1.0
 
   after_tests:
-    needs: [load-test-configurations, run-docker-tests, run-k8s-runner-tests]
+    needs: [build-test-configurations, run-docker-tests, run-k8s-runner-tests]
     if: always()
     name: After tests
     runs-on: ubuntu-latest
@@ -1018,7 +973,7 @@ jobs:
         with:
           path: test_results
           pattern:
-            test_result_${{ needs.load-test-configurations.outputs.workflow_id
+            test_result_${{ needs.build-test-configurations.outputs.workflow_id
             }}_*
 
       - name: Set detailed test results

From be6b6742031b65af86a2a33b049fbdc46048fc97 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 14:38:37 -0400
Subject: [PATCH 02/27] Remove dead steps

---
 .github/workflows/run-e2e-tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 2ff79376..8c53129b 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -792,8 +792,7 @@ jobs:
       [
         build-test-configurations,
         prepare-remote-runner-test-image,
-        require-chainlink-image-versions-in-qa-ecr,
-        require-chainlink-plugin-versions-in-qa-ecr,
+        wait-for-chainlink-image,
         get_latest_chainlink_release_version,
       ]
     if:

From a7123f248d57eeb82c2ed724a3a4c474e0d0e4da Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 14:51:44 -0400
Subject: [PATCH 03/27] Missing '

---
 .github/workflows/run-e2e-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 8c53129b..90c82192 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -371,7 +371,7 @@ jobs:
             MATRIX_JSON=$(echo '${{ inputs.custom_test_list_json }}' | jq -c '{tests: [.tests[] | select(.test_env_type == "docker")]}')
           else
             echo "Using default test list"
-            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'docker' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_trigger }}' --chainlink-image-type '${{ inputs.test_chainlink_image_type }})
+            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'docker' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_trigger }}' --chainlink-image-type '${{ inputs.test_chainlink_image_type }}')
           fi
 
           echo "Docker tests:"
@@ -388,7 +388,7 @@ jobs:
             MATRIX_JSON=$(echo '${{ inputs.custom_test_list_json }}' | jq -c '{tests: [.tests[] | select(.test_env_type == "k8s-remote-runner")]}')
           else
             echo "Using default test list"
-            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'k8s-remote-runner' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_trigger }}' --chainlink-image-type '${{ inputs.test_chainlink_image_type }})
+            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'k8s-remote-runner' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_trigger }}' --chainlink-image-type '${{ inputs.test_chainlink_image_type }}')
           fi
 
           echo "K8s tests:"

From 6316ef7ed3d9152e091018263b6e880b1a417f1c Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 15:16:53 -0400
Subject: [PATCH 04/27] Update CI tool

---
 .github/workflows/run-e2e-tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 90c82192..7a48ee1c 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -286,7 +286,7 @@ jobs:
       - name: Install citool
         shell: bash
         run: go install
-          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@95212946b92c2bacb36dc9382b6db62d4db3a24d # DEBUG: Testing version
+          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@b0d1a6bb6118fd2c024ffb0cb4c7d9d5b84b5e97 # DEBUG: Testing version
       - name: Run Check Tests Command
         run: |
           if ! citool check-tests ${{ github.workspace }}/${{ inputs.check_test_path }} ${{ github.workspace }}/${{ inputs.test_path }}; then
@@ -337,7 +337,7 @@ jobs:
       - name: Install citool
         shell: bash
         run: go install
-          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@95212946b92c2bacb36dc9382b6db62d4db3a24d # DEBUG: Testing version
+          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@b0d1a6bb6118fd2c024ffb0cb4c7d9d5b84b5e97 # DEBUG: Testing version
       - name: Install jq
         run: sudo apt-get install jq
 
@@ -359,7 +359,7 @@ jobs:
           echo "chainlink_image_tag=$chainlink_image_tag"
           echo "chainlink_image_tag=$chainlink_image_tag" >> $GITHUB_OUTPUT
           echo "## Chainlink Image" >> $GITHUB_STEP_SUMMARY
-          echo "${{ env.CHAINLINK_IMAGE }}:$chainlink_image_tag" >> $GITHUB_STEP_SUMMARY
+          echo "${{ env.CHAINLINK_IMAGE }}:`$chainlink_image_tag`" >> $GITHUB_STEP_SUMMARY
 
       - name: Generate Docker Tests Matrix
         id: set-docker-matrix

From 4afab2300055e8e90c665096b8b480ce6bc8bcb8 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 15:22:09 -0400
Subject: [PATCH 05/27] Workflow to trigger

---
 .github/workflows/run-e2e-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 7a48ee1c..6ae7b146 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -371,7 +371,7 @@ jobs:
             MATRIX_JSON=$(echo '${{ inputs.custom_test_list_json }}' | jq -c '{tests: [.tests[] | select(.test_env_type == "docker")]}')
           else
             echo "Using default test list"
-            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'docker' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_trigger }}' --chainlink-image-type '${{ inputs.test_chainlink_image_type }}')
+            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'docker' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --trigger '${{ inputs.test_trigger }}' --chainlink-image-type '${{ inputs.test_chainlink_image_type }}')
           fi
 
           echo "Docker tests:"
@@ -388,7 +388,7 @@ jobs:
             MATRIX_JSON=$(echo '${{ inputs.custom_test_list_json }}' | jq -c '{tests: [.tests[] | select(.test_env_type == "k8s-remote-runner")]}')
           else
             echo "Using default test list"
-            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'k8s-remote-runner' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_trigger }}' --chainlink-image-type '${{ inputs.test_chainlink_image_type }}')
+            MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/${{ inputs.test_path }} --test-env-type 'k8s-remote-runner' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --trigger '${{ inputs.test_trigger }}' --chainlink-image-type '${{ inputs.test_chainlink_image_type }}')
           fi
 
           echo "K8s tests:"

From 68e2b531ed4cd3f61fc95afd0ed5474530b382f1 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 15:26:38 -0400
Subject: [PATCH 06/27] Fix extra '

---
 .github/workflows/run-e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 6ae7b146..cade4bbe 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -481,7 +481,7 @@ jobs:
           START_TIME=$(date +%s)
           echo "Checking for '${{ env.CHAINLINK_IMAGE }}' in ECR..."
           while true; do
-            IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?contains(imageTags, `${{ needs.build-test-configurations.outputs.chainlink_image_tag }}`)].imageTags[]'' --output text)
+            IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?contains(imageTags, `${{ needs.build-test-configurations.outputs.chainlink_image_tag }}`)].imageTags[]')
             if [[ "$IMAGE" == "$IMAGE_TAG" ]]; then
                 echo "Found tag!"
                 exit 0

From 3a50c8a3d9b3997a8379a585c6d7a6a54860c489 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 15:31:07 -0400
Subject: [PATCH 07/27] Linting

---
 .github/workflows/run-e2e-tests.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index cade4bbe..3f1988d7 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -263,8 +263,8 @@ jobs:
     steps:
       - name: Check input conditions
         run: |
-          if [[ "${{ inputs.test_ids }}" != "" && "${{ inputs.test_workflow }}" != "" ]]; then
-            echo "::error::Error: Both 'test_ids' and 'test_workflow' are provided. Please specify only one."
+          if [[ "${{ inputs.test_ids }}" != "" && "${{ inputs.test_trigger }}" != "" ]]; then
+            echo "::error::Error: Both 'test_ids' and 'test_trigger' are provided. Please specify only one."
             exit 1
           fi
           if [[ "${{ inputs.test_chainlink_image_type }}" != "" && "${{ inputs.chainlink_version }}" != "" ]]; then
@@ -357,9 +357,9 @@ jobs:
           fi
 
           echo "chainlink_image_tag=$chainlink_image_tag"
-          echo "chainlink_image_tag=$chainlink_image_tag" >> $GITHUB_OUTPUT
-          echo "## Chainlink Image" >> $GITHUB_STEP_SUMMARY
-          echo "${{ env.CHAINLINK_IMAGE }}:`$chainlink_image_tag`" >> $GITHUB_STEP_SUMMARY
+          echo "chainlink_image_tag=$chainlink_image_tag" >> "$GITHUB_OUTPUT"
+          echo "## Chainlink Image" >> "$GITHUB_STEP_SUMMARY"
+          echo "${{ env.CHAINLINK_IMAGE }}:`$chainlink_image_tag`" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Generate Docker Tests Matrix
         id: set-docker-matrix
@@ -872,7 +872,8 @@ jobs:
           ENV_JOB_IMAGE:
             ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
             secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{
-            needs.get-remote-runner-test-image.outputs.remote-runner-version }}
+            needs.prepare-remote-runner-test-image.outputs.remote-runner-version
+            }}
           INTERNAL_DOCKER_REPO:
             ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
             secrets.QA_AWS_REGION }}.amazonaws.com

From ba22dbd4f0889a73cf579da0a1739650581b9424 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 15:40:39 -0400
Subject: [PATCH 08/27] Darn backticks

---
 .github/workflows/run-e2e-tests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 3f1988d7..a6d1a2d4 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -358,8 +358,8 @@ jobs:
 
           echo "chainlink_image_tag=$chainlink_image_tag"
           echo "chainlink_image_tag=$chainlink_image_tag" >> "$GITHUB_OUTPUT"
-          echo "## Chainlink Image" >> "$GITHUB_STEP_SUMMARY"
-          echo "${{ env.CHAINLINK_IMAGE }}:`$chainlink_image_tag`" >> "$GITHUB_STEP_SUMMARY"
+          echo "### Chainlink Image" >> "$GITHUB_STEP_SUMMARY"
+          echo "${{ env.CHAINLINK_IMAGE }}:\`$chainlink_image_tag\`" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Generate Docker Tests Matrix
         id: set-docker-matrix
@@ -479,9 +479,9 @@ jobs:
       - name: Wait for Chainlink Image
         run: |
           START_TIME=$(date +%s)
-          echo "Checking for '${{ env.CHAINLINK_IMAGE }}' in ECR..."
+          echo "Checking for '${{ env.CHAINLINK_IMAGE }}:${{ needs.build-test-configurations.outputs.chainlink_image_tag }}' in ECR..."
           while true; do
-            IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?contains(imageTags, `${{ needs.build-test-configurations.outputs.chainlink_image_tag }}`)].imageTags[]')
+            IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]')
             if [[ "$IMAGE" == "$IMAGE_TAG" ]]; then
                 echo "Found tag!"
                 exit 0

From 639132124927b61668f970e2fcebd982813be97c Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 15:47:39 -0400
Subject: [PATCH 09/27] Fix ecr error

---
 .github/workflows/run-e2e-tests.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index a6d1a2d4..440ae2ed 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -481,12 +481,13 @@ jobs:
           START_TIME=$(date +%s)
           echo "Checking for '${{ env.CHAINLINK_IMAGE }}:${{ needs.build-test-configurations.outputs.chainlink_image_tag }}' in ECR..."
           while true; do
-            IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]')
+            IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]' --output text 2>/dev/null)
             if [[ "$IMAGE" == "$IMAGE_TAG" ]]; then
                 echo "Found tag!"
                 exit 0
             else
-                echo "Not found, waiting $POLL_INTERVAL seconds..."
+                echo "Command Output: $IMAGE"
+                echo "Image not found, waiting $POLL_INTERVAL seconds..."
             fi
             
             CURRENT_TIME=$(date +%s)

From bfe49a93ce4760aad90416a20a607f56c6a4ae8b Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 16:07:22 -0400
Subject: [PATCH 10/27] Look for actual image tag

---
 .github/workflows/run-e2e-tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 440ae2ed..86f833d9 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -482,7 +482,8 @@ jobs:
           echo "Checking for '${{ env.CHAINLINK_IMAGE }}:${{ needs.build-test-configurations.outputs.chainlink_image_tag }}' in ECR..."
           while true; do
             IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]' --output text 2>/dev/null)
-            if [[ "$IMAGE" == "$IMAGE_TAG" ]]; then
+            if [[ "$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}" ]]; then
+                echo "$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}"
                 echo "Found tag!"
                 exit 0
             else

From 5e5a9c6c659a56d487d2450e3dd1e0ce164c9ef3 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 16:16:20 -0400
Subject: [PATCH 11/27] Smoother image output

---
 .github/workflows/run-e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 86f833d9..be89d4ab 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -482,12 +482,12 @@ jobs:
           echo "Checking for '${{ env.CHAINLINK_IMAGE }}:${{ needs.build-test-configurations.outputs.chainlink_image_tag }}' in ECR..."
           while true; do
             IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]' --output text 2>/dev/null)
+            echo "Command Output: $IMAGE"
             if [[ "$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}" ]]; then
                 echo "$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}"
                 echo "Found tag!"
                 exit 0
             else
-                echo "Command Output: $IMAGE"
                 echo "Image not found, waiting $POLL_INTERVAL seconds..."
             fi
             

From fad3424d769cd055cc77e54d7afa1618c8de1759 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 24 Sep 2024 16:29:00 -0400
Subject: [PATCH 12/27] Debug AWS error info

---
 .github/workflows/run-e2e-tests.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index be89d4ab..628bc16a 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -481,8 +481,7 @@ jobs:
           START_TIME=$(date +%s)
           echo "Checking for '${{ env.CHAINLINK_IMAGE }}:${{ needs.build-test-configurations.outputs.chainlink_image_tag }}' in ECR..."
           while true; do
-            IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]' --output text 2>/dev/null)
-            echo "Command Output: $IMAGE"
+            IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]' --output text 2>aws_error.log)
             if [[ "$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}" ]]; then
                 echo "$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}"
                 echo "Found tag!"
@@ -490,6 +489,12 @@ jobs:
             else
                 echo "Image not found, waiting $POLL_INTERVAL seconds..."
             fi
+
+            echo "Command Output: $IMAGE"
+            if [ -s aws_error.log ]; then
+              echo "Error running ECR command:"
+              cat aws_error.log
+            fi
             
             CURRENT_TIME=$(date +%s)
             ELAPSED_TIME=$((CURRENT_TIME - START_TIME))

From 318e93e08d1e44b01e00f5dbe3140262a30d22e1 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 1 Oct 2024 13:09:46 -0400
Subject: [PATCH 13/27] Debug image call

---
 .github/workflows/run-e2e-tests.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index d48065ff..96108475 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -489,14 +489,15 @@ jobs:
           while true; do
             IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]' --output text 2>aws_error.log)
             if [[ "$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}" ]]; then
-                echo "$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}"
+                echo '"$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}"'
                 echo "Found tag!"
                 exit 0
             else
                 echo "Image not found, waiting $POLL_INTERVAL seconds..."
             fi
 
-            echo "Command Output: $IMAGE"
+            echo "Command Output"
+            aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]' --output text
             if [ -s aws_error.log ]; then
               echo "Error running ECR command:"
               cat aws_error.log

From b88496960fefbc1cc450f2230b60aade45d992fe Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 1 Oct 2024 13:18:28 -0400
Subject: [PATCH 14/27] Debug image call

---
 .github/workflows/run-e2e-tests.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 96108475..19c6d389 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -496,8 +496,9 @@ jobs:
                 echo "Image not found, waiting $POLL_INTERVAL seconds..."
             fi
 
-            echo "Command Output"
-            aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]' --output text
+            echo "Command Output:"
+            aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]'
+            echo "End Command Output"
             if [ -s aws_error.log ]; then
               echo "Error running ECR command:"
               cat aws_error.log

From aa3376a29143a42790c6de8ba59bb782e6071c04 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 1 Oct 2024 13:19:41 -0400
Subject: [PATCH 15/27] Lint

---
 .github/workflows/run-e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 19c6d389..6309f09a 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -489,7 +489,7 @@ jobs:
           while true; do
             IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]' --output text 2>aws_error.log)
             if [[ "$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}" ]]; then
-                echo '"$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}"'
+                echo "$IMAGE == ${{ needs.build-test-configurations.outputs.chainlink_image_tag }}"
                 echo "Found tag!"
                 exit 0
             else

From bdc38aec556342b0c33ce9c9f884d0acb5805362 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 1 Oct 2024 13:35:29 -0400
Subject: [PATCH 16/27] Overhaul command

---
 .github/workflows/run-e2e-tests.yml | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 6309f09a..4ea52676 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -472,7 +472,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [build-test-configurations]
     env:
-      POLL_INTERVAL: 10
+      POLL_INTERVAL: 30
       TIMEOUT: 600
     steps:
       - name: Configure AWS credentials
@@ -487,22 +487,13 @@ jobs:
           START_TIME=$(date +%s)
           echo "Checking for '${{ env.CHAINLINK_IMAGE }}:${{ needs.build-test-configurations.outputs.chainlink_image_tag }}' in ECR..."
           while true; do
-            IMAGE=$(aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]' --output text 2>aws_error.log)
-            if [[ "$IMAGE" == "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}" ]]; then
-                echo "$IMAGE == ${{ needs.build-test-configurations.outputs.chainlink_image_tag }}"
-                echo "Found tag!"
-                exit 0
-            else
-                echo "Image not found, waiting $POLL_INTERVAL seconds..."
-            fi
-
             echo "Command Output:"
-            aws ecr describe-images --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" --query 'imageDetails[?imageTags && contains(imageTags, "${{ needs.build-test-configurations.outputs.chainlink_image_tag }}")].imageTags[]'
+            aws ecr describe-images \
+              --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" \
+              --filter "tagStatus=TAGGED" \
+              --query 'imageDetails[?contains(imageTags, `'${{ needs.build-test-configurations.outputs.chainlink_image_tag }}'`)]' \
+              --output json
             echo "End Command Output"
-            if [ -s aws_error.log ]; then
-              echo "Error running ECR command:"
-              cat aws_error.log
-            fi
             
             CURRENT_TIME=$(date +%s)
             ELAPSED_TIME=$((CURRENT_TIME - START_TIME))

From 96712e00e5d315ea4037adf414016f300576e7a6 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 1 Oct 2024 13:53:02 -0400
Subject: [PATCH 17/27] Better command and debug

---
 .github/workflows/run-e2e-tests.yml | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 4ea52676..ed50e654 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -472,7 +472,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [build-test-configurations]
     env:
-      POLL_INTERVAL: 30
+      POLL_INTERVAL: 15
       TIMEOUT: 600
     steps:
       - name: Configure AWS credentials
@@ -487,13 +487,20 @@ jobs:
           START_TIME=$(date +%s)
           echo "Checking for '${{ env.CHAINLINK_IMAGE }}:${{ needs.build-test-configurations.outputs.chainlink_image_tag }}' in ECR..."
           while true; do
-            echo "Command Output:"
-            aws ecr describe-images \
+            result=$(aws ecr describe-images \
               --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" \
               --filter "tagStatus=TAGGED" \
-              --query 'imageDetails[?contains(imageTags, `'${{ needs.build-test-configurations.outputs.chainlink_image_tag }}'`)]' \
-              --output json
+              --query 'imageDetails[?contains(imageTags, `'${{ needs.build-test-configurations.outputs.chainlink_image_tag }}'`)]'
+              --output json)
             echo "End Command Output"
+
+            if [[ "$result" != "[]" ]]; then
+                echo "Found ${{ needs.build-test-configurations.outputs.chainlink_image_tag }}"
+                return 0
+            else
+                echo "Chainlink image not found in ECR. Waiting..."
+                echo "Command Output: $result"
+            fi
             
             CURRENT_TIME=$(date +%s)
             ELAPSED_TIME=$((CURRENT_TIME - START_TIME))

From 65616d849cf4d302675b6d4e7640714c13062022 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 1 Oct 2024 13:57:29 -0400
Subject: [PATCH 18/27] Typo

---
 .github/workflows/run-e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index ed50e654..c3a68669 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -490,7 +490,7 @@ jobs:
             result=$(aws ecr describe-images \
               --repository-name "${{ inputs.chainlink_image_name || 'chainlink' }}" \
               --filter "tagStatus=TAGGED" \
-              --query 'imageDetails[?contains(imageTags, `'${{ needs.build-test-configurations.outputs.chainlink_image_tag }}'`)]'
+              --query 'imageDetails[?contains(imageTags, `'${{ needs.build-test-configurations.outputs.chainlink_image_tag }}'`)]' \
               --output json)
             echo "End Command Output"
 

From 6b1583f56c5adaa11e104351e21566d79c6acedc Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 1 Oct 2024 14:05:24 -0400
Subject: [PATCH 19/27] Better debug and exit

---
 .github/workflows/run-e2e-tests.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index c3a68669..157f8bac 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -492,14 +492,13 @@ jobs:
               --filter "tagStatus=TAGGED" \
               --query 'imageDetails[?contains(imageTags, `'${{ needs.build-test-configurations.outputs.chainlink_image_tag }}'`)]' \
               --output json)
-            echo "End Command Output"
 
+            echo "Command Output: $result"
             if [[ "$result" != "[]" ]]; then
                 echo "Found ${{ needs.build-test-configurations.outputs.chainlink_image_tag }}"
-                return 0
+                exit 0
             else
-                echo "Chainlink image not found in ECR. Waiting..."
-                echo "Command Output: $result"
+                echo "Chainlink image not found in ECR. Waiting $POLL_INTERVAL seconds..."
             fi
             
             CURRENT_TIME=$(date +%s)

From a7e41ced271293aac1cfb3ff7344553a752cebf5 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 1 Oct 2024 14:32:13 -0400
Subject: [PATCH 20/27] Fix upgrade image

---
 .github/workflows/run-e2e-tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 157f8bac..3b13739e 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -631,7 +631,8 @@ jobs:
         uses: smartcontractkit/.github/actions/ctf-run-tests@b8731364b119e88983e94b0c4da87fc27ddb41b8 # ctf-run-tests@0.0.0
         env:
           E2E_TEST_CHAINLINK_VERSION:
-            ${{ needs.build-test-configurations.outputs.chainlink_image_tag }}
+            ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION ||
+            needs.build-test-configurations.outputs.chainlink_image_tag }}
           E2E_TEST_CHAINLINK_UPGRADE_VERSION:
             ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION ||
             needs.build-test-configurations.outputs.chainlink_image_tag }}

From cc6f812e2dc949913abfe8da18955a1a7e029ade Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Fri, 18 Oct 2024 14:22:20 -0400
Subject: [PATCH 21/27] Typo

---
 .github/workflows/run-e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 51477940..5f1c8de5 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -359,7 +359,7 @@ jobs:
             chainlink_image_tag=${{ inputs.chainlink_version }}
           else
             echo "determining chainlink image tag based on workflow event"
-            chainlink_image_tag=${{ steps.get-chainlink-image-tag.outputs.base-image-tag }}-${{ inputs.test_chainlink_image_type }}
+            chainlink_image_tag=${{ steps.get-chainlink-image-tag.outputs.image-tag }}-${{ inputs.test_chainlink_image_type }}
           fi
 
           echo "chainlink_image_tag=$chainlink_image_tag"

From 3977e9db3de4a69d05958cdee68b3789e51b709f Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Mon, 21 Oct 2024 16:18:03 -0400
Subject: [PATCH 22/27] Debug AWS error info

---
 .github/workflows/run-e2e-tests.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 5f1c8de5..1189a330 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -347,10 +347,10 @@ jobs:
       - name: Install jq
         run: sudo apt-get install jq
 
-      - name: Get Chainlink Image Tag
+      - name: Get Chainlink Image Tag # Get the base image tag, e.g. pr-1234-sha
         id: get-chainlink-image-tag
         uses: ./.github/actions/get-chainlink-image-tag
-      - name: Determine Chainlink Image Tag
+      - name: Determine Chainlink Image Tag # Get the full image tag based on image type, e.g. pr-1234-sha-arm64
         id: determine-chainlink-image-tag
         run: |
           chainlink_image_tag=""
@@ -484,6 +484,7 @@ jobs:
           role-session-name: e2e-tests-wait-for-chainlink-image
       - name: Wait for Chainlink Image
         run: |
+          aws sts get-caller-identity # DEBUG: Check identity
           START_TIME=$(date +%s)
           echo "Checking for '${{ env.CHAINLINK_IMAGE }}:${{ needs.build-test-configurations.outputs.chainlink_image_tag }}' in ECR..."
           while true; do

From e087b0cd6a4acec2ef916aeee30c64c7cd55cd38 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 22 Oct 2024 12:23:57 -0400
Subject: [PATCH 23/27] Use arm64 images

---
 .github/workflows/run-e2e-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 1189a330..1d3d2ba5 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -292,7 +292,7 @@ jobs:
       - name: Install citool
         shell: bash
         run: go install
-          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@c2b6620f9743b326f3fcb82829c760d01b742306 # DEBUG: Testing version
+          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@c012a63734d3b2475a621470d920a0d9da91ba98 # DEBUG: Testing version
       - name: Run Check Tests Command
         run: |
           if ! citool check-tests ${{ github.workspace }}/${{ inputs.check_test_path }} ${{ github.workspace }}/${{ inputs.test_path }}; then
@@ -343,7 +343,7 @@ jobs:
       - name: Install citool
         shell: bash
         run: go install
-          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@c2b6620f9743b326f3fcb82829c760d01b742306 # DEBUG: Testing version
+          github.com/smartcontractkit/chainlink-testing-framework/tools/citool@c012a63734d3b2475a621470d920a0d9da91ba98 # DEBUG: Testing version
       - name: Install jq
         run: sudo apt-get install jq
 

From e2799159bf63e2dcb10ada0a244f0f6e8b3781c3 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Tue, 22 Oct 2024 12:27:54 -0400
Subject: [PATCH 24/27] Use arm64 runners

---
 .github/workflows/run-e2e-tests.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 1d3d2ba5..27976940 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -525,7 +525,10 @@ jobs:
     if:
       ${{ needs.build-test-configurations.outputs.run-docker-tests == 'true' &&
       always() && !failure() && !cancelled() }}
-    runs-on: ${{ matrix.tests.runs_on }}
+    # Check if the test is to be run on arm64, if so, run on the specified arm64 runner
+    runs-on:
+      ${{ inputs.test_chainlink_image_type == 'arm64' &&
+      matrix.tests.runs_on_arm64 || matrix.tests.runs_on }}
     strategy:
       fail-fast: false
       matrix: ${{fromJson(needs.build-test-configurations.outputs.docker-matrix)}}

From 9c1e41b71402b91ffbae6a219ef2dd0c20599e84 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Wed, 23 Oct 2024 10:59:58 -0400
Subject: [PATCH 25/27] Remove debug

---
 .github/workflows/run-e2e-tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 27976940..64f78947 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -484,7 +484,6 @@ jobs:
           role-session-name: e2e-tests-wait-for-chainlink-image
       - name: Wait for Chainlink Image
         run: |
-          aws sts get-caller-identity # DEBUG: Check identity
           START_TIME=$(date +%s)
           echo "Checking for '${{ env.CHAINLINK_IMAGE }}:${{ needs.build-test-configurations.outputs.chainlink_image_tag }}' in ECR..."
           while true; do
@@ -514,7 +513,7 @@ jobs:
 
   # Run Docker tests
   run-docker-tests:
-    name: ${{ matrix.tests.name || matrix.tests.id }}
+    name: ${{ matrix.tests.name || matrix.tests.id_sanitized }}
     needs:
       [
         build-test-configurations,

From 2c75afa810d99613d937f2a480361a804052fc3e Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Fri, 25 Oct 2024 18:19:53 -0400
Subject: [PATCH 26/27] Add check

---
 .github/workflows/run-e2e-tests.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index 64f78947..8d3d4a46 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -753,10 +753,6 @@ jobs:
           path: ${{ matrix.tests.test_artifacts_on_failure }}
           retention-days: 1
 
-      - name: Show Grafana url in test summary
-        if: always()
-        uses: smartcontractkit/.github/actions/ctf-show-grafana-in-test-summary@b6e37806737eef87e8c9137ceeb23ef0bff8b1db # ctf-show-grafana-in-test-summary@0.1.0
-
   # Run K8s tests using old remote runner
 
   prepare-remote-runner-test-image:

From 7f9b2536034cb5053e24feb7dc2ef0aeb390a5a8 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Mon, 28 Oct 2024 15:23:43 -0400
Subject: [PATCH 27/27] Build testcontainers hub prefix

---
 .github/workflows/run-e2e-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
index be9b4cf6..73e7b81f 100644
--- a/.github/workflows/run-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests.yml
@@ -261,6 +261,9 @@ env:
   E2E_JD_IMAGE:
     ${{ secrets.PROD_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
     secrets.AWS_REGION}}.amazonaws.com/job-distributor
+  TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX:
+    ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION
+    }}.amazonaws.com
 
 jobs:
   validate-inputs: