From c0885b44c1797c6a1199a7e218f7280fb0972a60 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 12:13:45 +0200
Subject: [PATCH 001/141] Update some test workflows to run with test config
 path

---
 .../workflows/automation-benchmark-tests.yml  | 44 +++++++------------
 .../on-demand-vrfv2plus-smoke-tests.yml       | 38 +++++++---------
 2 files changed, 33 insertions(+), 49 deletions(-)

diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml
index 8867a1250f7..0d7e9ab5d92 100644
--- a/.github/workflows/automation-benchmark-tests.yml
+++ b/.github/workflows/automation-benchmark-tests.yml
@@ -7,20 +7,20 @@ on:
         required: true
         default: benchmark
         type: string
-      base64Config:
-        description: base64-ed config
-        required: true
-        type: string
+      test_config_override_path:
+        description: Path to a test config file used to override the default test config
+        required: false
+        type: string  
+      test_secrets_override_key:
+        description: Key to run tests with custom test secrets
+        required: false
+        type: string 
       slackMemberID:
         description: Notifies test results (Not your @)
         required: true
         default: U02Q14G80TY
         type: string
-      test_secrets_override_key:
-        description: 'Key to run tests with custom test secrets'
-        required: false
-        type: string 
-
+  
 jobs:
   automation_benchmark:
     environment: integration
@@ -46,23 +46,11 @@ jobs:
           SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
           echo ::add-mask::$SLACK_USER
           echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
-
-          BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-      - name: Parse base64 config
-        uses: ./.github/actions/setup-parse-base64-config
-        with:
-          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}
-      - name: Send details to Step Summary
-        shell: bash
+      - name: Show test config override path in summary
+        if: ${{ inputs.test_config_override_path }}
         run: |
-          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_IMAGE }}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY
+          echo "### Test config override path" >>$GITHUB_STEP_SUMMARY
+          echo "\`${{ inputs.test_config_override_path }}\`" >>$GITHUB_STEP_SUMMARY
       - name: Build Test Image
         uses: ./.github/actions/build-test-image
         with:
@@ -82,13 +70,13 @@ jobs:
           TEST_TEST_TYPE: ${{ github.event.inputs.testType }}
           RR_MEM: 4Gi
           TEST_LOG_LEVEL: info
+          E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+          E2E_TEST_CHAINLINK_VERSION: ${{ github.sha }} # Default chainlink version if not overrided in the test config 
         with:
           test_command_to_run: cd integration-tests && go test -timeout 30m -v -run ^TestAutomationBenchmark$ ./benchmark -count=1
           test_download_vendor_packages_command: make gomod
           test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          test_config_override_path: ${{ inputs.test_config_override_path }}
           token: ${{ secrets.GITHUB_TOKEN }}
           should_cleanup: false
           go_mod_path: ./integration-tests/go.mod
diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
index 1a62e0fe061..8e1a9c10925 100644
--- a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
+++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
@@ -2,10 +2,6 @@ name: On Demand VRFV2 Plus Smoke Tests
 on:
   workflow_dispatch:
     inputs:
-      base64Config:
-        description: base64-ed config
-        required: true
-        type: string
       test_suite:
         description: "Test Suite to run"
         required: true
@@ -17,6 +13,10 @@ on:
         description: "Regex for 'Selected Tests' to run"
         required: false
         default: "TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS"
+      test_config_override_path:
+        description: Path to a test config file used to override the default test config
+        required: false
+        type: string          
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
@@ -59,27 +59,23 @@ jobs:
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
           fetch-depth: 0
-      - name: Mask base64 config
-        run: |
-          BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-      - name: Merge and export base64 config
-        uses: ./.github/actions/setup-merge-base64-config
-        with:
-          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}
-      - name: Send details to Step Summary
-        shell: bash
+      - name: Show test config override path in summary
+        if: ${{ inputs.test_config_override_path }}
         run: |
-          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_IMAGE }}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY
+          echo "### Test config override path" >>$GITHUB_STEP_SUMMARY
+          echo "\`${{ inputs.test_config_override_path }}\`" >>$GITHUB_STEP_SUMMARY          
       - name: Run Tests
         if: ${{ github.event.inputs.test_suite == 'Selected Tests' }}
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
+        env:
+          E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+          E2E_TEST_CHAINLINK_VERSION: ${{ github.sha }} # Default chainlink version if not overrided in the test config 
+          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
+          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
         with:
           test_command_to_run: cd ./integration-tests/smoke && go test -v -count=1 -parallel=1 -timeout 2h -run "${{ inputs.test_list_regex }}" 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download

From c6de43bc6e84c20b8dc4c6fe5b545d3f84964f19 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 12:41:36 +0200
Subject: [PATCH 002/141] Fix summary

---
 .github/workflows/automation-benchmark-tests.yml      | 4 ++--
 .github/workflows/on-demand-vrfv2plus-smoke-tests.yml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml
index 0d7e9ab5d92..d9a752dcccb 100644
--- a/.github/workflows/automation-benchmark-tests.yml
+++ b/.github/workflows/automation-benchmark-tests.yml
@@ -49,8 +49,8 @@ jobs:
       - name: Show test config override path in summary
         if: ${{ inputs.test_config_override_path }}
         run: |
-          echo "### Test config override path" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ inputs.test_config_override_path }}\`" >>$GITHUB_STEP_SUMMARY
+          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
+          echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY
       - name: Build Test Image
         uses: ./.github/actions/build-test-image
         with:
diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
index 8e1a9c10925..d3d4697c2cc 100644
--- a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
+++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
@@ -62,8 +62,8 @@ jobs:
       - name: Show test config override path in summary
         if: ${{ inputs.test_config_override_path }}
         run: |
-          echo "### Test config override path" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ inputs.test_config_override_path }}\`" >>$GITHUB_STEP_SUMMARY          
+          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
+          echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY
       - name: Run Tests
         if: ${{ github.event.inputs.test_suite == 'Selected Tests' }}
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32

From bf6897407954afe74a103fac129bec2206fcec2f Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 13:28:31 +0200
Subject: [PATCH 003/141] Fix

---
 .../on-demand-vrfv2plus-smoke-tests.yml         | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
index d3d4697c2cc..bd91add75ad 100644
--- a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
+++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
@@ -80,9 +80,7 @@ jobs:
           test_command_to_run: cd ./integration-tests/smoke && go test -v -count=1 -parallel=1 -timeout 2h -run "${{ inputs.test_list_regex }}" 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          test_config_override_path: ${{ inputs.test_config_override_path }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: vrf-test-logs
           artifacts_location: ./integration-tests/smoke/vrfv2plus/logs/
@@ -95,13 +93,20 @@ jobs:
       - name: Run Tests
         if: ${{ github.event.inputs.test_suite == 'All Tests' }}
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
+        env:
+          E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+          E2E_TEST_CHAINLINK_VERSION: ${{ github.sha }} # Default chainlink version if not overrided in the test config 
+          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
+          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}        
         with:
           test_command_to_run: cd ./integration-tests/smoke && go test -v -count=1 -parallel=1 -timeout 3h vrfv2plus_test.go 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          test_config_override_path: ${{ inputs.test_config_override_path }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: vrf-test-logs
           artifacts_location: ./integration-tests/smoke/vrfv2plus/logs/

From ee1e5a13e87336ce570f300c5b366f80a944fd49 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 13:48:28 +0200
Subject: [PATCH 004/141] Update on-demand-ocr-soak-test.yml

---
 .github/workflows/on-demand-ocr-soak-test.yml | 37 +++++++++----------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
index 3b827032b12..0fa9549e748 100644
--- a/.github/workflows/on-demand-ocr-soak-test.yml
+++ b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -18,9 +18,9 @@ on:
           - TestOCRSoak_ChangeBlockGasLimit
           - TestOCRSoak_RPCDownForAllCLNodes
           - TestOCRSoak_RPCDownForHalfCLNodes
-      base64Config:
-        description: base64-ed config
-        required: true
+      test_config_override_path:
+        description: Path to a test config file used to override the default test config
+        required: false
         type: string
       slackMemberID:
         description: Slack Member ID (Not your @)
@@ -64,28 +64,21 @@ jobs:
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
           ref: ${{ env.REF_NAME }}
-      - name: Get Slack config and mask base64 config
+      - name: Get Slack config
         run: |
           SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
           echo ::add-mask::$SLACK_USER
           echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
-
-          BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-      - name: Parse base64 config
-        uses: ./.github/actions/setup-parse-base64-config
-        with:
-          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}
       - name: Setup Push Tag
         shell: bash
         run: |
-          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_IMAGE }}\`" >>$GITHUB_STEP_SUMMARY
           echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
           echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY
+      - name: Show test config override path in summary
+        if: ${{ inputs.test_config_override_path }}
+        run: |
+          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
+          echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY          
       - name: Build Image
         uses: ./.github/actions/build-test-image
         with:
@@ -102,13 +95,19 @@ jobs:
           # We can comment these out when we have a stable soak test and aren't worried about resource consumption
           TEST_UPLOAD_CPU_PROFILE: true
           TEST_UPLOAD_MEM_PROFILE: true
+          E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+          E2E_TEST_CHAINLINK_VERSION: ${{ github.sha }} # Default chainlink version if not overrided in the test config 
+          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
+          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}          
         with:
           test_command_to_run: cd ./integration-tests && go test -v -count=1 -run ^${{ github.event.inputs.testToRun }}$ ./soak
           test_download_vendor_packages_command: make gomod
           test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          test_config_override_path: ${{ inputs.test_config_override_path }}
           token: ${{ secrets.GITHUB_TOKEN }}
           should_cleanup: false
           go_mod_path: ./integration-tests/go.mod

From 36a61c134a6d0f278925b4cc2e6c395c5029e6df Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 15:05:47 +0200
Subject: [PATCH 005/141] Run vrfplus tests on ARBITRUM_SEPOLIA on existing env

---
 .../testconfig/vrfv2plus/overrides/ilja_pavlovs.toml         | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml

diff --git a/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml b/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
new file mode 100644
index 00000000000..e65fcc00257
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["ARBITRUM_SEPOLIA"]
+
+[ARBITRUM_SEPOLIA.VRFv2Plus.General]
+use_existing_env = true
\ No newline at end of file

From 40dcd988e9ba49f4f488f63faf5e162bbdc0f2df Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 15:15:10 +0200
Subject: [PATCH 006/141] Add config for staging release testing on arbitrum
 sepolia

---
 .../arbitrum_sepolia_staging_release_test_config.toml        | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/arbitrum_sepolia_staging_release_test_config.toml

diff --git a/integration-tests/testconfig/vrfv2plus/overrides/arbitrum_sepolia_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/arbitrum_sepolia_staging_release_test_config.toml
new file mode 100644
index 00000000000..e65fcc00257
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/arbitrum_sepolia_staging_release_test_config.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["ARBITRUM_SEPOLIA"]
+
+[ARBITRUM_SEPOLIA.VRFv2Plus.General]
+use_existing_env = true
\ No newline at end of file

From 275cca794ef4f40c1619c45614895ed15847231e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 15:30:39 +0200
Subject: [PATCH 007/141] Show test_secrets_override_key in summary

---
 .github/workflows/on-demand-vrfv2plus-smoke-tests.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
index bd91add75ad..e8fb90d4c05 100644
--- a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
+++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
@@ -64,6 +64,11 @@ jobs:
         run: |
           echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
           echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY
+      - name: Show test secrets override key in summary
+        if: ${{ inputs.test_secrets_override_key }}
+        run: |
+          echo "### Test secrets override key" >> $GITHUB_STEP_SUMMARY
+          echo "${{ inputs.test_secrets_override_key }}" >> $GITHUB_STEP_SUMMARY          
       - name: Run Tests
         if: ${{ github.event.inputs.test_suite == 'Selected Tests' }}
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32

From 9376a908a715d3299b061051608b41658ab12da7 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 15:55:17 +0200
Subject: [PATCH 008/141] Generate hash of integration-tests/ for test runner
 image if tag not provided

---
 .github/actions/build-test-image/action.yml | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/.github/actions/build-test-image/action.yml b/.github/actions/build-test-image/action.yml
index dcc9fefda71..bcd8b4b27bc 100644
--- a/.github/actions/build-test-image/action.yml
+++ b/.github/actions/build-test-image/action.yml
@@ -7,8 +7,7 @@ inputs:
     default: chainlink-tests
     required: false
   tag:
-    description: The tag to use by default and to use for checking image existance
-    default: ${{ github.sha }}
+    description: The tag to use by default and to use for checking image existance. If not provided, the hash of the integration-tests/ directory will be used
     required: false
   other_tags:
     description: Other tags to push if needed
@@ -90,12 +89,22 @@ runs:
     # End Base Image Logic
 
     # Test Runner Logic
+    - name: Get hash of integration-tests/ for test runner image
+      id: test_runner_hash
+      if: ${{ inputs.tag == '' }}
+      # Do not include testconfig/ in the hash to avoid rebuilding the image when only testconfig/ changes
+      shell: sh
+      run: |
+        HASH_VALUE=$(echo ${{ hashFiles('integration-tests/**', '!integration-tests/testconfig/**') }})
+        echo "Computed hash: $HASH_VALUE"
+        echo "hash_value=$HASH_VALUE" >> $GITHUB_OUTPUT
+
     - name: Check if image exists
       id: check-image
       uses: smartcontractkit/chainlink-github-actions/docker/image-exists@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
       with:
         repository: ${{ inputs.repository }}
-        tag: ${{ inputs.tag }}
+        tag: ${{ inputs.tag || steps.test_runner_hash.outputs.hash_value }}
         AWS_REGION: ${{ inputs.QA_AWS_REGION }}
         AWS_ROLE_TO_ASSUME: ${{ inputs.QA_AWS_ROLE_TO_ASSUME }}
     - name: Build and Publish Test Runner
@@ -103,7 +112,7 @@ runs:
       uses: smartcontractkit/chainlink-github-actions/docker/build-push@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
       with:
         tags: |
-          ${{ inputs.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ inputs.QA_AWS_REGION }}.amazonaws.com/${{ inputs.repository }}:${{ inputs.tag }}
+          ${{ inputs.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ inputs.QA_AWS_REGION }}.amazonaws.com/${{ inputs.repository }}:${{ inputs.tag || steps.test_runner_hash.outputs.hash_value }}
           ${{ inputs.other_tags }}
         file: ./integration-tests/test.Dockerfile
         build-args: |
@@ -116,7 +125,7 @@ runs:
       shell: sh
       env:
         INPUTS_REPOSITORY: ${{ inputs.repository }}
-        INPUTS_TAG: ${{ inputs.tag }}
+        INPUTS_TAG: ${{ inputs.tag || steps.test_runner_hash.outputs.hash_value }}
       run: |
         echo "### ${INPUTS_REPOSITORY} image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
         echo "\`${INPUTS_TAG}\`" >>$GITHUB_STEP_SUMMARY

From 5b3665800c7ad29e2a0e7009fc40d2ceefd2f55e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 16:14:50 +0200
Subject: [PATCH 009/141] test updating config but not rebuilding test image

---
 .../testconfig/vrfv2plus/overrides/ilja_pavlovs.toml           | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml b/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
index e65fcc00257..a8f0173563b 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
+++ b/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
@@ -2,4 +2,5 @@
 selected_networks = ["ARBITRUM_SEPOLIA"]
 
 [ARBITRUM_SEPOLIA.VRFv2Plus.General]
-use_existing_env = true
\ No newline at end of file
+use_existing_env = true
+

From f3dee3f14e09921fd5ed052b73cbbb5896bc8031 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 16:46:57 +0200
Subject: [PATCH 010/141] Fix test_runner_hash

---
 .github/actions/build-test-image/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/build-test-image/action.yml b/.github/actions/build-test-image/action.yml
index bcd8b4b27bc..5c58150820f 100644
--- a/.github/actions/build-test-image/action.yml
+++ b/.github/actions/build-test-image/action.yml
@@ -95,7 +95,7 @@ runs:
       # Do not include testconfig/ in the hash to avoid rebuilding the image when only testconfig/ changes
       shell: sh
       run: |
-        HASH_VALUE=$(echo ${{ hashFiles('integration-tests/**', '!integration-tests/testconfig/**') }})
+        HASH_VALUE=$(find integration-tests -type f ! -path 'integration-tests/testconfig/*/*.toml' ! -path 'integration-tests/*/__debug_bin*' -exec sha256sum {} + | sort -k 2 | sha256sum | awk '{print $1}')
         echo "Computed hash: $HASH_VALUE"
         echo "hash_value=$HASH_VALUE" >> $GITHUB_OUTPUT
 

From 3725f76316f34ae39a8c0a58ca199ff98c188254 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 16:56:52 +0200
Subject: [PATCH 011/141] Commit all configs for staging

---
 .../testconfig/vrfv2plus/overrides/ilja_pavlovs.toml         | 1 -
 .../arbitrum_sepolia_staging_release_test_config.toml        | 0
 .../staging/avalanche_fuji_staging_release_test_config.toml  | 5 +++++
 .../staging/bsc_testnet_staging_release_test_config.toml     | 5 +++++
 .../staging/nexon_dev_staging_release_test_config.toml       | 5 +++++
 .../staging/nexon_qa_staging_release_test_config.toml        | 5 +++++
 .../staging/nexon_stage_staging_release_test_config.toml     | 5 +++++
 .../staging/nexon_test_staging_release_test_config copy.toml | 5 +++++
 .../staging/sepolia_staging_release_test_config.toml         | 5 +++++
 9 files changed, 35 insertions(+), 1 deletion(-)
 rename integration-tests/testconfig/vrfv2plus/overrides/{ => staging}/arbitrum_sepolia_staging_release_test_config.toml (100%)
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_release_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_release_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_release_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_release_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_release_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_release_test_config copy.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_release_test_config.toml

diff --git a/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml b/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
index a8f0173563b..07ced104913 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
+++ b/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
@@ -3,4 +3,3 @@ selected_networks = ["ARBITRUM_SEPOLIA"]
 
 [ARBITRUM_SEPOLIA.VRFv2Plus.General]
 use_existing_env = true
-
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/arbitrum_sepolia_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/arbitrum_sepolia_staging_release_test_config.toml
similarity index 100%
rename from integration-tests/testconfig/vrfv2plus/overrides/arbitrum_sepolia_staging_release_test_config.toml
rename to integration-tests/testconfig/vrfv2plus/overrides/staging/arbitrum_sepolia_staging_release_test_config.toml
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_release_test_config.toml
new file mode 100644
index 00000000000..acf5c921fc7
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_release_test_config.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["AVALANCHE_FUJI"]
+
+[AVALANCHE_FUJI.VRFv2Plus.General]
+use_existing_env = true
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_release_test_config.toml
new file mode 100644
index 00000000000..1aba4843bc3
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_release_test_config.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["BSC_TESTNET"]
+
+[BSC_TESTNET.VRFv2Plus.General]
+use_existing_env = true
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_release_test_config.toml
new file mode 100644
index 00000000000..b9db27d96fd
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_release_test_config.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["NEXON_DEV"]
+
+[NEXON_DEV.VRFv2Plus.General]
+use_existing_env = true
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_release_test_config.toml
new file mode 100644
index 00000000000..2d01703285a
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_release_test_config.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["NEXON_QA"]
+
+[NEXON_QA.VRFv2Plus.General]
+use_existing_env = true
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_release_test_config.toml
new file mode 100644
index 00000000000..0edcf71caab
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_release_test_config.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["NEXON_STAGE"]
+
+[NEXON_STAGE.VRFv2Plus.General]
+use_existing_env = true
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_release_test_config copy.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_release_test_config copy.toml
new file mode 100644
index 00000000000..6a3eabb46be
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_release_test_config copy.toml	
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["NEXON_TEST"]
+
+[NEXON_TEST.VRFv2Plus.General]
+use_existing_env = true
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_release_test_config.toml
new file mode 100644
index 00000000000..31185f2540b
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_release_test_config.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["SEPOLIA"]
+
+[SEPOLIA.VRFv2Plus.General]
+use_existing_env = true
\ No newline at end of file

From e34d7cf42e861ff8f1980b4c156865172c21b99b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 17:05:34 +0200
Subject: [PATCH 012/141] test config change

---
 .../testconfig/vrfv2plus/overrides/ilja_pavlovs.toml             | 1 +
 1 file changed, 1 insertion(+)

diff --git a/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml b/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
index 07ced104913..a8f0173563b 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
+++ b/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
@@ -3,3 +3,4 @@ selected_networks = ["ARBITRUM_SEPOLIA"]
 
 [ARBITRUM_SEPOLIA.VRFv2Plus.General]
 use_existing_env = true
+

From d9bf2e9e460064b6172640ec8b44a90a62a08f14 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 17:09:48 +0200
Subject: [PATCH 013/141] Update release configs

---
 .../arbitrum_sepolia_release_test_config.toml}                  | 2 +-
 .../avalanche_fuji_release_test_config.toml}                    | 2 +-
 .../bsc_testnet_release_test_config.toml}                       | 2 +-
 .../nexon_dev_release_test_config.toml}                         | 2 +-
 .../nexon_qa_release_test_config.toml}                          | 2 +-
 .../nexon_stage_release_test_config.toml}                       | 2 +-
 .../nexon_test_release_test_config copy.toml}                   | 2 +-
 .../sepolia_release_test_config.toml}                           | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)
 rename integration-tests/testconfig/vrfv2plus/overrides/{staging/arbitrum_sepolia_staging_release_test_config.toml => release_testing/arbitrum_sepolia_release_test_config.toml} (78%)
 rename integration-tests/testconfig/vrfv2plus/overrides/{staging/avalanche_fuji_staging_release_test_config.toml => release_testing/avalanche_fuji_release_test_config.toml} (77%)
 rename integration-tests/testconfig/vrfv2plus/overrides/{staging/bsc_testnet_staging_release_test_config.toml => release_testing/bsc_testnet_release_test_config.toml} (76%)
 rename integration-tests/testconfig/vrfv2plus/overrides/{staging/nexon_dev_staging_release_test_config.toml => release_testing/nexon_dev_release_test_config.toml} (75%)
 rename integration-tests/testconfig/vrfv2plus/overrides/{staging/nexon_qa_staging_release_test_config.toml => release_testing/nexon_qa_release_test_config.toml} (75%)
 rename integration-tests/testconfig/vrfv2plus/overrides/{staging/nexon_stage_staging_release_test_config.toml => release_testing/nexon_stage_release_test_config.toml} (76%)
 rename integration-tests/testconfig/vrfv2plus/overrides/{staging/nexon_test_staging_release_test_config copy.toml => release_testing/nexon_test_release_test_config copy.toml} (76%)
 rename integration-tests/testconfig/vrfv2plus/overrides/{staging/sepolia_staging_release_test_config.toml => release_testing/sepolia_release_test_config.toml} (74%)

diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/arbitrum_sepolia_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/arbitrum_sepolia_release_test_config.toml
similarity index 78%
rename from integration-tests/testconfig/vrfv2plus/overrides/staging/arbitrum_sepolia_staging_release_test_config.toml
rename to integration-tests/testconfig/vrfv2plus/overrides/release_testing/arbitrum_sepolia_release_test_config.toml
index e65fcc00257..a71e662cc40 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/staging/arbitrum_sepolia_staging_release_test_config.toml
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/arbitrum_sepolia_release_test_config.toml
@@ -2,4 +2,4 @@
 selected_networks = ["ARBITRUM_SEPOLIA"]
 
 [ARBITRUM_SEPOLIA.VRFv2Plus.General]
-use_existing_env = true
\ No newline at end of file
+use_existing_env = false
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/avalanche_fuji_release_test_config.toml
similarity index 77%
rename from integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_release_test_config.toml
rename to integration-tests/testconfig/vrfv2plus/overrides/release_testing/avalanche_fuji_release_test_config.toml
index acf5c921fc7..5219ebe4280 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_release_test_config.toml
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/avalanche_fuji_release_test_config.toml
@@ -2,4 +2,4 @@
 selected_networks = ["AVALANCHE_FUJI"]
 
 [AVALANCHE_FUJI.VRFv2Plus.General]
-use_existing_env = true
\ No newline at end of file
+use_existing_env = false
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/bsc_testnet_release_test_config.toml
similarity index 76%
rename from integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_release_test_config.toml
rename to integration-tests/testconfig/vrfv2plus/overrides/release_testing/bsc_testnet_release_test_config.toml
index 1aba4843bc3..8a8ab538f29 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_release_test_config.toml
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/bsc_testnet_release_test_config.toml
@@ -2,4 +2,4 @@
 selected_networks = ["BSC_TESTNET"]
 
 [BSC_TESTNET.VRFv2Plus.General]
-use_existing_env = true
\ No newline at end of file
+use_existing_env = false
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_dev_release_test_config.toml
similarity index 75%
rename from integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_release_test_config.toml
rename to integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_dev_release_test_config.toml
index b9db27d96fd..903e383886e 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_release_test_config.toml
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_dev_release_test_config.toml
@@ -2,4 +2,4 @@
 selected_networks = ["NEXON_DEV"]
 
 [NEXON_DEV.VRFv2Plus.General]
-use_existing_env = true
\ No newline at end of file
+use_existing_env = false
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_qa_release_test_config.toml
similarity index 75%
rename from integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_release_test_config.toml
rename to integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_qa_release_test_config.toml
index 2d01703285a..ff5c62bf04d 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_release_test_config.toml
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_qa_release_test_config.toml
@@ -2,4 +2,4 @@
 selected_networks = ["NEXON_QA"]
 
 [NEXON_QA.VRFv2Plus.General]
-use_existing_env = true
\ No newline at end of file
+use_existing_env = false
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_stage_release_test_config.toml
similarity index 76%
rename from integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_release_test_config.toml
rename to integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_stage_release_test_config.toml
index 0edcf71caab..a504bd8c720 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_release_test_config.toml
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_stage_release_test_config.toml
@@ -2,4 +2,4 @@
 selected_networks = ["NEXON_STAGE"]
 
 [NEXON_STAGE.VRFv2Plus.General]
-use_existing_env = true
\ No newline at end of file
+use_existing_env = false
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_release_test_config copy.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_test_release_test_config copy.toml
similarity index 76%
rename from integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_release_test_config copy.toml
rename to integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_test_release_test_config copy.toml
index 6a3eabb46be..7551f348fb8 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_release_test_config copy.toml	
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_test_release_test_config copy.toml	
@@ -2,4 +2,4 @@
 selected_networks = ["NEXON_TEST"]
 
 [NEXON_TEST.VRFv2Plus.General]
-use_existing_env = true
\ No newline at end of file
+use_existing_env = false
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/sepolia_release_test_config.toml
similarity index 74%
rename from integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_release_test_config.toml
rename to integration-tests/testconfig/vrfv2plus/overrides/release_testing/sepolia_release_test_config.toml
index 31185f2540b..cef5136feca 100644
--- a/integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_release_test_config.toml
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/sepolia_release_test_config.toml
@@ -2,4 +2,4 @@
 selected_networks = ["SEPOLIA"]
 
 [SEPOLIA.VRFv2Plus.General]
-use_existing_env = true
\ No newline at end of file
+use_existing_env = false
\ No newline at end of file

From 0cf8086c34d8055745897b9873cceba45c4d03bb Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 2 Sep 2024 17:23:41 +0200
Subject: [PATCH 014/141] Do not rebuild test image when md or .github files
 were changed

---
 .github/actions/build-test-image/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/build-test-image/action.yml b/.github/actions/build-test-image/action.yml
index 5c58150820f..38bcbf8dad3 100644
--- a/.github/actions/build-test-image/action.yml
+++ b/.github/actions/build-test-image/action.yml
@@ -95,7 +95,7 @@ runs:
       # Do not include testconfig/ in the hash to avoid rebuilding the image when only testconfig/ changes
       shell: sh
       run: |
-        HASH_VALUE=$(find integration-tests -type f ! -path 'integration-tests/testconfig/*/*.toml' ! -path 'integration-tests/*/__debug_bin*' -exec sha256sum {} + | sort -k 2 | sha256sum | awk '{print $1}')
+        HASH_VALUE=$(find integration-tests -type f ! -path 'integration-tests/testconfig/*/*.toml' ! -path '.github/*/*' ! -path '*/*.md' ! -path '*/*.MD' ! -path 'integration-tests/*/__debug_bin*' -exec sha256sum {} + | sort -k 2 | sha256sum | awk '{print $1}')
         echo "Computed hash: $HASH_VALUE"
         echo "hash_value=$HASH_VALUE" >> $GITHUB_OUTPUT
 

From e2d53eef85c5152b8d1ca49d0ce4df3e678a0a66 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 09:08:48 +0200
Subject: [PATCH 015/141] Add wemix testnet config for ocrv2 soak test

---
 .../ocr2/overrides/wemix_testnet.toml         | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml

diff --git a/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml b/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
new file mode 100644
index 00000000000..f428bfd4c88
--- /dev/null
+++ b/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
@@ -0,0 +1,25 @@
+[ChainlinkImage]
+version = "develop"
+# version = "2.14.0-beta0"
+
+[Network]
+selected_networks = ["WEMIX_TESTNET"]
+
+# soak test specific configuration
+[Soak.Common]
+chainlink_node_funding = 2 # Funding per node
+
+# Note this is for when you are running OCR2 tests
+# To run OCR1 tests use [Soak.OCR] instead
+[Soak.OCR2]
+# To run OCR1 tests use [Soak.OCR.Common] instead
+[Soak.OCR2.Common]
+test_duration = "2880m" # Full duration the test will run 1440 - 1 day, 7200 - 5 days
+
+# To run OCR1 tests use [Soak.OCR.Soak] instead
+[Soak.OCR2.Soak]
+number_of_contracts = 2    # Number of deployed contracts, 2 being default
+time_between_rounds = "5m" # Default 5 mins for a 1 day, 10 mins for a 5 day
+
+[Seth]
+experiments_enabled = ["slow_funds_return"]
\ No newline at end of file

From 13d4cd7347256fd598c576f0c8f7895a97420818 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 09:10:53 +0200
Subject: [PATCH 016/141] Use chainlink version from config

---
 .github/workflows/on-demand-ocr-soak-test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
index 0fa9549e748..4651e82523c 100644
--- a/.github/workflows/on-demand-ocr-soak-test.yml
+++ b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -96,7 +96,6 @@ jobs:
           TEST_UPLOAD_CPU_PROFILE: true
           TEST_UPLOAD_MEM_PROFILE: true
           E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-          E2E_TEST_CHAINLINK_VERSION: ${{ github.sha }} # Default chainlink version if not overrided in the test config 
           E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
           E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
           E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}

From 69b1adcb5be73d2a1173cb0fa49f388ef41f9bd2 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 09:22:44 +0200
Subject: [PATCH 017/141] Update reusable workflow inputs

---
 .../run-e2e-tests-reusable-workflow.yml       | 20 +++++--------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index de3f2545398..921f03ffc79 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -29,11 +29,10 @@ on:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
         type: string
-      # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392
-      # test_config_override_base64:
-      #   required: false
-      #   description: The base64-encoded test config override
-      #   type: string
+      test_config_override_path:
+        description: 'Path to a test config file used to override the default test config'
+        required: false
+        type: string  
       enable_check_test_configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -286,13 +285,6 @@ jobs:
       - name: Check if test config override is required for any test
         shell: bash
         run: |
-          # Check if the test config override is provided and skip the checks if it is non-empty
-          # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 
-          # if [ -n "${{ inputs.test_config_override_base64 }}" ]; then
-          #   echo "Test config override provided. Skipping checks for tests requiring config override."
-          #   exit 0
-          # fi
-
           # Parse the JSON to check for test_config_override_required in Docker matrix
           DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq 'if .tests then .tests[] | select(has("test_config_override_required") and .test_config_override_required) | .id else empty end' -r)
           # Parse the JSON to check for test_config_override_required in Kubernetes matrix
@@ -307,7 +299,7 @@ jobs:
             if [ ! -z "$K8S_TESTS_REQUIRING_CONFIG_OVERRIDE" ]; then
               echo $K8S_TESTS_REQUIRING_CONFIG_OVERRIDE
             fi
-            echo "::error::Error: Some of the tests require a test config override. Please see workflow logs and set 'test_config_override_base64' to run these tests."
+            echo "::error::Error: Some of the tests require a test config override. Please see workflow logs and set 'test_config_override_path' to run these tests."
             exit 1
           else
             echo "No tests require a configuration override. Proceeding without overrides."
@@ -510,8 +502,6 @@ jobs:
           test_download_vendor_packages_command: cd $(dirname ${{ matrix.tests.path }}) && go mod download
           test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}
           test_config_override_path: ${{ matrix.tests.test_config_override_path }}
-          # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392
-          # test_config_override_base64: ${{ inputs.test_config_override_base64 }}
           test_type: ${{ matrix.tests.test_env_vars.TEST_TYPE }}
           test_suite: ${{ matrix.tests.test_env_vars.TEST_SUITE }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}

From e096464fa2dc8b9f0d95fac23a5a14e96f85ccdc Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 09:23:09 +0200
Subject: [PATCH 018/141] Add soak/ocr_test.go:^TestOCRv2Soak$WemixTestnet to
 CI tests

---
 .github/e2e-tests.yml                                  | 10 ++++++++++
 .../testconfig/ocr2/overrides/wemix_testnet.toml       |  4 ----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 16a5152f140..5cc04798b37 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -41,6 +41,16 @@ runner-test-matrix:
     test_env_vars:
       TEST_SUITE: soak
 
+  - id: soak/ocr_test.go:TestOCRv2Soak_WemixTestnet
+    path: integration-tests/soak/ocr_test.go
+    test_env_type: k8s-remote-runner
+    runs_on: ubuntu-latest
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 -json
+    test_config_override_path: integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
+    test_secrets_required: true
+    test_env_vars:
+      TEST_SUITE: soak
+
   - id: soak/ocr_test.go:^TestForwarderOCRv1Soak$
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
diff --git a/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml b/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
index f428bfd4c88..58d3dd1fd15 100644
--- a/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
+++ b/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
@@ -1,7 +1,3 @@
-[ChainlinkImage]
-version = "develop"
-# version = "2.14.0-beta0"
-
 [Network]
 selected_networks = ["WEMIX_TESTNET"]
 

From 47d8d35db100b24bf3ef5ce627607ea588595561 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 09:35:52 +0200
Subject: [PATCH 019/141] Fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 921f03ffc79..819e0529db2 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -690,6 +690,7 @@ jobs:
           test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: make gomod
           test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}
+          test_config_override_path: ${{ matrix.tests.test_config_override_path }}
           test_type: ${{ matrix.tests.test_env_vars.TEST_TYPE }}
           test_suite: ${{ matrix.tests.test_env_vars.TEST_SUITE }}
           token: ${{ secrets.GH_TOKEN }}

From 3a0b5f06069ad738cff6dce7bcbfd8205a798555 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 09:44:34 +0200
Subject: [PATCH 020/141] Fix test image tag

---
 .github/actions/build-test-image/action.yml    | 18 ++++++++++++++++++
 .github/workflows/on-demand-ocr-soak-test.yml  |  3 ++-
 .../run-e2e-tests-reusable-workflow.yml        |  5 ++++-
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/.github/actions/build-test-image/action.yml b/.github/actions/build-test-image/action.yml
index 38bcbf8dad3..58a984bbb85 100644
--- a/.github/actions/build-test-image/action.yml
+++ b/.github/actions/build-test-image/action.yml
@@ -26,6 +26,17 @@ inputs:
     description: The AWS region the ECR repository is located in, should only be needed for public ECR repositories, used in configuring docker/login-action
     required: true
 
+outputs:
+  test_image:
+    description: The full name of the test image that was built
+    value: ${{ steps.image_outputs.outputs.test_image }}  
+  test_image_tag:
+    description: The tag of the test image that was built
+    value: ${{ steps.image_outputs.outputs.test_image_tag }}
+  test_image_repository:
+    description: The repository of the test image that was built
+    value: ${{ steps.image_outputs.outputs.test_image_repo }}  
+
 runs:
   using: composite
   steps:
@@ -129,4 +140,11 @@ runs:
       run: |
         echo "### ${INPUTS_REPOSITORY} image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
         echo "\`${INPUTS_TAG}\`" >>$GITHUB_STEP_SUMMARY
+    - name: Set outputs
+      id: image_outputs
+      shell: sh
+      run: |
+        echo "test_image_repo=${{ inputs.repository }}" >> $GITHUB_OUTPUT
+        echo "test_image_tag=${{ inputs.tag || steps.test_runner_hash.outputs.hash_value }}" >> $GITHUB_OUTPUT
+        echo "test_image=${{ inputs.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ inputs.QA_AWS_REGION }}.amazonaws.com/${{ inputs.repository }}:${{ inputs.tag || steps.test_runner_hash.outputs.hash_value }}" >> $GITHUB_OUTPUT
     # End Test Runner Logic
diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
index 4651e82523c..c3e895b428d 100644
--- a/.github/workflows/on-demand-ocr-soak-test.yml
+++ b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -80,6 +80,7 @@ jobs:
           echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
           echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY          
       - name: Build Image
+        id: build_test_image
         uses: ./.github/actions/build-test-image
         with:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
@@ -91,7 +92,7 @@ jobs:
           DETACH_RUNNER: true
           TEST_SUITE: soak
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
-          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
+          ENV_JOB_IMAGE: ${{ steps.build_test_image.outputs.test_image }}
           # We can comment these out when we have a stable soak test and aren't worried about resource consumption
           TEST_UPLOAD_CPU_PROFILE: true
           TEST_UPLOAD_MEM_PROFILE: true
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 819e0529db2..e624e5e6b01 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -598,6 +598,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
       - name: Build Test Runner Image
+        id: build-test-runner-image
         uses: ./.github/actions/build-test-image
         if: ${{ inputs.with_existing_remote_runner_version == '' }}
         with:
@@ -608,7 +609,9 @@ jobs:
         id: set-remote-runner-version
         run: |
           if [[ -z "${{ inputs.with_existing_remote_runner_version }}" ]]; then
-            echo "remote-runner-version=${{ github.sha }}" >> $GITHUB_OUTPUT
+            echo "remote-runner-image=${{ steps.build-test-runner-image.outputs.test_image }}" >> $GITHUB_OUTPUT
+            echo "remote-runner-repository=${{ steps.build-test-runner-image.outputs.test_image_repository }}" >> $GITHUB_OUTPUT
+            echo "remote-runner-version=${{ steps.build-test-runner-image.outputs.test_image_tag }}" >> $GITHUB_OUTPUT
           else
             echo "remote-runner-version=${{ inputs.with_existing_remote_runner_version }}" >> $GITHUB_OUTPUT
           fi

From 67b4664cbc70e345dfae663dd5400abab7d5b4f6 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 10:01:00 +0200
Subject: [PATCH 021/141] Remove image from default.toml

---
 integration-tests/testconfig/default.toml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/integration-tests/testconfig/default.toml b/integration-tests/testconfig/default.toml
index 2bb73f42ad9..f1dd65c5a82 100644
--- a/integration-tests/testconfig/default.toml
+++ b/integration-tests/testconfig/default.toml
@@ -13,10 +13,9 @@ log_producer_retry_limit = 10
 [ChainlinkImage]
 # postgres version to use
 postgres_version = "15.6"
-# chainlink image to use
-image = "public.ecr.aws/chainlink/chainlink"
 # chainlink image tag to use
 version = "2.12.0"
+# Set chainlink image using E2E_TEST_CHAINLINK_IMAGE env, as it is a test secret
 
 [Common]
 # chainlink node funding in native token

From e18c0c8645051ad04d814f330cfb66bb62acbb73 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 10:03:37 +0200
Subject: [PATCH 022/141] Fix chainlink_version

---
 .github/workflows/on-demand-ocr-soak-test.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
index c3e895b428d..75283cbaf05 100644
--- a/.github/workflows/on-demand-ocr-soak-test.yml
+++ b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -30,7 +30,12 @@ on:
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
-        type: string 
+        type: string
+      chainlink_version:
+        description: Chainlink image version to use
+        default: develop
+        required: true
+        type: string
         
 jobs:
   ocr_soak_test:
@@ -97,6 +102,7 @@ jobs:
           TEST_UPLOAD_CPU_PROFILE: true
           TEST_UPLOAD_MEM_PROFILE: true
           E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+          E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_version }}
           E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
           E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
           E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}

From 880441f80d3fc926b2be265d0a7cd2d0273892ef Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 10:15:15 +0200
Subject: [PATCH 023/141] Fix test image hash value

---
 .github/actions/build-test-image/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/build-test-image/action.yml b/.github/actions/build-test-image/action.yml
index 58a984bbb85..2463bc6edda 100644
--- a/.github/actions/build-test-image/action.yml
+++ b/.github/actions/build-test-image/action.yml
@@ -106,7 +106,7 @@ runs:
       # Do not include testconfig/ in the hash to avoid rebuilding the image when only testconfig/ changes
       shell: sh
       run: |
-        HASH_VALUE=$(find integration-tests -type f ! -path 'integration-tests/testconfig/*/*.toml' ! -path '.github/*/*' ! -path '*/*.md' ! -path '*/*.MD' ! -path 'integration-tests/*/__debug_bin*' -exec sha256sum {} + | sort -k 2 | sha256sum | awk '{print $1}')
+        HASH_VALUE=$(find integration-tests -type f ! -path 'integration-tests/testconfig/overrides/*.toml' ! -path 'integration-tests/testconfig/overrides/*/*.toml' ! -path 'integration-tests/testconfig/*/overrides/*.toml' ! -path 'integration-tests/testconfig/*/overrides/*/*.toml' ! -path 'integration-tests/ccip-tests/testconfig/*/overrides/*.toml' ! -path 'integration-tests/ccip-tests/testconfig/*/overrides/*/*.toml' ! -path '.github/*/*' ! -path '*/*.md' ! -path '*/*.MD' ! -path 'integration-tests/*/__debug_bin*' ! -path '*/*.MD' ! -path 'integration-tests/*/tmp-manifest*.yaml' ! -path '*/*.MD' ! -path 'integration-tests/*/*.log' ! -path 'integration-tests/*/*_dump.sql' ! -path 'integration-tests/*/.test_summary/*' -exec sha256sum {} + | sort -k 2 | sha256sum | awk '{print $1}')
         echo "Computed hash: $HASH_VALUE"
         echo "hash_value=$HASH_VALUE" >> $GITHUB_OUTPUT
 

From eb7854722d63816ff963da1946b03906665a48fc Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 10:32:33 +0200
Subject: [PATCH 024/141] Fix summary

---
 .github/workflows/on-demand-ocr-soak-test.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
index 75283cbaf05..9099591a7cc 100644
--- a/.github/workflows/on-demand-ocr-soak-test.yml
+++ b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -74,16 +74,15 @@ jobs:
           SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
           echo ::add-mask::$SLACK_USER
           echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
-      - name: Setup Push Tag
-        shell: bash
-        run: |
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
       - name: Show test config override path in summary
         if: ${{ inputs.test_config_override_path }}
         run: |
           echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
           echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY          
+      - name: Show chainlink version in summary
+        run: |
+          echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
+          echo "${{ inputs.chainlink_version }}" >> $GITHUB_STEP_SUMMARY
       - name: Build Image
         id: build_test_image
         uses: ./.github/actions/build-test-image

From ff864b94490f0ae75eb75f04eb5eb8d9345655d5 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 11:04:50 +0200
Subject: [PATCH 025/141] Update workflow

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index e624e5e6b01..6a0ce8fe8c0 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -478,7 +478,7 @@ jobs:
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
         env:
           DETACH_RUNNER: true
-          E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || inputs.chainlink_version || github.sha }}
+          E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}
           E2E_TEST_CHAINLINK_UPGRADE_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION }}
           E2E_TEST_CHAINLINK_POSTGRES_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_POSTGRES_VERSION }}
           E2E_TEST_SELECTED_NETWORK: ${{ matrix.tests.test_env_vars.E2E_TEST_SELECTED_NETWORK || env.SELECTED_NETWORKS }}
@@ -669,7 +669,7 @@ jobs:
           TEST_UPLOAD_CPU_PROFILE: true
           TEST_UPLOAD_MEM_PROFILE: true
           REF_NAME: ${{ github.head_ref || github.ref_name }}
-          E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || inputs.chainlink_version || github.sha }}
+          E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}
           E2E_TEST_CHAINLINK_UPGRADE_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION }}
           E2E_TEST_CHAINLINK_POSTGRES_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_POSTGRES_VERSION }}
           E2E_TEST_SELECTED_NETWORK: ${{ matrix.tests.test_env_vars.E2E_TEST_SELECTED_NETWORK || env.SELECTED_NETWORKS }}

From eb564bef1c851ebc4a85ee55b66d3a8eb2382269 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 11:06:20 +0200
Subject: [PATCH 026/141] Update config overrides, read from default, env vars
 and then from BASE64_CONFIG_OVERRIDE if exists

BASE64_CONFIG_OVERRIDE should be able to override everything that was already set in the test config
---
 .../ccip-tests/testconfig/global.go           | 33 ++++++++++---------
 integration-tests/testconfig/testconfig.go    | 25 +++++++-------
 2 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/integration-tests/ccip-tests/testconfig/global.go b/integration-tests/ccip-tests/testconfig/global.go
index c00bfdcfea1..a97b65e9d35 100644
--- a/integration-tests/ccip-tests/testconfig/global.go
+++ b/integration-tests/ccip-tests/testconfig/global.go
@@ -113,6 +113,23 @@ func NewConfig() (*Config, error) {
 		return nil, errors.Wrap(err, ErrReadConfig)
 	}
 
+	// read secrets for all products
+	if cfg.CCIP != nil {
+		err := ctfconfig.LoadSecretEnvsFromFiles()
+		if err != nil {
+			return nil, errors.Wrap(err, "error loading testsecrets files")
+		}
+		err = cfg.CCIP.LoadFromEnv()
+		if err != nil {
+			return nil, errors.Wrap(err, "error loading env vars into CCIP config")
+		}
+		// validate all products
+		err = cfg.CCIP.Validate()
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// load config overrides from env var if specified
 	// there can be multiple overrides separated by comma
 	rawConfigs, _ := osutil.GetEnv(OVERIDECONFIG)
@@ -137,22 +154,6 @@ func NewConfig() (*Config, error) {
 			}
 		}
 	}
-	// read secrets for all products
-	if cfg.CCIP != nil {
-		err := ctfconfig.LoadSecretEnvsFromFiles()
-		if err != nil {
-			return nil, errors.Wrap(err, "error loading testsecrets files")
-		}
-		err = cfg.CCIP.LoadFromEnv()
-		if err != nil {
-			return nil, errors.Wrap(err, "error loading env vars into CCIP config")
-		}
-		// validate all products
-		err = cfg.CCIP.Validate()
-		if err != nil {
-			return nil, err
-		}
-	}
 
 	return cfg, nil
 }
diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go
index d0839d72398..533720dc97e 100644
--- a/integration-tests/testconfig/testconfig.go
+++ b/integration-tests/testconfig/testconfig.go
@@ -353,13 +353,19 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error)
 		}
 	}
 
-	// it needs some custom logic, so we do it separately
-	err := testConfig.readNetworkConfiguration()
+	logger.Info().Msg("Setting env vars from testsecrets dot-env files")
+	err := ctf_config.LoadSecretEnvsFromFiles()
 	if err != nil {
-		return TestConfig{}, errors.Wrapf(err, "error reading network config")
+		return TestConfig{}, errors.Wrapf(err, "error reading test config values from ~/.testsecrets file")
+	}
+
+	logger.Info().Msg("Reading config values from existing env vars")
+	err = testConfig.ReadFromEnvVar()
+	if err != nil {
+		return TestConfig{}, errors.Wrapf(err, "error reading test config values from env vars")
 	}
 
-	logger.Info().Msg("Reading configs from Base64 override env var")
+	logger.Info().Msgf("Overriding config from %s env var", Base64OverrideEnvVarName)
 	configEncoded, isSet := os.LookupEnv(Base64OverrideEnvVarName)
 	if isSet && configEncoded != "" {
 		logger.Debug().Msgf("Found base64 config override environment variable '%s' found", Base64OverrideEnvVarName)
@@ -380,16 +386,9 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error)
 		logger.Debug().Msg("Base64 config override from environment variable not found")
 	}
 
-	logger.Info().Msg("Loading config values from default ~/.testsecrets env file")
-	err = ctf_config.LoadSecretEnvsFromFiles()
+	err = testConfig.readNetworkConfiguration()
 	if err != nil {
-		return TestConfig{}, errors.Wrapf(err, "error reading test config values from ~/.testsecrets file")
-	}
-
-	logger.Info().Msg("Reading values from environment variables")
-	err = testConfig.ReadFromEnvVar()
-	if err != nil {
-		return TestConfig{}, errors.Wrapf(err, "error reading test config values from env vars")
+		return TestConfig{}, errors.Wrapf(err, "error reading network config")
 	}
 
 	logger.Debug().Msg("Validating test config")

From f7a39921dd6fcd13219e5237708027d3993e86e7 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 11:33:47 +0200
Subject: [PATCH 027/141] Build chainlink image sha if required

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 6a0ce8fe8c0..a5973e2be8e 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -166,6 +166,9 @@ jobs:
           if [[ "${{ inputs.require_chainlink_image_versions_in_qa_ecr }}" != '' ]]; then
             image_versions=$(echo "${{ inputs.require_chainlink_image_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then [] else . end')
             echo "versions=$image_versions" >> $GITHUB_OUTPUT
+          elif [[ "${{ env.DEFAULT_CHAINLINK_VERSION }}" == "${{ github.sha }}" ]]; then
+            echo "Setting ${{env.DEFAULT_CHAINLINK_VERSION}} as a required Chainlink image version in QA ECR"
+            echo "versions=$env.DEFAULT_CHAINLINK_VERSION" >> $GITHUB_OUTPUT
           fi
       - name: Create matrix for required Chainlink plugin versions
         id: set-required-chainlink-plugin-versions-matrix

From e97ca858f2eec339e59e4ac49020605f4d9b6416 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 12:08:43 +0200
Subject: [PATCH 028/141] Fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index a5973e2be8e..5da4a24d5ed 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -504,7 +504,7 @@ jobs:
           test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd $(dirname ${{ matrix.tests.path }}) && go mod download
           test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}
-          test_config_override_path: ${{ matrix.tests.test_config_override_path }}
+          test_config_override_path: ${{ matrix.tests.test_config_override_path || inputs.test_config_override_path }}
           test_type: ${{ matrix.tests.test_env_vars.TEST_TYPE }}
           test_suite: ${{ matrix.tests.test_env_vars.TEST_SUITE }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
@@ -696,7 +696,7 @@ jobs:
           test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: make gomod
           test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}
-          test_config_override_path: ${{ matrix.tests.test_config_override_path }}
+          test_config_override_path: ${{ matrix.tests.test_config_override_path || inputs.test_config_override_path }}
           test_type: ${{ matrix.tests.test_env_vars.TEST_TYPE }}
           test_suite: ${{ matrix.tests.test_env_vars.TEST_SUITE }}
           token: ${{ secrets.GH_TOKEN }}

From 86c528a4b9851aaf4a8df0d92b03b23d2055ae42 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 12:16:02 +0200
Subject: [PATCH 029/141] Add new workflow

---
 .github/e2e-tests.yml                         | 14 +++---
 .../on-demand-vrfv2plus-smoke-tests-2.yml     | 47 +++++++++++++++++++
 .github/workflows/run-selected-e2e-tests.yml  |  2 -
 3 files changed, 55 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 5cc04798b37..072fc47e554 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -678,15 +678,17 @@ runner-test-matrix:
     workflows:
       - On Demand VRFV2 Performance Test
 
-  - id: smoke/vrfv2plus_test.go:^TestVRFv2Plus$/^Link_Billing$
+  - id: TestVRFv2Plus_LinkBilling
     path: integration-tests/smoke/vrfv2plus_test.go
-    runs_on: ubuntu22.04-8cores-32GB
+    runs_on: ubuntu-latest
     test_env_type: docker
     test_cmd: cd integration-tests && go test -v -test.run ^TestVRFv2Plus$/^Link_Billing$ smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
-    test_secrets_required: true
-    workflows:
-      - On Demand VRFV2Plus Smoke Test (Ethereum clients)
+
+  - id: TestVRFv2Plus_NativeBilling
+    path: integration-tests/smoke/vrfv2plus_test.go
+    runs_on: ubuntu-latest
+    test_env_type: docker
+    test_cmd: cd integration-tests && go test -v -test.run ^TestVRFv2Plus$/^Native_Billing$ smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
 
   - id: smoke/vrf_test.go:*
     path: integration-tests/smoke/vrf_test.go
diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml
new file mode 100644
index 00000000000..aeac10a5ccc
--- /dev/null
+++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml
@@ -0,0 +1,47 @@
+name: On Demand VRFV2 Plus Smoke Tests 2
+on:
+  workflow_dispatch:
+    inputs:
+      test_ids:
+        description: Commas separated list of test IDs to run (see .github/e2e-tests.yml)
+        required: false
+        default: TestVRFv2Plus_LinkBilling,TestVRFv2Plus_NativeBilling
+      test_config_override_path:
+        description: Path to a test config file used to override the default test config
+        required: false
+        type: string          
+      test_secrets_override_key:
+        description: 'Key to run tests with custom test secrets'
+        required: false
+        type: string
+      chainlink_version:
+        description: 'Enter Chainlink version to use for the tests. Example: develop or sha'
+        required: false
+        type: string
+        default: develop
+
+jobs:
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      chainlink_version: ${{ inputs.chainlink_version }}
+      test_ids: ${{ inputs.test_ids }}
+      test_config_override_path: ${{ inputs.test_config_override_path }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index ab064531fd3..4e5409ce655 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -45,8 +45,6 @@ jobs:
     with:
       chainlink_version: ${{ github.event.inputs.chainlink_version }}
       test_ids: ${{ github.event.inputs.test_ids }}
-      # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 
-      # test_config_override_base64: ${{ github.event.inputs.test_config_override_base64 }}
       with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
       # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677
       enable_check_test_configurations: ${{ fromJSON(github.event.inputs.enable_check_test_configurations) }}

From 40ac40c92baa81b87bfa70a7dc19b0dcc6237778 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 12:26:52 +0200
Subject: [PATCH 030/141] Show test config path in reusable workflow

---
 .../run-e2e-tests-reusable-workflow.yml       | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 5da4a24d5ed..637205229a2 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -419,6 +419,7 @@ jobs:
       contents: read
     env:
       LATEST_CHAINLINK_RELEASE_VERSION: ${{ needs.get_latest_chainlink_release_version.outputs.latest_chainlink_release_version }}
+      TEST_CONFIG_OVERRIDE_PATH: ${{ matrix.tests.test_config_override_path || inputs.test_config_override_path }}
     steps:
       - name: Collect Metrics
         if: always()
@@ -437,6 +438,13 @@ jobs:
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
       - name: Install jq
         run: sudo apt-get install -y jq
+
+      - name: Show test config override path in summary
+        if: ${{ env.TEST_CONFIG_OVERRIDE_PATH }}
+        run: |
+          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
+          echo "[${{ env.TEST_CONFIG_OVERRIDE_PATH }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ env.TEST_CONFIG_OVERRIDE_PATH }})" >> $GITHUB_STEP_SUMMARY
+
       - name: Show test configuration
         run: echo '${{ toJson(matrix.tests) }}' | jq .
 
@@ -504,7 +512,7 @@ jobs:
           test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd $(dirname ${{ matrix.tests.path }}) && go mod download
           test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}
-          test_config_override_path: ${{ matrix.tests.test_config_override_path || inputs.test_config_override_path }}
+          test_config_override_path: ${{ env.TEST_CONFIG_OVERRIDE_PATH }}
           test_type: ${{ matrix.tests.test_env_vars.TEST_TYPE }}
           test_suite: ${{ matrix.tests.test_env_vars.TEST_SUITE }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
@@ -636,6 +644,7 @@ jobs:
       contents: read
     env:
       LATEST_CHAINLINK_RELEASE_VERSION: ${{ needs.get_latest_chainlink_release_version.outputs.latest_chainlink_release_version }}
+      TEST_CONFIG_OVERRIDE_PATH: ${{ matrix.tests.test_config_override_path || inputs.test_config_override_path }}
     steps:
       - name: Collect Metrics
         if: always()
@@ -653,6 +662,13 @@ jobs:
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
       - name: Install jq
         run: sudo apt-get install -y jq
+
+      - name: Show test config override path in summary
+        if: ${{ env.TEST_CONFIG_OVERRIDE_PATH }}
+        run: |
+          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
+          echo "[${{ env.TEST_CONFIG_OVERRIDE_PATH }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ env.TEST_CONFIG_OVERRIDE_PATH }})" >> $GITHUB_STEP_SUMMARY
+
       - name: Show Test Configuration
         run: echo '${{ toJson(matrix.tests) }}' | jq .
       - name: Show Remote Runner Version
@@ -696,7 +712,7 @@ jobs:
           test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: make gomod
           test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}
-          test_config_override_path: ${{ matrix.tests.test_config_override_path || inputs.test_config_override_path }}
+          test_config_override_path: ${{ env.TEST_CONFIG_OVERRIDE_PATH }}
           test_type: ${{ matrix.tests.test_env_vars.TEST_TYPE }}
           test_suite: ${{ matrix.tests.test_env_vars.TEST_SUITE }}
           token: ${{ secrets.GH_TOKEN }}

From 66fe2265ef3a053853c25e09f5957fdc439c0d66 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 12:31:55 +0200
Subject: [PATCH 031/141] Update list of tests

---
 .github/e2e-tests.yml                                | 12 ++++++++++++
 .../workflows/on-demand-vrfv2plus-smoke-tests-2.yml  |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 072fc47e554..ddd877eda21 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -690,6 +690,18 @@ runner-test-matrix:
     test_env_type: docker
     test_cmd: cd integration-tests && go test -v -test.run ^TestVRFv2Plus$/^Native_Billing$ smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
 
+  - id: TestVRFv2Plus_DirectFunding
+    path: integration-tests/smoke/vrfv2plus_test.go
+    runs_on: ubuntu-latest
+    test_env_type: docker
+    test_cmd: cd integration-tests && go test -v -test.run ^TestVRFv2Plus$/^DirectFunding$ smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
+
+  - id: TestVRFV2PlusWithBHS
+    path: integration-tests/smoke/vrfv2plus_test.go
+    runs_on: ubuntu-latest
+    test_env_type: docker
+    test_cmd: cd integration-tests && go test -v -test.run TestVRFV2PlusWithBHS smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
+
   - id: smoke/vrf_test.go:*
     path: integration-tests/smoke/vrf_test.go
     test_env_type: docker
diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml
index aeac10a5ccc..879ce1ecfa2 100644
--- a/.github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml
+++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml
@@ -5,7 +5,7 @@ on:
       test_ids:
         description: Commas separated list of test IDs to run (see .github/e2e-tests.yml)
         required: false
-        default: TestVRFv2Plus_LinkBilling,TestVRFv2Plus_NativeBilling
+        default: TestVRFv2Plus_LinkBilling,TestVRFv2Plus_NativeBilling,TestVRFv2Plus_DirectFunding,TestVRFV2PlusWithBHS
       test_config_override_path:
         description: Path to a test config file used to override the default test config
         required: false

From e28054aee51e36c1ff38d35c6d3504ab6b975003 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 12:33:39 +0200
Subject: [PATCH 032/141] Update run-selected-tests workflow

---
 .github/workflows/run-selected-e2e-tests.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 4e5409ce655..c204be6a564 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -16,11 +16,10 @@ on:
         description: 'Enter the secret key to override test secrets'
         required: false
         type: string 
-      # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 
-      # test_config_override_base64:
-      #   required: false
-      #   description: The base64-encoded test config override
-      #   type: string        
+      test_config_override_path:
+        description: 'Path to a test config file used to override the default test config'
+        required: false
+        type: string
       enable_check_test_configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -45,6 +44,7 @@ jobs:
     with:
       chainlink_version: ${{ github.event.inputs.chainlink_version }}
       test_ids: ${{ github.event.inputs.test_ids }}
+      test_config_override_path: ${{ github.event.inputs.test_config_override_path }}
       with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
       # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677
       enable_check_test_configurations: ${{ fromJSON(github.event.inputs.enable_check_test_configurations) }}

From 92ff1eca7b105b73c404d9cc9aa2f89723ac3e7e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 12:46:34 +0200
Subject: [PATCH 033/141] Fix

---
 .../run-e2e-tests-reusable-workflow.yml       | 46 +++++++++++++++----
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 637205229a2..f23678bd05f 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -160,16 +160,29 @@ jobs:
           fi
       - name: Install jq
         run: sudo apt-get install jq
+
       - name: Create matrix for required Chainlink image versions
         id: set-required-chainlink-image-versions-matrix
         run: |
-          if [[ "${{ inputs.require_chainlink_image_versions_in_qa_ecr }}" != '' ]]; then
-            image_versions=$(echo "${{ inputs.require_chainlink_image_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then [] else . end')
-            echo "versions=$image_versions" >> $GITHUB_OUTPUT
-          elif [[ "${{ env.DEFAULT_CHAINLINK_VERSION }}" == "${{ github.sha }}" ]]; then
-            echo "Setting ${{env.DEFAULT_CHAINLINK_VERSION}} as a required Chainlink image version in QA ECR"
-            echo "versions=$env.DEFAULT_CHAINLINK_VERSION" >> $GITHUB_OUTPUT
+          image_versions="${{ inputs.require_chainlink_image_versions_in_qa_ecr }}"
+          default_version="${{ env.DEFAULT_CHAINLINK_VERSION }}"
+          current_sha="${{ github.sha }}"
+
+          if [[ "$default_version" == "$current_sha" ]]; then
+            # Append the current SHA to required image versions
+            if [[ -z "$image_versions" ]]; then
+              image_versions="${{ github.sha }}"
+            else
+              image_versions+=",${{ github.sha }}"
+            fi            
           fi
+
+          # Convert the comma-separated string to a JSON array
+          image_versions=$(echo "$image_versions" | jq -Rc 'split(",") | if . == [""] then [] else . end')
+
+          echo "Required Chainlink image versions: $image_versions"   
+          echo "versions=$image_versions" >> $GITHUB_OUTPUT
+
       - name: Create matrix for required Chainlink plugin versions
         id: set-required-chainlink-plugin-versions-matrix
         run: |
@@ -445,7 +458,12 @@ jobs:
           echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
           echo "[${{ env.TEST_CONFIG_OVERRIDE_PATH }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ env.TEST_CONFIG_OVERRIDE_PATH }})" >> $GITHUB_STEP_SUMMARY
 
-      - name: Show test configuration
+      - name: Show chainlink version in summary
+        run: |
+          echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
+          echo "${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}" >> $GITHUB_STEP_SUMMARY
+      
+      - name: Show test configuration in logs
         run: echo '${{ toJson(matrix.tests) }}' | jq .
 
       - name: Setup GAP for Grafana
@@ -669,11 +687,19 @@ jobs:
           echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
           echo "[${{ env.TEST_CONFIG_OVERRIDE_PATH }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ env.TEST_CONFIG_OVERRIDE_PATH }})" >> $GITHUB_STEP_SUMMARY
 
-      - name: Show Test Configuration
-        run: echo '${{ toJson(matrix.tests) }}' | jq .
-      - name: Show Remote Runner Version
+      - name: Show chainlink version in summary
+        run: |
+          echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
+          echo "${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}" >> $GITHUB_STEP_SUMMARY                    
+
+      - name: Show remote runner version in summary
         run: |
           echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
+          echo "### Remote Runner Version" >> $GITHUB_STEP_SUMMARY
+          echo "${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}" >> $GITHUB_STEP_SUMMARY
+
+      - name: Show test configuration in logs
+        run: echo '${{ toJson(matrix.tests) }}' | jq .
 
       - name: Run tests
         id: run_tests

From 4e9998e1adb95453b4b4668018aff05b30aa821f Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 13:00:36 +0200
Subject: [PATCH 034/141] Rename steps

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index f23678bd05f..a8dd7e42a0d 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -374,7 +374,7 @@ jobs:
       - name: Checkout the repo
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
 
-      - name: Build Chainlink image for ${{ matrix.version }} and push it to QA ECR
+      - name: Build Chainlink image
         uses: ./.github/actions/build-chainlink-image
         with:
           dockerfile: core/chainlink.Dockerfile
@@ -403,7 +403,7 @@ jobs:
       - name: Checkout the repo
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
 
-      - name: Build Chainlink plugins image for ${{ matrix.version }}
+      - name: Build Chainlink plugins image
         uses: ./.github/actions/build-chainlink-image
         with:
           dockerfile: plugins/chainlink.Dockerfile

From 798db7ac392d857d53e9d94641548d9f1b8e5c87 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 13:48:27 +0200
Subject: [PATCH 035/141] fix workflow

---
 .github/workflows/automation-benchmark-tests.yml | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml
index d9a752dcccb..7dbc6475f21 100644
--- a/.github/workflows/automation-benchmark-tests.yml
+++ b/.github/workflows/automation-benchmark-tests.yml
@@ -20,7 +20,12 @@ on:
         required: true
         default: U02Q14G80TY
         type: string
-  
+      chainlink_version:
+        description: Chainlink image version to use
+        default: develop
+        required: true
+        type: string
+
 jobs:
   automation_benchmark:
     environment: integration
@@ -51,7 +56,12 @@ jobs:
         run: |
           echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
           echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY
+      - name: Show chainlink version in summary
+        run: |
+          echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
+          echo "${{ inputs.chainlink_version }}" >> $GITHUB_STEP_SUMMARY          
       - name: Build Test Image
+        id: build_test_image
         uses: ./.github/actions/build-test-image
         with:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
@@ -64,14 +74,14 @@ jobs:
           DETACH_RUNNER: true
           TEST_SUITE: benchmark
           TEST_ARGS: -test.timeout 720h
-          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
+          ENV_JOB_IMAGE: ${{ steps.build_test_image.outputs.test_image }}
           INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
           TEST_TYPE: ${{ github.event.inputs.testType }}
           TEST_TEST_TYPE: ${{ github.event.inputs.testType }}
           RR_MEM: 4Gi
           TEST_LOG_LEVEL: info
           E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-          E2E_TEST_CHAINLINK_VERSION: ${{ github.sha }} # Default chainlink version if not overrided in the test config 
+          E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_version }}
         with:
           test_command_to_run: cd integration-tests && go test -timeout 30m -v -run ^TestAutomationBenchmark$ ./benchmark -count=1
           test_download_vendor_packages_command: make gomod

From 1b28f4b15fd3ff4a87b76ce0806c8452ed065a45 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 14:35:44 +0200
Subject: [PATCH 036/141] Add TestVRFv2Plus_LiveTestnets to CI tests definition

---
 .github/e2e-tests.yml                         |  7 +++
 .../on-demand-vrfv2plus-smoke-tests-2.yml     | 47 -------------------
 2 files changed, 7 insertions(+), 47 deletions(-)
 delete mode 100644 .github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index ddd877eda21..4e31a14f5ac 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -702,6 +702,13 @@ runner-test-matrix:
     test_env_type: docker
     test_cmd: cd integration-tests && go test -v -test.run TestVRFV2PlusWithBHS smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
 
+  # Run all VRFv2Plus tests on live testnets (test_config_override_path required)
+  - id: TestVRFv2Plus_LiveTestnets
+    path: integration-tests/smoke/vrfv2plus_test.go
+    runs_on: ubuntu-latest
+    test_env_type: docker
+    test_cmd: cd integration-tests && go test -v -test.run "TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS" -test.parallel=1 -timeout 2h -count=1 -json
+
   - id: smoke/vrf_test.go:*
     path: integration-tests/smoke/vrf_test.go
     test_env_type: docker
diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml
deleted file mode 100644
index 879ce1ecfa2..00000000000
--- a/.github/workflows/on-demand-vrfv2plus-smoke-tests-2.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-name: On Demand VRFV2 Plus Smoke Tests 2
-on:
-  workflow_dispatch:
-    inputs:
-      test_ids:
-        description: Commas separated list of test IDs to run (see .github/e2e-tests.yml)
-        required: false
-        default: TestVRFv2Plus_LinkBilling,TestVRFv2Plus_NativeBilling,TestVRFv2Plus_DirectFunding,TestVRFV2PlusWithBHS
-      test_config_override_path:
-        description: Path to a test config file used to override the default test config
-        required: false
-        type: string          
-      test_secrets_override_key:
-        description: 'Key to run tests with custom test secrets'
-        required: false
-        type: string
-      chainlink_version:
-        description: 'Enter Chainlink version to use for the tests. Example: develop or sha'
-        required: false
-        type: string
-        default: develop
-
-jobs:
-  run-e2e-tests-workflow:
-    name: Run E2E Tests
-    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
-    with:
-      chainlink_version: ${{ inputs.chainlink_version }}
-      test_ids: ${{ inputs.test_ids }}
-      test_config_override_path: ${{ inputs.test_config_override_path }}
-    secrets:
-      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
-      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
-      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

From 139cde3a660ad65987e92b133c6a5c05a07e092d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 14:52:29 +0200
Subject: [PATCH 037/141] Fix

---
 .github/e2e-tests.yml                                 |  1 +
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 11 +++++------
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 4e31a14f5ac..1ee1198e316 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -703,6 +703,7 @@ runner-test-matrix:
     test_cmd: cd integration-tests && go test -v -test.run TestVRFV2PlusWithBHS smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
 
   # Run all VRFv2Plus tests on live testnets (test_config_override_path required)
+  # Tests have to run in sequence because of a single private key used
   - id: TestVRFv2Plus_LiveTestnets
     path: integration-tests/smoke/vrfv2plus_test.go
     runs_on: ubuntu-latest
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index a8dd7e42a0d..5d4ab78311e 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -186,10 +186,9 @@ jobs:
       - name: Create matrix for required Chainlink plugin versions
         id: set-required-chainlink-plugin-versions-matrix
         run: |
-          if [[ "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" != '' ]]; then
-            image_versions=$(echo "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then [] else . end')
-            echo "versions=$image_versions" >> $GITHUB_OUTPUT
-          fi          
+          image_versions=$(echo "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then [] else . end')
+          echo "Required Chainlink plugin image versions: $image_versions"   
+          echo "versions=$image_versions" >> $GITHUB_OUTPUT
 
   check-test-configurations:
     name: Check test configurations
@@ -359,7 +358,7 @@ jobs:
   require-chainlink-image-versions-in-qa-ecr:
     name: Build Chainlink image
     needs: [validate-inputs, load-test-configurations]
-    if: ${{ needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix != '' }}
+    if: fromJson(needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix).length > 0
     runs-on: ubuntu-latest
     environment: integration
     permissions:
@@ -388,7 +387,7 @@ jobs:
   require-chainlink-plugin-versions-in-qa-ecr:
     name: Build Chainlink plugins
     needs: [validate-inputs, load-test-configurations]
-    if: ${{ needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix != '' }}
+    if: fromJson(needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix).length > 0
     runs-on: ubuntu-latest
     environment: integration
     permissions:

From abf6fd84c1d7335e35fc488f0ad97d2281f03862 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 14:58:30 +0200
Subject: [PATCH 038/141] Run vrfv2plus tests on simulated network by default

---
 integration-tests/testconfig/vrfv2plus/vrfv2plus.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/integration-tests/testconfig/vrfv2plus/vrfv2plus.toml b/integration-tests/testconfig/vrfv2plus/vrfv2plus.toml
index 1b43d73e174..84af6d965bc 100644
--- a/integration-tests/testconfig/vrfv2plus/vrfv2plus.toml
+++ b/integration-tests/testconfig/vrfv2plus/vrfv2plus.toml
@@ -1,4 +1,8 @@
 # default config
+
+[Network]
+selected_networks = ["SIMULATED"]
+
 [NodeConfig]
 BaseConfigTOML = """
 [Feature]

From 61e1255198396054e3559551e1de3f947102ddc3 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 15:01:57 +0200
Subject: [PATCH 039/141] Fix

---
 .github/e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 1ee1198e316..7e72d3f1f0b 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -708,7 +708,7 @@ runner-test-matrix:
     path: integration-tests/smoke/vrfv2plus_test.go
     runs_on: ubuntu-latest
     test_env_type: docker
-    test_cmd: cd integration-tests && go test -v -test.run "TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS" -test.parallel=1 -timeout 2h -count=1 -json
+    test_cmd: cd integration-tests/smoke && go test -v -test.run "TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS" -test.parallel=1 -timeout 2h -count=1 -json
 
   - id: smoke/vrf_test.go:*
     path: integration-tests/smoke/vrf_test.go

From 474fcbb028d4e980a394be44ea779e01037c1ead Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 15:10:57 +0200
Subject: [PATCH 040/141] fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 5d4ab78311e..2bebc7a5bf9 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -178,7 +178,7 @@ jobs:
           fi
 
           # Convert the comma-separated string to a JSON array
-          image_versions=$(echo "$image_versions" | jq -Rc 'split(",") | if . == [""] then [] else . end')
+          image_versions=$(echo "$image_versions" | jq -Rc 'split(",") | if . == [""] then "" else . end')
 
           echo "Required Chainlink image versions: $image_versions"   
           echo "versions=$image_versions" >> $GITHUB_OUTPUT
@@ -186,7 +186,7 @@ jobs:
       - name: Create matrix for required Chainlink plugin versions
         id: set-required-chainlink-plugin-versions-matrix
         run: |
-          image_versions=$(echo "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then [] else . end')
+          image_versions=$(echo "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then "" else . end')
           echo "Required Chainlink plugin image versions: $image_versions"   
           echo "versions=$image_versions" >> $GITHUB_OUTPUT
 
@@ -358,7 +358,7 @@ jobs:
   require-chainlink-image-versions-in-qa-ecr:
     name: Build Chainlink image
     needs: [validate-inputs, load-test-configurations]
-    if: fromJson(needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix).length > 0
+    if: needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix != ''
     runs-on: ubuntu-latest
     environment: integration
     permissions:
@@ -387,7 +387,7 @@ jobs:
   require-chainlink-plugin-versions-in-qa-ecr:
     name: Build Chainlink plugins
     needs: [validate-inputs, load-test-configurations]
-    if: fromJson(needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix).length > 0
+    if: needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix != ''
     runs-on: ubuntu-latest
     environment: integration
     permissions:

From 472568491f6ee25d962f73d300eb0ab1720be7ed Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 15:21:27 +0200
Subject: [PATCH 041/141] fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 2bebc7a5bf9..5364d9a7ddb 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -178,7 +178,7 @@ jobs:
           fi
 
           # Convert the comma-separated string to a JSON array
-          image_versions=$(echo "$image_versions" | jq -Rc 'split(",") | if . == [""] then "" else . end')
+          image_versions=$(echo "$image_versions" | jq -Rc 'if . == "" then "" else split(",") | if . == [""] then "" else . end end')
 
           echo "Required Chainlink image versions: $image_versions"   
           echo "versions=$image_versions" >> $GITHUB_OUTPUT
@@ -186,7 +186,7 @@ jobs:
       - name: Create matrix for required Chainlink plugin versions
         id: set-required-chainlink-plugin-versions-matrix
         run: |
-          image_versions=$(echo "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then "" else . end')
+          image_versions=$(echo "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" | jq -Rc 'if . == "" then "" else split(",") | if . == [""] then "" else . end end')
           echo "Required Chainlink plugin image versions: $image_versions"   
           echo "versions=$image_versions" >> $GITHUB_OUTPUT
 
@@ -358,7 +358,7 @@ jobs:
   require-chainlink-image-versions-in-qa-ecr:
     name: Build Chainlink image
     needs: [validate-inputs, load-test-configurations]
-    if: needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix != ''
+    if: ${{ fromJson(needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix) != '' }}
     runs-on: ubuntu-latest
     environment: integration
     permissions:
@@ -387,7 +387,7 @@ jobs:
   require-chainlink-plugin-versions-in-qa-ecr:
     name: Build Chainlink plugins
     needs: [validate-inputs, load-test-configurations]
-    if: needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix != ''
+    if: ${{ fromJson(needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix) != '' }}  
     runs-on: ubuntu-latest
     environment: integration
     permissions:

From 722a88ce380733006bee42949e9eace3d5a6fa2d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 15:34:48 +0200
Subject: [PATCH 042/141] Update test definition

---
 .github/e2e-tests.yml | 40 ++++++++--------------------------------
 1 file changed, 8 insertions(+), 32 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 7e72d3f1f0b..7a476903c21 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -678,38 +678,6 @@ runner-test-matrix:
     workflows:
       - On Demand VRFV2 Performance Test
 
-  - id: TestVRFv2Plus_LinkBilling
-    path: integration-tests/smoke/vrfv2plus_test.go
-    runs_on: ubuntu-latest
-    test_env_type: docker
-    test_cmd: cd integration-tests && go test -v -test.run ^TestVRFv2Plus$/^Link_Billing$ smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
-
-  - id: TestVRFv2Plus_NativeBilling
-    path: integration-tests/smoke/vrfv2plus_test.go
-    runs_on: ubuntu-latest
-    test_env_type: docker
-    test_cmd: cd integration-tests && go test -v -test.run ^TestVRFv2Plus$/^Native_Billing$ smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
-
-  - id: TestVRFv2Plus_DirectFunding
-    path: integration-tests/smoke/vrfv2plus_test.go
-    runs_on: ubuntu-latest
-    test_env_type: docker
-    test_cmd: cd integration-tests && go test -v -test.run ^TestVRFv2Plus$/^DirectFunding$ smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
-
-  - id: TestVRFV2PlusWithBHS
-    path: integration-tests/smoke/vrfv2plus_test.go
-    runs_on: ubuntu-latest
-    test_env_type: docker
-    test_cmd: cd integration-tests && go test -v -test.run TestVRFV2PlusWithBHS smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
-
-  # Run all VRFv2Plus tests on live testnets (test_config_override_path required)
-  # Tests have to run in sequence because of a single private key used
-  - id: TestVRFv2Plus_LiveTestnets
-    path: integration-tests/smoke/vrfv2plus_test.go
-    runs_on: ubuntu-latest
-    test_env_type: docker
-    test_cmd: cd integration-tests/smoke && go test -v -test.run "TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS" -test.parallel=1 -timeout 2h -count=1 -json
-
   - id: smoke/vrf_test.go:*
     path: integration-tests/smoke/vrf_test.go
     test_env_type: docker
@@ -740,6 +708,14 @@ runner-test-matrix:
     test_cmd: cd integration-tests/ && go test smoke/vrfv2plus_test.go -timeout 30m -count=1 -test.parallel=9 -json
     pyroscope_env: ci-smoke-vrf2plus-evm-simulated
 
+  # Run VRFv2Plus tests on live testnets (test_config_override_path required)
+  # Tests have to run in sequence because of a single private key used
+  - id: TestVRFv2Plus_LiveTestnets
+    path: integration-tests/smoke/vrfv2plus_test.go
+    runs_on: ubuntu-latest
+    test_env_type: docker
+    test_cmd: cd integration-tests/smoke && go test -v -test.run "TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS" -test.parallel=1 -timeout 2h -count=1 -json
+
   # END: VRF tests
 
   # START: LogPoller tests

From edc5f1e1337c1704ffe8b024debf3af988bb7da6 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 15:42:54 +0200
Subject: [PATCH 043/141] Add TestVRFv2Plus_Release_Sepolia

---
 .github/e2e-tests.yml | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 7a476903c21..25ed541298f 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -708,7 +708,7 @@ runner-test-matrix:
     test_cmd: cd integration-tests/ && go test smoke/vrfv2plus_test.go -timeout 30m -count=1 -test.parallel=9 -json
     pyroscope_env: ci-smoke-vrf2plus-evm-simulated
 
-  # Run VRFv2Plus tests on live testnets (test_config_override_path required)
+  # VRFv2Plus tests on any live testnet (test_config_override_path required)
   # Tests have to run in sequence because of a single private key used
   - id: TestVRFv2Plus_LiveTestnets
     path: integration-tests/smoke/vrfv2plus_test.go
@@ -716,6 +716,16 @@ runner-test-matrix:
     test_env_type: docker
     test_cmd: cd integration-tests/smoke && go test -v -test.run "TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS" -test.parallel=1 -timeout 2h -count=1 -json
 
+  # VRFv2Plus release tests on Sepolia testnet
+  - id: TestVRFv2Plus_Release_Sepolia
+    path: integration-tests/smoke/vrfv2plus_test.go
+    runs_on: ubuntu-latest
+    test_env_type: docker
+    test_cmd: cd integration-tests/smoke && go test -v -test.run "TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS" -test.parallel=1 -timeout 2h -count=1 -json
+    test_config_override_path: integration-tests/testconfig/vrfv2plus/overrides/release_testing/sepolia_release_test_config.toml
+    # workflows:
+    #   - Nightly E2E Tests
+
   # END: VRF tests
 
   # START: LogPoller tests

From 304e36a6adfec2dd3f6e1b451c5ac22d2497b218 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 15:51:16 +0200
Subject: [PATCH 044/141] Add workflow to run vrf e2e release tests

---
 .github/e2e-tests.yml                       |  7 ++--
 .github/workflows/vrf-e2e-release-tests.yml | 38 +++++++++++++++++++++
 2 files changed, 42 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/vrf-e2e-release-tests.yml

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 25ed541298f..bf52535b65f 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -722,9 +722,10 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     test_env_type: docker
     test_cmd: cd integration-tests/smoke && go test -v -test.run "TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS" -test.parallel=1 -timeout 2h -count=1 -json
-    test_config_override_path: integration-tests/testconfig/vrfv2plus/overrides/release_testing/sepolia_release_test_config.toml
-    # workflows:
-    #   - Nightly E2E Tests
+    # test_config_override_path: integration-tests/testconfig/vrfv2plus/overrides/release_testing/sepolia_release_test_config.toml
+    test_config_override_path: integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
+    workflows:
+      - VRF E2E Release Tests
 
   # END: VRF tests
 
diff --git a/.github/workflows/vrf-e2e-release-tests.yml b/.github/workflows/vrf-e2e-release-tests.yml
new file mode 100644
index 00000000000..33b6ee51476
--- /dev/null
+++ b/.github/workflows/vrf-e2e-release-tests.yml
@@ -0,0 +1,38 @@
+name: Run VRF E2E Release Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      chainlink_version:
+        description: Chainlink image version to use
+        default: develop
+        required: true
+        type: string
+  push:
+
+jobs:
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      chainlink_version: ${{ github.event.inputs.chainlink_version }}
+      test_workflow: VRF E2E Release Tests
+      slack_notification_after_tests: true
+      slack_notification_after_tests_channel_id: "#team-test-tooling-internal"
+      slack_notification_after_tests_name: JUST TESTING PLEASE IGNORE - VRF E2E Release Tests
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}    
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

From 97c0b79fd6bf6faffd3ffda77fea9df2006a6606 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:15:36 +0200
Subject: [PATCH 045/141] test new workflow with slack notification

---
 .../run-e2e-tests-reusable-workflow.yml       | 30 +++++++++++++++++++
 .github/workflows/vrf-e2e-release-tests.yml   |  7 +++--
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 5364d9a7ddb..dd63ae69591 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -62,6 +62,10 @@ on:
         description: 'Name of the slack notification'
         required: false
         type: string
+      slack_notification_after_tests_notify_user_id_on_failure:
+        description: 'Set Slack user id to notify on test failure'
+        required: false
+        type: string
       test_log_upload_on_failure:
         description: 'Set to "true" to upload the test log on failure as Github artifact'
         required: false
@@ -820,6 +824,32 @@ jobs:
                 }
               ]
             }
+        
+      - name: Notify user in Slack message if tests failed
+        # if: ${{ inputs.slack_notification_after_tests == 'on_failure' && contains(join(needs.*.result, ','), 'failure') && inputs.slack_notification_after_tests_notify_user_id_on_failure != '' }}
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        with:
+          channel-id: ${{ inputs.slack_notification_after_tests_channel_id }}
+          payload: |
+            {
+              "thread_ts": "${{ steps.slack.outputs.thread_ts }}",
+              "attachments": [
+                {
+                  "color": "${{ '#C62828' }}",
+                  "blocks": [
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "Notifying <@${{ inputs.slack_notification_after_tests_notify_user_id_on_failure }}>, please check the test results."
+                      }
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
 
   # Run K8s tests using new remote runner
   # remote-runner-k8s-tests:
diff --git a/.github/workflows/vrf-e2e-release-tests.yml b/.github/workflows/vrf-e2e-release-tests.yml
index 33b6ee51476..70923e01471 100644
--- a/.github/workflows/vrf-e2e-release-tests.yml
+++ b/.github/workflows/vrf-e2e-release-tests.yml
@@ -16,10 +16,13 @@ jobs:
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
       chainlink_version: ${{ github.event.inputs.chainlink_version }}
-      test_workflow: VRF E2E Release Tests
+      # test_workflow: VRF E2E Release Tests
+      test_ids: 'smoke/vrfv2plus_test.go:*'
       slack_notification_after_tests: true
-      slack_notification_after_tests_channel_id: "#team-test-tooling-internal"
+      # slack_notification_after_tests_channel_id: "#team-test-tooling-internal"
+      slack_notification_after_tests_channel_id: "#lf-slack-notification-tests"
       slack_notification_after_tests_name: JUST TESTING PLEASE IGNORE - VRF E2E Release Tests
+      slack_notification_after_tests_notify_user_id_on_failure: 'D04ENK9VDFA'
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From db4d5fae14c69d936d052b29683beed5ddcbdaa4 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:16:44 +0200
Subject: [PATCH 046/141] fix

---
 .github/workflows/vrf-e2e-release-tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/vrf-e2e-release-tests.yml b/.github/workflows/vrf-e2e-release-tests.yml
index 70923e01471..78d12bddc92 100644
--- a/.github/workflows/vrf-e2e-release-tests.yml
+++ b/.github/workflows/vrf-e2e-release-tests.yml
@@ -8,7 +8,6 @@ on:
         default: develop
         required: true
         type: string
-  push:
 
 jobs:
   run-e2e-tests-workflow:

From c5b1cace792c652d099ae1b1e0bd9cbb849818f0 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:34:02 +0200
Subject: [PATCH 047/141] fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index dd63ae69591..142cb2f17b3 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -828,6 +828,8 @@ jobs:
       - name: Notify user in Slack message if tests failed
         # if: ${{ inputs.slack_notification_after_tests == 'on_failure' && contains(join(needs.*.result, ','), 'failure') && inputs.slack_notification_after_tests_notify_user_id_on_failure != '' }}
         uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
         with:
           channel-id: ${{ inputs.slack_notification_after_tests_channel_id }}
           payload: |
@@ -848,8 +850,6 @@ jobs:
                 }
               ]
             }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
 
   # Run K8s tests using new remote runner
   # remote-runner-k8s-tests:

From 73dd7d5781b5d1fd4bf8fdda4617e47387894996 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:39:21 +0200
Subject: [PATCH 048/141] fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 .github/workflows/vrf-e2e-release-tests.yml           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 142cb2f17b3..1f2930f16fe 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -807,6 +807,7 @@ jobs:
           channel-id: ${{ inputs.slack_notification_after_tests_channel_id }}
           payload: |
             {
+              "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || '#2E7D32' }}",
               "blocks": [
                 {
                   "type": "section",
@@ -837,7 +838,6 @@ jobs:
               "thread_ts": "${{ steps.slack.outputs.thread_ts }}",
               "attachments": [
                 {
-                  "color": "${{ '#C62828' }}",
                   "blocks": [
                     {
                       "type": "section",
diff --git a/.github/workflows/vrf-e2e-release-tests.yml b/.github/workflows/vrf-e2e-release-tests.yml
index 78d12bddc92..552d96a8f6a 100644
--- a/.github/workflows/vrf-e2e-release-tests.yml
+++ b/.github/workflows/vrf-e2e-release-tests.yml
@@ -21,7 +21,7 @@ jobs:
       # slack_notification_after_tests_channel_id: "#team-test-tooling-internal"
       slack_notification_after_tests_channel_id: "#lf-slack-notification-tests"
       slack_notification_after_tests_name: JUST TESTING PLEASE IGNORE - VRF E2E Release Tests
-      slack_notification_after_tests_notify_user_id_on_failure: 'D04ENK9VDFA'
+      slack_notification_after_tests_notify_user_id_on_failure: 'U04DYU1KLGJ'
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From 6df843a20854df864aa7de650b776d6f75c312fc Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:43:18 +0200
Subject: [PATCH 049/141] fix color

---
 .../run-e2e-tests-reusable-workflow.yml       | 34 +++++++++++--------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 1f2930f16fe..d5f647539ea 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -807,21 +807,25 @@ jobs:
           channel-id: ${{ inputs.slack_notification_after_tests_channel_id }}
           payload: |
             {
-              "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || '#2E7D32' }}",
-              "blocks": [
-                {
-                  "type": "section",
-                  "text": {
-                    "type": "mrkdwn",
-                    "text": "${{ inputs.slack_notification_after_tests_name }} - ${{ contains(join(needs.*.result, ','), 'failure') && 'Failed :x:' || contains(join(needs.*.result, ','), 'cancelled') && 'Cancelled :warning:' || 'Passed :white_check_mark:' }}"
-                  }
-                },
+              "attachments": [
                 {
-                  "type": "section",
-                  "text": {
-                    "type": "mrkdwn",
-                    "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Build Details>"
-                  }
+                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || '#2E7D32' }}",
+                  "blocks": [
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "${{ inputs.slack_notification_after_tests_name }} - ${{ contains(join(needs.*.result, ','), 'failure') && 'Failed :x:' || contains(join(needs.*.result, ','), 'cancelled') && 'Cancelled :warning:' || 'Passed :white_check_mark:' }}"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Build Details>"
+                      }
+                    }
+                  ]
                 }
               ]
             }
@@ -842,7 +846,7 @@ jobs:
                     {
                       "type": "section",
                       "text": {
-                        "type": "mrkdwn",
+                        "type": "plain_text",
                         "text": "Notifying <@${{ inputs.slack_notification_after_tests_notify_user_id_on_failure }}>, please check the test results."
                       }
                     }

From f0a084f0bcb915d38041b2b1903839562cb22c7f Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:45:05 +0200
Subject: [PATCH 050/141] fix

---
 .../run-e2e-tests-reusable-workflow.yml          | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index d5f647539ea..be202fc469d 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -840,17 +840,13 @@ jobs:
           payload: |
             {
               "thread_ts": "${{ steps.slack.outputs.thread_ts }}",
-              "attachments": [
+              "blocks": [
                 {
-                  "blocks": [
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "plain_text",
-                        "text": "Notifying <@${{ inputs.slack_notification_after_tests_notify_user_id_on_failure }}>, please check the test results."
-                      }
-                    }
-                  ]
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "Notifying <@${{ inputs.slack_notification_after_tests_notify_user_id_on_failure }}>, please check the test results."
+                  }
                 }
               ]
             }

From 60defc9f1525bf2582658871b8b44ac2d103670e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:47:22 +0200
Subject: [PATCH 051/141] revert if

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index be202fc469d..7c6f4771184 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -831,7 +831,7 @@ jobs:
             }
         
       - name: Notify user in Slack message if tests failed
-        # if: ${{ inputs.slack_notification_after_tests == 'on_failure' && contains(join(needs.*.result, ','), 'failure') && inputs.slack_notification_after_tests_notify_user_id_on_failure != '' }}
+        if: ${{ inputs.slack_notification_after_tests == 'on_failure' && contains(join(needs.*.result, ','), 'failure') && inputs.slack_notification_after_tests_notify_user_id_on_failure != '' }}
         uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

From 01c42597a4d6e2734ea3c2960cac54528d580bc1 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:48:27 +0200
Subject: [PATCH 052/141] revert test definition changes

---
 .github/e2e-tests.yml                       | 3 +--
 .github/workflows/vrf-e2e-release-tests.yml | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index bf52535b65f..30d1bf28a84 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -722,8 +722,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     test_env_type: docker
     test_cmd: cd integration-tests/smoke && go test -v -test.run "TestVRFv2Plus$/(Link_Billing|Native_Billing|Direct_Funding)|TestVRFV2PlusWithBHS" -test.parallel=1 -timeout 2h -count=1 -json
-    # test_config_override_path: integration-tests/testconfig/vrfv2plus/overrides/release_testing/sepolia_release_test_config.toml
-    test_config_override_path: integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
+    test_config_override_path: integration-tests/testconfig/vrfv2plus/overrides/release_testing/sepolia_release_test_config.toml
     workflows:
       - VRF E2E Release Tests
 
diff --git a/.github/workflows/vrf-e2e-release-tests.yml b/.github/workflows/vrf-e2e-release-tests.yml
index 552d96a8f6a..54d91d2da92 100644
--- a/.github/workflows/vrf-e2e-release-tests.yml
+++ b/.github/workflows/vrf-e2e-release-tests.yml
@@ -15,8 +15,7 @@ jobs:
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
       chainlink_version: ${{ github.event.inputs.chainlink_version }}
-      # test_workflow: VRF E2E Release Tests
-      test_ids: 'smoke/vrfv2plus_test.go:*'
+      test_workflow: VRF E2E Release Tests
       slack_notification_after_tests: true
       # slack_notification_after_tests_channel_id: "#team-test-tooling-internal"
       slack_notification_after_tests_channel_id: "#lf-slack-notification-tests"

From d26eff57ee0be1071ccfc23d7008e68270bb1a3b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:58:46 +0200
Subject: [PATCH 053/141] Add test_secrets_override_key

---
 .github/workflows/vrf-e2e-release-tests.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/vrf-e2e-release-tests.yml b/.github/workflows/vrf-e2e-release-tests.yml
index 54d91d2da92..7b1f0092dda 100644
--- a/.github/workflows/vrf-e2e-release-tests.yml
+++ b/.github/workflows/vrf-e2e-release-tests.yml
@@ -8,6 +8,10 @@ on:
         default: develop
         required: true
         type: string
+      test_secrets_override_key:
+        description: Key to override default test secrets
+        required: true
+        type: string         
 
 jobs:
   run-e2e-tests-workflow:
@@ -37,3 +41,4 @@ jobs:
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
       AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}    
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}

From 5fbc05ed23b0aad50013b97aed06138978c3b4a2 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 3 Sep 2024 17:23:05 +0200
Subject: [PATCH 054/141] fix for slack notification after test

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 7c6f4771184..8079f731fde 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -831,7 +831,7 @@ jobs:
             }
         
       - name: Notify user in Slack message if tests failed
-        if: ${{ inputs.slack_notification_after_tests == 'on_failure' && contains(join(needs.*.result, ','), 'failure') && inputs.slack_notification_after_tests_notify_user_id_on_failure != '' }}
+        if: ${{ inputs.slack_notification_after_tests != '' && contains(join(needs.*.result, ','), 'failure') && inputs.slack_notification_after_tests_notify_user_id_on_failure != '' }}
         uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

From 7711363af12472938e2661e6db9366e6718dd202 Mon Sep 17 00:00:00 2001
From: Ilja Pavlovs <ilja.pavlovs@gmail.com>
Date: Tue, 3 Sep 2024 21:52:58 +0300
Subject: [PATCH 055/141] TT-1550: reorganizing VRF V2 Plus configs

---
 .../base_sepolia_release_test_config.toml     |   5 +
 ...ml => nexon_test_release_test_config.toml} |   0
 .../optimism_sepolia_release_test_config.toml |   5 +
 .../polygon_amoy_release_test_config.toml     |   5 +
 .../arbitrum_sepolia_staging_test_config.toml |  36 +
 .../avalanche_fuji_staging_test_config.toml   |  18 +
 .../base_sepolia_staging_test_config.toml     |  20 +
 .../bsc_testnet_staging_test_config.toml      |  18 +
 .../nexon_dev_staging_test_config.toml        |  18 +
 .../staging/nexon_qa_staging_test_config.toml |  19 +
 .../nexon_stage_staging_test_config.toml      |  15 +
 .../nexon_test_staging_test_config.toml       |  19 +
 .../optimism_sepolia_staging_test_config.toml |  20 +
 .../polygon_amoy_staging_test_config.toml     |  32 +
 .../staging/sepolia_staging_test_config.toml  |  18 +
 .../testconfig/vrfv2plus/vrfv2plus.toml       | 755 +++++++-----------
 16 files changed, 553 insertions(+), 450 deletions(-)
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/release_testing/base_sepolia_release_test_config.toml
 rename integration-tests/testconfig/vrfv2plus/overrides/release_testing/{nexon_test_release_test_config copy.toml => nexon_test_release_test_config.toml} (100%)
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/release_testing/optimism_sepolia_release_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/release_testing/polygon_amoy_release_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/arbitrum_sepolia_staging_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/base_sepolia_staging_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/optimism_sepolia_staging_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/polygon_amoy_staging_test_config.toml
 create mode 100644 integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_test_config.toml

diff --git a/integration-tests/testconfig/vrfv2plus/overrides/release_testing/base_sepolia_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/base_sepolia_release_test_config.toml
new file mode 100644
index 00000000000..ffc209d2ad5
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/base_sepolia_release_test_config.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["BASE_SEPOLIA"]
+
+[BASE_SEPOLIA.VRFv2Plus.General]
+use_existing_env = false
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_test_release_test_config copy.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_test_release_test_config.toml
similarity index 100%
rename from integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_test_release_test_config copy.toml
rename to integration-tests/testconfig/vrfv2plus/overrides/release_testing/nexon_test_release_test_config.toml
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/release_testing/optimism_sepolia_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/optimism_sepolia_release_test_config.toml
new file mode 100644
index 00000000000..6ccfa1c4cad
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/optimism_sepolia_release_test_config.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["OPTIMISM_SEPOLIA"]
+
+[OPTIMISM_SEPOLIA.VRFv2Plus.General]
+use_existing_env = false
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/release_testing/polygon_amoy_release_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/polygon_amoy_release_test_config.toml
new file mode 100644
index 00000000000..e462be8acc1
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/release_testing/polygon_amoy_release_test_config.toml
@@ -0,0 +1,5 @@
+[Network]
+selected_networks = ["POLYGON_AMOY"]
+
+[POLYGON_AMOY.VRFv2Plus.General]
+use_existing_env = false
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/arbitrum_sepolia_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/arbitrum_sepolia_staging_test_config.toml
new file mode 100644
index 00000000000..8ba2128f7ab
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/arbitrum_sepolia_staging_test_config.toml
@@ -0,0 +1,36 @@
+[Network]
+selected_networks = ["ARBITRUM_SEPOLIA"]
+
+[ARBITRUM_SEPOLIA.VRFv2Plus.General]
+use_existing_env = true
+
+[ARBITRUM_SEPOLIA.VRFv2Plus.ExistingEnv]
+coordinator_address = "0xF7ba1Cf141F9729abC43c146dc2bf86EbcfD8603"
+consumer_address = ""
+sub_id = ""
+key_hash = "0xe13aa26fe94bfcd2ae055911f4d3bf1aed54ca6cf77af34e17f918802fd69ba1"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 20
+node_sending_keys = [
+    "0xbE21ae371FcA1aC2d8A152e707D21e68d7d99252",
+    "0xb13e9BA0aE94FD3b89B13b092e6d41614c805134",
+    "0x27aa703e585Ee165B7c977EAA652eCFa13b08294",
+    "0x9324643ACD2ec5b0813488E5EdAb64C3758ae4Ee",
+    "0x7CBA8c8e86f23f23363051650Fe5AE4DE78c3652",
+    "0x9A0143a4BAB55A826331A8ef82462557633aA016",
+    "0xD4259633F8e87949F683433a17e1fFcCE27865AC",
+    "0x5e47B71d6F95f68cd5538907ec6A9635b1Fe30Fa",
+    "0xa850a1a257FDF439c8f854ce3b89dd5b6F411827",
+    "0x7c82D56087c10aF2c3970f9a9Be7786B2850ce91",
+    "0x9545CB59956347d3debf27f5029754bBE1d398FA",
+    "0xEb8C69ac19709f27A97FB4A561f51AD2F9b34c5B",
+    #   BHS
+    "0xf0e8cF7Fbd28Fc4D412B76B744CDA269df671F8e",
+    "0x317A02A658d12E5Bb4A6270171E7385928dD2d53",
+    "0x480f1dbcEc118Bd91e4dbb7e45bFa4A73180d21f",
+    "0x500a2662FaF981EC4669f791349D37Cbf1bE7d85",
+    "0x2ad350374B904c10B47c64ECdBD9e70BB0833Be2",
+    "0x0b946F0bF4e63C12b5157137f1c130f0788bC1b1",
+    #    BHF
+    "0x571BBF4a5b07fc3F47Bd3B65CE2FE73739f86623"
+]
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_test_config.toml
new file mode 100644
index 00000000000..9bf6c723cb2
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/avalanche_fuji_staging_test_config.toml
@@ -0,0 +1,18 @@
+[Network]
+selected_networks = ["AVALANCHE_FUJI"]
+
+[AVALANCHE_FUJI.VRFv2Plus.General]
+use_existing_env = true
+
+[AVALANCHE_FUJI.VRFv2Plus.ExistingEnv]
+coordinator_address = "0xE122bf3Badd6545bDec5D4627a6DAd16352A1b36"
+consumer_address = ""
+sub_id = ""
+key_hash = "0x5b03254a80ea3eb72139ff0423cb88be42612780c3dd25f1d95a5ba7708a4be1"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 30
+node_sending_keys = [
+    "0x3D7Da5D6A23CA2240CE576C8638C1798a023920a",
+    #   BHS
+    "0x72c8565279430F5179b0090d51ab8BB53Da323B5"
+]
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/base_sepolia_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/base_sepolia_staging_test_config.toml
new file mode 100644
index 00000000000..aaab02df7d9
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/base_sepolia_staging_test_config.toml
@@ -0,0 +1,20 @@
+[Network]
+selected_networks = ["BASE_SEPOLIA"]
+
+[BASE_SEPOLIA.VRFv2Plus.General]
+use_existing_env = true
+
+[BASE_SEPOLIA.VRFv2Plus.ExistingEnv]
+coordinator_address = "0x2Cf7Bb5923FA4dBdf92981fDBbEe27d13A896705"
+consumer_address = ""
+sub_id = ""
+key_hash = "0x01d1eb450e0271ac86d3b78c7cc799f80e7f80863a4875f6fc7b66629419c951"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 30
+node_sending_keys = [
+    "0x5d621FF993B1a990d189936E70021c585e3B0880",
+    "0x87F701AD21BfAF99eF64596c5CE8762a5Cb9ED13",
+    "0xe8B0865e9Aae9DE628BE14965Bbd1C0c9C80d245",
+    #    BHS
+    "0x65C853683beB6363869DDda8534b2aD45786d380",
+]
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_test_config.toml
new file mode 100644
index 00000000000..d1716413f12
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/bsc_testnet_staging_test_config.toml
@@ -0,0 +1,18 @@
+[Network]
+selected_networks = ["BSC_TESTNET"]
+
+[BSC_TESTNET.VRFv2Plus.General]
+use_existing_env = true
+
+[BSC_TESTNET.VRFv2Plus.ExistingEnv]
+coordinator_address = "0x84A477F6ebF33501eE3ACA86fEcB980b1fC99AC2"
+consumer_address = ""
+sub_id = ""
+key_hash = "0x4d43763d3eff849a89cf578a42787baa32132d7a80032125710e95b3972cd214"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 30
+node_sending_keys = [
+    "0x4EE2Cc6D50E8acb6BaEf673B03559525a6c92fB8",
+    #   BHS
+    "0xAFB44568f7DAc218EA6e1C71c366692ED4758A07"
+]
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_test_config.toml
new file mode 100644
index 00000000000..597abe13aff
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_dev_staging_test_config.toml
@@ -0,0 +1,18 @@
+[Network]
+selected_networks = ["NEXON_DEV"]
+
+[NEXON_DEV.VRFv2Plus.General]
+use_existing_env = true
+
+[NEXON_DEV.VRFv2Plus.ExistingEnv]
+coordinator_address = "0x6901d7236A823E7B7911d90FBe46E6FA770CC823"
+consumer_address = ""
+sub_id = ""
+key_hash = "0xdc023892a41e5fe74ec7c4c2e8c0a808b01aea7acaf2b2ae30f4e08df877c48b"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 30
+node_sending_keys = [
+    "0xF3d9879a75BBD85890056D7c6cB37C555F9b41A3",
+    #   BHS
+    "0xb544f9D7c16a30af0EEd0afcC4132D1c63bAF8AC",
+]
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_test_config.toml
new file mode 100644
index 00000000000..5f5feab74ab
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_qa_staging_test_config.toml
@@ -0,0 +1,19 @@
+[Network]
+selected_networks = ["NEXON_QA"]
+
+[NEXON_QA.VRFv2Plus.General]
+use_existing_env = true
+
+[NEXON_QA.VRFv2Plus.ExistingEnv]
+coordinator_address = "0xF1F0beBcc284591FCD28d8f2BAc9f30efdA3E0ea"
+consumer_address = ""
+sub_id = ""
+key_hash = "0x7d5692e71807c4c02f5a109627a9ad2b12a361a346790a306983af9a5e3a186f"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 30
+node_sending_keys = [
+    "0xB97c0C52A2B957b45DA213e652c76090DDd0FEc6",
+    "0xe205F5d4a99ca0f474d0b4d12f60a0153c786B4E",
+    #   BHS
+    "0xf85E291edF0352435f2fD5e817030f6542375a99",
+]
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_test_config.toml
new file mode 100644
index 00000000000..1a50876a730
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_stage_staging_test_config.toml
@@ -0,0 +1,15 @@
+[Network]
+selected_networks = ["NEXON_STAGE"]
+
+[NEXON_STAGE.VRFv2Plus.General]
+use_existing_env = true
+
+[NEXON_STAGE.VRFv2Plus.ExistingEnv]
+coordinator_address = "0xF705dD3e7E717F32de0Cc5F833f8009f16122AD1"
+consumer_address = ""
+sub_id = ""
+key_hash = "0xbc9f525e3e1d9e2336f7c77d5f33f5b60aab3765944617fed7f66a6afecac616"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 30
+node_sending_keys = [
+]
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_test_config.toml
new file mode 100644
index 00000000000..aeb4cbdb297
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/nexon_test_staging_test_config.toml
@@ -0,0 +1,19 @@
+[Network]
+selected_networks = ["NEXON_TEST"]
+
+[NEXON_TEST.VRFv2Plus.General]
+use_existing_env = true
+
+[NEXON_TEST.VRFv2Plus.ExistingEnv]
+coordinator_address = "0xAa92Ba21168B48195cAdB87cfaB3eB70B2499F55"
+consumer_address = ""
+sub_id = ""
+key_hash = "0x0cb2a18e8b762cb4c8f7b17a6cc02ac7b9d2a3346f048cfd2f5d37677f8747d8"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 30
+node_sending_keys = [
+    "0xBFD780Af421e98C35918e10B9d6da7389C3e1D10",
+    "0xbf6c76024672F233aB8164EC00683e1AE774F6b0",
+    #   BHS
+    "0x2a3900Ac77de110670E060DBFf4fCbe36c6f8170",
+]
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/optimism_sepolia_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/optimism_sepolia_staging_test_config.toml
new file mode 100644
index 00000000000..c707b83f8d2
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/optimism_sepolia_staging_test_config.toml
@@ -0,0 +1,20 @@
+[Network]
+selected_networks = ["OPTIMISM_SEPOLIA"]
+
+[OPTIMISM_SEPOLIA.VRFv2Plus.General]
+use_existing_env = true
+
+[OPTIMISM_SEPOLIA.VRFv2Plus.ExistingEnv]
+coordinator_address = "0xA4a64A217bE85680e0ebB454CD5BF4A1c274Fc7B"
+consumer_address = ""
+sub_id = ""
+key_hash = "0x4b4838bbf22a7c5a871ada8ceab6ded3c2de6cadc037371146ee70f6435325c1"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 30
+node_sending_keys = [
+    "0x17509D615052DE3A81a684755Ec118d62C4e1Cd1",
+    "0x5118ece61294f4dFf4a1fb63b3f036969516dF0a",
+    "0x89554391652616ea06a408263b9B2b9a70E87204",
+    #    BHS
+    "0x8DE6446b5022C68F38CD32d04AA0E3b8F4C1aaB6",
+]
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/polygon_amoy_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/polygon_amoy_staging_test_config.toml
new file mode 100644
index 00000000000..0b49f66ec30
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/polygon_amoy_staging_test_config.toml
@@ -0,0 +1,32 @@
+[Network]
+selected_networks = ["POLYGON_AMOY"]
+
+[POLYGON_AMOY.VRFv2Plus.General]
+use_existing_env = true
+
+[POLYGON_AMOY.VRFv2Plus.ExistingEnv]
+coordinator_address = "0x7541EbaE23f32B4A1A2e7a8Cbf9da9582767A9B4"
+consumer_address = ""
+sub_id = ""
+key_hash = "0xd360445bacd26df47086ccf255c4f932d297ed8d5c7334b51eed32f61c541601"
+#key_hash = "0x2328cbee29e32d0b6662d6df82ff0fea7be300bd310561c92f515c9ee19464f1"
+#key_hash = "0x25f4e2d0509f42ec77db5380f3433a89fe623fa75f65d5b398d5f498327be4dd"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 10
+node_sending_keys = [
+    "0xD96013C241f1741C35a135321969f92Aae02A12F",
+    "0x0580E61a5523F5CAAC4968E4f8FE63b59596BdD7",
+    "0xd15FcEa6a6AA17085930Fbd5647A9F7fD2Ff58b8",
+    "0xB7277cBb6E7028AE65235b8ee9201AcBb14B11d4",
+    "0x6D36a1dC1eEd25C75961E989c4d01Cd4453bE465",
+    "0xd299Cd7C0073b71e620bf8A3bfD50F75c0b49af8",
+    "0x48BE7BAED0b65776D85DF971fA901c637cFC5e87",
+    #    BHS
+    "0x638372de870eF0F8E675A3f67F18D5bd4A2fd804",
+    "0xF9eF03816411D037202d5ed4457dC1613e3bd729",
+    "0xCD66973f8fbaE787211EC20228c6bd90D83562A8",
+    "0x242ea1F4Bb72EF643B2D8EF22e18a89f00742F40",
+    "0xaA09B4F9B5710b239fdbf1D0f535dd7f86F91219",
+    "0xe6b72B647B8B45C5562F7a5259E187889C747d3b",
+    "0x2c1185C4d3B0B4a577d4079Ee193A4e293164D9d"
+]
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_test_config.toml b/integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_test_config.toml
new file mode 100644
index 00000000000..346b1d792c7
--- /dev/null
+++ b/integration-tests/testconfig/vrfv2plus/overrides/staging/sepolia_staging_test_config.toml
@@ -0,0 +1,18 @@
+[Network]
+selected_networks = ["SEPOLIA"]
+
+[SEPOLIA.VRFv2Plus.General]
+use_existing_env = true
+
+[SEPOLIA.VRFv2Plus.ExistingEnv]
+coordinator_address = "0x2F3b892710523Ee9A85c3155a42089fFe99Ca31e"
+consumer_address = ""
+sub_id = ""
+key_hash = "0xf5b4a359df0598eef89872ea2170f2afa844dbf74b417e6d44d4bda9420aceb2"
+create_fund_subs_and_add_consumers = true
+node_sending_key_funding_min = 30
+node_sending_keys = [
+    "0x0c0DC7f33A1256f0247c5ea75861d385fa5FED31",
+    #    BHS
+    "0xEd8A4b792d16484f6c9B4df1e721e8280925Db80",
+]
\ No newline at end of file
diff --git a/integration-tests/testconfig/vrfv2plus/vrfv2plus.toml b/integration-tests/testconfig/vrfv2plus/vrfv2plus.toml
index 84af6d965bc..3365974f278 100644
--- a/integration-tests/testconfig/vrfv2plus/vrfv2plus.toml
+++ b/integration-tests/testconfig/vrfv2plus/vrfv2plus.toml
@@ -1,8 +1,5 @@
 # default config
 
-[Network]
-selected_networks = ["SIMULATED"]
-
 [NodeConfig]
 BaseConfigTOML = """
 [Feature]
@@ -295,19 +292,17 @@ bhs_test_rate_limit_unit_duration = "3s"
 bhs_test_rps = 1
 
 #SOAK TEST CONFIG
-[Soak.Common]
-chainlink_node_funding = 0.1
 
 [Soak.VRFv2Plus.General]
 randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
 number_of_sending_keys_to_create = 0
-subscription_funding_amount_link = 50
-subscription_funding_amount_native=10
+subscription_funding_amount_link = 500
+subscription_funding_amount_native = 100
 
 [Soak.VRFv2Plus.Performance]
-test_duration = "2m"
+test_duration = "2h"
 rate_limit_unit_duration = "10s"
 rps = 1
 bhs_test_duration = "1m"
@@ -339,16 +334,16 @@ bhs_test_rps = 1
 chainlink_node_funding = 0.1
 
 [Stress.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+randomness_request_count_per_request = 30 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
 number_of_sending_keys_to_create = 0
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native=1
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
 [Stress.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "10m"
+rate_limit_unit_duration = "10s"
 rps = 1
 bhs_test_duration = "1m"
 bhs_test_rate_limit_unit_duration = "3s"
@@ -406,33 +401,6 @@ bhs_job_poll_period = "2s"
 bhs_job_run_timeout = "30s"
 # NEW ENV CONFIG END
 
-[POLYGON_AMOY.VRFv2Plus.ExistingEnv]
-coordinator_address = "0x7541EbaE23f32B4A1A2e7a8Cbf9da9582767A9B4"
-consumer_address = ""
-sub_id = ""
-key_hash = "0xd360445bacd26df47086ccf255c4f932d297ed8d5c7334b51eed32f61c541601"
-#key_hash = "0x2328cbee29e32d0b6662d6df82ff0fea7be300bd310561c92f515c9ee19464f1"
-#key_hash = "0x25f4e2d0509f42ec77db5380f3433a89fe623fa75f65d5b398d5f498327be4dd"
-create_fund_subs_and_add_consumers = true
-node_sending_key_funding_min = 10
-node_sending_keys = [
-    "0xD96013C241f1741C35a135321969f92Aae02A12F",
-    "0x0580E61a5523F5CAAC4968E4f8FE63b59596BdD7",
-    "0xd15FcEa6a6AA17085930Fbd5647A9F7fD2Ff58b8",
-    "0xB7277cBb6E7028AE65235b8ee9201AcBb14B11d4",
-    "0x6D36a1dC1eEd25C75961E989c4d01Cd4453bE465",
-    "0xd299Cd7C0073b71e620bf8A3bfD50F75c0b49af8",
-    "0x48BE7BAED0b65776D85DF971fA901c637cFC5e87",
-    #    BHS
-    "0x638372de870eF0F8E675A3f67F18D5bd4A2fd804",
-    "0xF9eF03816411D037202d5ed4457dC1613e3bd729",
-    "0xCD66973f8fbaE787211EC20228c6bd90D83562A8",
-    "0x242ea1F4Bb72EF643B2D8EF22e18a89f00742F40",
-    "0xaA09B4F9B5710b239fdbf1D0f535dd7f86F91219",
-    "0xe6b72B647B8B45C5562F7a5259E187889C747d3b",
-    "0x2c1185C4d3B0B4a577d4079Ee193A4e293164D9d"
-]
-
 #SMOKE TEST CONFIG
 [POLYGON_AMOY-Smoke.VRFv2Plus.General]
 randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
@@ -449,55 +417,33 @@ wait_for_256_blocks_timeout = "15m"
 wrapper_consumer_funding_amount_native_token = 1.0
 wrapper_consumer_funding_amount_link = 3
 
-
-[POLYGON_AMOY-Smoke.VRFv2Plus.Performance]
-test_duration = "2s"
-rate_limit_unit_duration = "15s"
-rps = 1
-
-
-#SOAK TEST CONFIG
 [POLYGON_AMOY-Soak.VRFv2Plus.General]
-randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
+number_of_sending_keys_to_create = 0
 subscription_funding_amount_link = 500
-subscription_funding_amount_native = 200
+subscription_funding_amount_native = 100
 
 [POLYGON_AMOY-Soak.VRFv2Plus.Performance]
-test_duration = "5h"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-# LOAD TEST CONFIG
-[POLYGON_AMOY-Load.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
-number_of_sub_to_create = 1
-subscription_funding_amount_link = 300
-subscription_funding_amount_native = 300
-
-[POLYGON_AMOY-Load.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "2h"
+rate_limit_unit_duration = "10s"
 rps = 1
 
-
-# STRESS TEST CONFIG
 [POLYGON_AMOY-Stress.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+randomness_request_count_per_request = 150 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native = 0.1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
 [POLYGON_AMOY-Stress.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
 rps = 1
 
 ### ARBITRUM SEPOLIA Config
-
 [ARBITRUM_SEPOLIA.VRFv2Plus.General]
 use_test_coordinator = true
 #todo - need to have separate minimum_confirmations config for Coordinator, CL Node and Consumer request
@@ -547,36 +493,7 @@ bhs_job_poll_period = "300ms"
 bhs_job_run_timeout = "30s"
 # NEW ENV CONFIG END
 
-[ARBITRUM_SEPOLIA.VRFv2Plus.ExistingEnv]
-coordinator_address = "0xF7ba1Cf141F9729abC43c146dc2bf86EbcfD8603"
-consumer_address = ""
-sub_id = ""
-key_hash = "0xe13aa26fe94bfcd2ae055911f4d3bf1aed54ca6cf77af34e17f918802fd69ba1"
-create_fund_subs_and_add_consumers = true
-node_sending_key_funding_min = 20
-node_sending_keys = [
-    "0xbE21ae371FcA1aC2d8A152e707D21e68d7d99252",
-    "0xb13e9BA0aE94FD3b89B13b092e6d41614c805134",
-    "0x27aa703e585Ee165B7c977EAA652eCFa13b08294",
-    "0x9324643ACD2ec5b0813488E5EdAb64C3758ae4Ee",
-    "0x7CBA8c8e86f23f23363051650Fe5AE4DE78c3652",
-    "0x9A0143a4BAB55A826331A8ef82462557633aA016",
-    "0xD4259633F8e87949F683433a17e1fFcCE27865AC",
-    "0x5e47B71d6F95f68cd5538907ec6A9635b1Fe30Fa",
-    "0xa850a1a257FDF439c8f854ce3b89dd5b6F411827",
-    "0x7c82D56087c10aF2c3970f9a9Be7786B2850ce91",
-    "0x9545CB59956347d3debf27f5029754bBE1d398FA",
-    "0xEb8C69ac19709f27A97FB4A561f51AD2F9b34c5B",
-    #   BHS
-    "0xf0e8cF7Fbd28Fc4D412B76B744CDA269df671F8e",
-    "0x317A02A658d12E5Bb4A6270171E7385928dD2d53",
-    "0x480f1dbcEc118Bd91e4dbb7e45bFa4A73180d21f",
-    "0x500a2662FaF981EC4669f791349D37Cbf1bE7d85",
-    "0x2ad350374B904c10B47c64ECdBD9e70BB0833Be2",
-    "0x0b946F0bF4e63C12b5157137f1c130f0788bC1b1",
-    #    BHF
-    "0x571BBF4a5b07fc3F47Bd3B65CE2FE73739f86623"
-]
+
 
 #SMOKE TEST CONFIG
 [ARBITRUM_SEPOLIA-Smoke.VRFv2Plus.General]
@@ -594,53 +511,32 @@ wait_for_256_blocks_timeout = "100s"
 wrapper_consumer_funding_amount_native_token = 1.0
 wrapper_consumer_funding_amount_link = 5
 
-[ARBITRUM_SEPOLIA-Smoke.VRFv2Plus.Performance]
-test_duration = "1s"
-rate_limit_unit_duration = "10s"
-rps = 1
-
-
-#SOAK TEST CONFIG
 [ARBITRUM_SEPOLIA-Soak.VRFv2Plus.General]
 randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 300
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
 subscription_funding_amount_native = 100
 
 [ARBITRUM_SEPOLIA-Soak.VRFv2Plus.Performance]
-test_duration = "1h"
+test_duration = "2h"
 rate_limit_unit_duration = "10s"
 rps = 1
 
-# LOAD TEST CONFIG
-[ARBITRUM_SEPOLIA-Load.VRFv2Plus.General]
-randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
-number_of_sub_to_create = 1
-subscription_funding_amount_link = 100
-subscription_funding_amount_native = 60
-
-[ARBITRUM_SEPOLIA-Load.VRFv2Plus.Performance]
-test_duration = "30m"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-
-# STRESS TEST CONFIG
 [ARBITRUM_SEPOLIA-Stress.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+randomness_request_count_per_request = 150 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native = 0.1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
 [ARBITRUM_SEPOLIA-Stress.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
 rps = 1
 
-
 ### AVALANCHE FUJI Config
 [AVALANCHE_FUJI.Common]
 chainlink_node_funding = 3
@@ -693,19 +589,6 @@ bhs_job_poll_period = "2s"
 bhs_job_run_timeout = "30s"
 # NEW ENV CONFIG END
 
-[AVALANCHE_FUJI.VRFv2Plus.ExistingEnv]
-coordinator_address = "0xE122bf3Badd6545bDec5D4627a6DAd16352A1b36"
-consumer_address = ""
-sub_id = ""
-key_hash = "0x5b03254a80ea3eb72139ff0423cb88be42612780c3dd25f1d95a5ba7708a4be1"
-create_fund_subs_and_add_consumers = true
-node_sending_key_funding_min = 50
-node_sending_keys = [
-    "0x3D7Da5D6A23CA2240CE576C8638C1798a023920a",
-    #   BHS
-    "0x72c8565279430F5179b0090d51ab8BB53Da323B5"
-]
-
 #SMOKE TEST CONFIG
 [AVALANCHE_FUJI-Smoke.VRFv2Plus.General]
 randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
@@ -722,52 +605,222 @@ wait_for_256_blocks_timeout = "10m"
 wrapper_consumer_funding_amount_native_token = 1.0
 wrapper_consumer_funding_amount_link = 5
 
-[AVALANCHE_FUJI-Smoke.VRFv2Plus.Performance]
-test_duration = "2s"
+[AVALANCHE_FUJI-Soak.VRFv2Plus.General]
+randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
+number_of_sub_to_create = 1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native = 100
+
+[AVALANCHE_FUJI-Soak.VRFv2Plus.Performance]
+test_duration = "2h"
 rate_limit_unit_duration = "10s"
 rps = 1
 
+[AVALANCHE_FUJI-Stress.VRFv2Plus.General]
+randomness_request_count_per_request = 60 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
+number_of_sub_to_create = 1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
-#SOAK TEST CONFIG
-[AVALANCHE_FUJI-Soak.VRFv2Plus.General]
+[AVALANCHE_FUJI-Stress.VRFv2Plus.Performance]
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
+rps = 1
+
+### BASE SEPOLIA Config
+[BASE_SEPOLIA.Common]
+chainlink_node_funding = 5
+[BASE_SEPOLIA.VRFv2Plus.General]
+use_test_coordinator = false
+#todo - need to have separate minimum_confirmations config for Coordinator, CL Node and Consumer request
+minimum_confirmations = 0
+
+# Consumer Request config
+subscription_billing_type = "LINK_AND_NATIVE"
+callback_gas_limit = 1000000
+
+# NEW ENV CONFIG
+# CL Node config
+cl_node_max_gas_price_gwei = 30
+number_of_sending_keys_to_create = 0
+
+# Coordinator config
+max_gas_limit_coordinator_config = 2500000
+fallback_wei_per_unit_link = "3962147213857640"
+staleness_seconds = 172_800
+gas_after_payment_calculation = 42_500
+fulfillment_flat_fee_native_ppm = 0
+fulfillment_flat_fee_link_discount_ppm = 0
+native_premium_percentage = 60
+link_premium_percentage = 50
+
+# Wrapper config
+wrapped_gas_overhead = 13_400
+coordinator_gas_overhead_native = 128_500
+coordinator_gas_overhead_link = 150_400
+coordinator_gas_overhead_per_word = 435
+coordinator_native_premium_percentage = 60
+coordinator_link_premium_percentage = 50
+wrapper_max_number_of_words = 10
+
+# VRF Job config
+vrf_job_forwarding_allowed = false
+vrf_job_estimate_gas_multiplier = 1.15
+vrf_job_batch_fulfillment_enabled = true
+vrf_job_batch_fulfillment_gas_multiplier = 1.1
+vrf_job_poll_period = "2s"
+vrf_job_request_timeout = "2h0m0s"
+vrf_job_simulation_block = "pending"
+
+# BHS Job config
+bhs_job_wait_blocks = 30
+bhs_job_lookback_blocks = 200
+bhs_job_poll_period = "2s"
+bhs_job_run_timeout = "30s"
+# NEW ENV CONFIG END
+
+#SMOKE TEST CONFIG
+[BASE_SEPOLIA-Smoke.VRFv2Plus.General]
 randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 400
-subscription_funding_amount_native = 200
+subscription_funding_amount_link = 3
+subscription_funding_amount_native = 1
+subscription_refunding_amount_link = 3
+subscription_refunding_amount_native = 1
+number_of_words = 1
+random_words_fulfilled_event_timeout = "1m30s"
+wait_for_256_blocks_timeout = "10m"
 
-[AVALANCHE_FUJI-Soak.VRFv2Plus.Performance]
-test_duration = "5h"
-rate_limit_unit_duration = "3s"
+wrapper_consumer_funding_amount_native_token = 1.0
+wrapper_consumer_funding_amount_link = 5
+
+[BASE_SEPOLIA-Soak.VRFv2Plus.General]
+randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
+number_of_sub_to_create = 1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native = 100
+
+[BASE_SEPOLIA-Soak.VRFv2Plus.Performance]
+test_duration = "2h"
+rate_limit_unit_duration = "10s"
 rps = 1
 
-# LOAD TEST CONFIG
-[AVALANCHE_FUJI-Load.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+[BASE_SEPOLIA-Stress.VRFv2Plus.General]
+randomness_request_count_per_request = 60 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 300
-subscription_funding_amount_native = 300
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
-[AVALANCHE_FUJI-Load.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+[BASE_SEPOLIA-Stress.VRFv2Plus.Performance]
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
 rps = 1
 
 
-# STRESS TEST CONFIG
-[AVALANCHE_FUJI-Stress.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+### OPTIMISM SEPOLIA Config
+[OPTIMISM_SEPOLIA.Common]
+chainlink_node_funding = 5
+[OPTIMISM_SEPOLIA.VRFv2Plus.General]
+use_test_coordinator = false
+#todo - need to have separate minimum_confirmations config for Coordinator, CL Node and Consumer request
+minimum_confirmations = 0
+
+# Consumer Request config
+subscription_billing_type = "LINK_AND_NATIVE"
+callback_gas_limit = 1000000
+
+# NEW ENV CONFIG
+# CL Node config
+cl_node_max_gas_price_gwei = 30
+number_of_sending_keys_to_create = 0
+
+# Coordinator config
+max_gas_limit_coordinator_config = 2500000
+fallback_wei_per_unit_link = "3896881047706782"
+staleness_seconds = 172_800
+gas_after_payment_calculation = 42_500
+fulfillment_flat_fee_native_ppm = 0
+fulfillment_flat_fee_link_discount_ppm = 0
+native_premium_percentage = 60
+link_premium_percentage = 50
+
+# Wrapper config
+wrapped_gas_overhead = 13_400
+coordinator_gas_overhead_native = 128_500
+coordinator_gas_overhead_link = 150_400
+coordinator_gas_overhead_per_word = 435
+coordinator_native_premium_percentage = 60
+coordinator_link_premium_percentage = 50
+wrapper_max_number_of_words = 10
+
+# VRF Job config
+vrf_job_forwarding_allowed = false
+vrf_job_estimate_gas_multiplier = 1.15
+vrf_job_batch_fulfillment_enabled = true
+vrf_job_batch_fulfillment_gas_multiplier = 1.1
+vrf_job_poll_period = "2s"
+vrf_job_request_timeout = "2h0m0s"
+vrf_job_simulation_block = "pending"
+
+# BHS Job config
+bhs_job_wait_blocks = 30
+bhs_job_lookback_blocks = 200
+bhs_job_poll_period = "2s"
+bhs_job_run_timeout = "30s"
+# NEW ENV CONFIG END
+
+#SMOKE TEST CONFIG
+[OPTIMISM_SEPOLIA-Smoke.VRFv2Plus.General]
+randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native = 0.1
+subscription_funding_amount_link = 3
+subscription_funding_amount_native = 1
+subscription_refunding_amount_link = 3
+subscription_refunding_amount_native = 1
+number_of_words = 1
+random_words_fulfilled_event_timeout = "1m30s"
+wait_for_256_blocks_timeout = "10m"
 
-[AVALANCHE_FUJI-Stress.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+wrapper_consumer_funding_amount_native_token = 1.0
+wrapper_consumer_funding_amount_link = 5
+
+[OPTIMISM_SEPOLIA-Soak.VRFv2Plus.General]
+randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
+number_of_sub_to_create = 1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native = 100
+
+[OPTIMISM_SEPOLIA-Soak.VRFv2Plus.Performance]
+test_duration = "2h"
+rate_limit_unit_duration = "10s"
+rps = 1
+
+[OPTIMISM_SEPOLIA-Stress.VRFv2Plus.General]
+randomness_request_count_per_request = 60 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
+number_of_sub_to_create = 1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
+
+[OPTIMISM_SEPOLIA-Stress.VRFv2Plus.Performance]
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
 rps = 1
 
+
 ### ETH SEPOLIA Config
 [SEPOLIA.VRFv2Plus.General]
 use_test_coordinator = true
@@ -818,19 +871,6 @@ bhs_job_poll_period = "30s"
 bhs_job_run_timeout = "1m0s"
 # NEW ENV CONFIG END
 
-[SEPOLIA.VRFv2Plus.ExistingEnv]
-coordinator_address = "0x2F3b892710523Ee9A85c3155a42089fFe99Ca31e"
-consumer_address = ""
-sub_id = ""
-key_hash = "0xf5b4a359df0598eef89872ea2170f2afa844dbf74b417e6d44d4bda9420aceb2"
-create_fund_subs_and_add_consumers = true
-node_sending_key_funding_min = 50
-node_sending_keys = [
-    "0x0c0DC7f33A1256f0247c5ea75861d385fa5FED31",
-    #    BHS
-    "0xEd8A4b792d16484f6c9B4df1e721e8280925Db80",
-]
-
 #SMOKE TEST CONFIG
 [SEPOLIA-Smoke.VRFv2Plus.General]
 randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
@@ -847,48 +887,30 @@ wait_for_256_blocks_timeout = "70m"
 wrapper_consumer_funding_amount_native_token = 1.0
 wrapper_consumer_funding_amount_link = 3
 
-[SEPOLIA-Smoke.VRFv2Plus.Performance]
-test_duration = "2s"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-#SOAK TEST CONFIG
 [SEPOLIA-Soak.VRFv2Plus.General]
-randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
+number_of_sending_keys_to_create = 0
 subscription_funding_amount_link = 500
-subscription_funding_amount_native = 200
+subscription_funding_amount_native = 100
 
 [SEPOLIA-Soak.VRFv2Plus.Performance]
 test_duration = "2h"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-# LOAD TEST CONFIG
-[SEPOLIA-Load.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
-number_of_sub_to_create = 1
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native = 30
-
-[SEPOLIA-Load.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+rate_limit_unit_duration = "10s"
 rps = 1
 
-# STRESS TEST CONFIG
 [SEPOLIA-Stress.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+randomness_request_count_per_request = 30 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native = 0.1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
 [SEPOLIA-Stress.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
 rps = 1
 
 ### BSC TESTNET Config
@@ -941,19 +963,6 @@ bhs_job_poll_period = "2s"
 bhs_job_run_timeout = "30s"
 # NEW ENV CONFIG END
 
-[BSC_TESTNET.VRFv2Plus.ExistingEnv]
-coordinator_address = "0x84A477F6ebF33501eE3ACA86fEcB980b1fC99AC2"
-consumer_address = ""
-sub_id = ""
-key_hash = "0x4d43763d3eff849a89cf578a42787baa32132d7a80032125710e95b3972cd214"
-create_fund_subs_and_add_consumers = true
-node_sending_key_funding_min = 150
-node_sending_keys = [
-    "0x4EE2Cc6D50E8acb6BaEf673B03559525a6c92fB8",
-    #   BHS
-    "0xAFB44568f7DAc218EA6e1C71c366692ED4758A07"
-]
-
 #SMOKE TEST CONFIG
 [BSC_TESTNET-Smoke.VRFv2Plus.General]
 randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
@@ -970,53 +979,32 @@ wait_for_256_blocks_timeout = "15m"
 wrapper_consumer_funding_amount_native_token = 1.0
 wrapper_consumer_funding_amount_link = 5
 
-[BSC_TESTNET-Smoke.VRFv2Plus.Performance]
-test_duration = "15s"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-
-#SOAK TEST CONFIG
 [BSC_TESTNET-Soak.VRFv2Plus.General]
-randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 400
-subscription_funding_amount_native = 200
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native = 100
 
 [BSC_TESTNET-Soak.VRFv2Plus.Performance]
-test_duration = "5h"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-# LOAD TEST CONFIG
-[BSC_TESTNET-Load.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
-number_of_sub_to_create = 1
-subscription_funding_amount_link = 300
-subscription_funding_amount_native = 300
-
-[BSC_TESTNET-Load.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "2h"
+rate_limit_unit_duration = "10s"
 rps = 1
 
-
-# STRESS TEST CONFIG
 [BSC_TESTNET-Stress.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+randomness_request_count_per_request = 60 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native = 0.1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
 [BSC_TESTNET-Stress.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
 rps = 1
 
-
 ### NEXON QA Config
 [NEXON_QA.VRFv2Plus.General]
 use_test_coordinator = true
@@ -1028,20 +1016,6 @@ generate_txs_on_chain = true
 subscription_billing_type = "LINK_AND_NATIVE"
 callback_gas_limit = 1000000
 
-[NEXON_QA.VRFv2Plus.ExistingEnv]
-coordinator_address = "0xF1F0beBcc284591FCD28d8f2BAc9f30efdA3E0ea"
-consumer_address = ""
-sub_id = ""
-key_hash = "0x7d5692e71807c4c02f5a109627a9ad2b12a361a346790a306983af9a5e3a186f"
-create_fund_subs_and_add_consumers = true
-node_sending_key_funding_min = 30
-node_sending_keys = [
-    "0xB97c0C52A2B957b45DA213e652c76090DDd0FEc6",
-    "0xe205F5d4a99ca0f474d0b4d12f60a0153c786B4E",
-    #   BHS
-    "0xf85E291edF0352435f2fD5e817030f6542375a99",
-]
-
 #SMOKE TEST CONFIG
 [NEXON_QA-Smoke.VRFv2Plus.General]
 randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
@@ -1057,51 +1031,30 @@ wait_for_256_blocks_timeout = "25m"
 wrapper_consumer_funding_amount_link = 21
 wrapper_consumer_funding_amount_native_token = 3
 
-
-[NEXON_QA-Smoke.VRFv2Plus.Performance]
-test_duration = "2s"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-
-#SOAK TEST CONFIG
 [NEXON_QA-Soak.VRFv2Plus.General]
-randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 400
-subscription_funding_amount_native = 200
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native = 100
 
 [NEXON_QA-Soak.VRFv2Plus.Performance]
-test_duration = "5h"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-# LOAD TEST CONFIG
-[NEXON_QA-Load.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
-number_of_sub_to_create = 1
-subscription_funding_amount_link = 300
-subscription_funding_amount_native = 300
-
-[NEXON_QA-Load.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "2h"
+rate_limit_unit_duration = "10s"
 rps = 1
 
-
-# STRESS TEST CONFIG
 [NEXON_QA-Stress.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+randomness_request_count_per_request = 60 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native = 2
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
 [NEXON_QA-Stress.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
 rps = 1
 
 ### NEXON DEV Config
@@ -1115,19 +1068,6 @@ generate_txs_on_chain = true
 subscription_billing_type = "LINK_AND_NATIVE"
 callback_gas_limit = 1000000
 
-[NEXON_DEV.VRFv2Plus.ExistingEnv]
-coordinator_address = "0x6901d7236A823E7B7911d90FBe46E6FA770CC823"
-consumer_address = ""
-sub_id = ""
-key_hash = "0xdc023892a41e5fe74ec7c4c2e8c0a808b01aea7acaf2b2ae30f4e08df877c48b"
-create_fund_subs_and_add_consumers = true
-node_sending_key_funding_min = 30
-node_sending_keys = [
-    "0xF3d9879a75BBD85890056D7c6cB37C555F9b41A3",
-    #   BHS
-    "0xb544f9D7c16a30af0EEd0afcC4132D1c63bAF8AC",
-]
-
 #SMOKE TEST CONFIG
 [NEXON_DEV-Smoke.VRFv2Plus.General]
 randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
@@ -1143,53 +1083,32 @@ wait_for_256_blocks_timeout = "25m"
 wrapper_consumer_funding_amount_link = 21
 wrapper_consumer_funding_amount_native_token = 3
 
-[NEXON_DEV-Smoke.VRFv2Plus.Performance]
-test_duration = "2s"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-
-#SOAK TEST CONFIG
 [NEXON_DEV-Soak.VRFv2Plus.General]
-randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 400
-subscription_funding_amount_native = 200
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native = 100
 
 [NEXON_DEV-Soak.VRFv2Plus.Performance]
-test_duration = "5h"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-# LOAD TEST CONFIG
-[NEXON_DEV-Load.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
-number_of_sub_to_create = 1
-subscription_funding_amount_link = 300
-subscription_funding_amount_native = 300
-
-[NEXON_DEV-Load.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "2h"
+rate_limit_unit_duration = "10s"
 rps = 1
 
-
-# STRESS TEST CONFIG
 [NEXON_DEV-Stress.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+randomness_request_count_per_request = 60 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native = 0.1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
 [NEXON_DEV-Stress.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
 rps = 1
 
-
 ### NEXON TEST Config
 [NEXON_TEST.VRFv2Plus.General]
 use_test_coordinator = true
@@ -1201,19 +1120,6 @@ generate_txs_on_chain = true
 subscription_billing_type = "LINK_AND_NATIVE"
 callback_gas_limit = 1000000
 
-[NEXON_TEST.VRFv2Plus.ExistingEnv]
-coordinator_address = "0xAa92Ba21168B48195cAdB87cfaB3eB70B2499F55"
-consumer_address = ""
-sub_id = ""
-key_hash = "0x0cb2a18e8b762cb4c8f7b17a6cc02ac7b9d2a3346f048cfd2f5d37677f8747d8"
-create_fund_subs_and_add_consumers = true
-node_sending_key_funding_min = 30
-node_sending_keys = [
-    "0xBFD780Af421e98C35918e10B9d6da7389C3e1D10",
-    "0xbf6c76024672F233aB8164EC00683e1AE774F6b0",
-    #   BHS
-    "0x2a3900Ac77de110670E060DBFf4fCbe36c6f8170",
-]
 
 #SMOKE TEST CONFIG
 [NEXON_TEST-Smoke.VRFv2Plus.General]
@@ -1230,50 +1136,30 @@ wait_for_256_blocks_timeout = "25m"
 wrapper_consumer_funding_amount_link = 5
 wrapper_consumer_funding_amount_native_token = 3
 
-[NEXON_TEST-Smoke.VRFv2Plus.Performance]
-test_duration = "2s"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-
-#SOAK TEST CONFIG
 [NEXON_TEST-Soak.VRFv2Plus.General]
-randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 400
-subscription_funding_amount_native = 200
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native = 100
 
 [NEXON_TEST-Soak.VRFv2Plus.Performance]
-test_duration = "5h"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-# LOAD TEST CONFIG
-[NEXON_TEST-Load.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
-number_of_sub_to_create = 1
-subscription_funding_amount_link = 300
-subscription_funding_amount_native = 300
-
-[NEXON_TEST-Load.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "2h"
+rate_limit_unit_duration = "10s"
 rps = 1
 
-
-# STRESS TEST CONFIG
 [NEXON_TEST-Stress.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+randomness_request_count_per_request = 60 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native = 0.1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
 [NEXON_TEST-Stress.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
 rps = 1
 
 
@@ -1288,15 +1174,6 @@ generate_txs_on_chain = true
 subscription_billing_type = "LINK_AND_NATIVE"
 callback_gas_limit = 1000000
 
-[NEXON_STAGE.VRFv2Plus.ExistingEnv]
-coordinator_address = "0xF705dD3e7E717F32de0Cc5F833f8009f16122AD1"
-consumer_address = ""
-sub_id = ""
-key_hash = "0xbc9f525e3e1d9e2336f7c77d5f33f5b60aab3765944617fed7f66a6afecac616"
-create_fund_subs_and_add_consumers = true
-node_sending_key_funding_min = 30
-node_sending_keys = [
-]
 
 #SMOKE TEST CONFIG
 [NEXON_STAGE-Smoke.VRFv2Plus.General]
@@ -1313,50 +1190,28 @@ wait_for_256_blocks_timeout = "25m"
 wrapper_consumer_funding_amount_link = 21
 wrapper_consumer_funding_amount_native_token = 3
 
-[NEXON_STAGE-Smoke.VRFv2Plus.Performance]
-test_duration = "2s"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-
-#SOAK TEST CONFIG
 [NEXON_STAGE-Soak.VRFv2Plus.General]
-randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request
 randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 400
-subscription_funding_amount_native = 200
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native = 100
 
 [NEXON_STAGE-Soak.VRFv2Plus.Performance]
-test_duration = "5h"
-rate_limit_unit_duration = "3s"
-rps = 1
-
-# LOAD TEST CONFIG
-[NEXON_STAGE-Load.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
-number_of_sub_to_create = 1
-subscription_funding_amount_link = 300
-subscription_funding_amount_native = 300
-
-[NEXON_STAGE-Load.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "2h"
+rate_limit_unit_duration = "10s"
 rps = 1
 
-
-# STRESS TEST CONFIG
 [NEXON_STAGE-Stress.VRFv2Plus.General]
-randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request
-randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting
+randomness_request_count_per_request = 60 # amount of randomness requests to make per one TX request
+randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting
 number_of_sub_to_create = 1
-subscription_funding_amount_link = 5.0
-subscription_funding_amount_native = 0.1
+number_of_sending_keys_to_create = 0
+subscription_funding_amount_link = 500
+subscription_funding_amount_native=100
 
 [NEXON_STAGE-Stress.VRFv2Plus.Performance]
-test_duration = "2m"
-rate_limit_unit_duration = "3s"
+test_duration = "10m"
+rate_limit_unit_duration = "1m"
 rps = 1
-
-

From 32c71775ac2820335b18887b2ea194526c2e4f7d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 4 Sep 2024 10:14:21 +0200
Subject: [PATCH 056/141] Use test_config_override_path instead of base64 in
 other VRFv2Plus workflows

---
 .../on-demand-vrfv2plus-eth2-clients-test.yml | 65 +++++++++----------
 .../on-demand-vrfv2plus-performance-test.yml  | 54 ++++++++-------
 2 files changed, 63 insertions(+), 56 deletions(-)

diff --git a/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml b/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml
index 652c51095f1..0636a299442 100644
--- a/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml
+++ b/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml
@@ -2,14 +2,19 @@ name: On Demand VRFV2Plus Smoke Test (Ethereum clients)
 on:
   workflow_dispatch:
     inputs:
-      base64Config:
-        description: base64-ed config
-        required: true
-        type: string
+      test_config_override_path:
+        description: Path to a test config file used to override the default test config
+        required: false
+        type: string     
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
         type: string
+      chainlink_version:
+        description: Chainlink image version to use
+        default: develop
+        required: true
+        type: string        
 
 jobs:
   vrfv2plus_smoke_test:
@@ -29,35 +34,36 @@ jobs:
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
           fetch-depth: 0
-      - name: Mask base64 config
+      - name: Show test config override path in summary
+        if: ${{ inputs.test_config_override_path }}
         run: |
-          BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-      - name: Parse base64 config
-        uses: ./.github/actions/setup-parse-base64-config
-        with:
-          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}
-      - name: Send details to Step Summary
-        shell: bash
+          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
+          echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY
+      - name: Show test secrets override key in summary
+        if: ${{ inputs.test_secrets_override_key }}
         run: |
-          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_IMAGE }}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### Execution client used" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.ETH2_EL_CLIENT }}\`" >>$GITHUB_STEP_SUMMARY
+          echo "### Test secrets override key" >> $GITHUB_STEP_SUMMARY
+          echo "${{ inputs.test_secrets_override_key }}" >> $GITHUB_STEP_SUMMARY 
+      - name: Show chainlink version in summary
+        run: |
+          echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
+          echo "${{ inputs.chainlink_version }}" >> $GITHUB_STEP_SUMMARY 
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
+        env:
+          E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+          E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_version }} # Default chainlink version if not overrided in the test config 
+          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
+          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}        
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -run ^TestVRFv2Plus$/^Link_Billing$ ./smoke/vrfv2plus_test.go 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          test_config_override_path: ${{ inputs.test_config_override_path }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: vrfplus-test-logs
           artifacts_location: |
@@ -69,11 +75,4 @@ jobs:
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
-        env:
-          E2E_TEST_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
-          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
-          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+
diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
index 04e0c9318a8..0aa7146a638 100644
--- a/.github/workflows/on-demand-vrfv2plus-performance-test.yml
+++ b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
@@ -2,10 +2,6 @@ name: On Demand VRFV2 Plus Performance Test
 on:
   workflow_dispatch:
     inputs:
-      base64Config:
-        description: base64-ed config
-        required: true
-        type: string          
       performanceTestType:
         description: Performance Test Type of test to run
         type: string
@@ -14,10 +10,19 @@ on:
         description: "Regex for tests to run"
         required: false
         default: "(TestVRFV2PlusPerformance)"
+      test_config_override_path:
+        description: Path to a test config file used to override the default test config
+        required: false
+        type: string               
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
         type: string 
+      chainlink_version:
+        description: Chainlink image version to use
+        default: develop
+        required: true
+        type: string        
 
 jobs:
   vrfv2plus_performance_test:
@@ -57,33 +62,36 @@ jobs:
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
           fetch-depth: 0
-      - name: Mask base64 config
+      - name: Show test config override path in summary
+        if: ${{ inputs.test_config_override_path }}
         run: |
-          BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV          
-      - name: Merge and export base64 config
-        uses: ./.github/actions/setup-merge-base64-config
-        with:
-          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}
-      - name: Send details to Step Summary
-        shell: bash
+          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
+          echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY
+      - name: Show test secrets override key in summary
+        if: ${{ inputs.test_secrets_override_key }}
+        run: |
+          echo "### Test secrets override key" >> $GITHUB_STEP_SUMMARY
+          echo "${{ inputs.test_secrets_override_key }}" >> $GITHUB_STEP_SUMMARY           
+      - name: Show chainlink version in summary
         run: |
-          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_IMAGE }}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY                
+          echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
+          echo "${{ inputs.chainlink_version }}" >> $GITHUB_STEP_SUMMARY                  
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
+        env:
+          E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+          E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_version }} # Default chainlink version if not overrided in the test config 
+          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
+          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
         with:
           test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2plus
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          test_config_override_path: ${{ inputs.test_config_override_path }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: vrf-test-logs
           artifacts_location: ./integration-tests/load/vrfv2plus/logs/

From b039eb64a226e1bad78973cb5c20228054ef824e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 6 Sep 2024 15:02:34 +0200
Subject: [PATCH 057/141] remove file

---
 .../testconfig/vrfv2plus/overrides/ilja_pavlovs.toml        | 6 ------
 1 file changed, 6 deletions(-)
 delete mode 100644 integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml

diff --git a/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml b/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
deleted file mode 100644
index a8f0173563b..00000000000
--- a/integration-tests/testconfig/vrfv2plus/overrides/ilja_pavlovs.toml
+++ /dev/null
@@ -1,6 +0,0 @@
-[Network]
-selected_networks = ["ARBITRUM_SEPOLIA"]
-
-[ARBITRUM_SEPOLIA.VRFv2Plus.General]
-use_existing_env = true
-

From 64fe10321a7c3815956084bbf97d82d93119b037 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 6 Sep 2024 15:34:19 +0200
Subject: [PATCH 058/141] Refactor on-demand-vrfv2plus-performance-test.yml

---
 .../on-demand-vrfv2plus-performance-test.yml  | 134 ++++++++----------
 .../run-e2e-tests-reusable-workflow.yml       |  13 +-
 2 files changed, 71 insertions(+), 76 deletions(-)

diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
index 4cb99eed628..52c7249c645 100644
--- a/.github/workflows/on-demand-vrfv2plus-performance-test.yml
+++ b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
@@ -25,79 +25,63 @@ on:
         type: string        
 
 jobs:
-  vrfv2plus_performance_test:
-    name: VRFV2 Plus Performance Test
-    environment: integration
-    runs-on: ubuntu22.04-8cores-32GB
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      LOKI_URL: ${{ secrets.LOKI_URL }}
-      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
-      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
-      TEST_TYPE: ${{ inputs.performanceTestType }}
-      TEST_LOG_LEVEL: debug
-      REF_NAME: ${{ github.head_ref || github.ref_name }}
-      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
-      SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }}
-      GRAFANA_URL: "http://localhost:8080/primary"
-      GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-      GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-      WASP_LOG_LEVEL: info
-    steps:      
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
-        with:
-          id: on-demand-vrfv2-plus-performance-test
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: ${{ inputs.network }} VRFV2 Plus Performance Test
-        continue-on-error: true
-      - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Show test config override path in summary
-        if: ${{ inputs.test_config_override_path }}
-        run: |
-          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
-          echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY
-      - name: Show test secrets override key in summary
-        if: ${{ inputs.test_secrets_override_key }}
+  set-tests-to-run:
+    name: Set tests to run
+    runs-on: ubuntu-latest
+    outputs:
+      test_list: ${{ steps.set-tests.outputs.test_list }}
+    steps:
+      - name: Generate Test List JSON
+        id: set-tests
         run: |
-          echo "### Test secrets override key" >> $GITHUB_STEP_SUMMARY
-          echo "${{ inputs.test_secrets_override_key }}" >> $GITHUB_STEP_SUMMARY           
-      - name: Show chainlink version in summary
-        run: |
-          echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
-          echo "${{ inputs.chainlink_version }}" >> $GITHUB_STEP_SUMMARY                  
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
-        env:
-          E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-          E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_version }} # Default chainlink version if not overrided in the test config 
-          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
-          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-        with:
-          test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2plus
-          test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_path: ${{ inputs.test_config_override_path }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          artifacts_name: vrf-test-logs
-          artifacts_location: ./integration-tests/load/vrfv2plus/logs/
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          should_cleanup: false
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          TEST_CMD='cd integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2plus'
+          TEST_CONFIG_OVERRIDE_PATH=${{ inputs.test_config_override_path }}
+          TEST_TYPE=${{ inputs.performanceTestType }}
+
+          TEST_LIST=$(jq -n -c \
+            --arg test_cmd "$TEST_CMD" \
+            --arg test_config_override_path "$TEST_CONFIG_OVERRIDE_PATH" \
+            --arg TEST_TYPE "$TEST_TYPE" \
+            '{
+              "tests": [
+                {
+                  "id": "TestVRFv2Plus_Performance",
+                  "path": "integration-tests/load/vrfv2plus/vrfv2plus_test.go",
+                  "runs_on": "ubuntu22.04-8cores-32GB",
+                  "test_env_type": "docker",
+                  "test_cmd": $test_cmd,
+                  "test_config_override_path": $test_config_override_path,
+                  "test_env_vars": {
+                    "TEST_TYPE": $TEST_TYPE   
+                  }
+                }
+              ]
+            }')
+
+          echo "test_list=$TEST_LIST" >> $GITHUB_OUTPUT
+
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    needs: set-tests-to-run
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
+      chainlink_version: ${{ inputs.chainlink_version }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
+      SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }}      
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index c4ab54fc689..e4d6a81b5dc 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -123,6 +123,9 @@ on:
     secrets:
       TEST_SECRETS_OVERRIDE_BASE64:
         required: false
+      # Use instead of slack_notification_after_tests_channel_id if channel id is secret
+      SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID:
+        required: false              
       QA_AWS_REGION:
         required: true
       QA_AWS_ROLE_TO_ASSUME:
@@ -153,6 +156,12 @@ on:
         required: true
       SLACK_BOT_TOKEN:
         required: false
+      # Used in some tests to send slack notifications  
+      SLACK_API_KEY:
+        required: false
+      # Used in some tests to send slack notifications  
+      SLACK_CHANNEL:
+        required: false    
 
 env:
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
@@ -165,6 +174,8 @@ env:
   MOD_CACHE_VERSION: 1
   TEST_LOG_LEVEL: ${{ inputs.test_log_level }}
   METRICS_COLLECTION_ID: chainlink-e2e-tests
+  SLACK_API_KEY: ${{ secrets.SLACK_API_KEY }}
+  SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
 
 jobs:
   validate-inputs:
@@ -844,7 +855,7 @@ jobs:
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
         with:
-          channel-id: ${{ inputs.slack_notification_after_tests_channel_id }}
+          channel-id: ${{ inputs.slack_notification_after_tests_channel_id || secrets.SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID }}
           payload: |
             {
               "attachments": [

From 2e2badd590ff6a873cd4435d408c692376fd2b86 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 6 Sep 2024 15:36:45 +0200
Subject: [PATCH 059/141] Add slack notification for
 on-demand-vrfv2plus-smoke-tests.yml

---
 .github/workflows/on-demand-vrfv2plus-smoke-tests.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
index 268bb7e7d7a..652d983c07f 100644
--- a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
+++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
@@ -70,6 +70,8 @@ jobs:
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}
+      slack_notification_after_tests: always
+      slack_notification_after_tests_name: VRFV2Plus Smoke Tests
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
@@ -86,3 +88,5 @@ jobs:
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
       AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
       TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
+      SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID: ${{ secrets.QA_VRF_SLACK_CHANNEL }}

From af190087abcf48950544f27ead1971b74f91c71d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 6 Sep 2024 15:40:37 +0200
Subject: [PATCH 060/141] Fix

---
 .github/workflows/on-demand-vrfv2plus-smoke-tests.yml | 2 +-
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
index 652d983c07f..deb1b57df4f 100644
--- a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
+++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
@@ -71,7 +71,7 @@ jobs:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}
       slack_notification_after_tests: always
-      slack_notification_after_tests_name: VRFV2Plus Smoke Tests
+      slack_notification_after_tests_name: VRF V2 Plus Smoke Tests
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index e4d6a81b5dc..6eef915ae18 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -887,7 +887,7 @@ jobs:
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
         with:
-          channel-id: ${{ inputs.slack_notification_after_tests_channel_id }}
+          channel-id: ${{ inputs.slack_notification_after_tests_channel_id || secrets.SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID }}
           payload: |
             {
               "thread_ts": "${{ steps.slack.outputs.thread_ts }}",

From 7ba4fe2d463942670aec03a63af907d0895f2293 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 6 Sep 2024 15:53:46 +0200
Subject: [PATCH 061/141] fix 2

---
 .../workflows/run-e2e-tests-reusable-workflow.yml   | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 6eef915ae18..5dc338f538e 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -122,10 +122,7 @@ on:
         value: ${{ jobs.after_tests.outputs.test_results }}        
     secrets:
       TEST_SECRETS_OVERRIDE_BASE64:
-        required: false
-      # Use instead of slack_notification_after_tests_channel_id if channel id is secret
-      SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID:
-        required: false              
+        required: false        
       QA_AWS_REGION:
         required: true
       QA_AWS_ROLE_TO_ASSUME:
@@ -156,6 +153,9 @@ on:
         required: true
       SLACK_BOT_TOKEN:
         required: false
+      # Use instead of slack_notification_after_tests_channel_id if channel id is secret
+      SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID:
+        required: false        
       # Used in some tests to send slack notifications  
       SLACK_API_KEY:
         required: false
@@ -822,9 +822,14 @@ jobs:
     if: always()
     name: After tests
     runs-on: ubuntu-latest
+    # Set to access secrets like secrets.QA_SLACK_API_KEY that are set in the "integration" environment
+    environment: integration
     outputs:
       test_results: ${{ steps.set_test_results.outputs.results }}    
     steps:
+      - name: Echo SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID
+        run: echo ${{ secrets.SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID }}
+
       - name: Download all test result artifacts
         uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
         with:

From 043e019ae835364849e1808809b30094bd8cedab Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 6 Sep 2024 16:19:05 +0200
Subject: [PATCH 062/141] remove debug info

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 5dc338f538e..281ac6dfaf4 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -827,9 +827,6 @@ jobs:
     outputs:
       test_results: ${{ steps.set_test_results.outputs.results }}    
     steps:
-      - name: Echo SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID
-        run: echo ${{ secrets.SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID }}
-
       - name: Download all test result artifacts
         uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
         with:

From ef1b88a36e2a820fe97a33047e7a842d627a3c74 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 6 Sep 2024 16:24:26 +0200
Subject: [PATCH 063/141] update color of slack notification

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 281ac6dfaf4..38ab56fb9d8 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -862,7 +862,7 @@ jobs:
             {
               "attachments": [
                 {
-                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || '#2E7D32' }}",
+                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || '#3fae55' }}",
                   "blocks": [
                     {
                       "type": "section",

From cec3c5953af9ec9590999bfeb71ebed6b7a861c8 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 10:04:33 +0200
Subject: [PATCH 064/141] Migrate on-demand-vrfv2-eth2-clients-test

---
 .../on-demand-vrfv2plus-eth2-clients-test.yml | 93 +++++++------------
 1 file changed, 34 insertions(+), 59 deletions(-)

diff --git a/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml b/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml
index 0636a299442..f3921a814f2 100644
--- a/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml
+++ b/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml
@@ -17,62 +17,37 @@ on:
         type: string        
 
 jobs:
-  vrfv2plus_smoke_test:
-    name: VRFV2Plus Smoke Test with custom EL client
-    environment: integration
-    runs-on: ubuntu22.04-8cores-32GB
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      TEST_LOG_LEVEL: debug
-      REF_NAME: ${{ github.head_ref || github.ref_name }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Show test config override path in summary
-        if: ${{ inputs.test_config_override_path }}
-        run: |
-          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
-          echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY
-      - name: Show test secrets override key in summary
-        if: ${{ inputs.test_secrets_override_key }}
-        run: |
-          echo "### Test secrets override key" >> $GITHUB_STEP_SUMMARY
-          echo "${{ inputs.test_secrets_override_key }}" >> $GITHUB_STEP_SUMMARY 
-      - name: Show chainlink version in summary
-        run: |
-          echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
-          echo "${{ inputs.chainlink_version }}" >> $GITHUB_STEP_SUMMARY 
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
-        env:
-          E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-          E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_version }} # Default chainlink version if not overrided in the test config 
-          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
-          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}        
-        with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -run ^TestVRFv2Plus$/^Link_Billing$ ./smoke/vrfv2plus_test.go 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
-          test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_path: ${{ inputs.test_config_override_path }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          artifacts_name: vrfplus-test-logs
-          artifacts_location: |
-            ./integration-tests/smoke/logs/
-            ./integration-tests/smoke/db_dumps/
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          should_cleanup: false
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ""
-
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      custom_test_list_json: >
+        {
+          "tests": [
+            {
+              "id": "TestVRFv2Plus",
+              "path": "integration-tests/smoke/vrfv2plus_test.go",
+              "runs_on": "ubuntu-latest",
+              "test_env_type": "docker",
+              "test_cmd": "cd integration-tests/smoke && go test -timeout 30m -count=1 -json -run ^TestVRFv2Plus$/^Link_Billing$ .vrfv2plus_test.go
+            }
+          ]
+        }  
+      test_config_override_path: ${{ inputs.test_config_override_path }}
+      chainlink_version: ${{ inputs.chainlink_version }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}      
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}

From 58390ae3488be2abee9b00c82e611395006e855c Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 10:43:33 +0200
Subject: [PATCH 065/141] Migrate on-demand-ocr-soak-test.yml (WIP)

---
 .github/e2e-tests.yml                         |  10 +-
 .github/workflows/on-demand-ocr-soak-test.yml | 127 +++++-------------
 .../run-e2e-tests-reusable-workflow.yml       |  23 ----
 3 files changed, 37 insertions(+), 123 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 30d1bf28a84..0b29c727be1 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -21,22 +21,20 @@ runner-test-matrix:
     pyroscope_env: ci-smoke-ocr-evm-simulated
 
   # Example of a configuration for running a single soak test in Kubernetes Remote Runner
-  - id: soak/ocr_test.go:^TestOCRv1Soak$
+  - id: soak/ocr_test.go:TestOCRv1Soak
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true    
     test_env_vars:
       TEST_SUITE: soak
 
-  - id: soak/ocr_test.go:^TestOCRv2Soak$
+  - id: soak/ocr_test.go:TestOCRv2Soak
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run TestOCRv2Soak -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak
diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
index 9099591a7cc..d1d0e4192ba 100644
--- a/.github/workflows/on-demand-ocr-soak-test.yml
+++ b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -8,25 +8,20 @@ on:
         default: TestOCRSoak
         type: choice
         options:
-          - TestOCRv1Soak
-          - TestOCRv2Soak
-          - TestForwarderOCRv1Soak
-          - TestForwarderOCRv2Soak
-          - TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled
-          - TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled
-          - TestOCRSoak_GasSpike
-          - TestOCRSoak_ChangeBlockGasLimit
-          - TestOCRSoak_RPCDownForAllCLNodes
-          - TestOCRSoak_RPCDownForHalfCLNodes
+          - soak/ocr_test.go:TestOCRv1Soak
+          - soak/ocr_test.go:TestOCRv2Soak
+          # - TestForwarderOCRv1Soak
+          # - TestForwarderOCRv2Soak
+          # - TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled
+          # - TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled
+          # - TestOCRSoak_GasSpike
+          # - TestOCRSoak_ChangeBlockGasLimit
+          # - TestOCRSoak_RPCDownForAllCLNodes
+          # - TestOCRSoak_RPCDownForHalfCLNodes
       test_config_override_path:
         description: Path to a test config file used to override the default test config
         required: false
         type: string
-      slackMemberID:
-        description: Slack Member ID (Not your @)
-        required: true
-        default: U01A2B2C3D4
-        type: string
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
@@ -38,84 +33,28 @@ on:
         type: string
         
 jobs:
-  ocr_soak_test:
-    name: OCR Soak Test
-    environment: integration
-    runs-on: ubuntu-latest
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      CHAINLINK_ENV_USER: ${{ github.actor }}
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      test_ids: ${{ inputs.testToRun}}
+      test_config_override_path: ${{ inputs.test_config_override_path }}
+      chainlink_version: ${{ inputs.chainlink_version }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
       SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
       SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
-      TEST_LOG_LEVEL: debug
-      REF_NAME: ${{ github.head_ref || github.ref_name }}
-      ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
-    steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
-        with:
-          id: on-demand-ocr-soak-test
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: ${{ inputs.network }} OCR Soak Test
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          ref: ${{ env.REF_NAME }}
-      - name: Get Slack config
-        run: |
-          SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$SLACK_USER
-          echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
-      - name: Show test config override path in summary
-        if: ${{ inputs.test_config_override_path }}
-        run: |
-          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
-          echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY          
-      - name: Show chainlink version in summary
-        run: |
-          echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
-          echo "${{ inputs.chainlink_version }}" >> $GITHUB_STEP_SUMMARY
-      - name: Build Image
-        id: build_test_image
-        uses: ./.github/actions/build-test-image
-        with:
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
-        env:
-          DETACH_RUNNER: true
-          TEST_SUITE: soak
-          TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
-          ENV_JOB_IMAGE: ${{ steps.build_test_image.outputs.test_image }}
-          # We can comment these out when we have a stable soak test and aren't worried about resource consumption
-          TEST_UPLOAD_CPU_PROFILE: true
-          TEST_UPLOAD_MEM_PROFILE: true
-          E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-          E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_version }}
-          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
-          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}          
-        with:
-          test_command_to_run: cd ./integration-tests && go test -v -count=1 -run ^${{ github.event.inputs.testToRun }}$ ./soak
-          test_download_vendor_packages_command: make gomod
-          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_path: ${{ inputs.test_config_override_path }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          should_cleanup: false
-          go_mod_path: ./integration-tests/go.mod
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 38ab56fb9d8..56c4f490c6c 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -352,29 +352,6 @@ jobs:
           fi
         shell: bash
 
-      - name: Check if test config override is required for any test
-        shell: bash
-        run: |
-          # Parse the JSON to check for test_config_override_required in Docker matrix
-          DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq 'if .tests then .tests[] | select(has("test_config_override_required") and .test_config_override_required) | .id else empty end' -r)
-          # Parse the JSON to check for test_config_override_required in Kubernetes matrix
-          K8S_TESTS_REQUIRING_CONFIG_OVERRIDE=$(echo '${{ steps.set-k8s-runner-matrix.outputs.matrix }}' | jq 'if .tests then .tests[] | select(has("test_config_override_required") and .test_config_override_required) | .id else empty end' -r)
-
-          # Determine if any tests require a configuration override
-          if [ ! -z "$DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE" ] || [ ! -z "$K8S_TESTS_REQUIRING_CONFIG_OVERRIDE" ]; then
-            echo "Tests in .github/e2e-tests.yml requiring test config override:"
-            if [ ! -z "$DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE" ]; then
-              echo $DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE
-            fi
-            if [ ! -z "$K8S_TESTS_REQUIRING_CONFIG_OVERRIDE" ]; then
-              echo $K8S_TESTS_REQUIRING_CONFIG_OVERRIDE
-            fi
-            echo "::error::Error: Some of the tests require a test config override. Please see workflow logs and set 'test_config_override_path' to run these tests."
-            exit 1
-          else
-            echo "No tests require a configuration override. Proceeding without overrides."
-          fi
-
       - name: Check if test secrets are required for any test
         shell: bash
         run: |

From d7f4241c6743ebc0fcbcb480ff0b1198fb5bffcc Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 10:53:35 +0200
Subject: [PATCH 066/141] Rename step

---
 .../workflows/run-e2e-tests-reusable-workflow.yml    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 56c4f490c6c..dce753d78c9 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -638,10 +638,10 @@ jobs:
 
   # Run K8s tests using old remote runner
 
-  prepare-remote-runner-test-image:
+  get-remote-runner-test-image:
     needs: [load-test-configurations, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr]
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' && always() && !failure() && !cancelled() }}
-    name: Prepare remote runner test image
+    name: Get emote runner test image
     runs-on: ubuntu-latest
     environment: integration
     permissions:
@@ -677,7 +677,7 @@ jobs:
           fi
 
   run-k8s-runner-tests:
-    needs: [load-test-configurations, prepare-remote-runner-test-image, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr, get_latest_chainlink_release_version]
+    needs: [load-test-configurations, get-remote-runner-test-image, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr, get_latest_chainlink_release_version]
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' && always() && !failure() && !cancelled() }}
     name: ${{ matrix.tests.id }}
     runs-on: ${{ matrix.tests.runs_on }}
@@ -725,9 +725,9 @@ jobs:
 
       - name: Show remote runner version in summary
         run: |
-          echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
+          echo "Remote Runner Version: ${{ needs.get-remote-runner-test-image.outputs.remote-runner-version }}"
           echo "### Remote Runner Version" >> $GITHUB_STEP_SUMMARY
-          echo "${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}" >> $GITHUB_STEP_SUMMARY
+          echo "${{ needs.get-remote-runner-test-image.outputs.remote-runner-version }}" >> $GITHUB_STEP_SUMMARY
 
       - name: Show test configuration in logs
         run: echo '${{ toJson(matrix.tests) }}' | jq .
@@ -739,7 +739,7 @@ jobs:
           DETACH_RUNNER: true
           RR_MEM: ${{ matrix.tests.remote_runner_memory }}
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
-          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}
+          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.get-remote-runner-test-image.outputs.remote-runner-version }}
           INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
           # We can comment these out when we have a stable soak test and aren't worried about resource consumption
           TEST_UPLOAD_CPU_PROFILE: true

From c7d8fb3f9bbb2c8bc0af63a85052a825cb4465e5 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 11:03:09 +0200
Subject: [PATCH 067/141] update test names

---
 .github/e2e-tests.yml                         | 45 ++++++++-----------
 .github/workflows/on-demand-ocr-soak-test.yml | 16 +++----
 2 files changed, 26 insertions(+), 35 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 0b29c727be1..49d50c4a685 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -20,7 +20,6 @@ runner-test-matrix:
     test_cmd: cd integration-tests/ && go test smoke/ocr_test.go -timeout 30m -count=1 -test.parallel=2 -json
     pyroscope_env: ci-smoke-ocr-evm-simulated
 
-  # Example of a configuration for running a single soak test in Kubernetes Remote Runner
   - id: soak/ocr_test.go:TestOCRv1Soak
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
@@ -34,7 +33,7 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run TestOCRv2Soak -test.parallel=1 -timeout 900h -count=1 -json
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run $TestOCRv2Soak$ -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak
@@ -43,88 +42,80 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 -json
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 900h -count=1 -json
     test_config_override_path: integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak
 
-  - id: soak/ocr_test.go:^TestForwarderOCRv1Soak$
+  - id: soak/ocr_test.go:TestForwarderOCRv1Soak
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv1Soak$ -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak
 
-  - id: soak/ocr_test.go:^TestForwarderOCRv2Soak$
+  - id: soak/ocr_test.go:TestForwarderOCRv2Soak
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv2Soak$ -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak
       
-  - id: soak/ocr_test.go:^TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled$
+  - id: soak/ocr_test.go:TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled$ -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak      
 
-  - id: soak/ocr_test.go:^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$
+  - id: soak/ocr_test.go:TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$ -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$ -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak      
 
-  - id: soak/ocr_test.go:^TestOCRSoak_GasSpike$
+  - id: soak/ocr_test.go:TestOCRSoak_GasSpike
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GasSpike$ -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GasSpike$ -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak      
 
-  - id: soak/ocr_test.go:^TestOCRSoak_ChangeBlockGasLimit$
+  - id: soak/ocr_test.go:TestOCRSoak_ChangeBlockGasLimit
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_ChangeBlockGasLimit$ -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_ChangeBlockGasLimit$ -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak   
 
-  - id: soak/ocr_test.go:^TestOCRSoak_RPCDownForAllCLNodes$
+  - id: soak/ocr_test.go:TestOCRSoak_RPCDownForAllCLNodes
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForAllCLNodes$ -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForAllCLNodes$ -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak  
 
-  - id: soak/ocr_test.go:^TestOCRSoak_RPCDownForHalfCLNodes$
+  - id: soak/ocr_test.go:TestOCRSoak_RPCDownForHalfCLNodes
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForHalfCLNodes$ -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForHalfCLNodes$ -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak                             
diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
index d1d0e4192ba..0a988ca090b 100644
--- a/.github/workflows/on-demand-ocr-soak-test.yml
+++ b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -10,14 +10,14 @@ on:
         options:
           - soak/ocr_test.go:TestOCRv1Soak
           - soak/ocr_test.go:TestOCRv2Soak
-          # - TestForwarderOCRv1Soak
-          # - TestForwarderOCRv2Soak
-          # - TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled
-          # - TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled
-          # - TestOCRSoak_GasSpike
-          # - TestOCRSoak_ChangeBlockGasLimit
-          # - TestOCRSoak_RPCDownForAllCLNodes
-          # - TestOCRSoak_RPCDownForHalfCLNodes
+          - soak/ocr_test.go:TestForwarderOCRv1Soak
+          - soak/ocr_test.go:TestForwarderOCRv2Soak
+          - soak/ocr_test.go:TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled
+          - soak/ocr_test.go:TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled
+          - soak/ocr_test.go:TestOCRSoak_GasSpike
+          - soak/ocr_test.go:TestOCRSoak_ChangeBlockGasLimit
+          - soak/ocr_test.go:TestOCRSoak_RPCDownForAllCLNodes
+          - soak/ocr_test.go:TestOCRSoak_RPCDownForHalfCLNodes
       test_config_override_path:
         description: Path to a test config file used to override the default test config
         required: false

From e917232b303b7be1585d31ec0c77e5369c7f27f7 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 11:10:08 +0200
Subject: [PATCH 068/141] fix

---
 .github/workflows/on-demand-ocr-soak-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
index 0a988ca090b..32a16a69d3c 100644
--- a/.github/workflows/on-demand-ocr-soak-test.yml
+++ b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -3,7 +3,7 @@ on:
   workflow_dispatch:
     inputs:
       testToRun:
-        description: Select a test to run
+        description: Select a test to run from .github/e2e-tests.yml
         required: true
         default: TestOCRSoak
         type: choice

From cc45e7a39f6a2a0dd2f535b763c2cbc79b735290 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 11:10:46 +0200
Subject: [PATCH 069/141] fix name

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index dce753d78c9..dfe4a8ac164 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -641,7 +641,7 @@ jobs:
   get-remote-runner-test-image:
     needs: [load-test-configurations, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr]
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' && always() && !failure() && !cancelled() }}
-    name: Get emote runner test image
+    name: Get remote runner test image
     runs-on: ubuntu-latest
     environment: integration
     permissions:

From 9bc9744bb7d189ad3a5762115fea971e8811db42 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 12:05:42 +0200
Subject: [PATCH 070/141] fix

---
 .github/e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 49d50c4a685..2636cdedb9c 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -33,7 +33,7 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run $TestOCRv2Soak$ -test.parallel=1 -timeout 900h -count=1 -json
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 900h -count=1 -json
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak

From cad06ca2848b39e62212ac1589930b1eda470a34 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 13:12:10 +0200
Subject: [PATCH 071/141] Migrate automation-nightly-tests.yml

---
 .github/e2e-tests.yml                         |   6 +-
 .../workflows/automation-nightly-tests.yml    | 315 ++----------------
 .../run-e2e-tests-reusable-workflow.yml       |   1 +
 3 files changed, 32 insertions(+), 290 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 2636cdedb9c..261ea78402a 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -505,7 +505,7 @@ runner-test-matrix:
       E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
       E2E_TEST_CHAINLINK_VERSION: latest
       E2E_TEST_CHAINLINK_UPGRADE_IMAGE: '{{ env.QA_CHAINLINK_IMAGE }}'
-      E2E_TEST_CHAINLINK_UPGRADE_VERSION: develop
+      E2E_TEST_CHAINLINK_UPGRADE_VERSION: '{{ env.DEFAULT_CHAINLINK_UPGRADE_VERSION }}'
     pyroscope_env: ci-smoke-automation-upgrade-tests
 
   - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1
@@ -519,7 +519,7 @@ runner-test-matrix:
       E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
       E2E_TEST_CHAINLINK_VERSION: latest
       E2E_TEST_CHAINLINK_UPGRADE_IMAGE: '{{ env.QA_CHAINLINK_IMAGE }}'
-      E2E_TEST_CHAINLINK_UPGRADE_VERSION: develop
+      E2E_TEST_CHAINLINK_UPGRADE_VERSION: '{{ env.DEFAULT_CHAINLINK_UPGRADE_VERSION }}'
     pyroscope_env: ci-smoke-automation-upgrade-tests
 
   - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2
@@ -533,7 +533,7 @@ runner-test-matrix:
       E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
       E2E_TEST_CHAINLINK_VERSION: latest
       E2E_TEST_CHAINLINK_UPGRADE_IMAGE: '{{ env.QA_CHAINLINK_IMAGE }}'
-      E2E_TEST_CHAINLINK_UPGRADE_VERSION: develop
+      E2E_TEST_CHAINLINK_UPGRADE_VERSION: '{{ env.DEFAULT_CHAINLINK_UPGRADE_VERSION }}'
     pyroscope_env: ci-smoke-automation-upgrade-tests
 
   - id: reorg/automation_reorg_test.go^TestAutomationReorg/registry_2_0
diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml
index 59d11728954..6b88c0a7164 100644
--- a/.github/workflows/automation-nightly-tests.yml
+++ b/.github/workflows/automation-nightly-tests.yml
@@ -7,291 +7,32 @@ on:
       - "*"
   workflow_dispatch:
 
-env:
-  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-
 jobs:
-  build-chainlink:
-    environment: integration
-    permissions:
-      id-token: write
-      contents: read
-    name: Build Chainlink Image
-    runs-on: ubuntu22.04-16cores-64GB
-    steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
-        with:
-          id: automation-nightly-build-chainlink
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Build Chainlink Image
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
-      - name: Build Chainlink Image
-        uses: ./.github/actions/build-chainlink-image
-        with:
-          tag_suffix: ""
-          dockerfile: core/chainlink.Dockerfile
-          git_commit_sha: ${{ github.sha }}
-          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-
-  automation-upgrade-tests:
-    environment: integration
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink]
-    env:
-      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_LOG_LEVEL: info
-      SELECTED_NETWORKS: "SIMULATED"
-    strategy:
-      fail-fast: false
-      matrix:
-        tests:
-          - name: Upgrade 2.0
-            id: upgrade-2-0
-            suite: smoke
-            nodes: 1
-            os: ubuntu22.04-8cores-32GB
-            network: SIMULATED
-            command: -run ^TestAutomationNodeUpgrade/registry_2_0 ./smoke
-          - name: Upgrade 2.1
-            id: upgrade-2-1
-            suite: smoke
-            nodes: 5
-            os: ubuntu22.04-8cores-32GB
-            network: SIMULATED
-            command: -run ^TestAutomationNodeUpgrade/registry_2_1 ./smoke
-          - name: Upgrade 2.2
-            id: upgrade-2-2
-            suite: smoke
-            nodes: 5
-            os: ubuntu22.04-8cores-32GB
-            network: SIMULATED
-            command: -run ^TestAutomationNodeUpgrade/registry_2_2 ./smoke
-    runs-on: ${{ matrix.tests.os }}
-    name: Automation ${{ matrix.tests.name }} Test
-    steps:
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          ref: ${{ github.head_ref || github.ref_name }}
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
-        env:
-          TEST_SUITE: ${{ matrix.tests.suite }}
-          E2E_TEST_SELECTED_NETWORK: ${{ env.SELECTED_NETWORKS }}
-          E2E_TEST_CHAINLINK_IMAGE: "public.ecr.aws/chainlink/chainlink"
-          E2E_TEST_CHAINLINK_VERSION: "latest"
-          E2E_TEST_CHAINLINK_UPGRADE_IMAGE: ${{ env.CHAINLINK_IMAGE }}
-          E2E_TEST_CHAINLINK_UPGRADE_VERSION: ${{ github.sha }}
-          E2E_TEST_LOGGING_RUN_ID: ${{ github.run_id }}
-          E2E_TEST_LOG_COLLECT: "true"
-          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          E2E_TEST_LOG_STREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
-          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-        with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 60m -count=1 -json -test.parallel=${{ matrix.tests.nodes }} ${{ matrix.tests.command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
-          test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          cl_repo: 'public.ecr.aws/chainlink/chainlink'
-          cl_image_tag: 'latest'
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          artifacts_location: ./integration-tests/${{ matrix.tests.suite }}/logs
-          artifacts_name: testcontainers-logs-${{ matrix.tests.name }}
-          publish_check_name: Automation Results ${{ matrix.tests.name }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Upload test log
-        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
-        if: failure()
-        with:
-          name: gotest-logs-${{ matrix.tests.name }}
-          path: /tmp/gotest.log
-          retention-days: 7
-        continue-on-error: true
-      - name: Collect Metrics
-        if: always()
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
-        with:
-          id: automation-nightly-upgrade-tests-${{ matrix.tests.id }}
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Automation ${{ matrix.tests.name }} Test
-          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
-        continue-on-error: true
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
-
-  test-notify:
-    name: Start Slack Thread
-    if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
-    environment: integration
-    outputs:
-      thread_ts: ${{ steps.slack.outputs.thread_ts }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs: [ automation-upgrade-tests ]
-    steps:
-      - name: Debug Result
-        run: echo ${{ join(needs.*.result, ',') }}
-      - name: Main Slack Notification
-        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
-        id: slack
-        with:
-          channel-id: C03KJ5S7KEK
-          payload: |
-            {
-              "attachments": [
-                {
-                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}",
-                  "blocks": [
-                    {
-                      "type": "header",
-                      "text": {
-                        "type": "plain_text",
-                        "text": "Automation Nightly Tests ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}",
-                        "emoji": true
-                      }
-                    },
-                    {
-                      "type": "divider"
-                    },
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "mrkdwn",
-                        "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>"
-                      }
-                    }
-                  ]
-                }
-              ]
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
-
-  test-results:
-    name: Post Test Results for ${{ matrix.name }}
-    if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
-    environment: integration
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs: test-notify
-    strategy:
-      fail-fast: false
-      matrix:
-        name: [ Upgrade 2.0, Upgrade 2.1, Upgrade 2.2 ]
-    steps:
-      - name: Get Results
-        id: test-results
-        run: |
-          # I feel like there's some clever, fully jq way to do this, but I ain't got the motivation to figure it out
-          echo "Querying test results"
-
-          PARSED_RESULTS=$(curl \
-          -H "Authorization: Bearer ${{ github.token }}" \
-          'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \
-          | jq -r --arg pattern "${{ matrix.name }} Test" '.jobs[]
-          | select(.name | test($pattern)) as $job
-          | $job.steps[]
-          | select(.name == "Run Tests")
-          | { conclusion: (if .conclusion == "success" then ":white_check_mark:" else ":x:" end), product: ("*" + ($job.name | capture($pattern).product) + "*") }')
-
-          echo "Parsed Results:"
-          echo $PARSED_RESULTS
-
-          ALL_SUCCESS=true
-          for row in $(echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'); do
-              success=false
-              break
-          done
-
-          echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT
-
-          FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[] 
-          | { 
-            conclusion: .conclusion, 
-            product: .product 
-            }
-          ]
-          | map("{\"type\": \"section\", \"text\": {\"type\": \"mrkdwn\", \"text\": \"\(.product): \(.conclusion)\"}}")
-          | join(",")')
-
-          echo "Formatted Results:"
-          echo $FORMATTED_RESULTS
-
-          # Cleans out backslashes and quotes from jq
-          CLEAN_RESULTS=$(echo "$FORMATTED_RESULTS" | sed 's/\\\"/"/g' | sed 's/^"//;s/"$//')
-
-          echo "Clean Results"
-          echo $CLEAN_RESULTS
-
-          echo results=$CLEAN_RESULTS >> $GITHUB_OUTPUT
-
-      - name: Test Details
-        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
-        with:
-          channel-id: C03KJ5S7KEK
-          payload: |
-            {
-              "thread_ts": "${{ needs.test-notify.outputs.thread_ts }}",
-              "attachments": [
-                {
-                  "color": "${{ steps.test-results.outputs.all_success && '#2E7D32' || '#C62828' }}",
-                  "blocks": [
-                    {
-                      "type": "header",
-                      "text": {
-                        "type": "plain_text",
-                        "text": "${{ matrix.name }} ${{ steps.test-results.outputs.all_success && ':white_check_mark:' || ':x: Notifying <@U02Q14G80TY>'}}",
-                        "emoji": true
-                      }
-                    },
-                    {
-                      "type": "divider"
-                    },
-                    ${{ steps.test-results.outputs.results }}
-                  ]
-                }
-              ]
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
\ No newline at end of file
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      test_ids: 'smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2'
+      chainlink_version: ${{ github.sha }}
+      slack_notification_after_tests: true
+      slack_notification_after_tests_channel_id: "#lf-slack-notification-tests"
+      slack_notification_after_tests_name: Automation Nightly E2E Tests
+      # slack_notification_after_tests_notify_user_id_on_failure: U0XXXXXXX
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
+      SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index dfe4a8ac164..d35cf94a9f0 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -168,6 +168,7 @@ env:
   QA_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
   DEFAULT_CHAINLINK_VERSION: ${{ inputs.chainlink_version || github.sha }}
   DEFAULT_CHAINLINK_PLUGINS_VERSION: ${{ inputs.chainlink_version != '' && format('{0}-plugins', inputs.chainlink_version) || format('{0}-plugins', github.sha) }}
+  DEFAULT_CHAINLINK_UPGRADE_VERSION: ${{ inputs.chainlink_version || github.sha }}
   CHAINLINK_ENV_USER: ${{ github.actor }}
   CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
   SELECTED_NETWORKS: SIMULATED

From 4beb6aab08250c818733c93d34004148a9864471 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 13:15:08 +0200
Subject: [PATCH 072/141] Build remote runner image asap

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index d35cf94a9f0..593f480f13e 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -640,7 +640,7 @@ jobs:
   # Run K8s tests using old remote runner
 
   get-remote-runner-test-image:
-    needs: [load-test-configurations, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr]
+    needs: [load-test-configurations]
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' && always() && !failure() && !cancelled() }}
     name: Get remote runner test image
     runs-on: ubuntu-latest

From bfb8152db5593aabb52bdbfadb9266b4b985c383 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 13:19:59 +0200
Subject: [PATCH 073/141] Fix

---
 .github/e2e-tests.yml                          | 6 +++---
 .github/workflows/automation-nightly-tests.yml | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 261ea78402a..65bdc4dfdf5 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -499,7 +499,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu22.04-8cores-32GB
     workflows:
-      - Run Automation Product Nightly E2E Tests
+      - Automation Nightly Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
     test_env_vars:
       E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
@@ -513,7 +513,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu22.04-8cores-32GB
     workflows:
-      - Run Automation Product Nightly E2E Tests    
+      - Automation Nightly Tests    
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json
     test_env_vars:
       E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
@@ -527,7 +527,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu22.04-8cores-32GB
     workflows:
-      - Run Automation Product Nightly E2E Tests
+      - Automation Nightly Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json
     test_env_vars:
       E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml
index 6b88c0a7164..f27e98494d6 100644
--- a/.github/workflows/automation-nightly-tests.yml
+++ b/.github/workflows/automation-nightly-tests.yml
@@ -12,7 +12,7 @@ jobs:
     name: Run E2E Tests
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      test_ids: 'smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2'
+      test_workflow: Automation Nightly Tests
       chainlink_version: ${{ github.sha }}
       slack_notification_after_tests: true
       slack_notification_after_tests_channel_id: "#lf-slack-notification-tests"

From 046eb7eb00b587101d7d5673e7efddce008685c1 Mon Sep 17 00:00:00 2001
From: joaoluisam <joao.luismoreira@smartcontract.com>
Date: Mon, 9 Sep 2024 12:43:05 +0100
Subject: [PATCH 074/141] add ocr2 celo_alfajores and base_sepolia test configs

---
 .../testconfig/ocr2/overrides/base_sepolia.toml | 17 +++++++++++++++++
 .../ocr2/overrides/celo_alfajores.toml          | 17 +++++++++++++++++
 .../ocr2/overrides/wemix_testnet.toml           |  7 ++-----
 3 files changed, 36 insertions(+), 5 deletions(-)
 create mode 100644 integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
 create mode 100644 integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml

diff --git a/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml b/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
new file mode 100644
index 00000000000..2bd6f503014
--- /dev/null
+++ b/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
@@ -0,0 +1,17 @@
+[Network]
+selected_networks = ["BASE_SEPOLIA"]
+
+# soak test specific configuration
+[Soak.Common]
+chainlink_node_funding = 0.5 # Funding per node
+
+[Soak.OCR2]
+[Soak.OCR2.Common]
+test_duration = "24h" # Full duration the test will run 1440 - 1 day, 7200 - 5 days
+
+[Soak.OCR2.Soak]
+number_of_contracts = 2    # Number of deployed contracts, 2 being default
+time_between_rounds = "5m" # Default 5 mins for a 1 day, 10 mins for a 5 day
+
+[Seth]
+experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml b/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml
new file mode 100644
index 00000000000..2bc5c240787
--- /dev/null
+++ b/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml
@@ -0,0 +1,17 @@
+[Network]
+selected_networks = ["CELO_ALFAJORES"]
+
+# soak test specific configuration
+[Soak.Common]
+chainlink_node_funding = 2 # Funding per node
+
+[Soak.OCR2]
+[Soak.OCR2.Common]
+test_duration = "15m" # Full duration the test will run 1440 - 1 day, 7200 - 5 days
+
+[Soak.OCR2.Soak]
+number_of_contracts = 2    # Number of deployed contracts, 2 being default
+time_between_rounds = "5m" # Default 5 mins for a 1 day, 10 mins for a 5 day
+
+[Seth]
+experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml b/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
index 58d3dd1fd15..dac4819d2f6 100644
--- a/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
+++ b/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
@@ -5,17 +5,14 @@ selected_networks = ["WEMIX_TESTNET"]
 [Soak.Common]
 chainlink_node_funding = 2 # Funding per node
 
-# Note this is for when you are running OCR2 tests
-# To run OCR1 tests use [Soak.OCR] instead
 [Soak.OCR2]
-# To run OCR1 tests use [Soak.OCR.Common] instead
+
 [Soak.OCR2.Common]
 test_duration = "2880m" # Full duration the test will run 1440 - 1 day, 7200 - 5 days
 
-# To run OCR1 tests use [Soak.OCR.Soak] instead
 [Soak.OCR2.Soak]
 number_of_contracts = 2    # Number of deployed contracts, 2 being default
 time_between_rounds = "5m" # Default 5 mins for a 1 day, 10 mins for a 5 day
 
 [Seth]
-experiments_enabled = ["slow_funds_return"]
\ No newline at end of file
+experiments_enabled = ["slow_funds_return"]

From d3c9f4536613f29f9d418cd086a4c9b0eea00bac Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 13:50:48 +0200
Subject: [PATCH 075/141] Fix

---
 .github/workflows/automation-nightly-tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml
index f27e98494d6..c7c4cc1a3f9 100644
--- a/.github/workflows/automation-nightly-tests.yml
+++ b/.github/workflows/automation-nightly-tests.yml
@@ -34,5 +34,4 @@ jobs:
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
       AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
       TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
-      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
-      SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

From 0eb157f3ff280145ca606d405914b3435d15764e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 14:22:12 +0200
Subject: [PATCH 076/141] Fix

---
 .github/workflows/automation-nightly-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml
index c7c4cc1a3f9..9075897fa47 100644
--- a/.github/workflows/automation-nightly-tests.yml
+++ b/.github/workflows/automation-nightly-tests.yml
@@ -15,7 +15,7 @@ jobs:
       test_workflow: Automation Nightly Tests
       chainlink_version: ${{ github.sha }}
       slack_notification_after_tests: true
-      slack_notification_after_tests_channel_id: "#lf-slack-notification-tests"
+      slack_notification_after_tests_channel_id: "#automation-test-notifications"
       slack_notification_after_tests_name: Automation Nightly E2E Tests
       # slack_notification_after_tests_notify_user_id_on_failure: U0XXXXXXX
     secrets:

From 06565ac68af0e580a542da96511c6b36614f4459 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 14:28:13 +0200
Subject: [PATCH 077/141] Migrate automation-benchmark-tests.yml

---
 .../workflows/automation-benchmark-tests.yml  | 111 ++++--------------
 .../run-e2e-tests-reusable-workflow.yml       |   2 +-
 2 files changed, 26 insertions(+), 87 deletions(-)

diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml
index 7dbc6475f21..61a55aaf698 100644
--- a/.github/workflows/automation-benchmark-tests.yml
+++ b/.github/workflows/automation-benchmark-tests.yml
@@ -2,11 +2,6 @@ name: Automation Benchmark Test
 on:
   workflow_dispatch:
     inputs:
-      testType:
-        description: Type of test to run (benchmark, soak)
-        required: true
-        default: benchmark
-        type: string
       test_config_override_path:
         description: Path to a test config file used to override the default test config
         required: false
@@ -15,11 +10,6 @@ on:
         description: Key to run tests with custom test secrets
         required: false
         type: string 
-      slackMemberID:
-        description: Notifies test results (Not your @)
-        required: true
-        default: U02Q14G80TY
-        type: string
       chainlink_version:
         description: Chainlink image version to use
         default: develop
@@ -27,80 +17,29 @@ on:
         type: string
 
 jobs:
-  automation_benchmark:
-    environment: integration
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    name: Automation Benchmark Test
-    runs-on: ubuntu22.04-16cores-64GB
-    env:
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      test_ids: 'benchmark/keeper_test.go:TestAutomationBenchmark'
+      test_config_override_path: ${{ inputs.test_config_override_path }}
+      chainlink_version: ${{ github.sha }}
+      slack_notification_after_tests_channel_id: C03KJ5S7KEK
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
       SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
-      SLACK_CHANNEL: C03KJ5S7KEK
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      REF_NAME: ${{ github.head_ref || github.ref_name }}
-    steps:
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          ref: ${{ env.REF_NAME }}
-      - name: Get Slack config and mask base64 config
-        run: |
-          SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$SLACK_USER
-          echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
-      - name: Show test config override path in summary
-        if: ${{ inputs.test_config_override_path }}
-        run: |
-          echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
-          echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY
-      - name: Show chainlink version in summary
-        run: |
-          echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
-          echo "${{ inputs.chainlink_version }}" >> $GITHUB_STEP_SUMMARY          
-      - name: Build Test Image
-        id: build_test_image
-        uses: ./.github/actions/build-test-image
-        with:
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          suites: benchmark chaos reorg load
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
-        env:
-          DETACH_RUNNER: true
-          TEST_SUITE: benchmark
-          TEST_ARGS: -test.timeout 720h
-          ENV_JOB_IMAGE: ${{ steps.build_test_image.outputs.test_image }}
-          INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
-          TEST_TYPE: ${{ github.event.inputs.testType }}
-          TEST_TEST_TYPE: ${{ github.event.inputs.testType }}
-          RR_MEM: 4Gi
-          TEST_LOG_LEVEL: info
-          E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-          E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_version }}
-        with:
-          test_command_to_run: cd integration-tests && go test -timeout 30m -v -run ^TestAutomationBenchmark$ ./benchmark -count=1
-          test_download_vendor_packages_command: make gomod
-          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_path: ${{ inputs.test_config_override_path }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          should_cleanup: false
-          go_mod_path: ./integration-tests/go.mod
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Collect Metrics
-        if: always()
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
-        with:
-          id: automation-benchmark-build-test-image
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Automation Benchmark Test
-        continue-on-error: true
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 593f480f13e..72be0f42ff3 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -176,7 +176,7 @@ env:
   TEST_LOG_LEVEL: ${{ inputs.test_log_level }}
   METRICS_COLLECTION_ID: chainlink-e2e-tests
   SLACK_API_KEY: ${{ secrets.SLACK_API_KEY }}
-  SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+  SLACK_CHANNEL: ${{ inputs.slack_notification_after_tests_channel_id || secrets.SLACK_CHANNEL }}
 
 jobs:
   validate-inputs:

From 7380ca5e21f300f6eb2b02ee9de9293243fef9ef Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 14:29:52 +0200
Subject: [PATCH 078/141] fix

---
 .github/e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 65bdc4dfdf5..6c89102354e 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -592,7 +592,7 @@ runner-test-matrix:
     test_env_vars:
       TEST_SUITE: chaos
 
-  - id: benchmark/keeper_test.go:^TestAutomationBenchmark$
+  - id: benchmark/keeper_test.go:TestAutomationBenchmark
     path: integration-tests/benchmark/keeper_test.go
     test_env_type: k8s-remote-runner
     remote_runner_memory: 4Gi

From b2662124a73f84f5389ec85387b8a4fbb99933f3 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 14:34:09 +0200
Subject: [PATCH 079/141] Migrate automation-load-tests.yml

---
 .github/e2e-tests.yml                       |   3 -
 .github/workflows/automation-load-tests.yml | 141 ++++----------------
 2 files changed, 29 insertions(+), 115 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 6c89102354e..8a3cc469d1f 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -486,12 +486,9 @@ runner-test-matrix:
     test_env_type: k8s-remote-runner
     test_cmd: cd integration-tests/load/automationv2_1 && go test -test.run TestLogTrigger -test.parallel=1 -timeout 60m -count=1 -json
     remote_runner_memory: 4Gi
-    test_config_override_required: true
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: automationv2_1
-    workflows:
-      - Automation Load Test
     pyroscope_env: automation-load-test
 
   - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0
diff --git a/.github/workflows/automation-load-tests.yml b/.github/workflows/automation-load-tests.yml
index 45a1dc422bc..e83f9fec847 100644
--- a/.github/workflows/automation-load-tests.yml
+++ b/.github/workflows/automation-load-tests.yml
@@ -2,122 +2,39 @@ name: Automation Load Test
 on:
   workflow_dispatch:
     inputs:
-      base64Config:
-        description: base64-ed config
-        required: true
-        type: string      
-      slackMemberID:
-        description: Notifies test results (Not your @)
-        required: true
-        default: U02Q14G80TY
-        type: string
+      test_config_override_path:
+        description: Path to a test config file used to override the default test config
+        required: false
+        type: string    
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
         type: string 
 
 jobs:
-  automation_load:
-    environment: integration
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    name: Automation Load Test
-    runs-on: ubuntu22.04-16cores-64GB
-    env:
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      test_ids: 'load/automationv2_1/automationv2_1_test.go:TestLogTrigger'
+      test_config_override_path: ${{ inputs.test_config_override_path }}
+      chainlink_version: ${{ github.sha }}
+      slack_notification_after_tests_channel_id: C03KJ5S7KEK
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
       SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
-      SLACK_CHANNEL: C03KJ5S7KEK
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      REF_NAME: ${{ github.head_ref || github.ref_name }}
-    steps:
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          ref: ${{ env.REF_NAME }}
-      - name: Get Slack config and mask base64 config
-        run: |
-          SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$SLACK_USER
-          echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
-
-          BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-      - name: Merge Pyrsoscope config
-        env:
-          PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-          PYROSCOPE_ENVIRONMENT: "automation-load-test"
-          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-        run: |
-          decoded_toml=$(echo $BASE64_CONFIG_OVERRIDE | base64 -d)
-
-          # use Pyroscope config from GH secrets and merge it with base64 input
-          cat << EOF > config.toml
-          server_url="$PYROSCOPE_SERVER"
-          environment="$PYROSCOPE_ENVIRONMENT"
-          key_secret="$PYROSCOPE_KEY"
-          EOF
-
-          echo "$decoded_toml" >> final_config.toml
-          cat config.toml >> final_config.toml
-          BASE64_CONFIG_OVERRIDE=$(cat final_config.toml | base64 -w 0)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-      - name: Parse base64 config
-        uses: ./.github/actions/setup-parse-base64-config
-        with:
-          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}
-      - name: Send details to Step Summary
-        shell: bash
-        run: |
-          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_IMAGE }}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY          
-      - name: Build Test Image
-        uses: ./.github/actions/build-test-image
-        with:
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          suites: benchmark chaos reorg load
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
-        env:
-          RR_CPU: 4000m
-          RR_MEM: 4Gi
-          DETACH_RUNNER: true
-          TEST_SUITE: automationv2_1
-          TEST_ARGS: -test.timeout 720h
-          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
-          INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
-          PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-        with:
-          test_command_to_run: cd integration-tests/load && go test -timeout 1h -v -run TestLogTrigger ./automationv2_1 -count=1
-          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
-          test_download_vendor_packages_command: make gomod
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          should_cleanup: false
-          go_mod_path: ./integration-tests/go.mod
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Collect Metrics
-        if: always()
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
-        with:
-          id: automation-load-test
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Automation Load Test
-        continue-on-error: true

From ddf0aeb322e215b31e3e50da31453a995c556883 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 14:38:44 +0200
Subject: [PATCH 080/141] Migrate on-demand-vrfv2-eth2-clients-test.yml

---
 .github/e2e-tests.yml                         |  1 -
 .../on-demand-vrfv2-eth2-clients-test.yml     | 94 +++++--------------
 2 files changed, 26 insertions(+), 69 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 8a3cc469d1f..41a8a8ae1f9 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -611,7 +611,6 @@ runner-test-matrix:
     runs_on: ubuntu22.04-8cores-32GB
     test_env_type: docker
     test_cmd: cd integration-tests/smoke && go test -v -test.run TestVRFv2Basic -test.parallel=1 -timeout 30m -count=1 -json
-    test_config_override_required: true
     test_secrets_required: true
     workflows:
       - On Demand VRFV2 Smoke Test (Ethereum clients)
diff --git a/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml b/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml
index b949db09119..166fe8e1eed 100644
--- a/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml
+++ b/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml
@@ -2,9 +2,9 @@ name: On Demand VRFV2 Smoke Test (Ethereum clients)
 on:
   workflow_dispatch:
     inputs:
-      base64Config:
-        description: base64-ed config
-        required: true
+      test_config_override_path:
+        description: Path to a test config file used to override the default test config
+        required: false
         type: string
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
@@ -12,68 +12,26 @@ on:
         type: string
 
 jobs:
-  vrfv2_smoke_test:
-    name: VRFV2 Smoke Test with custom EL client client
-    environment: integration
-    runs-on: ubuntu22.04-8cores-32GB
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      TEST_LOG_LEVEL: debug
-      REF_NAME: ${{ github.head_ref || github.ref_name }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Mask base64 config
-        run: |
-          BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-      - name: Parse base64 config
-        uses: ./.github/actions/setup-parse-base64-config
-        with:
-          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}
-      - name: Send details to Step Summary
-        shell: bash
-        run: |
-          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_IMAGE }}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### Execution client used" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.ETH2_EL_CLIENT }}\`" >>$GITHUB_STEP_SUMMARY
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
-        with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -run TestVRFv2Basic ./smoke/vrfv2_test.go 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
-          test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          artifacts_name: vrf-test-logs
-          artifacts_location: |
-            ./integration-tests/smoke/logs/
-            ./integration-tests/smoke/db_dumps/
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          should_cleanup: false
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ""
-        env:
-          E2E_TEST_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
-          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
-          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      test_ids: smoke/vrfv2_test.go:TestVRFv2Basic
+      test_config_override_path: ${{ inputs.test_config_override_path }}
+      chainlink_version: ${{ github.sha }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}

From bc2ff037e444941826b54e4efbf8fc76c326b9d5 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 14:41:27 +0200
Subject: [PATCH 081/141] Migrate on-demand-vrfv2-performance-test.yml

---
 .../on-demand-vrfv2-performance-test.yml      | 137 ++++++++----------
 1 file changed, 62 insertions(+), 75 deletions(-)

diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml
index b5a4c40d129..e2dc04011d4 100644
--- a/.github/workflows/on-demand-vrfv2-performance-test.yml
+++ b/.github/workflows/on-demand-vrfv2-performance-test.yml
@@ -2,10 +2,6 @@ name: On Demand VRFV2 Performance Test
 on:
   workflow_dispatch:
     inputs:
-      base64Config:
-        description: base64-ed config
-        required: true
-        type: string 
       performanceTestType:
         description: Performance Test Type of test to run
         type: choice
@@ -19,82 +15,73 @@ on:
         description: "Regex for tests to run"
         required: false
         default: "(TestVRFV2Performance)"
+      test_config_override_path:
+        description: Path to a test config file used to override the default test config
+        required: false
+        type: string               
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
         type: string 
 
 jobs:
-  vrfv2_performance_test:
-    name: VRFV2 Performance Test
-    environment: integration
-    runs-on: ubuntu22.04-8cores-32GB
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      LOKI_URL: ${{ secrets.LOKI_URL }}
-      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
-      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
-      TEST_TYPE: ${{ inputs.performanceTestType }}
-      TEST_LOG_LEVEL: debug
-      REF_NAME: ${{ github.head_ref || github.ref_name }}
-      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
-      SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }}
-      GRAFANA_URL: "http://localhost:8080/primary"
-      GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-      GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-      WASP_LOG_LEVEL: info
+  set-tests-to-run:
+    name: Set tests to run
+    runs-on: ubuntu-latest
+    outputs:
+      test_list: ${{ steps.set-tests.outputs.test_list }}
     steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
-        with:
-          id: on-demand-vrfv2-performance-test
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: ${{ inputs.network }} VRFV2 Performance Test
-        continue-on-error: true
-      - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Mask base64 config
-        run: |
-          BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV          
-      - name: Merge and export base64 config
-        uses: ./.github/actions/setup-merge-base64-config
-        with:
-          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}
-      - name: Send details to Step Summary
-        shell: bash
+      - name: Generate Test List JSON
+        id: set-tests
         run: |
-          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_IMAGE }}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
-          echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY          
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
-        with:
-          test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2
-          test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          artifacts_name: vrf-test-logs
-          artifacts_location: ./integration-tests/load/vrfv2/logs/
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          should_cleanup: false
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          TEST_CMD='cd integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2'
+          TEST_CONFIG_OVERRIDE_PATH=${{ inputs.test_config_override_path }}
+          TEST_TYPE=${{ inputs.performanceTestType }}
+
+          TEST_LIST=$(jq -n -c \
+            --arg test_cmd "$TEST_CMD" \
+            --arg test_config_override_path "$TEST_CONFIG_OVERRIDE_PATH" \
+            --arg TEST_TYPE "$TEST_TYPE" \
+            '{
+              "tests": [
+                {
+                  "id": "TestVRFv2Plus_Performance",
+                  "path": "integration-tests/load/vrfv2plus/vrfv2plus_test.go",
+                  "runs_on": "ubuntu22.04-8cores-32GB",
+                  "test_env_type": "docker",
+                  "test_cmd": $test_cmd,
+                  "test_config_override_path": $test_config_override_path,
+                  "test_env_vars": {
+                    "TEST_TYPE": $TEST_TYPE   
+                  }
+                }
+              ]
+            }')
+
+          echo "test_list=$TEST_LIST" >> $GITHUB_OUTPUT
+
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    needs: set-tests-to-run
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
+      chainlink_version: ${{ inputs.chainlink_version }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
+      SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }}      

From eeb5a039f83a67c866328a54dfd55ebf86e022e1 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 14:44:59 +0200
Subject: [PATCH 082/141] Remove unused actions

---
 .../action.yml                                | 125 ------------------
 .../setup-merge-base64-config/action.yml      |  70 ----------
 .../setup-parse-base64-config/action.yml      |  44 ------
 3 files changed, 239 deletions(-)
 delete mode 100644 .github/actions/setup-create-base64-upgrade-config/action.yml
 delete mode 100644 .github/actions/setup-merge-base64-config/action.yml
 delete mode 100644 .github/actions/setup-parse-base64-config/action.yml

diff --git a/.github/actions/setup-create-base64-upgrade-config/action.yml b/.github/actions/setup-create-base64-upgrade-config/action.yml
deleted file mode 100644
index c2d0bc19f35..00000000000
--- a/.github/actions/setup-create-base64-upgrade-config/action.yml
+++ /dev/null
@@ -1,125 +0,0 @@
-name: Create Base64 Upgrade Config
-description: A composite action that creates a base64-encoded config to be used by Chainlink version upgrade tests
-
-inputs:
-  selectedNetworks:
-    description: The networks to run tests against
-  chainlinkImage:
-    description: The chainlink image to upgrade from
-    default: "public.ecr.aws/chainlink/chainlink"
-  chainlinkVersion:
-    description: The git commit sha to use for the image tag
-  chainlinkPostgresVersion:
-    description: The postgres version to use with the chainlink node
-    default: "15.6"
-  upgradeImage:
-    description: The chainlink image to upgrade to
-    default: "public.ecr.aws/chainlink/chainlink"
-  upgradeVersion:
-    description: The git commit sha to use for the image tag
-  runId:
-    description: The run id
-  testLogCollect:
-    description: Whether to always collect logs, even for passing tests
-    default: "false"
-  lokiEndpoint:
-    description: Loki push endpoint
-  lokiTenantId:
-    description: Loki tenant id
-  lokiBasicAuth:
-    description: Loki basic auth
-  logstreamLogTargets:
-    description: Where to send logs (e.g. file, loki)
-  grafanaUrl:
-    description: Grafana URL
-  grafanaDashboardUrl:
-    description: Grafana dashboard URL
-  grafanaBearerToken:
-    description: Grafana bearer token
-
-runs:
-  using: composite
-  steps:
-    - name: Prepare Base64 TOML override
-      shell: bash
-      id: base64-config-override
-      env:
-        SELECTED_NETWORKS: ${{ inputs.selectedNetworks }}
-        CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }}
-        CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }}
-        CHAINLINK_POSTGRES_VERSION: ${{ inputs.chainlinkPostgresVersion }}
-        UPGRADE_IMAGE: ${{ inputs.upgradeImage }}
-        UPGRADE_VERSION: ${{ inputs.upgradeVersion }}
-        RUN_ID: ${{ inputs.runId }}
-        TEST_LOG_COLLECT: ${{ inputs.testLogCollect }}
-        LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }}
-        LOKI_TENANT_ID: ${{ inputs.lokiTenantId }}
-        LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }}
-        LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }}
-        GRAFANA_URL: ${{ inputs.grafanaUrl }}
-        GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }}
-        GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }}
-      run: |
-        function convert_to_toml_array() {
-          local IFS=','
-          local input_array=($1)
-          local toml_array_format="["
-
-          for element in "${input_array[@]}"; do
-              toml_array_format+="\"$element\","
-          done
-
-          toml_array_format="${toml_array_format%,}]"
-          echo "$toml_array_format"
-        }
-
-        selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS")
-
-        if [ -n "$TEST_LOG_COLLECT" ]; then
-          test_log_collect=true
-        else
-          test_log_collect=false
-        fi
-
-        log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
-
-        grafana_bearer_token=""
-        if [ -n "$GRAFANA_BEARER_TOKEN" ]; then
-          grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\""
-        fi
-
-        cat << EOF > config.toml
-        [Network]
-        selected_networks=$selected_networks
-
-        [ChainlinkImage]
-        image="$CHAINLINK_IMAGE"
-        version="$CHAINLINK_VERSION"
-        postgres_version="$CHAINLINK_POSTGRES_VERSION"
-
-        [ChainlinkUpgradeImage]
-        image="$UPGRADE_IMAGE"
-        version="$UPGRADE_VERSION"
-        postgres_version="$CHAINLINK_POSTGRES_VERSION"
-
-        [Logging]
-        test_log_collect=$test_log_collect
-        run_id="$RUN_ID"
-
-        [Logging.LogStream]
-        log_targets=$log_targets
-
-        [Logging.Loki]
-        tenant_id="$LOKI_TENANT_ID"
-        endpoint="$LOKI_ENDPOINT"
-        basic_auth_secret="$LOKI_BASIC_AUTH"
-
-        [Logging.Grafana]
-        base_url="$GRAFANA_URL"
-        dashboard_url="$GRAFANA_DASHBOARD_URL"
-        $grafana_bearer_token
-        EOF
-
-        BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0)
-        echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-        echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
diff --git a/.github/actions/setup-merge-base64-config/action.yml b/.github/actions/setup-merge-base64-config/action.yml
deleted file mode 100644
index 79dc8758315..00000000000
--- a/.github/actions/setup-merge-base64-config/action.yml
+++ /dev/null
@@ -1,70 +0,0 @@
-name: Merge Base64 Config
-description: A composite action that merges user-provided Base64-encoded config with repository's secrets
-
-inputs:
-  base64Config:
-    description: Base64-encoded config to decode
-
-runs:
-  using: composite
-  steps:
-    - name: Install dasel
-      shell: bash
-      run: |
-        if ! which dasel > /dev/null; then
-          curl -L -o dasel "https://github.com/TomWright/dasel/releases/download/v2.6.0/dasel_linux_amd64" && chmod +x dasel && sudo mv dasel /usr/local/bin/
-        else
-          echo "Dasel is already installed."
-        fi
-    - name: Add masks and export base64 config
-      shell: bash
-      run: |
-        BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
-        echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-        echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-
-        decoded_toml=$(echo $BASE64_CONFIG_OVERRIDE | base64 -d)
-        CHAINLINK_IMAGE=$(echo "$decoded_toml" | { dasel -r toml 'ChainlinkImage.image' 2>/dev/null || echo ''; })
-        echo ::add-mask::$CHAINLINK_IMAGE
-        CHAINLINK_VERSION=$(echo "$decoded_toml" | { dasel -r toml 'ChainlinkImage.version' 2>/dev/null || echo ''; })
-        NETWORKS=$(echo "$decoded_toml" | awk -F'=' '/^[[:space:]]*selected_networks[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2}' 2>/dev/null)
-
-        if [ -n "$CHAINLINK_IMAGE" ]; then
-          echo "CHAINLINK_IMAGE=$CHAINLINK_IMAGE" >> $GITHUB_ENV
-        else
-          echo "No Chainlink Image found in base64-ed config"
-        fi
-        if [ -n "$CHAINLINK_VERSION" ]; then
-          echo "CHAINLINK_VERSION=$CHAINLINK_VERSION" >> $GITHUB_ENV
-        else
-          echo "No Chainlink Version found in base64-ed config"
-        fi
-        if [ -n "$NETWORKS" ]; then
-          echo "NETWORKS=$NETWORKS" >> $GITHUB_ENV
-        fi
-
-        grafana_bearer_token=""
-        if [ -n "$GRAFANA_BEARER_TOKEN" ]; then
-          grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\""
-        fi
-
-        # use Loki config from GH secrets and merge it with base64 input
-        cat << EOF > config.toml
-        [Logging.Loki]
-        tenant_id="$LOKI_TENANT_ID"
-        endpoint="$LOKI_URL"
-        basic_auth_secret="$LOKI_BASIC_AUTH"
-        # legacy, you only need this to access the cloud version
-        # bearer_token_secret="bearer_token"
-        
-        [Logging.Grafana]
-        base_url="$GRAFANA_URL"
-        dashboard_url="$GRAFANA_DASHBOARD_URL"
-        $grafana_bearer_token
-        EOF
-
-        echo "$decoded_toml" >> final_config.toml
-        cat config.toml >> final_config.toml
-        BASE64_CONFIG_OVERRIDE=$(cat final_config.toml | base64 -w 0)
-        echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-        echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
\ No newline at end of file
diff --git a/.github/actions/setup-parse-base64-config/action.yml b/.github/actions/setup-parse-base64-config/action.yml
deleted file mode 100644
index 72e8982e6d0..00000000000
--- a/.github/actions/setup-parse-base64-config/action.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-name: Parse Base64 Config
-description: A composite action that extracts the chainlink image, version and network from a base64-encoded config
-
-inputs:
-  base64Config:
-    description: Base64-encoded config to decode
-
-runs:
-  using: composite
-  steps:
-    - name: Install dasel
-      shell: bash
-      run: |
-        if ! which dasel > /dev/null; then
-          curl -L -o dasel "https://github.com/TomWright/dasel/releases/download/v2.6.0/dasel_linux_amd64" && chmod +x dasel && sudo mv dasel /usr/local/bin/
-        else
-          echo "Dasel is already installed."
-        fi
-    - name: Add masks and export base64 config
-      shell: bash
-      run: |
-        decoded_toml=$(echo $BASE64_CONFIG_OVERRIDE | base64 -d)
-        CHAINLINK_IMAGE=$(echo "$decoded_toml" | { dasel -r toml 'ChainlinkImage.image' 2>/dev/null || echo ''; })
-        echo ::add-mask::$CHAINLINK_IMAGE
-        CHAINLINK_VERSION=$(echo "$decoded_toml" | { dasel -r toml 'ChainlinkImage.version' 2>/dev/null || echo ''; })
-        NETWORKS=$(echo "$decoded_toml" | awk -F'=' '/^[[:space:]]*selected_networks[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2}' 2>/dev/null)
-        ETH2_EL_CLIENT=$(echo "$decoded_toml" | awk -F'=' '/^[[:space:]]*execution_layer[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2}' 2>/dev/null)
-
-        if [ -n "$CHAINLINK_IMAGE" ]; then
-          echo "CHAINLINK_IMAGE=$CHAINLINK_IMAGE" >> $GITHUB_ENV
-        else
-          echo "No Chainlink Image found in base64-ed config"
-        fi
-        if [ -n "$CHAINLINK_VERSION" ]; then
-          echo "CHAINLINK_VERSION=$CHAINLINK_VERSION" >> $GITHUB_ENV
-        else
-          echo "No Chainlink Version found in base64-ed config. Exiting"
-        fi
-        if [ -n "$NETWORKS" ]; then
-          echo "NETWORKS=$NETWORKS" >> $GITHUB_ENV
-        fi
-        if [ -n "$ETH2_EL_CLIENT" ]; then
-          echo "ETH2_EL_CLIENT=$ETH2_EL_CLIENT" >> $GITHUB_ENV
-        fi
\ No newline at end of file

From dbfbdf7a1940b47d277e65ba4268190232e3a979 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 16:00:49 +0200
Subject: [PATCH 083/141] Always show console logs for ocr tests

---
 .github/e2e-tests.yml                                 | 2 ++
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 8 ++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 41a8a8ae1f9..4eb7180b15d 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -25,6 +25,7 @@ runner-test-matrix:
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
     test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 900h -count=1 -json
+    test_cmd_opts: 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false
     test_secrets_required: true    
     test_env_vars:
       TEST_SUITE: soak
@@ -34,6 +35,7 @@ runner-test-matrix:
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
     test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 900h -count=1 -json
+    test_cmd_opts: 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false
     test_secrets_required: true
     test_env_vars:
       TEST_SUITE: soak
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 72be0f42ff3..bdee8814dc7 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -240,7 +240,7 @@ jobs:
           path: core
       - name: Install citool
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@a49f2dff000fcf020ab9978b33e3726d7df5bf96 # v1.34.4
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@a5fa738dad7dfca70322d44877bdb1bc8eb8659d # v1.34.4
       - name: Run Check Tests Command
         run: |
           if ! citool check-tests ${{ github.workspace }}/integration-tests ${{ github.workspace }}/.github/e2e-tests.yml; then
@@ -287,7 +287,7 @@ jobs:
           check-latest: true
       - name: Install citool
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@a49f2dff000fcf020ab9978b33e3726d7df5bf96 # v1.34.4
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@a5fa738dad7dfca70322d44877bdb1bc8eb8659d # v1.34.4
       - name: Install jq
         run: sudo apt-get install jq
 
@@ -557,7 +557,7 @@ jobs:
           E2E_TEST_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }}
           E2E_TEST_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }}
         with:
-          test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
+          test_command_to_run: ${{ matrix.tests.test_cmd }} ${{ matrix.tests.test_cmd_opts || '2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs' }} 
           test_download_vendor_packages_command: cd $(dirname ${{ matrix.tests.path }}) && go mod download
           test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}
           test_config_override_path: ${{ env.TEST_CONFIG_OVERRIDE_PATH }}
@@ -765,7 +765,7 @@ jobs:
           E2E_TEST_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }}
           DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
         with:
-          test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
+          test_command_to_run: ${{ matrix.tests.test_cmd }} ${{ matrix.tests.test_cmd_opts || '2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs' }} 
           test_download_vendor_packages_command: make gomod
           test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}
           test_config_override_path: ${{ env.TEST_CONFIG_OVERRIDE_PATH }}

From d3643d2b6e4df2c18c0afadea7126fad8e89bfa2 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 16:50:57 +0200
Subject: [PATCH 084/141] Allow custom docker test artifacts

---
 .github/e2e-tests.yml                         | 10 ++++++++
 .../run-e2e-tests-reusable-workflow.yml       | 23 ++++++++++++++-----
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 4eb7180b15d..ee3f75db760 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -983,6 +983,16 @@ runner-test-matrix:
     test_env_vars:
       E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2      
 
+  - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPS
+    path: integration-tests/ccip-tests/load/ccip_test.go
+    test_env_type: docker
+    runs_on: ubuntu-latest
+    test_cmd: cd integration-tests/ccip-tests/load && go test -test.run TestLoadCCIPStableRPS -timeout 70m -count=1 -test.parallel=1 -json
+    test_env_vars:
+      E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
+    artifact_paths: 
+      - ./integration-tests/load/logs/payload_ccip.json
+
   - id: ccip-tests/chaos/ccip_test.go
     path: integration-tests/ccip-tests/chaos/ccip_test.go
     test_env_type: k8s-remote-runner
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index bdee8814dc7..6093ebd5e04 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -240,7 +240,7 @@ jobs:
           path: core
       - name: Install citool
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@a5fa738dad7dfca70322d44877bdb1bc8eb8659d # v1.34.4
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@7deba4b43ab4f70d8b6446286bd4ea25377b44d2 # v1.34.4
       - name: Run Check Tests Command
         run: |
           if ! citool check-tests ${{ github.workspace }}/integration-tests ${{ github.workspace }}/.github/e2e-tests.yml; then
@@ -287,7 +287,7 @@ jobs:
           check-latest: true
       - name: Install citool
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@a5fa738dad7dfca70322d44877bdb1bc8eb8659d # v1.34.4
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@7deba4b43ab4f70d8b6446286bd4ea25377b44d2 # v1.34.4
       - name: Install jq
         run: sudo apt-get install jq
 
@@ -534,6 +534,20 @@ jobs:
             --user "$(id -u):$(id -g)" \
             -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml
 
+      - name: Get test artifact paths
+        id: get_artifact_paths
+        run: |
+          DEFAULT_PATHS=(
+            "./integration-tests/smoke/logs/"
+            "./integration-tests/smoke/db_dumps/"
+            "/tmp/gotest.log"
+            "${{ matrix.tests.artifact_paths }}"
+          )
+          # Convert array to newline-separated string for output
+          printf -v PATHS_STR '%s\n' "${DEFAULT_PATHS[@]}"
+          echo "Test artifact paths: $PATHS_STR"  # Echoing the PATHS_STR to the logs
+          echo "paths=$PATHS_STR" >> $GITHUB_OUTPUT
+
       - name: Run tests
         id: run_tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@4f377a6b1cc07f0eca82745782736b4908a1da30 # v2.3.32
@@ -567,10 +581,7 @@ jobs:
           default_e2e_test_chainlink_upgrade_image: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_IMAGE }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: ${{ matrix.tests.id_sanitized }}-test-logs
-          artifacts_location: |
-            ./integration-tests/smoke/logs/
-            ./integration-tests/smoke/db_dumps/
-            /tmp/gotest.log
+          artifacts_location: ${{ steps.get_artifact_paths.outputs.paths }}
           publish_check_name: ${{ matrix.tests.id_sanitized }}
           token: ${{ secrets.GH_TOKEN }}
           cache_key_id: e2e-tests

From 4b41da7cc2345e6be06041d856bf94114cf51918 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 17:02:39 +0200
Subject: [PATCH 085/141] Migrate ccip-load-tests.yml

---
 .github/e2e-tests.yml                 |  15 ++
 .github/workflows/ccip-load-tests.yml | 303 +++-----------------------
 2 files changed, 45 insertions(+), 273 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index ee3f75db760..becf3c32de5 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -990,6 +990,21 @@ runner-test-matrix:
     test_cmd: cd integration-tests/ccip-tests/load && go test -test.run TestLoadCCIPStableRPS -timeout 70m -count=1 -test.parallel=1 -json
     test_env_vars:
       E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
+    workflows:
+      - E2E CCIP Load Tests  
+    artifact_paths: 
+      - ./integration-tests/load/logs/payload_ccip.json
+
+  - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPSAfterARMCurseAndUncurse
+    path: integration-tests/ccip-tests/load/ccip_test.go
+    test_env_type: docker
+    runs_on: ubuntu-latest
+    test_cmd: cd integration-tests/ccip-tests/load && go test -test.run $TestLoadCCIPStableRPSAfterARMCurseAndUncurse$ -timeout 70m -count=1 -test.parallel=1 -json
+    test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/load-with-arm-curse-uncurse.toml
+    test_env_vars:
+      E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
+    workflows:
+      - E2E CCIP Load Tests  
     artifact_paths: 
       - ./integration-tests/load/logs/payload_ccip.json
 
diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml
index fdf77067a53..e4078b51731 100644
--- a/.github/workflows/ccip-load-tests.yml
+++ b/.github/workflows/ccip-load-tests.yml
@@ -10,9 +10,10 @@ on:
       - '*'
   workflow_dispatch:
     inputs:
-      base64_test_input: # base64 encoded toml for test input
-        description: 'Base64 encoded toml test input'
+      test_config_override_path:
+        description: Path to a test config file used to override the default test config
         required: false
+        type: string
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
@@ -23,276 +24,32 @@ concurrency:
   group: load-ccip-tests-chainlink-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  # TODO: TT-1470 - Update image names as we solidify new realease strategy
-  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-  CHAINLINK_VERSION: ${{ github.sha}}
-  INPUT_CHAINLINK_TEST_VERSION: ${{ github.sha}}
-  ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
-  INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
-  AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws
-  MOD_CACHE_VERSION: 1
-
 jobs:
-  build-chainlink:
-    environment: integration
-    permissions:
-      id-token: write
-      contents: read
-    name: Build Chainlink Image
-    runs-on: ubuntu20.04-16cores-64GB
-    steps:
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Check if image exists
-        id: check-image
-        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
-        with:
-          repository: chainlink
-          tag: ${{ env.CHAINLINK_VERSION }}
-          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      - name: Build Image
-        if: steps.check-image.outputs.exists == 'false'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          cl_repo: smartcontractkit/chainlink
-          cl_ref: ${{ env.CHAINLINK_VERSION }}
-          push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ env.CHAINLINK_VERSION }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      - name: Collect Metrics
-        if: always()
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
-        with:
-          id: ccip-load-test-build-chainlink-image
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Build Chainlink Image
-        continue-on-error: true
-
-  build-test-image:
-    environment: integration
-    permissions:
-      id-token: write
-      contents: read
-    name: Build Test Image
-    runs-on: ubuntu20.04-16cores-64GB
-    steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
-        with:
-          id: ccip-load-test-build-test-image
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Build Test Image
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Build Test Image
-        uses: ./.github/actions/build-test-image
-        with:
-          tag: ${{ env.INPUT_CHAINLINK_TEST_VERSION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-
-  ccip-load-test:
-    environment: integration
-    needs: [ build-chainlink, build-test-image ]
-    if: ${{ always() && !contains(needs.*.result, 'failure') }}
-    permissions:
-      issues: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      CHAINLINK_ENV_USER: ${{ github.actor }}
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      test_workflow: E2E CCIP Load Tests
+      test_config_override_path: ${{ inputs.test_config_override_path }}
+      chainlink_version: ${{ github.sha }}
+      slack_notification_after_tests: always
+      slack_notification_after_tests_channel_id: '#ccip-testing'
+      slack_notification_after_tests_name: CCIP E2E Load Tests
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
       SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
-      SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
-      TEST_LOG_LEVEL: info
-      REF_NAME: ${{ github.head_ref || github.ref_name }}
-      BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }}
-    strategy:
-      fail-fast: false
-      matrix:
-        type:
-          - name: stable-load
-            run: ^TestLoadCCIPStableRPS$
-            os: ubuntu-latest
-          - name: load-with-arm-curse-uncurse
-            run: ^TestLoadCCIPStableRPSAfterARMCurseAndUncurse$
-            config_path: ./integration-tests/ccip-tests/testconfig/tomls/load-with-arm-curse-uncurse.toml
-            os: ubuntu-latest
-    runs-on: ${{ matrix.type.os }}
-    name: CCIP ${{ matrix.type.name }}
-    steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
-        with:
-          id: ccip-load-test-${{ matrix.type.name }}
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: CCIP ${{ matrix.type.name }}
-          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          ref: ${{ env.REF_NAME }}
-      - name: Sets env vars
-        id: set_override_config
-        shell: bash
-        run: |
-          # if the matrix.type.config_path is set, use it as the override config
-          if [ -n "${{ matrix.type.config_path }}" ]; then
-              BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ${{ matrix.type.config_path }})
-              echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
-              echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
-          fi
-          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH)
-            echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
-            echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
-          fi
-      - name: step summary
-        shell: bash
-        run: |
-          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.INPUT_CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-      - name: Prepare Base64 TOML override for CCIP secrets
-        uses: ./.github/actions/setup-create-base64-config-ccip
-        id: setup_create_base64_config_ccip
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkVersion: ${{ github.sha }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
-        env:
-          TEST_SUITE: load
-          TEST_ARGS: -test.timeout 900h
-          DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
-          RR_MEM: 8Gi
-          RR_CPU: 4
-          TEST_TRIGGERED_BY: ccip-load-test-ci-${{ matrix.type.name }}
-          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
-          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}          
-          E2E_TEST_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
-          E2E_TEST_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
-          E2E_TEST_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }}
-          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
-          E2E_TEST_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
-          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
-        with:
-          test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ${{ matrix.type.run }} ./load 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
-          test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
-          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-          triggered_by: ${{ env.TEST_TRIGGERED_BY }}
-          publish_check_name: ${{ matrix.type.name }}
-          artifacts_location: ./integration-tests/load/logs/payload_ccip.json
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          should_cleanup: "true"
-
-  # Reporting Jobs
-  start-slack-thread:
-    name: Start Slack Thread
-    if: ${{ failure() && needs.ccip-load-test.result != 'skipped' && needs.ccip-load-test.result != 'cancelled' }}
-    environment: integration
-    outputs:
-      thread_ts: ${{ steps.slack.outputs.thread_ts }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs: [ccip-load-test]
-    steps:
-      - name: Debug Result
-        run: echo ${{ join(needs.*.result, ',') }}
-      - name: Main Slack Notification
-        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
-        id: slack
-        with:
-          channel-id: "#ccip-testing"
-          payload: |
-            {
-              "attachments": [
-                {
-                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}",
-                  "blocks": [
-                    {
-                      "type": "header",
-                      "text": {
-                        "type": "plain_text",
-                        "text": "CCIP load tests results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}",
-                        "emoji": true
-                      }
-                    },
-                    {
-                      "type": "divider"
-                    },
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "mrkdwn",
-                        "text": "<${{ github.server_url }}/${{ github.repository }}/${{contains(github.ref_name, 'release') && 'releases/tag' || 'tree'}}/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>"
-                      }
-                    }
-                  ]
-                }
-              ]
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
-
-  post-test-results-to-slack:
-    name: Post Test Results
-    if: ${{ failure() && needs.start-slack-thread.result != 'skipped' && needs.start-slack-thread.result != 'cancelled' }}
-    environment: integration
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs: start-slack-thread
-    steps:
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
-      - name: Post Test Results
-        uses: ./.github/actions/notify-slack-jobs-result
-        with:
-          github_token: ${{ github.token }}
-          github_repository: ${{ github.repository }}
-          workflow_run_id: ${{ github.run_id }}
-          github_job_name_regex: ^CCIP (.*)$
-          message_title: CCIP Jobs
-          slack_channel_id: "#ccip-testing"
-          slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }}
-          slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }}
-
-  # End Reporting Jobs
\ No newline at end of file

From c51c5b5a351bf5e564c2314f11e1544b3021046a Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 17:03:46 +0200
Subject: [PATCH 086/141] revert this commits

---
 .github/workflows/ccip-load-tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml
index e4078b51731..f3bf3c022bd 100644
--- a/.github/workflows/ccip-load-tests.yml
+++ b/.github/workflows/ccip-load-tests.yml
@@ -33,7 +33,8 @@ jobs:
       test_config_override_path: ${{ inputs.test_config_override_path }}
       chainlink_version: ${{ github.sha }}
       slack_notification_after_tests: always
-      slack_notification_after_tests_channel_id: '#ccip-testing'
+      slack_notification_after_tests_channel_id: '#lf-slack-notification-tests'
+      # slack_notification_after_tests_channel_id: '#ccip-testing'
       slack_notification_after_tests_name: CCIP E2E Load Tests
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}

From 3909b4a41304231d0ed42a28c92fc5ed9c6c0354 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 17:05:58 +0200
Subject: [PATCH 087/141] fix ARTIFACT_PATHS

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 6093ebd5e04..274ef96d621 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -546,7 +546,7 @@ jobs:
           # Convert array to newline-separated string for output
           printf -v PATHS_STR '%s\n' "${DEFAULT_PATHS[@]}"
           echo "Test artifact paths: $PATHS_STR"  # Echoing the PATHS_STR to the logs
-          echo "paths=$PATHS_STR" >> $GITHUB_OUTPUT
+          echo "ARTIFACT_PATHS=$PATHS_STR" >> $GITHUB_ENV  # Store the multiline paths in an environment variable
 
       - name: Run tests
         id: run_tests
@@ -581,7 +581,7 @@ jobs:
           default_e2e_test_chainlink_upgrade_image: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_IMAGE }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: ${{ matrix.tests.id_sanitized }}-test-logs
-          artifacts_location: ${{ steps.get_artifact_paths.outputs.paths }}
+          artifacts_location: ${{ env.ARTIFACT_PATHS }}
           publish_check_name: ${{ matrix.tests.id_sanitized }}
           token: ${{ secrets.GH_TOKEN }}
           cache_key_id: e2e-tests

From 3ba73493fa2711d2b1d886950e241c5b4d442245 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 17:14:56 +0200
Subject: [PATCH 088/141] fix

---
 .../run-e2e-tests-reusable-workflow.yml       | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 274ef96d621..7202dadb953 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -534,20 +534,6 @@ jobs:
             --user "$(id -u):$(id -g)" \
             -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml
 
-      - name: Get test artifact paths
-        id: get_artifact_paths
-        run: |
-          DEFAULT_PATHS=(
-            "./integration-tests/smoke/logs/"
-            "./integration-tests/smoke/db_dumps/"
-            "/tmp/gotest.log"
-            "${{ matrix.tests.artifact_paths }}"
-          )
-          # Convert array to newline-separated string for output
-          printf -v PATHS_STR '%s\n' "${DEFAULT_PATHS[@]}"
-          echo "Test artifact paths: $PATHS_STR"  # Echoing the PATHS_STR to the logs
-          echo "ARTIFACT_PATHS=$PATHS_STR" >> $GITHUB_ENV  # Store the multiline paths in an environment variable
-
       - name: Run tests
         id: run_tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@4f377a6b1cc07f0eca82745782736b4908a1da30 # v2.3.32
@@ -581,7 +567,10 @@ jobs:
           default_e2e_test_chainlink_upgrade_image: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_IMAGE }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: ${{ matrix.tests.id_sanitized }}-test-logs
-          artifacts_location: ${{ env.ARTIFACT_PATHS }}
+          artifacts_location: |
+            ./integration-tests/smoke/logs/
+            ./integration-tests/smoke/db_dumps/
+            /tmp/gotest.log
           publish_check_name: ${{ matrix.tests.id_sanitized }}
           token: ${{ secrets.GH_TOKEN }}
           cache_key_id: e2e-tests

From 909ac9cbfe7a019ce48f6e6a5da68a9ae149472d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 17:19:29 +0200
Subject: [PATCH 089/141] Allow to upload custom test artifacts

---
 .github/e2e-tests.yml                         |  4 +--
 .../run-e2e-tests-reusable-workflow.yml       | 27 ++++++++++++++-----
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index becf3c32de5..4ecf29077ab 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -992,7 +992,7 @@ runner-test-matrix:
       E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
     workflows:
       - E2E CCIP Load Tests  
-    artifact_paths: 
+    test_artifacts: 
       - ./integration-tests/load/logs/payload_ccip.json
 
   - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPSAfterARMCurseAndUncurse
@@ -1005,7 +1005,7 @@ runner-test-matrix:
       E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
     workflows:
       - E2E CCIP Load Tests  
-    artifact_paths: 
+    test_artifacts: 
       - ./integration-tests/load/logs/payload_ccip.json
 
   - id: ccip-tests/chaos/ccip_test.go
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 7202dadb953..23e98773b95 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -240,7 +240,7 @@ jobs:
           path: core
       - name: Install citool
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@7deba4b43ab4f70d8b6446286bd4ea25377b44d2 # v1.34.4
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@809763a82dd14028cbc403df9daffeacd352ffda # v1.34.4
       - name: Run Check Tests Command
         run: |
           if ! citool check-tests ${{ github.workspace }}/integration-tests ${{ github.workspace }}/.github/e2e-tests.yml; then
@@ -287,7 +287,7 @@ jobs:
           check-latest: true
       - name: Install citool
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@7deba4b43ab4f70d8b6446286bd4ea25377b44d2 # v1.34.4
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@809763a82dd14028cbc403df9daffeacd352ffda # v1.34.4
       - name: Install jq
         run: sudo apt-get install jq
 
@@ -592,14 +592,14 @@ jobs:
         run: |
           ls -l ./integration-tests/smoke/traces
 
-      - name: Upload trace data as Github artifact
+      - name: Upload trace data as artifact
         if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_env_vars.ENABLE_OTEL_TRACES == 'true'
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: trace-data
           path: ./integration-tests/smoke/traces/trace-data.json          
           
-      - name: Upload test log as Github artifact
+      - name: Upload test log as artifact
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: inputs.test_log_upload_on_failure && failure()
         with:
@@ -608,7 +608,7 @@ jobs:
           retention-days: ${{ inputs.test_log_upload_retention_days }}
         continue-on-error: true
         
-      - name: Upload cl node coverage data as Github artifact
+      - name: Upload cl node coverage data as artifact
         if: inputs.upload_cl_node_coverage_artifact
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         timeout-minutes: 2
@@ -632,11 +632,17 @@ jobs:
           path: test_result.json        
           retention-days: 1
 
+      - name: Upload custom test artifacts
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: custom_test_artifacts_${{ needs.load-test-configurations.outputs.workflow_id }}_${{ matrix.tests.id_sanitized }}
+          path: ${{ matrix.tests.test_artifacts }}        
+          retention-days: 1      
+
       - name: Print failed test summary
         if: always()
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@70ccaef155381025e411cf7cd1fa5ef8f668ed75 # v2.3.25
 
-
   # Run K8s tests using old remote runner
 
   get-remote-runner-test-image:
@@ -781,7 +787,7 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
 
-      - name: Upload test log as Github artifact
+      - name: Upload test log as artifact
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: inputs.test_log_upload_on_failure && failure()
         with:
@@ -790,6 +796,13 @@ jobs:
           retention-days: ${{ inputs.test_log_upload_retention_days }}
         continue-on-error: true
 
+      - name: Upload custom test artifacts
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: custom_test_artifacts_${{ needs.load-test-configurations.outputs.workflow_id }}_${{ matrix.tests.id_sanitized }}
+          path: ${{ matrix.tests.test_artifacts }}        
+          retention-days: 1              
+
       # TODO: move to run-tests GHA
       - name: Print failed test summary
         if: always()

From be08c28b59f6e71f872adf6a70e1e465f8628222 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 17:22:16 +0200
Subject: [PATCH 090/141] Fux

---
 .github/workflows/ccip-load-tests.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml
index f3bf3c022bd..040adbaa4c8 100644
--- a/.github/workflows/ccip-load-tests.yml
+++ b/.github/workflows/ccip-load-tests.yml
@@ -18,6 +18,10 @@ on:
         description: 'Key to run tests with custom test secrets'
         required: false
         type: string
+      chainlink_version:
+        description: Chainlink image version to use. Commit sha if not provided
+        required: false
+        type: string        
 
 # Only run 1 of this workflow at a time per PR
 concurrency:
@@ -31,7 +35,7 @@ jobs:
     with:
       test_workflow: E2E CCIP Load Tests
       test_config_override_path: ${{ inputs.test_config_override_path }}
-      chainlink_version: ${{ github.sha }}
+      chainlink_version: ${{ inputs.chainlink_version || github.sha }} 
       slack_notification_after_tests: always
       slack_notification_after_tests_channel_id: '#lf-slack-notification-tests'
       # slack_notification_after_tests_channel_id: '#ccip-testing'

From 1ba81ee69674d164bb02068776ff3b762259aee7 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 17:38:39 +0200
Subject: [PATCH 091/141] Fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 23e98773b95..e3f8a457cf8 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -240,7 +240,7 @@ jobs:
           path: core
       - name: Install citool
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@809763a82dd14028cbc403df9daffeacd352ffda # v1.34.4
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@f03af143be18178b618e6e5ea43ef828f52d2ea6 # v1.34.4
       - name: Run Check Tests Command
         run: |
           if ! citool check-tests ${{ github.workspace }}/integration-tests ${{ github.workspace }}/.github/e2e-tests.yml; then
@@ -287,7 +287,7 @@ jobs:
           check-latest: true
       - name: Install citool
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@809763a82dd14028cbc403df9daffeacd352ffda # v1.34.4
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@f03af143be18178b618e6e5ea43ef828f52d2ea6 # v1.34.4
       - name: Install jq
         run: sudo apt-get install jq
 
@@ -633,10 +633,11 @@ jobs:
           retention-days: 1
 
       - name: Upload custom test artifacts
+        if: inputs.test_log_upload_on_failure && failure()
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: custom_test_artifacts_${{ needs.load-test-configurations.outputs.workflow_id }}_${{ matrix.tests.id_sanitized }}
-          path: ${{ matrix.tests.test_artifacts }}        
+          path: ${{ matrix.tests.test_artifacts_on_failure }}        
           retention-days: 1      
 
       - name: Print failed test summary
@@ -797,10 +798,11 @@ jobs:
         continue-on-error: true
 
       - name: Upload custom test artifacts
+        if: inputs.test_log_upload_on_failure && failure() 
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: custom_test_artifacts_${{ needs.load-test-configurations.outputs.workflow_id }}_${{ matrix.tests.id_sanitized }}
-          path: ${{ matrix.tests.test_artifacts }}        
+          path: ${{ matrix.tests.test_artifacts_on_failure }}        
           retention-days: 1              
 
       # TODO: move to run-tests GHA

From beba4c3eddf53aba6371d753039efa2be8a490ba Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 17:51:55 +0200
Subject: [PATCH 092/141] Clean up toml examples

---
 .../testconfig/automation/example.toml        | 30 -------------------
 .../testconfig/forwarder_ocr/example.toml     | 29 ------------------
 .../testconfig/forwarder_ocr2/example.toml    | 28 -----------------
 .../testconfig/functions/example.toml         | 29 ------------------
 .../testconfig/keeper/example.toml            | 29 ------------------
 .../testconfig/log_poller/example.toml        | 29 ------------------
 .../testconfig/node/example.toml              | 30 -------------------
 integration-tests/testconfig/ocr/example.toml | 29 ------------------
 .../testconfig/ocr2/example.toml              | 29 ------------------
 .../testconfig/vrfv2/example.toml             | 29 ------------------
 .../testconfig/vrfv2plus/example.toml         | 29 ------------------
 11 files changed, 320 deletions(-)

diff --git a/integration-tests/testconfig/automation/example.toml b/integration-tests/testconfig/automation/example.toml
index 3b48e89a548..3bbe78d693d 100644
--- a/integration-tests/testconfig/automation/example.toml
+++ b/integration-tests/testconfig/automation/example.toml
@@ -1,7 +1,6 @@
 # Example of full config with all fields
 # General part
 [ChainlinkImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.7.0"
 
 [Logging]
@@ -16,38 +15,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use polygon_mumbial
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"
@@ -163,5 +134,4 @@ use_prometheus=false
 
 # upgrade test specific override
 [TestAutomationNodeUpgrade.ChainlinkUpgradeImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.8.0"
\ No newline at end of file
diff --git a/integration-tests/testconfig/forwarder_ocr/example.toml b/integration-tests/testconfig/forwarder_ocr/example.toml
index 0b762299af1..0d15845baa4 100644
--- a/integration-tests/testconfig/forwarder_ocr/example.toml
+++ b/integration-tests/testconfig/forwarder_ocr/example.toml
@@ -1,7 +1,6 @@
 # Example of full config with all fields
 # General part
 [ChainlinkImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.7.0"
 
 [Logging]
@@ -16,38 +15,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use polygon_mumbial
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"
diff --git a/integration-tests/testconfig/forwarder_ocr2/example.toml b/integration-tests/testconfig/forwarder_ocr2/example.toml
index b3bc45d270c..2109937487b 100644
--- a/integration-tests/testconfig/forwarder_ocr2/example.toml
+++ b/integration-tests/testconfig/forwarder_ocr2/example.toml
@@ -16,38 +16,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use polygon_mumbial
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"
diff --git a/integration-tests/testconfig/functions/example.toml b/integration-tests/testconfig/functions/example.toml
index 7502a6fc440..74d931632a8 100644
--- a/integration-tests/testconfig/functions/example.toml
+++ b/integration-tests/testconfig/functions/example.toml
@@ -1,7 +1,6 @@
 # Example of full config with all fields
 # General part
 [ChainlinkImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.7.0"
 
 [Logging]
@@ -16,38 +15,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use simulated network
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"
diff --git a/integration-tests/testconfig/keeper/example.toml b/integration-tests/testconfig/keeper/example.toml
index 5abb5835629..4efbf974827 100644
--- a/integration-tests/testconfig/keeper/example.toml
+++ b/integration-tests/testconfig/keeper/example.toml
@@ -1,7 +1,6 @@
 # Example of full config with all fields
 # General part
 [ChainlinkImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.7.0"
 
 [Logging]
@@ -16,38 +15,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use polygon_mumbial
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"
diff --git a/integration-tests/testconfig/log_poller/example.toml b/integration-tests/testconfig/log_poller/example.toml
index c28d36ae12f..78f3b5482d9 100644
--- a/integration-tests/testconfig/log_poller/example.toml
+++ b/integration-tests/testconfig/log_poller/example.toml
@@ -1,7 +1,6 @@
 # Example of full config with all fields
 # General part
 [ChainlinkImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.7.0"
 
 [Logging]
@@ -16,38 +15,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use polygon_mumbial
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"
diff --git a/integration-tests/testconfig/node/example.toml b/integration-tests/testconfig/node/example.toml
index 510379b4f05..bc5628e46b3 100644
--- a/integration-tests/testconfig/node/example.toml
+++ b/integration-tests/testconfig/node/example.toml
@@ -1,7 +1,6 @@
 # Example of full config with all fields
 # General part
 [ChainlinkImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.7.0"
 
 [Logging]
@@ -16,38 +15,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use polygon_mumbial
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"
@@ -77,5 +48,4 @@ chainlink_node_funding = 0.5
 
 # Test-specific part
 [ChainlinkUpgradeImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.8.0"
\ No newline at end of file
diff --git a/integration-tests/testconfig/ocr/example.toml b/integration-tests/testconfig/ocr/example.toml
index 92262241dff..b1e497f90a0 100644
--- a/integration-tests/testconfig/ocr/example.toml
+++ b/integration-tests/testconfig/ocr/example.toml
@@ -1,7 +1,6 @@
 # Example of full config with all fields
 # General part
 [ChainlinkImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.7.0"
 
 [Logging]
@@ -16,38 +15,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use polygon_mumbial
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"
diff --git a/integration-tests/testconfig/ocr2/example.toml b/integration-tests/testconfig/ocr2/example.toml
index 36e3105f219..238c7f53a63 100644
--- a/integration-tests/testconfig/ocr2/example.toml
+++ b/integration-tests/testconfig/ocr2/example.toml
@@ -1,7 +1,6 @@
 # Example of full config with all fields
 # General part
 [ChainlinkImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.7.0"
 
 [Logging]
@@ -16,38 +15,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use polygon_mumbial
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"
diff --git a/integration-tests/testconfig/vrfv2/example.toml b/integration-tests/testconfig/vrfv2/example.toml
index 9417a422cf0..13af6dee620 100644
--- a/integration-tests/testconfig/vrfv2/example.toml
+++ b/integration-tests/testconfig/vrfv2/example.toml
@@ -1,7 +1,6 @@
 # Example of full config with all fields
 # General part
 [ChainlinkImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.7.0"
 
 [Logging]
@@ -16,38 +15,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use polygon_mumbial
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"
diff --git a/integration-tests/testconfig/vrfv2plus/example.toml b/integration-tests/testconfig/vrfv2plus/example.toml
index d6e7a3f28d5..160e9ba03a9 100644
--- a/integration-tests/testconfig/vrfv2plus/example.toml
+++ b/integration-tests/testconfig/vrfv2plus/example.toml
@@ -1,7 +1,6 @@
 # Example of full config with all fields
 # General part
 [ChainlinkImage]
-image="public.ecr.aws/chainlink/chainlink"
 version="2.7.0"
 
 [Logging]
@@ -16,38 +15,10 @@ log_producer_timeout="10s"
 # number of retries before log producer gives up and stops listening to logs
 log_producer_retry_limit=10
 
-[Logging.Loki]
-tenant_id="tenant_id"
-# full URL of Loki ingest endpoint
-endpoint="https://loki.url/api/v3/push"
-# currently only needed when using public instance
-basic_auth_secret="loki-basic-auth"
-# only needed for cloud grafana
-bearer_token_secret="bearer_token"
-
-# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
-[Logging.Grafana]
-# grafana url (trailing "/" will be stripped)
-base_url="http://grafana.url"
-# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
-dashboard_url="/d/your-dashboard"
-# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model 
-dashboard_uid="dashboard-uid-to-annotate"
-bearer_token_secret="my-awesome-token"
-
 # if you want to use polygon_mumbial
 [Network]
 selected_networks=["polygon_mumbai"]
 
-[Network.RpcHttpUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.RpcWsUrls]
-polygon_mumbai = ["https://my-rpc-endpoint.io"]
-
-[Network.WalletKeys]
-polygon_mumbai = ["change-me-to-your-PK"]
-
 [PrivateEthereumNetwork]
 # pos or pow
 consensus_type="pos"

From ba70c4918fe89a00d417691ef8b5263949dd474d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 18:07:38 +0200
Subject: [PATCH 093/141] Remove unused live-testnet-tests.yml workflow

Confirmed with Adam
---
 .../action.yml                                |  144 ---
 .github/workflows/live-testnet-tests.yml      | 1119 -----------------
 2 files changed, 1263 deletions(-)
 delete mode 100644 .github/actions/setup-create-base64-config-live-testnets/action.yml
 delete mode 100644 .github/workflows/live-testnet-tests.yml

diff --git a/.github/actions/setup-create-base64-config-live-testnets/action.yml b/.github/actions/setup-create-base64-config-live-testnets/action.yml
deleted file mode 100644
index 64fc134b46e..00000000000
--- a/.github/actions/setup-create-base64-config-live-testnets/action.yml
+++ /dev/null
@@ -1,144 +0,0 @@
-name: Create Base64 Config
-description: A composite action that creates a base64-encoded config to be used by integration tests
-
-inputs:
-  runId:
-    description: The run id
-  testLogCollect:
-    description: Whether to always collect logs, even for passing tests
-    default: "false"
-  chainlinkImage:
-    description: The chainlink image to use
-    default: "public.ecr.aws/chainlink/chainlink"
-  chainlinkVersion:
-    description: The git commit sha to use for the image tag
-  chainlinkPostgresVersion:
-    description: The postgres version to use with the chainlink node
-    default: "15.6"
-  pyroscopeServer:
-    description: URL of Pyroscope server
-  pyroscopeEnvironment:
-    description: Name of Pyroscope environment
-  pyroscopeKey:
-    description: Pyroscope server key
-  lokiEndpoint:
-    description: Loki push endpoint
-  lokiTenantId:
-    description: Loki tenant id
-  lokiBasicAuth:
-    description: Loki basic auth
-  logstreamLogTargets:
-    description: Where to send logs (e.g. file, loki)
-  grafanaUrl:
-    description: Grafana URL
-  grafanaDashboardUrl:
-    description: Grafana dashboard URL
-  grafanaBearerToken:
-    description: Grafana bearer token
-  network:
-    description: Network to run tests on
-  httpEndpoints:
-    description: HTTP endpoints to use for network
-  wsEndpoints:
-    description: WS endpoints to use for network
-  fundingKeys:
-    description: Funding keys to use for network
-
-runs:
-  using: composite
-  steps:
-    - name: Prepare Base64 TOML override
-      shell: bash
-      id: base64-config-override
-      env:
-        RUN_ID: ${{ inputs.runId }}
-        PYROSCOPE_SERVER: ${{ inputs.pyroscopeServer }}
-        PYROSCOPE_ENVIRONMENT: ${{ inputs.pyroscopeEnvironment }}
-        PYROSCOPE_KEY: ${{ inputs.pyroscopeKey }}
-        CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }}
-        CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }}
-        CHAINLINK_POSTGRES_VERSION: ${{ inputs.chainlinkPostgresVersion }}
-        LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }}
-        LOKI_TENANT_ID: ${{ inputs.lokiTenantId }}
-        LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }}
-        LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }}
-        GRAFANA_URL: ${{ inputs.grafanaUrl }}
-        GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }}
-        GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }}
-        NETWORK: ${{ inputs.network }}
-        HTTP_ENDPOINTS: ${{ inputs.httpEndpoints }}
-        WS_ENDPOINTS: ${{ inputs.wsEndpoints }}
-        FUNDING_KEYS: ${{ inputs.fundingKeys }}
-      run: |
-        convert_to_toml_array() {
-            local IFS=','
-            local input_array=($1)
-            local toml_array_format="["
-
-            for element in "${input_array[@]}"; do
-                toml_array_format+="\"$element\","
-            done
-
-            toml_array_format="${toml_array_format%,}]"
-            echo "$toml_array_format"
-        }
-
-        if [ -n "$PYROSCOPE_SERVER" ]; then
-          pyroscope_enabled=true
-        else
-          pyroscope_enabled=false
-        fi
-
-        grafana_bearer_token=""
-        if [ -n "$GRAFANA_BEARER_TOKEN" ]; then
-          grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\""
-        fi
-
-        cat << EOF > config.toml
-        [Common]
-        chainlink_node_funding=0.5
-
-        [ChainlinkImage]
-        image="$CHAINLINK_IMAGE"
-        version="$CHAINLINK_VERSION"
-        postgres_version="$CHAINLINK_POSTGRES_VERSION"
-
-        [Pyroscope]
-        enabled=$pyroscope_enabled
-        server_url="$PYROSCOPE_SERVER"
-        environment="$PYROSCOPE_ENVIRONMENT"
-        key_secret="$PYROSCOPE_KEY"
-
-        [Logging]
-        run_id="$RUN_ID"
-
-        [Logging.LogStream]
-        log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
-
-        [Logging.Loki]
-        tenant_id="$LOKI_TENANT_ID"
-        endpoint="$LOKI_URL"
-        basic_auth_secret="$LOKI_BASIC_AUTH"
-
-        [Logging.Grafana]
-        base_url="$GRAFANA_URL"
-        dashboard_url="$GRAFANA_DASHBOARD_URL"
-        $grafana_bearer_token
-
-        [Network]
-        selected_networks=["$NETWORK"]
-
-        [Network.RpcHttpUrls]
-        "$NETWORK" = $(convert_to_toml_array "$HTTP_ENDPOINTS")
-
-        [Network.RpcWsUrls]
-        "$NETWORK" = $(convert_to_toml_array "$WS_ENDPOINTS")
-
-        [Network.WalletKeys]
-        "$NETWORK" = $(convert_to_toml_array "$FUNDING_KEYS")
-        EOF
-
-        BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0)
-        echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-        echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-        touch .root_dir
diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml
deleted file mode 100644
index bcf4dfea199..00000000000
--- a/.github/workflows/live-testnet-tests.yml
+++ /dev/null
@@ -1,1119 +0,0 @@
-# ***
-# This workflow is a monstrosity of copy-paste, and that's to increase legibility in reporting and running, so the code be damned.
-# I suspect this can be cleaned up significantly with some clever trickery of the GitHub actions matrices, but I am not that clever.
-# We want each chain to run in parallel, but each test within the chain needs to be able to run sequentially
-# (we're trying to eliminate this as a requirement, should make it a lot easier).
-# Each chain can have a variety of tests to run.
-# We also want reporting to be clear in the start-slack-thread and post-test-results-to-slack jobs.
-# Funding address: 0xC1107e57082945E28d3202A81B1520DEA3AE6AEC
-# ***
-
-name: Live Testnet Tests
-on:
-  # Disable refular runs for now until we can fix some test client flakiness and improve stability
-  # schedule:
-  #   - cron: "0 5 * * *" # Run every night at midnight EST
-  # push:
-  #   tags:
-  #     - "*"
-  workflow_dispatch:
-    inputs:
-      slack_user_id:
-        description: "The Slack member ID to notify"
-        required: true
-        type: string
-      network:
-        description: "The network to run tests on"
-        required: true
-        type: choice
-        options:
-          - "All"
-          - "Sepolia"
-          - "Optimism Sepolia"
-          - "Arbitrum Sepolia"
-          - "Base Sepolia"
-          - "Polygon Mumbai"
-          - "Avalanche Fuji"
-          - "Fantom Testnet"
-          - "Celo Alfajores"
-          - "Linea Goerli"
-          - "BSC Testnet"
-
-env:
-  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-  INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
-  MOD_CACHE_VERSION: 2
-  CHAINLINK_NODE_FUNDING: .5
-  PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-  LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
-  LOKI_URL: ${{ secrets.LOKI_URL }}
-  LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
-  LOGSTREAM_LOG_TARGETS: loki
-  GRAFANA_URL: ${{ vars.GRAFANA_URL }}
-  RUN_ID: ${{ github.run_id }}
-
-  CHAINLINK_COMMIT_SHA: ${{ github.sha }}
-  CHAINLINK_ENV_USER: ${{ github.actor }}
-  TEST_LOG_LEVEL: debug
-
-jobs:
-
-  # Build Test Dependencies
-
-  build-chainlink:
-    environment: integration
-    permissions:
-      id-token: write
-      contents: read
-    name: Build Chainlink Image
-    runs-on: ubuntu-latest
-    steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
-        with:
-          id: live-testnet-build-chainlink
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Build Chainlink Image
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
-      - name: Build Chainlink Image
-        uses: ./.github/actions/build-chainlink-image
-        with:
-          tag_suffix: ""
-          dockerfile: core/chainlink.Dockerfile
-          git_commit_sha: ${{ github.sha }}
-          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-
-  build-tests:
-    environment: integration
-    permissions:
-      id-token: write
-      contents: read
-    name: Build Tests Binary
-    runs-on: ubuntu-latest
-    steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
-        with:
-          id: live-testnet-build-test-image
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Build Tests Binary
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
-      - name: Build Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          go_tags: embed
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          binary_name: tests
-
-  # End Build Test Dependencies
-
-  # Reporting Jobs
-
-  start-slack-thread:
-    name: Start Slack Thread
-    if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
-    environment: integration
-    outputs:
-      thread_ts: ${{ steps.slack.outputs.thread_ts }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs: [sepolia-smoke-tests, optimism-sepolia-smoke-tests, arbitrum-sepolia-smoke-tests, base-sepolia-smoke-tests, polygon-mumbai-smoke-tests, avalanche-fuji-smoke-tests, fantom-testnet-smoke-tests, celo-alfajores-smoke-tests, linea-goerli-smoke-tests, bsc-testnet-smoke-tests]
-    steps:
-      - name: Debug Result
-        run: echo ${{ join(needs.*.result, ',') }}
-      - name: Main Slack Notification
-        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
-        id: slack
-        with:
-          channel-id: ${{ secrets.QA_SLACK_CHANNEL }}
-          payload: |
-            {
-              "attachments": [
-                {
-                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}",
-                  "blocks": [
-                    {
-                      "type": "header",
-                      "text": {
-                        "type": "plain_text",
-                        "text": "Live Smoke Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}",
-                        "emoji": true
-                      }
-                    },
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "mrkdwn",
-                        "text": "Notifying <@${{ inputs.slack_user_id }}>"
-                      }
-                    },
-                    {
-                      "type": "divider"
-                    },
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "mrkdwn",
-                        "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>\nThe funding address for all tests and networks is `0xC1107e57082945E28d3202A81B1520DEA3AE6AEC`"
-                      }
-                    }
-                  ]
-                }
-              ]
-            }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
-
-  post-test-results-to-slack:
-    name: Post Test Results for ${{ matrix.network }}
-    if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
-    environment: integration
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs: start-slack-thread
-    strategy:
-      fail-fast: false
-      matrix:
-        network: [Sepolia, Optimism Sepolia, Arbitrum Sepolia, Base Sepolia, Polygon Mumbai, Avalanche Fuji, Fantom Testnet, Celo Alfajores, Linea Goerli, BSC Testnet]
-    steps:
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
-      - name: Post Test Results
-        uses: ./.github/actions/notify-slack-jobs-result
-        with:
-          github_token: ${{ github.token }}
-          github_repository: ${{ github.repository }}
-          workflow_run_id: ${{ github.run_id }}
-          github_job_name_regex: ^${{ matrix.network }} (.*?) Tests$
-          message_title: ${{ matrix.network }}
-          slack_channel_id: ${{ secrets.QA_SLACK_CHANNEL }}
-          slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }}
-          slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }}
-
-  # End Reporting Jobs
-
-  sepolia-smoke-tests:
-    environment: integration
-    if: ${{ (github.event.inputs.network == 'All' || github.event.inputs.network == 'Sepolia') }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-          - product: Automation Conditional
-            test: TestAutomationBasic/registry_2_1_conditional
-          - product: Automation Log Trigger
-            test: TestAutomationBasic/registry_2_1_logtrigger
-    name: Sepolia ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-sepolia
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "sepolia"
-          httpEndpoints: ${{ secrets.QA_SEPOLIA_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_SEPOLIA_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directories: "./"
-
-  bsc-testnet-smoke-tests:
-    environment: integration
-    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'BSC Testnet' }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-          - product: Automation Conditional
-            test: TestAutomationBasic/registry_2_1_conditional
-          - product: Automation Log Trigger
-            test: TestAutomationBasic/registry_2_1_logtrigger
-    name: BSC Testnet ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-bsc-testnet
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "bsc_testnet"
-          httpEndpoints: ${{ secrets.QA_BSC_TESTNET_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_BSC_TESTNET_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directory: "./"
-
-  optimism-sepolia-smoke-tests:
-    environment: integration
-    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Optimism Sepolia' }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-          - product: Automation Conditional
-            test: TestAutomationBasic/registry_2_1_conditional
-          - product: Automation Log Trigger
-            test: TestAutomationBasic/registry_2_1_logtrigger
-    name: Optimism Sepolia ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-optimism-sepolia
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "optimism_sepolia"
-          httpEndpoints: ${{ secrets.QA_OPTIMISM_SEPOLIA_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_OPTIMISM_SEPOLIA_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directory: "./"
-
-  arbitrum-sepolia-smoke-tests:
-    environment: integration
-    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Arbitrum Sepolia' }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-          - product: Automation Conditional
-            test: TestAutomationBasic/registry_2_1_conditional
-          - product: Automation Log Trigger
-            test: TestAutomationBasic/registry_2_1_logtrigger
-    name: Arbitrum Sepolia ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-arbitrum-sepolia
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "arbitrum_sepolia"
-          httpEndpoints: ${{ secrets.QA_ARBITRUM_SEPOLIA_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_ARBITRUM_SEPOLIA_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directory: "./"
-
-  base-sepolia-smoke-tests:
-    environment: integration
-    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Base Sepolia' }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-    name: Base Sepolia ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-base-sepolia
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "base_sepolia"
-          httpEndpoints: ${{ secrets.QA_BASE_SEPOLIA_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_BASE_SEPOLIA_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directory: "./"
-
-  polygon-mumbai-smoke-tests:
-    environment: integration
-    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Polygon Mumbai' }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-          - product: Automation Conditional
-            test: TestAutomationBasic/registry_2_1_conditional
-          - product: Automation Log Trigger
-            test: TestAutomationBasic/registry_2_1_logtrigger
-    name: Polygon Mumbai ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-polygon-mumbai
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "polygon_mumbai"
-          httpEndpoints: ${{ secrets.QA_POLYGON_MUMBAI_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_POLYGON_MUMBAI_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directory: "./"
-
-  avalanche-fuji-smoke-tests:
-    environment: integration
-    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Avalanche Fuji' }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-          - product: Automation Conditional
-            test: TestAutomationBasic/registry_2_1_conditional
-          - product: Automation Log Trigger
-            test: TestAutomationBasic/registry_2_1_logtrigger
-    name: Avalanche Fuji ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-avalanche-fuji
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "avalanche_fuji"
-          httpEndpoints: ${{ secrets.QA_AVALANCHE_FUJI_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_AVALANCHE_FUJI_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directory: "./"
-
-  fantom-testnet-smoke-tests:
-    environment: integration
-    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Fantom Testnet' }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-          - product: Automation Conditional
-            test: TestAutomationBasic/registry_2_1_conditional
-          - product: Automation Log Trigger
-            test: TestAutomationBasic/registry_2_1_logtrigger
-    name: Fantom Testnet ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-fantom-testnet
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "fantom_testnet"
-          httpEndpoints: ${{ secrets.QA_FANTOM_TESTNET_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_FANTOM_TESTNET_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directory: "./"
-
-  celo-alfajores-smoke-tests:
-    environment: integration
-    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Celo Alfajores' }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-    name: Celo Alfajores ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-celo-alfajores
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "celo_alfajores"
-          httpEndpoints: ${{ secrets.QA_CELO_ALFAJORES_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_CELO_ALFAJORES_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directory: "./"
-
-  scroll-sepolia-smoke-tests:
-    if: false
-    environment: integration
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-    name: Scroll Sepolia ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-scroll-sepolia
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "scroll_sepolia"
-          httpEndpoints: ${{ secrets.QA_SCROLL_SEPOLIA_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_SCROLL_SEPOLIA_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directory: "./"
-
-  linea-goerli-smoke-tests:
-    environment: integration
-    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Linea Goerli' }}
-    permissions:
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    needs: [build-chainlink, build-tests]
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations
-          - product: OCR
-            test: TestOCRBasic
-    name: Linea Goerli ${{ matrix.product }} Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          fetch-depth: 0
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
-        with:
-          # aws inputs
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          # other inputs
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config-live-testnets
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-smoke-${{ matrix.product }}-linea-goerli
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          network: "linea_goerli"
-          httpEndpoints: ${{ secrets.QA_LINEA_GOERLI_HTTP_URLS }}
-          wsEndpoints: ${{ secrets.QA_LINEA_GOERLI_URLS }}
-          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
-      - name: Download Tests Binary
-        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-        with:
-          name: tests
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_command_to_run: ./tests -test.timeout 30m -test.count=1 -test.parallel=1 -test.run ${{ matrix.test }}
-          binary_name: tests
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
-          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
-          artifacts_location: ./logs
-          token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      - name: Print failed test summary
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-        with:
-          test_directory: "./"

From 2564ca5572f67dec1c91fd9612eab865d8d2fe37 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 18:08:34 +0200
Subject: [PATCH 094/141] Remove old action

---
 .../action.yml                                | 151 ------------------
 1 file changed, 151 deletions(-)
 delete mode 100644 .github/actions/setup-create-base64-config-ccip/action.yml

diff --git a/.github/actions/setup-create-base64-config-ccip/action.yml b/.github/actions/setup-create-base64-config-ccip/action.yml
deleted file mode 100644
index cb20c886e38..00000000000
--- a/.github/actions/setup-create-base64-config-ccip/action.yml
+++ /dev/null
@@ -1,151 +0,0 @@
-name: Create Base64 Config for CCIP Tests
-description: A composite action that creates a base64-encoded config to be used by ccip integration tests
-
-inputs:
-  runId:
-    description: The run id
-  existingNamespace:
-    description: If test needs to run against already deployed namespace
-  testLogCollect:
-    description: Whether to always collect logs, even for passing tests
-    default: "false"
-  selectedNetworks:
-    description: The networks to run tests against
-  chainlinkVersion:
-    description: The git commit sha to use for the image tag
-  upgradeVersion:
-    description: The git commit sha to use for the image tag    
-  logstreamLogTargets:
-    description: Where to send logs (e.g. file, loki)
-  customEvmNodes:
-    description: Custom EVM nodes to use  in key=value format, where key is chain id and value is docker image to use. If they are provided the number of networksSelected must be equal to the number of customEvmNodes
-  evmNodeLogLevel:
-    description: Log level for the custom EVM nodes
-    default: "info"
-outputs:
-  base64_config:
-    description: The base64-encoded config
-    value: ${{ steps.base64_config_override.outputs.base64_config }}
-
-runs:
-  using: composite
-  steps:
-    - name: Prepare Base64 TOML override
-      shell: bash
-      id: base64_config_override
-      env:
-        RUN_ID: ${{ inputs.runId }}
-        SELECTED_NETWORKS: ${{ inputs.selectedNetworks }}
-        EXISTING_NAMESPACE: ${{ inputs.existingNamespace }}
-        TEST_LOG_COLLECT: ${{ inputs.testLogCollect }}
-        CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }}
-        UPGRADE_VERSION: ${{ inputs.upgradeVersion }}
-        LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }}
-        CUSTOM_EVM_NODES: ${{ inputs.customEvmNodes }}
-        EVM_NODE_LOG_LEVEL: ${{ inputs.evmNodeLogLevel }}
-      run: |
-        function convert_to_toml_array() {
-          local IFS=','
-          local input_array=($1)
-          local toml_array_format="["
-
-          for element in "${input_array[@]}"; do
-              toml_array_format+="\"$element\","
-          done
-
-          toml_array_format="${toml_array_format%,}]"
-          echo "$toml_array_format"
-        }
-
-        selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS")
-        log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")      
-
-        if [ -n "$TEST_LOG_COLLECT" ]; then
-          test_log_collect=true
-        else
-          test_log_collect=false
-        fi
-
-        # make sure the number of networks and nodes match
-        IFS=',' read -r -a networks_array <<< "$SELECTED_NETWORKS"
-        IFS=',' read -r -a nodes_array <<< "$CUSTOM_EVM_NODES"
-
-        networks_count=${#networks_array[@]}
-        nodes_count=${#nodes_array[@]}
-
-        # Initialize or clear CONFIG_TOML environment variable
-        custom_nodes_toml=""
-
-        # Check if the number of CUSTOM_EVM_NODES is zero
-        if [ $nodes_count -eq 0 ]; then
-          echo "The number of CUSTOM_EVM_NODES is zero, won't output any custom private Ethereum network configurations."
-        else
-          if [ $networks_count -ne $nodes_count ]; then
-            echo "The number of elements in SELECTED_NETWORKS (${networks_count}) and CUSTOM_EVM_NODES does not match (${nodes_count})."
-            exit 1
-          else
-            for i in "${!networks_array[@]}"; do
-              IFS='=' read -r chain_id docker_image <<< "${nodes_array[i]}"
-              custom_nodes_toml+="
-        [CCIP.Env.PrivateEthereumNetworks.${networks_array[i]}]
-        ethereum_version=\"\"
-        execution_layer=\"\"
-
-        [CCIP.Env.PrivateEthereumNetworks.${networks_array[i]}.EthereumChainConfig]
-        seconds_per_slot=3
-        slots_per_epoch=2
-        genesis_delay=15
-        validator_count=4
-        chain_id=${chain_id}
-        addresses_to_fund=[\"0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266\", \"0x70997970C51812dc3A010C7d01b50e0d17dc79C8\"]
-        node_log_level=\"${EVM_NODES_LOG_LEVEL}\"
-
-        [CCIP.Env.PrivateEthereumNetworks.${networks_array[i]}.EthereumChainConfig.HardForkEpochs]
-        Deneb=500
-
-        [CCIP.Env.PrivateEthereumNetworks.${networks_array[i]}.CustomDockerImages]
-        execution_layer=\"${docker_image}\"
-        "
-            done
-          fi
-        fi
-
-        cat << EOF > config.toml
-        [CCIP]
-        [CCIP.Env]
-        EnvToConnect="$EXISTING_NAMESPACE"
-        [CCIP.Env.Network]
-        selected_networks = $selected_networks
-        [CCIP.Env.NewCLCluster]
-        [CCIP.Env.NewCLCluster.Common]
-        [CCIP.Env.NewCLCluster.Common.ChainlinkImage]
-        version="$CHAINLINK_VERSION"
-
-        $custom_nodes_toml
-
-        [CCIP.Env.Logging]
-        test_log_collect=$test_log_collect
-        run_id="$RUN_ID"
-
-        [CCIP.Env.Logging.LogStream]
-        log_targets=$log_targets
-
-        [CCIP.Groups.load]
-        TestRunName = '$EXISTING_NAMESPACE'
-
-        [CCIP.Groups.smoke]
-        TestRunName = '$EXISTING_NAMESPACE'
-
-        EOF
-
-        # Check if UPGRADE_VERSION is not empty and append to config.toml
-        if [ -n "$UPGRADE_VERSION" ]; then
-          cat << EOF >> config.toml
-        [CCIP.Env.NewCLCluster.Common.ChainlinkUpgradeImage]
-        version="$UPGRADE_VERSION"
-        EOF
-        fi     
-
-        BASE64_CONFIG=$(cat config.toml | base64 -w 0)
-        echo ::add-mask::$BASE64_CONFIG
-        echo "base64_config=$BASE64_CONFIG" >> $GITHUB_OUTPUT
\ No newline at end of file

From 93c6d308ae650ef50841ff61971ee74cb7a51937 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 18:21:33 +0200
Subject: [PATCH 095/141] Fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index e3f8a457cf8..920478f237f 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -633,10 +633,10 @@ jobs:
           retention-days: 1
 
       - name: Upload custom test artifacts
-        if: inputs.test_log_upload_on_failure && failure()
+        if: failure() && matrix.tests.test_artifacts_on_failure != 'null'
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
-          name: custom_test_artifacts_${{ needs.load-test-configurations.outputs.workflow_id }}_${{ matrix.tests.id_sanitized }}
+          name: custom_test_artifacts_${{ matrix.tests.id_sanitized }}_${{ needs.load-test-configurations.outputs.workflow_id }}
           path: ${{ matrix.tests.test_artifacts_on_failure }}        
           retention-days: 1      
 
@@ -798,10 +798,10 @@ jobs:
         continue-on-error: true
 
       - name: Upload custom test artifacts
-        if: inputs.test_log_upload_on_failure && failure() 
+        if: failure() && matrix.tests.test_artifacts_on_failure != 'null'
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
-          name: custom_test_artifacts_${{ needs.load-test-configurations.outputs.workflow_id }}_${{ matrix.tests.id_sanitized }}
+          name: custom_test_artifacts_${{ matrix.tests.id_sanitized }}_${{ needs.load-test-configurations.outputs.workflow_id }}
           path: ${{ matrix.tests.test_artifacts_on_failure }}        
           retention-days: 1              
 

From 98814e7fb58669dac2f6ea113344fc08abf9fbad Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 9 Sep 2024 18:29:08 +0200
Subject: [PATCH 096/141] Fix

---
 .github/e2e-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 4ecf29077ab..2c6bb124b55 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -992,7 +992,7 @@ runner-test-matrix:
       E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
     workflows:
       - E2E CCIP Load Tests  
-    test_artifacts: 
+    test_artifacts_on_failure: 
       - ./integration-tests/load/logs/payload_ccip.json
 
   - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPSAfterARMCurseAndUncurse
@@ -1005,7 +1005,7 @@ runner-test-matrix:
       E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
     workflows:
       - E2E CCIP Load Tests  
-    test_artifacts: 
+    test_artifacts_on_failure: 
       - ./integration-tests/load/logs/payload_ccip.json
 
   - id: ccip-tests/chaos/ccip_test.go

From 61b47a891359a4077c723ed9e56ccd45ac986ef2 Mon Sep 17 00:00:00 2001
From: joaoluisam <joao.luismoreira@smartcontract.com>
Date: Mon, 9 Sep 2024 21:11:12 +0100
Subject: [PATCH 097/141] update test duration to 24h

---
 integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml b/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml
index 2bc5c240787..42a71171d11 100644
--- a/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml
+++ b/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml
@@ -7,7 +7,7 @@ chainlink_node_funding = 2 # Funding per node
 
 [Soak.OCR2]
 [Soak.OCR2.Common]
-test_duration = "15m" # Full duration the test will run 1440 - 1 day, 7200 - 5 days
+test_duration = "24h" # Full duration the test will run 1440 - 1 day, 7200 - 5 days
 
 [Soak.OCR2.Soak]
 number_of_contracts = 2    # Number of deployed contracts, 2 being default

From 1ed7f595dc38241d437db774eebe99b7c8320cc9 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 10 Sep 2024 11:14:17 +0200
Subject: [PATCH 098/141] Fix citool

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 920478f237f..52d572d289d 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -240,7 +240,7 @@ jobs:
           path: core
       - name: Install citool
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@f03af143be18178b618e6e5ea43ef828f52d2ea6 # v1.34.4
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@1d8d5b55bf6379b969dbcde99abc87faa5963ea1 # v1.34.4
       - name: Run Check Tests Command
         run: |
           if ! citool check-tests ${{ github.workspace }}/integration-tests ${{ github.workspace }}/.github/e2e-tests.yml; then
@@ -287,7 +287,7 @@ jobs:
           check-latest: true
       - name: Install citool
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@f03af143be18178b618e6e5ea43ef828f52d2ea6 # v1.34.4
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/citool@1d8d5b55bf6379b969dbcde99abc87faa5963ea1 # v1.34.4
       - name: Install jq
         run: sudo apt-get install jq
 

From a26a58cabf5449716ed748402a722c07f1d4ef5e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 10 Sep 2024 11:14:26 +0200
Subject: [PATCH 099/141] debug artifacts on failure

---
 .github/e2e-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 2c6bb124b55..c33e2ec2683 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -994,6 +994,7 @@ runner-test-matrix:
       - E2E CCIP Load Tests  
     test_artifacts_on_failure: 
       - ./integration-tests/load/logs/payload_ccip.json
+      - ./integration-tests/load/logs/payload_ccip2.json
 
   - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPSAfterARMCurseAndUncurse
     path: integration-tests/ccip-tests/load/ccip_test.go

From 013f5c198430c20597122fede71c7321085d7fcd Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 10 Sep 2024 11:47:44 +0200
Subject: [PATCH 100/141] debug

---
 .github/e2e-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index c33e2ec2683..d2ee17dee7e 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -993,8 +993,8 @@ runner-test-matrix:
     workflows:
       - E2E CCIP Load Tests  
     test_artifacts_on_failure: 
-      - ./integration-tests/load/logs/payload_ccip.json
-      - ./integration-tests/load/logs/payload_ccip2.json
+      - integration-tests/smoke/automation_test.go
+      - integration-tests/ccip-tests/load/ccip_test.go
 
   - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPSAfterARMCurseAndUncurse
     path: integration-tests/ccip-tests/load/ccip_test.go

From 39c5dfc86be95cc943232282e54dba71ee6620b8 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 10 Sep 2024 11:59:33 +0200
Subject: [PATCH 101/141] Fix

---
 .github/e2e-tests.yml                                 | 3 +--
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index d2ee17dee7e..2c6bb124b55 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -993,8 +993,7 @@ runner-test-matrix:
     workflows:
       - E2E CCIP Load Tests  
     test_artifacts_on_failure: 
-      - integration-tests/smoke/automation_test.go
-      - integration-tests/ccip-tests/load/ccip_test.go
+      - ./integration-tests/load/logs/payload_ccip.json
 
   - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPSAfterARMCurseAndUncurse
     path: integration-tests/ccip-tests/load/ccip_test.go
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 52d572d289d..e51456fde59 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -633,7 +633,7 @@ jobs:
           retention-days: 1
 
       - name: Upload custom test artifacts
-        if: failure() && matrix.tests.test_artifacts_on_failure != 'null'
+        if: failure() && matrix.tests.test_artifacts_on_failure != ''
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: custom_test_artifacts_${{ matrix.tests.id_sanitized }}_${{ needs.load-test-configurations.outputs.workflow_id }}
@@ -798,7 +798,7 @@ jobs:
         continue-on-error: true
 
       - name: Upload custom test artifacts
-        if: failure() && matrix.tests.test_artifacts_on_failure != 'null'
+        if: failure() && matrix.tests.test_artifacts_on_failure != ''
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: custom_test_artifacts_${{ matrix.tests.id_sanitized }}_${{ needs.load-test-configurations.outputs.workflow_id }}

From 3a5699404a0378df9d197a93b59b4d733a34286d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 10 Sep 2024 13:04:30 +0200
Subject: [PATCH 102/141] Use ubuntu-latest

---
 .github/workflows/on-demand-vrfv2plus-performance-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
index 52c7249c645..d242008643d 100644
--- a/.github/workflows/on-demand-vrfv2plus-performance-test.yml
+++ b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
@@ -47,7 +47,7 @@ jobs:
                 {
                   "id": "TestVRFv2Plus_Performance",
                   "path": "integration-tests/load/vrfv2plus/vrfv2plus_test.go",
-                  "runs_on": "ubuntu22.04-8cores-32GB",
+                  "runs_on": "ubuntu-latest",
                   "test_env_type": "docker",
                   "test_cmd": $test_cmd,
                   "test_config_override_path": $test_config_override_path,

From 42362da909d56ea05e52d5fd0aa739e01ed0e0f4 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 10 Sep 2024 13:10:40 +0200
Subject: [PATCH 103/141] Update workflow slack notifications

---
 .github/workflows/on-demand-vrfv2plus-performance-test.yml | 3 +++
 .github/workflows/on-demand-vrfv2plus-smoke-tests.yml      | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
index d242008643d..22aee4870be 100644
--- a/.github/workflows/on-demand-vrfv2plus-performance-test.yml
+++ b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
@@ -67,6 +67,9 @@ jobs:
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}
+      slack_notification_after_tests: always
+      slack_notification_after_tests_name: "VRF V2 Plus Performance Tests with test config: ${{ inputs.test_config_override_path || 'default' }}"
+      slack_notification_after_tests_notify_user_id_on_failure: U03PJF3VDB8
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
index deb1b57df4f..f6a2cebd0c7 100644
--- a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
+++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
@@ -71,7 +71,8 @@ jobs:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}
       slack_notification_after_tests: always
-      slack_notification_after_tests_name: VRF V2 Plus Smoke Tests
+      slack_notification_after_tests_name: "VRF V2 Plus Smoke Tests with test config: ${{ inputs.test_config_override_path || 'default' }}"
+      slack_notification_after_tests_notify_user_id_on_failure: U03PJF3VDB8
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From 84eb57ae5dabc6180fdc527a6237197487e082bc Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 10 Sep 2024 13:19:12 +0200
Subject: [PATCH 104/141] Fix test id when running custom tests

---
 .../run-e2e-tests-reusable-workflow.yml       | 22 ++++++++++---------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index e51456fde59..dc2ecdcead0 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -465,13 +465,14 @@ jobs:
     env:
       LATEST_CHAINLINK_RELEASE_VERSION: ${{ needs.get_latest_chainlink_release_version.outputs.latest_chainlink_release_version }}
       TEST_CONFIG_OVERRIDE_PATH: ${{ matrix.tests.test_config_override_path || inputs.test_config_override_path }}
+      TEST_ID: ${{ matrix.tests.id_sanitized || matrix.tests.id }}
     steps:
       - name: Collect Metrics
         if: always()
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
         with:
-          id: e2e_tests_${{ matrix.tests.id_sanitized }}
+          id: e2e_tests_${{ env.TEST_ID }}
           org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
           basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
           hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
@@ -566,12 +567,12 @@ jobs:
           default_e2e_test_chainlink_image: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_IMAGE || env.CHAINLINK_IMAGE }}
           default_e2e_test_chainlink_upgrade_image: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_IMAGE }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          artifacts_name: ${{ matrix.tests.id_sanitized }}-test-logs
+          artifacts_name: ${{ env.TEST_ID }}-test-logs
           artifacts_location: |
             ./integration-tests/smoke/logs/
             ./integration-tests/smoke/db_dumps/
             /tmp/gotest.log
-          publish_check_name: ${{ matrix.tests.id_sanitized }}
+          publish_check_name: ${{ env.TEST_ID }}
           token: ${{ secrets.GH_TOKEN }}
           cache_key_id: e2e-tests
           go_mod_path: ./integration-tests/go.mod
@@ -603,7 +604,7 @@ jobs:
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: inputs.test_log_upload_on_failure && failure()
         with:
-          name: test_log_${{ matrix.tests.id_sanitized }}
+          name: test_log_${{ env.TEST_ID }}
           path: /tmp/gotest.log
           retention-days: ${{ inputs.test_log_upload_retention_days }}
         continue-on-error: true
@@ -614,7 +615,7 @@ jobs:
         timeout-minutes: 2
         continue-on-error: true
         with:
-          name: ${{ inputs.upload_cl_node_coverage_artifact_prefix }}${{ matrix.tests.id_sanitized }}
+          name: ${{ inputs.upload_cl_node_coverage_artifact_prefix }}${{ env.TEST_ID }}
           path: .covdata
           retention-days: 1
 
@@ -628,7 +629,7 @@ jobs:
       - name: Upload test result as artifact
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
-          name: test_result_${{ needs.load-test-configurations.outputs.workflow_id }}_${{ matrix.tests.id_sanitized }}
+          name: test_result_${{ needs.load-test-configurations.outputs.workflow_id }}_${{ env.TEST_ID }}
           path: test_result.json        
           retention-days: 1
 
@@ -636,7 +637,7 @@ jobs:
         if: failure() && matrix.tests.test_artifacts_on_failure != ''
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
-          name: custom_test_artifacts_${{ matrix.tests.id_sanitized }}_${{ needs.load-test-configurations.outputs.workflow_id }}
+          name: custom_test_artifacts_${{ env.TEST_ID }}_${{ needs.load-test-configurations.outputs.workflow_id }}
           path: ${{ matrix.tests.test_artifacts_on_failure }}        
           retention-days: 1      
 
@@ -702,13 +703,14 @@ jobs:
     env:
       LATEST_CHAINLINK_RELEASE_VERSION: ${{ needs.get_latest_chainlink_release_version.outputs.latest_chainlink_release_version }}
       TEST_CONFIG_OVERRIDE_PATH: ${{ matrix.tests.test_config_override_path || inputs.test_config_override_path }}
+      TEST_ID: ${{ matrix.tests.id_sanitized || matrix.tests.id }}
     steps:
       - name: Collect Metrics
         if: always()
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
         with:
-          id: e2e_tests_${{ matrix.tests.id_sanitized }}
+          id: e2e_tests_${{ env.TEST_ID }}
           org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
           basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
           hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
@@ -792,7 +794,7 @@ jobs:
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: inputs.test_log_upload_on_failure && failure()
         with:
-          name: test_log_${{ matrix.tests.id_sanitized }}
+          name: test_log_${{ env.TEST_ID }}
           path: /tmp/gotest.log
           retention-days: ${{ inputs.test_log_upload_retention_days }}
         continue-on-error: true
@@ -801,7 +803,7 @@ jobs:
         if: failure() && matrix.tests.test_artifacts_on_failure != ''
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
-          name: custom_test_artifacts_${{ matrix.tests.id_sanitized }}_${{ needs.load-test-configurations.outputs.workflow_id }}
+          name: custom_test_artifacts_${{ env.TEST_ID }}_${{ needs.load-test-configurations.outputs.workflow_id }}
           path: ${{ matrix.tests.test_artifacts_on_failure }}        
           retention-days: 1              
 

From 0c72f4f9906785c791a4ef5792a9a5fb8db4ce23 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 11 Sep 2024 16:52:32 +0200
Subject: [PATCH 105/141] Fix

---
 .github/e2e-tests.yml                         | 26 ++++++++++---------
 .../run-e2e-tests-reusable-workflow.yml       |  6 +++++
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 8367f89e698..628b701ab39 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -1005,18 +1005,20 @@ runner-test-matrix:
     test_artifacts_on_failure: 
       - ./integration-tests/load/logs/payload_ccip.json
 
-  - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPSAfterARMCurseAndUncurse
-    path: integration-tests/ccip-tests/load/ccip_test.go
-    test_env_type: docker
-    runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ccip-tests/load && go test -test.run $TestLoadCCIPStableRPSAfterARMCurseAndUncurse$ -timeout 70m -count=1 -test.parallel=1 -json
-    test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/load-with-arm-curse-uncurse.toml
-    test_env_vars:
-      E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
-    workflows:
-      - E2E CCIP Load Tests  
-    test_artifacts_on_failure: 
-      - ./integration-tests/load/logs/payload_ccip.json
+  # Enable when CCIP-2277 is resolved
+  #
+  # - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPSAfterARMCurseAndUncurse
+  #   path: integration-tests/ccip-tests/load/ccip_test.go
+  #   test_env_type: docker
+  #   runs_on: ubuntu-latest
+  #   test_cmd: cd integration-tests/ccip-tests/load && go test -test.run $TestLoadCCIPStableRPSAfterARMCurseAndUncurse$ -timeout 70m -count=1 -test.parallel=1 -json
+  #   test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/load-with-arm-curse-uncurse.toml
+  #   test_env_vars:
+  #     E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
+  #   workflows:
+  #     - E2E CCIP Load Tests  
+  #   test_artifacts_on_failure: 
+  #     - ./integration-tests/load/logs/payload_ccip.json
 
   - id: ccip-tests/chaos/ccip_test.go
     path: integration-tests/ccip-tests/chaos/ccip_test.go
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index d4dca5d45af..44ce2bfd02a 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -63,6 +63,11 @@ on:
         description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"'
         required: false
         type: string
+      test_image_suites:
+        description: 'Suites to build in the test image. Space separated'
+        required: false
+        type: string
+        default: chaos migration reorg smoke soak benchmark load
       require_chainlink_image_versions_in_qa_ecr:
         description: 'Check Chainlink image versions to be present in QA ECR. If not, build and push the image to QA ECR. Takes comma separated list of Chainlink image versions. Example: "5733cdcda9a9fc6da6343798b119b2ae136146cd,0b7d2c497a508efa5a827714780d908b7b8eda19"'
         required: false
@@ -674,6 +679,7 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          suites: ${{ inputs.test_image_suites }}
       - name: Set Remote Runner Version
         id: set-remote-runner-version
         run: |

From f93f493c35727e902e66f4cd41c1e6acb4dd44a0 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 11 Sep 2024 16:56:38 +0200
Subject: [PATCH 106/141] Fix

---
 .github/workflows/ccip-load-tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml
index f1c30d8f016..e8101223ba8 100644
--- a/.github/workflows/ccip-load-tests.yml
+++ b/.github/workflows/ccip-load-tests.yml
@@ -37,8 +37,7 @@ jobs:
       test_config_override_path: ${{ inputs.test_config_override_path }}
       chainlink_version: ${{ inputs.chainlink_version || github.sha }} 
       slack_notification_after_tests: always
-      slack_notification_after_tests_channel_id: '#lf-slack-notification-tests'
-      # slack_notification_after_tests_channel_id: '#ccip-testing'
+      slack_notification_after_tests_channel_id: '#ccip-testing'
       slack_notification_after_tests_name: CCIP E2E Load Tests
       test_image_suites: ccip-load
     secrets:

From 91ed43b6678e878debd048f2cef70929c33ebb29 Mon Sep 17 00:00:00 2001
From: davidcauchi <david.cauchi@smartcontract.com>
Date: Wed, 11 Sep 2024 20:47:38 +0200
Subject: [PATCH 107/141] Add OCR test configs

---
 integration-tests/testconfig/default.toml     | 127 +++++-------------
 .../ocr/overrides/arbitrum_mainnet.toml       |  16 +++
 .../ocr/overrides/arbitrum_sepolia.toml       |  13 ++
 .../ocr/overrides/base_mainnet.toml           |  16 +++
 .../ocr/overrides/base_sepolia.toml           |  14 ++
 .../ocr/overrides/celo_alfajores.toml         |  13 ++
 .../ocr/overrides/ethereum_sepolia.toml       |  13 ++
 .../ocr/overrides/linea_sepolia.toml          |  13 ++
 .../ocr/overrides/optimism_mainnet.toml       |  16 +++
 .../ocr/overrides/optimism_sepolia.toml       |  13 ++
 .../ocr/overrides/scroll_sepolia.toml         |  13 ++
 .../ocr/overrides/wemix_mainnet.toml          |  16 +++
 .../ocr2/overrides/base_sepolia.toml          |   9 +-
 .../ocr2/overrides/celo_alfajores.toml        |  17 ---
 .../ocr2/overrides/ethereum_sepolia.toml      |  13 ++
 .../ocr2/overrides/polygon_amoy.toml          |  13 ++
 .../ocr2/overrides/polygon_mainnet.toml       |  16 +++
 .../ocr2/overrides/wemix_testnet.toml         |  13 +-
 .../ocr2/overrides/xlayer_sepolia.toml        |  13 ++
 19 files changed, 250 insertions(+), 127 deletions(-)
 create mode 100644 integration-tests/testconfig/ocr/overrides/arbitrum_mainnet.toml
 create mode 100644 integration-tests/testconfig/ocr/overrides/arbitrum_sepolia.toml
 create mode 100644 integration-tests/testconfig/ocr/overrides/base_mainnet.toml
 create mode 100644 integration-tests/testconfig/ocr/overrides/base_sepolia.toml
 create mode 100644 integration-tests/testconfig/ocr/overrides/celo_alfajores.toml
 create mode 100644 integration-tests/testconfig/ocr/overrides/ethereum_sepolia.toml
 create mode 100644 integration-tests/testconfig/ocr/overrides/linea_sepolia.toml
 create mode 100644 integration-tests/testconfig/ocr/overrides/optimism_mainnet.toml
 create mode 100644 integration-tests/testconfig/ocr/overrides/optimism_sepolia.toml
 create mode 100644 integration-tests/testconfig/ocr/overrides/scroll_sepolia.toml
 create mode 100644 integration-tests/testconfig/ocr/overrides/wemix_mainnet.toml
 delete mode 100644 integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml
 create mode 100644 integration-tests/testconfig/ocr2/overrides/ethereum_sepolia.toml
 create mode 100644 integration-tests/testconfig/ocr2/overrides/polygon_amoy.toml
 create mode 100644 integration-tests/testconfig/ocr2/overrides/polygon_mainnet.toml
 create mode 100644 integration-tests/testconfig/ocr2/overrides/xlayer_sepolia.toml

diff --git a/integration-tests/testconfig/default.toml b/integration-tests/testconfig/default.toml
index f1dd65c5a82..48a5cd35b85 100644
--- a/integration-tests/testconfig/default.toml
+++ b/integration-tests/testconfig/default.toml
@@ -254,66 +254,14 @@ gas_tip_cap = 1_800_000_000
 [[Seth.networks]]
 name = "Sepolia Testnet"
 transaction_timeout = "3m"
-eip_1559_dynamic_fees = false
-
-# automated gas estimation for live networks
-# if set to true we will dynamically estimate gas for every transaction (based on suggested values, priority and congestion rate for last X blocks)
-# gas_price_estimation_enabled = true
-# number of blocks to use for congestion rate estimation (it will determine buffer added on top of suggested values)
-# gas_price_estimation_blocks = 100
-# transaction priority, which determines adjustment factor multiplier applied to suggested values (fast - 1.2x, standard - 1x, slow - 0.8x)
-# gas_price_estimation_tx_priority = "standard"
-
-# URLs
-# if set they will overwrite URLs from EVMNetwork that Seth uses, can be either WS(S) or HTTP(S)
-# urls_secret = ["ws://your-ws-url:8546"]
-
-# gas_limits
-# gas limit should be explicitly set only if you are connecting to a node that's incapable of estimating gas limit itself (should only happen for very old versions)
-# gas_limit = 14_000_000
-# transfer_gas_fee is gas limit that will be used, when funding CL nodes and returning funds from there and when funding and returning funds from ephemeral keys
-# we use hardcoded value in order to be estimate how much funds are available for sending or returning after tx costs have been paid
-transfer_gas_fee = 21_000
-
-# manual settings, used when gas_price_estimation_enabled is false or when it fails
-# legacy transactions
-gas_price = 50_000_000_000
-
-# EIP-1559 transactions
-gas_fee_cap = 45_000_000_000
-gas_tip_cap = 10_000_000_000
-
-[[Seth.networks]]
-name = "Polygon Mumbai"
-transaction_timeout = "3m"
 eip_1559_dynamic_fees = true
-
-# automated gas estimation for live networks
-# if set to true we will dynamically estimate gas for every transaction (based on suggested values, priority and congestion rate for last X blocks)
-# gas_price_estimation_enabled = true
-# number of blocks to use for congestion rate estimation (it will determine buffer added on top of suggested values)
-# gas_price_estimation_blocks = 100
-# transaction priority, which determines adjustment factor multiplier applied to suggested values (fast - 1.2x, standard - 1x, slow - 0.8x)
-# gas_price_estimation_tx_priority = "standard"
-
-# URLs
-# if set they will overwrite URLs from EVMNetwork that Seth uses, can be either WS(S) or HTTP(S)
-# urls_secret = ["ws://your-ws-url:8546"]
-
-# gas_limits
-# gas limit should be explicitly set only if you are connecting to a node that's incapable of estimating gas limit itself (should only happen for very old versions)
-# gas_limit = 6_000_000
-# transfer_gas_fee is gas limit that will be used, when funding CL nodes and returning funds from there and when funding and returning funds from ephemeral keys
-# we use hardcoded value in order to be estimate how much funds are available for sending or returning after tx costs have been paid
 transfer_gas_fee = 21_000
-
-# manual settings, used when gas_price_estimation_enabled is false or when it fails
-# legacy transactions
-gas_price = 1_800_000_000
-
-# EIP-1559 transactions
-gas_fee_cap = 3_800_000_000
-gas_tip_cap = 1_800_000_000
+gas_price = 105_000_000_000
+gas_fee_cap = 150_312_843_059
+gas_tip_cap = 40_416_094
+gas_price_estimation_enabled = true
+gas_price_estimation_blocks = 1000
+gas_price_estimation_tx_priority = "standard"
 
 [[Seth.networks]]
 name = "Polygon Amoy"
@@ -322,11 +270,11 @@ eip_1559_dynamic_fees = true
 
 # automated gas estimation for live networks
 # if set to true we will dynamically estimate gas for every transaction (based on suggested values, priority and congestion rate for last X blocks)
- gas_price_estimation_enabled = true
+gas_price_estimation_enabled = true
 # number of blocks to use for congestion rate estimation (it will determine buffer added on top of suggested values)
- gas_price_estimation_blocks = 100
+gas_price_estimation_blocks = 100
 # transaction priority, which determines adjustment factor multiplier applied to suggested values (fast - 1.2x, standard - 1x, slow - 0.8x)
- gas_price_estimation_tx_priority = "standard"
+gas_price_estimation_tx_priority = "standard"
 
 # URLs
 # if set they will overwrite URLs from EVMNetwork that Seth uses, can be either WS(S) or HTTP(S)
@@ -347,38 +295,6 @@ gas_price = 200_000_000_000
 gas_fee_cap = 200_000_000_000
 gas_tip_cap = 25_000_000_000
 
-[[Seth.networks]]
-name = "Polygon zkEVM Goerli"
-transaction_timeout = "3m"
-eip_1559_dynamic_fees = false
-
-# automated gas estimation for live networks
-# if set to true we will dynamically estimate gas for every transaction (based on suggested values, priority and congestion rate for last X blocks)
-# gas_price_estimation_enabled = true
-# number of blocks to use for congestion rate estimation (it will determine buffer added on top of suggested values)
-# gas_price_estimation_blocks = 100
-# transaction priority, which determines adjustment factor multiplier applied to suggested values (fast - 1.2x, standard - 1x, slow - 0.8x)
-# gas_price_estimation_tx_priority = "standard"
-
-# URLs
-# if set they will overwrite URLs from EVMNetwork that Seth uses, can be either WS(S) or HTTP(S)
-# urls_secret = ["ws://your-ws-url:8546"]
-
-# gas_limits
-# gas limit should be explicitly set only if you are connecting to a node that's incapable of estimating gas limit itself (should only happen for very old versions)
-# gas_limit = 9_000_000
-# transfer_gas_fee is gas limit that will be used, when funding CL nodes and returning funds from there and when funding and returning funds from ephemeral keys
-# we use hardcoded value in order to be estimate how much funds are available for sending or returning after tx costs have been paid
-transfer_gas_fee = 21_000
-
-# manual settings, used when gas_price_estimation_enabled is false or when it fails
-# legacy transactions
-gas_price = 50_000_000
-
-# EIP-1559 transactions
-gas_fee_cap = 3_800_000_000
-gas_tip_cap = 1_800_000_000
-
 [[Seth.networks]]
 name = "Optimism Sepolia"
 transaction_timeout = "3m"
@@ -681,7 +597,6 @@ gas_price_estimation_tx_priority = "standard"
 
 [[Seth.networks]]
 name = "TREASURE_RUBY"
-chain_id = "978657"
 transaction_timeout = "10m"
 transfer_gas_fee = 21_000
 gas_price = 100_000_000
@@ -694,7 +609,6 @@ gas_price_estimation_tx_priority = "standard"
 
 [[Seth.networks]]
 name = "XLAYER_MAINNET"
-chain_id = "196"
 transaction_timeout = "10m"
 transfer_gas_fee = 21_000
 gas_price = 13_400_000_000
@@ -707,7 +621,6 @@ gas_price_estimation_tx_priority = "standard"
 
 [[Seth.networks]]
 name = "XLAYER_SEPOLIA"
-chain_id = "195"
 transaction_timeout = "10m"
 transfer_gas_fee = 21_000
 gas_price = 200_000_000_000
@@ -717,3 +630,25 @@ gas_tip_cap = 30_000_000_000
 gas_price_estimation_enabled = true
 gas_price_estimation_blocks = 500
 gas_price_estimation_tx_priority = "standard"
+
+[[Seth.networks]]
+name = "POLYGON_MAINNET"
+transaction_timeout = "3m"
+transfer_gas_fee = 21_000
+gas_price = 31_000_000_000
+eip_1559_dynamic_fees = true
+gas_fee_cap = 61_000_000_000
+gas_tip_cap = 30_000_000_000
+gas_price_estimation_enabled = true
+gas_price_estimation_blocks = 1000
+
+[[Seth.networks]]
+name = "ARBITRUM_MAINNET"
+transaction_timeout = "10m"
+transfer_gas_fee = 21_000
+gas_price = 10_000_000
+eip_1559_dynamic_fees = true
+gas_fee_cap = 10_000_000
+gas_tip_cap = 0
+gas_price_estimation_enabled = true
+gas_price_estimation_blocks = 1000
diff --git a/integration-tests/testconfig/ocr/overrides/arbitrum_mainnet.toml b/integration-tests/testconfig/ocr/overrides/arbitrum_mainnet.toml
new file mode 100644
index 00000000000..fa6a2173998
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/arbitrum_mainnet.toml
@@ -0,0 +1,16 @@
+[Network]
+selected_networks = ["ARBITRUM_MAINNET"]
+
+[Soak.Common]
+chainlink_node_funding = 0.1
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "10m"
+
+[Seth]
+experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr/overrides/arbitrum_sepolia.toml b/integration-tests/testconfig/ocr/overrides/arbitrum_sepolia.toml
new file mode 100644
index 00000000000..4eb9f70b9b9
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/arbitrum_sepolia.toml
@@ -0,0 +1,13 @@
+[Network]
+selected_networks = ["ARBITRUM_SEPOLIA"]
+
+[Soak.Common]
+chainlink_node_funding = 1
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "2m"
diff --git a/integration-tests/testconfig/ocr/overrides/base_mainnet.toml b/integration-tests/testconfig/ocr/overrides/base_mainnet.toml
new file mode 100644
index 00000000000..8a8cfc34ae5
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/base_mainnet.toml
@@ -0,0 +1,16 @@
+[Network]
+selected_networks = ["BASE_MAINNET"]
+
+[Soak.Common]
+chainlink_node_funding = 5
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "10m"
+
+[Seth]
+experiments_enabled = ["slow_funds_return"]
\ No newline at end of file
diff --git a/integration-tests/testconfig/ocr/overrides/base_sepolia.toml b/integration-tests/testconfig/ocr/overrides/base_sepolia.toml
new file mode 100644
index 00000000000..21c5975d890
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/base_sepolia.toml
@@ -0,0 +1,14 @@
+[Network]
+selected_networks = ["BASE_SEPOLIA"]
+
+[Soak.Common]
+chainlink_node_funding = 1
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "2m"
+
diff --git a/integration-tests/testconfig/ocr/overrides/celo_alfajores.toml b/integration-tests/testconfig/ocr/overrides/celo_alfajores.toml
new file mode 100644
index 00000000000..c051cea64e7
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/celo_alfajores.toml
@@ -0,0 +1,13 @@
+[Network]
+selected_networks = ["CELO_ALFAJORES"]
+
+[Soak.Common]
+chainlink_node_funding = 10
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "2m"
diff --git a/integration-tests/testconfig/ocr/overrides/ethereum_sepolia.toml b/integration-tests/testconfig/ocr/overrides/ethereum_sepolia.toml
new file mode 100644
index 00000000000..d0fa75565eb
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/ethereum_sepolia.toml
@@ -0,0 +1,13 @@
+[Network]
+selected_networks = ["SEPOLIA"]
+
+[Soak.Common]
+chainlink_node_funding = 1
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "2m"
diff --git a/integration-tests/testconfig/ocr/overrides/linea_sepolia.toml b/integration-tests/testconfig/ocr/overrides/linea_sepolia.toml
new file mode 100644
index 00000000000..9f9c5b2e3a6
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/linea_sepolia.toml
@@ -0,0 +1,13 @@
+[Network]
+selected_networks = ["Linea_Sepolia"]
+
+[Soak.Common]
+chainlink_node_funding = 1
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "2m"
diff --git a/integration-tests/testconfig/ocr/overrides/optimism_mainnet.toml b/integration-tests/testconfig/ocr/overrides/optimism_mainnet.toml
new file mode 100644
index 00000000000..1996a89ec73
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/optimism_mainnet.toml
@@ -0,0 +1,16 @@
+[Network]
+selected_networks = ["OPTIMISM_MAINNET"]
+
+[Soak.Common]
+chainlink_node_funding = 0.1
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "10m"
+
+[Seth]
+experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr/overrides/optimism_sepolia.toml b/integration-tests/testconfig/ocr/overrides/optimism_sepolia.toml
new file mode 100644
index 00000000000..804e977db8c
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/optimism_sepolia.toml
@@ -0,0 +1,13 @@
+[Network]
+selected_networks = ["OPTIMISM_SEPOLIA"]
+
+[Soak.Common]
+chainlink_node_funding = 1
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "2m"
diff --git a/integration-tests/testconfig/ocr/overrides/scroll_sepolia.toml b/integration-tests/testconfig/ocr/overrides/scroll_sepolia.toml
new file mode 100644
index 00000000000..964756d5fbd
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/scroll_sepolia.toml
@@ -0,0 +1,13 @@
+[Network]
+selected_networks = ["SCROLL_SEPOLIA"]
+
+[Soak.Common]
+chainlink_node_funding = 1
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "2m"
diff --git a/integration-tests/testconfig/ocr/overrides/wemix_mainnet.toml b/integration-tests/testconfig/ocr/overrides/wemix_mainnet.toml
new file mode 100644
index 00000000000..138cd7d46f7
--- /dev/null
+++ b/integration-tests/testconfig/ocr/overrides/wemix_mainnet.toml
@@ -0,0 +1,16 @@
+[Network]
+selected_networks = ["WEMIX_MAINNET"]
+
+[Soak.Common]
+chainlink_node_funding = 5
+
+[Soak.OCR]
+[Soak.OCR.Common]
+test_duration = "24h"
+
+[Soak.OCR.Soak]
+number_of_contracts = 2
+time_between_rounds = "10m"
+
+[Seth]
+experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml b/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
index 2bd6f503014..9ed03bf078f 100644
--- a/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
+++ b/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
@@ -1,17 +1,16 @@
 [Network]
 selected_networks = ["BASE_SEPOLIA"]
 
-# soak test specific configuration
 [Soak.Common]
-chainlink_node_funding = 0.5 # Funding per node
+chainlink_node_funding = 1
 
 [Soak.OCR2]
 [Soak.OCR2.Common]
-test_duration = "24h" # Full duration the test will run 1440 - 1 day, 7200 - 5 days
+test_duration = "24h"
 
 [Soak.OCR2.Soak]
-number_of_contracts = 2    # Number of deployed contracts, 2 being default
-time_between_rounds = "5m" # Default 5 mins for a 1 day, 10 mins for a 5 day
+number_of_contracts = 2
+time_between_rounds = "2m"
 
 [Seth]
 experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml b/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml
deleted file mode 100644
index 42a71171d11..00000000000
--- a/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml
+++ /dev/null
@@ -1,17 +0,0 @@
-[Network]
-selected_networks = ["CELO_ALFAJORES"]
-
-# soak test specific configuration
-[Soak.Common]
-chainlink_node_funding = 2 # Funding per node
-
-[Soak.OCR2]
-[Soak.OCR2.Common]
-test_duration = "24h" # Full duration the test will run 1440 - 1 day, 7200 - 5 days
-
-[Soak.OCR2.Soak]
-number_of_contracts = 2    # Number of deployed contracts, 2 being default
-time_between_rounds = "5m" # Default 5 mins for a 1 day, 10 mins for a 5 day
-
-[Seth]
-experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr2/overrides/ethereum_sepolia.toml b/integration-tests/testconfig/ocr2/overrides/ethereum_sepolia.toml
new file mode 100644
index 00000000000..f7dadcf7337
--- /dev/null
+++ b/integration-tests/testconfig/ocr2/overrides/ethereum_sepolia.toml
@@ -0,0 +1,13 @@
+[Network]
+selected_networks = ["SEPOLIA"]
+
+[Soak.Common]
+chainlink_node_funding = 1
+
+[Soak.OCR2]
+[Soak.OCR2.Common]
+test_duration = "24h"
+
+[Soak.OCR2.Soak]
+number_of_contracts = 2
+time_between_rounds = "2m"
diff --git a/integration-tests/testconfig/ocr2/overrides/polygon_amoy.toml b/integration-tests/testconfig/ocr2/overrides/polygon_amoy.toml
new file mode 100644
index 00000000000..6af0aa556ec
--- /dev/null
+++ b/integration-tests/testconfig/ocr2/overrides/polygon_amoy.toml
@@ -0,0 +1,13 @@
+[Network]
+selected_networks = ["POLYGON_AMOY"]
+
+[Soak.Common]
+chainlink_node_funding = 20
+
+[Soak.OCR2]
+[Soak.OCR2.Common]
+test_duration = "24h"
+
+[Soak.OCR2.Soak]
+number_of_contracts = 2
+time_between_rounds = "2m"
diff --git a/integration-tests/testconfig/ocr2/overrides/polygon_mainnet.toml b/integration-tests/testconfig/ocr2/overrides/polygon_mainnet.toml
new file mode 100644
index 00000000000..fd814c206b3
--- /dev/null
+++ b/integration-tests/testconfig/ocr2/overrides/polygon_mainnet.toml
@@ -0,0 +1,16 @@
+[Network]
+selected_networks = ["POLYGON_MAINNET"]
+
+[Soak.Common]
+chainlink_node_funding = 5
+
+[Soak.OCR2]
+[Soak.OCR2.Common]
+test_duration = "24h"
+
+[Soak.OCR2.Soak]
+number_of_contracts = 2
+time_between_rounds = "10m"
+
+[Seth]
+experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml b/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
index dac4819d2f6..a48c76eaccb 100644
--- a/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
+++ b/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
@@ -1,18 +1,13 @@
 [Network]
 selected_networks = ["WEMIX_TESTNET"]
 
-# soak test specific configuration
 [Soak.Common]
-chainlink_node_funding = 2 # Funding per node
+chainlink_node_funding = 10
 
 [Soak.OCR2]
-
 [Soak.OCR2.Common]
-test_duration = "2880m" # Full duration the test will run 1440 - 1 day, 7200 - 5 days
+test_duration = "24h"
 
 [Soak.OCR2.Soak]
-number_of_contracts = 2    # Number of deployed contracts, 2 being default
-time_between_rounds = "5m" # Default 5 mins for a 1 day, 10 mins for a 5 day
-
-[Seth]
-experiments_enabled = ["slow_funds_return"]
+number_of_contracts = 2
+time_between_rounds = "2m"
diff --git a/integration-tests/testconfig/ocr2/overrides/xlayer_sepolia.toml b/integration-tests/testconfig/ocr2/overrides/xlayer_sepolia.toml
new file mode 100644
index 00000000000..3d94da583fa
--- /dev/null
+++ b/integration-tests/testconfig/ocr2/overrides/xlayer_sepolia.toml
@@ -0,0 +1,13 @@
+[Network]
+selected_networks = ["xlayer_sepolia"]
+
+[Soak.Common]
+chainlink_node_funding = 1
+
+[Soak.OCR2]
+[Soak.OCR2.Common]
+test_duration = "24h"
+
+[Soak.OCR2.Soak]
+number_of_contracts = 2
+time_between_rounds = "2m"

From 20046e1469e9fe3ffa379c0dec7c17eabe663c9b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Thu, 12 Sep 2024 10:16:07 +0200
Subject: [PATCH 108/141] remove unused workflow

---
 .github/workflows/vrf-e2e-release-tests.yml | 44 ---------------------
 1 file changed, 44 deletions(-)
 delete mode 100644 .github/workflows/vrf-e2e-release-tests.yml

diff --git a/.github/workflows/vrf-e2e-release-tests.yml b/.github/workflows/vrf-e2e-release-tests.yml
deleted file mode 100644
index 7b1f0092dda..00000000000
--- a/.github/workflows/vrf-e2e-release-tests.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-name: Run VRF E2E Release Tests
-
-on:
-  workflow_dispatch:
-    inputs:
-      chainlink_version:
-        description: Chainlink image version to use
-        default: develop
-        required: true
-        type: string
-      test_secrets_override_key:
-        description: Key to override default test secrets
-        required: true
-        type: string         
-
-jobs:
-  run-e2e-tests-workflow:
-    name: Run E2E Tests
-    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
-    with:
-      chainlink_version: ${{ github.event.inputs.chainlink_version }}
-      test_workflow: VRF E2E Release Tests
-      slack_notification_after_tests: true
-      # slack_notification_after_tests_channel_id: "#team-test-tooling-internal"
-      slack_notification_after_tests_channel_id: "#lf-slack-notification-tests"
-      slack_notification_after_tests_name: JUST TESTING PLEASE IGNORE - VRF E2E Release Tests
-      slack_notification_after_tests_notify_user_id_on_failure: 'U04DYU1KLGJ'
-    secrets:
-      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
-      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}    
-      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
-      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}

From 345bd2817affb3bbdd05f5a6c9482cde8c650d7e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Thu, 12 Sep 2024 10:19:03 +0200
Subject: [PATCH 109/141] Add slack notifications for vrfv2 workflows

---
 .../workflows/on-demand-vrfv2-eth2-clients-test.yml  | 11 ++++++++++-
 .../workflows/on-demand-vrfv2-performance-test.yml   | 12 ++++++++++--
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml b/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml
index 166fe8e1eed..4113454542c 100644
--- a/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml
+++ b/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml
@@ -10,7 +10,11 @@ on:
         description: 'Key to run tests with custom test secrets'
         required: false
         type: string
-
+      notify_user_id_on_failure:
+        description: 'Enter Slack user ID to notify on test failure'
+        required: false
+        type: string    
+        
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
@@ -19,6 +23,9 @@ jobs:
       test_ids: smoke/vrfv2_test.go:TestVRFv2Basic
       test_config_override_path: ${{ inputs.test_config_override_path }}
       chainlink_version: ${{ github.sha }}
+      slack_notification_after_tests: always
+      slack_notification_after_tests_name: "VRF V2 ETH2 Clients with test config: ${{ inputs.test_config_override_path || 'default' }}"
+      slack_notification_after_tests_notify_user_id_on_failure: ${{ inputs.notify_user_id_on_failure }}      
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
@@ -35,3 +42,5 @@ jobs:
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
       AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
       TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
+      SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }}
diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml
index e2dc04011d4..ad8640ccfac 100644
--- a/.github/workflows/on-demand-vrfv2-performance-test.yml
+++ b/.github/workflows/on-demand-vrfv2-performance-test.yml
@@ -23,7 +23,11 @@ on:
         description: 'Key to run tests with custom test secrets'
         required: false
         type: string 
-
+      notify_user_id_on_failure:
+        description: 'Enter Slack user ID to notify on test failure'
+        required: false
+        type: string    
+        
 jobs:
   set-tests-to-run:
     name: Set tests to run
@@ -67,6 +71,9 @@ jobs:
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}
+      slack_notification_after_tests: always
+      slack_notification_after_tests_name: "VRF V2 Performance Tests with test config: ${{ inputs.test_config_override_path || 'default' }}"
+      slack_notification_after_tests_notify_user_id_on_failure: ${{ inputs.notify_user_id_on_failure }}      
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
@@ -81,7 +88,8 @@ jobs:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}      
       TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
       SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
       SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }}      

From 43bfc9a77ff3d2b591ab5b2717f92244ca6f42bf Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 17 Sep 2024 14:24:38 +0200
Subject: [PATCH 110/141] Fix slack notifications for automation workflows

---
 .github/workflows/automation-benchmark-tests.yml      |  8 +++++++-
 .github/workflows/automation-load-tests.yml           | 10 ++++++++--
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 11 ++++++++++-
 3 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml
index 61a55aaf698..1b6df7f5e4b 100644
--- a/.github/workflows/automation-benchmark-tests.yml
+++ b/.github/workflows/automation-benchmark-tests.yml
@@ -15,6 +15,11 @@ on:
         default: develop
         required: true
         type: string
+      slackMemberID:
+        description: Notifies test results (Not your @)
+        required: true
+        default: U02Q14G80TY
+        type: string     
 
 jobs:
   run-e2e-tests-workflow:
@@ -24,7 +29,8 @@ jobs:
       test_ids: 'benchmark/keeper_test.go:TestAutomationBenchmark'
       test_config_override_path: ${{ inputs.test_config_override_path }}
       chainlink_version: ${{ github.sha }}
-      slack_notification_after_tests_channel_id: C03KJ5S7KEK
+      SLACK_USER: ${{ inputs.slackMemberID }}
+      SLACK_CHANNEL: C03KJ5S7KEK
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/automation-load-tests.yml b/.github/workflows/automation-load-tests.yml
index e83f9fec847..6ec6d242726 100644
--- a/.github/workflows/automation-load-tests.yml
+++ b/.github/workflows/automation-load-tests.yml
@@ -9,7 +9,12 @@ on:
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
-        type: string 
+        type: string
+      slackMemberID:
+        description: Notifies test results (Not your @)
+        required: true
+        default: U02Q14G80TY
+        type: string         
 
 jobs:
   run-e2e-tests-workflow:
@@ -19,7 +24,8 @@ jobs:
       test_ids: 'load/automationv2_1/automationv2_1_test.go:TestLogTrigger'
       test_config_override_path: ${{ inputs.test_config_override_path }}
       chainlink_version: ${{ github.sha }}
-      slack_notification_after_tests_channel_id: C03KJ5S7KEK
+      SLACK_USER: ${{ inputs.slackMemberID }}
+      SLACK_CHANNEL: C03KJ5S7KEK
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 220c6862040..fc3065c80bf 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -121,6 +121,14 @@ on:
         required: false
         type: boolean
         default: false
+      SLACK_CHANNEL:
+        description: 'SLACK_CHANNEL env used to send Slack notifications from test code'
+        required: false
+        type: string        
+      SLACK_USER:
+        description: 'SLACK_USER env used to send Slack notifications from test code'
+        required: false
+        type: string  
     outputs:
       test_results:
         description: 'Test results from all executed tests'
@@ -181,7 +189,8 @@ env:
   TEST_LOG_LEVEL: ${{ inputs.test_log_level }}
   METRICS_COLLECTION_ID: chainlink-e2e-tests
   SLACK_API_KEY: ${{ secrets.SLACK_API_KEY }}
-  SLACK_CHANNEL: ${{ inputs.slack_notification_after_tests_channel_id || secrets.SLACK_CHANNEL }}
+  SLACK_CHANNEL: ${{ inputs.slack_notification_after_tests_channel_id || inputs.SLACK_CHANNEL || secrets.SLACK_CHANNEL }}
+  SLACK_USER: ${{ inputs.SLACK_USER }}
 
 jobs:
   validate-inputs:

From 76293feebf4215427ef7be0e90469ecd0e6ee009 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 17 Sep 2024 14:55:49 +0200
Subject: [PATCH 111/141] Fix SLACK_USER input in on-demand-ocr-soak-test.yml

---
 .github/workflows/on-demand-ocr-soak-test.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
index 32a16a69d3c..454c0806b91 100644
--- a/.github/workflows/on-demand-ocr-soak-test.yml
+++ b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -31,6 +31,10 @@ on:
         default: develop
         required: true
         type: string
+      slackMemberID:
+        description: Slack Member ID (Not your @)
+        required: true
+        default: U01A2B2C3D4        
         
 jobs:
   run-e2e-tests-workflow:
@@ -40,6 +44,7 @@ jobs:
       test_ids: ${{ inputs.testToRun}}
       test_config_override_path: ${{ inputs.test_config_override_path }}
       chainlink_version: ${{ inputs.chainlink_version }}
+      SLACK_USER: ${{ inputs.slackMemberID }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From f82450a88884a9f986c62d4a51da046f89e3ba07 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 17 Sep 2024 15:11:41 +0200
Subject: [PATCH 112/141] Do not use github.sha as default chainlink version

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index fc3065c80bf..70999e8c81b 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -179,11 +179,11 @@ on:
 env:
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
   QA_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-  DEFAULT_CHAINLINK_VERSION: ${{ inputs.chainlink_version || github.sha }}
-  DEFAULT_CHAINLINK_PLUGINS_VERSION: ${{ inputs.chainlink_version != '' && format('{0}-plugins', inputs.chainlink_version) || format('{0}-plugins', github.sha) }}
-  DEFAULT_CHAINLINK_UPGRADE_VERSION: ${{ inputs.chainlink_version || github.sha }}
+  DEFAULT_CHAINLINK_VERSION: ${{ inputs.chainlink_version }}
+  DEFAULT_CHAINLINK_PLUGINS_VERSION: ${{ inputs.chainlink_version != '' && format('{0}-plugins', inputs.chainlink_version) }}
+  DEFAULT_CHAINLINK_UPGRADE_VERSION: ${{ inputs.chainlink_version }}
   CHAINLINK_ENV_USER: ${{ github.actor }}
-  CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
+  CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref }}
   SELECTED_NETWORKS: SIMULATED
   MOD_CACHE_VERSION: 1
   TEST_LOG_LEVEL: ${{ inputs.test_log_level }}

From 83ac7d52403f3330960afdf46eb405de0c34bee8 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 17 Sep 2024 15:12:41 +0200
Subject: [PATCH 113/141] Remove chainlink_version input form automation
 workflows. Use config instead

---
 .github/workflows/automation-benchmark-tests.yml | 6 ------
 .github/workflows/automation-load-tests.yml      | 1 -
 2 files changed, 7 deletions(-)

diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml
index 1b6df7f5e4b..59bbcc02f6a 100644
--- a/.github/workflows/automation-benchmark-tests.yml
+++ b/.github/workflows/automation-benchmark-tests.yml
@@ -10,11 +10,6 @@ on:
         description: Key to run tests with custom test secrets
         required: false
         type: string 
-      chainlink_version:
-        description: Chainlink image version to use
-        default: develop
-        required: true
-        type: string
       slackMemberID:
         description: Notifies test results (Not your @)
         required: true
@@ -28,7 +23,6 @@ jobs:
     with:
       test_ids: 'benchmark/keeper_test.go:TestAutomationBenchmark'
       test_config_override_path: ${{ inputs.test_config_override_path }}
-      chainlink_version: ${{ github.sha }}
       SLACK_USER: ${{ inputs.slackMemberID }}
       SLACK_CHANNEL: C03KJ5S7KEK
     secrets:
diff --git a/.github/workflows/automation-load-tests.yml b/.github/workflows/automation-load-tests.yml
index 6ec6d242726..81bf2633f70 100644
--- a/.github/workflows/automation-load-tests.yml
+++ b/.github/workflows/automation-load-tests.yml
@@ -23,7 +23,6 @@ jobs:
     with:
       test_ids: 'load/automationv2_1/automationv2_1_test.go:TestLogTrigger'
       test_config_override_path: ${{ inputs.test_config_override_path }}
-      chainlink_version: ${{ github.sha }}
       SLACK_USER: ${{ inputs.slackMemberID }}
       SLACK_CHANNEL: C03KJ5S7KEK
     secrets:

From d5f7d717d8ef70f993a127600e4376546a273c26 Mon Sep 17 00:00:00 2001
From: joaoluisam <joao.luismoreira@smartcontract.com>
Date: Tue, 17 Sep 2024 14:19:40 +0100
Subject: [PATCH 114/141] update base sepolia test config duration to 1h

---
 integration-tests/testconfig/ocr2/overrides/base_sepolia.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml b/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
index 9ed03bf078f..aa12107ae36 100644
--- a/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
+++ b/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
@@ -6,7 +6,7 @@ chainlink_node_funding = 1
 
 [Soak.OCR2]
 [Soak.OCR2.Common]
-test_duration = "24h"
+test_duration = "1h"
 
 [Soak.OCR2.Soak]
 number_of_contracts = 2

From 60cca4303f2b7286a88b9b5477f41054dc20f94f Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 17 Sep 2024 15:23:45 +0200
Subject: [PATCH 115/141] Fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 70999e8c81b..056a32f0bfe 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -506,6 +506,7 @@ jobs:
           echo "[${{ env.TEST_CONFIG_OVERRIDE_PATH }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ env.TEST_CONFIG_OVERRIDE_PATH }})" >> $GITHUB_STEP_SUMMARY
 
       - name: Show chainlink version in summary
+        if: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}
         run: |
           echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
           echo "${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}" >> $GITHUB_STEP_SUMMARY
@@ -744,6 +745,7 @@ jobs:
           echo "[${{ env.TEST_CONFIG_OVERRIDE_PATH }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ env.TEST_CONFIG_OVERRIDE_PATH }})" >> $GITHUB_STEP_SUMMARY
 
       - name: Show chainlink version in summary
+        if: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}
         run: |
           echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
           echo "${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}" >> $GITHUB_STEP_SUMMARY                    

From 672344b582641fcbc51b63894b7b267e3d3ab660 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 17 Sep 2024 15:26:37 +0200
Subject: [PATCH 116/141] Do not upload cpu and memory profile by default

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 056a32f0bfe..84a4a9919c9 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -769,8 +769,8 @@ jobs:
           ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.get-remote-runner-test-image.outputs.remote-runner-version }}
           INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
           # We can comment these out when we have a stable soak test and aren't worried about resource consumption
-          TEST_UPLOAD_CPU_PROFILE: true
-          TEST_UPLOAD_MEM_PROFILE: true
+          TEST_UPLOAD_CPU_PROFILE: ${{ matrix.tests.test_env_vars.TEST_UPLOAD_CPU_PROFILE || 'false' }}
+          TEST_UPLOAD_MEM_PROFILE: ${{ matrix.tests.test_env_vars.TEST_UPLOAD_MEM_PROFILE || 'false' }}
           REF_NAME: ${{ github.head_ref || github.ref_name }}
           E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}
           E2E_TEST_CHAINLINK_UPGRADE_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION }}

From a5ecc1ad6b60966530ca6379411decac330dd58d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 17 Sep 2024 15:36:48 +0200
Subject: [PATCH 117/141] Do not set E2E_TEST_PYROSCOPE_ENABLED. Get it from
 toml config instead

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 84a4a9919c9..dbebbac69b6 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -571,7 +571,6 @@ jobs:
           E2E_TEST_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
           E2E_TEST_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }}
           E2E_TEST_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }}
-          E2E_TEST_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }}
         with:
           test_command_to_run: ${{ matrix.tests.test_cmd }} ${{ matrix.tests.test_cmd_opts || '2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false' }} 
           test_download_vendor_packages_command: cd $(dirname ${{ matrix.tests.path }}) && go mod download
@@ -788,7 +787,6 @@ jobs:
           E2E_TEST_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
           E2E_TEST_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }}
           E2E_TEST_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }}
-          E2E_TEST_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }}
           DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
         with:
           test_command_to_run: ${{ matrix.tests.test_cmd }} ${{ matrix.tests.test_cmd_opts || '2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false' }} 

From 36b5a7a610f91c5d51c9149dfbc459180eaf21ca Mon Sep 17 00:00:00 2001
From: joaoluisam <joao.luismoreira@smartcontract.com>
Date: Tue, 17 Sep 2024 14:43:05 +0100
Subject: [PATCH 118/141] reverting base sepolia duration to 24h

---
 integration-tests/testconfig/ocr2/overrides/base_sepolia.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml b/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
index aa12107ae36..9ed03bf078f 100644
--- a/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
+++ b/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
@@ -6,7 +6,7 @@ chainlink_node_funding = 1
 
 [Soak.OCR2]
 [Soak.OCR2.Common]
-test_duration = "1h"
+test_duration = "24h"
 
 [Soak.OCR2.Soak]
 number_of_contracts = 2

From 58f5ae6ad549da7ab50ac100fe6126f1450a776e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 17 Sep 2024 16:13:49 +0200
Subject: [PATCH 119/141] Fix TEST_UPLOAD_CPU_PROFILE and
 TEST_UPLOAD_MEM_PROFILE

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index dbebbac69b6..745ebe4bc56 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -768,8 +768,8 @@ jobs:
           ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.get-remote-runner-test-image.outputs.remote-runner-version }}
           INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
           # We can comment these out when we have a stable soak test and aren't worried about resource consumption
-          TEST_UPLOAD_CPU_PROFILE: ${{ matrix.tests.test_env_vars.TEST_UPLOAD_CPU_PROFILE || 'false' }}
-          TEST_UPLOAD_MEM_PROFILE: ${{ matrix.tests.test_env_vars.TEST_UPLOAD_MEM_PROFILE || 'false' }}
+          TEST_UPLOAD_CPU_PROFILE: ${{ matrix.tests.test_env_vars.TEST_UPLOAD_CPU_PROFILE }}
+          TEST_UPLOAD_MEM_PROFILE: ${{ matrix.tests.test_env_vars.TEST_UPLOAD_MEM_PROFILE }}
           REF_NAME: ${{ github.head_ref || github.ref_name }}
           E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}
           E2E_TEST_CHAINLINK_UPGRADE_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION }}

From 2549c351d37c5dd9cfc23a2d3761af854a2860a1 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 18 Sep 2024 09:12:01 +0200
Subject: [PATCH 120/141] debug TEST_UPLOAD_CPU_PROFILE

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 745ebe4bc56..b567bd3a466 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -758,6 +758,12 @@ jobs:
       - name: Show test configuration in logs
         run: echo '${{ toJson(matrix.tests) }}' | jq .
 
+      - name: Show TEST_UPLOAD_CPU_PROFILE
+        run: echo '${{ matrix.tests.test_env_vars.TEST_UPLOAD_CPU_PROFILE }}'
+
+      - name: Show TEST_UPLOAD_MEM_PROFILE
+        run: echo '${{ matrix.tests.test_env_vars.TEST_UPLOAD_MEM_PROFILE }}'
+
       - name: Run tests
         id: run_tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@4f377a6b1cc07f0eca82745782736b4908a1da30 # v2.3.32

From 2b2fb180dbf936bf8549fb7d40e2084b83e5820b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 18 Sep 2024 09:22:39 +0200
Subject: [PATCH 121/141] Read config from env var first then override with
 TOML

---
 integration-tests/testconfig/testconfig.go | 24 +++++++++++-----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go
index fb692c56a75..2ec0a4c66c6 100644
--- a/integration-tests/testconfig/testconfig.go
+++ b/integration-tests/testconfig/testconfig.go
@@ -312,6 +312,18 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error)
 	testConfig := TestConfig{}
 	testConfig.ConfigurationNames = configurationNames
 
+	logger.Info().Msg("Setting env vars from testsecrets dot-env files")
+	err := ctf_config.LoadSecretEnvsFromFiles()
+	if err != nil {
+		return TestConfig{}, errors.Wrapf(err, "error reading test config values from ~/.testsecrets file")
+	}
+
+	logger.Info().Msg("Reading config values from existing env vars")
+	err = testConfig.ReadFromEnvVar()
+	if err != nil {
+		return TestConfig{}, errors.Wrapf(err, "error reading test config values from env vars")
+	}
+
 	logger.Debug().Msgf("Will apply configurations named '%s' if they are found in any of the configs", strings.Join(configurationNames, ","))
 
 	// read embedded configs is build tag "embed" is set
@@ -365,18 +377,6 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error)
 		}
 	}
 
-	logger.Info().Msg("Setting env vars from testsecrets dot-env files")
-	err := ctf_config.LoadSecretEnvsFromFiles()
-	if err != nil {
-		return TestConfig{}, errors.Wrapf(err, "error reading test config values from ~/.testsecrets file")
-	}
-
-	logger.Info().Msg("Reading config values from existing env vars")
-	err = testConfig.ReadFromEnvVar()
-	if err != nil {
-		return TestConfig{}, errors.Wrapf(err, "error reading test config values from env vars")
-	}
-
 	logger.Info().Msgf("Overriding config from %s env var", Base64OverrideEnvVarName)
 	configEncoded, isSet := os.LookupEnv(Base64OverrideEnvVarName)
 	if isSet && configEncoded != "" {

From e7050be57403df029da13389176668851163616d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 18 Sep 2024 09:25:43 +0200
Subject: [PATCH 122/141] remove debug info

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index b567bd3a466..745ebe4bc56 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -758,12 +758,6 @@ jobs:
       - name: Show test configuration in logs
         run: echo '${{ toJson(matrix.tests) }}' | jq .
 
-      - name: Show TEST_UPLOAD_CPU_PROFILE
-        run: echo '${{ matrix.tests.test_env_vars.TEST_UPLOAD_CPU_PROFILE }}'
-
-      - name: Show TEST_UPLOAD_MEM_PROFILE
-        run: echo '${{ matrix.tests.test_env_vars.TEST_UPLOAD_MEM_PROFILE }}'
-
       - name: Run tests
         id: run_tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@4f377a6b1cc07f0eca82745782736b4908a1da30 # v2.3.32

From e1bc8704e9cdbc0d1352502a0dee0d1bde8cf8f0 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 18 Sep 2024 09:34:04 +0200
Subject: [PATCH 123/141] Include tag and sha in workflow slack notification

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 745ebe4bc56..87edcb7381c 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -885,7 +885,7 @@ jobs:
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Build Details>"
+                        "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|tag: ${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|sha: ${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Details>"
                       }
                     }
                   ]

From f3b60d5466a787c7d5949897ea5e353f9ff73ec3 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 18 Sep 2024 10:54:22 +0200
Subject: [PATCH 124/141] Fix notification

Do not use link for tag because it may not exist. Use short sha.
---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 87edcb7381c..b8c4f2c3e2b 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -860,6 +860,11 @@ jobs:
             echo "results=[]" >> $GITHUB_OUTPUT
           fi
 
+      - name: Set short SHA
+        id: set_short_sha
+        shell: bash
+        run: echo "short_sha=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
+
       - name: Send Slack notification
         uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
         if: ${{ inputs.slack_notification_after_tests == 'true' || inputs.slack_notification_after_tests == 'always' || (inputs.slack_notification_after_tests == 'on_failure' && contains(join(needs.*.result, ','), 'failure')) }}
@@ -885,7 +890,7 @@ jobs:
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|tag: ${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|sha: ${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Details>"
+                        "text": "${{ github.ref_name }} | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ steps.set_short_sha.outputs.short_sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Details>"
                       }
                     }
                   ]

From 9a5d891220d26db11959f393338a0e11d33b8fe7 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 18 Sep 2024 10:54:46 +0200
Subject: [PATCH 125/141] Revert "Read config from env var first then override
 with TOML"

This reverts commit 2b2fb180dbf936bf8549fb7d40e2084b83e5820b.
---
 integration-tests/testconfig/testconfig.go | 24 +++++++++++-----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go
index 2ec0a4c66c6..fb692c56a75 100644
--- a/integration-tests/testconfig/testconfig.go
+++ b/integration-tests/testconfig/testconfig.go
@@ -312,18 +312,6 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error)
 	testConfig := TestConfig{}
 	testConfig.ConfigurationNames = configurationNames
 
-	logger.Info().Msg("Setting env vars from testsecrets dot-env files")
-	err := ctf_config.LoadSecretEnvsFromFiles()
-	if err != nil {
-		return TestConfig{}, errors.Wrapf(err, "error reading test config values from ~/.testsecrets file")
-	}
-
-	logger.Info().Msg("Reading config values from existing env vars")
-	err = testConfig.ReadFromEnvVar()
-	if err != nil {
-		return TestConfig{}, errors.Wrapf(err, "error reading test config values from env vars")
-	}
-
 	logger.Debug().Msgf("Will apply configurations named '%s' if they are found in any of the configs", strings.Join(configurationNames, ","))
 
 	// read embedded configs is build tag "embed" is set
@@ -377,6 +365,18 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error)
 		}
 	}
 
+	logger.Info().Msg("Setting env vars from testsecrets dot-env files")
+	err := ctf_config.LoadSecretEnvsFromFiles()
+	if err != nil {
+		return TestConfig{}, errors.Wrapf(err, "error reading test config values from ~/.testsecrets file")
+	}
+
+	logger.Info().Msg("Reading config values from existing env vars")
+	err = testConfig.ReadFromEnvVar()
+	if err != nil {
+		return TestConfig{}, errors.Wrapf(err, "error reading test config values from env vars")
+	}
+
 	logger.Info().Msgf("Overriding config from %s env var", Base64OverrideEnvVarName)
 	configEncoded, isSet := os.LookupEnv(Base64OverrideEnvVarName)
 	if isSet && configEncoded != "" {

From a31918280a2bd503ec91fbbca2d6b01a22144041 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 18 Sep 2024 15:44:18 +0200
Subject: [PATCH 126/141] Fix and bump ctf

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 6 ++----
 integration-tests/go.mod                              | 2 +-
 integration-tests/go.sum                              | 4 ++--
 integration-tests/load/go.mod                         | 2 +-
 integration-tests/load/go.sum                         | 4 ++--
 integration-tests/testconfig/default.toml             | 5 +++++
 6 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index b8c4f2c3e2b..45232c3e46e 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -565,8 +565,7 @@ jobs:
           E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
           E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
           E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
-          E2E_TEST_GRAFANA_DASHBOARD_URL: ${{ matrix.tests.test_env_vars.E2E_TEST_GRAFANA_DASHBOARD_URL || '/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs' }} 
+          E2E_TEST_GRAFANA_DASHBOARD_URL: ${{ matrix.tests.test_env_vars.E2E_TEST_GRAFANA_DASHBOARD_URL }}
           E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
           E2E_TEST_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
           E2E_TEST_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }}
@@ -781,8 +780,7 @@ jobs:
           E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
           E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
           E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
-          E2E_TEST_GRAFANA_DASHBOARD_URL: ${{ matrix.tests.test_env_vars.E2E_TEST_GRAFANA_DASHBOARD_URL || '/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs' }} 
+          E2E_TEST_GRAFANA_DASHBOARD_URL: ${{ matrix.tests.test_env_vars.E2E_TEST_GRAFANA_DASHBOARD_URL }}
           E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
           E2E_TEST_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
           E2E_TEST_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }}
diff --git a/integration-tests/go.mod b/integration-tests/go.mod
index 40c8964c031..160f682a23d 100644
--- a/integration-tests/go.mod
+++ b/integration-tests/go.mod
@@ -41,7 +41,7 @@ require (
 	github.com/smartcontractkit/chainlink-ccip v0.0.0-20240916150615-85b8aa5fa7e6
 	github.com/smartcontractkit/chainlink-common v0.2.2-0.20240913161926-ce5d667907ce
 	github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0
-	github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5
+	github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09
 	github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0
 	github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1
 	github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0
diff --git a/integration-tests/go.sum b/integration-tests/go.sum
index db4c6ed040d..af7432cf895 100644
--- a/integration-tests/go.sum
+++ b/integration-tests/go.sum
@@ -1439,8 +1439,8 @@ github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.202409
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240911194142-506bc469d8ae/go.mod h1:ec/a20UZ7YRK4oxJcnTBFzp1+DBcJcwqEaerUMsktMs=
 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0 h1:mgjBQIEy+3V3G6K8e+6by3xndgsXdYYsdy+7kzQZwSk=
 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0/go.mod h1:pdIxrooP5CFGmC0p5NTOBiZAFtMw+5pTT4de5GY3ywA=
-github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 h1:Owb1MQZn0NZHwtZAnPZE6TVoTx6xLrfPaUdeOYswE9M=
-github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5/go.mod h1:hS4yNF94C1lkS9gvtFXW8Km8K9NzGeR20aNfkqo5qbE=
+github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09 h1:oHTXpi+BX0of/jDqQg66/3uN2MKPR51CbaubO+Y/uew=
+github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09/go.mod h1:7R5wGWWJi0dr5Y5cXbLQ4vSeIj0ElvhBaymcfvqqUmo=
 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg=
 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM=
 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs=
diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod
index 00c6453814f..f800ed65b6d 100644
--- a/integration-tests/load/go.mod
+++ b/integration-tests/load/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.2.2-0.20240913161926-ce5d667907ce
-	github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5
+	github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09
 	github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1
 	github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum
index bf4ca8c6ba6..3d477ba5cb1 100644
--- a/integration-tests/load/go.sum
+++ b/integration-tests/load/go.sum
@@ -1413,8 +1413,8 @@ github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.202409
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240911194142-506bc469d8ae/go.mod h1:ec/a20UZ7YRK4oxJcnTBFzp1+DBcJcwqEaerUMsktMs=
 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0 h1:mgjBQIEy+3V3G6K8e+6by3xndgsXdYYsdy+7kzQZwSk=
 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0/go.mod h1:pdIxrooP5CFGmC0p5NTOBiZAFtMw+5pTT4de5GY3ywA=
-github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 h1:Owb1MQZn0NZHwtZAnPZE6TVoTx6xLrfPaUdeOYswE9M=
-github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5/go.mod h1:hS4yNF94C1lkS9gvtFXW8Km8K9NzGeR20aNfkqo5qbE=
+github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09 h1:oHTXpi+BX0of/jDqQg66/3uN2MKPR51CbaubO+Y/uew=
+github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09/go.mod h1:7R5wGWWJi0dr5Y5cXbLQ4vSeIj0ElvhBaymcfvqqUmo=
 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg=
 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM=
 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs=
diff --git a/integration-tests/testconfig/default.toml b/integration-tests/testconfig/default.toml
index 48a5cd35b85..3370b0fd4e4 100644
--- a/integration-tests/testconfig/default.toml
+++ b/integration-tests/testconfig/default.toml
@@ -2,6 +2,11 @@
 # set to true to flush logs to selected target regardless of test result; otherwise logs are only flushed if test failed
 test_log_collect = false
 
+[Logging.Grafana]
+base_url="https://grafana.ops.prod.cldev.sh"
+base_url_github_ci="http://localhost:8080/primary"
+dashboard_url="/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+
 [Logging.LogStream]
 # supported targets: file, loki, in-memory. if empty no logs will be persisted
 log_targets = ["file"]

From 8bdcb34308911ff6ab5a8a99542fbec21d0a207b Mon Sep 17 00:00:00 2001
From: anirudhwarrier <12178754+anirudhwarrier@users.noreply.github.com>
Date: Wed, 18 Sep 2024 22:10:01 +0530
Subject: [PATCH 127/141] go mod tidy

---
 integration-tests/load/go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod
index 673191e145b..642762d7990 100644
--- a/integration-tests/load/go.mod
+++ b/integration-tests/load/go.mod
@@ -16,7 +16,7 @@ require (
 	github.com/rs/zerolog v1.33.0
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-common v0.2.2-0.20240916150342-36cb47701edf
-	github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5
+	github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09
 	github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1
 	github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c

From 3058c42a2658f4e27a3da0cf094c0af134c6226e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 18 Sep 2024 19:05:27 +0200
Subject: [PATCH 128/141] bump ctf

---
 integration-tests/go.mod      | 2 +-
 integration-tests/go.sum      | 4 ++--
 integration-tests/load/go.mod | 2 +-
 integration-tests/load/go.sum | 4 ++--
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/integration-tests/go.mod b/integration-tests/go.mod
index 4a1be879b9d..0dd71432ccd 100644
--- a/integration-tests/go.mod
+++ b/integration-tests/go.mod
@@ -41,7 +41,7 @@ require (
 	github.com/smartcontractkit/chainlink-ccip v0.0.0-20240917180332-5a68498d1612
 	github.com/smartcontractkit/chainlink-common v0.2.2-0.20240916150342-36cb47701edf
 	github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0
-	github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09
+	github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6
 	github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0
 	github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1
 	github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0
diff --git a/integration-tests/go.sum b/integration-tests/go.sum
index 805cd37a636..2b15c5a3204 100644
--- a/integration-tests/go.sum
+++ b/integration-tests/go.sum
@@ -1439,8 +1439,8 @@ github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.202409
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240911194142-506bc469d8ae/go.mod h1:ec/a20UZ7YRK4oxJcnTBFzp1+DBcJcwqEaerUMsktMs=
 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0 h1:mgjBQIEy+3V3G6K8e+6by3xndgsXdYYsdy+7kzQZwSk=
 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0/go.mod h1:pdIxrooP5CFGmC0p5NTOBiZAFtMw+5pTT4de5GY3ywA=
-github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09 h1:oHTXpi+BX0of/jDqQg66/3uN2MKPR51CbaubO+Y/uew=
-github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09/go.mod h1:7R5wGWWJi0dr5Y5cXbLQ4vSeIj0ElvhBaymcfvqqUmo=
+github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6 h1:Pzr5VAMdI2CjFftodGkilMTFlIjCHJ7oqWAD7aZvFeI=
+github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6/go.mod h1:7R5wGWWJi0dr5Y5cXbLQ4vSeIj0ElvhBaymcfvqqUmo=
 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg=
 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM=
 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs=
diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod
index 642762d7990..741ff752398 100644
--- a/integration-tests/load/go.mod
+++ b/integration-tests/load/go.mod
@@ -16,7 +16,7 @@ require (
 	github.com/rs/zerolog v1.33.0
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-common v0.2.2-0.20240916150342-36cb47701edf
-	github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09
+	github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6
 	github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1
 	github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum
index a3245180e53..a3a1bb53777 100644
--- a/integration-tests/load/go.sum
+++ b/integration-tests/load/go.sum
@@ -1413,8 +1413,8 @@ github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.202409
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240911194142-506bc469d8ae/go.mod h1:ec/a20UZ7YRK4oxJcnTBFzp1+DBcJcwqEaerUMsktMs=
 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0 h1:mgjBQIEy+3V3G6K8e+6by3xndgsXdYYsdy+7kzQZwSk=
 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0/go.mod h1:pdIxrooP5CFGmC0p5NTOBiZAFtMw+5pTT4de5GY3ywA=
-github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09 h1:oHTXpi+BX0of/jDqQg66/3uN2MKPR51CbaubO+Y/uew=
-github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6-0.20240918133844-005e8bc2da09/go.mod h1:7R5wGWWJi0dr5Y5cXbLQ4vSeIj0ElvhBaymcfvqqUmo=
+github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6 h1:Pzr5VAMdI2CjFftodGkilMTFlIjCHJ7oqWAD7aZvFeI=
+github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.6/go.mod h1:7R5wGWWJi0dr5Y5cXbLQ4vSeIj0ElvhBaymcfvqqUmo=
 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg=
 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM=
 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs=

From 7e6f7f4a7baf3fb4134fd1e5ce47e6de96c2d607 Mon Sep 17 00:00:00 2001
From: Anirudh Warrier <12178754+anirudhwarrier@users.noreply.github.com>
Date: Thu, 19 Sep 2024 09:23:11 +0400
Subject: [PATCH 129/141] add automation test configs (#14455)

* add automation test configs

* update grafana config

* update testid in workflow

* update testid in e2e-tests

* restructure configs

* update config

* add testType input in benchmark workflow

* add testType input in e2e reusable workflow

* Revert "add testType input in e2e reusable workflow"

This reverts commit 937607cccfbf52f579a6b2ae09ff8fae9f51cac9.

* update benchmark workflow

* update test log level for load and benchmark

---------

Co-authored-by: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
---
 .github/e2e-tests.yml                         | 18 ++++++-
 .../workflows/automation-benchmark-tests.yml  |  9 +++-
 .../testconfig/automation/automation.toml     | 13 +++++
 .../benchmark/1000Upkeeps-1h-2_1.toml         | 15 ++++++
 .../benchmark/1000Upkeeps-1h-2_3.toml         | 15 ++++++
 .../overrides/load/500Upkeeps-1x-1h.toml      | 47 +++++++++++++++++++
 .../overrides/load/50Upkeeps-1x-12h.toml      | 43 +++++++++++++++++
 .../overrides/load/50Upkeeps-1x-1h.toml       | 43 +++++++++++++++++
 .../overrides/soak/50Upkeeps-8h-2_1.toml      | 15 ++++++
 .../overrides/soak/50Upkeeps-8h-2_3.toml      | 15 ++++++
 .../testconfig/keeper/keeper.toml             |  2 +-
 11 files changed, 231 insertions(+), 4 deletions(-)
 create mode 100644 integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_1.toml
 create mode 100644 integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_3.toml
 create mode 100644 integration-tests/testconfig/automation/overrides/load/500Upkeeps-1x-1h.toml
 create mode 100644 integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-12h.toml
 create mode 100644 integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-1h.toml
 create mode 100644 integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_1.toml
 create mode 100644 integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_3.toml

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 594cb9a4392..fe178a0df68 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -490,6 +490,7 @@ runner-test-matrix:
     remote_runner_memory: 4Gi
     test_secrets_required: true
     test_env_vars:
+      TEST_LOG_LEVEL: info
       TEST_SUITE: automationv2_1
     pyroscope_env: automation-load-test
 
@@ -591,7 +592,7 @@ runner-test-matrix:
     test_env_vars:
       TEST_SUITE: chaos
 
-  - id: benchmark/automation_test.go:^TestAutomationBenchmark$
+  - id: benchmark/automation_test.go:TestAutomationBenchmark
     path: integration-tests/benchmark/automation_test.go
     test_env_type: k8s-remote-runner
     remote_runner_memory: 4Gi
@@ -601,9 +602,24 @@ runner-test-matrix:
     test_cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-benchmark-automation-nightly
     test_env_vars:
+      TEST_LOG_LEVEL: info
       TEST_SUITE: benchmark
       TEST_TYPE: benchmark
 
+  - id: soak/automation_test.go:TestAutomationBenchmark
+    path: integration-tests/benchmark/automation_test.go
+    test_env_type: k8s-remote-runner
+    remote_runner_memory: 4Gi
+    runs_on: ubuntu-latest
+    # workflows:
+    # - Nightly E2E Tests
+    test_cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 -json
+    pyroscope_env: ci-benchmark-automation-nightly
+    test_env_vars:
+      TEST_LOG_LEVEL: info
+      TEST_SUITE: benchmark
+      TEST_TYPE: soak
+
   # END: Automation tests
 
   # START: VRF tests
diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml
index 59bbcc02f6a..6e446959295 100644
--- a/.github/workflows/automation-benchmark-tests.yml
+++ b/.github/workflows/automation-benchmark-tests.yml
@@ -14,14 +14,19 @@ on:
         description: Notifies test results (Not your @)
         required: true
         default: U02Q14G80TY
-        type: string     
+        type: string
+      testType:
+        description: Type of test to run (benchmark, soak)
+        required: true
+        default: benchmark
+        type: string
 
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      test_ids: 'benchmark/keeper_test.go:TestAutomationBenchmark'
+      test_ids: '${{ inputs.testType }}/automation_test.go:TestAutomationBenchmark'
       test_config_override_path: ${{ inputs.test_config_override_path }}
       SLACK_USER: ${{ inputs.slackMemberID }}
       SLACK_CHANNEL: C03KJ5S7KEK
diff --git a/integration-tests/testconfig/automation/automation.toml b/integration-tests/testconfig/automation/automation.toml
index f07136e998e..b2e87fa34f9 100644
--- a/integration-tests/testconfig/automation/automation.toml
+++ b/integration-tests/testconfig/automation/automation.toml
@@ -2,6 +2,9 @@
 [Common]
 chainlink_node_funding = 2.0
 
+[Pyroscope]
+enabled=false
+
 [NodeConfig]
 BaseConfigTOML = """
 [Feature]
@@ -207,6 +210,10 @@ max_perform_data_size=5_000
 max_revert_data_size=5_000
 
 # load test specific overrides
+[Load.Logging.Grafana]
+base_url="https://grafana.ops.prod.cldev.sh"
+dashboard_url="/d/a4899f53-f709-430a-aec2-24f32198dcc9/chainlink-automation-v2-load-test"
+
 [Load.Seth]
 ephemeral_addresses_number = 100
 root_key_funds_buffer = 1_000_000
@@ -301,6 +308,9 @@ max_revert_data_size=5_000
 enabled=false
 
 # automation benchmark test specific overrides
+[Benchmark.Logging.Grafana]
+base_url="https://grafana.ops.prod.cldev.sh"
+dashboard_url="/d/Q8n6m1unz/chainlink-automation-benchmark-test"
 
 # will retry roughly for 1h before giving up (900 * 4s)
 [Benchmark.Automation.Resiliency]
@@ -420,6 +430,9 @@ max_perform_data_size=5_000
 max_revert_data_size=5_000
 
 # automation soak test specific overrides
+[Soak.Logging.Grafana]
+base_url="https://grafana.ops.prod.cldev.sh"
+dashboard_url="/d/Q8n6m1unz/chainlink-automation-benchmark-test"
 
 # will retry roughly for 1h before giving up (900 * 4s)
 [Soak.Automation.Resiliency]
diff --git a/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_1.toml b/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_1.toml
new file mode 100644
index 00000000000..0b704524e9d
--- /dev/null
+++ b/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_1.toml
@@ -0,0 +1,15 @@
+[ChainlinkImage]
+version="latest"
+
+[Benchmark.Automation.Benchmark]
+registry_to_test = "2_1"
+number_of_registries = 1
+number_of_nodes = 6
+number_of_upkeeps = 1000
+upkeep_gas_limit = 1500000
+check_gas_to_burn = 10000
+perform_gas_to_burn = 1000
+block_range = 3600
+block_interval = 60
+forces_single_tx_key = false
+delete_jobs_on_end = true
\ No newline at end of file
diff --git a/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_3.toml b/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_3.toml
new file mode 100644
index 00000000000..f00bb5ed47b
--- /dev/null
+++ b/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_3.toml
@@ -0,0 +1,15 @@
+[ChainlinkImage]
+version="latest"
+
+[Benchmark.Automation.Benchmark]
+registry_to_test = "2_3"
+number_of_registries = 1
+number_of_nodes = 6
+number_of_upkeeps = 1000
+upkeep_gas_limit = 1500000
+check_gas_to_burn = 10000
+perform_gas_to_burn = 1000
+block_range = 3600
+block_interval = 60
+forces_single_tx_key = false
+delete_jobs_on_end = true
\ No newline at end of file
diff --git a/integration-tests/testconfig/automation/overrides/load/500Upkeeps-1x-1h.toml b/integration-tests/testconfig/automation/overrides/load/500Upkeeps-1x-1h.toml
new file mode 100644
index 00000000000..6d58253f526
--- /dev/null
+++ b/integration-tests/testconfig/automation/overrides/load/500Upkeeps-1x-1h.toml
@@ -0,0 +1,47 @@
+[ChainlinkImage]
+version="latest"
+
+[Load.Seth]
+root_key_funds_buffer = 1_000_000
+ephemeral_addresses_number = 300
+
+[Load.Seth.nonce_manager]
+key_sync_timeout = "100s"
+
+[Load.Common]
+chainlink_node_funding = 1000
+
+[Load.Automation.AutomationConfig]
+use_log_buffer_v1=false
+
+[Load.Automation.AutomationConfig.PluginConfig.LogProviderConfig]
+block_rate=1
+log_limit=2
+
+[Load.Automation]
+[Load.Automation.General]
+number_of_nodes=6
+duration=3600
+block_time=1
+spec_type="recommended"
+chainlink_node_log_level="debug"
+use_prometheus=true
+remove_namespace = true
+
+[Load.Automation.DataStreams]
+enabled=false
+
+[[Load.Automation.Load]]
+number_of_upkeeps=500
+number_of_events = 1
+number_of_spam_matching_events = 0
+number_of_spam_non_matching_events = 0
+check_burn_amount = 0
+perform_burn_amount = 0
+upkeep_gas_limit = 1000000
+shared_trigger = false
+is_streams_lookup = false
+feeds = []
+
+[Pyroscope]
+enabled=false
\ No newline at end of file
diff --git a/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-12h.toml b/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-12h.toml
new file mode 100644
index 00000000000..8991a6eb903
--- /dev/null
+++ b/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-12h.toml
@@ -0,0 +1,43 @@
+[ChainlinkImage]
+version="latest"
+
+[Load.Seth]
+root_key_funds_buffer = 1_000_000
+
+[Load.Common]
+chainlink_node_funding = 10000
+
+[Load.Automation.AutomationConfig]
+use_log_buffer_v1=false
+
+[Load.Automation.AutomationConfig.PluginConfig.LogProviderConfig]
+block_rate=1
+log_limit=2
+
+[Load.Automation]
+[Load.Automation.General]
+number_of_nodes=6
+duration=43200
+block_time=1
+spec_type="recommended"
+chainlink_node_log_level="debug"
+use_prometheus=true
+remove_namespace = true
+
+[Load.Automation.DataStreams]
+enabled=false
+
+[[Load.Automation.Load]]
+number_of_upkeeps=50
+number_of_events = 1
+number_of_spam_matching_events = 0
+number_of_spam_non_matching_events = 0
+check_burn_amount = 0
+perform_burn_amount = 0
+upkeep_gas_limit = 1000000
+shared_trigger = false
+is_streams_lookup = false
+feeds = []
+
+[Pyroscope]
+enabled=false
\ No newline at end of file
diff --git a/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-1h.toml b/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-1h.toml
new file mode 100644
index 00000000000..a133fce6dcf
--- /dev/null
+++ b/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-1h.toml
@@ -0,0 +1,43 @@
+[ChainlinkImage]
+version="latest"
+
+[Load.Seth]
+root_key_funds_buffer = 1_000_000
+
+[Load.Common]
+chainlink_node_funding = 1000
+
+[Load.Automation.AutomationConfig]
+use_log_buffer_v1=false
+
+[Load.Automation.AutomationConfig.PluginConfig.LogProviderConfig]
+block_rate=1
+log_limit=2
+
+[Load.Automation]
+[Load.Automation.General]
+number_of_nodes=6
+duration=3600
+block_time=1
+spec_type="recommended"
+chainlink_node_log_level="debug"
+use_prometheus=true
+remove_namespace = true
+
+[Load.Automation.DataStreams]
+enabled=false
+
+[[Load.Automation.Load]]
+number_of_upkeeps=50
+number_of_events = 1
+number_of_spam_matching_events = 0
+number_of_spam_non_matching_events = 0
+check_burn_amount = 0
+perform_burn_amount = 0
+upkeep_gas_limit = 1000000
+shared_trigger = false
+is_streams_lookup = false
+feeds = []
+
+[Pyroscope]
+enabled=false
\ No newline at end of file
diff --git a/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_1.toml b/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_1.toml
new file mode 100644
index 00000000000..fda5cb6ea29
--- /dev/null
+++ b/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_1.toml
@@ -0,0 +1,15 @@
+[ChainlinkImage]
+version="latest"
+
+[Soak.Automation.Benchmark]
+registry_to_test = "2_1"
+number_of_registries = 1
+number_of_nodes = 6
+number_of_upkeeps = 50
+upkeep_gas_limit = 1500000
+check_gas_to_burn = 10000
+perform_gas_to_burn = 1000
+block_range = 28800
+block_interval = 300
+forces_single_tx_key = false
+delete_jobs_on_end = true
\ No newline at end of file
diff --git a/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_3.toml b/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_3.toml
new file mode 100644
index 00000000000..46ba1ad3e85
--- /dev/null
+++ b/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_3.toml
@@ -0,0 +1,15 @@
+[ChainlinkImage]
+version="latest"
+
+[Soak.Automation.Benchmark]
+registry_to_test = "2_3"
+number_of_registries = 1
+number_of_nodes = 6
+number_of_upkeeps = 50
+upkeep_gas_limit = 1500000
+check_gas_to_burn = 10000
+perform_gas_to_burn = 1000
+block_range = 28800
+block_interval = 300
+forces_single_tx_key = false
+delete_jobs_on_end = true
\ No newline at end of file
diff --git a/integration-tests/testconfig/keeper/keeper.toml b/integration-tests/testconfig/keeper/keeper.toml
index b4f544165a9..d483d6df493 100644
--- a/integration-tests/testconfig/keeper/keeper.toml
+++ b/integration-tests/testconfig/keeper/keeper.toml
@@ -32,4 +32,4 @@ HTTPSPort = 0
 
 [Keeper]
 TurnLookBack = 0
-"""
+"""
\ No newline at end of file

From ef4f1b486e751ffdc9edd4999d1849f6f50906f7 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Thu, 19 Sep 2024 10:13:27 +0200
Subject: [PATCH 130/141] Increase test_log_upload_retention_days from 3 to 5

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 45232c3e46e..b1a7fb8ae55 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -101,7 +101,7 @@ on:
         description: 'Number of days to retain the test log. Default is 3 days'
         required: false
         type: number
-        default: 3
+        default: 5
       test_log_level:
         description: 'Set the log level for the tests. Default is "debug"'
         required: false

From 8e47f488ccb2056b45ede09a76361b5d00889297 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Thu, 19 Sep 2024 11:49:43 +0200
Subject: [PATCH 131/141] Fix

---
 .../workflows/automation-benchmark-tests.yml  |  3 +++
 .github/workflows/automation-load-tests.yml   |  3 +++
 .../workflows/automation-nightly-tests.yml    |  3 +++
 .../workflows/automation-ondemand-tests.yml   |  3 +++
 .github/workflows/ccip-chaos-tests.yml        |  3 +++
 .github/workflows/ccip-load-tests.yml         |  3 +++
 .github/workflows/integration-chaos-tests.yml |  3 +++
 .github/workflows/integration-tests.yml       |  6 +++++
 .github/workflows/on-demand-ocr-soak-test.yml |  3 +++
 .../on-demand-vrfv2-performance-test.yml      |  3 +++
 .../workflows/on-demand-vrfv2-smoke-tests.yml |  3 +++
 .../on-demand-vrfv2plus-performance-test.yml  |  3 +++
 .../on-demand-vrfv2plus-smoke-tests.yml       |  3 +++
 .../run-e2e-tests-reusable-workflow.yml       | 23 +++++++++++--------
 .github/workflows/run-nightly-e2e-tests.yml   |  3 +++
 .github/workflows/run-selected-e2e-tests.yml  |  4 +++-
 16 files changed, 62 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml
index 6e446959295..d1af80fb91e 100644
--- a/.github/workflows/automation-benchmark-tests.yml
+++ b/.github/workflows/automation-benchmark-tests.yml
@@ -41,6 +41,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/automation-load-tests.yml b/.github/workflows/automation-load-tests.yml
index 81bf2633f70..23c053203a1 100644
--- a/.github/workflows/automation-load-tests.yml
+++ b/.github/workflows/automation-load-tests.yml
@@ -36,6 +36,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml
index 9075897fa47..06dc00b3a14 100644
--- a/.github/workflows/automation-nightly-tests.yml
+++ b/.github/workflows/automation-nightly-tests.yml
@@ -29,6 +29,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml
index ed69bd5ac05..a5b26744253 100644
--- a/.github/workflows/automation-ondemand-tests.yml
+++ b/.github/workflows/automation-ondemand-tests.yml
@@ -171,6 +171,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/ccip-chaos-tests.yml b/.github/workflows/ccip-chaos-tests.yml
index 1d37551d9fe..3a6cae796d2 100644
--- a/.github/workflows/ccip-chaos-tests.yml
+++ b/.github/workflows/ccip-chaos-tests.yml
@@ -34,6 +34,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml
index e8101223ba8..235b9b0f67e 100644
--- a/.github/workflows/ccip-load-tests.yml
+++ b/.github/workflows/ccip-load-tests.yml
@@ -51,6 +51,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/integration-chaos-tests.yml b/.github/workflows/integration-chaos-tests.yml
index 1f7e6f40fb1..c9da7d84381 100644
--- a/.github/workflows/integration-chaos-tests.yml
+++ b/.github/workflows/integration-chaos-tests.yml
@@ -27,6 +27,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index a8884c306f9..cf5789bcc04 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -262,6 +262,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
@@ -298,6 +301,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
index 454c0806b91..cdb0b5da01a 100644
--- a/.github/workflows/on-demand-ocr-soak-test.yml
+++ b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -56,6 +56,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml
index ad8640ccfac..d5eefcc348f 100644
--- a/.github/workflows/on-demand-vrfv2-performance-test.yml
+++ b/.github/workflows/on-demand-vrfv2-performance-test.yml
@@ -85,6 +85,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/on-demand-vrfv2-smoke-tests.yml b/.github/workflows/on-demand-vrfv2-smoke-tests.yml
index ede71267b47..ea42a9014da 100644
--- a/.github/workflows/on-demand-vrfv2-smoke-tests.yml
+++ b/.github/workflows/on-demand-vrfv2-smoke-tests.yml
@@ -88,6 +88,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
index 0744a93b26e..f026086fff7 100644
--- a/.github/workflows/on-demand-vrfv2plus-performance-test.yml
+++ b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
@@ -85,6 +85,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
index 5330edf294b..e1821336c63 100644
--- a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
+++ b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
@@ -88,6 +88,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index b1a7fb8ae55..a5194dfaa80 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -148,6 +148,12 @@ on:
         required: true
       QA_KUBECONFIG:
         required: true
+      LOKI_TENANT_ID:
+        required: true
+      LOKI_URL:
+        required: true
+      LOKI_BASIC_AUTH:
+        required: true        
       GRAFANA_INTERNAL_TENANT_ID:
         required: true
       GRAFANA_INTERNAL_BASIC_AUTH:
@@ -184,7 +190,6 @@ env:
   DEFAULT_CHAINLINK_UPGRADE_VERSION: ${{ inputs.chainlink_version }}
   CHAINLINK_ENV_USER: ${{ github.actor }}
   CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref }}
-  SELECTED_NETWORKS: SIMULATED
   MOD_CACHE_VERSION: 1
   TEST_LOG_LEVEL: ${{ inputs.test_log_level }}
   METRICS_COLLECTION_ID: chainlink-e2e-tests
@@ -558,13 +563,13 @@ jobs:
           E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}
           E2E_TEST_CHAINLINK_UPGRADE_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION }}
           E2E_TEST_CHAINLINK_POSTGRES_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_POSTGRES_VERSION }}
-          E2E_TEST_SELECTED_NETWORK: ${{ matrix.tests.test_env_vars.E2E_TEST_SELECTED_NETWORK || env.SELECTED_NETWORKS }}
+          E2E_TEST_SELECTED_NETWORK: ${{ matrix.tests.test_env_vars.E2E_TEST_SELECTED_NETWORK }}
           E2E_TEST_LOGGING_RUN_ID: ${{ github.run_id }}
           E2E_TEST_LOG_STREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
           E2E_TEST_LOG_COLLECT: ${{ vars.TEST_LOG_COLLECT }}
-          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+          E2E_TEST_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }}
+          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
           E2E_TEST_GRAFANA_DASHBOARD_URL: ${{ matrix.tests.test_env_vars.E2E_TEST_GRAFANA_DASHBOARD_URL }}
           E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
           E2E_TEST_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
@@ -773,13 +778,13 @@ jobs:
           E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}
           E2E_TEST_CHAINLINK_UPGRADE_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION }}
           E2E_TEST_CHAINLINK_POSTGRES_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_POSTGRES_VERSION }}
-          E2E_TEST_SELECTED_NETWORK: ${{ matrix.tests.test_env_vars.E2E_TEST_SELECTED_NETWORK || env.SELECTED_NETWORKS }}
+          E2E_TEST_SELECTED_NETWORK: ${{ matrix.tests.test_env_vars.E2E_TEST_SELECTED_NETWORK }}
           E2E_TEST_LOGGING_RUN_ID: ${{ github.run_id }}
           E2E_TEST_LOG_STREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
           E2E_TEST_LOG_COLLECT: ${{ vars.TEST_LOG_COLLECT }}
-          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          E2E_TEST_LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+          E2E_TEST_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }}
+          E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
           E2E_TEST_GRAFANA_DASHBOARD_URL: ${{ matrix.tests.test_env_vars.E2E_TEST_GRAFANA_DASHBOARD_URL }}
           E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
           E2E_TEST_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml
index 6d7056ed04d..151217180b0 100644
--- a/.github/workflows/run-nightly-e2e-tests.yml
+++ b/.github/workflows/run-nightly-e2e-tests.yml
@@ -27,6 +27,9 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index c204be6a564..2ff3fbb979d 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -59,10 +59,12 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
       AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
       TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
-

From 16efbab1c7bc18a7785dbaee1b6a44be438aaf96 Mon Sep 17 00:00:00 2001
From: Adam Hamrick <adam.hamrick@smartcontract.com>
Date: Thu, 19 Sep 2024 08:41:39 -0400
Subject: [PATCH 132/141] Debug CCIP Load Test in E2E Workflow Conversion
 (#14479)

Fixes CCIP load tests
---
 .github/e2e-tests.yml                                 | 8 ++++----
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 5 +++--
 integration-tests/scripts/buildTests                  | 1 +
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index fe178a0df68..5d7d2537f33 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -1047,9 +1047,9 @@ runner-test-matrix:
 
   - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPS
     path: integration-tests/ccip-tests/load/ccip_test.go
-    test_env_type: docker
+    test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ccip-tests/load && go test -test.run TestLoadCCIPStableRPS -timeout 70m -count=1 -test.parallel=1 -json
+    test_cmd: cd integration-tests/ccip-tests/load && DETACH_RUNNER=false go test -test.run ^TestLoadCCIPStableRPS$ -timeout 70m -count=1 -test.parallel=1 -json
     test_env_vars:
       E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
     workflows:
@@ -1061,9 +1061,9 @@ runner-test-matrix:
   #
   # - id: integration-tests/ccip-tests/load/ccip_test.go:TestLoadCCIPStableRPSAfterARMCurseAndUncurse
   #   path: integration-tests/ccip-tests/load/ccip_test.go
-  #   test_env_type: docker
+  #   test_env_type: k8s-remote-runner
   #   runs_on: ubuntu-latest
-  #   test_cmd: cd integration-tests/ccip-tests/load && go test -test.run $TestLoadCCIPStableRPSAfterARMCurseAndUncurse$ -timeout 70m -count=1 -test.parallel=1 -json
+  #   test_cmd: cd integration-tests/ccip-tests/load && DETACH_RUNNER=false go test -test.run $TestLoadCCIPStableRPSAfterARMCurseAndUncurse$ -timeout 70m -count=1 -test.parallel=1 -json
   #   test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/load-with-arm-curse-uncurse.toml
   #   test_env_vars:
   #     E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index a5194dfaa80..fc0c53215c4 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -767,6 +767,7 @@ jobs:
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@4f377a6b1cc07f0eca82745782736b4908a1da30 # v2.3.32
         env:
           DETACH_RUNNER: true
+          TEST_SUITE: ${{ inputs.test_image_suites }}
           RR_MEM: ${{ matrix.tests.remote_runner_memory }}
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
           ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.get-remote-runner-test-image.outputs.remote-runner-version }}
@@ -880,13 +881,13 @@ jobs:
             {
               "attachments": [
                 {
-                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || '2E7D32' }}",
+                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || contains(join(needs.*.result, ','), 'skipped') && '#FFA000' || '2E7D32' }}",
                   "blocks": [
                     {
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "${{ inputs.slack_notification_after_tests_name }} - ${{ contains(join(needs.*.result, ','), 'failure') && 'Failed :x:' || contains(join(needs.*.result, ','), 'cancelled') && 'Cancelled :warning:' || 'Passed :white_check_mark:' }}"
+                        "text": "${{ inputs.slack_notification_after_tests_name }} - ${{ contains(join(needs.*.result, ','), 'failure') && 'Failed :x:' || contains(join(needs.*.result, ','), 'cancelled') && 'Cancelled :warning:' || contains(join(needs.*.result, ','), 'skipped') && 'Skipped :warning:' || 'Passed :white_check_mark:' }}"
                       }
                     },
                     {
diff --git a/integration-tests/scripts/buildTests b/integration-tests/scripts/buildTests
index 04e9cea94b3..b5e8fc3080e 100755
--- a/integration-tests/scripts/buildTests
+++ b/integration-tests/scripts/buildTests
@@ -28,6 +28,7 @@ do
     elif [ "$x" = "ccip-load" ]; then
       echo "Changing directory and executing go test -c ./... for 'ccip-load' package"
       pushd "./ccip-tests/load" && go test -c -tags embed -o ../ ./...
+      mv ../load.test ../ccip-load.test # rename the binary to match the suite name
       popd
     else
       go test -c -tags embed ./"${x}"

From af3577dc23873d0a483eed23ea0463cb9bd8d827 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Thu, 19 Sep 2024 16:19:40 +0200
Subject: [PATCH 133/141] Run selected e2e tests in merge queue

---
 .github/e2e-tests.yml                   |  3 +-
 .github/workflows/integration-tests.yml | 68 +++++++++++++++++++++----
 2 files changed, 60 insertions(+), 11 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 5d7d2537f33..22f2826788e 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -338,6 +338,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestUpdateCheckData$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated  
@@ -357,7 +358,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperBasicSmoke$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated  
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index cf5789bcc04..bee79ae6cb8 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -232,7 +232,7 @@ jobs:
           AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           dep_evm_sha: ${{ inputs.evm-ref }}
 
-  run-core-e2e-tests-workflow:
+  run-core-e2e-tests-in-pr:
     name: Run Core E2E Tests
     permissions:
       actions: read
@@ -271,7 +271,51 @@ jobs:
       AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
 
-  run-ccip-e2e-tests-workflow:
+  run-core-e2e-tests-in-merge-queue:
+    name: Run Core E2E Tests In Merge Queue
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    needs: [build-chainlink, changes]
+    # Enable when done debugging
+    # if: ${{ github.event_name == 'merge_group' }} 
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      workflow_name: Run Core E2E Tests In Merge Queue
+      chainlink_version: ${{ inputs.evm-ref || github.sha }}
+      chainlink_upgrade_version: ${{ github.sha }}
+      test_workflow: Merge Queue E2E Core Tests
+      upload_cl_node_coverage_artifact: true
+      upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_
+      enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }}
+      # Notify Test Tooling team in slack when merge queue tests fail
+      slack_notification_after_tests: on_failure
+      slack_notification_after_tests_channel_id: "#team-test-tooling-internal"
+      slack_notification_after_tests_name: Core E2E Tests In Merge Queue
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
+              
+  run-ccip-e2e-tests-in-pr:
     name: Run CCIP E2E Tests
     permissions:
       actions: read
@@ -314,12 +358,12 @@ jobs:
     if: always()
     name: ETH Smoke Tests
     runs-on: ubuntu-latest
-    needs: [lint-integration-tests, run-core-e2e-tests-workflow, run-ccip-e2e-tests-workflow]
+    needs: [lint-integration-tests, run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue]
     steps:
       - name: Check Core test results
         id: check_core_results
         run: |
-          results='${{ needs.run-core-e2e-tests-workflow.outputs.test_results }}'
+          results='${{ needs.run-core-e2e-tests-in-pr.outputs.test_results }}'
           echo "Core test results:"
           echo "$results" | jq .
           
@@ -329,8 +373,8 @@ jobs:
       - name: Check CCIP test results
         id: check_ccip_results
         run: |
-          if [[ '${{ needs.run-ccip-e2e-tests-workflow.result }}' != 'skipped' ]]; then
-            results='${{ needs.run-ccip-e2e-tests-workflow.outputs.test_results }}'
+          if [[ '${{ needs.run-ccip-e2e-tests-in-pr.result }}' != 'skipped' ]]; then
+            results='${{ needs.run-ccip-e2e-tests-in-pr.outputs.test_results }}'
             echo "CCIP test results:"
             echo "$results" | jq .
           else
@@ -346,10 +390,14 @@ jobs:
           channel-id: "#team-test-tooling-internal"
           slack-message: ":x: :mild-panic-intensifies: Node Migration Tests Failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}\n${{ format('Notifying <!subteam^{0}|{0}>', secrets.GUARDIAN_SLACK_NOTIFICATION_HANDLE) }}"
 
-      - name: Fail the job if core tests not successful
-        if: always() && needs.run-core-e2e-tests-workflow.result == 'failure'
+      - name: Fail the job if core tests in PR not successful
+        if: always() && needs.run-core-e2e-tests-in-pr.result == 'failure'
         run: exit 1
 
+      - name: Fail the job if core tests in merge queue not successful
+        if: always() && needs.run-core-e2e-tests-in-merge-queue.result == 'failure'
+        run: exit 1        
+
       - name: Fail the job if lint not successful
         if: always() && needs.lint-integration-tests.result == 'failure'
         run: exit 1
@@ -357,7 +405,7 @@ jobs:
   cleanup:
     name: Clean up integration environment deployments
     if: always()
-    needs: [run-core-e2e-tests-workflow, run-ccip-e2e-tests-workflow]
+    needs: [run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
@@ -389,7 +437,7 @@ jobs:
   show-chainlink-node-coverage:
     name: Show Chainlink Node Go Coverage
     if: always()
-    needs: [run-core-e2e-tests-workflow, run-ccip-e2e-tests-workflow]
+    needs: [run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the repo

From a8e8bdf541e68d33496c2235e65a232c2424d2e7 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Thu, 19 Sep 2024 16:46:32 +0200
Subject: [PATCH 134/141] Run selected e2e tests in merge queue part 2

---
 .github/e2e-tests.yml                   | 73 +++++++++++++++++++++----
 .github/workflows/integration-tests.yml | 52 +++++++++++++++---
 2 files changed, 106 insertions(+), 19 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 22f2826788e..d5c257de8fd 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -16,6 +16,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/ocr_test.go -timeout 30m -count=1 -test.parallel=2 -json
     pyroscope_env: ci-smoke-ocr-evm-simulated
@@ -128,6 +129,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/forwarder_ocr_test.go -timeout 30m -count=1 -test.parallel=2 -json
     pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated
@@ -138,6 +140,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/forwarders_ocr2_test.go -timeout 30m -count=1 -test.parallel=2 -json
     pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated
@@ -148,6 +151,7 @@ runner-test-matrix:
     runs_on: ubuntu22.04-16cores-64GB
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -timeout 30m -count=1 -test.parallel=6 -json
     pyroscope_env: ci-smoke-ocr2-evm-simulated
@@ -160,6 +164,7 @@ runner-test-matrix:
     runs_on: ubuntu22.04-16cores-64GB
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -timeout 30m -count=1 -test.parallel=6 -json
     pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated
@@ -188,6 +193,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_0|TestAutomationBasic/registry_2_1_conditional|TestAutomationBasic/registry_2_1_logtrigger$" -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated
@@ -198,6 +204,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_1_with_mercury_v02|TestAutomationBasic/registry_2_1_with_mercury_v03|TestAutomationBasic/registry_2_1_with_logtrigger_and_mercury_v02$" -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated
@@ -208,6 +215,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_2_conditional|TestAutomationBasic/registry_2_2_logtrigger|TestAutomationBasic/registry_2_2_with_mercury_v02$" -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated
@@ -218,6 +226,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_2_with_mercury_v03|TestAutomationBasic/registry_2_2_with_logtrigger_and_mercury_v02$" -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated
@@ -228,6 +237,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_3_conditional_native|TestAutomationBasic/registry_2_3_conditional_link$" -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated
@@ -238,6 +248,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_3_logtrigger_native|TestAutomationBasic/registry_2_3_logtrigger_link$" -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated
@@ -248,6 +259,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_3_with_mercury_v03_link|TestAutomationBasic/registry_2_3_with_logtrigger_and_mercury_v02_link$" -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated                
@@ -258,6 +270,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestSetUpkeepTriggerConfig$ -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated  
@@ -268,6 +281,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationAddFunds$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated  
@@ -278,6 +292,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationPauseUnPause$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated  
@@ -288,6 +303,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationRegisterUpkeep$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated  
@@ -298,6 +314,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationPauseRegistry$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated  
@@ -308,6 +325,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationKeeperNodesDown$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated  
@@ -318,6 +336,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationPerformSimulation$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated  
@@ -328,6 +347,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationCheckPerformGasLimit$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated  
@@ -349,6 +369,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestSetOffchainConfigWithMaxGasPrice$ -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-automation-evm-simulated  
@@ -368,7 +389,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperBlockCountPerTurn$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated  
@@ -378,7 +399,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperSimulation$ -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated  
@@ -388,7 +409,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperCheckPerformGasLimit$ -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated  
@@ -398,7 +419,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperRegisterUpkeep$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated  
@@ -408,7 +429,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperAddFunds$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated  
@@ -418,7 +439,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperRemove$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated     
@@ -428,7 +449,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperPauseRegistry$ -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated  
@@ -438,7 +459,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperMigrateRegistry$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated  
@@ -448,7 +469,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperNodeDown$ -test.parallel=3 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated  
@@ -458,7 +479,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperPauseUnPauseUpkeep$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated         
@@ -468,7 +489,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperUpdateCheckData$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated         
@@ -478,7 +499,7 @@ runner-test-matrix:
     test_env_type: docker
     runs_on: ubuntu-latest
     workflows:
-      - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperJobReplacement$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-keeper-evm-simulated                           
@@ -688,6 +709,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/vrf_test.go -timeout 30m -count=1 -test.parallel=2 -json
     pyroscope_env: ci-smoke-vrf-evm-simulated
@@ -698,6 +720,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/vrfv2_test.go -timeout 30m -count=1 -test.parallel=6 -json
     pyroscope_env: ci-smoke-vrf2-evm-simulated
@@ -708,6 +731,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/vrfv2plus_test.go -timeout 30m -count=1 -test.parallel=9 -json
     pyroscope_env: ci-smoke-vrf2plus-evm-simulated
@@ -740,6 +764,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerFewFiltersFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-log_poller-evm-simulated
@@ -750,6 +775,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerFewFiltersFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-log_poller-evm-simulated
@@ -760,6 +786,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-log_poller-evm-simulated
@@ -770,6 +797,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-log_poller-evm-simulated
@@ -780,6 +808,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosPostgresFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-log_poller-evm-simulated
@@ -790,6 +819,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosPostgresFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-log_poller-evm-simulated
@@ -800,6 +830,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerReplayFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-log_poller-evm-simulated
@@ -810,6 +841,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerReplayFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-log_poller-evm-simulated
@@ -824,6 +856,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/runlog_test.go -timeout 30m -test.parallel=2 -count=1 -json
     pyroscope_env: ci-smoke-runlog-evm-simulated
@@ -834,6 +867,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/cron_test.go -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-cron-evm-simulated
@@ -844,6 +878,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/flux_test.go -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-flux-evm-simulated    
@@ -854,6 +889,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/reorg_above_finality_test.go -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-reorg-above-finality-evm-simulated
@@ -864,6 +900,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/migration && go test upgrade_version_test.go -timeout 30m -count=1 -test.parallel=2 -json
     test_env_vars:
@@ -878,6 +915,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E Core Tests
+      - Merge Queue E2E Core Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ && go test smoke/job_distributor_test.go -timeout 30m -count=1 -json
     pyroscope_env: ci-smoke-jd-evm-simulated
@@ -892,6 +930,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPForBidirectionalLane$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
@@ -903,6 +942,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPForBidirectionalLane$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
@@ -915,6 +955,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPForBidirectionalLane$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
@@ -927,6 +968,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPForBidirectionalLane$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
@@ -952,6 +994,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPTokenPoolRateLimits$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
@@ -963,6 +1006,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPMulticall$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
@@ -974,6 +1018,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPManuallyExecuteAfterExecutionFailingDueToInsufficientGas$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
@@ -985,6 +1030,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPOnRampLimits$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
@@ -1016,6 +1062,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPReorgBelowFinality$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
@@ -1028,6 +1075,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPReorgAboveFinalityAtDestination$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
@@ -1040,6 +1088,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     workflows:
       - PR E2E CCIP Tests
+      - Merge Queue E2E CCIP Tests
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPReorgAboveFinalityAtSource$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index bee79ae6cb8..896b96766a1 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -241,7 +241,7 @@ jobs:
       id-token: write
       contents: read
     needs: [build-chainlink, changes]
-    if: needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true'
+    if: github.event_name == 'pull_request' && ( needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
       workflow_name: Run Core E2E Tests
@@ -280,8 +280,7 @@ jobs:
       id-token: write
       contents: read
     needs: [build-chainlink, changes]
-    # Enable when done debugging
-    # if: ${{ github.event_name == 'merge_group' }} 
+    if: github.event_name == 'merge_group'
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
       workflow_name: Run Core E2E Tests In Merge Queue
@@ -324,7 +323,7 @@ jobs:
       id-token: write
       contents: read
     needs: [build-chainlink, changes]
-    if: needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true'
+    if: github.event_name == 'pull_request' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
       workflow_name: Run CCIP E2E Tests
@@ -334,6 +333,45 @@ jobs:
       upload_cl_node_coverage_artifact: true
       upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_
       enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}      
+              
+  run-ccip-e2e-tests-in-merge-queue:
+    name: Run CCIP E2E Tests In Merge Queue
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    needs: [build-chainlink, changes]
+    if: github.event_name == 'merge_group' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      workflow_name: Run CCIP E2E Tests In Merge Queue
+      chainlink_version: ${{ inputs.evm-ref || github.sha }}
+      chainlink_upgrade_version: ${{ github.sha }}
+      test_workflow: Merge Queue E2E CCIP Tests
+      upload_cl_node_coverage_artifact: true
+      upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_
+      enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
@@ -358,7 +396,7 @@ jobs:
     if: always()
     name: ETH Smoke Tests
     runs-on: ubuntu-latest
-    needs: [lint-integration-tests, run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue]
+    needs: [lint-integration-tests, run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue, run-ccip-e2e-tests-in-merge-queue]
     steps:
       - name: Check Core test results
         id: check_core_results
@@ -405,7 +443,7 @@ jobs:
   cleanup:
     name: Clean up integration environment deployments
     if: always()
-    needs: [run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue]
+    needs: [run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue, run-ccip-e2e-tests-in-merge-queue]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
@@ -437,7 +475,7 @@ jobs:
   show-chainlink-node-coverage:
     name: Show Chainlink Node Go Coverage
     if: always()
-    needs: [run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue]
+    needs: [run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue, run-ccip-e2e-tests-in-merge-queue]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the repo

From bc79402dceb2bb5af772526d5dab148222ea47fe Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Thu, 19 Sep 2024 17:14:03 +0200
Subject: [PATCH 135/141] Run selected e2e tests in merge queue part 3

---
 .github/workflows/integration-tests.yml | 40 ++++++++++++-------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 896b96766a1..2746565c2fb 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -232,8 +232,8 @@ jobs:
           AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           dep_evm_sha: ${{ inputs.evm-ref }}
 
-  run-core-e2e-tests-in-pr:
-    name: Run Core E2E Tests
+  run-core-e2e-tests-for-pr:
+    name: Run Core E2E Tests For PR
     permissions:
       actions: read
       checks: write
@@ -244,7 +244,7 @@ jobs:
     if: github.event_name == 'pull_request' && ( needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      workflow_name: Run Core E2E Tests
+      workflow_name: Run Core E2E Tests For PR
       chainlink_version: ${{ inputs.evm-ref || github.sha }}
       chainlink_upgrade_version: ${{ github.sha }}
       test_workflow: PR E2E Core Tests
@@ -271,8 +271,8 @@ jobs:
       AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
 
-  run-core-e2e-tests-in-merge-queue:
-    name: Run Core E2E Tests In Merge Queue
+  run-core-e2e-tests-for-merge-queue:
+    name: Run Core E2E Tests For Merge Queue
     permissions:
       actions: read
       checks: write
@@ -283,7 +283,7 @@ jobs:
     if: github.event_name == 'merge_group'
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      workflow_name: Run Core E2E Tests In Merge Queue
+      workflow_name: Run Core E2E Tests For Merge Queue
       chainlink_version: ${{ inputs.evm-ref || github.sha }}
       chainlink_upgrade_version: ${{ github.sha }}
       test_workflow: Merge Queue E2E Core Tests
@@ -314,8 +314,8 @@ jobs:
       AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
               
-  run-ccip-e2e-tests-in-pr:
-    name: Run CCIP E2E Tests
+  run-ccip-e2e-tests-for-pr:
+    name: Run CCIP E2E Tests For PR
     permissions:
       actions: read
       checks: write
@@ -326,7 +326,7 @@ jobs:
     if: github.event_name == 'pull_request' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      workflow_name: Run CCIP E2E Tests
+      workflow_name: Run CCIP E2E Tests For PR
       chainlink_version: ${{ inputs.evm-ref || github.sha }}
       chainlink_upgrade_version: ${{ github.sha }}
       test_workflow: PR E2E CCIP Tests
@@ -353,8 +353,8 @@ jobs:
       AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}      
               
-  run-ccip-e2e-tests-in-merge-queue:
-    name: Run CCIP E2E Tests In Merge Queue
+  run-ccip-e2e-tests-for-merge-queue:
+    name: Run CCIP E2E Tests For Merge Queue
     permissions:
       actions: read
       checks: write
@@ -365,7 +365,7 @@ jobs:
     if: github.event_name == 'merge_group' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      workflow_name: Run CCIP E2E Tests In Merge Queue
+      workflow_name: Run CCIP E2E Tests For Merge Queue
       chainlink_version: ${{ inputs.evm-ref || github.sha }}
       chainlink_upgrade_version: ${{ github.sha }}
       test_workflow: Merge Queue E2E CCIP Tests
@@ -396,12 +396,12 @@ jobs:
     if: always()
     name: ETH Smoke Tests
     runs-on: ubuntu-latest
-    needs: [lint-integration-tests, run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue, run-ccip-e2e-tests-in-merge-queue]
+    needs: [lint-integration-tests, run-core-e2e-tests-for-pr, run-ccip-e2e-tests-for-pr, run-core-e2e-tests-for-merge-queue, run-ccip-e2e-tests-for-merge-queue]
     steps:
       - name: Check Core test results
         id: check_core_results
         run: |
-          results='${{ needs.run-core-e2e-tests-in-pr.outputs.test_results }}'
+          results='${{ needs.run-core-e2e-tests-for-pr.outputs.test_results }}'
           echo "Core test results:"
           echo "$results" | jq .
           
@@ -411,8 +411,8 @@ jobs:
       - name: Check CCIP test results
         id: check_ccip_results
         run: |
-          if [[ '${{ needs.run-ccip-e2e-tests-in-pr.result }}' != 'skipped' ]]; then
-            results='${{ needs.run-ccip-e2e-tests-in-pr.outputs.test_results }}'
+          if [[ '${{ needs.run-ccip-e2e-tests-for-pr.result }}' != 'skipped' ]]; then
+            results='${{ needs.run-ccip-e2e-tests-for-pr.outputs.test_results }}'
             echo "CCIP test results:"
             echo "$results" | jq .
           else
@@ -429,11 +429,11 @@ jobs:
           slack-message: ":x: :mild-panic-intensifies: Node Migration Tests Failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}\n${{ format('Notifying <!subteam^{0}|{0}>', secrets.GUARDIAN_SLACK_NOTIFICATION_HANDLE) }}"
 
       - name: Fail the job if core tests in PR not successful
-        if: always() && needs.run-core-e2e-tests-in-pr.result == 'failure'
+        if: always() && needs.run-core-e2e-tests-for-pr.result == 'failure'
         run: exit 1
 
       - name: Fail the job if core tests in merge queue not successful
-        if: always() && needs.run-core-e2e-tests-in-merge-queue.result == 'failure'
+        if: always() && needs.run-core-e2e-tests-for-merge-queue.result == 'failure'
         run: exit 1        
 
       - name: Fail the job if lint not successful
@@ -443,7 +443,7 @@ jobs:
   cleanup:
     name: Clean up integration environment deployments
     if: always()
-    needs: [run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue, run-ccip-e2e-tests-in-merge-queue]
+    needs: [run-core-e2e-tests-for-pr, run-ccip-e2e-tests-for-pr, run-core-e2e-tests-for-merge-queue, run-ccip-e2e-tests-for-merge-queue]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
@@ -475,7 +475,7 @@ jobs:
   show-chainlink-node-coverage:
     name: Show Chainlink Node Go Coverage
     if: always()
-    needs: [run-core-e2e-tests-in-pr, run-ccip-e2e-tests-in-pr, run-core-e2e-tests-in-merge-queue, run-ccip-e2e-tests-in-merge-queue]
+    needs: [run-core-e2e-tests-for-pr, run-ccip-e2e-tests-for-pr, run-core-e2e-tests-for-merge-queue, run-ccip-e2e-tests-for-merge-queue]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the repo

From bd15eee395bf45ffcec3c3b2c5017fc97bdb6552 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 20 Sep 2024 10:04:33 +0200
Subject: [PATCH 136/141] Use ctf-build-test-image@0.1.0

---
 .github/actions/build-test-image/action.yml   | 150 ------------------
 .../workflows/integration-tests-publish.yml   |   2 +-
 .github/workflows/integration-tests.yml       |   3 +-
 .../run-e2e-tests-reusable-workflow.yml       |   2 +-
 4 files changed, 3 insertions(+), 154 deletions(-)
 delete mode 100644 .github/actions/build-test-image/action.yml

diff --git a/.github/actions/build-test-image/action.yml b/.github/actions/build-test-image/action.yml
deleted file mode 100644
index bc81226b329..00000000000
--- a/.github/actions/build-test-image/action.yml
+++ /dev/null
@@ -1,150 +0,0 @@
-name: Build Test Image
-description: A composite action that allows building and publishing the test remote runner image
-
-inputs:
-  repository:
-    description: The docker repository for the image
-    default: chainlink-tests
-    required: false
-  tag:
-    description: The tag to use by default and to use for checking image existance. If not provided, the hash of the integration-tests/ directory will be used
-    required: false
-  other_tags:
-    description: Other tags to push if needed
-    required: false
-  suites:
-    description: The test suites to build into the image
-    default: chaos migration reorg smoke soak benchmark load
-    required: false
-  QA_AWS_ROLE_TO_ASSUME:
-    description: The AWS role to assume as the CD user, if any. Used in configuring the docker/login-action
-    required: true
-  QA_AWS_REGION:
-    description: The AWS region the ECR repository is located in, should only be needed for public ECR repositories, used in configuring docker/login-action
-    required: true
-  QA_AWS_ACCOUNT_NUMBER:
-    description: The AWS region the ECR repository is located in, should only be needed for public ECR repositories, used in configuring docker/login-action
-    required: true
-
-outputs:
-  test_image:
-    description: The full name of the test image that was built
-    value: ${{ steps.image_outputs.outputs.test_image }}  
-  test_image_tag:
-    description: The tag of the test image that was built
-    value: ${{ steps.image_outputs.outputs.test_image_tag }}
-  test_image_repository:
-    description: The repository of the test image that was built
-    value: ${{ steps.image_outputs.outputs.test_image_repo }}  
-
-runs:
-  using: composite
-  steps:
-
-    # Base Test Image Logic
-    - name: Get CTF Version
-      id: version
-      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/mod-version@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
-      with:
-        go-project-path: ./integration-tests
-        module-name: github.com/smartcontractkit/chainlink-testing-framework/lib
-        enforce-semantic-tag: false
-    - name: Get CTF sha
-      if: steps.version.outputs.is_semantic == 'false'
-      id: short_sha
-      env:
-        VERSION: ${{ steps.version.outputs.version }}
-      shell: bash
-      run: |
-        short_sha="${VERSION##*-}"
-        echo "short sha is: ${short_sha}"
-        echo "short_sha=${short_sha}" >> "$GITHUB_OUTPUT"
-    - name: Checkout chainlink-testing-framework
-      if: steps.version.outputs.is_semantic == 'false'
-      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      with:
-        repository: smartcontractkit/chainlink-testing-framework
-        ref: main
-        fetch-depth: 0
-        path: ctf
-    - name: Get long sha
-      if: steps.version.outputs.is_semantic == 'false'
-      id: long_sha
-      env:
-        SHORT_SHA: ${{ steps.short_sha.outputs.short_sha }}
-      shell: bash
-      run: |
-        cd ctf
-        long_sha=$(git rev-parse ${SHORT_SHA})
-        echo "sha is: ${long_sha}"
-        echo "long_sha=${long_sha}" >> "$GITHUB_OUTPUT"
-    - name: Check if test base image exists
-      if: steps.version.outputs.is_semantic == 'false'
-      id: check-base-image
-      uses: smartcontractkit/chainlink-github-actions/docker/image-exists@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-      with:
-        repository: test-base-image
-        tag: ${{ steps.long_sha.outputs.long_sha }}
-        AWS_REGION: ${{ inputs.QA_AWS_REGION }}
-        AWS_ROLE_TO_ASSUME: ${{ inputs.QA_AWS_ROLE_TO_ASSUME }}
-    - name: Build Base Image
-      if: steps.version.outputs.is_semantic == 'false' && steps.check-base-image.outputs.exists == 'false'
-      uses: smartcontractkit/chainlink-github-actions/docker/build-push@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-      env:
-        BASE_IMAGE_NAME: ${{ inputs.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ inputs.QA_AWS_REGION }}.amazonaws.com/test-base-image:${{ steps.long_sha.outputs.long_sha }}
-      with:
-        tags: ${{ env.BASE_IMAGE_NAME }}
-        file: ctf/lib/k8s/Dockerfile.base
-        AWS_REGION: ${{ inputs.QA_AWS_REGION }}
-        AWS_ROLE_TO_ASSUME: ${{ inputs.QA_AWS_ROLE_TO_ASSUME }}
-    # End Base Image Logic
-
-    # Test Runner Logic
-    - name: Get hash of integration-tests/ for test runner image
-      id: test_runner_hash
-      if: ${{ inputs.tag == '' }}
-      # Do not include testconfig/ in the hash to avoid rebuilding the image when only testconfig/ changes
-      shell: sh
-      run: |
-        HASH_VALUE=$(find integration-tests -type f ! -path 'integration-tests/testconfig/overrides/*.toml' ! -path 'integration-tests/testconfig/overrides/*/*.toml' ! -path 'integration-tests/testconfig/*/overrides/*.toml' ! -path 'integration-tests/testconfig/*/overrides/*/*.toml' ! -path 'integration-tests/ccip-tests/testconfig/*/overrides/*.toml' ! -path 'integration-tests/ccip-tests/testconfig/*/overrides/*/*.toml' ! -path '.github/*/*' ! -path '*/*.md' ! -path '*/*.MD' ! -path 'integration-tests/*/__debug_bin*' ! -path '*/*.MD' ! -path 'integration-tests/*/tmp-manifest*.yaml' ! -path '*/*.MD' ! -path 'integration-tests/*/*.log' ! -path 'integration-tests/*/*_dump.sql' ! -path 'integration-tests/*/.test_summary/*' -exec sha256sum {} + | sort -k 2 | sha256sum | awk '{print $1}')
-        echo "Computed hash: $HASH_VALUE"
-        echo "hash_value=$HASH_VALUE" >> $GITHUB_OUTPUT
-
-    - name: Check if image exists
-      id: check-image
-      uses: smartcontractkit/chainlink-github-actions/docker/image-exists@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-      with:
-        repository: ${{ inputs.repository }}
-        tag: ${{ inputs.tag || steps.test_runner_hash.outputs.hash_value }}
-        AWS_REGION: ${{ inputs.QA_AWS_REGION }}
-        AWS_ROLE_TO_ASSUME: ${{ inputs.QA_AWS_ROLE_TO_ASSUME }}
-    - name: Build and Publish Test Runner
-      if: steps.check-image.outputs.exists == 'false'
-      uses: smartcontractkit/chainlink-github-actions/docker/build-push@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
-      with:
-        tags: |
-          ${{ inputs.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ inputs.QA_AWS_REGION }}.amazonaws.com/${{ inputs.repository }}:${{ inputs.tag || steps.test_runner_hash.outputs.hash_value }}
-          ${{ inputs.other_tags }}
-        file: ./integration-tests/test.Dockerfile
-        build-args: |
-          BASE_IMAGE=${{ inputs.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ inputs.QA_AWS_REGION }}.amazonaws.com/test-base-image
-          IMAGE_VERSION=${{ steps.long_sha.outputs.long_sha || steps.version.outputs.version }}
-          SUITES="${{ inputs.suites }}"
-        AWS_REGION: ${{ inputs.QA_AWS_REGION }}
-        AWS_ROLE_TO_ASSUME: ${{ inputs.QA_AWS_ROLE_TO_ASSUME }}
-    - name: Print Image Built
-      shell: sh
-      env:
-        INPUTS_REPOSITORY: ${{ inputs.repository }}
-        INPUTS_TAG: ${{ inputs.tag || steps.test_runner_hash.outputs.hash_value }}
-      run: |
-        echo "### ${INPUTS_REPOSITORY} image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-        echo "\`${INPUTS_TAG}\`" >>$GITHUB_STEP_SUMMARY
-    - name: Set outputs
-      id: image_outputs
-      shell: sh
-      run: |
-        echo "test_image_repo=${{ inputs.repository }}" >> $GITHUB_OUTPUT
-        echo "test_image_tag=${{ inputs.tag || steps.test_runner_hash.outputs.hash_value }}" >> $GITHUB_OUTPUT
-        echo "test_image=${{ inputs.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ inputs.QA_AWS_REGION }}.amazonaws.com/${{ inputs.repository }}:${{ inputs.tag || steps.test_runner_hash.outputs.hash_value }}" >> $GITHUB_OUTPUT
-    # End Test Runner Logic
diff --git a/.github/workflows/integration-tests-publish.yml b/.github/workflows/integration-tests-publish.yml
index de551fedce1..b74902cdc43 100644
--- a/.github/workflows/integration-tests-publish.yml
+++ b/.github/workflows/integration-tests-publish.yml
@@ -40,7 +40,7 @@ jobs:
         run: |
           echo "other_tags=${ECR_TAG}" >> $GITHUB_OUTPUT
       - name: Build Image
-        uses: ./.github/actions/build-test-image
+        uses: smartcontractkit/.github/actions/ctf-build-test-image@a5e4f4c8fbb8e15ab2ad131552eca6ac83c4f4b3 # ctf-build-test-image@0.1.0
         with:
           other_tags: ${{ steps.tags.outputs.other_tags }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 2746565c2fb..85a7a54dcf1 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -659,10 +659,9 @@ jobs:
           ref: ${{ needs.get_solana_sha.outputs.sha }}
       - name: Build Test Image
         if: (needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
-        uses: ./.github/actions/build-test-image
+        uses: smartcontractkit/.github/actions/ctf-build-test-image@a5e4f4c8fbb8e15ab2ad131552eca6ac83c4f4b3 # ctf-build-test-image@0.1.0
         with:
           tag: ${{ needs.get_solana_sha.outputs.sha }}
-          artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index fc0c53215c4..2b35b46b1ae 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -686,7 +686,7 @@ jobs:
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
       - name: Build Test Runner Image
         id: build-test-runner-image
-        uses: ./.github/actions/build-test-image
+        uses: smartcontractkit/.github/actions/ctf-build-test-image@a5e4f4c8fbb8e15ab2ad131552eca6ac83c4f4b3 # ctf-build-test-image@0.1.0
         if: ${{ inputs.with_existing_remote_runner_version == '' }}
         with:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From c7f28fcdb5d7448ceed1e3d42130994696816de8 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 20 Sep 2024 11:08:38 +0200
Subject: [PATCH 137/141] Fix suites for k8s tests

Use test_suite input and matrix.tests.test_env_vars.TEST_SUITE to set test suites for tests in e2e-tests.yml
---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 2b35b46b1ae..664e35de958 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -767,7 +767,6 @@ jobs:
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@4f377a6b1cc07f0eca82745782736b4908a1da30 # v2.3.32
         env:
           DETACH_RUNNER: true
-          TEST_SUITE: ${{ inputs.test_image_suites }}
           RR_MEM: ${{ matrix.tests.remote_runner_memory }}
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
           ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.get-remote-runner-test-image.outputs.remote-runner-version }}

From 16ce1402c2b1b4300bb5b6d10c5f5cb236530721 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 20 Sep 2024 14:11:25 +0200
Subject: [PATCH 138/141] Update gha

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 664e35de958..48f96be64a1 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -661,7 +661,7 @@ jobs:
 
       - name: Print failed test summary
         if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@70ccaef155381025e411cf7cd1fa5ef8f668ed75 # v2.3.25
+        uses: smartcontractkit/.github/actions/ctf-show-grafana-in-test-summary@70fcaef0bf3a5a7d8aa681861d2f76e4188863d9 # ctf-show-grafana-in-test-summary@0.0.0
 
   # Run K8s tests using old remote runner
 
@@ -825,10 +825,9 @@ jobs:
           path: ${{ matrix.tests.test_artifacts_on_failure }}        
           retention-days: 1              
 
-      # TODO: move to run-tests GHA
       - name: Print failed test summary
         if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@70ccaef155381025e411cf7cd1fa5ef8f668ed75 # v2.3.25        
+        uses: smartcontractkit/.github/actions/ctf-show-grafana-in-test-summary@70fcaef0bf3a5a7d8aa681861d2f76e4188863d9 # ctf-show-grafana-in-test-summary@0.0.0
 
   after_tests:
     needs: [load-test-configurations, run-docker-tests, run-k8s-runner-tests]

From 73546a4a95cabccc051bfa3b55ddaeffd5bd8120 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 20 Sep 2024 14:26:38 +0200
Subject: [PATCH 139/141] Rename step

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 48f96be64a1..d519a58e03e 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -659,7 +659,7 @@ jobs:
           path: ${{ matrix.tests.test_artifacts_on_failure }}        
           retention-days: 1      
 
-      - name: Print failed test summary
+      - name: Show Grafana url in test summary
         if: always()
         uses: smartcontractkit/.github/actions/ctf-show-grafana-in-test-summary@70fcaef0bf3a5a7d8aa681861d2f76e4188863d9 # ctf-show-grafana-in-test-summary@0.0.0
 
@@ -825,7 +825,7 @@ jobs:
           path: ${{ matrix.tests.test_artifacts_on_failure }}        
           retention-days: 1              
 
-      - name: Print failed test summary
+      - name: Show Grafana url in test summary
         if: always()
         uses: smartcontractkit/.github/actions/ctf-show-grafana-in-test-summary@70fcaef0bf3a5a7d8aa681861d2f76e4188863d9 # ctf-show-grafana-in-test-summary@0.0.0
 

From f3c31170f79fcfca2fc71dd7e5eaf2b87ff3cdcd Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 20 Sep 2024 14:41:11 +0200
Subject: [PATCH 140/141] Update run-tests GHA

---
 .github/actions/build-chainlink-image/action.yml      | 2 +-
 .github/workflows/integration-tests.yml               | 8 ++++----
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/actions/build-chainlink-image/action.yml b/.github/actions/build-chainlink-image/action.yml
index 4934e579aec..0e457560bf0 100644
--- a/.github/actions/build-chainlink-image/action.yml
+++ b/.github/actions/build-chainlink-image/action.yml
@@ -37,7 +37,7 @@ runs:
         AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
     - name: Build Image
       if: steps.check-image.outputs.exists != 'true'
-      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
+      uses: smartcontractkit/.github/actions/ctf-build-image@1a26fe378d7ebdc34ab1fe31ec4a6d1c376199f8 # ctf-build-image@0.0.0
       with:
         cl_repo: smartcontractkit/chainlink
         cl_ref: ${{ inputs.git_commit_sha }}
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 85a7a54dcf1..032ec40b4ea 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -71,7 +71,7 @@ jobs:
           echo "should-enforce=$SHOULD_ENFORCE" >> $GITHUB_OUTPUT
       - name: Enforce CTF Version
         if: steps.condition-check.outputs.should-enforce == 'true'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/mod-version@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
+        uses: smartcontractkit/.github/actions/ctf-check-mod-version@21b0189c5fdca0318617d259634b1a91e6d80262 # ctf-check-mod-version@0.0.0
         with:
           go-project-path: ./integration-tests
           module-name: github.com/smartcontractkit/chainlink-testing-framework/lib
@@ -160,7 +160,7 @@ jobs:
           repository: smartcontractkit/chainlink
           ref: ${{ inputs.cl_ref }}
       - name: Setup Go
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@0ce1e67b254a4f041e03cc6f0e3afc987b47c7bd # v2.3.30
+        uses: smartcontractkit/.github/actions/ctf-setup-go@b0d756c57fcdbcff187e74166562a029fdd5d1b9 # ctf-setup-go@0.0.0
         with:
           test_download_vendor_packages_command: cd ${{ matrix.project.path }} && go mod download
           go_mod_path: ${{ matrix.project.path }}/go.mod
@@ -711,7 +711,7 @@ jobs:
           ref: ${{ needs.get_solana_sha.outputs.sha }}
       - name: Run Setup
         if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@0ce1e67b254a4f041e03cc6f0e3afc987b47c7bd # v2.3.30
+        uses: smartcontractkit/.github/actions/ctf-setup-run-tests-environment@49cb1613e96c9ce17f7290e4dabd38f43aa9bd4d # ctf-setup-run-tests-environment@0.0.0
         with:
           go_mod_path: ./integration-tests/go.mod
           cache_restore_only: true
@@ -763,7 +763,7 @@ jobs:
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
       - name: Run Tests
         if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d2f9642bcc24a73400568756f24b72c188ac7a9a # v2.3.31
+        uses: smartcontractkit/.github/actions/ctf-run-tests@b8731364b119e88983e94b0c4da87fc27ddb41b8 # ctf-run-tests@0.0.0
         with:
           test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke
           test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index d519a58e03e..8278828a38d 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -557,7 +557,7 @@ jobs:
 
       - name: Run tests
         id: run_tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@4f377a6b1cc07f0eca82745782736b4908a1da30 # v2.3.32
+        uses: smartcontractkit/.github/actions/ctf-run-tests@b8731364b119e88983e94b0c4da87fc27ddb41b8 # ctf-run-tests@0.0.0
         env:
           DETACH_RUNNER: true
           E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || env.DEFAULT_CHAINLINK_VERSION }}
@@ -764,7 +764,7 @@ jobs:
 
       - name: Run tests
         id: run_tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@4f377a6b1cc07f0eca82745782736b4908a1da30 # v2.3.32
+        uses: smartcontractkit/.github/actions/ctf-run-tests@b8731364b119e88983e94b0c4da87fc27ddb41b8 # ctf-run-tests@0.0.0
         env:
           DETACH_RUNNER: true
           RR_MEM: ${{ matrix.tests.remote_runner_memory }}

From b696d8eb4bc1e6694f97390a0c574d1597c4605f Mon Sep 17 00:00:00 2001
From: joaoluisam <joao.luismoreira@smartcontract.com>
Date: Sun, 22 Sep 2024 14:35:22 +0100
Subject: [PATCH 141/141] update testsconfigs to use ocr common

---
 .../testconfig/ocr/overrides/arbitrum_mainnet.toml          | 4 +++-
 .../testconfig/ocr/overrides/arbitrum_sepolia.toml          | 4 +++-
 .../testconfig/ocr/overrides/base_mainnet.toml              | 6 ++++--
 .../testconfig/ocr/overrides/base_sepolia.toml              | 3 ++-
 .../testconfig/ocr/overrides/celo_alfajores.toml            | 4 +++-
 .../testconfig/ocr/overrides/ethereum_sepolia.toml          | 4 +++-
 .../testconfig/ocr/overrides/linea_sepolia.toml             | 4 +++-
 .../testconfig/ocr/overrides/optimism_mainnet.toml          | 4 +++-
 .../testconfig/ocr/overrides/optimism_sepolia.toml          | 4 +++-
 .../testconfig/ocr/overrides/scroll_sepolia.toml            | 4 +++-
 .../testconfig/ocr/overrides/wemix_mainnet.toml             | 4 +++-
 .../testconfig/ocr2/overrides/base_sepolia.toml             | 4 +++-
 .../testconfig/ocr2/overrides/ethereum_sepolia.toml         | 4 +++-
 .../testconfig/ocr2/overrides/polygon_amoy.toml             | 4 +++-
 .../testconfig/ocr2/overrides/polygon_mainnet.toml          | 4 +++-
 .../testconfig/ocr2/overrides/wemix_testnet.toml            | 4 +++-
 .../testconfig/ocr2/overrides/xlayer_sepolia.toml           | 4 +++-
 17 files changed, 51 insertions(+), 18 deletions(-)

diff --git a/integration-tests/testconfig/ocr/overrides/arbitrum_mainnet.toml b/integration-tests/testconfig/ocr/overrides/arbitrum_mainnet.toml
index fa6a2173998..953d9f351a4 100644
--- a/integration-tests/testconfig/ocr/overrides/arbitrum_mainnet.toml
+++ b/integration-tests/testconfig/ocr/overrides/arbitrum_mainnet.toml
@@ -9,8 +9,10 @@ chainlink_node_funding = 0.1
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "10m"
 
+[OCR.Common]
+number_of_contracts = 2
+
 [Seth]
 experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr/overrides/arbitrum_sepolia.toml b/integration-tests/testconfig/ocr/overrides/arbitrum_sepolia.toml
index 4eb9f70b9b9..1428e50b0e3 100644
--- a/integration-tests/testconfig/ocr/overrides/arbitrum_sepolia.toml
+++ b/integration-tests/testconfig/ocr/overrides/arbitrum_sepolia.toml
@@ -9,5 +9,7 @@ chainlink_node_funding = 1
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
+
+[OCR.Common]
+number_of_contracts = 2
diff --git a/integration-tests/testconfig/ocr/overrides/base_mainnet.toml b/integration-tests/testconfig/ocr/overrides/base_mainnet.toml
index 8a8cfc34ae5..a285c23758e 100644
--- a/integration-tests/testconfig/ocr/overrides/base_mainnet.toml
+++ b/integration-tests/testconfig/ocr/overrides/base_mainnet.toml
@@ -9,8 +9,10 @@ chainlink_node_funding = 5
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "10m"
 
+[OCR.Common]
+number_of_contracts = 2
+
 [Seth]
-experiments_enabled = ["slow_funds_return"]
\ No newline at end of file
+experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr/overrides/base_sepolia.toml b/integration-tests/testconfig/ocr/overrides/base_sepolia.toml
index 21c5975d890..3dbbadcbef2 100644
--- a/integration-tests/testconfig/ocr/overrides/base_sepolia.toml
+++ b/integration-tests/testconfig/ocr/overrides/base_sepolia.toml
@@ -9,6 +9,7 @@ chainlink_node_funding = 1
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
 
+[OCR.Common]
+number_of_contracts = 2
diff --git a/integration-tests/testconfig/ocr/overrides/celo_alfajores.toml b/integration-tests/testconfig/ocr/overrides/celo_alfajores.toml
index c051cea64e7..37c4a8cf16d 100644
--- a/integration-tests/testconfig/ocr/overrides/celo_alfajores.toml
+++ b/integration-tests/testconfig/ocr/overrides/celo_alfajores.toml
@@ -9,5 +9,7 @@ chainlink_node_funding = 10
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
+
+[OCR.Common]
+number_of_contracts = 2
diff --git a/integration-tests/testconfig/ocr/overrides/ethereum_sepolia.toml b/integration-tests/testconfig/ocr/overrides/ethereum_sepolia.toml
index d0fa75565eb..612e47506a7 100644
--- a/integration-tests/testconfig/ocr/overrides/ethereum_sepolia.toml
+++ b/integration-tests/testconfig/ocr/overrides/ethereum_sepolia.toml
@@ -9,5 +9,7 @@ chainlink_node_funding = 1
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
+
+[OCR.Common]
+number_of_contracts = 2
diff --git a/integration-tests/testconfig/ocr/overrides/linea_sepolia.toml b/integration-tests/testconfig/ocr/overrides/linea_sepolia.toml
index 9f9c5b2e3a6..6fa6218d541 100644
--- a/integration-tests/testconfig/ocr/overrides/linea_sepolia.toml
+++ b/integration-tests/testconfig/ocr/overrides/linea_sepolia.toml
@@ -9,5 +9,7 @@ chainlink_node_funding = 1
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
+
+[OCR.Common]
+number_of_contracts = 2
diff --git a/integration-tests/testconfig/ocr/overrides/optimism_mainnet.toml b/integration-tests/testconfig/ocr/overrides/optimism_mainnet.toml
index 1996a89ec73..eec0640baa3 100644
--- a/integration-tests/testconfig/ocr/overrides/optimism_mainnet.toml
+++ b/integration-tests/testconfig/ocr/overrides/optimism_mainnet.toml
@@ -9,8 +9,10 @@ chainlink_node_funding = 0.1
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "10m"
 
+[OCR.Common]
+number_of_contracts = 2
+
 [Seth]
 experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr/overrides/optimism_sepolia.toml b/integration-tests/testconfig/ocr/overrides/optimism_sepolia.toml
index 804e977db8c..fe666b6aa9e 100644
--- a/integration-tests/testconfig/ocr/overrides/optimism_sepolia.toml
+++ b/integration-tests/testconfig/ocr/overrides/optimism_sepolia.toml
@@ -9,5 +9,7 @@ chainlink_node_funding = 1
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
+
+[OCR.Common]
+number_of_contracts = 2
diff --git a/integration-tests/testconfig/ocr/overrides/scroll_sepolia.toml b/integration-tests/testconfig/ocr/overrides/scroll_sepolia.toml
index 964756d5fbd..0beedfd05ea 100644
--- a/integration-tests/testconfig/ocr/overrides/scroll_sepolia.toml
+++ b/integration-tests/testconfig/ocr/overrides/scroll_sepolia.toml
@@ -9,5 +9,7 @@ chainlink_node_funding = 1
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
+
+[OCR.Common]
+number_of_contracts = 2
diff --git a/integration-tests/testconfig/ocr/overrides/wemix_mainnet.toml b/integration-tests/testconfig/ocr/overrides/wemix_mainnet.toml
index 138cd7d46f7..4a7859db3c9 100644
--- a/integration-tests/testconfig/ocr/overrides/wemix_mainnet.toml
+++ b/integration-tests/testconfig/ocr/overrides/wemix_mainnet.toml
@@ -9,8 +9,10 @@ chainlink_node_funding = 5
 test_duration = "24h"
 
 [Soak.OCR.Soak]
-number_of_contracts = 2
 time_between_rounds = "10m"
 
+[OCR.Common]
+number_of_contracts = 2
+
 [Seth]
 experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml b/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
index 9ed03bf078f..76e4fc4722d 100644
--- a/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
+++ b/integration-tests/testconfig/ocr2/overrides/base_sepolia.toml
@@ -9,8 +9,10 @@ chainlink_node_funding = 1
 test_duration = "24h"
 
 [Soak.OCR2.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
 
+[OCR2.Common]
+number_of_contracts = 2
+
 [Seth]
 experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr2/overrides/ethereum_sepolia.toml b/integration-tests/testconfig/ocr2/overrides/ethereum_sepolia.toml
index f7dadcf7337..f7e02407808 100644
--- a/integration-tests/testconfig/ocr2/overrides/ethereum_sepolia.toml
+++ b/integration-tests/testconfig/ocr2/overrides/ethereum_sepolia.toml
@@ -9,5 +9,7 @@ chainlink_node_funding = 1
 test_duration = "24h"
 
 [Soak.OCR2.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
+
+[OCR2.Common]
+number_of_contracts = 2
diff --git a/integration-tests/testconfig/ocr2/overrides/polygon_amoy.toml b/integration-tests/testconfig/ocr2/overrides/polygon_amoy.toml
index 6af0aa556ec..41a31897c5f 100644
--- a/integration-tests/testconfig/ocr2/overrides/polygon_amoy.toml
+++ b/integration-tests/testconfig/ocr2/overrides/polygon_amoy.toml
@@ -9,5 +9,7 @@ chainlink_node_funding = 20
 test_duration = "24h"
 
 [Soak.OCR2.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
+
+[OCR2.Common]
+number_of_contracts = 2
diff --git a/integration-tests/testconfig/ocr2/overrides/polygon_mainnet.toml b/integration-tests/testconfig/ocr2/overrides/polygon_mainnet.toml
index fd814c206b3..51da2793133 100644
--- a/integration-tests/testconfig/ocr2/overrides/polygon_mainnet.toml
+++ b/integration-tests/testconfig/ocr2/overrides/polygon_mainnet.toml
@@ -9,8 +9,10 @@ chainlink_node_funding = 5
 test_duration = "24h"
 
 [Soak.OCR2.Soak]
-number_of_contracts = 2
 time_between_rounds = "10m"
 
+[OCR2.Common]
+number_of_contracts = 2
+
 [Seth]
 experiments_enabled = ["slow_funds_return"]
diff --git a/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml b/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
index a48c76eaccb..82bc06c17ee 100644
--- a/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
+++ b/integration-tests/testconfig/ocr2/overrides/wemix_testnet.toml
@@ -9,5 +9,7 @@ chainlink_node_funding = 10
 test_duration = "24h"
 
 [Soak.OCR2.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
+
+[OCR2.Common]
+number_of_contracts = 2
diff --git a/integration-tests/testconfig/ocr2/overrides/xlayer_sepolia.toml b/integration-tests/testconfig/ocr2/overrides/xlayer_sepolia.toml
index 3d94da583fa..d58098c5b0f 100644
--- a/integration-tests/testconfig/ocr2/overrides/xlayer_sepolia.toml
+++ b/integration-tests/testconfig/ocr2/overrides/xlayer_sepolia.toml
@@ -9,5 +9,7 @@ chainlink_node_funding = 1
 test_duration = "24h"
 
 [Soak.OCR2.Soak]
-number_of_contracts = 2
 time_between_rounds = "2m"
+
+[OCR2.Common]
+number_of_contracts = 2