Run some DATs as part of base-normalization tests (#14312)

airbytehq · Jul 1, 2022 · bb74373 · bb74373
1 parent dcfa1fb
commit bb74373
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 0 deletions.
diff --git a/.github/workflows/build-connector-command.yml b/.github/workflows/build-connector-command.yml
@@ -135,6 +135,12 @@ jobs:
         run: |
           source venv/bin/activate
           ci_credentials ${{ github.event.inputs.connector }}
+          # normalization also runs destination-specific tests, so fetch their creds also
+          if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ]; then
+            ci_credentials destination-bigquery
+            ci_credentials destination-postgres
+            ci_credentials destination-snowflake
+          fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
       # TODO: seems like this should run in post-merge workflow

diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml
@@ -266,6 +266,12 @@ jobs:
         run: |
           source venv/bin/activate
           ci_credentials ${{ matrix.connector }}
+          # normalization also runs destination-specific tests, so fetch their creds also
+          if [ 'bases/base-normalization' = "${{ matrix.connector }}" ]; then
+            ci_credentials destination-bigquery
+            ci_credentials destination-postgres
+            ci_credentials destination-snowflake
+          fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
       - name: Set Name and Version Environment Vars

diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml
@@ -110,6 +110,12 @@ jobs:
         run: |
           source venv/bin/activate
           ci_credentials ${{ github.event.inputs.connector }}
+          # normalization also runs destination-specific tests, so fetch their creds also
+          if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ]; then
+            ci_credentials destination-bigquery
+            ci_credentials destination-postgres
+            ci_credentials destination-snowflake
+          fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
 

diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml
@@ -116,6 +116,12 @@ jobs:
         run: |
           source venv/bin/activate
           ci_credentials ${{ github.event.inputs.connector }}
+          # normalization also runs destination-specific tests, so fetch their creds also
+          if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ]; then
+            ci_credentials destination-bigquery
+            ci_credentials destination-postgres
+            ci_credentials destination-snowflake
+          fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
       - run: |

diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle
@@ -102,6 +102,14 @@ task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs
     dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker'
 }
 
+// DATs have some additional tests that exercise normalization code paths,
+// so we want to run these in addition to the base-normalization integration tests.
+// If you add more items here, make sure to also to have CI fetch their credentials.
+// See git history for an example.
+integrationTest.dependsOn(":airbyte-integrations:connectors:destination-bigquery:integrationTest")
+integrationTest.dependsOn(":airbyte-integrations:connectors:destination-postgres:integrationTest")
+integrationTest.dependsOn(":airbyte-integrations:connectors:destination-snowflake:integrationTest")
+
 integrationTest.dependsOn("customIntegrationTestPython")
 customIntegrationTests.dependsOn("customIntegrationTestPython")