From d9ca2b56569b4b512ce38730e14b9eb1eae9bc4c Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 22 Jul 2024 13:29:37 -0400 Subject: [PATCH 01/21] Create run-name --- .github/workflows/integration.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f6d3addd2..08121eb52 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,6 +20,8 @@ name: Adapter Integration Tests +run-name: Run Integration ${{ inputs.name }} by @${{ github.actor }} + on: # pushes to release branches push: @@ -38,6 +40,11 @@ on: description: "branch of dbt-core to use in dev-requirements.txt" required: false type: string + name: + description: "name to associate with run" + required: false + type: string + default: "Adapter Integration Tests" # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all From be9ff67b70664bbbe978f5a53f64a3512a991cf1 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 22 Jul 2024 15:40:16 -0400 Subject: [PATCH 02/21] Update integration.yml --- .github/scripts/update_dev_packages.sh | 17 +++++++++++ .github/workflows/integration.yml | 40 ++++++++++++++++++++------ 2 files changed, 49 insertions(+), 8 deletions(-) create mode 100755 .github/scripts/update_dev_packages.sh diff --git a/.github/scripts/update_dev_packages.sh b/.github/scripts/update_dev_packages.sh new file mode 100755 index 000000000..c0f207b4e --- /dev/null +++ b/.github/scripts/update_dev_packages.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e +set -e + + +adapters_git_branch=$1 +core_git_branch=$2 +target_req_file="pyproject.toml" +core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${core_git_branch}#subdirectory=core|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${adapters_git_branch}|g" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$adapters_req_sed_pattern" $target_req_file +else + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$adapters_req_sed_pattern" $target_req_file +fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 08121eb52..650681395 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -36,15 +36,38 @@ on: # manual trigger workflow_dispatch: inputs: - dbt-core-branch: + dbt_core_branch: description: "branch of dbt-core to use in dev-requirements.txt" required: false type: string + default: main name: description: "name to associate with run" required: false type: string default: "Adapter Integration Tests" + dbt_adapters_branch: + description: "The branch of dbt-adapters to use" + type: string + required: false + default: main + workflow_call: + inputs: + dbt_core_branch: + description: "branch of dbt-core to use in dev-requirements.txt" + required: false + type: string + default: main + name: + description: "name to associate with run" + required: false + type: string + default: "Adapter Integration Tests" + dbt_adapters_branch: + description: "The branch of dbt-adapters to use" + type: string + required: false + default: main # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -153,7 +176,7 @@ jobs: with: persist-credentials: false - # explicity checkout the branch for the PR, + # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event - name: Check out the repository (PR) if: github.event_name == 'pull_request_target' @@ -167,6 +190,13 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Update Adapters and Core branches (update dev_requirements.txt) + if: ${{ github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' }} + run: | + ./.github/scripts/update_dev_packages.sh \ + ${{ inputs.dbt_adapters_branch }} \ + ${{ inputs.dbt_core_branch }} + - name: Install python dependencies run: | python -m pip install --user --upgrade pip @@ -174,12 +204,6 @@ jobs: python -m pip --version tox --version - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' - run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - name: Run tox (snowflake) if: matrix.adapter == 'snowflake' env: From e3b033e083a17df265392910c1346f5660fe152a Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 22 Jul 2024 17:26:57 -0400 Subject: [PATCH 03/21] update update_dev_packages.sh --- .github/scripts/update_dev_packages.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/update_dev_packages.sh b/.github/scripts/update_dev_packages.sh index c0f207b4e..fb26c39b5 100755 --- a/.github/scripts/update_dev_packages.sh +++ b/.github/scripts/update_dev_packages.sh @@ -4,9 +4,9 @@ set -e adapters_git_branch=$1 core_git_branch=$2 -target_req_file="pyproject.toml" -core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${core_git_branch}#subdirectory=core|g" -adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${adapters_git_branch}|g" +target_req_file="dev-requirements.txt" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${core_git_branch}#egg=dbt-core|g" +adapters_req_sed_pattern="s|dbt-adapters.git.*#egg=dbt-adapters|dbt-adapters.git@${adapters_git_branch}#egg=dbt-adapters|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file From 2cbbddd6338834f18d1ece185523ac40f5ebae6e Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 22 Jul 2024 17:27:54 -0400 Subject: [PATCH 04/21] Tweak run-name --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 650681395..9850dc130 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,7 +20,7 @@ name: Adapter Integration Tests -run-name: Run Integration ${{ inputs.name }} by @${{ github.actor }} +run-name: ${{ inputs.name }} by @${{ github.actor }} on: # pushes to release branches From af6dbb32e990b48a84a2d58caf6843c65bee6de7 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 23 Jul 2024 11:48:23 -0400 Subject: [PATCH 05/21] Adjust run-name, add additional checkout option --- .github/workflows/integration.yml | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9850dc130..27dfec6f4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,7 +20,12 @@ name: Adapter Integration Tests -run-name: ${{ inputs.name }} by @${{ github.actor }} +run-name: >- +if [${{ github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' }}]; then +"${{ inputs.name }} by @${{ github.actor }}" +else +"Integration tests: ${{ github.ref_name }}" +fi on: # pushes to release branches @@ -51,6 +56,11 @@ on: type: string required: false default: main + adapter_branch: + description: "The branch of this adapter repo to use" + type: string + required: false + default: main workflow_call: inputs: dbt_core_branch: @@ -68,6 +78,11 @@ on: type: string required: false default: main + adapter_branch: + description: "The branch of this adapter repo to use" + type: string + required: false + default: main # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -95,8 +110,15 @@ jobs: matrix: ${{ steps.generate-matrix.outputs.result }} steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ inputs.adapter_branch }} + + - name: Check out the repository (non-PR, non workflow_dispatch) + if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' uses: actions/checkout@v4 with: persist-credentials: false From 9f33856c9196929639146aa67a770d1b47253ca0 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 23 Jul 2024 11:53:34 -0400 Subject: [PATCH 06/21] Revert "Adjust run-name, add additional checkout option" This reverts commit af6dbb32e990b48a84a2d58caf6843c65bee6de7. --- .github/workflows/integration.yml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 27dfec6f4..9850dc130 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,12 +20,7 @@ name: Adapter Integration Tests -run-name: >- -if [${{ github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' }}]; then -"${{ inputs.name }} by @${{ github.actor }}" -else -"Integration tests: ${{ github.ref_name }}" -fi +run-name: ${{ inputs.name }} by @${{ github.actor }} on: # pushes to release branches @@ -56,11 +51,6 @@ on: type: string required: false default: main - adapter_branch: - description: "The branch of this adapter repo to use" - type: string - required: false - default: main workflow_call: inputs: dbt_core_branch: @@ -78,11 +68,6 @@ on: type: string required: false default: main - adapter_branch: - description: "The branch of this adapter repo to use" - type: string - required: false - default: main # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -110,15 +95,8 @@ jobs: matrix: ${{ steps.generate-matrix.outputs.result }} steps: - - name: Check out the repository (workflow_dispatch) - if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' - uses: actions/checkout@v4 - with: - persist-credentials: false - ref: ${{ inputs.adapter_branch }} - - - name: Check out the repository (non-PR, non workflow_dispatch) - if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' + - name: Check out the repository (non-PR) + if: github.event_name != 'pull_request_target' uses: actions/checkout@v4 with: persist-credentials: false From 5fdc7041fcf02be79e859bc3bb2c9aadce9463ab Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 23 Jul 2024 12:07:51 -0400 Subject: [PATCH 07/21] Try another run-name --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9850dc130..f296f7b89 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,7 +20,7 @@ name: Adapter Integration Tests -run-name: ${{ inputs.name }} by @${{ github.actor }} +run-name: "${{ contains(github.event_name, 'workflow_') && inputs.name || github.event_name }}: ${{ github.ref_name }} by @${{ github.actor }}" on: # pushes to release branches From 6912bb394d584215d3db10b9cbb6395ae0c82222 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 23 Jul 2024 12:15:59 -0400 Subject: [PATCH 08/21] Try another run-name tweak --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f296f7b89..a9cf050d9 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,7 +20,7 @@ name: Adapter Integration Tests -run-name: "${{ contains(github.event_name, 'workflow_') && inputs.name || github.event_name }}: ${{ github.ref_name }} by @${{ github.actor }}" +run-name: "${{ contains(github.event_name, 'workflow_') && inputs.name || github.event_name }}: ${{ contains(github.event_name, 'workflow_') && inputs.adapter_branch || github.ref_name }} by @${{ github.actor }}" on: # pushes to release branches From 103e4037960797b7f21b8d0bc2c51e2b70ac8f34 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 23 Jul 2024 12:25:19 -0400 Subject: [PATCH 09/21] Add checking out adapter branch --- .github/workflows/integration.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a9cf050d9..6b45827fc 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -171,10 +171,17 @@ jobs: steps: - name: Check out the repository - if: github.event_name != 'pull_request_target' + if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' uses: actions/checkout@v4 with: persist-credentials: false + ref: ${{ inputs.adapter_branch }} # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event From 6cf2424ae4687b998c3a41725ca8a0d4890ff51a Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 23 Jul 2024 12:40:04 -0400 Subject: [PATCH 10/21] actually add adapter_branch input --- .github/workflows/integration.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6b45827fc..5700496ec 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -51,6 +51,11 @@ on: type: string required: false default: main + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: main workflow_call: inputs: dbt_core_branch: @@ -68,6 +73,11 @@ on: type: string required: false default: main + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: main # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all From 39f922b443e49d3c07774eb9a0fdd57b1d82ce13 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 23 Jul 2024 12:51:46 -0400 Subject: [PATCH 11/21] Don't change adapter branch for now --- .github/workflows/integration.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 5700496ec..8373a791e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -191,7 +191,9 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - ref: ${{ inputs.adapter_branch }} + # We can't actually change branch until this is in main, because it + # won't find the update_dev_packages.sh script. + # ref: ${{ inputs.adapter_branch }} # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event From 60c53062d7d5bf9a67b0e200e7077b35abb5611a Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 24 Jul 2024 10:34:52 -0400 Subject: [PATCH 12/21] Try cat dev-requirements.txt --- .github/workflows/integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8373a791e..b774e7cf4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -215,6 +215,7 @@ jobs: ./.github/scripts/update_dev_packages.sh \ ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} + cat dev-requirements.txt - name: Install python dependencies run: | From 4d63b29707820af49b77289c7c75ccf06686ef62 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 24 Jul 2024 10:50:31 -0400 Subject: [PATCH 13/21] Fix update_dev_packages.sh --- .github/scripts/update_dev_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/update_dev_packages.sh b/.github/scripts/update_dev_packages.sh index fb26c39b5..7fdd47c35 100755 --- a/.github/scripts/update_dev_packages.sh +++ b/.github/scripts/update_dev_packages.sh @@ -6,7 +6,7 @@ adapters_git_branch=$1 core_git_branch=$2 target_req_file="dev-requirements.txt" core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${core_git_branch}#egg=dbt-core|g" -adapters_req_sed_pattern="s|dbt-adapters.git.*#egg=dbt-adapters|dbt-adapters.git@${adapters_git_branch}#egg=dbt-adapters|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${adapters_git_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file From e9e5673a7f47313b8c3043ed344b804445d592c0 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 24 Jul 2024 17:40:46 -0400 Subject: [PATCH 14/21] Remove workflow_call --- .github/workflows/integration.yml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4c741506f..f1df034e2 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -56,28 +56,6 @@ on: type: string required: false default: main - workflow_call: - inputs: - dbt_core_branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - default: main - name: - description: "name to associate with run" - required: false - type: string - default: "Adapter Integration Tests" - dbt_adapters_branch: - description: "The branch of dbt-adapters to use" - type: string - required: false - default: main - adapter_branch: - description: "The branch of this adapter repository to use" - type: string - required: false - default: main # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -181,13 +159,13 @@ jobs: steps: - name: Check out the repository - if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' + if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false - name: Check out the repository (workflow_dispatch) - if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false @@ -210,7 +188,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Update Adapters and Core branches (update dev_requirements.txt) - if: ${{ github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' }} run: | ./.github/scripts/update_dev_packages.sh \ ${{ inputs.dbt_adapters_branch }} \ From f3781be8404f47f1b4e8d524e8756e0f19a6ca5a Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 25 Jul 2024 16:58:42 -0400 Subject: [PATCH 15/21] Remove update_dbt_core_branch.sh --- .github/scripts/update_dbt_core_branch.sh | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100755 .github/scripts/update_dbt_core_branch.sh diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh deleted file mode 100755 index d28a40c35..000000000 --- a/.github/scripts/update_dbt_core_branch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi -core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) -bumpversion --allow-dirty --new-version "$core_version" major From fea92563f74a15780320bb37b58f39bb23ab3718 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 25 Jul 2024 17:10:22 -0400 Subject: [PATCH 16/21] Use adapter_branch --- .github/workflows/integration.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f1df034e2..7bc6729b5 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -169,9 +169,7 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - # We can't actually change branch until this is in main, because it - # won't find the update_dev_packages.sh script. - # ref: ${{ inputs.adapter_branch }} + ref: ${{ inputs.adapter_branch }} # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event From fa42ff2d475fa6c6881406529aec186fc6d29014 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 29 Jul 2024 10:39:25 -0400 Subject: [PATCH 17/21] Rename update_dev_dependency_branches, add dbt-common --- ...s.sh => update_dev_dependency_branches.sh} | 16 +++++++---- .github/workflows/integration.yml | 28 +++++++++++-------- 2 files changed, 27 insertions(+), 17 deletions(-) rename .github/scripts/{update_dev_packages.sh => update_dev_dependency_branches.sh} (62%) diff --git a/.github/scripts/update_dev_packages.sh b/.github/scripts/update_dev_dependency_branches.sh similarity index 62% rename from .github/scripts/update_dev_packages.sh rename to .github/scripts/update_dev_dependency_branches.sh index 7fdd47c35..022df6a8a 100755 --- a/.github/scripts/update_dev_packages.sh +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -2,16 +2,20 @@ set -e -adapters_git_branch=$1 -core_git_branch=$2 +dbt_adapters_branch=$1 +dbt_core_branch=$2 +dbt_common_branch=$3 target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${core_git_branch}#egg=dbt-core|g" -adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${adapters_git_branch}|g" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" +common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file sed -i "" "$adapters_req_sed_pattern" $target_req_file + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$common_req_sed_pattern" $target_req_file else - sed -i "$core_req_sed_pattern" $target_req_file sed -i "$adapters_req_sed_pattern" $target_req_file + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$common_req_sed_pattern" $target_req_file fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7bc6729b5..52729fba7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,7 +20,7 @@ name: Adapter Integration Tests -run-name: "${{ contains(github.event_name, 'workflow_') && inputs.name || github.event_name }}: ${{ contains(github.event_name, 'workflow_') && inputs.adapter_branch || github.ref_name }} by @${{ github.actor }}" +run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') && inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}" on: # pushes to release branches @@ -36,26 +36,31 @@ on: # manual trigger workflow_dispatch: inputs: - dbt_core_branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - default: main name: - description: "name to associate with run" + description: "Name to associate with run (example: 'dbt-adapters-242')" required: false type: string default: "Adapter Integration Tests" + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: main dbt_adapters_branch: description: "The branch of dbt-adapters to use" type: string required: false - default: main - adapter_branch: - description: "The branch of this adapter repository to use" + default: "main" + dbt_core_branch: + description: "The branch of dbt-core to use" type: string required: false - default: main + default: "main" + dbt_common_branch: + description: "The branch of dbt-common to use" + type: string + required: false + default: "main" # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -190,6 +195,7 @@ jobs: run: | ./.github/scripts/update_dev_packages.sh \ ${{ inputs.dbt_adapters_branch }} \ + ${{ inputs.dbt_core_branch }} \ ${{ inputs.dbt_core_branch }} cat dev-requirements.txt From 65008475776c79276244b6743205a4e06b6deea2 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 29 Jul 2024 10:50:10 -0400 Subject: [PATCH 18/21] Fix script name --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 52729fba7..f23326a78 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -193,7 +193,7 @@ jobs: - name: Update Adapters and Core branches (update dev_requirements.txt) if: ${{ github.event_name == 'workflow_dispatch' }} run: | - ./.github/scripts/update_dev_packages.sh \ + ./.github/scripts/update_dev_dependency_branches.sh \ ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} \ ${{ inputs.dbt_core_branch }} From b49d02b881649561de1c55da337699f54997c6d9 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 29 Jul 2024 13:48:52 -0400 Subject: [PATCH 19/21] Add quotes to default --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f23326a78..8006f2266 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -45,7 +45,7 @@ on: description: "The branch of this adapter repository to use" type: string required: false - default: main + default: "main" dbt_adapters_branch: description: "The branch of dbt-adapters to use" type: string From cb1e38c16c87e265ce5ced844e550d86cc44e0b4 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 29 Jul 2024 15:07:03 -0400 Subject: [PATCH 20/21] Tweak if for repository checkout --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8006f2266..af220655b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -164,7 +164,7 @@ jobs: steps: - name: Check out the repository - if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' + if: github.event_name == 'push' uses: actions/checkout@v4 with: persist-credentials: false From 4918412ec9d0ff75906427b8a1af844c5440c6c1 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 30 Jul 2024 10:48:23 -0400 Subject: [PATCH 21/21] Fix typo --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index af220655b..b3662d5c0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -196,7 +196,7 @@ jobs: ./.github/scripts/update_dev_dependency_branches.sh \ ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} \ - ${{ inputs.dbt_core_branch }} + ${{ inputs.dbt_common_branch }} cat dev-requirements.txt - name: Install python dependencies