From a3fab0b5a96526904befc592a8b917391924e654 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Mon, 17 Apr 2023 20:57:20 -0400 Subject: [PATCH 01/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 32 +++++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index f7b8dc29543..1ecc8cec5f0 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -27,23 +27,22 @@ on: jobs: get_version_meta: - name: Get version meta + name: Get version metadata runs-on: ubuntu-latest + outputs: - major: ${{ steps.version.outputs.major }} - minor: ${{ steps.version.outputs.minor }} - patch: ${{ steps.version.outputs.patch }} latest: ${{ steps.latest.outputs.latest }} minor_latest: ${{ steps.latest.outputs.minor_latest }} + steps: - - uses: actions/checkout@v1 - - name: Split version - id: version + - name: "[DEBUG] Print Variables" run: | - IFS="." read -r MAJOR MINOR PATCH <<< ${{ github.event.inputs.version_number }} - echo "major=$MAJOR" >> $GITHUB_OUTPUT - echo "minor=$MINOR" >> $GITHUB_OUTPUT - echo "patch=$PATCH" >> $GITHUB_OUTPUT + echo "all variables defined as inputs" + echo "The package: ${{ github.event.inputs.package }}" + echo "The version_number: ${{ github.event.inputs.version_number }}" + + - name: "Check out the repository" + uses: actions/checkout@v3 - name: Is pkg 'latest' id: latest @@ -67,6 +66,13 @@ jobs: runs-on: ubuntu-latest needs: [setup_image_builder, get_version_meta] steps: + + - name: "Parse Version Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1 + with: + version: ${{ github.event.inputs.version_number }} + - name: Get docker build arg id: build_arg run: | @@ -91,7 +97,7 @@ jobs: build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ github.event.inputs.version_number }} + ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ ${{ steps.parse-valid.outputs.base-version }} }} - name: Build and push MINOR.latest tag uses: docker/build-push-action@v2 @@ -103,7 +109,7 @@ jobs: build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ needs.get_version_meta.outputs.major }}.${{ needs.get_version_meta.outputs.minor }}.latest + ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.major }}.${{ steps.parse-valid.outputs.minor }}.latest - name: Build and push latest tag uses: docker/build-push-action@v2 From d0d4eba4778de147b39ad9fa41d4533a0fdd0cbd Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Mon, 17 Apr 2023 21:02:10 -0400 Subject: [PATCH 02/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 1ecc8cec5f0..8a5349bd16c 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -89,7 +89,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push MAJOR.MINOR.PATCH tag - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: file: docker/Dockerfile push: True @@ -100,7 +100,7 @@ jobs: ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ ${{ steps.parse-valid.outputs.base-version }} }} - name: Build and push MINOR.latest tag - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 if: ${{ needs.get_version_meta.outputs.minor_latest == 'True' }} with: file: docker/Dockerfile @@ -112,7 +112,7 @@ jobs: ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.major }}.${{ steps.parse-valid.outputs.minor }}.latest - name: Build and push latest tag - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 if: ${{ needs.get_version_meta.outputs.latest == 'True' }} with: file: docker/Dockerfile From 23a9504a519093710dbf51dda46ce767dd8d0e7f Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Mon, 17 Apr 2023 21:03:29 -0400 Subject: [PATCH 03/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 8a5349bd16c..ad2fbb6bbf1 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -97,7 +97,7 @@ jobs: build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ ${{ steps.parse-valid.outputs.base-version }} }} + ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.base-version }} - name: Build and push MINOR.latest tag uses: docker/build-push-action@v4 From 4f068a45ff79999424c4bd9b1ff69730685f078f Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Mon, 17 Apr 2023 21:06:40 -0400 Subject: [PATCH 04/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index ad2fbb6bbf1..7171bd404e7 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -97,7 +97,7 @@ jobs: build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.base-version }} + ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.base-version }} - name: Build and push MINOR.latest tag uses: docker/build-push-action@v4 @@ -109,7 +109,7 @@ jobs: build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.major }}.${{ steps.parse-valid.outputs.minor }}.latest + ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.major }}.${{ steps.parse-valid.outputs.minor }}.latest - name: Build and push latest tag uses: docker/build-push-action@v4 @@ -121,4 +121,4 @@ jobs: build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/dbt-labs/${{ github.event.inputs.package }}:latest + ghcr.io/leahwicz/${{ github.event.inputs.package }}:latest From 3b3def5b8ad75b299b987d859b0e4780de5d45ad Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Mon, 17 Apr 2023 21:11:34 -0400 Subject: [PATCH 05/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 7171bd404e7..cf77a56f555 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -59,7 +59,7 @@ jobs: needs: [get_version_meta] steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 build_and_push: name: Build images and push to GHCR From 50b3d1deaa75a3457310489f8850a3a810ec724f Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Mon, 17 Apr 2023 21:22:55 -0400 Subject: [PATCH 06/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index cf77a56f555..e22be252823 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -97,7 +97,7 @@ jobs: build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.base-version }} + ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.semver.outputs.base-version }} - name: Build and push MINOR.latest tag uses: docker/build-push-action@v4 @@ -109,7 +109,7 @@ jobs: build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.major }}.${{ steps.parse-valid.outputs.minor }}.latest + ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.latest - name: Build and push latest tag uses: docker/build-push-action@v4 From 88ae1f8871ebf3282635725c255e8b88abda2951 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 18 Apr 2023 09:15:19 -0400 Subject: [PATCH 07/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index e22be252823..e5192b36848 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -95,9 +95,9 @@ jobs: push: True target: ${{ github.event.inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.parse-valid.outputs.version }} tags: | - ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.semver.outputs.base-version }} + ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.version }} - name: Build and push MINOR.latest tag uses: docker/build-push-action@v4 @@ -107,7 +107,7 @@ jobs: push: True target: ${{ github.event.inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.parse-valid.outputs.version }} tags: | ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.latest @@ -119,6 +119,6 @@ jobs: push: True target: ${{ github.event.inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.parse-valid.outputs.version }} tags: | ghcr.io/leahwicz/${{ github.event.inputs.package }}:latest From d5da0a8093e54c3a91bfe71e4fa9f179b8d84d0a Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 18 Apr 2023 21:36:29 -0400 Subject: [PATCH 08/47] Update release.yml --- .github/workflows/release.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e3722ea19cd..8b131c90c74 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -208,6 +208,31 @@ jobs: secrets: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + + + docker-core-release: + name: Docker dbt-core Release + if: ${{ inputs.test_run }} + + needs: [pypi-release] + + uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main + + with: + package: 'dbt-core' + version_number: ${{ inputs.version_number }} + + + docker-postgres-release: + name: Docker dbt-postgres Release + + needs: [docker-core-release] + + uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main + + with: + package: 'dbt-postgres' + version_number: ${{ inputs.version_number }} slack-notification: name: Slack Notification From 89cd24388d9833366df2c90328afcd82c69ff726 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 18 Apr 2023 21:40:55 -0400 Subject: [PATCH 09/47] Update release.yml --- .github/workflows/release.yml | 84 +---------------------------------- 1 file changed, 1 insertion(+), 83 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8b131c90c74..f22807fb3c5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -130,7 +130,7 @@ jobs: echo Test run: ${{ inputs.test_run }} echo Nightly release: ${{ inputs.nightly_release }} - bump-version-generate-changelog: + pypi-release: name: Bump package version, Generate changelog uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main @@ -145,70 +145,6 @@ jobs: secrets: inherit - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} - - needs: [bump-version-generate-changelog] - - runs-on: ubuntu-latest - - steps: - - name: Print variables - run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - - build-test-package: - name: Build, Test, Package - if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog] - - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build_script_path: ${{ inputs.build_script_path }} - s3_bucket_name: ${{ inputs.s3_bucket_name }} - package_test_command: ${{ inputs.package_test_command }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - github-release: - name: GitHub Release - if: ${{ !failure() && !cancelled() }} - - needs: [bump-version-generate-changelog, build-test-package] - - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.test_run }} - - pypi-release: - name: PyPI Release - - needs: [github-release] - - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - - with: - version_number: ${{ inputs.version_number }} - test_run: ${{ inputs.test_run }} - - secrets: - PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - docker-core-release: name: Docker dbt-core Release @@ -234,21 +170,3 @@ jobs: package: 'dbt-postgres' version_number: ${{ inputs.version_number }} - slack-notification: - name: Slack Notification - if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - - needs: - [ - bump-version-generate-changelog, - build-test-package, - github-release, - pypi-release, - ] - - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main - with: - status: "failure" - - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }} From cc39fe51b35e80bd5bfaaa5ab14a07813540ec14 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 18 Apr 2023 21:44:08 -0400 Subject: [PATCH 10/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index e5192b36848..f57c369fcd2 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -16,14 +16,26 @@ permissions: packages: write on: + workflow_call: + inputs: + package: + description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] + required: true + type: string + version_number: + description: The release version number (i.e. 1.0.0b1). Do not include `latest` tags or a leading `v`! + required: true + type: string workflow_dispatch: inputs: package: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true + type: string version_number: description: The release version number (i.e. 1.0.0b1). Do not include `latest` tags or a leading `v`! required: true + type: string jobs: get_version_meta: From 3e4c61d020e3931a11ee5f17466c8c7acef6e515 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 18 Apr 2023 21:46:26 -0400 Subject: [PATCH 11/47] Update release.yml --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f22807fb3c5..b0eafd25f7d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -108,6 +108,7 @@ on: permissions: contents: write # this is the permission that allows creating a new release + packages: write defaults: run: From 5df501a2812b62a4588f087b5579a01fdde03b2c Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 18 Apr 2023 21:48:49 -0400 Subject: [PATCH 12/47] Update release.yml --- .github/workflows/release.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b0eafd25f7d..42e568a45b8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -115,7 +115,7 @@ defaults: shell: bash jobs: - log-inputs: + pypi-release: name: Log Inputs runs-on: ubuntu-latest steps: @@ -131,21 +131,6 @@ jobs: echo Test run: ${{ inputs.test_run }} echo Nightly release: ${{ inputs.nightly_release }} - pypi-release: - name: Bump package version, Generate changelog - - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main - - with: - sha: ${{ inputs.sha }} - version_number: ${{ inputs.version_number }} - target_branch: ${{ inputs.target_branch }} - env_setup_script_path: ${{ inputs.env_setup_script_path }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - - secrets: inherit - docker-core-release: name: Docker dbt-core Release From 412ac8d1b9cbf313e2bfea19d0de93f9ca74ebb3 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Wed, 19 Apr 2023 09:08:39 -0400 Subject: [PATCH 13/47] Update release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 42e568a45b8..b6ed69042a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -134,7 +134,7 @@ jobs: docker-core-release: name: Docker dbt-core Release - if: ${{ inputs.test_run }} + if: ${{ !inputs.test_run }} needs: [pypi-release] From 0ada5e8bf7ceaeec24a57369621a017893350eb8 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 12:17:26 -0400 Subject: [PATCH 14/47] Create testing.yml --- .github/workflows/testing.yml | 285 ++++++++++++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 .github/workflows/testing.yml diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 00000000000..33f3498313b --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,285 @@ +# **what?** +# This workflow models the performance characteristics of a point in time in dbt. +# It runs specific dbt commands on committed projects multiple times to create and +# commit information about the distribution to the current branch. For more information +# see the readme in the performance module at /performance/README.md. +# +# **why?** +# When developing new features, we can take quick performance samples and compare +# them against the commited baseline measurements produced by this workflow to detect +# some performance regressions at development time before they reach users. +# +# **when?** +# This is only run once directly after each release. If for some reason the results of +# a run are not satisfactory, it can also be triggered manually. + +name: Model Performance Characteristics + +on: + # runs after non-prereleases are published. + release: + types: [released] + # run manually from the actions tab + workflow_dispatch: + inputs: + # if we ever want to model pre-releases, we would need to handle pre-release version numbers + # here and in the runner. Even if we use a semver library I suspect it will be rather difficult. + release_id: + description: 'dbt version to model (must be non-prerelease in Pypi)' + type: string + required: true + open_prs: + description: Open PRs to main and release branch? (branch name inferred from provided version) + type: boolean + required: true + pull_request: # TODO: remove, just for testing purposes + +env: + RUNNER_CACHE_PATH: performance/runner/target/release/runner + +# both jobs need to write +permissions: + contents: write + pull-requests: write + +jobs: + latest-runner: + name: Build or Fetch Runner + runs-on: ubuntu-latest + env: + RUSTFLAGS: "-D warnings" + outputs: + cache_key: ${{ steps.variables.outputs.cache_key }} + release_id: ${{ steps.variables.outputs.release_id }} + open_prs: ${{ steps.variables.outputs.open_prs }} + release_branch: ${{ steps.variables.outputs.release_branch }} + steps: + + # explicitly checkout the performance runner from main regardless of which + # version we are modeling. + - name: Checkout + uses: actions/checkout@v2 + with: + ref: main + + # collect all the variables that need to be used in subsequent jobs + - name: Set Variables + id: variables + run: | + # create a cache key that will be used in the next job. without this the + # next job would have to checkout from main an hash the files itself. + echo "cache_key=${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" >> $GITHUB_OUTPUT + # echo "::set-output name=cache_key::${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" + + # this value gets used to create other values locally so it gets a local definition too. + local_release_id='' + # users are prompted to input with the correct format + if [[ $GITHUB_EVENT_NAME == "workflow_dispatch" ]]; then + echo "Workflow dispatch event detected" + local_release_id=${{github.event.inputs.release_id}} + # echo "::set-output name=release_id::${{github.event.inputs.release_id}}" + # echo "::set-output name=open_prs::${{github.event.inputs.open_prs}}" + echo "release_id=${{github.event.inputs.release_id}}" >> $GITHUB_OUTPUT + echo "open_prs=${{github.event.inputs.open_prs}}" >> $GITHUB_OUTPUT + # release.tag_name has a v prepended. we must remove it. + else + echo "release event detected" + with_v=${{github.event.release.tag_name}} + without_v=${with_v:1} + without_v=1.2.1 # TODO: just for testing + local_release_id=$without_v + # echo "::set-output name=release_id::$without_v" + # echo "::set-output name=open_prs::true" + echo "release_id={$without_v}" >> $GITHUB_OUTPUT + echo "open_prs={true}" >> $GITHUB_OUTPUT + fi + + # string manipulation to get the branch name. It can't be discovered from the github api + # for release triggers so we're stuck with this. If we change our branch naming strategy + # we have to update this code. example: 1.0.0 -> 1.0.latest. the sed command takes into + # account multiple digits like 1.0.999 -> 1.0.latest + + no_patch=$(sed "s|\(.*\)\..*|\1|" <<< $local_release_id) + branch_name="${no_patch}.latest" + # echo "::set-output name=release_branch::$branch_name" + echo "release_branch={$branch_name}" >> $GITHUB_OUTPUT + echo "release branch is inferred to be ${branch_name}" + + # attempts to access a previously cached runner + - uses: actions/cache@v2 + id: cache + with: + path: ${{ env.RUNNER_CACHE_PATH }} + key: ${{ steps.variables.outputs.cache_key }} + + - name: Fetch Rust Toolchain + if: steps.cache.outputs.cache-hit != 'true' + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - name: Add fmt + if: steps.cache.outputs.cache-hit != 'true' + run: rustup component add rustfmt + + - name: Cargo fmt + if: steps.cache.outputs.cache-hit != 'true' + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --manifest-path performance/runner/Cargo.toml --all -- --check + + - name: Test + if: steps.cache.outputs.cache-hit != 'true' + uses: actions-rs/cargo@v1 + with: + command: test + args: --manifest-path performance/runner/Cargo.toml + + - name: Build (optimized) + if: steps.cache.outputs.cache-hit != 'true' + uses: actions-rs/cargo@v1 + with: + command: build + args: --release --manifest-path performance/runner/Cargo.toml + # the cache action automatically caches this binary at the end of the job + + model: + # depends on `latest-runner` as a separate job so that failures in this job do not prevent + # a successfully tested and built binary from being cached. + needs: [latest-runner] + name: Model a release + runs-on: ubuntu-latest + steps: + + - name: '[DEBUG] print variables' + run: | + echo "all variables defined in latest-runner > Set Variables > outputs" + echo "cache_key: ${{ needs.latest-runner.outputs.cache_key }}" + echo "release_id: ${{ needs.latest-runner.outputs.release_id }}" + echo "open_prs: ${{ needs.latest-runner.outputs.open_prs }}" + echo "release_branch: ${{ needs.latest-runner.outputs.release_branch }}" + + - name: Setup Python + uses: actions/setup-python@v2.2.2 + with: + python-version: "3.8" + + - name: Install dbt + run: pip install dbt-postgres==${{ needs.latest-runner.outputs.release_id }} + + - name: Install Hyperfine + run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb + + # explicitly checkout main to get the latest project definitions + - name: Checkout + uses: actions/checkout@v2 + with: + ref: main + + # this was built in the previous job so it will be there. + - name: Fetch Runner + uses: actions/cache@v2 + id: cache + with: + path: ${{ env.RUNNER_CACHE_PATH }} + key: ${{ needs.latest-runner.outputs.cache_key }} + + - name: Move Runner + run: mv performance/runner/target/release/runner performance/app + + - name: Change Runner Permissions + run: chmod +x ./performance/app + + - name: '[DEBUG] ls baseline directory before run' + run: ls -R performance/baselines/ + + # `${{ github.workspace }}` is used to pass the absolute path + - name: Create directories + run: | + mkdir ${{ github.workspace }}/performance/tmp/ + mkdir -p performance/baselines/${{ needs.latest-runner.outputs.release_id }}/ + + # TODO CHANGE NUMBER OF RUNS BEFORE MERGING + # TODO this isn't putting the baseline in the right directory. it's putting it one level up. + # Run modeling with taking 20 samples + - name: Run Measurement + run: | + performance/app model -v ${{ needs.latest-runner.outputs.release_id }} -b ${{ github.workspace }}/performance/baselines/ -p ${{ github.workspace }}/performance/projects/ -t ${{ github.workspace }}/performance/tmp/ -n 2 + + - name: '[DEBUG] ls baseline directory after run' + run: ls -R performance/baselines/ + + - uses: actions/upload-artifact@v3 + with: + name: baseline + path: performance/baselines/${{ needs.latest-runner.outputs.release_id }}/ + + create-pr: + name: Open PR for ${{ matrix.base-branch }} + if: ${{ needs.latest-runner.outputs.open_prs }} + + # depends on `model` as a separate job so that the baseline can be committed to more than one branch + # i.e. release branch and main + needs: [latest-runner, model] + + runs-on: ubuntu-latest + + strategy: + matrix: + include: + - base-branch: "refs/heads/main" + target-branch: "performance-bot/main_${{needs.latest-runner.outputs.release_id}}_${{GITHUB.RUN_ID}}" + - base-branch: "refs/heads/${{ needs.latest-runner.outputs.release_branch }}" + target-branch: "performance-bot/release_${{needs.latest-runner.outputs.release_id}}_${{GITHUB.RUN_ID}}" + + steps: + - name: '[DEBUG] print variables' + run: | + echo "all variables defined in latest-runner > Set Variables > outputs" + echo "cache_key: ${{ needs.latest-runner.outputs.cache_key }}" + echo "release_id: ${{ needs.latest-runner.outputs.release_id }}" + echo "open_prs: ${{ needs.latest-runner.outputs.open_prs }}" + echo "release_branch: ${{ needs.latest-runner.outputs.release_branch }}" + + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ matrix.base-branch }} + + - name: Create PR branch + run: | + git checkout -b ${{ matrix.target-branch }} + git push origin ${{ matrix.target-branch }} + git branch --set-upstream-to=origin/${{ matrix.target-branch }} ${{ matrix.target-branch }} + + - uses: actions/download-artifact@v3 + with: + name: baseline + path: performance/baselines/${{ needs.latest-runner.outputs.release_id }} + + - name: '[DEBUG] ls baselines after artifact download' + run: ls -R performance/baselines/ + + - name: Commit baseline + uses: EndBug/add-and-commit@v9 + with: + add: 'performance/baselines/*' + author_name: 'Github Build Bot' + author_email: 'buildbot@fishtownanalytics.com' + message: 'adding performance baseline for ${{ needs.latest-runner.outputs.release_id }}' + branch: '${{ matrix.target-branch }}' + push: 'origin origin/${{ matrix.target-branch }}' + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + with: + author: 'Github Build Bot ' + base: ${{ matrix.base-branch }} + title: 'Adding performance modeling for ${{needs.latest-runner.outputs.release_id}} to ${{ matrix.base-branch }}' + branch: '${{ matrix.target-branch }}' + labels: | + Skip Changelog + Performance From 15c7b589c2dba7e405070facd5cab96f5dc925e0 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 13:06:35 -0400 Subject: [PATCH 15/47] Update testing.yml --- .github/workflows/testing.yml | 88 ++++++++++++++--------------------- 1 file changed, 35 insertions(+), 53 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 33f3498313b..7030523b3bb 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -28,10 +28,6 @@ on: description: 'dbt version to model (must be non-prerelease in Pypi)' type: string required: true - open_prs: - description: Open PRs to main and release branch? (branch name inferred from provided version) - type: boolean - required: true pull_request: # TODO: remove, just for testing purposes env: @@ -43,17 +39,46 @@ permissions: pull-requests: write jobs: + set-variables: + name: Setting Variables + runs-on: ubuntu-latest + outputs: + cache_key: ${{ steps.variables.outputs.cache_key }} + release_id: ${{ steps.semver.outputs.base-version }} + release_branch: ${{ steps.variables.outputs.release_branch }} + steps: + + - name: "Parse Version Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1 + with: + version: ${{ github.event.inputs.release_id || github.event.release.tag_name }} + + # collect all the variables that need to be used in subsequent jobs + - name: Set variables for workflow dispatch event + if: $GITHUB_EVENT_NAME == "workflow_dispatch" + run: | + # create a cache key that will be used in the next job. without this the + # next job would have to checkout from main and hash the files itself. + echo "cache_key=${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" >> $GITHUB_OUTPUT + + branch_name="${steps.semver.outputs.major}.${steps.semver.outputs.minor}.latest" + echo "release_branch=$branch_name" >> $GITHUB_OUTPUT + echo "release branch is inferred to be ${branch_name}" + latest-runner: name: Build or Fetch Runner runs-on: ubuntu-latest + needs: [set-variables] env: RUSTFLAGS: "-D warnings" - outputs: - cache_key: ${{ steps.variables.outputs.cache_key }} - release_id: ${{ steps.variables.outputs.release_id }} - open_prs: ${{ steps.variables.outputs.open_prs }} - release_branch: ${{ steps.variables.outputs.release_branch }} steps: + - name: '[DEBUG] print variables' + run: | + echo "all variables defined in set-variables" + echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}" + echo "release_id: ${{ needs.set-variables.outputs.release_id }}" + echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" # explicitly checkout the performance runner from main regardless of which # version we are modeling. @@ -62,49 +87,6 @@ jobs: with: ref: main - # collect all the variables that need to be used in subsequent jobs - - name: Set Variables - id: variables - run: | - # create a cache key that will be used in the next job. without this the - # next job would have to checkout from main an hash the files itself. - echo "cache_key=${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" >> $GITHUB_OUTPUT - # echo "::set-output name=cache_key::${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" - - # this value gets used to create other values locally so it gets a local definition too. - local_release_id='' - # users are prompted to input with the correct format - if [[ $GITHUB_EVENT_NAME == "workflow_dispatch" ]]; then - echo "Workflow dispatch event detected" - local_release_id=${{github.event.inputs.release_id}} - # echo "::set-output name=release_id::${{github.event.inputs.release_id}}" - # echo "::set-output name=open_prs::${{github.event.inputs.open_prs}}" - echo "release_id=${{github.event.inputs.release_id}}" >> $GITHUB_OUTPUT - echo "open_prs=${{github.event.inputs.open_prs}}" >> $GITHUB_OUTPUT - # release.tag_name has a v prepended. we must remove it. - else - echo "release event detected" - with_v=${{github.event.release.tag_name}} - without_v=${with_v:1} - without_v=1.2.1 # TODO: just for testing - local_release_id=$without_v - # echo "::set-output name=release_id::$without_v" - # echo "::set-output name=open_prs::true" - echo "release_id={$without_v}" >> $GITHUB_OUTPUT - echo "open_prs={true}" >> $GITHUB_OUTPUT - fi - - # string manipulation to get the branch name. It can't be discovered from the github api - # for release triggers so we're stuck with this. If we change our branch naming strategy - # we have to update this code. example: 1.0.0 -> 1.0.latest. the sed command takes into - # account multiple digits like 1.0.999 -> 1.0.latest - - no_patch=$(sed "s|\(.*\)\..*|\1|" <<< $local_release_id) - branch_name="${no_patch}.latest" - # echo "::set-output name=release_branch::$branch_name" - echo "release_branch={$branch_name}" >> $GITHUB_OUTPUT - echo "release branch is inferred to be ${branch_name}" - # attempts to access a previously cached runner - uses: actions/cache@v2 id: cache @@ -145,7 +127,7 @@ jobs: command: build args: --release --manifest-path performance/runner/Cargo.toml # the cache action automatically caches this binary at the end of the job - + model: # depends on `latest-runner` as a separate job so that failures in this job do not prevent # a successfully tested and built binary from being cached. From c5267335a3e3086799c4c0f60f6c07bb39957112 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 13:08:00 -0400 Subject: [PATCH 16/47] Update testing.yml --- .github/workflows/testing.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 7030523b3bb..f22bad1f800 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -48,15 +48,14 @@ jobs: release_branch: ${{ steps.variables.outputs.release_branch }} steps: - - name: "Parse Version Into Parts" + - name: Parse version into parts id: semver uses: dbt-labs/actions/parse-semver@v1 with: version: ${{ github.event.inputs.release_id || github.event.release.tag_name }} # collect all the variables that need to be used in subsequent jobs - - name: Set variables for workflow dispatch event - if: $GITHUB_EVENT_NAME == "workflow_dispatch" + - name: Set variables run: | # create a cache key that will be used in the next job. without this the # next job would have to checkout from main and hash the files itself. From 35d2fc11580c4a182e0f407cfafd6c57341d0cca Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 13:10:04 -0400 Subject: [PATCH 17/47] Update testing.yml --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f22bad1f800..a1e4f3d802e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -61,7 +61,7 @@ jobs: # next job would have to checkout from main and hash the files itself. echo "cache_key=${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" >> $GITHUB_OUTPUT - branch_name="${steps.semver.outputs.major}.${steps.semver.outputs.minor}.latest" + branch_name="${{steps.semver.outputs.major}}.${{steps.semver.outputs.minor}}.latest" echo "release_branch=$branch_name" >> $GITHUB_OUTPUT echo "release branch is inferred to be ${branch_name}" From dede0e9747c3869f606f9788c8aac1674d7175a6 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 13:11:57 -0400 Subject: [PATCH 18/47] Update testing.yml --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a1e4f3d802e..f1920cfe264 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -56,6 +56,7 @@ jobs: # collect all the variables that need to be used in subsequent jobs - name: Set variables + id: variables run: | # create a cache key that will be used in the next job. without this the # next job would have to checkout from main and hash the files itself. From 57da3e51cd503105e25fc6280368cf3acd377250 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 13:14:50 -0400 Subject: [PATCH 19/47] Update testing.yml --- .github/workflows/testing.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f1920cfe264..e1b1a896d67 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -47,6 +47,12 @@ jobs: release_id: ${{ steps.semver.outputs.base-version }} release_branch: ${{ steps.variables.outputs.release_branch }} steps: + # explicitly checkout the performance runner from main regardless of which + # version we are modeling. + - name: Checkout + uses: actions/checkout@v2 + with: + ref: main - name: Parse version into parts id: semver From 2b8564b16fbb174e2c62dada1c8c805301c9f89d Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 16:55:21 -0400 Subject: [PATCH 20/47] Update testing.yml --- .github/workflows/testing.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e1b1a896d67..0b8566ca95d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -98,7 +98,7 @@ jobs: id: cache with: path: ${{ env.RUNNER_CACHE_PATH }} - key: ${{ steps.variables.outputs.cache_key }} + key: ${{ needs.set-variables.outputs.cache_key }} - name: Fetch Rust Toolchain if: steps.cache.outputs.cache-hit != 'true' @@ -147,7 +147,6 @@ jobs: echo "all variables defined in latest-runner > Set Variables > outputs" echo "cache_key: ${{ needs.latest-runner.outputs.cache_key }}" echo "release_id: ${{ needs.latest-runner.outputs.release_id }}" - echo "open_prs: ${{ needs.latest-runner.outputs.open_prs }}" echo "release_branch: ${{ needs.latest-runner.outputs.release_branch }}" - name: Setup Python @@ -207,7 +206,6 @@ jobs: create-pr: name: Open PR for ${{ matrix.base-branch }} - if: ${{ needs.latest-runner.outputs.open_prs }} # depends on `model` as a separate job so that the baseline can be committed to more than one branch # i.e. release branch and main @@ -229,7 +227,6 @@ jobs: echo "all variables defined in latest-runner > Set Variables > outputs" echo "cache_key: ${{ needs.latest-runner.outputs.cache_key }}" echo "release_id: ${{ needs.latest-runner.outputs.release_id }}" - echo "open_prs: ${{ needs.latest-runner.outputs.open_prs }}" echo "release_branch: ${{ needs.latest-runner.outputs.release_branch }}" - name: Checkout From 955dcec68b4dfb3c67476b5ad274cfd3dd72057a Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 17:03:23 -0400 Subject: [PATCH 21/47] Update testing.yml --- .github/workflows/testing.yml | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 0b8566ca95d..86b9d2ab2d3 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -144,10 +144,10 @@ jobs: - name: '[DEBUG] print variables' run: | - echo "all variables defined in latest-runner > Set Variables > outputs" - echo "cache_key: ${{ needs.latest-runner.outputs.cache_key }}" - echo "release_id: ${{ needs.latest-runner.outputs.release_id }}" - echo "release_branch: ${{ needs.latest-runner.outputs.release_branch }}" + echo "all variables defined in set-variables" + echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}" + echo "release_id: ${{ needs.set-variables.outputs.release_id }}" + echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" - name: Setup Python uses: actions/setup-python@v2.2.2 @@ -155,7 +155,7 @@ jobs: python-version: "3.8" - name: Install dbt - run: pip install dbt-postgres==${{ needs.latest-runner.outputs.release_id }} + run: pip install dbt-postgres==${{ needs.set-variables.outputs.release_id }} - name: Install Hyperfine run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb @@ -172,7 +172,7 @@ jobs: id: cache with: path: ${{ env.RUNNER_CACHE_PATH }} - key: ${{ needs.latest-runner.outputs.cache_key }} + key: ${{ needs.set-variables.outputs.cache_key }} - name: Move Runner run: mv performance/runner/target/release/runner performance/app @@ -187,14 +187,14 @@ jobs: - name: Create directories run: | mkdir ${{ github.workspace }}/performance/tmp/ - mkdir -p performance/baselines/${{ needs.latest-runner.outputs.release_id }}/ + mkdir -p performance/baselines/${{ needs.set-variables.outputs.release_id }}/ # TODO CHANGE NUMBER OF RUNS BEFORE MERGING # TODO this isn't putting the baseline in the right directory. it's putting it one level up. # Run modeling with taking 20 samples - name: Run Measurement run: | - performance/app model -v ${{ needs.latest-runner.outputs.release_id }} -b ${{ github.workspace }}/performance/baselines/ -p ${{ github.workspace }}/performance/projects/ -t ${{ github.workspace }}/performance/tmp/ -n 2 + performance/app model -v ${{ needs.set-variables.outputs.release_id }} -b ${{ github.workspace }}/performance/baselines/ -p ${{ github.workspace }}/performance/projects/ -t ${{ github.workspace }}/performance/tmp/ -n 2 - name: '[DEBUG] ls baseline directory after run' run: ls -R performance/baselines/ @@ -202,7 +202,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: baseline - path: performance/baselines/${{ needs.latest-runner.outputs.release_id }}/ + path: performance/baselines/${{ needs.set-variables.outputs.release_id }}/ create-pr: name: Open PR for ${{ matrix.base-branch }} @@ -217,17 +217,17 @@ jobs: matrix: include: - base-branch: "refs/heads/main" - target-branch: "performance-bot/main_${{needs.latest-runner.outputs.release_id}}_${{GITHUB.RUN_ID}}" - - base-branch: "refs/heads/${{ needs.latest-runner.outputs.release_branch }}" - target-branch: "performance-bot/release_${{needs.latest-runner.outputs.release_id}}_${{GITHUB.RUN_ID}}" + target-branch: "performance-bot/main_${{needs.set-variables.outputs.release_id}}_${{GITHUB.RUN_ID}}" + - base-branch: "refs/heads/${{ needs.set-variables.outputs.release_branch }}" + target-branch: "performance-bot/release_${{needs.set-variables.outputs.release_id}}_${{GITHUB.RUN_ID}}" steps: - name: '[DEBUG] print variables' run: | - echo "all variables defined in latest-runner > Set Variables > outputs" - echo "cache_key: ${{ needs.latest-runner.outputs.cache_key }}" - echo "release_id: ${{ needs.latest-runner.outputs.release_id }}" - echo "release_branch: ${{ needs.latest-runner.outputs.release_branch }}" + echo "all variables defined in set-variables > Set Variables > outputs" + echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}" + echo "release_id: ${{ needs.set-variables.outputs.release_id }}" + echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" - name: Checkout uses: actions/checkout@v2 @@ -243,7 +243,7 @@ jobs: - uses: actions/download-artifact@v3 with: name: baseline - path: performance/baselines/${{ needs.latest-runner.outputs.release_id }} + path: performance/baselines/${{ needs.set-variables.outputs.release_id }} - name: '[DEBUG] ls baselines after artifact download' run: ls -R performance/baselines/ @@ -254,7 +254,7 @@ jobs: add: 'performance/baselines/*' author_name: 'Github Build Bot' author_email: 'buildbot@fishtownanalytics.com' - message: 'adding performance baseline for ${{ needs.latest-runner.outputs.release_id }}' + message: 'adding performance baseline for ${{ needs.set-variables.outputs.release_id }}' branch: '${{ matrix.target-branch }}' push: 'origin origin/${{ matrix.target-branch }}' @@ -263,7 +263,7 @@ jobs: with: author: 'Github Build Bot ' base: ${{ matrix.base-branch }} - title: 'Adding performance modeling for ${{needs.latest-runner.outputs.release_id}} to ${{ matrix.base-branch }}' + title: 'Adding performance modeling for ${{needs.set-variables.outputs.release_id}} to ${{ matrix.base-branch }}' branch: '${{ matrix.target-branch }}' labels: | Skip Changelog From 20fdf55bf6efc4820adf688cd75240c0f20c9665 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 17:15:07 -0400 Subject: [PATCH 22/47] Update testing.yml --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 86b9d2ab2d3..e0c319d057f 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -137,7 +137,7 @@ jobs: model: # depends on `latest-runner` as a separate job so that failures in this job do not prevent # a successfully tested and built binary from being cached. - needs: [latest-runner] + needs: [set-variables, latest-runner] name: Model a release runs-on: ubuntu-latest steps: From cc5f15885d0bd73bcfc63258a4d070238cc6d325 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 17:29:33 -0400 Subject: [PATCH 23/47] Update release.yml --- .github/workflows/release.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b6ed69042a3..4a5ecc4ac84 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -135,11 +135,9 @@ jobs: docker-core-release: name: Docker dbt-core Release if: ${{ !inputs.test_run }} - needs: [pypi-release] uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main - with: package: 'dbt-core' version_number: ${{ inputs.version_number }} @@ -147,11 +145,9 @@ jobs: docker-postgres-release: name: Docker dbt-postgres Release - needs: [docker-core-release] uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main - with: package: 'dbt-postgres' version_number: ${{ inputs.version_number }} From 8bcbf73aaa16e566e6edd641ac46fdabd88c9f68 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 17:31:20 -0400 Subject: [PATCH 24/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index f57c369fcd2..93493edea0e 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -107,9 +107,9 @@ jobs: push: True target: ${{ github.event.inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.parse-valid.outputs.version }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} tags: | - ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.parse-valid.outputs.version }} + ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.semver.outputs.version }} - name: Build and push MINOR.latest tag uses: docker/build-push-action@v4 @@ -119,7 +119,7 @@ jobs: push: True target: ${{ github.event.inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.parse-valid.outputs.version }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} tags: | ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.latest @@ -131,6 +131,6 @@ jobs: push: True target: ${{ github.event.inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.parse-valid.outputs.version }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} tags: | ghcr.io/leahwicz/${{ github.event.inputs.package }}:latest From 83d3421e726ba5765915be7fbaa777c998d9fa72 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 17:55:56 -0400 Subject: [PATCH 25/47] Update testing.yml --- .github/workflows/testing.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e0c319d057f..6ec32c464b8 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -209,22 +209,22 @@ jobs: # depends on `model` as a separate job so that the baseline can be committed to more than one branch # i.e. release branch and main - needs: [latest-runner, model] + needs: [set-variables, latest-runner, model] runs-on: ubuntu-latest strategy: matrix: include: - - base-branch: "refs/heads/main" - target-branch: "performance-bot/main_${{needs.set-variables.outputs.release_id}}_${{GITHUB.RUN_ID}}" - - base-branch: "refs/heads/${{ needs.set-variables.outputs.release_branch }}" - target-branch: "performance-bot/release_${{needs.set-variables.outputs.release_id}}_${{GITHUB.RUN_ID}}" + - base-branch: refs/heads/main + target-branch: performance-bot/main_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}} + - base-branch: refs/heads/${{ needs.set-variables.outputs.release_branch }} + target-branch: performance-bot/release_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}} steps: - name: '[DEBUG] print variables' run: | - echo "all variables defined in set-variables > Set Variables > outputs" + echo "all variables defined in set-variables" echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}" echo "release_id: ${{ needs.set-variables.outputs.release_id }}" echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" From fbd12e78c912f2ba0c78f2bfe6ab157e8a517c0c Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 18:40:41 -0400 Subject: [PATCH 26/47] Update testing.yml --- .github/workflows/testing.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6ec32c464b8..1bc8836a24a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -50,7 +50,7 @@ jobs: # explicitly checkout the performance runner from main regardless of which # version we are modeling. - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: main @@ -89,7 +89,7 @@ jobs: # explicitly checkout the performance runner from main regardless of which # version we are modeling. - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: main @@ -150,7 +150,7 @@ jobs: echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" - name: Setup Python - uses: actions/setup-python@v2.2.2 + uses: actions/setup-python@v4 with: python-version: "3.8" @@ -162,7 +162,7 @@ jobs: # explicitly checkout main to get the latest project definitions - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: main @@ -230,7 +230,7 @@ jobs: echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ matrix.base-branch }} @@ -259,12 +259,13 @@ jobs: push: 'origin origin/${{ matrix.target-branch }}' - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v5 with: author: 'Github Build Bot ' base: ${{ matrix.base-branch }} title: 'Adding performance modeling for ${{needs.set-variables.outputs.release_id}} to ${{ matrix.base-branch }}' branch: '${{ matrix.target-branch }}' + body: 'Committing perf results for tracking for the ${{needs.set-variables.outputs.release_id}}' labels: | Skip Changelog Performance From 4210d17f1481be361d89d9c62c7426b99a03b6f6 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 19:03:15 -0400 Subject: [PATCH 27/47] Update testing.yml --- .github/workflows/testing.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1bc8836a24a..fb19a8613a1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -94,7 +94,7 @@ jobs: ref: main # attempts to access a previously cached runner - - uses: actions/cache@v2 + - uses: actions/cache@v3 id: cache with: path: ${{ env.RUNNER_CACHE_PATH }} @@ -168,7 +168,7 @@ jobs: # this was built in the previous job so it will be there. - name: Fetch Runner - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache with: path: ${{ env.RUNNER_CACHE_PATH }} @@ -264,7 +264,6 @@ jobs: author: 'Github Build Bot ' base: ${{ matrix.base-branch }} title: 'Adding performance modeling for ${{needs.set-variables.outputs.release_id}} to ${{ matrix.base-branch }}' - branch: '${{ matrix.target-branch }}' body: 'Committing perf results for tracking for the ${{needs.set-variables.outputs.release_id}}' labels: | Skip Changelog From d4d099007257bce06dd0f427fdbd848d887c7ebd Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 19:04:40 -0400 Subject: [PATCH 28/47] Update release.yml --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a5ecc4ac84..98e429684cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,7 +139,7 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package: 'dbt-core' + package: dbt-core version_number: ${{ inputs.version_number }} @@ -149,6 +149,6 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package: 'dbt-postgres' + package: dbt-postgres version_number: ${{ inputs.version_number }} From 388838aa9968a1b0df30c6f2844715680e2c7df5 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 19:20:06 -0400 Subject: [PATCH 29/47] Update testing.yml --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index fb19a8613a1..4b4947aea2a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -255,7 +255,6 @@ jobs: author_name: 'Github Build Bot' author_email: 'buildbot@fishtownanalytics.com' message: 'adding performance baseline for ${{ needs.set-variables.outputs.release_id }}' - branch: '${{ matrix.target-branch }}' push: 'origin origin/${{ matrix.target-branch }}' - name: Create Pull Request @@ -263,6 +262,7 @@ jobs: with: author: 'Github Build Bot ' base: ${{ matrix.base-branch }} + branch: '${{ matrix.target-branch }}' title: 'Adding performance modeling for ${{needs.set-variables.outputs.release_id}} to ${{ matrix.base-branch }}' body: 'Committing perf results for tracking for the ${{needs.set-variables.outputs.release_id}}' labels: | From e77f1c3b0f3e2cde40f2164595b907f16f960606 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 19:24:59 -0400 Subject: [PATCH 30/47] Update release.yml --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 98e429684cb..c56b4ed81ca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,7 +139,7 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package: dbt-core + package: "dbt-core" version_number: ${{ inputs.version_number }} @@ -149,6 +149,6 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package: dbt-postgres + package: "dbt-postgres" version_number: ${{ inputs.version_number }} From 42935cce05a17ad2597e435d0ecda07f20c2580c Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 21:52:56 -0400 Subject: [PATCH 31/47] Update release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c56b4ed81ca..5e8f8a5380f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,7 +139,7 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package: "dbt-core" + package: ${{ inputs.version_number }} # "dbt-core" version_number: ${{ inputs.version_number }} From 48ad13de00aa4ccedb9b491c5f7fa5b22c9a453a Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 21:57:19 -0400 Subject: [PATCH 32/47] Update release.yml --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5e8f8a5380f..98e429684cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,7 +139,7 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package: ${{ inputs.version_number }} # "dbt-core" + package: dbt-core version_number: ${{ inputs.version_number }} @@ -149,6 +149,6 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package: "dbt-postgres" + package: dbt-postgres version_number: ${{ inputs.version_number }} From 36d9f841d64a71a14a5333512abfe0f2ba6e23f0 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:01:27 -0400 Subject: [PATCH 33/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 93493edea0e..1a131e0d8f1 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -49,9 +49,10 @@ jobs: steps: - name: "[DEBUG] Print Variables" run: | - echo "all variables defined as inputs" + echo "All variables defined as inputs" echo "The package: ${{ github.event.inputs.package }}" echo "The version_number: ${{ github.event.inputs.version_number }}" + echo "The version_number: ${{ github.event.inputs }}" - name: "Check out the repository" uses: actions/checkout@v3 From 3adc6dca610bfe017614ce681d7c3243fe596acc Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:03:15 -0400 Subject: [PATCH 34/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 1a131e0d8f1..b7e624cc816 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -52,7 +52,7 @@ jobs: echo "All variables defined as inputs" echo "The package: ${{ github.event.inputs.package }}" echo "The version_number: ${{ github.event.inputs.version_number }}" - echo "The version_number: ${{ github.event.inputs }}" + echo "The version_number: ${{ github.event.inputs.* }}" - name: "Check out the repository" uses: actions/checkout@v3 From 1d193a9ab99b268ea6e75f699caf80e69db43c26 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:07:53 -0400 Subject: [PATCH 35/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index b7e624cc816..8ffaa9f8f15 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -18,7 +18,7 @@ permissions: on: workflow_call: inputs: - package: + package_name: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true type: string @@ -28,7 +28,7 @@ on: type: string workflow_dispatch: inputs: - package: + package_name: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true type: string @@ -50,9 +50,8 @@ jobs: - name: "[DEBUG] Print Variables" run: | echo "All variables defined as inputs" - echo "The package: ${{ github.event.inputs.package }}" + echo "The package: ${{ github.event.inputs.package_name }}" echo "The version_number: ${{ github.event.inputs.version_number }}" - echo "The version_number: ${{ github.event.inputs.* }}" - name: "Check out the repository" uses: actions/checkout@v3 From 0930c9c05980701676b7f06a4021cff4e0e30046 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:08:11 -0400 Subject: [PATCH 36/47] Update release.yml --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 98e429684cb..1cc8f5dd835 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,7 +139,7 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package: dbt-core + package_name: dbt-core version_number: ${{ inputs.version_number }} @@ -149,6 +149,6 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package: dbt-postgres + package_name: dbt-postgres version_number: ${{ inputs.version_number }} From 65f41a1e36dc63f04f170cf7f0d3d6d59fe48609 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:12:39 -0400 Subject: [PATCH 37/47] Update testing.yml --- .github/workflows/testing.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4b4947aea2a..ef28473a077 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -10,8 +10,8 @@ # some performance regressions at development time before they reach users. # # **when?** -# This is only run once directly after each release. If for some reason the results of -# a run are not satisfactory, it can also be triggered manually. +# This is only run once directly after each release (for non-prereleases). If for some +# reason the results of a run are not satisfactory, it can also be triggered manually. name: Model Performance Characteristics @@ -22,8 +22,6 @@ on: # run manually from the actions tab workflow_dispatch: inputs: - # if we ever want to model pre-releases, we would need to handle pre-release version numbers - # here and in the runner. Even if we use a semver library I suspect it will be rather difficult. release_id: description: 'dbt version to model (must be non-prerelease in Pypi)' type: string @@ -47,6 +45,7 @@ jobs: release_id: ${{ steps.semver.outputs.base-version }} release_branch: ${{ steps.variables.outputs.release_branch }} steps: + # explicitly checkout the performance runner from main regardless of which # version we are modeling. - name: Checkout @@ -210,7 +209,6 @@ jobs: # depends on `model` as a separate job so that the baseline can be committed to more than one branch # i.e. release branch and main needs: [set-variables, latest-runner, model] - runs-on: ubuntu-latest strategy: From ba5ab2114083a0c05bcb19e8bfac77ef9eeb968c Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:27:57 -0400 Subject: [PATCH 38/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 8ffaa9f8f15..aa107d3dbc9 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -50,8 +50,9 @@ jobs: - name: "[DEBUG] Print Variables" run: | echo "All variables defined as inputs" - echo "The package: ${{ github.event.inputs.package_name }}" + echo "The package_name: ${{ github.event.inputs.package_name }}" echo "The version_number: ${{ github.event.inputs.version_number }}" + echo "The ====: ${{ toJSON(github.event.inputs) }}" - name: "Check out the repository" uses: actions/checkout@v3 From da34b80c264b056cd20e6a432ffb2db93fc8351e Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:43:59 -0400 Subject: [PATCH 39/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index aa107d3dbc9..4def820c571 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -50,9 +50,9 @@ jobs: - name: "[DEBUG] Print Variables" run: | echo "All variables defined as inputs" - echo "The package_name: ${{ github.event.inputs.package_name }}" - echo "The version_number: ${{ github.event.inputs.version_number }}" - echo "The ====: ${{ toJSON(github.event.inputs) }}" + echo "The package_name: ${{ inputs.package_name }}" + echo "The version_number: ${{ inputs.version_number }}" + echo "The ====: ${{ toJSON(inputs) }}" - name: "Check out the repository" uses: actions/checkout@v3 From a3fb66daa4a12da9924607f313a7874b78c631f3 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:45:41 -0400 Subject: [PATCH 40/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 4def820c571..41f3104111d 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -22,7 +22,7 @@ on: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true type: string - version_number: + version_num: description: The release version number (i.e. 1.0.0b1). Do not include `latest` tags or a leading `v`! required: true type: string @@ -32,7 +32,7 @@ on: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true type: string - version_number: + version_num: description: The release version number (i.e. 1.0.0b1). Do not include `latest` tags or a leading `v`! required: true type: string @@ -51,7 +51,7 @@ jobs: run: | echo "All variables defined as inputs" echo "The package_name: ${{ inputs.package_name }}" - echo "The version_number: ${{ inputs.version_number }}" + echo "The version_num: ${{ inputs.version_num }}" echo "The ====: ${{ toJSON(inputs) }}" - name: "Check out the repository" @@ -62,7 +62,7 @@ jobs: uses: ./.github/actions/latest-wrangler with: package: ${{ github.event.inputs.package }} - new_version: ${{ github.event.inputs.version_number }} + new_version: ${{ github.event.inputs.version_num }} gh_token: ${{ secrets.GITHUB_TOKEN }} halt_on_missing: False @@ -84,7 +84,7 @@ jobs: id: semver uses: dbt-labs/actions/parse-semver@v1 with: - version: ${{ github.event.inputs.version_number }} + version: ${{ github.event.inputs.version_num }} - name: Get docker build arg id: build_arg From 718482fb025ef12cf57d4bcf1742baf00dc5f42d Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:46:13 -0400 Subject: [PATCH 41/47] Update release.yml --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1cc8f5dd835..ef2ad60e2ae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -140,7 +140,7 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: package_name: dbt-core - version_number: ${{ inputs.version_number }} + version_num: ${{ inputs.version_number }} docker-postgres-release: @@ -150,5 +150,5 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: package_name: dbt-postgres - version_number: ${{ inputs.version_number }} + version_num: ${{ inputs.version_number }} From 0796c84da56a381b9a5bb611df2226b03f14f3f5 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:50:18 -0400 Subject: [PATCH 42/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 41f3104111d..d852c9ac72e 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -22,10 +22,6 @@ on: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true type: string - version_num: - description: The release version number (i.e. 1.0.0b1). Do not include `latest` tags or a leading `v`! - required: true - type: string workflow_dispatch: inputs: package_name: @@ -51,7 +47,7 @@ jobs: run: | echo "All variables defined as inputs" echo "The package_name: ${{ inputs.package_name }}" - echo "The version_num: ${{ inputs.version_num }}" + echo "The version_number: ${{ inputs.version_number }}" echo "The ====: ${{ toJSON(inputs) }}" - name: "Check out the repository" @@ -62,7 +58,7 @@ jobs: uses: ./.github/actions/latest-wrangler with: package: ${{ github.event.inputs.package }} - new_version: ${{ github.event.inputs.version_num }} + new_version: ${{ github.event.inputs.version_number }} gh_token: ${{ secrets.GITHUB_TOKEN }} halt_on_missing: False @@ -84,7 +80,7 @@ jobs: id: semver uses: dbt-labs/actions/parse-semver@v1 with: - version: ${{ github.event.inputs.version_num }} + version: ${{ github.event.inputs.version_number }} - name: Get docker build arg id: build_arg From 70246c3f86817da6bb0c8aeb213698164d62f6d5 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 22:50:35 -0400 Subject: [PATCH 43/47] Update release.yml --- .github/workflows/release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ef2ad60e2ae..54d301ffaec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -140,7 +140,6 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: package_name: dbt-core - version_num: ${{ inputs.version_number }} docker-postgres-release: @@ -150,5 +149,4 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: package_name: dbt-postgres - version_num: ${{ inputs.version_number }} From e7489383a21c414bbcbdd1aecb0616e51a344cab Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 23:11:27 -0400 Subject: [PATCH 44/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index d852c9ac72e..7b11b9d01f4 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -18,13 +18,13 @@ permissions: on: workflow_call: inputs: - package_name: + package: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true type: string workflow_dispatch: inputs: - package_name: + package: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true type: string @@ -46,9 +46,8 @@ jobs: - name: "[DEBUG] Print Variables" run: | echo "All variables defined as inputs" - echo "The package_name: ${{ inputs.package_name }}" + echo "The package: ${{ inputs.package }}" echo "The version_number: ${{ inputs.version_number }}" - echo "The ====: ${{ toJSON(inputs) }}" - name: "Check out the repository" uses: actions/checkout@v3 From e3692a6a3dbf1871cf1ded0604b9bdbe1f7f36c3 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 23:11:40 -0400 Subject: [PATCH 45/47] Update release.yml --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 54d301ffaec..a65cd43d15d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,7 +139,7 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package_name: dbt-core + package: dbt-core docker-postgres-release: @@ -148,5 +148,5 @@ jobs: uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package_name: dbt-postgres + package: dbt-postgres From bcf140b3c10ffc03d299abef23c80e183772dfd5 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Tue, 2 May 2023 23:21:08 -0400 Subject: [PATCH 46/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 7b11b9d01f4..a5d1e88ca84 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -56,8 +56,8 @@ jobs: id: latest uses: ./.github/actions/latest-wrangler with: - package: ${{ github.event.inputs.package }} - new_version: ${{ github.event.inputs.version_number }} + package: ${{ inputs.package }} + new_version: ${{ inputs.version_number }} gh_token: ${{ secrets.GITHUB_TOKEN }} halt_on_missing: False @@ -79,13 +79,13 @@ jobs: id: semver uses: dbt-labs/actions/parse-semver@v1 with: - version: ${{ github.event.inputs.version_number }} + version: ${{ inputs.version_number }} - name: Get docker build arg id: build_arg run: | - BUILD_ARG_NAME=$(echo ${{ github.event.inputs.package }} | sed 's/\-/_/g') - BUILD_ARG_VALUE=$(echo ${{ github.event.inputs.package }} | sed 's/postgres/core/g') + BUILD_ARG_NAME=$(echo ${{ inputs.package }} | sed 's/\-/_/g') + BUILD_ARG_VALUE=$(echo ${{ inputs.package }} | sed 's/postgres/core/g') echo "build_arg_name=$BUILD_ARG_NAME" >> $GITHUB_OUTPUT echo "build_arg_value=$BUILD_ARG_VALUE" >> $GITHUB_OUTPUT @@ -101,11 +101,11 @@ jobs: with: file: docker/Dockerfile push: True - target: ${{ github.event.inputs.package }} + target: ${{ inputs.package }} build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} tags: | - ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.semver.outputs.version }} + ghcr.io/leahwicz/${{ inputs.package }}:${{ steps.semver.outputs.version }} - name: Build and push MINOR.latest tag uses: docker/build-push-action@v4 @@ -113,11 +113,11 @@ jobs: with: file: docker/Dockerfile push: True - target: ${{ github.event.inputs.package }} + target: ${{ inputs.package }} build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} tags: | - ghcr.io/leahwicz/${{ github.event.inputs.package }}:${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.latest + ghcr.io/leahwicz/${{ inputs.package }}:${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.latest - name: Build and push latest tag uses: docker/build-push-action@v4 @@ -125,8 +125,8 @@ jobs: with: file: docker/Dockerfile push: True - target: ${{ github.event.inputs.package }} + target: ${{ inputs.package }} build-args: | ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} tags: | - ghcr.io/leahwicz/${{ github.event.inputs.package }}:latest + ghcr.io/leahwicz/${{ inputs.package }}:latest From 22c40a4766896f93c55094f3de7fef78d9a2dbff Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Wed, 3 May 2023 09:33:31 -0400 Subject: [PATCH 47/47] Update release-docker.yml --- .github/workflows/release-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index a5d1e88ca84..ea36594c1b6 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -28,7 +28,7 @@ on: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true type: string - version_num: + version_number: description: The release version number (i.e. 1.0.0b1). Do not include `latest` tags or a leading `v`! required: true type: string