From 24777517a576febf43c740f23a122de4f8bea1a5 Mon Sep 17 00:00:00 2001 From: Leah Antkiewicz Date: Mon, 11 Sep 2023 21:49:51 -0400 Subject: [PATCH] Revert "Merge branch 'main' into main" This reverts commit c93cba460301a20a2938428cda1788eb44c1b93a, reversing changes made to 971669016f48acd7338ea8df417bd95be9696835. --- .github/workflows/release-docker.yml | 63 +++---- .github/workflows/release.yml | 105 +++++++++-- .github/workflows/testing.yml | 268 --------------------------- 3 files changed, 115 insertions(+), 321 deletions(-) delete mode 100644 .github/workflows/testing.yml diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 7e33e9a636e..ae83eb5f8a3 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -16,49 +16,41 @@ permissions: packages: write on: - workflow_call: - inputs: - package: - description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] - required: true - type: string workflow_dispatch: inputs: package: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true - type: string version_number: description: The release version number (i.e. 1.0.0b1). Do not include `latest` tags or a leading `v`! required: true - type: string jobs: get_version_meta: - name: Get version metadata + name: Get version meta runs-on: ubuntu-latest - outputs: + major: ${{ steps.version.outputs.major }} + minor: ${{ steps.version.outputs.minor }} + patch: ${{ steps.version.outputs.patch }} latest: ${{ steps.latest.outputs.latest }} minor_latest: ${{ steps.latest.outputs.minor_latest }} - steps: - - name: "[DEBUG] Print Variables" - + - uses: actions/checkout@v3 + - name: Split version + id: version run: | - echo "All variables defined as inputs" - echo "The package: ${{ inputs.package }}" - echo "The version_number: ${{ inputs.version_number }}" - - - name: "Check out the repository" - uses: actions/checkout@v3 + IFS="." read -r MAJOR MINOR PATCH <<< ${{ github.event.inputs.version_number }} + echo "major=$MAJOR" >> $GITHUB_OUTPUT + echo "minor=$MINOR" >> $GITHUB_OUTPUT + echo "patch=$PATCH" >> $GITHUB_OUTPUT - name: Is pkg 'latest' id: latest uses: ./.github/actions/latest-wrangler with: - package: ${{ inputs.package }} - new_version: ${{ inputs.version_number }} + package: ${{ github.event.inputs.package }} + new_version: ${{ github.event.inputs.version_number }} gh_token: ${{ secrets.GITHUB_TOKEN }} halt_on_missing: False @@ -75,18 +67,11 @@ jobs: runs-on: ubuntu-latest needs: [setup_image_builder, get_version_meta] steps: - - - name: "Parse Version Into Parts" - id: semver - uses: dbt-labs/actions/parse-semver@v1 - with: - version: ${{ inputs.version_number }} - - name: Get docker build arg id: build_arg run: | - BUILD_ARG_NAME=$(echo ${{ inputs.package }} | sed 's/\-/_/g') - BUILD_ARG_VALUE=$(echo ${{ inputs.package }} | sed 's/postgres/core/g') + BUILD_ARG_NAME=$(echo ${{ github.event.inputs.package }} | sed 's/\-/_/g') + BUILD_ARG_VALUE=$(echo ${{ github.event.inputs.package }} | sed 's/postgres/core/g') echo "build_arg_name=$BUILD_ARG_NAME" >> $GITHUB_OUTPUT echo "build_arg_value=$BUILD_ARG_VALUE" >> $GITHUB_OUTPUT @@ -102,11 +87,11 @@ jobs: with: file: docker/Dockerfile push: True - target: ${{ inputs.package }} + target: ${{ github.event.inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/leahwicz/${{ inputs.package }}:${{ steps.semver.outputs.version }} + ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ github.event.inputs.version_number }} - name: Build and push MINOR.latest tag uses: docker/build-push-action@v4 @@ -114,11 +99,11 @@ jobs: with: file: docker/Dockerfile push: True - target: ${{ inputs.package }} + target: ${{ github.event.inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/leahwicz/${{ inputs.package }}:${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.latest + ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ needs.get_version_meta.outputs.major }}.${{ needs.get_version_meta.outputs.minor }}.latest - name: Build and push latest tag uses: docker/build-push-action@v4 @@ -126,8 +111,8 @@ jobs: with: file: docker/Dockerfile push: True - target: ${{ inputs.package }} + target: ${{ github.event.inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} tags: | - ghcr.io/leahwicz/${{ inputs.package }}:latest + ghcr.io/dbt-labs/${{ github.event.inputs.package }}:latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a65cd43d15d..e3722ea19cd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -108,14 +108,13 @@ on: permissions: contents: write # this is the permission that allows creating a new release - packages: write defaults: run: shell: bash jobs: - pypi-release: + log-inputs: name: Log Inputs runs-on: ubuntu-latest steps: @@ -131,22 +130,100 @@ jobs: echo Test run: ${{ inputs.test_run }} echo Nightly release: ${{ inputs.nightly_release }} + bump-version-generate-changelog: + name: Bump package version, Generate changelog + + uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main + + with: + sha: ${{ inputs.sha }} + version_number: ${{ inputs.version_number }} + target_branch: ${{ inputs.target_branch }} + env_setup_script_path: ${{ inputs.env_setup_script_path }} + test_run: ${{ inputs.test_run }} + nightly_release: ${{ inputs.nightly_release }} + + secrets: inherit + + log-outputs-bump-version-generate-changelog: + name: "[Log output] Bump package version, Generate changelog" + if: ${{ !failure() && !cancelled() }} + + needs: [bump-version-generate-changelog] - docker-core-release: - name: Docker dbt-core Release - if: ${{ !inputs.test_run }} - needs: [pypi-release] + runs-on: ubuntu-latest + + steps: + - name: Print variables + run: | + echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} + echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + + build-test-package: + name: Build, Test, Package + if: ${{ !failure() && !cancelled() }} + needs: [bump-version-generate-changelog] + + uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - package: dbt-core - + sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} + version_number: ${{ inputs.version_number }} + changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + build_script_path: ${{ inputs.build_script_path }} + s3_bucket_name: ${{ inputs.s3_bucket_name }} + package_test_command: ${{ inputs.package_test_command }} + test_run: ${{ inputs.test_run }} + nightly_release: ${{ inputs.nightly_release }} + + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + github-release: + name: GitHub Release + if: ${{ !failure() && !cancelled() }} + + needs: [bump-version-generate-changelog, build-test-package] + + uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main + + with: + sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} + version_number: ${{ inputs.version_number }} + changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + test_run: ${{ inputs.test_run }} + + pypi-release: + name: PyPI Release + + needs: [github-release] + + uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main + + with: + version_number: ${{ inputs.version_number }} + test_run: ${{ inputs.test_run }} + + secrets: + PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + + slack-notification: + name: Slack Notification + if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - docker-postgres-release: - name: Docker dbt-postgres Release - needs: [docker-core-release] + needs: + [ + bump-version-generate-changelog, + build-test-package, + github-release, + pypi-release, + ] - uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main + uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main with: - package: dbt-postgres + status: "failure" + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml deleted file mode 100644 index ef28473a077..00000000000 --- a/.github/workflows/testing.yml +++ /dev/null @@ -1,268 +0,0 @@ -# **what?** -# This workflow models the performance characteristics of a point in time in dbt. -# It runs specific dbt commands on committed projects multiple times to create and -# commit information about the distribution to the current branch. For more information -# see the readme in the performance module at /performance/README.md. -# -# **why?** -# When developing new features, we can take quick performance samples and compare -# them against the commited baseline measurements produced by this workflow to detect -# some performance regressions at development time before they reach users. -# -# **when?** -# This is only run once directly after each release (for non-prereleases). If for some -# reason the results of a run are not satisfactory, it can also be triggered manually. - -name: Model Performance Characteristics - -on: - # runs after non-prereleases are published. - release: - types: [released] - # run manually from the actions tab - workflow_dispatch: - inputs: - release_id: - description: 'dbt version to model (must be non-prerelease in Pypi)' - type: string - required: true - pull_request: # TODO: remove, just for testing purposes - -env: - RUNNER_CACHE_PATH: performance/runner/target/release/runner - -# both jobs need to write -permissions: - contents: write - pull-requests: write - -jobs: - set-variables: - name: Setting Variables - runs-on: ubuntu-latest - outputs: - cache_key: ${{ steps.variables.outputs.cache_key }} - release_id: ${{ steps.semver.outputs.base-version }} - release_branch: ${{ steps.variables.outputs.release_branch }} - steps: - - # explicitly checkout the performance runner from main regardless of which - # version we are modeling. - - name: Checkout - uses: actions/checkout@v3 - with: - ref: main - - - name: Parse version into parts - id: semver - uses: dbt-labs/actions/parse-semver@v1 - with: - version: ${{ github.event.inputs.release_id || github.event.release.tag_name }} - - # collect all the variables that need to be used in subsequent jobs - - name: Set variables - id: variables - run: | - # create a cache key that will be used in the next job. without this the - # next job would have to checkout from main and hash the files itself. - echo "cache_key=${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" >> $GITHUB_OUTPUT - - branch_name="${{steps.semver.outputs.major}}.${{steps.semver.outputs.minor}}.latest" - echo "release_branch=$branch_name" >> $GITHUB_OUTPUT - echo "release branch is inferred to be ${branch_name}" - - latest-runner: - name: Build or Fetch Runner - runs-on: ubuntu-latest - needs: [set-variables] - env: - RUSTFLAGS: "-D warnings" - steps: - - name: '[DEBUG] print variables' - run: | - echo "all variables defined in set-variables" - echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}" - echo "release_id: ${{ needs.set-variables.outputs.release_id }}" - echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" - - # explicitly checkout the performance runner from main regardless of which - # version we are modeling. - - name: Checkout - uses: actions/checkout@v3 - with: - ref: main - - # attempts to access a previously cached runner - - uses: actions/cache@v3 - id: cache - with: - path: ${{ env.RUNNER_CACHE_PATH }} - key: ${{ needs.set-variables.outputs.cache_key }} - - - name: Fetch Rust Toolchain - if: steps.cache.outputs.cache-hit != 'true' - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - - name: Add fmt - if: steps.cache.outputs.cache-hit != 'true' - run: rustup component add rustfmt - - - name: Cargo fmt - if: steps.cache.outputs.cache-hit != 'true' - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --manifest-path performance/runner/Cargo.toml --all -- --check - - - name: Test - if: steps.cache.outputs.cache-hit != 'true' - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path performance/runner/Cargo.toml - - - name: Build (optimized) - if: steps.cache.outputs.cache-hit != 'true' - uses: actions-rs/cargo@v1 - with: - command: build - args: --release --manifest-path performance/runner/Cargo.toml - # the cache action automatically caches this binary at the end of the job - - model: - # depends on `latest-runner` as a separate job so that failures in this job do not prevent - # a successfully tested and built binary from being cached. - needs: [set-variables, latest-runner] - name: Model a release - runs-on: ubuntu-latest - steps: - - - name: '[DEBUG] print variables' - run: | - echo "all variables defined in set-variables" - echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}" - echo "release_id: ${{ needs.set-variables.outputs.release_id }}" - echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install dbt - run: pip install dbt-postgres==${{ needs.set-variables.outputs.release_id }} - - - name: Install Hyperfine - run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb - - # explicitly checkout main to get the latest project definitions - - name: Checkout - uses: actions/checkout@v3 - with: - ref: main - - # this was built in the previous job so it will be there. - - name: Fetch Runner - uses: actions/cache@v3 - id: cache - with: - path: ${{ env.RUNNER_CACHE_PATH }} - key: ${{ needs.set-variables.outputs.cache_key }} - - - name: Move Runner - run: mv performance/runner/target/release/runner performance/app - - - name: Change Runner Permissions - run: chmod +x ./performance/app - - - name: '[DEBUG] ls baseline directory before run' - run: ls -R performance/baselines/ - - # `${{ github.workspace }}` is used to pass the absolute path - - name: Create directories - run: | - mkdir ${{ github.workspace }}/performance/tmp/ - mkdir -p performance/baselines/${{ needs.set-variables.outputs.release_id }}/ - - # TODO CHANGE NUMBER OF RUNS BEFORE MERGING - # TODO this isn't putting the baseline in the right directory. it's putting it one level up. - # Run modeling with taking 20 samples - - name: Run Measurement - run: | - performance/app model -v ${{ needs.set-variables.outputs.release_id }} -b ${{ github.workspace }}/performance/baselines/ -p ${{ github.workspace }}/performance/projects/ -t ${{ github.workspace }}/performance/tmp/ -n 2 - - - name: '[DEBUG] ls baseline directory after run' - run: ls -R performance/baselines/ - - - uses: actions/upload-artifact@v3 - with: - name: baseline - path: performance/baselines/${{ needs.set-variables.outputs.release_id }}/ - - create-pr: - name: Open PR for ${{ matrix.base-branch }} - - # depends on `model` as a separate job so that the baseline can be committed to more than one branch - # i.e. release branch and main - needs: [set-variables, latest-runner, model] - runs-on: ubuntu-latest - - strategy: - matrix: - include: - - base-branch: refs/heads/main - target-branch: performance-bot/main_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}} - - base-branch: refs/heads/${{ needs.set-variables.outputs.release_branch }} - target-branch: performance-bot/release_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}} - - steps: - - name: '[DEBUG] print variables' - run: | - echo "all variables defined in set-variables" - echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}" - echo "release_id: ${{ needs.set-variables.outputs.release_id }}" - echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" - - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ matrix.base-branch }} - - - name: Create PR branch - run: | - git checkout -b ${{ matrix.target-branch }} - git push origin ${{ matrix.target-branch }} - git branch --set-upstream-to=origin/${{ matrix.target-branch }} ${{ matrix.target-branch }} - - - uses: actions/download-artifact@v3 - with: - name: baseline - path: performance/baselines/${{ needs.set-variables.outputs.release_id }} - - - name: '[DEBUG] ls baselines after artifact download' - run: ls -R performance/baselines/ - - - name: Commit baseline - uses: EndBug/add-and-commit@v9 - with: - add: 'performance/baselines/*' - author_name: 'Github Build Bot' - author_email: 'buildbot@fishtownanalytics.com' - message: 'adding performance baseline for ${{ needs.set-variables.outputs.release_id }}' - push: 'origin origin/${{ matrix.target-branch }}' - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 - with: - author: 'Github Build Bot ' - base: ${{ matrix.base-branch }} - branch: '${{ matrix.target-branch }}' - title: 'Adding performance modeling for ${{needs.set-variables.outputs.release_id}} to ${{ matrix.base-branch }}' - body: 'Committing perf results for tracking for the ${{needs.set-variables.outputs.release_id}}' - labels: | - Skip Changelog - Performance