diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index ae83eb5f8a3..7e33e9a636e 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -16,41 +16,49 @@ permissions: packages: write on: + workflow_call: + inputs: + package: + description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] + required: true + type: string workflow_dispatch: inputs: package: description: The package to release. _One_ of [dbt-core, dbt-redshift, dbt-bigquery, dbt-snowflake, dbt-spark, dbt-postgres] required: true + type: string version_number: description: The release version number (i.e. 1.0.0b1). Do not include `latest` tags or a leading `v`! required: true + type: string jobs: get_version_meta: - name: Get version meta + name: Get version metadata runs-on: ubuntu-latest + outputs: - major: ${{ steps.version.outputs.major }} - minor: ${{ steps.version.outputs.minor }} - patch: ${{ steps.version.outputs.patch }} latest: ${{ steps.latest.outputs.latest }} minor_latest: ${{ steps.latest.outputs.minor_latest }} + steps: - - uses: actions/checkout@v3 - - name: Split version - id: version + - name: "[DEBUG] Print Variables" + run: | - IFS="." read -r MAJOR MINOR PATCH <<< ${{ github.event.inputs.version_number }} - echo "major=$MAJOR" >> $GITHUB_OUTPUT - echo "minor=$MINOR" >> $GITHUB_OUTPUT - echo "patch=$PATCH" >> $GITHUB_OUTPUT + echo "All variables defined as inputs" + echo "The package: ${{ inputs.package }}" + echo "The version_number: ${{ inputs.version_number }}" + + - name: "Check out the repository" + uses: actions/checkout@v3 - name: Is pkg 'latest' id: latest uses: ./.github/actions/latest-wrangler with: - package: ${{ github.event.inputs.package }} - new_version: ${{ github.event.inputs.version_number }} + package: ${{ inputs.package }} + new_version: ${{ inputs.version_number }} gh_token: ${{ secrets.GITHUB_TOKEN }} halt_on_missing: False @@ -67,11 +75,18 @@ jobs: runs-on: ubuntu-latest needs: [setup_image_builder, get_version_meta] steps: + + - name: "Parse Version Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1 + with: + version: ${{ inputs.version_number }} + - name: Get docker build arg id: build_arg run: | - BUILD_ARG_NAME=$(echo ${{ github.event.inputs.package }} | sed 's/\-/_/g') - BUILD_ARG_VALUE=$(echo ${{ github.event.inputs.package }} | sed 's/postgres/core/g') + BUILD_ARG_NAME=$(echo ${{ inputs.package }} | sed 's/\-/_/g') + BUILD_ARG_VALUE=$(echo ${{ inputs.package }} | sed 's/postgres/core/g') echo "build_arg_name=$BUILD_ARG_NAME" >> $GITHUB_OUTPUT echo "build_arg_value=$BUILD_ARG_VALUE" >> $GITHUB_OUTPUT @@ -87,11 +102,11 @@ jobs: with: file: docker/Dockerfile push: True - target: ${{ github.event.inputs.package }} + target: ${{ inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} tags: | - ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ github.event.inputs.version_number }} + ghcr.io/leahwicz/${{ inputs.package }}:${{ steps.semver.outputs.version }} - name: Build and push MINOR.latest tag uses: docker/build-push-action@v4 @@ -99,11 +114,11 @@ jobs: with: file: docker/Dockerfile push: True - target: ${{ github.event.inputs.package }} + target: ${{ inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} tags: | - ghcr.io/dbt-labs/${{ github.event.inputs.package }}:${{ needs.get_version_meta.outputs.major }}.${{ needs.get_version_meta.outputs.minor }}.latest + ghcr.io/leahwicz/${{ inputs.package }}:${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.latest - name: Build and push latest tag uses: docker/build-push-action@v4 @@ -111,8 +126,8 @@ jobs: with: file: docker/Dockerfile push: True - target: ${{ github.event.inputs.package }} + target: ${{ inputs.package }} build-args: | - ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ github.event.inputs.version_number }} + ${{ steps.build_arg.outputs.build_arg_name }}_ref=${{ steps.build_arg.outputs.build_arg_value }}@v${{ steps.semver.outputs.version }} tags: | - ghcr.io/dbt-labs/${{ github.event.inputs.package }}:latest + ghcr.io/leahwicz/${{ inputs.package }}:latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e3722ea19cd..a65cd43d15d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -108,13 +108,14 @@ on: permissions: contents: write # this is the permission that allows creating a new release + packages: write defaults: run: shell: bash jobs: - log-inputs: + pypi-release: name: Log Inputs runs-on: ubuntu-latest steps: @@ -130,100 +131,22 @@ jobs: echo Test run: ${{ inputs.test_run }} echo Nightly release: ${{ inputs.nightly_release }} - bump-version-generate-changelog: - name: Bump package version, Generate changelog - - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main - - with: - sha: ${{ inputs.sha }} - version_number: ${{ inputs.version_number }} - target_branch: ${{ inputs.target_branch }} - env_setup_script_path: ${{ inputs.env_setup_script_path }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - - secrets: inherit - - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} - - needs: [bump-version-generate-changelog] - runs-on: ubuntu-latest - - steps: - - name: Print variables - run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - - build-test-package: - name: Build, Test, Package - if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog] - - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main + docker-core-release: + name: Docker dbt-core Release + if: ${{ !inputs.test_run }} + needs: [pypi-release] + uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build_script_path: ${{ inputs.build_script_path }} - s3_bucket_name: ${{ inputs.s3_bucket_name }} - package_test_command: ${{ inputs.package_test_command }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - github-release: - name: GitHub Release - if: ${{ !failure() && !cancelled() }} - - needs: [bump-version-generate-changelog, build-test-package] - - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.test_run }} - - pypi-release: - name: PyPI Release - - needs: [github-release] - - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - - with: - version_number: ${{ inputs.version_number }} - test_run: ${{ inputs.test_run }} - - secrets: - PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - - slack-notification: - name: Slack Notification - if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} + package: dbt-core + - needs: - [ - bump-version-generate-changelog, - build-test-package, - github-release, - pypi-release, - ] + docker-postgres-release: + name: Docker dbt-postgres Release + needs: [docker-core-release] - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main + uses: leahwicz/dbt-core/.github/workflows/release-docker.yml@main with: - status: "failure" + package: dbt-postgres - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 00000000000..ef28473a077 --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,268 @@ +# **what?** +# This workflow models the performance characteristics of a point in time in dbt. +# It runs specific dbt commands on committed projects multiple times to create and +# commit information about the distribution to the current branch. For more information +# see the readme in the performance module at /performance/README.md. +# +# **why?** +# When developing new features, we can take quick performance samples and compare +# them against the commited baseline measurements produced by this workflow to detect +# some performance regressions at development time before they reach users. +# +# **when?** +# This is only run once directly after each release (for non-prereleases). If for some +# reason the results of a run are not satisfactory, it can also be triggered manually. + +name: Model Performance Characteristics + +on: + # runs after non-prereleases are published. + release: + types: [released] + # run manually from the actions tab + workflow_dispatch: + inputs: + release_id: + description: 'dbt version to model (must be non-prerelease in Pypi)' + type: string + required: true + pull_request: # TODO: remove, just for testing purposes + +env: + RUNNER_CACHE_PATH: performance/runner/target/release/runner + +# both jobs need to write +permissions: + contents: write + pull-requests: write + +jobs: + set-variables: + name: Setting Variables + runs-on: ubuntu-latest + outputs: + cache_key: ${{ steps.variables.outputs.cache_key }} + release_id: ${{ steps.semver.outputs.base-version }} + release_branch: ${{ steps.variables.outputs.release_branch }} + steps: + + # explicitly checkout the performance runner from main regardless of which + # version we are modeling. + - name: Checkout + uses: actions/checkout@v3 + with: + ref: main + + - name: Parse version into parts + id: semver + uses: dbt-labs/actions/parse-semver@v1 + with: + version: ${{ github.event.inputs.release_id || github.event.release.tag_name }} + + # collect all the variables that need to be used in subsequent jobs + - name: Set variables + id: variables + run: | + # create a cache key that will be used in the next job. without this the + # next job would have to checkout from main and hash the files itself. + echo "cache_key=${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" >> $GITHUB_OUTPUT + + branch_name="${{steps.semver.outputs.major}}.${{steps.semver.outputs.minor}}.latest" + echo "release_branch=$branch_name" >> $GITHUB_OUTPUT + echo "release branch is inferred to be ${branch_name}" + + latest-runner: + name: Build or Fetch Runner + runs-on: ubuntu-latest + needs: [set-variables] + env: + RUSTFLAGS: "-D warnings" + steps: + - name: '[DEBUG] print variables' + run: | + echo "all variables defined in set-variables" + echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}" + echo "release_id: ${{ needs.set-variables.outputs.release_id }}" + echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" + + # explicitly checkout the performance runner from main regardless of which + # version we are modeling. + - name: Checkout + uses: actions/checkout@v3 + with: + ref: main + + # attempts to access a previously cached runner + - uses: actions/cache@v3 + id: cache + with: + path: ${{ env.RUNNER_CACHE_PATH }} + key: ${{ needs.set-variables.outputs.cache_key }} + + - name: Fetch Rust Toolchain + if: steps.cache.outputs.cache-hit != 'true' + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - name: Add fmt + if: steps.cache.outputs.cache-hit != 'true' + run: rustup component add rustfmt + + - name: Cargo fmt + if: steps.cache.outputs.cache-hit != 'true' + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --manifest-path performance/runner/Cargo.toml --all -- --check + + - name: Test + if: steps.cache.outputs.cache-hit != 'true' + uses: actions-rs/cargo@v1 + with: + command: test + args: --manifest-path performance/runner/Cargo.toml + + - name: Build (optimized) + if: steps.cache.outputs.cache-hit != 'true' + uses: actions-rs/cargo@v1 + with: + command: build + args: --release --manifest-path performance/runner/Cargo.toml + # the cache action automatically caches this binary at the end of the job + + model: + # depends on `latest-runner` as a separate job so that failures in this job do not prevent + # a successfully tested and built binary from being cached. + needs: [set-variables, latest-runner] + name: Model a release + runs-on: ubuntu-latest + steps: + + - name: '[DEBUG] print variables' + run: | + echo "all variables defined in set-variables" + echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}" + echo "release_id: ${{ needs.set-variables.outputs.release_id }}" + echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - name: Install dbt + run: pip install dbt-postgres==${{ needs.set-variables.outputs.release_id }} + + - name: Install Hyperfine + run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb + + # explicitly checkout main to get the latest project definitions + - name: Checkout + uses: actions/checkout@v3 + with: + ref: main + + # this was built in the previous job so it will be there. + - name: Fetch Runner + uses: actions/cache@v3 + id: cache + with: + path: ${{ env.RUNNER_CACHE_PATH }} + key: ${{ needs.set-variables.outputs.cache_key }} + + - name: Move Runner + run: mv performance/runner/target/release/runner performance/app + + - name: Change Runner Permissions + run: chmod +x ./performance/app + + - name: '[DEBUG] ls baseline directory before run' + run: ls -R performance/baselines/ + + # `${{ github.workspace }}` is used to pass the absolute path + - name: Create directories + run: | + mkdir ${{ github.workspace }}/performance/tmp/ + mkdir -p performance/baselines/${{ needs.set-variables.outputs.release_id }}/ + + # TODO CHANGE NUMBER OF RUNS BEFORE MERGING + # TODO this isn't putting the baseline in the right directory. it's putting it one level up. + # Run modeling with taking 20 samples + - name: Run Measurement + run: | + performance/app model -v ${{ needs.set-variables.outputs.release_id }} -b ${{ github.workspace }}/performance/baselines/ -p ${{ github.workspace }}/performance/projects/ -t ${{ github.workspace }}/performance/tmp/ -n 2 + + - name: '[DEBUG] ls baseline directory after run' + run: ls -R performance/baselines/ + + - uses: actions/upload-artifact@v3 + with: + name: baseline + path: performance/baselines/${{ needs.set-variables.outputs.release_id }}/ + + create-pr: + name: Open PR for ${{ matrix.base-branch }} + + # depends on `model` as a separate job so that the baseline can be committed to more than one branch + # i.e. release branch and main + needs: [set-variables, latest-runner, model] + runs-on: ubuntu-latest + + strategy: + matrix: + include: + - base-branch: refs/heads/main + target-branch: performance-bot/main_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}} + - base-branch: refs/heads/${{ needs.set-variables.outputs.release_branch }} + target-branch: performance-bot/release_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}} + + steps: + - name: '[DEBUG] print variables' + run: | + echo "all variables defined in set-variables" + echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}" + echo "release_id: ${{ needs.set-variables.outputs.release_id }}" + echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}" + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ matrix.base-branch }} + + - name: Create PR branch + run: | + git checkout -b ${{ matrix.target-branch }} + git push origin ${{ matrix.target-branch }} + git branch --set-upstream-to=origin/${{ matrix.target-branch }} ${{ matrix.target-branch }} + + - uses: actions/download-artifact@v3 + with: + name: baseline + path: performance/baselines/${{ needs.set-variables.outputs.release_id }} + + - name: '[DEBUG] ls baselines after artifact download' + run: ls -R performance/baselines/ + + - name: Commit baseline + uses: EndBug/add-and-commit@v9 + with: + add: 'performance/baselines/*' + author_name: 'Github Build Bot' + author_email: 'buildbot@fishtownanalytics.com' + message: 'adding performance baseline for ${{ needs.set-variables.outputs.release_id }}' + push: 'origin origin/${{ matrix.target-branch }}' + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + author: 'Github Build Bot ' + base: ${{ matrix.base-branch }} + branch: '${{ matrix.target-branch }}' + title: 'Adding performance modeling for ${{needs.set-variables.outputs.release_id}} to ${{ matrix.base-branch }}' + body: 'Committing perf results for tracking for the ${{needs.set-variables.outputs.release_id}}' + labels: | + Skip Changelog + Performance