Skip to content

Commit

Permalink
[CI/CD] Backport release workflow to 1.4.latest (#619)
Browse files Browse the repository at this point in the history 
* [CI/CD] Update release workflow (#592)

* Update release workflow

* Fix format

* Set default `test_run` value to `true`

* Update Slack secret

* Resolve review comments

* Update release workflow (#613)

- Update AWS secrets
- Rework condition for Slack notification

* update regex for version bump (#630)

* update regex for version bump

* update to prekind

* more renaming

* finish up adding nightly release check for future proofing
# Conflicts:
#	.bumpversion.cfg

---------

Co-authored-by: Emily Rockman <emily.rockman@dbtlabs.com>
  • Loading branch information
@alexander-smolyakov @emmyoop
alexander-smolyakov and emmyoop authored Feb 3, 2023
1 parent ff9f9ff commit 5ab6aaf
Show file tree
Hide file tree
Showing 2 changed files with 138 additions and 157 deletions.
24 changes: 18 additions & 6 deletions .bumpversion.cfg
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,26 @@
[bumpversion]
current_version = 1.4.1
parse = (?P<major>\d+)
\.(?P<minor>\d+)
\.(?P<patch>\d+)
((?P<prerelease>a|b|rc)(?P<num>\d+))?

# `parse` allows parsing the version into the parts we need to check. There are some
# unnamed groups and that's okay because they do not need to be audited. If any part
# of the version passed and does not match the regex, it will fail.
# expected matches: `1.5.0`, `1.5.0a1`, `1.5.0a1.dev123457+nightly`
# excepted failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0`
parse = (?P<major>[\d]+) # major version number
\.(?P<minor>[\d]+) # minor version number
\.(?P<patch>[\d]+) # patch version number
(((?P<prekind>a|b|rc) # optional pre-release type
?(?P<num>[\d]+?)) # optional pre-release version number
\.?(?P<nightly>[a-z0-9]+\+[a-z]+)? # optional nightly release indicator
)?
serialize =
{major}.{minor}.{patch}{prerelease}{num}
{major}.{minor}.{patch}{prekind}{num}.{nightly}
{major}.{minor}.{patch}{prekind}{num}
{major}.{minor}.{patch}
commit = False
tag = False

[bumpversion:part:prerelease]
[bumpversion:part:prekind]
first_value = a
optional_value = final
values =
Expand All @@ -22,6 +32,8 @@ values =
[bumpversion:part:num]
first_value = 1

[bumpversion:part:nightly]

[bumpversion:file:setup.py]

[bumpversion:file:dbt/adapters/spark/__version__.py]
271 changes: 120 additions & 151 deletions .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,20 @@
# **what?**
# Take the given commit, run unit tests specifically on that sha, build and
# package it, and then release to GitHub with that specific build (PyPi to follow later)

# Release workflow provides the following steps:
# - checkout the given commit;
# - validate version in sources and changelog file for given version;
# - run unit tests against given commit;
# - build and package that SHA;
# - release it to GitHub and PyPI with that specific build;
#
# **why?**
# Ensure an automated and tested release process

#
# **when?**
# This will only run manually with a given sha and version
# This will only run manually. Run this workflow only after the
# version bump workflow is completed and related changes are reviewed and merged.
#

name: Build, Test, and Package
name: Release to GitHub and PyPI

on:
workflow_dispatch:
Expand All @@ -17,28 +23,43 @@ on:
description: "The last commit sha in the release"
type: string
required: true
changelog_path:
description: "Path to changes log"
target_branch:
description: "The branch to release from"
type: string
default: "./CHANGELOG.md"
required: false
required: true
version_number:
description: "The release version number (i.e. 1.0.0b1)"
type: string
required: true
build_script_path:
description: "Build script path"
type: string
default: "scripts/build-dist.sh"
required: true
s3_bucket_name:
description: "AWS S3 bucket name"
type: string
default: "core-team-artifacts"
required: true
package_test_command:
description: "Package test command"
type: string
default: "dbt --version"
required: true
env_setup_script_path:
description: "Environment setup script path"
type: string
default: ""
required: false
test_run:
description: "Test run (Publish release as draft to GitHub)"
description: "Test run (Publish release as draft)"
type: boolean
default: false
default: true
required: false

permissions:
contents: write # this is the permission that allows creating a new release

env:
PYTHON_TARGET_VERSION: 3.8
ARTIFACT_RETENTION_DAYS: 2

defaults:
run:
shell: bash
Expand All @@ -50,166 +71,114 @@ jobs:
steps:
- name: "[DEBUG] Print Variables"
run: |
echo The last commit sha in the release: ${{ inputs.sha }}
echo The release version number: ${{ inputs.version_number }}
echo The path to the changelog markdpown: ${{ inputs.changelog_path }}
echo This is a test run: ${{ inputs.test_run }}
echo Python target version: ${{ env.PYTHON_TARGET_VERSION }}
echo Artifact retention days: ${{ env.ARTIFACT_RETENTION_DAYS }}
unit:
name: Unit Test
runs-on: ubuntu-latest
echo The last commit sha in the release: ${{ inputs.sha }}
echo The branch to release from: ${{ inputs.target_branch }}
echo The release version number: ${{ inputs.version_number }}
echo Build script path: ${{ inputs.build_script_path }}
echo Environment setup script path: ${{ inputs.env_setup_script_path }}
echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }}
echo Package test command: ${{ inputs.package_test_command }}
echo Test run: ${{ inputs.test_run }}
# The Spark repository uses CircleCI to run integration tests.
# Because of this, the process of version bumps will be manual
# which means that this stage will be used to audit the version
# and changelog in sources.
# We are passing `env_setup_script_path` as an empty string
# so that the integration tests stage will be skipped.
audit-version-and-changelog:
name: Bump package version, Generate changelog

uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main

env:
TOXENV: "unit"
with:
sha: ${{ inputs.sha }}
version_number: ${{ inputs.version_number }}
target_branch: ${{ inputs.target_branch }}
env_setup_script_path: ""
test_run: ${{ inputs.test_run }}

steps:
- name: "Checkout Commit - ${{ inputs.sha }}"
uses: actions/checkout@v3
with:
persist-credentials: false
ref: ${{ github.event.inputs.sha }}

- name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}"
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_TARGET_VERSION }}

- name: "Install Python Dependencies"
run: |
sudo apt-get install libsasl2-dev
python -m pip install --user --upgrade pip
python -m pip install tox
python -m pip --version
python -m tox --version
secrets:
FISHTOWN_BOT_PAT: ${{ secrets.FISHTOWN_BOT_PAT }}

- name: "Run Tox"
run: tox
log-outputs-audit-version-and-changelog:
name: "[Log output] Bump package version, Generate changelog"
if: ${{ !failure() && !cancelled() }}

build:
name: Build Packages
needs: [audit-version-and-changelog]

runs-on: ubuntu-latest

steps:
- name: "Checkout Commit - ${{ inputs.sha }}"
uses: actions/checkout@v3
with:
persist-credentials: false
ref: ${{ inputs.sha }}

- name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}"
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_TARGET_VERSION }}

- name: "Install Python Dependencies"
run: |
sudo apt-get install libsasl2-dev
python -m pip install --user --upgrade pip
python -m pip install --upgrade setuptools wheel twine check-wheel-contents
python -m pip --version
- name: "Build Distributions"
run: ./scripts/build-dist.sh

- name: "[DEBUG] Show Distributions"
run: ls -lh dist/

- name: "Check Distribution Descriptions"
- name: Print variables
run: |
twine check dist/*
- name: "[DEBUG] Check Wheel Contents"
run: |
check-wheel-contents dist/*.whl --ignore W007,W008
- name: "Upload Build Artifact - ${{ inputs.version_number }}"
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.version_number }}
path: |
dist/
!dist/dbt-${{ inputs.version_number }}.tar.gz
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}

test-build:
name: Verify Packages
echo Final SHA : ${{ needs.audit-version-and-changelog.outputs.final_sha }}
echo Changelog path: ${{ needs.audit-version-and-changelog.outputs.changelog_path }}
needs: [unit, build]

runs-on: ubuntu-latest
build-test-package:
name: Build, Test, Package
if: ${{ !failure() && !cancelled() }}
needs: [audit-version-and-changelog]

steps:
- name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}"
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_TARGET_VERSION }}
uses: dbt-labs/dbt-release/.github/workflows/build.yml@main

- name: "Install Python Dependencies"
run: |
sudo apt-get install libsasl2-dev
python -m pip install --user --upgrade pip
python -m pip install --upgrade wheel
python -m pip --version
with:
sha: ${{ needs.audit-version-and-changelog.outputs.final_sha }}
version_number: ${{ inputs.version_number }}
changelog_path: ${{ needs.audit-version-and-changelog.outputs.changelog_path }}
build_script_path: ${{ inputs.build_script_path }}
s3_bucket_name: ${{ inputs.s3_bucket_name }}
package_test_command: ${{ inputs.package_test_command }}
test_run: ${{ inputs.test_run }}

- name: "Download Build Artifact - ${{ inputs.version_number }}"
uses: actions/download-artifact@v3
with:
name: ${{ inputs.version_number }}
path: dist/
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

- name: "[DEBUG] Show Distributions"
run: ls -lh dist/
github-release:
name: GitHub Release
if: ${{ !failure() && !cancelled() }}

- name: "Install Wheel Distributions"
run: |
find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
needs: [audit-version-and-changelog, build-test-package]

- name: "[DEBUG] Check Wheel Distributions"
run: |
dbt --version
uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main

- name: "Install Source Distributions"
run: |
find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
with:
sha: ${{ needs.audit-version-and-changelog.outputs.final_sha }}
version_number: ${{ inputs.version_number }}
changelog_path: ${{ needs.audit-version-and-changelog.outputs.changelog_path }}
test_run: ${{ inputs.test_run }}

- name: "[DEBUG] Check Source Distributions"
run: |
dbt --version
pypi-release:
name: PyPI Release

github-release:
name: GitHub Release
if: ${{ !failure() && !cancelled() }}
needs: test-build
needs: [github-release]

# pin to commit since this is workflow is WIP but this commit has been tested as working
uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@7b6e01d73d2c8454e06302cc66ef4c2dbd4dbe4e
uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main

with:
sha: ${{ inputs.sha }}
version_number: ${{ inputs.version_number }}
changelog_path: ${{ inputs.changelog_path }}
test_run: ${{ inputs.test_run }}

pypi-release:
name: Pypi release
# only release to PyPi if we're not testing - will release to PyPi test when workflow gets rewritten
if: inputs.test_run == 'false'
secrets:
PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}

runs-on: ubuntu-latest
slack-notification:
name: Slack Notification
if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }}

needs: github-release
needs:
[
audit-version-and-changelog,
build-test-package,
github-release,
pypi-release,
]

environment: PypiProd
steps:
- uses: actions/download-artifact@v2
with:
name: dist
path: 'dist'

- name: Publish distribution to PyPI
uses: pypa/gh-action-pypi-publish@v1.4.2
with:
password: ${{ secrets.PYPI_API_TOKEN }}
uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main
with:
status: "failure"

secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }}

0 comments on commit 5ab6aaf

Please sign in to comment.