From f01cbe2a9a10929557d3b78d32152df54ba533c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Thu, 18 Jul 2024 02:06:48 +0200 Subject: [PATCH] Release automation (#9687) * ci: Add workflow for code-freeze Signed-off-by: Oliver Koenig * ci: Add workflow for releasing NeMo Tookit Signed-off-by: Oliver Koenig --------- Signed-off-by: Oliver Koenig --- .github/workflows/release-freeze.yml | 192 +++++++++++++++++++++++++++ .github/workflows/release.yml | 179 +++++++++++++++++++++++++ MANIFEST.in | 1 + 3 files changed, 372 insertions(+) create mode 100644 .github/workflows/release-freeze.yml create mode 100644 .github/workflows/release.yml create mode 100644 MANIFEST.in diff --git a/.github/workflows/release-freeze.yml b/.github/workflows/release-freeze.yml new file mode 100644 index 000000000000..f8d037271f36 --- /dev/null +++ b/.github/workflows/release-freeze.yml @@ -0,0 +1,192 @@ +name: "NeMo Code freeze" + +on: + workflow_dispatch: + inputs: + next_version: + description: 'MAJOR.MINOR.PATCH[rcN] (Example: 2.0.0rc1, or 2.1.0)' + required: true + type: string + mcore_version: + description: 'Version of MCore to use (must be a valid git ref)' + required: true + type: string +jobs: + create-release-branch: + runs-on: ubuntu-latest + if: contains(fromJSON('["ko3n1g"]'), github.actor) + environment: + name: main + outputs: + version: ${{ steps.release-branch.outputs.version }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + path: ${{ github.run_id }} + fetch-depth: 0 + fetch-tags: true + ref: main + + - name: Get Previous tag + id: previous-tag + # git for-each-ref --sort=-creatordate --format '%(refname)' refs/tags ==> refs/tags/vX.Y.Z in descending order of date + # awk 'FNR == 2 {print substr($1, 11, length($1))}') ==> Selects the 2nd tag from the list, then strips the /refs/tags/ part of the tag + # set-output name=tag_name:: ==> Takes the clean tag vX.Y.Z and sets it to steps.previous_tag.outputs.tag_name + run: | + TAG=$(git for-each-ref --sort=-creatordate --format '%(refname)' refs/tags | awk 'FNR == 2 {print substr($1, 11, length($1))}') + echo "tag-name=$TAG" >> "$GITHUB_OUTPUT" + + - name: Get release branch ref + id: release-branch + run: | + cd ${{ github.run_id }} + + VERSION=$(python -c 'import nemo; print(nemo.__version__)') + echo "Release version r$VERSION" > version + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Pin branch name in Notebooks + run: | + cd ${{ github.run_id }} + find tutorials -type f -name "*.ipynb" -exec sed -i "s/BRANCH = 'main'/BRANCH = 'r${{ steps.release-branch.outputs.version }}'/g" {} + + + - name: Pin MCore in Dockerfile + run: | + cd ${{ github.run_id }} + sed -i 's/^ARG MCORE_TAG=.*$/ARG MCORE_TAG=${{ inputs.mcore_version }}/' Dockerfile.ci + + - name: Build Changelog + id: build-changelog + uses: mikepenz/release-changelog-builder-action@v3.3.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # Configuration file is setup with filters for domains + # owner:repo must point to current repo + # fromTag: Auto resolved from historical tag order (previous tag compared to current tag) + # toTag: Current tag reference + configuration: ".github/workflows/config/changelog-config.json" + owner: ${{ github.repository_owner }} + repo: ${{ github.event.repository.name }} + ignorePreReleases: "false" + failOnError: "false" + fromTag: ${{ steps.previous-tag.outputs.tag-name }} + toTag: main + + - name: Append Changelog + run: | + echo "${{ steps.build-changelog.outputs.changelog }}" + + - name: Create Release PR + uses: peter-evans/create-pull-request@v6 + id: create-pull-request + with: + path: ${{ github.run_id }} + branch: r${{ steps.release-branch.outputs.version }} + title: 'Release `${{ steps.release-branch.outputs.version }}`' + body: | + 🚀 PR to release NeMo `${{ steps.release-branch.outputs.version }}`. + + 📝 Please remember the following to-do's before merge: + - [ ] Fill-in the comment `Highlights` + - [ ] Review the comment `Detailed Changelogs` + + 🚨 Please also keep in mind to _not_ delete the headings of the task commits. They are required by the post-merge automation. + + 🙏 Please merge this PR only if the CI workflow completed successfully. + + commit-message: "[🤠]: Howdy folks, let's release NeMo `${{ steps.release-branch.outputs.version }}` !" + signoff: true + assignees: okoenig + labels: 'Run CICD' + + - name: Add Summary comment + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ steps.create-pull-request.outputs.pull-request-number }} + body: | + # Highlights + __ + + - name: Add Changelog comment + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ steps.create-pull-request.outputs.pull-request-number }} + body: | + # Detailed Changelogs + ${{ steps.build-changelog.outputs.changelog }} + + bump-next-version: + runs-on: ubuntu-latest + needs: [create-release-branch] + environment: + name: main + env: + VERSION_FILE: nemo/package_info.py + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + path: ${{ github.run_id }} + fetch-depth: 0 + fetch-tags: true + ref: main + token: ${{ secrets.PAT }} + + - name: Bump version + id: bump-version + run: | + cd ${{ github.run_id }} + FULL_VERSION_NUM=${{ inputs.next_version }} + VERSION=${FULL_VERSION_NUM%%rc*} + MAJOR=$(echo "$VERSION" | cut -d. -f1) + MINOR=$(echo "$VERSION" | cut -d. -f2) + PATCH=$(echo "$VERSION" | cut -d. -f3) + PRE_RELEASE=${FULL_VERSION_NUM#$VERSION} + + sed -i 's/^MAJOR\s*=\s*[0-9]\+/MAJOR = '$MAJOR'/' $VERSION_FILE + sed -i 's/^MINOR\s*=\s*[0-9]\+/MINOR = '$MINOR'/' $VERSION_FILE + sed -i 's/^PATCH\s*=\s*[0-9]\+/PATCH = '$PATCH'/' $VERSION_FILE + sed -i 's/^PRE_RELEASE\s*=\s*'.*'/PRE_RELEASE = '\'$PRE_RELEASE\''/' $VERSION_FILE + + cat $VERSION_FILE + PRE_RELEASE=$(echo $PRE_RELEASE | tr -d "'") + echo "version=$MAJOR.$MINOR.$PATCH$PRE_RELEASE" >> "$GITHUB_OUTPUT" + + - name: Create Version Bump PR + uses: peter-evans/create-pull-request@v6 + id: create-pull-request + with: + path: ${{ github.run_id }} + branch: bot/chore/version-bump-${{ inputs.next_version }} + title: 'Version bump to `${{ inputs.next_version }}`' + body: | + 🚀 Version bump NeMo toolkit to `${{ inputs.next_version }}` + + commit-message: "[🤠]: Howdy folks, let's bump NeMo `${{ inputs.next_version }}` !" + signoff: true + assignees: okoenig + labels: 'Run CICD' + + notify: + runs-on: ubuntu-latest + needs: [create-release-branch, bump-next-version] + environment: + name: main + steps: + - name: Main + run: | + MESSAGE='{ + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Releasebot 🤖: NeMo Toolkit has been frozen 🎉 to branch `r${{ needs.create-release-branch.outputs.version }}`" + } + } + ] + }' + + curl -X POST -H "Content-type: application/json" --data "$MESSAGE" ${{ secrets.SLACK_RELEASE_ENDPOINT }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000000..3f4c4f3c19de --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,179 @@ +name: "NeMo Code release" + +on: + issue_comment: + types: [created] + +jobs: + main: + if: > + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/release-please') && + contains(fromJSON('["ko3n1g"]'), github.actor) + runs-on: ubuntu-latest + environment: + name: main + steps: + - name: Update PR issue comment + shell: bash + env: + message: ${{ github.event.comment.body }} + run: | + message="$message + + --- + + Releasebot 🤖: Release processes started... + " + message="${message//$'\n'/
}" + + curl -L \ + -X PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }} \ + -d '{"body":"'"$message"'"}' + + - name: Get PR number + shell: bash + id: get-pr-num + run: | + PR_URL="${{ github.event.issue.pull_request.url }}" + PR_NUM=${PR_URL##*/} + echo "pr_number=$PR_NUM" >> $GITHUB_OUTPUT + + - name: Get Pull Request Information + uses: actions/github-script@v6 + id: get-pr-branch + with: + result-encoding: string + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: ${{ steps.get-pr-num.outputs.pr_number }} + }); + console.log('Pull Request Information:', pr.data); + return pr.data.head.ref; + + - name: Checkout repository + uses: actions/checkout@v4 + with: + path: ${{ github.run_id }} + ref: ${{ steps.get-pr-branch.outputs.result }} + + - name: Get version number + id: version-number + run: | + cd ${{ github.run_id }} + VERSION=$(python -c "import nemo; print(nemo.__version__)") + echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Extract changelog + id: extract-changelog + uses: peter-evans/find-comment@v3 + with: + issue-number: ${{ steps.get-pr-num.outputs.pr_number }} + body-includes: '# Detailed Changelogs' + + - name: Extract summary + id: extract-summary + uses: peter-evans/find-comment@v3 + with: + issue-number: ${{ steps.get-pr-num.outputs.pr_number }} + body-includes: '# Highlights' + + - name: Create Release doc + id: create-release-doc + env: + SUMMARY: ${{ steps.extract-summary.outputs.comment-body }} + CHANGELOG: ${{ steps.extract-changelog.outputs.comment-body }} + run: | + + echo "TITLE<> $GITHUB_ENV + echo "NVIDIA Neural Modules ${{ steps.version-number.outputs.VERSION }}" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + echo "BODY<> $GITHUB_ENV + echo "$SUMMARY" >> $GITHUB_ENV + echo "$CHANGELOG" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + name: ${{ env.TITLE }} + tag_name: ${{ steps.version-number.outputs.VERSION }} + body: ${{ env.BODY }} + + - name: Build, test, and release wheel + env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + run: | + cd ${{ github.run_id }} + python3 -m pip install --upgrade build + python3 -m build + + pip install dist/*.whl + + cd ../ + + INSTALLED_VERSION=$(python -c 'import nemo; print(nemo.__version__)') + EXPECTED_VERSION=${{ steps.version-number.outputs.VERSION }} + + if [[ "$INSTALLED_VERSION" != "$EXPECTED_VERSION" ]]; then + echo 'Wheel has an outdated version, mission abort immediately!' + exit 1 + fi + + echo Proceed with uploading wheel... + cd ${{ github.run_id }} + python3 -m pip install --upgrade twine + python3 -m twine upload --repository pypi dist/* + + - name: Update PR issue comment + shell: bash + env: + message: ${{ github.event.comment.body }} + run: | + message="$message + + --- + + Releasebot 🤖: Release done 🎉 + " + message="${message//$'\n'/
}" + + curl -L \ + -X PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }} \ + -d '{"body":"'"$message"'"}' + + - name: Close Pull + run: | + cd ${{ github.run_id }} + gh pr close --comment "Releasebot 🤖: Closing PR" "${{ steps.get-pr-num.outputs.pr_number }}" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: notify + run: | + MESSAGE='{ + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Releasebot 🤖: NeMo Toolkit released `${{ steps.version-number.outputs.VERSION }}` 🚀" + } + } + ] + }' + + curl -X POST -H "Content-type: application/json" --data "$MESSAGE" ${{ secrets.SLACK_RELEASE_ENDPOINT }} \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000000..cfcd6ee939cb --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include requirements/* \ No newline at end of file