Release automation (NVIDIA#9687)

* ci: Add workflow for code-freeze Signed-off-by: Oliver Koenig <okoenig@nvidia.com> * ci: Add workflow for releasing NeMo Tookit Signed-off-by: Oliver Koenig <okoenig@nvidia.com> --------- Signed-off-by: Oliver Koenig <okoenig@nvidia.com> Signed-off-by: Malay Nagda <malayn@malayn-mlt.client.nvidia.com>
malay-nagda · Jul 26, 2024 · 79c08b0 · 79c08b0
1 parent c06e0ae
commit 79c08b0
Show file tree

Hide file tree

Showing 3 changed files with 372 additions and 0 deletions.
diff --git a/.github/workflows/release-freeze.yml b/.github/workflows/release-freeze.yml
@@ -0,0 +1,192 @@
+name: "NeMo Code freeze"
+
+on:
+  workflow_dispatch:
+    inputs:
+      next_version:
+        description: 'MAJOR.MINOR.PATCH[rcN] (Example: 2.0.0rc1, or 2.1.0)'
+        required: true
+        type: string
+      mcore_version:
+        description: 'Version of MCore to use (must be a valid git ref)'
+        required: true
+        type: string
+jobs:
+  create-release-branch:
+    runs-on: ubuntu-latest
+    if: contains(fromJSON('["ko3n1g"]'), github.actor)
+    environment: 
+      name: main
+    outputs:
+      version: ${{ steps.release-branch.outputs.version }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          path: ${{ github.run_id }}
+          fetch-depth: 0
+          fetch-tags: true
+          ref: main
+
+      - name: Get Previous tag
+        id: previous-tag
+        # git for-each-ref --sort=-creatordate --format '%(refname)' refs/tags ==> refs/tags/vX.Y.Z in descending order of date
+        # awk 'FNR == 2 {print substr($1, 11, length($1))}') ==> Selects the 2nd tag from the list, then strips the /refs/tags/ part of the tag
+        # set-output name=tag_name:: ==> Takes the clean tag vX.Y.Z and sets it to steps.previous_tag.outputs.tag_name
+        run: |
+          TAG=$(git for-each-ref --sort=-creatordate --format '%(refname)' refs/tags | awk 'FNR == 2 {print substr($1, 11, length($1))}')
+          echo "tag-name=$TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Get release branch ref
+        id: release-branch
+        run: |
+          cd ${{ github.run_id }}
+          
+          VERSION=$(python -c 'import nemo; print(nemo.__version__)')
+                    echo "Release version r$VERSION" > version
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Pin branch name in Notebooks
+        run: |
+          cd ${{ github.run_id }}
+          find tutorials -type f -name "*.ipynb" -exec sed -i "s/BRANCH = 'main'/BRANCH = 'r${{ steps.release-branch.outputs.version }}'/g" {} +
+
+      - name: Pin MCore in Dockerfile
+        run: |
+          cd ${{ github.run_id }}
+          sed -i 's/^ARG MCORE_TAG=.*$/ARG MCORE_TAG=${{ inputs.mcore_version }}/' Dockerfile.ci
+
+      - name: Build Changelog
+        id: build-changelog
+        uses: mikepenz/release-changelog-builder-action@v3.3.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          # Configuration file is setup with filters for domains
+          # owner:repo must point to current repo
+          # fromTag: Auto resolved from historical tag order (previous tag compared to current tag)
+          # toTag: Current tag reference
+          configuration: ".github/workflows/config/changelog-config.json"
+          owner: ${{ github.repository_owner }}
+          repo: ${{ github.event.repository.name }} 
+          ignorePreReleases: "false"
+          failOnError: "false"
+          fromTag: ${{ steps.previous-tag.outputs.tag-name }}
+          toTag: main
+
+      - name: Append Changelog
+        run: |
+          echo "${{ steps.build-changelog.outputs.changelog }}"
+
+      - name: Create Release PR
+        uses: peter-evans/create-pull-request@v6
+        id: create-pull-request
+        with:
+          path: ${{ github.run_id }}
+          branch: r${{ steps.release-branch.outputs.version }}
+          title: 'Release `${{ steps.release-branch.outputs.version }}`'
+          body: |
+            🚀 PR to release NeMo `${{ steps.release-branch.outputs.version }}`.
+
+            📝 Please remember the following to-do's before merge:
+            - [ ] Fill-in the comment `Highlights`
+            - [ ] Review the comment `Detailed Changelogs`
+
+            🚨 Please also keep in mind to _not_ delete the headings of the task commits. They are required by the post-merge automation.
+
+            🙏 Please merge this PR only if the CI workflow completed successfully.
+
+          commit-message: "[🤠]: Howdy folks, let's release NeMo `${{ steps.release-branch.outputs.version }}` !"
+          signoff: true
+          assignees: okoenig
+          labels: 'Run CICD'
+
+      - name: Add Summary comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ steps.create-pull-request.outputs.pull-request-number }}
+          body: |
+            # Highlights
+            _<here-goes-the-summary...>_
+
+      - name: Add Changelog comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ steps.create-pull-request.outputs.pull-request-number }}
+          body: |
+            # Detailed Changelogs
+            ${{ steps.build-changelog.outputs.changelog }}
+
+  bump-next-version:
+    runs-on: ubuntu-latest
+    needs: [create-release-branch]
+    environment: 
+      name: main
+    env:
+      VERSION_FILE: nemo/package_info.py
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          path: ${{ github.run_id }}
+          fetch-depth: 0
+          fetch-tags: true
+          ref: main
+          token: ${{ secrets.PAT }}
+
+      - name: Bump version
+        id: bump-version
+        run: |
+          cd ${{ github.run_id }}
+          FULL_VERSION_NUM=${{ inputs.next_version }}
+          VERSION=${FULL_VERSION_NUM%%rc*}
+          MAJOR=$(echo "$VERSION" | cut -d. -f1)
+          MINOR=$(echo "$VERSION" | cut -d. -f2)
+          PATCH=$(echo "$VERSION" | cut -d. -f3)
+          PRE_RELEASE=${FULL_VERSION_NUM#$VERSION}
+          
+          sed -i 's/^MAJOR\s*=\s*[0-9]\+/MAJOR = '$MAJOR'/' $VERSION_FILE
+          sed -i 's/^MINOR\s*=\s*[0-9]\+/MINOR = '$MINOR'/' $VERSION_FILE
+          sed -i 's/^PATCH\s*=\s*[0-9]\+/PATCH = '$PATCH'/' $VERSION_FILE
+          sed -i 's/^PRE_RELEASE\s*=\s*'.*'/PRE_RELEASE = '\'$PRE_RELEASE\''/' $VERSION_FILE
+
+          cat $VERSION_FILE
+          PRE_RELEASE=$(echo $PRE_RELEASE | tr -d "'")
+          echo "version=$MAJOR.$MINOR.$PATCH$PRE_RELEASE" >> "$GITHUB_OUTPUT"
+
+      - name: Create Version Bump PR
+        uses: peter-evans/create-pull-request@v6
+        id: create-pull-request
+        with:
+          path: ${{ github.run_id }}
+          branch: bot/chore/version-bump-${{ inputs.next_version }}
+          title: 'Version bump to `${{ inputs.next_version }}`'
+          body: |
+            🚀 Version bump NeMo toolkit to `${{ inputs.next_version }}`
+
+          commit-message: "[🤠]: Howdy folks, let's bump NeMo `${{ inputs.next_version }}` !"
+          signoff: true
+          assignees: okoenig
+          labels: 'Run CICD'
+
+  notify:
+    runs-on: ubuntu-latest
+    needs: [create-release-branch, bump-next-version]
+    environment: 
+      name: main
+    steps:
+      - name: Main
+        run: |
+          MESSAGE='{
+            "blocks": [
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Releasebot 🤖: NeMo Toolkit has been frozen 🎉 to branch `r${{ needs.create-release-branch.outputs.version }}`"
+                }
+              }
+            ]
+          }'
+
+          curl -X POST -H "Content-type: application/json" --data "$MESSAGE" ${{ secrets.SLACK_RELEASE_ENDPOINT }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,179 @@
+name: "NeMo Code release"
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs: 
+  main:
+    if: >
+      github.event_name == 'issue_comment' &&
+      github.event.issue.pull_request &&
+      startsWith(github.event.comment.body, '/release-please') &&
+      contains(fromJSON('["ko3n1g"]'), github.actor)
+    runs-on: ubuntu-latest
+    environment: 
+      name: main
+    steps:  
+      - name: Update PR issue comment
+        shell: bash
+        env:
+          message: ${{ github.event.comment.body }}
+        run: |
+          message="$message
+
+          ---
+
+          Releasebot 🤖: Release processes started...
+          "
+          message="${message//$'\n'/<br>}"
+
+          curl -L \
+            -X PATCH \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }} \
+            -d '{"body":"'"$message"'"}'
+
+      - name: Get PR number
+        shell: bash
+        id: get-pr-num
+        run: |
+          PR_URL="${{ github.event.issue.pull_request.url }}"
+          PR_NUM=${PR_URL##*/}
+          echo "pr_number=$PR_NUM" >> $GITHUB_OUTPUT
+
+      - name: Get Pull Request Information
+        uses: actions/github-script@v6
+        id: get-pr-branch
+        with:
+          result-encoding: string
+          script: |
+            const pr = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: ${{ steps.get-pr-num.outputs.pr_number }}
+            });
+            console.log('Pull Request Information:', pr.data);
+            return pr.data.head.ref;
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          path: ${{ github.run_id }}
+          ref: ${{ steps.get-pr-branch.outputs.result }}
+
+      - name: Get version number
+        id: version-number
+        run: |
+          cd ${{ github.run_id }}
+          VERSION=$(python -c "import nemo; print(nemo.__version__)")
+          echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Extract changelog
+        id: extract-changelog
+        uses: peter-evans/find-comment@v3
+        with:
+          issue-number: ${{ steps.get-pr-num.outputs.pr_number }}
+          body-includes: '# Detailed Changelogs'
+
+      - name: Extract summary
+        id: extract-summary
+        uses: peter-evans/find-comment@v3
+        with:
+          issue-number: ${{ steps.get-pr-num.outputs.pr_number }}
+          body-includes: '# Highlights'
+
+      - name: Create Release doc
+        id: create-release-doc
+        env:
+          SUMMARY: ${{ steps.extract-summary.outputs.comment-body }}
+          CHANGELOG: ${{ steps.extract-changelog.outputs.comment-body }}
+        run: |
+          
+          echo "TITLE<<EOF" >> $GITHUB_ENV
+          echo "NVIDIA Neural Modules ${{ steps.version-number.outputs.VERSION }}" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+          echo "BODY<<EOF" >> $GITHUB_ENV
+          echo "$SUMMARY" >> $GITHUB_ENV
+          echo "$CHANGELOG" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Create Release
+        uses: softprops/action-gh-release@v2
+        with:
+          name: ${{ env.TITLE }}
+          tag_name: ${{ steps.version-number.outputs.VERSION }}
+          body: ${{ env.BODY }}
+
+      - name: Build, test, and release wheel
+        env:
+          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+        run: |
+          cd ${{ github.run_id }}
+          python3 -m pip install --upgrade build
+          python3 -m build
+
+          pip install dist/*.whl
+
+          cd ../
+
+          INSTALLED_VERSION=$(python -c 'import nemo; print(nemo.__version__)')
+          EXPECTED_VERSION=${{ steps.version-number.outputs.VERSION }}
+          
+          if [[ "$INSTALLED_VERSION" != "$EXPECTED_VERSION" ]]; then
+            echo 'Wheel has an outdated version, mission abort immediately!'
+            exit 1
+          fi
+
+          echo Proceed with uploading wheel...
+          cd ${{ github.run_id }}
+          python3 -m pip install --upgrade twine
+          python3 -m twine upload --repository pypi dist/*
+
+      - name: Update PR issue comment
+        shell: bash
+        env:
+          message: ${{ github.event.comment.body }}
+        run: |
+          message="$message
+
+          ---
+
+          Releasebot 🤖: Release done 🎉
+          "
+          message="${message//$'\n'/<br>}"
+
+          curl -L \
+            -X PATCH \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }} \
+            -d '{"body":"'"$message"'"}'
+
+      - name: Close Pull
+        run: |
+          cd ${{ github.run_id }}
+          gh pr close --comment "Releasebot 🤖: Closing PR" "${{ steps.get-pr-num.outputs.pr_number }}"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: notify
+        run: |
+          MESSAGE='{
+            "blocks": [
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Releasebot 🤖: NeMo Toolkit released `${{ steps.version-number.outputs.VERSION }}` 🚀"
+                }
+              }
+            ]
+          }'
+
+          curl -X POST -H "Content-type: application/json" --data "$MESSAGE" ${{ secrets.SLACK_RELEASE_ENDPOINT }}
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1 @@
+include requirements/*