build_release_candidate.yml

name: build_release_candidate

# Workflow added after https://github.com/apache/beam/commit/4183e747becebd18becee5fff547af365910fc9c
# If help is needed debugging issues, you can view the release guide at that commit for guidance on how to do this manually.
# (https://github.com/apache/beam/blob/4183e747becebd18becee5fff547af365910fc9c/website/www/site/content/en/contribute/release-guide.md)
on:
  workflow_dispatch:
    inputs:
      RELEASE:
        description: Beam version of current release (e.g. 2.XX.0)
        required: true
        default: '2.XX.0'
      RC:
        description: Integer RC version for the release (e.g. 3 for RC3)
        required: true
      APACHE_ID:
        description: Your Apache id. Required if you want to stage artifacts into https://dist.apache.org/repos/dist/dev/beam/
        required: false
      APACHE_PASSWORD:
        description: Your Apache password. Required if you want to stage artifacts into https://dist.apache.org/repos/dist/dev/beam/
        required: false
      REPO_TOKEN:
        description: |
          Github Personal Access Token with repo permissions if you want to create the beam-site docs PR,
          create a draft release, or upload prism artifacts to that release.
          See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens.
        default: ''
      STAGE:
        description: |
          Configure which actions this workflow should perform, by setting the desired fields to "yes".
          --
          java_artifacts: publish java artifacts to https://repository.apache.org/#stagingRepositories
          --
          java_source: java source into https://dist.apache.org/repos/dist/dev/beam/
          --
          docker_artifacts: stage SDK docker images to docker hub Apache organization
          --
          python_artifacts: stage the python artifacts into https://dist.apache.org/repos/dist/dev/beam/
          --
          beam_site_pr: create the documentation update PR against apache/beam-site.
          --
          prism: build and upload the artifacts to the release for this tag
        required: true
        default: |
           {java_artifacts: "no",
            java_source: "no",
            docker_artifacts: "no",
            python_artifacts: "no",
            beam_site_pr: "no",
            prism: "no"}

env:
  DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}

jobs:
  publish_java_artifacts:
    if:  ${{ fromJson(github.event.inputs.STAGE).java_artifacts == 'yes'}}
    runs-on: [self-hosted, ubuntu-20.04, main]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
          repository: apache/beam
      - name: Install Java 8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '8'
      - name: Import GPG key
        id: import_gpg
        uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549
        with:
          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
      - name: Auth for nexus
        run: |
          mkdir -p ${HOME}/.m2
          echo "<settings>
           <servers>
             <server>
               <id>apache.releases.https</id>
               <username>${{ secrets.NEXUS_STAGE_DEPLOYER_USER }}</username>
               <password>${{ secrets.NEXUS_STAGE_DEPLOYER_PW }}</password>
             </server>
             <server>
               <id>apache.snapshots.https</id>
               <username>${{ secrets.NEXUS_USER }}</username>
               <password>${{ secrets.NEXUS_PW }}</password>
             </server>
           </servers>
          </settings>" > ${HOME}/.m2/settings.xml
      - name: Configure git
        run: git config credential.helper store
      - name: Stage Java Artifacts into Maven
        run: ./gradlew publish -Psigning.gnupg.keyName=${{steps.import_gpg.outputs.fingerprint}} -PisRelease --no-daemon --no-parallel


  stage_java_source:
    if:  ${{ fromJson(github.event.inputs.STAGE).java_source == 'yes'}}
    runs-on: ubuntu-latest
    steps:
      - name: Mask Apache Password
        run: |
          # Workaround for Actions bug - https://github.com/actions/runner/issues/643
          APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH)
          echo "::add-mask::$APACHE_PASS"
      - name: Validate apache id/password
        run: |
          if [ "${{ github.event.inputs.APACHE_ID }}" == "" ]
          then
            echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
          fi
          if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ]
          then
            echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
          fi
      - name: Install Java 8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '8'
      - name: Import GPG key
        id: import_gpg
        uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549
        with:
          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
      - name: stage source
        run: |
            svn co https://dist.apache.org/repos/dist/dev/beam
            mkdir -p beam/${{ github.event.inputs.RELEASE }}
            cd beam/${{ github.event.inputs.RELEASE }}
            RC_DIR="beam-${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
            RC_ZIP="${RC_DIR}.zip"
            RELEASE_DIR="beam-${{ github.event.inputs.RELEASE }}"
            RC_TAG="v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
            SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.zip"
              # Check whether there is an existing dist dir
            if (svn ls "${SOURCE_RELEASE_ZIP}"); then
              echo "Removing existing ${SOURCE_RELEASE_ZIP}."
              svn delete "${SOURCE_RELEASE_ZIP}"
            fi

            echo "Downloading: https://github.com/apache/beam/archive/${RC_TAG}.zip"
            wget https://github.com/apache/beam/archive/${RC_TAG}.zip  -O "${RC_ZIP}"

            unzip "$RC_ZIP"
            rm "$RC_ZIP"
            mv "$RC_DIR" "$RELEASE_DIR"
            zip -r "${SOURCE_RELEASE_ZIP}" "$RELEASE_DIR"
            rm -r "$RELEASE_DIR"

            echo "----Signing Source Release ${SOURCE_RELEASE_ZIP}-----"
            gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${SOURCE_RELEASE_ZIP}"

            echo "----Creating Hash Value for ${SOURCE_RELEASE_ZIP}----"
            sha512sum ${SOURCE_RELEASE_ZIP} > ${SOURCE_RELEASE_ZIP}.sha512

            svn add --force .
            svn status
            svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"

  stage_python_artifacts:
    if:  ${{ fromJson(github.event.inputs.STAGE).python_artifacts == 'yes'}}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Mask Apache Password
        run: |
          # Workaround for Actions bug - https://github.com/actions/runner/issues/643
          APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH)
          echo "::add-mask::$APACHE_PASS"
      - name: Mask apache id/password
        run: |
          if [ "${{ github.event.inputs.APACHE_ID }}" == "" ]
          then
            echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
          fi
          if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ]
          then
            echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
          fi
      - name: Setup environment
        uses: ./.github/actions/setup-environment-action
        with:
          python-version: default
          disable-cache: true
      - name: Import GPG key
        id: import_gpg
        uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549
        with:
          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
      - name: Install dependencies
        run: |
          pip install python-dateutil
          pip install requests
      - name: stage python artifacts
        env:
          RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
          GIT_REPO_BASE_URL: https://github.com/apache/beam
          RELEASE_DIR: "beam/${{ github.event.inputs.RELEASE }}"
          RELEASE: "${{ github.event.inputs.RELEASE }}"
          SCRIPT_DIR: release/src/main/scripts
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SVN_ARTIFACTS_DIR: "beam/${{ github.event.inputs.RELEASE }}/python"
        run: |
          svn co https://dist.apache.org/repos/dist/dev/beam
          mkdir -p "${SVN_ARTIFACTS_DIR}"

          git fetch --all --tags --prune
          RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}")

          python "${SCRIPT_DIR}/download_github_actions_artifacts.py" \
            --github-token-var GITHUB_TOKEN \
            --repo-url "apache/beam" \
            --rc-tag "${RC_TAG}" \
            --release-commit "${RELEASE_COMMIT}" \
            --artifacts_dir "${RELEASE_DIR}/python" \
            --yes True

          cd "${RELEASE_DIR}"/python
          ls

          echo "------Checking Hash Value for apache_beam-${RELEASE}.tar.gz-----"
          sha512sum -c "apache_beam-${RELEASE}.tar.gz.sha512"

          echo "------Signing Source Release apache_beam-${RELEASE}.tar.gz------"
          gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache_beam-${RELEASE}.tar.gz"

          for artifact in *.whl; do
            echo "----------Checking Hash Value for ${artifact} wheel-----------"
            sha512sum -c "${artifact}.sha512"
          done

          for artifact in *.whl; do
            echo "------------------Signing ${artifact} wheel-------------------"
            gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${artifact}"
          done

          cd ..
          svn add --force python
          svn status
          svn commit -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"


  stage_docker:
    if:  ${{ fromJson(github.event.inputs.STAGE).docker_artifacts == 'yes'}}
    # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
          repository: apache/beam
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@v1.3.0
      - name: Install Java 11
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '11'
      - name: Install Python 3.8
        uses: actions/setup-python@v5
        with:
          python-version: '3.8'
      - run: echo $JAVA_HOME
      - run: echo "JAVA11_HOME=${JAVA_HOME}" >> "$GITHUB_OUTPUT"
        id: export-java11
      - name: Install Java 8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '8'
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Remove default github maven configuration
        # This step is a workaround to avoid a decryption issue of Beam's
        # net.linguica.gradle.maven.settings plugin and github's provided maven
        # settings.xml file
        run: rm ~/.m2/settings.xml || true
      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Push docker images
        run: ./gradlew :pushAllDockerImages -PisRelease -Pdocker-pull-licenses -Pprune-images -Pdocker-tag=${{ github.event.inputs.RELEASE }}rc${{ github.event.inputs.RC }} -Pjava11Home=${{steps.export-java11.outputs.JAVA11_HOME}} --no-daemon --no-parallel

  beam_site_pr:
    if:  ${{ fromJson(github.event.inputs.STAGE).beam_site_pr == 'yes'}}
    # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step
    runs-on: ubuntu-latest
    env:
      RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
      BRANCH_NAME: updates_release_${{ github.event.inputs.RELEASE }}
      BEAM_ROOT_DIR: ${{ github.workspace }}/beam
      SITE_ROOT_DIR: ${{ github.workspace }}/beam-site
    steps:
      - name: Checkout Beam Repo
        uses: actions/checkout@v4
        with:
          ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
          repository: apache/beam
          path: beam
      - name: Checkout Beam Site Repo
        uses: actions/checkout@v4
        with:
          repository: apache/beam-site
          path: beam-site
          token: ${{ github.event.inputs.REPO_TOKEN }}
          ref: release-docs
      - name: Install Python 3.8
        uses: actions/setup-python@v5
        with:
          python-version: '3.8'
      - name: Install node
        uses: actions/setup-node@v4
        with:
          node-version: '16'
      - name: Install Java 8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '8'
      - name: Remove default github maven configuration
        # This step is a workaround to avoid a decryption issue of Beam's
        # net.linguica.gradle.maven.settings plugin and github's provided maven
        # settings.xml file
        run: rm ~/.m2/settings.xml || true
      - name: Create documentation commit for website
        run: |
            echo "OK!"
      - name: Build Python Docs
        working-directory: beam/sdks/python
        run: |
            pip install --upgrade pip setuptools wheel
            pip install -U pip
            pip install tox
            # TODO(https://github.com/apache/beam/issues/20209): Don't hardcode py version in this file.
            tox -e docs
            rm -rf target/docs/_build/.doctrees
      - name: Build Typescript Docs
        working-directory: beam/sdks/typescript
        run: |
            npm ci && npm run docs
      - name: Build Java Docs
        working-directory: beam
        run: |
            ./gradlew :sdks:java:javadoc:aggregateJavadoc -PisRelease --no-daemon --no-parallel
      - name: Build YAML Docs
        working-directory: beam
        run: |
            ./gradlew :sdks:python:generateYamlDocs -PisRelease
      - name: Add canonical link into javadocs
        uses: cicirello/javadoc-cleanup@v1
        with:
          path-to-root: beam/sdks/java/javadoc/build/docs/javadoc
          base-url-path: https://beam.apache.org/releases/javadoc/current/
      - name: Consolidate Release Docs to beam-site branch with symlinks
        working-directory: beam-site
        run: |
            git checkout -b $BRANCH_NAME release-docs

            echo "..........Copying generated javadoc into beam-site.........."
            cp -r ${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/ javadoc/${{ github.event.inputs.RELEASE }}
            # Update current symlink to point to the latest release
            unlink javadoc/current
            ln -s ${{ github.event.inputs.RELEASE }} javadoc/current

            echo "............Copying generated pydoc into beam-site.........."
            cp -r ${BEAM_ROOT_DIR}/sdks/python/target/docs/_build pydoc/${{  github.event.inputs.RELEASE }}
            # Update current symlink to point to the latest release
            unlink pydoc/current
            ln -s ${{ github.event.inputs.RELEASE }} pydoc/current

            echo "............Copying generated typedoc into beam-site.........."
            mkdir -p typedoc
            cp -r ${BEAM_ROOT_DIR}/sdks/typescript/docs typedoc/${{ github.event.inputs.RELEASE }}
            # Update current symlink to point to the latest release
            unlink typedoc/current | true
            ln -s ${{ github.event.inputs.RELEASE }} typedoc/current

            echo "............Copying generated yamldoc into beam-site.........."
            mkdir -p yamldoc/${{ github.event.inputs.RELEASE }}
            cp -r ${BEAM_ROOT_DIR}/sdks/python/build/yaml-ref.html yamldoc/${{ github.event.inputs.RELEASE }}/index.html
            # Update current symlink to point to the latest release
            unlink yamldoc/current | true
            ln -s ${{ github.event.inputs.RELEASE }} yamldoc/current
      - name: Create commit on beam-site branch
        working-directory: beam
        run: |
            # Get the commit from the beam repo, not the beam-site repo.
            RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}")
            cd ${SITE_ROOT_DIR}
            git config user.name $GITHUB_ACTOR
            git config user.email actions@"$RUNNER_NAME".local
            git add -A
            git commit -m "Update beam-site for release ${{ github.event.inputs.RELEASE }}." -m "Content generated from commit ${RELEASE_COMMIT}."
            git push -f --set-upstream origin $BRANCH_NAME
      - name: Create beam-site PR
        working-directory: beam-site
        env:
          GH_TOKEN: ${{ github.event.inputs.REPO_TOKEN }}
          PR_TITLE: "Publish docs for ${{ github.event.inputs.RELEASE }} release"
          PR_BODY: "Content generated from https://github.com/apache/beam/tree/${{ env.RC_TAG }}."
        run: |
            gh pr create -t "$PR_TITLE" -b "$PR_BODY" --base release-docs

  build_and_stage_prism:
    if: ${{ fromJson(github.event.inputs.STAGE).prism == 'yes'}}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
          repository: apache/beam
      - name: Mask Apache Password
        run: |
          # Workaround for Actions bug - https://github.com/actions/runner/issues/643
          APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH)
          echo "::add-mask::$APACHE_PASS"
      - name: Mask apache id/password
        run: |
          if [ "${{ github.event.inputs.APACHE_ID }}" == "" ]
          then
            echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
          fi
          if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ]
          then
            echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
          fi
      - uses: actions/setup-go@v5
        with:
          go-version: '1.22'
      - name: Import GPG key
        id: import_gpg
        uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549
        with:
          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
      - name: Build prism artifacts
        env:
          RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
          GIT_REPO_BASE_URL: https://github.com/apache/beam
          RELEASE_DIR: "beam/${{ github.event.inputs.RELEASE }}"
          RELEASE: "${{ github.event.inputs.RELEASE }}"
          GH_TOKEN: ${{ github.event.inputs.REPO_TOKEN  }}
          SVN_ARTIFACTS_DIR: "beam/${{ github.event.inputs.RELEASE }}/prism"
        run: |
          # TODO stop the action if a non-prerelease release exists
          # TODO create the draft release automatically if none exist.
          # TODO update the draft release's RC tag to current.

          export GH_RELEASE_ID=`gh api \
           -H "Accept: application/vnd.github+json" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
           /repos/apache/beam/releases | jq '.[] | select(.tag_name | contains(env.RC_TAG)) .id'`

          echo "with gh release id $GH_RELEASE_ID for tag $RC_TAG"

          # Store a reference so we can get back to the start.
          ROOT_DIR=`pwd`
          echo "ROOT_DIR=$ROOT_DIR"
          echo "............Checking out Apache Dev Repo.........."

          svn co https://dist.apache.org/repos/dist/dev/beam
          mkdir -p "${SVN_ARTIFACTS_DIR}"

          OUTPUT_DIR=$ROOT_DIR/$SVN_ARTIFACTS_DIR
          mkdir -p $OUTPUT_DIR
          echo "OUTPUT_DIR=$OUTPUT_DIR"

          # Remove existing prism contents from the svn repo.
          cd $OUTPUT_DIR
          rm -rf *
          cd $ROOT_DIR

          # Navigate to the prism command directory.
          cd sdks/go/cmd/prism

          # Store a reference to where we're building from.
          BUILD_DIR=`pwd`

          echo "............Build and stage prism artifacts in the Github Release $GH_RELEASE_ID for tag $RC_TAG.........."


          # Loop through and build desired set from allowed types.
          for OS in linux windows darwin; do
            for ARCH in amd64 arm64; do
              TARGET_DIR="$OUTPUT_DIR/$OS/$ARCH"
              TARGET_NAME="apache_beam-v$RELEASE-prism-$OS-$ARCH"

              mkdir -p "$TARGET_DIR"
              OUTPUT_FILE="$TARGET_DIR/$TARGET_NAME"
              if GOOS=$OS GOARCH=$ARCH CGO_ENABLED=0 go build -trimpath -buildvcs=false -o "$OUTPUT_FILE" . > /dev/null 2>&1; then
                cd $TARGET_DIR
                # Extract real output name. Windows builds automatically have .exe added.
                ARTIFACT=`ls`
                echo "target built - $ARTIFACT"

                ZIP_NAME="$ARTIFACT.zip"
                zip -r $ZIP_NAME $ARTIFACT

                # Sign and hash
                gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig $ZIP_NAME
                sha512sum $ZIP_NAME > "${ZIP_NAME}.sha512"

                # Upload to the release.
                gh release upload $RC_TAG $ZIP_NAME ${ZIP_NAME}.sha512 ${ZIP_NAME}.asc --clobber

                # Remove the binary from the release directory
                rm $ARTIFACT

                # Return to our root build dir.
                cd $BUILD_DIR
              else
                echo "target failed for $OS - $ARCH"
                rm -f "$OUTPUT_FILE"
                exit 1 # Do not continue action
              fi
            done
          done
          echo "...........Prism builds done!..........."
          cd $ROOT_DIR
          cd $RELEASE_DIR

          echo "...........Adding prism artifacts to the Dev Apache SVN repo..........."

          svn add --force --parents prism
          svn status
          svn commit -m "Staging Prism artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"