From bef5b77749d47842ba27df0ef89250066a4d490d Mon Sep 17 00:00:00 2001 From: Mihai Nita Date: Sun, 29 Sep 2024 18:41:58 -0700 Subject: [PATCH] ICU-22606 Create full release from CI - workflow files The Ubuntu and Fedora files are very similar. But I am not sure (yet) what is the best way to share. The files uploaded to the GitHub Release use the exact same names as before. The .jar files are binary identical to the ones that go to Maven (checked). The Fedora Docker image should be built and uploaded from a dev machine. There is a workflow (from GitHub) doing that, but has a warning that: > This workflow uses actions that are not certified by GitHub. > They are provided by a third-party and are governed by > separate terms of service, privacy policy, and support documentation. --- .github/Dockerfile_fedora | 9 +++ .github/README.md | 25 ++++++ .github/workflows/release-check-sign.yml | 69 +++++++++++++++++ .github/workflows/release-icu4c-fedora.yml | 89 ++++++++++++++++++++++ .github/workflows/release-icu4c-ubuntu.yml | 85 +++++++++++++++++++++ .github/workflows/release-icu4j-maven.yml | 54 ++++++++----- 6 files changed, 313 insertions(+), 18 deletions(-) create mode 100644 .github/Dockerfile_fedora create mode 100644 .github/README.md create mode 100644 .github/workflows/release-check-sign.yml create mode 100644 .github/workflows/release-icu4c-fedora.yml create mode 100644 .github/workflows/release-icu4c-ubuntu.yml diff --git a/.github/Dockerfile_fedora b/.github/Dockerfile_fedora new file mode 100644 index 000000000000..dd7a354337f6 --- /dev/null +++ b/.github/Dockerfile_fedora @@ -0,0 +1,9 @@ +FROM fedora:latest + +RUN dnf install -y gcc-c++ zip unzip git-core git-lfs doxygen +RUN git lfs install --skip-repo \ + && ln -s /usr/bin/python3 /usr/bin/python + +WORKDIR /root + +ENTRYPOINT [ "/bin/bash" ] diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 000000000000..b182072df31a --- /dev/null +++ b/.github/README.md @@ -0,0 +1,25 @@ +## How to create a Fedora docker image + +Run +``` +docker login ghcr.io +``` + +When prompted use these: + +* **User:** the github user +* **Password:** the github key + +Update the timestamp (`20240929`) with the current date, ISO style: +``` +docker build --tag ghcr.io/unicode-org/fedora-docker-gcr:20240929 -f Dockerfile_fedora . +docker push ghcr.io/unicode-org/fedora-docker-gcr:20240929 +``` + +See: +https://docs.github.com/en/actions/use-cases-and-examples/publishing-packages/publishing-docker-images + +Also: +https://stackoverflow.com/questions/64033686/how-can-i-use-private-docker-image-in-github-actions + +To consider: generate and publish the docker image from a GitHub action. diff --git a/.github/workflows/release-check-sign.yml b/.github/workflows/release-check-sign.yml new file mode 100644 index 000000000000..96464ace309e --- /dev/null +++ b/.github/workflows/release-check-sign.yml @@ -0,0 +1,69 @@ +name: Release - Create checksums and GPG sign + +on: + workflow_dispatch: + inputs: + gitReleaseTag: + description: 'Release tag to upload to. Must start with "release-"' + type: string + +env: + RELEASE_FOLDER: '${{ github.workspace }}/releaseDist' + +jobs: + sign_and_checksums: + if: ${{ inputs.gitReleaseTag && startsWith(inputs.gitReleaseTag, 'release-') }} + runs-on: ubuntu-latest + environment: release-env + + permissions: + contents: write # So that we can upload to release + + steps: + + - name: Checkout and setup + uses: actions/checkout@v4 + with: + lfs: true + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Get all release files + run: | + mkdir -p ${RELEASE_FOLDER} + pushd ${RELEASE_FOLDER} + gh release download ${{ inputs.gitReleaseTag }} -p "*.zip" -p "*.tgz" -p "*.jar" --repo=${{ github.repository }} + popd + env: + GH_TOKEN: ${{ github.token }} + + - name: Checksums and sign + run: | + source icu4j/releases_tools/shared.sh + # Convert 76.1 to 76_1 + underscore_version=$(echo $artifact_version | sed 's/\./_/g') + pushd ${RELEASE_FOLDER} + sha512sum -b icu4c* > SHASUM512.txt + md5sum -b *.jar > icu4j-${artifact_version}.md5 + md5sum -b icu4c-*-data-bin-*.zip > icu4c-${underscore_version}-binary.md5 + md5sum -b icu4c-*-src.* > icu4c-${underscore_version}-sources.md5 + find . -type f -name 'icu4c*' -exec gpg --no-tty --batch --pinentry-mode loopback --passphrase=$MAVEN_GPG_PASSPHRASE -a --output {}.asc --detach-sig {} \; + gpg --no-tty --batch --pinentry-mode loopback --passphrase=$MAVEN_GPG_PASSPHRASE -a --output SHASUM512.txt.asc --detach-sig SHASUM512.txt + popd + env: + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + + - name: Upload to release + run: | + gh release upload ${{ inputs.gitReleaseTag }} LICENSE --clobber --repo=${{ github.repository }} + gh release upload ${{ inputs.gitReleaseTag }} ${RELEASE_FOLDER}/*.md5 --clobber --repo=${{ github.repository }} + gh release upload ${{ inputs.gitReleaseTag }} ${RELEASE_FOLDER}/*.asc --clobber --repo=${{ github.repository }} + gh release upload ${{ inputs.gitReleaseTag }} ${RELEASE_FOLDER}/SHASUM512.txt --clobber --repo=${{ github.repository }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/release-icu4c-fedora.yml b/.github/workflows/release-icu4c-fedora.yml new file mode 100644 index 000000000000..b8757fa1fb1c --- /dev/null +++ b/.github/workflows/release-icu4c-fedora.yml @@ -0,0 +1,89 @@ +name: Release - ICU4C artifacts on Fedora + +on: + workflow_dispatch: + inputs: + runTests: + description: 'Run the tests.' + type: boolean + default: true + gitReleaseTag: + description: 'Release tag to upload to. Must start with "release-"' + type: string + +env: + RELEASE_FOLDER: '${{ github.workspace }}/releaseDist' + +jobs: + build: + runs-on: ubuntu-latest + environment: release-env + + container: + image: ghcr.io/${{ github.repository_owner }}/fedora-docker-gcr:latest + credentials: + username: ${{ secrets.DOCKER_CONTAINER_USER_NAME }} + password: ${{ secrets.DOCKER_CONTAINER_REGISTRY_TOKEN }} + + permissions: + contents: write # So that we can upload to release + + steps: + + - name: Install gh (GitHub CLI) + run: | + # Don't install it in the docker image, get the latest (pros and cons) + dnf install -y gh + + - name: Checkout and setup + uses: actions/checkout@v4 + with: + lfs: true + + - name: Config and build ICU4C proper + run: | + pushd icu4c/source + ./runConfigureICU Linux/gcc + make -j8 + popd + + - name: Run tests + if: ${{ inputs.runTests }} + run: | + pushd icu4c/source + make check + popd + + - name: Build release ICU4C + run: | + pushd icu4c/source + make DESTDIR=${RELEASE_FOLDER}/icu releaseDist + popd + + - name: Collect artifacts in one folder + run: | + # Get the OS version in VERSION_ID + source /etc/os-release + # Get the ICU version in artifact_version + source icu4j/releases_tools/shared.sh + # Convert 76.1 to 76_1 + underscore_version=$(echo $artifact_version | sed 's/\./_/g') + pushd ${RELEASE_FOLDER} + tar -czf icu4c-${underscore_version}-Fedora_Linux${VERSION_ID}-x64.tgz icu + rm -fr icu + popd + + - name: Upload build results + uses: actions/upload-artifact@v4.3.6 + with: + name: icu4c-fedora-binaries + path: ${{ env.RELEASE_FOLDER }} + retention-days: 3 # TBD if we want to keep them longer + overwrite: true + + - name: Upload to release + if: ${{ inputs.gitReleaseTag && startsWith(inputs.gitReleaseTag, 'release-') }} + run: | + gh release upload ${{ inputs.gitReleaseTag }} ${RELEASE_FOLDER}/* --clobber --repo=${{ github.repository }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/release-icu4c-ubuntu.yml b/.github/workflows/release-icu4c-ubuntu.yml new file mode 100644 index 000000000000..c53ee7cbe289 --- /dev/null +++ b/.github/workflows/release-icu4c-ubuntu.yml @@ -0,0 +1,85 @@ +name: Release - ICU4C artifacts on Ubuntu + +on: + workflow_dispatch: + inputs: + runTests: + description: 'Run the tests.' + type: boolean + default: true + gitReleaseTag: + description: 'Release tag to upload to. Must start with "release-"' + type: string + +env: + RELEASE_FOLDER: '${{ github.workspace }}/releaseDist' + +jobs: + build: + runs-on: ubuntu-latest + environment: release-env + + permissions: + contents: write # So that we can upload to release + + steps: + + - name: Install doxygen + run: | + sudo apt-get -y install doxygen + + - name: Checkout and setup + uses: actions/checkout@v4 + with: + lfs: true + + - name: Config and build ICU4C proper + run: | + pushd icu4c/source + ./runConfigureICU Linux/gcc + make -j8 + popd + + - name: Run tests + if: ${{ inputs.runTests }} + run: | + pushd icu4c/source + make check + popd + + - name: Build release ICU4C + run: | + pushd icu4c/source + make dist + make DESTDIR=${RELEASE_FOLDER}/icu releaseDist + popd + + - name: Collect artifacts in one folder + run: | + # Get the OS version in VERSION_ID + source /etc/os-release + # Get the ICU version in artifact_version + source icu4j/releases_tools/shared.sh + # Convert 76.1 to 76_1 + underscore_version=$(echo $artifact_version | sed 's/\./_/g') + pushd ${RELEASE_FOLDER} + tar -czf icu4c-${underscore_version}-Ubuntu${VERSION_ID}-x64.tgz icu + rm -fr icu + popd + mv icu4c/source/dist/icu4c-76_1-d* ${RELEASE_FOLDER} + mv icu4c/source/dist/icu4c-76_1-src.* ${RELEASE_FOLDER} + + - name: Upload build results + uses: actions/upload-artifact@v4.3.6 + with: + name: icu4c-ubuntu-binaries + path: ${{ env.RELEASE_FOLDER }} + retention-days: 3 # TBD if we want to keep them longer + overwrite: true + + - name: Upload to release + if: ${{ inputs.gitReleaseTag && startsWith(inputs.gitReleaseTag, 'release-') }} + run: | + gh release upload ${{ inputs.gitReleaseTag }} ${RELEASE_FOLDER}/* --clobber --repo=${{ github.repository }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/release-icu4j-maven.yml b/.github/workflows/release-icu4j-maven.yml index fe14fec48264..c7f918be0cc4 100644 --- a/.github/workflows/release-icu4j-maven.yml +++ b/.github/workflows/release-icu4j-maven.yml @@ -1,7 +1,7 @@ # This workflow will build a package using Maven and then publish it to GitHub packages when a release is created # For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path -name: Publish to Nexus Sonatype OSS +name: Release - ICU4J publish to Maven Central # For this to run you should define the follwing GitHub Secrets in the proper Environment. # In Settings -- Environments -- release_env -- Environment secrets: @@ -21,24 +21,29 @@ on: # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: inputs: - gitTag: - # TODO: make this mandatory and validate that it is in a release* branch and looks like - # 'release-\d+.\d+ or something like that. - # For now we don't do it so that we can test. - description: 'Git tag at which to sync for deploy and release' + runTests: + description: 'Run the tests.' + type: boolean + default: true + deployToMaven: + description: 'Deploy to Maven Central (using Sonatype OSSRH)' + type: boolean + default: false + gitReleaseTag: + description: 'Release tag to upload to. Must start with "release-"' type: string env: SHARED_MVN_ARGS: '--no-transfer-progress --show-version --batch-mode' - RELEASE_FOLDER: 'target/release_files' + RELEASE_FOLDER: '${{ github.workspace }}/releaseDist' jobs: publish: runs-on: ubuntu-latest environment: release-env + permissions: - contents: read - packages: write + contents: write # So that we can upload to release steps: @@ -59,8 +64,11 @@ jobs: gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase settings-path: ${{ github.workspace }} # location for the settings.xml file - # TODO: enable tests? We don't want to release until we are 100% sure everything works. - - name: Build all of ICU + - name: Run tests + if: ${{ inputs.runTests }} + run: mvn install --file icu4j ${SHARED_MVN_ARGS} --errors --fail-at-end + + - name: Build javadoc taglets run: | # Really important to do this first because we need `tools/build` for the javadoc applets mvn install --file icu4j/tools/build \ @@ -69,8 +77,15 @@ jobs: - name: Build and deploy to Maven Central run: | + if [ "${{ inputs.deployToMaven }}" != "true" ]; then + echo Deploy to local folder + # Run the deploy, but do it to a local folder, not to Sonatype / Maven Central + export MAVEN_LOCAL='-DaltDeploymentRepository=tempid::file:./local-deploy-folder' + else + echo Deploy to Maven Central using Sonatype OSSRH + fi mvn deploy --file icu4j \ - ${SHARED_MVN_ARGS} \ + ${SHARED_MVN_ARGS} ${MAVEN_LOCAL} \ --settings $GITHUB_WORKSPACE/settings.xml \ --errors --fail-at-end -DdeployAtEnd=true \ -DskipTests -DskipITs \ @@ -97,19 +112,22 @@ jobs: ${SHARED_MVN_ARGS} \ -DskipITs -DskipTests \ -P with_full_javadoc + # Archive the full doc in a jar. Should also be posted to the web as official doc. jar -Mcf ${RELEASE_FOLDER}/icu4j-${ICU_VERSION}-fulljavadoc.jar \ -C target/site/apidocs/ . - # Create the file with MD5 checksums - pushd ${RELEASE_FOLDER} - md5sum *.jar > icu4j-${ICU_VERSION}.md5 - popd # out of $RELEASE_FOLDER popd # out of icu4j - # TODO: add them to the GitHub "Release" page automatically - name: Upload build results uses: actions/upload-artifact@v4.3.6 with: - path: ./icu4j/${{ env.RELEASE_FOLDER }}/* + name: icu4j-binaries + path: ${{ env.RELEASE_FOLDER }} retention-days: 3 # TBD if we want to keep them longer overwrite: true + - name: Upload to release + if: ${{ inputs.gitReleaseTag && startsWith(inputs.gitReleaseTag, 'release-') }} + run: | + gh release upload ${{ inputs.gitReleaseTag }} ${RELEASE_FOLDER}/* --clobber --repo=${{ github.repository }} + env: + GH_TOKEN: ${{ github.token }}