From 38c1a73b9d12ec03af88801b91ebbe0dc2ff13d0 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Wed, 29 Mar 2023 19:13:54 +0200 Subject: [PATCH 1/3] ci: split out release check into `is-workflow-valid.sh` This change refactors the script logic that checks if a workflow in fact built the commit matching our release tag out into a separate script. This is mainly an improvement in clarity. --- RELEASING.md | 13 +++++-- ci/download-workflow-artifacts.sh | 56 ++++++----------------------- ci/is-workflow-valid.sh | 58 +++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 49 deletions(-) create mode 100755 ci/is-workflow-valid.sh diff --git a/RELEASING.md b/RELEASING.md index 9f8dcf794..80f7b6e6b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -24,7 +24,14 @@ To publish a new version of `wasi-sdk` as a GitHub release: [actions]: https://github.com/WebAssembly/wasi-sdk/actions [tokens]: https://github.com/settings/tokens -3. Download and unzip the workflow artifacts. Note that artifacts with `+m` or +3. Check that the workflow built the artifacts for the given tag and that the + workflow completed successfully: + + ```shell script + ci/is-worfklow-valid.sh $TAG $WORKFLOW_RUN_ID $GITHUB_TOKEN + ``` + +4. Download and unzip the workflow artifacts. Note that artifacts with `+m` or `.m` suffixes indicate that the Git tree was modified. Expect some duplicates since some of the same artifacts are built on multiple CI runners (e.g., Windows, MacOS, Linux). The following script does all of this automatically: @@ -33,7 +40,7 @@ To publish a new version of `wasi-sdk` as a GitHub release: ci/download-workflow-artifacts.sh $TAG $WORKFLOW_RUN_ID $GITHUB_TOKEN ``` -4. Draft a new release. This could be done [manually][releases] but the +5. Draft a new release. This could be done [manually][releases] but the following script simplifies the uploading of all the files and auto-generates the release description: @@ -43,6 +50,6 @@ To publish a new version of `wasi-sdk` as a GitHub release: [releases]: https://github.com/WebAssembly/wasi-sdk/releases -5. Publish the release; the previous step only creates a draft. Follow the link +6. Publish the release; the previous step only creates a draft. Follow the link in the previous step or navigate to the GitHub [releases] to review the description, commit, tag, and assets before clicking "Publish" diff --git a/ci/download-workflow-artifacts.sh b/ci/download-workflow-artifacts.sh index a4849cca0..5b53868f6 100755 --- a/ci/download-workflow-artifacts.sh +++ b/ci/download-workflow-artifacts.sh @@ -1,59 +1,24 @@ #!/usr/bin/env bash set -e -# This script downloads and unzips the artifacts produced in a workflow run. It -# also checks that the workflow commit corresponds to the tag commit that these -# artifacts will be released under. The script has several pre-requisites: +# This script downloads and unzips the artifacts produced in a workflow run. The +# script has several pre-requisites: # - some standard Bash tools (curl, unzip) and one slightly more rare one (jq) -# - an already-created tag in the repository (this marks the code to release) # - the ID of a workflow run that has run successfully--this is where we # retrieve the artifacts from # - a GitHub access token, see https://github.com/settings/tokens # -# Usage: download-workflow-artifacts.sh +# Usage: download-workflow-artifacts.sh -TAG=$1 -WORKFLOW_RUN_ID=$2 -GITHUB_TOKEN=$3 +WORKFLOW_RUN_ID=$1 +GITHUB_TOKEN=$2 GITHUB_API_VERSION=2022-11-28 GITHUB_API_URL=https://api.github.com/repos/WebAssembly/wasi-sdk TMP_DIR=$(mktemp -d -t wasi-sdk-artifacts.XXXXXXX) -if [ -z "${TAG}" ] || [ -z "${WORKFLOW_RUN_ID}" ] || [ -z "${GITHUB_TOKEN}" ]; then +if [ -z "${WORKFLOW_RUN_ID}" ] || [ -z "${GITHUB_TOKEN}" ]; then >&2 echo "Missing parameter; exiting..." - >&2 echo "Usage: download-worfklow-artifacts.sh " - exit 1 -fi - -# Get the commit SHA for the passed tag. -# See https://docs.github.com/en/rest/commits/commits#get-a-commit -MATCHING_COMMIT=$(curl \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - -H "X-GitHub-Api-Version: ${GITHUB_API_VERSION}" \ - "${GITHUB_API_URL}/commits/${TAG}") -COMMIT=$(echo $MATCHING_COMMIT | jq -r '.sha') ->&2 echo "===== Found commit for tag ${TAG}: ${COMMIT} =====" - -# Check that the commit of the workflow run matches the tag commit and that the -# workflow was successful. -# See https://docs.github.com/en/rest/actions/workflow-runs#get-a-workflow-run -WORKFLOW_RUN=$(curl \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - -H "X-GitHub-Api-Version: ${GITHUB_API_VERSION}" \ - "${GITHUB_API_URL}/actions/runs/${WORKFLOW_RUN_ID}") -WORKFLOW_COMMIT=$(echo $WORKFLOW_RUN | jq -r '.head_sha') -WORKFLOW_STATUS=$(echo $WORKFLOW_RUN | jq -r '.status') ->&2 echo "===== Found commit for workflow ${WORKFLOW_RUN_ID}: ${WORKFLOW_COMMIT} =====" -if [ "${COMMIT}" != "${WORKFLOW_COMMIT}" ]; then - >&2 echo "Commit at tag ${TAG} did not match the commit for workflow ${WORKFLOW_RUN_ID}, exiting...:" - >&2 echo " ${COMMIT} != ${WORKFLOW_COMMIT}" - exit 1 -fi -if [ "${WORKFLOW_STATUS}" != "completed" ]; then - >&2 echo "Workflow ${WORKFLOW_RUN_ID} did not end successfully, exiting...:" - >&2 echo " status = ${WORKFLOW_STATUS}" + >&2 echo "Usage: download-worfklow-artifacts.sh " exit 1 fi @@ -72,10 +37,9 @@ for A in $ARTIFACTS; do URL=$(echo $A | cut -d ',' -f 3) TO=$TMP_DIR/$NAME.zip # Exclude dist-ubuntu-latest to prefer dist-ubuntu-bionic, which is - # compatible with wider distributions. - # cf. - # https://github.com/WebAssembly/wasi-sdk/pull/273#issuecomment-1373879491 - # https://github.com/WebAssembly/wasi-sdk/issues/303 + # compatible with wider distributions. See: + # - https://github.com/WebAssembly/wasi-sdk/pull/273#issuecomment-1373879491 + # - https://github.com/WebAssembly/wasi-sdk/issues/303 if [ "${NAME}" = "dist-ubuntu-latest" ]; then continue fi diff --git a/ci/is-workflow-valid.sh b/ci/is-workflow-valid.sh new file mode 100755 index 000000000..b4097f899 --- /dev/null +++ b/ci/is-workflow-valid.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -e + +# This script checks 1) that the workflow commit corresponds to the commit for +# the given tag and 2) that the workflow has completed. This is a sanity check +# to ensure the artifacts we are about to publish are in fact built from the +# commit/tag we think. The script has several pre-requisites: +# - some standard Bash tools (curl, unzip) and one slightly more rare one (jq) +# - an already-created tag in the repository (this marks the code to release) +# - the ID of a workflow run that has run successfully--this is where we +# retrieve the artifacts from +# - a GitHub access token, see https://github.com/settings/tokens +# +# Usage: is-workflow-valid.sh + +TAG=$1 +WORKFLOW_RUN_ID=$2 +GITHUB_TOKEN=$3 +GITHUB_API_VERSION=2022-11-28 +GITHUB_API_URL=https://api.github.com/repos/WebAssembly/wasi-sdk + +if [ -z "${TAG}" ] || [ -z "${WORKFLOW_RUN_ID}" ] || [ -z "${GITHUB_TOKEN}" ]; then + >&2 echo "Missing parameter; exiting..." + >&2 echo "Usage: is-workflow-valid.sh " + exit 1 +fi + +# Get the commit SHA for the passed tag. +# See https://docs.github.com/en/rest/commits/commits#get-a-commit +MATCHING_COMMIT=$(curl \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: ${GITHUB_API_VERSION}" \ + "${GITHUB_API_URL}/commits/${TAG}") +COMMIT=$(echo $MATCHING_COMMIT | jq -r '.sha') +>&2 echo "===== Found commit for tag ${TAG}: ${COMMIT} =====" + +# Check that the commit of the workflow run matches the tag commit and that the +# workflow was successful. +# See https://docs.github.com/en/rest/actions/workflow-runs#get-a-workflow-run +WORKFLOW_RUN=$(curl \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: ${GITHUB_API_VERSION}" \ + "${GITHUB_API_URL}/actions/runs/${WORKFLOW_RUN_ID}") +WORKFLOW_COMMIT=$(echo $WORKFLOW_RUN | jq -r '.head_sha') +WORKFLOW_STATUS=$(echo $WORKFLOW_RUN | jq -r '.status') +>&2 echo "===== Found commit for workflow ${WORKFLOW_RUN_ID}: ${WORKFLOW_COMMIT} =====" +if [ "${COMMIT}" != "${WORKFLOW_COMMIT}" ]; then + >&2 echo "Commit at tag ${TAG} did not match the commit for workflow ${WORKFLOW_RUN_ID}, exiting...:" + >&2 echo " ${COMMIT} != ${WORKFLOW_COMMIT}" + exit 1 +fi +if [ "${WORKFLOW_STATUS}" != "completed" ]; then + >&2 echo "Workflow ${WORKFLOW_RUN_ID} did not end successfully, exiting...:" + >&2 echo " status = ${WORKFLOW_STATUS}" + exit 1 +fi From d315cc148981c8f3a4dce7cdab2410654b7e25a8 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Wed, 29 Mar 2023 19:20:21 +0200 Subject: [PATCH 2/3] ci: allow overriding script inputs from the environment When testing out these scripts on a fork, I realized that it was more convenient to use a single environment variable (e.g., `GITHUB_API_URL=... ci/draft-release.sh`) to change some of the script's input parameters. --- ci/download-workflow-artifacts.sh | 8 ++++---- ci/draft-release.sh | 10 +++++----- ci/get-workflows-for-tag.sh | 8 ++++---- ci/is-workflow-valid.sh | 10 +++++----- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ci/download-workflow-artifacts.sh b/ci/download-workflow-artifacts.sh index 5b53868f6..409b484e0 100755 --- a/ci/download-workflow-artifacts.sh +++ b/ci/download-workflow-artifacts.sh @@ -10,10 +10,10 @@ set -e # # Usage: download-workflow-artifacts.sh -WORKFLOW_RUN_ID=$1 -GITHUB_TOKEN=$2 -GITHUB_API_VERSION=2022-11-28 -GITHUB_API_URL=https://api.github.com/repos/WebAssembly/wasi-sdk +WORKFLOW_RUN_ID=${WORKFLOW_RUN_ID:-$1} +GITHUB_TOKEN=${GITHUB_TOKEN:-$2} +GITHUB_API_VERSION=${GITHUB_API_VERSION:-2022-11-28} +GITHUB_API_URL=${GITHUB_API_URL:-https://api.github.com/repos/WebAssembly/wasi-sdk} TMP_DIR=$(mktemp -d -t wasi-sdk-artifacts.XXXXXXX) if [ -z "${WORKFLOW_RUN_ID}" ] || [ -z "${GITHUB_TOKEN}" ]; then diff --git a/ci/draft-release.sh b/ci/draft-release.sh index 207da9796..905ecb9a7 100755 --- a/ci/draft-release.sh +++ b/ci/draft-release.sh @@ -12,11 +12,11 @@ set -e # # Usage: draft-release.sh -TAG=$1 -ARTIFACTS_DIR=$2 -GITHUB_TOKEN=$3 -GITHUB_API_VERSION=2022-11-28 -GITHUB_API_URL=https://api.github.com/repos/WebAssembly/wasi-sdk +TAG=${TAG:-$1} +ARTIFACTS_DIR=${ARTIFACTS_DIR:-$2} +GITHUB_TOKEN=${GITHUB_TOKEN:-$3} +GITHUB_API_VERSION=${GITHUB_API_VERSION:-2022-11-28} +GITHUB_API_URL=${GITHUB_API_URL:-https://api.github.com/repos/WebAssembly/wasi-sdk} TMP_DIR=$(mktemp -d -t release.sh.XXXXXXX) if [ -z "${TAG}" ] || [ -z "${ARTIFACTS_DIR}" ] || [ -z "${GITHUB_TOKEN}" ]; then diff --git a/ci/get-workflows-for-tag.sh b/ci/get-workflows-for-tag.sh index 1b117cb72..879204184 100755 --- a/ci/get-workflows-for-tag.sh +++ b/ci/get-workflows-for-tag.sh @@ -10,10 +10,10 @@ set -e # # Usage: get-workflows-for-tag.sh -TAG=$1 -GITHUB_TOKEN=$2 -GITHUB_API_VERSION=2022-11-28 -GITHUB_API_URL=https://api.github.com/repos/WebAssembly/wasi-sdk +TAG=${TAG:-$1} +GITHUB_TOKEN=${GITHUB_TOKEN:-$2} +GITHUB_API_VERSION=${GITHUB_API_VERSION:-2022-11-28} +GITHUB_API_URL=${GITHUB_API_URL:-https://api.github.com/repos/WebAssembly/wasi-sdk} if [ -z "${TAG}" ] || [ -z "${GITHUB_TOKEN}" ]; then >&2 echo "Missing parameter; exiting..." diff --git a/ci/is-workflow-valid.sh b/ci/is-workflow-valid.sh index b4097f899..a7f0c80ab 100755 --- a/ci/is-workflow-valid.sh +++ b/ci/is-workflow-valid.sh @@ -13,11 +13,11 @@ set -e # # Usage: is-workflow-valid.sh -TAG=$1 -WORKFLOW_RUN_ID=$2 -GITHUB_TOKEN=$3 -GITHUB_API_VERSION=2022-11-28 -GITHUB_API_URL=https://api.github.com/repos/WebAssembly/wasi-sdk +TAG=${TAG:-$1} +WORKFLOW_RUN_ID=${WORKFLOW_RUN_ID:-$2} +GITHUB_TOKEN=${GITHUB_TOKEN:-$3} +GITHUB_API_VERSION=${GITHUB_API_VERSION:-2022-11-28} +GITHUB_API_URL=${GITHUB_API_URL:-https://api.github.com/repos/WebAssembly/wasi-sdk} if [ -z "${TAG}" ] || [ -z "${WORKFLOW_RUN_ID}" ] || [ -z "${GITHUB_TOKEN}" ]; then >&2 echo "Missing parameter; exiting..." From 2299db96abb4b27697a1189b31d1c47d91946406 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Thu, 30 Mar 2023 08:24:42 +0200 Subject: [PATCH 3/3] ci: add note about clearing the cache I found the artifacts created to have incorrect version numbers; clearing the Actions cache should resolve this. --- RELEASING.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/RELEASING.md b/RELEASING.md index 80f7b6e6b..8c65cc8e8 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -5,6 +5,8 @@ To publish a new version of `wasi-sdk` as a GitHub release: 1. Tag a commit with an annotated tag. Note that this must be an annotated tag, not a lightweight tag, so that `version.sh` can use it for calculating the package version (use `git show wasi-sdk-...` to show other tag messages). + Note that you may need to clear the repository cache to avoid problems with + cached artifacts [^cache]. ```shell script TAG=wasi-sdk-1 @@ -53,3 +55,11 @@ To publish a new version of `wasi-sdk` as a GitHub release: 6. Publish the release; the previous step only creates a draft. Follow the link in the previous step or navigate to the GitHub [releases] to review the description, commit, tag, and assets before clicking "Publish" + +[^cache]: Here is an example of how to clear a cache with the GitHub CLI: + + ```shell script + URL=/repos/WebAssembly/wasi-sdk/actions/caches + gh api $URL -q '.actions_caches[].id' \ + | xargs -I {} gh api --method DELETE $URL/{} + ```