From d5adac5552376e0289dac555b30e4e4c0f2becd7 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Mon, 1 Apr 2024 15:23:35 -0400 Subject: [PATCH 01/29] Add UPLOAD_MATLAB env variable to 05-binary-upload.sh --- dev/release/05-binary-upload.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh index ae240c02ddf62..d1265bf93214f 100755 --- a/dev/release/05-binary-upload.sh +++ b/dev/release/05-binary-upload.sh @@ -72,6 +72,7 @@ fi : ${UPLOAD_CENTOS:=${UPLOAD_DEFAULT}} : ${UPLOAD_DEBIAN:=${UPLOAD_DEFAULT}} : ${UPLOAD_DOCS:=${UPLOAD_DEFAULT}} +: ${UPLOAD_MATLAB:=${UPLOAD_DEFAULT}} : ${UPLOAD_NUGET:=${UPLOAD_DEFAULT}} : ${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}} : ${UPLOAD_R:=${UPLOAD_DEFAULT}} From 386c50bfb44785cbfa5c18fe717d2d902417fd96 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Mon, 1 Apr 2024 16:18:39 -0400 Subject: [PATCH 02/29] Add logic to upload MLTBX release candidates to apache/arrow's GitHub Releases Area --- dev/release/05-binary-upload.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh index d1265bf93214f..751ea0c1f12ab 100755 --- a/dev/release/05-binary-upload.sh +++ b/dev/release/05-binary-upload.sh @@ -136,3 +136,13 @@ docker_run \ VERBOSE=${VERBOSE:-no} \ VERSION=${version} \ YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}") + + +if [ ${UPLOAD_MATLAB} -gt 0 ]; then + mltbx_file="${ARROW_ARTIFACTS_DIR}/matlab-dist/matlab-arrow-${version_with_rc}.mltbx" + release_tag=${version_with_rc} + target_branch=${version_with_rc} + release_notes="Release Candidate: ${version_with_rc}" + repository="https://github.com/apache/arrow" + gh release create ${release_tag} ${mltbx_file} --prerelease --target ${target_branch} --notes ${release_notes} --repo ${repository} +fi \ No newline at end of file From 378af3a84454e6773833cdda168ace188aa0e15d Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Mon, 1 Apr 2024 16:23:06 -0400 Subject: [PATCH 03/29] Fix indentation --- dev/release/05-binary-upload.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh index 751ea0c1f12ab..b4e85efc1548b 100755 --- a/dev/release/05-binary-upload.sh +++ b/dev/release/05-binary-upload.sh @@ -139,10 +139,10 @@ docker_run \ if [ ${UPLOAD_MATLAB} -gt 0 ]; then - mltbx_file="${ARROW_ARTIFACTS_DIR}/matlab-dist/matlab-arrow-${version_with_rc}.mltbx" - release_tag=${version_with_rc} - target_branch=${version_with_rc} - release_notes="Release Candidate: ${version_with_rc}" - repository="https://github.com/apache/arrow" - gh release create ${release_tag} ${mltbx_file} --prerelease --target ${target_branch} --notes ${release_notes} --repo ${repository} + mltbx_file="${ARROW_ARTIFACTS_DIR}/matlab-dist/matlab-arrow-${version_with_rc}.mltbx" + release_tag=${version_with_rc} + target_branch=${version_with_rc} + release_notes="Release Candidate: ${version_with_rc}" + repository="https://github.com/apache/arrow" + gh release create ${release_tag} ${mltbx_file} --prerelease --target ${target_branch} --notes ${release_notes} --repo ${repository} fi \ No newline at end of file From ee9c171d206b3334a02a809f0054d062a1389e4f Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Mon, 1 Apr 2024 16:38:58 -0400 Subject: [PATCH 04/29] Split gh command into multiple lines --- dev/release/05-binary-upload.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh index b4e85efc1548b..7692f5bd3ab3d 100755 --- a/dev/release/05-binary-upload.sh +++ b/dev/release/05-binary-upload.sh @@ -144,5 +144,11 @@ if [ ${UPLOAD_MATLAB} -gt 0 ]; then target_branch=${version_with_rc} release_notes="Release Candidate: ${version_with_rc}" repository="https://github.com/apache/arrow" - gh release create ${release_tag} ${mltbx_file} --prerelease --target ${target_branch} --notes ${release_notes} --repo ${repository} + gh release create \ + ${release_tag} \ + ${mltbx_file} \ + --prerelease \ + --target ${target_branch} \ + --notes ${release_notes} \ + --repo ${repository} fi \ No newline at end of file From 6f6d67d77340c57af15cb62d5e509f9e3503a15b Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Mon, 1 Apr 2024 17:09:34 -0400 Subject: [PATCH 05/29] Add quotes around release notes argument. --- dev/release/05-binary-upload.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh index 7692f5bd3ab3d..0ba631905cf75 100755 --- a/dev/release/05-binary-upload.sh +++ b/dev/release/05-binary-upload.sh @@ -149,6 +149,6 @@ if [ ${UPLOAD_MATLAB} -gt 0 ]; then ${mltbx_file} \ --prerelease \ --target ${target_branch} \ - --notes ${release_notes} \ + --notes "${release_notes}" \ --repo ${repository} -fi \ No newline at end of file +fi From db4ed2a932ac94adbf6cc70e6a2eb8161fc0ad50 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Mon, 1 Apr 2024 17:34:05 -0400 Subject: [PATCH 06/29] Add gpg signing and SHA512 checksum calculation for MLTBX file. --- dev/release/05-binary-upload.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh index 0ba631905cf75..476a8f5988590 100755 --- a/dev/release/05-binary-upload.sh +++ b/dev/release/05-binary-upload.sh @@ -137,9 +137,21 @@ docker_run \ VERSION=${version} \ YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}") +# Upload the MATLAB MLTBX Release Candidate to the GitHub Releases +# area of the Apache Arrow GitHub project. +mltbx_file="${ARROW_ARTIFACTS_DIR}/matlab-dist/matlab-arrow-${version_with_rc}.mltbx" +mltbx_signature_gpg_ascii_armor=${mltbx_file}.asc +mltbx_checksum_sha512=${mltbx_file}.sha512 +# Sign the MLTBX file and create a detached (--deatch-sign) ASCII armor (--armor) GPG signature file. +gpg --detach-sign --local-user "${GPG_KEY_ID}" --armor ${mltbx_file} + +# Compute the SHA512 checksum of the MLTBX file. +shasum --algorithm 512 ${mltbx_file} > ${mltbx_checksum_sha512} + +# Upload artifacts to the Apache Arrow +# GitHub Releases area and mark as a "Prerelease". if [ ${UPLOAD_MATLAB} -gt 0 ]; then - mltbx_file="${ARROW_ARTIFACTS_DIR}/matlab-dist/matlab-arrow-${version_with_rc}.mltbx" release_tag=${version_with_rc} target_branch=${version_with_rc} release_notes="Release Candidate: ${version_with_rc}" @@ -147,6 +159,8 @@ if [ ${UPLOAD_MATLAB} -gt 0 ]; then gh release create \ ${release_tag} \ ${mltbx_file} \ + ${mltbx_signature_gpg_ascii_armor} \ + ${mltbx_checksum_sha512} \ --prerelease \ --target ${target_branch} \ --notes "${release_notes}" \ From 9dc7015b89a60ce76a7c8fb02995dfe378e7be25 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 2 Apr 2024 15:02:08 -0400 Subject: [PATCH 07/29] 1. Update Release Title 2. Update git tag 3. Fix target branch name --- dev/release/05-binary-upload.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh index 476a8f5988590..6a60a14d823af 100755 --- a/dev/release/05-binary-upload.sh +++ b/dev/release/05-binary-upload.sh @@ -152,9 +152,10 @@ shasum --algorithm 512 ${mltbx_file} > ${mltbx_checksum_sha512} # Upload artifacts to the Apache Arrow # GitHub Releases area and mark as a "Prerelease". if [ ${UPLOAD_MATLAB} -gt 0 ]; then - release_tag=${version_with_rc} - target_branch=${version_with_rc} - release_notes="Release Candidate: ${version_with_rc}" + release_tag=apache-arrow-${version_with_rc} + target_branch=release-${version_with_rc} + release_notes="Release Candidate: ${version} RC${rc}" + title="Apache Arrow ${version} RC${rc}" repository="https://github.com/apache/arrow" gh release create \ ${release_tag} \ @@ -164,5 +165,6 @@ if [ ${UPLOAD_MATLAB} -gt 0 ]; then --prerelease \ --target ${target_branch} \ --notes "${release_notes}" \ + --title "${title}" \ --repo ${repository} fi From f9d9f65229474a45090eba4440fbe8b5de289805 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 4 Apr 2024 15:29:55 -0400 Subject: [PATCH 08/29] Add initial .github/workflows/package.yml workflow --- .github/workflows/package.yml | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/package.yml diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 0000000000000..8583ce55fc9c2 --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Package + +on: + push: + tags: + - "apache-arrow-*-rc*" + +jobs: + release: + name: Release Candidate + runs-on: ubuntu-latest + if: github.ref_type == 'tag' + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Update tags + run: | + git fetch --tags --force origin + - name: Extract crossbow job id + run: | + CROSSBOW_JOB_ID=$(git for-each-ref refs/tags/$GIT_TAG_NAME --format='%(contents:subject)') + echo $CROSSBOW_JOB_ID >> $GITHUB_ENV + env: + GIT_TAG_NAME: ${{github.ref_name}} + - name: Setup Archery + run: | + python3 -m pip install -e dev/archery[crossbow] + - name: Download artifacts + run: | + archery crossbow download-artifacts -f matlab -t release-artifacts $CROSSBOW_JOB_ID + From e108422857a462387b7cdfc31973241c09667dad Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 4 Apr 2024 16:19:04 -0400 Subject: [PATCH 09/29] Add Cut Prerelease step to package.yml workflow --- .github/workflows/package.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 8583ce55fc9c2..da70ee3de9e5a 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -47,4 +47,29 @@ jobs: - name: Download artifacts run: | archery crossbow download-artifacts -f matlab -t release-artifacts $CROSSBOW_JOB_ID + - name: Cut Pre-Release + run: | + mltbx_file=$(find release-artifacts/$CROSSBOW_JOB_ID -name '*.mltbx' -type f | xargs basename) + mltbx_path=release-artifacts/$CROSSBOW_JOB_ID/$mltbx_file + version_with_rc=${GIT_TAG_NAME#apache-arrow-} + version=${version_with_rc%-rc*} + rc=${version_with_rc#${version}-rc} + + target_branch=release-${version_with_rc} + title="Apache Arrow {version} RC${rc}" + repository=${{github.repository}} + release_notes="Release Candidate: ${version} RC${rc}" + + gh release create \ + ${GIT_TAG_NAME} \ + ${mltbx_path} \ + --prerelease \ + --target ${target_branch} \ + --notes "${release_notes}" \ + --title "${title}" \ + --repo ${repository} + + env: + GIT_TAG_NAME: ${{github.ref_name}} + From 6c6bb9282fdb708d4dcc4eb6f6bcfaf9b79f8522 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 4 Apr 2024 16:37:09 -0400 Subject: [PATCH 10/29] Fix syntax error when appending env var to GITHUB_ENV --- .github/workflows/package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index da70ee3de9e5a..5afa70c0b3954 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -37,8 +37,8 @@ jobs: git fetch --tags --force origin - name: Extract crossbow job id run: | - CROSSBOW_JOB_ID=$(git for-each-ref refs/tags/$GIT_TAG_NAME --format='%(contents:subject)') - echo $CROSSBOW_JOB_ID >> $GITHUB_ENV + crossbow_job_id=$(git for-each-ref refs/tags/$GIT_TAG_NAME --format='%(contents:subject)') + echo "CROSSBOW_JOB_ID=$crossbow_job_id" >> $GITHUB_ENV env: GIT_TAG_NAME: ${{github.ref_name}} - name: Setup Archery From 2fb2d55162076d361147cd2a2012a75a07da68f8 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 4 Apr 2024 16:55:37 -0400 Subject: [PATCH 11/29] 1. Comment package.yml steps 2. Fix syntax errors --- .github/workflows/package.yml | 59 +++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 5afa70c0b3954..2a9d7397bf52f 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -35,41 +35,52 @@ jobs: - name: Update tags run: | git fetch --tags --force origin + - name: Store release tag name as environment variable + run: | + echo "RELEASE_TAG_NAME=${{github.ref_name}}" >> "$GITHUB_ENV" + - name: Prepare Version + # Extract the version string, version with the release-candidate + # number string, and release-candidate number from the git tag name + # (i.e. RELEASE_TAG_NAME). Store extracted values as environment + # variables in GITHUB_ENV for use in subsequent steps. + run: | + version_with_rc=${RELEASE_TAG_NAME#apache-arrow-} + version=${version_with_rc%-rc*} + rc=${version_with_rc#${version}-rc} + echo "ARROW_VERSION_WITH_RC=$version_with_rc" >> "$GITHUB_ENV" + echo "ARROW_VERSION=$version" >> $GITHUB_ENV + echo "ARROW_RC=$rc" >> $GITHUB_ENV - name: Extract crossbow job id + # Extract the crossbow job ID from the subject associated with the git + # tag message. Store the extracted ID as an environment variable in + # GITHUB_ENV for use in subsequent steps. run: | - crossbow_job_id=$(git for-each-ref refs/tags/$GIT_TAG_NAME --format='%(contents:subject)') - echo "CROSSBOW_JOB_ID=$crossbow_job_id" >> $GITHUB_ENV - env: - GIT_TAG_NAME: ${{github.ref_name}} + crossbow_job_id=$(git for-each-ref refs/tags/$RELEASE_TAG_NAME --format='%(contents:subject)') + echo "CROSSBOW_JOB_ID=$crossbow_job_id" >> "$GITHUB_ENV" - name: Setup Archery run: | python3 -m pip install -e dev/archery[crossbow] - name: Download artifacts + # Download the MLTBX file associated with the crossbow job ID. + # Store the path to the downloaded file as an environment variable + # in GITHUB_ENV for use in subsequent steps. run: | archery crossbow download-artifacts -f matlab -t release-artifacts $CROSSBOW_JOB_ID + basefilename=$(find release-artifacts/$CROSSBOW_JOB_ID -name '*.mltbx' -type f | xargs basename) + mltbx_file=release-artifacts/$CROSSBOW_JOB_ID/$basefilename + echo "ARROW_MLTBX_FILE=$mltbx_file" >> $GITHUB_ENV - name: Cut Pre-Release run: | - mltbx_file=$(find release-artifacts/$CROSSBOW_JOB_ID -name '*.mltbx' -type f | xargs basename) - mltbx_path=release-artifacts/$CROSSBOW_JOB_ID/$mltbx_file - version_with_rc=${GIT_TAG_NAME#apache-arrow-} - version=${version_with_rc%-rc*} - rc=${version_with_rc#${version}-rc} - - target_branch=release-${version_with_rc} - title="Apache Arrow {version} RC${rc}" + target_branch=release-$ARROW_VERSION_WITH_RC + title="Apache Arrow $ARROW_VERSION RC$ARROW_RC" repository=${{github.repository}} - release_notes="Release Candidate: ${version} RC${rc}" - + release_notes="Release Candidate: $ARROW_VERSION RC$ARROW_RC" gh release create \ - ${GIT_TAG_NAME} \ - ${mltbx_path} \ + $RELEASE_TAG_NAME \ + $ARROW_MLTBX_FILE \ --prerelease \ - --target ${target_branch} \ - --notes "${release_notes}" \ - --title "${title}" \ - --repo ${repository} - - env: - GIT_TAG_NAME: ${{github.ref_name}} - + --target $target_branch \ + --notes "$release_notes" \ + --title "$title" \ + --repo $repository From ac77f5b6e6360888e846f12c23c3c7ceffb01667 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 11:22:06 -0400 Subject: [PATCH 12/29] Use dev/release/04-binary-download.sh to download the release candidate MLTBX file --- .github/workflows/package.yml | 54 ++++++++++++++++------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 2a9d7397bf52f..57171cf0fadb7 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -35,52 +35,46 @@ jobs: - name: Update tags run: | git fetch --tags --force origin - - name: Store release tag name as environment variable + - name: Store release tag name as an environment variable run: | echo "RELEASE_TAG_NAME=${{github.ref_name}}" >> "$GITHUB_ENV" - - name: Prepare Version + - name: Extract version strings # Extract the version string, version with the release-candidate - # number string, and release-candidate number from the git tag name - # (i.e. RELEASE_TAG_NAME). Store extracted values as environment - # variables in GITHUB_ENV for use in subsequent steps. + # number string, and release-candidate number from the name of + # the git tag. (i.e. RELEASE_TAG_NAME). run: | version_with_rc=${RELEASE_TAG_NAME#apache-arrow-} version=${version_with_rc%-rc*} - rc=${version_with_rc#${version}-rc} - echo "ARROW_VERSION_WITH_RC=$version_with_rc" >> "$GITHUB_ENV" - echo "ARROW_VERSION=$version" >> $GITHUB_ENV - echo "ARROW_RC=$rc" >> $GITHUB_ENV + rc_num=${version_with_rc#${version}-rc} + echo "ARROW_VERSION_WITH_RC=${version_with_rc}" >> "${GITHUB_ENV}" + echo "ARROW_VERSION=${version}" >> ${GITHUB_ENV} + echo "ARROW_RC_NUM=${rc_num}" >> ${GITHUB_ENV} - name: Extract crossbow job id - # Extract the crossbow job ID from the subject associated with the git - # tag message. Store the extracted ID as an environment variable in - # GITHUB_ENV for use in subsequent steps. + # Extract the crossbow job ID from the subject field of the git tag subject. run: | - crossbow_job_id=$(git for-each-ref refs/tags/$RELEASE_TAG_NAME --format='%(contents:subject)') - echo "CROSSBOW_JOB_ID=$crossbow_job_id" >> "$GITHUB_ENV" + crossbow_job_id=$(git for-each-ref refs/tags/${RELEASE_TAG_NAME} --format='%(contents:subject)') + echo "CROSSBOW_JOB_ID=${crossbow_job_id}" >> "${GITHUB_ENV}" - name: Setup Archery run: | python3 -m pip install -e dev/archery[crossbow] - name: Download artifacts # Download the MLTBX file associated with the crossbow job ID. - # Store the path to the downloaded file as an environment variable - # in GITHUB_ENV for use in subsequent steps. run: | - archery crossbow download-artifacts -f matlab -t release-artifacts $CROSSBOW_JOB_ID - basefilename=$(find release-artifacts/$CROSSBOW_JOB_ID -name '*.mltbx' -type f | xargs basename) - mltbx_file=release-artifacts/$CROSSBOW_JOB_ID/$basefilename - echo "ARROW_MLTBX_FILE=$mltbx_file" >> $GITHUB_ENV + ./dev/release/04-binary-download.sh ${ARROW_VERSION} ${ARROW_RC_NUM} --task-filter matlab + base_filename=$(find packages/${CROSSBOW_JOB_ID} -name '*.mltbx' -type f | xargs basename) + mltbx_file=packages/$CROSSBOW_JOB_ID/${base_filename} + echo "ARROW_MLTBX_FILE=${mltbx_file}" >> $GITHUB_ENV - name: Cut Pre-Release run: | - target_branch=release-$ARROW_VERSION_WITH_RC - title="Apache Arrow $ARROW_VERSION RC$ARROW_RC" + target_branch=release-${ARROW_VERSION_WITH_RC} + title="Apache Arrow ${ARROW_VERSION} RC${ARROW_RC_NUM}" repository=${{github.repository}} - release_notes="Release Candidate: $ARROW_VERSION RC$ARROW_RC" + release_notes="Release Candidate: ${ARROW_VERSION} RC${ARROW_RC_NUM}" gh release create \ - $RELEASE_TAG_NAME \ - $ARROW_MLTBX_FILE \ + ${RELEASE_TAG_NAME} \ + ${ARROW_MLTBX_FILE} \ --prerelease \ - --target $target_branch \ - --notes "$release_notes" \ - --title "$title" \ - --repo $repository - + --target ${target_branch} \ + --notes "${release_notes}" \ + --title "${title}" \ + --repo ${repository} From d489d60d22d1f2d94ac20787dce31a8f4a70bb9c Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 11:29:46 -0400 Subject: [PATCH 13/29] Remove step extracting crossbow id from git tag message --- .github/workflows/package.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 57171cf0fadb7..0fccd9326e7ed 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -49,11 +49,6 @@ jobs: echo "ARROW_VERSION_WITH_RC=${version_with_rc}" >> "${GITHUB_ENV}" echo "ARROW_VERSION=${version}" >> ${GITHUB_ENV} echo "ARROW_RC_NUM=${rc_num}" >> ${GITHUB_ENV} - - name: Extract crossbow job id - # Extract the crossbow job ID from the subject field of the git tag subject. - run: | - crossbow_job_id=$(git for-each-ref refs/tags/${RELEASE_TAG_NAME} --format='%(contents:subject)') - echo "CROSSBOW_JOB_ID=${crossbow_job_id}" >> "${GITHUB_ENV}" - name: Setup Archery run: | python3 -m pip install -e dev/archery[crossbow] From 8fd9b4c0b6efb5925ea2d791b839f15b4ce03c95 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 16:12:17 -0400 Subject: [PATCH 14/29] Change job name from release to publish-release-candidate --- .github/workflows/package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 0fccd9326e7ed..230a6af83a869 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -23,8 +23,8 @@ on: - "apache-arrow-*-rc*" jobs: - release: - name: Release Candidate + publish-release-candidate: + name: Publish Release Candidate to GitHub Releases runs-on: ubuntu-latest if: github.ref_type == 'tag' steps: From 37171a54684015f721a745db49d3319245c42ec2 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 16:15:06 -0400 Subject: [PATCH 15/29] add comment explaining when the workflow is triggered --- .github/workflows/package.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 230a6af83a869..e58ad5c586d61 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -20,6 +20,8 @@ name: Package on: push: tags: + # A release candidate should be published whenever a tag with + # the name below is pushed to the apache/arrow repository. - "apache-arrow-*-rc*" jobs: From 1623ede6cfbd20961940c4c8f4ffb5c03124c1c1 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 16:28:24 -0400 Subject: [PATCH 16/29] Add step to checkout crossbow --- .github/workflows/package.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index e58ad5c586d61..f292c9545dca4 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -30,9 +30,16 @@ jobs: runs-on: ubuntu-latest if: github.ref_type == 'tag' steps: - - name: Check out repository + - name: Checkout apache/arrow uses: actions/checkout@v4 with: + path: arrow + fetch-depth: 0 + - name: Checkout ursacomputing/crossbow + uses: actions/checkout@v4 + with: + repository: ursacomputing/crossbow + path: crossbow fetch-depth: 0 - name: Update tags run: | From c0a9d8e3fb12b098ad62dc47398d8b88224d71df Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 16:28:47 -0400 Subject: [PATCH 17/29] Remove step updating git tags --- .github/workflows/package.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index f292c9545dca4..0f9701f84f21b 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -41,9 +41,6 @@ jobs: repository: ursacomputing/crossbow path: crossbow fetch-depth: 0 - - name: Update tags - run: | - git fetch --tags --force origin - name: Store release tag name as an environment variable run: | echo "RELEASE_TAG_NAME=${{github.ref_name}}" >> "$GITHUB_ENV" From 2ea8b1160f7a1d9cba45145e389adf5ea043b11c Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 16:33:59 -0400 Subject: [PATCH 18/29] Rename env var RELEASE_TAG_NAME to ARROW_RELEASE_TAG_NAME --- .github/workflows/package.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 0f9701f84f21b..cba7462a3ca32 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -41,9 +41,11 @@ jobs: repository: ursacomputing/crossbow path: crossbow fetch-depth: 0 - - name: Store release tag name as an environment variable - run: | - echo "RELEASE_TAG_NAME=${{github.ref_name}}" >> "$GITHUB_ENV" + - name: Store Release Tag Name as Environment Variable + # For convenience, store the release tag name as an environment + # variable for use in subsequent steps. + run: | + echo "ARROW_RELEASE_TAG_NAME=${{github.ref_name}}" >> "$GITHUB_ENV" - name: Extract version strings # Extract the version string, version with the release-candidate # number string, and release-candidate number from the name of From bc0b411fa2835eca052103cd19fca7f83b016f06 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 16:36:17 -0400 Subject: [PATCH 19/29] Rename Extract version strings step to Store Version and Release Candidate Number as Environment Variables --- .github/workflows/package.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index cba7462a3ca32..1392cac769ff0 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -46,10 +46,9 @@ jobs: # variable for use in subsequent steps. run: | echo "ARROW_RELEASE_TAG_NAME=${{github.ref_name}}" >> "$GITHUB_ENV" - - name: Extract version strings - # Extract the version string, version with the release-candidate - # number string, and release-candidate number from the name of - # the git tag. (i.e. RELEASE_TAG_NAME). + - name: Store Version and Release Candidate Number as Environment Variables + # From environment variable ARROW_RELEASE_TAG_NAME, extract the version and + # release candidate number. Store these values as environment variables. run: | version_with_rc=${RELEASE_TAG_NAME#apache-arrow-} version=${version_with_rc%-rc*} From 61f62ec1f55fe943375c7036e8638b99b72a9a3e Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 16:37:24 -0400 Subject: [PATCH 20/29] 1. Rename Setup Archery step to Install Archery 2. Fix archery install path --- .github/workflows/package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 1392cac769ff0..3e8a56484c6c9 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -56,9 +56,9 @@ jobs: echo "ARROW_VERSION_WITH_RC=${version_with_rc}" >> "${GITHUB_ENV}" echo "ARROW_VERSION=${version}" >> ${GITHUB_ENV} echo "ARROW_RC_NUM=${rc_num}" >> ${GITHUB_ENV} - - name: Setup Archery + - name: Install Archery run: | - python3 -m pip install -e dev/archery[crossbow] + python3 -m pip install -e arrow/dev/archery[crossbow] - name: Download artifacts # Download the MLTBX file associated with the crossbow job ID. run: | From 47e213a9e6c9f4fdb34c3eabcad74244c14c6685 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 16:41:39 -0400 Subject: [PATCH 21/29] 1. Rename Download artifacts step to Download MLTBX Release Candidate 2. ./dev/release/04-binary-download.sh -> ./arrow/release/04-binary-download.sh 3. Fix directory path provided to find --- .github/workflows/package.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 3e8a56484c6c9..abc4c1c6d0204 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -59,12 +59,11 @@ jobs: - name: Install Archery run: | python3 -m pip install -e arrow/dev/archery[crossbow] - - name: Download artifacts - # Download the MLTBX file associated with the crossbow job ID. + - name: Download MLTBX Release Candidate + # Download the MLTBX file from the crossbow GitHub Release's area. run: | - ./dev/release/04-binary-download.sh ${ARROW_VERSION} ${ARROW_RC_NUM} --task-filter matlab - base_filename=$(find packages/${CROSSBOW_JOB_ID} -name '*.mltbx' -type f | xargs basename) - mltbx_file=packages/$CROSSBOW_JOB_ID/${base_filename} + .arrow/dev/release/04-binary-download.sh ${ARROW_VERSION} ${ARROW_RC_NUM} --task-filter matlab + mltbx_file=$(find arrow/packages -name '*.mltbx' -type f) echo "ARROW_MLTBX_FILE=${mltbx_file}" >> $GITHUB_ENV - name: Cut Pre-Release run: | From e619b8ba2b439646c634a9baa709969c494bf4db Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 16:47:35 -0400 Subject: [PATCH 22/29] 1. Rename Cut Pre-Release to Create GitHub Release for Release Candidate 2. Set environment variable GH_TOKEN in Create GitHub Release for Release Candidate step --- .github/workflows/package.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index abc4c1c6d0204..c21e115ffc449 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -59,13 +59,15 @@ jobs: - name: Install Archery run: | python3 -m pip install -e arrow/dev/archery[crossbow] - - name: Download MLTBX Release Candidate + - name: Download Release Candidate MLTBX # Download the MLTBX file from the crossbow GitHub Release's area. run: | .arrow/dev/release/04-binary-download.sh ${ARROW_VERSION} ${ARROW_RC_NUM} --task-filter matlab mltbx_file=$(find arrow/packages -name '*.mltbx' -type f) echo "ARROW_MLTBX_FILE=${mltbx_file}" >> $GITHUB_ENV - - name: Cut Pre-Release + - name: Create GitHub Release for Release Candidate + # Create a pre-release in apache/arrow's GitHub Releases area. + # Attach the release candidate MLTBX file as an asset. run: | target_branch=release-${ARROW_VERSION_WITH_RC} title="Apache Arrow ${ARROW_VERSION} RC${ARROW_RC_NUM}" @@ -79,3 +81,5 @@ jobs: --notes "${release_notes}" \ --title "${title}" \ --repo ${repository} + env: + GH_TOKEN: ${{ github.token }} From 592e168b88c7c0110207120d3b2f86e93fad7898 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 17:01:05 -0400 Subject: [PATCH 23/29] Fix indentation in package.yml --- .github/workflows/package.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index c21e115ffc449..9effbcbd1ca54 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -42,10 +42,10 @@ jobs: path: crossbow fetch-depth: 0 - name: Store Release Tag Name as Environment Variable - # For convenience, store the release tag name as an environment - # variable for use in subsequent steps. - run: | - echo "ARROW_RELEASE_TAG_NAME=${{github.ref_name}}" >> "$GITHUB_ENV" + # For convenience, store the release tag name as an environment + # variable for use in subsequent steps. + run: | + echo "ARROW_RELEASE_TAG_NAME=${{github.ref_name}}" >> "$GITHUB_ENV" - name: Store Version and Release Candidate Number as Environment Variables # From environment variable ARROW_RELEASE_TAG_NAME, extract the version and # release candidate number. Store these values as environment variables. From bdf2a77422ebffede06a5595aa93dc4fd5b0200c Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 17:02:48 -0400 Subject: [PATCH 24/29] Fix call to 04-binary-download.sh --- .github/workflows/package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 9effbcbd1ca54..8b47b7bcf4ec1 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -62,7 +62,7 @@ jobs: - name: Download Release Candidate MLTBX # Download the MLTBX file from the crossbow GitHub Release's area. run: | - .arrow/dev/release/04-binary-download.sh ${ARROW_VERSION} ${ARROW_RC_NUM} --task-filter matlab + ./arrow/dev/release/04-binary-download.sh ${ARROW_VERSION} ${ARROW_RC_NUM} --task-filter matlab mltbx_file=$(find arrow/packages -name '*.mltbx' -type f) echo "ARROW_MLTBX_FILE=${mltbx_file}" >> $GITHUB_ENV - name: Create GitHub Release for Release Candidate From 153ec84b551999cd1e05e5b99f0858696dab04a6 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 17:06:22 -0400 Subject: [PATCH 25/29] Update old references to RELEASE_TAG_NAME to ARROW_RELEASE_TAG_NAME --- .github/workflows/package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 8b47b7bcf4ec1..72766d486392d 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -50,7 +50,7 @@ jobs: # From environment variable ARROW_RELEASE_TAG_NAME, extract the version and # release candidate number. Store these values as environment variables. run: | - version_with_rc=${RELEASE_TAG_NAME#apache-arrow-} + version_with_rc=${ARROW_RELEASE_TAG_NAME#apache-arrow-} version=${version_with_rc%-rc*} rc_num=${version_with_rc#${version}-rc} echo "ARROW_VERSION_WITH_RC=${version_with_rc}" >> "${GITHUB_ENV}" @@ -74,7 +74,7 @@ jobs: repository=${{github.repository}} release_notes="Release Candidate: ${ARROW_VERSION} RC${ARROW_RC_NUM}" gh release create \ - ${RELEASE_TAG_NAME} \ + ${ARROW_RELEASE_TAG_NAME} \ ${ARROW_MLTBX_FILE} \ --prerelease \ --target ${target_branch} \ From 243ea583661c43e8886f27adeaedaaafc49917ba Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 17:46:58 -0400 Subject: [PATCH 26/29] Update 05-binary-upload.sh to trigger the Package workflow --- dev/release/05-binary-upload.sh | 41 ++++++++++----------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh index 6a60a14d823af..b43812f299fe9 100755 --- a/dev/release/05-binary-upload.sh +++ b/dev/release/05-binary-upload.sh @@ -137,34 +137,17 @@ docker_run \ VERSION=${version} \ YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}") -# Upload the MATLAB MLTBX Release Candidate to the GitHub Releases -# area of the Apache Arrow GitHub project. -mltbx_file="${ARROW_ARTIFACTS_DIR}/matlab-dist/matlab-arrow-${version_with_rc}.mltbx" -mltbx_signature_gpg_ascii_armor=${mltbx_file}.asc -mltbx_checksum_sha512=${mltbx_file}.sha512 - -# Sign the MLTBX file and create a detached (--deatch-sign) ASCII armor (--armor) GPG signature file. -gpg --detach-sign --local-user "${GPG_KEY_ID}" --armor ${mltbx_file} - -# Compute the SHA512 checksum of the MLTBX file. -shasum --algorithm 512 ${mltbx_file} > ${mltbx_checksum_sha512} - -# Upload artifacts to the Apache Arrow -# GitHub Releases area and mark as a "Prerelease". +# Create a git tag to associate with a GitHub Release that will +# have the release candidate MLTBX file as an asset. +# +# Push this tag (e.g. apache-arrow-15.0.2-rc0) to the remote +# apache/arrow repository to trigger the GitHub Actions Workflow +# that creates the GitHub Release and uploads the MLTBX file. +# +# See .github/workflows/package.yml for details. if [ ${UPLOAD_MATLAB} -gt 0 ]; then - release_tag=apache-arrow-${version_with_rc} - target_branch=release-${version_with_rc} - release_notes="Release Candidate: ${version} RC${rc}" - title="Apache Arrow ${version} RC${rc}" - repository="https://github.com/apache/arrow" - gh release create \ - ${release_tag} \ - ${mltbx_file} \ - ${mltbx_signature_gpg_ascii_armor} \ - ${mltbx_checksum_sha512} \ - --prerelease \ - --target ${target_branch} \ - --notes "${release_notes}" \ - --title "${title}" \ - --repo ${repository} + release_tag="apache-arrow-${version_with_rc}" + tag_message="Release candidate: ${version_with_rc}" + git tag -a ${release_tag} -m ${tag_message} + git push apache ${release_tag} fi From af1910a69095699d8b5ad76a2dba7f11b1f3896f Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 5 Apr 2024 17:54:06 -0400 Subject: [PATCH 27/29] Add stencil utils-sign-artifact.sh file --- dev/release/utils-sign-artifact.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 dev/release/utils-sign-artifact.sh diff --git a/dev/release/utils-sign-artifact.sh b/dev/release/utils-sign-artifact.sh new file mode 100644 index 0000000000000..dd600b809af5d --- /dev/null +++ b/dev/release/utils-sign-artifact.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# From a6e6624ed6713b56d77fdb73bada1e4faada1ad4 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Sat, 6 Apr 2024 14:03:17 -0400 Subject: [PATCH 28/29] Remove utils-sign-artifact.sh --- dev/release/utils-sign-artifact.sh | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 dev/release/utils-sign-artifact.sh diff --git a/dev/release/utils-sign-artifact.sh b/dev/release/utils-sign-artifact.sh deleted file mode 100644 index dd600b809af5d..0000000000000 --- a/dev/release/utils-sign-artifact.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# From b1e8e15f8e9e1ff747708336834338bea73ab583 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 16 Apr 2024 13:56:09 -0400 Subject: [PATCH 29/29] Update release documentaion --- docs/source/developers/release.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/developers/release.rst b/docs/source/developers/release.rst index e7431ce0fb7b9..30fe290cfe853 100644 --- a/docs/source/developers/release.rst +++ b/docs/source/developers/release.rst @@ -228,6 +228,11 @@ Build source and binaries and submit them # Start verifications for binaries and wheels dev/release/07-binary-verify.sh + # Sign and upload MATLAB artifacts to the GitHub Releases area. + # + # NOTE: You must have GitHub CLI installed to run this script. + dev/release/08-matlab-upload.sh + Verify the Release ------------------