Skip to content

Commit

Permalink
Use more robust method to launch twine (#52)
Browse files Browse the repository at this point in the history
Using the twine executable directly relies on the Python bin directory
being added to the path, which is not reliably the case in different
environments. I've created a new copy of the script since I'm iterating
very actively right now in the new wheels workflow. We can get rid of
the old script once I've verified that everything is working as
expected.
  • Loading branch information
vyasr authored Apr 28, 2023
1 parent 6e26093 commit 254889a
Showing 1 changed file with 57 additions and 0 deletions.
57 changes: 57 additions & 0 deletions tools/rapids-twine-new
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/bin/bash
# A utility script that wraps twine to upload all pip wheels of a workflow run
#
# Positional Arguments:
# 1) wheel name
set -exou pipefail
source rapids-constants
export RAPIDS_SCRIPT_NAME="rapids-twine-new"

if [ -z "$1" ]; then
rapids-echo-stderr "Must specify input arguments: WHEEL_NAME"
exit 1
fi
WHEEL_NAME="$1"

WHEEL_SEARCH_KEY="wheel_python_${WHEEL_NAME}"

WHEEL_DIR="./dist"
mkdir -p "${WHEEL_DIR}"

S3_PATH=$(rapids-s3-path)
BUCKET_PREFIX=${S3_PATH/s3:\/\/${RAPIDS_DOWNLOADS_BUCKET}\//} # removes s3://rapids-downloads/ from s3://rapids-downloads/ci/rmm/...

# shellcheck disable=SC2016
WHEEL_TARBALLS=$(
set -eo pipefail;
aws \
--output json \
s3api list-objects \
--bucket "${RAPIDS_DOWNLOADS_BUCKET}" \
--prefix "${BUCKET_PREFIX}" \
--page-size 100 \
--query "Contents[?contains(Key, '${WHEEL_SEARCH_KEY}')].Key" \
| jq -c
)
export WHEEL_TARBALLS

# first untar them all
for OBJ in $(jq -nr 'env.WHEEL_TARBALLS | fromjson | .[]'); do
FILENAME=$(basename "${OBJ}")
S3_URI="${S3_PATH}${FILENAME}"

rapids-echo-stderr "Untarring ${S3_URI} into ${WHEEL_DIR}"
aws s3 cp --only-show-errors "${S3_URI}" - | tar xzf - -C "${WHEEL_DIR}"
done

# then run twine on all wheels
export RAPIDS_RETRY_SLEEP=180
# shellcheck disable=SC2086
rapids-retry python -m twine \
upload \
--disable-progress-bar \
--non-interactive \
--skip-existing \
"${WHEEL_DIR}"/*.whl

echo ""

0 comments on commit 254889a

Please sign in to comment.