-
Notifications
You must be signed in to change notification settings - Fork 18
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use more robust method to launch twine (#52)
Using the twine executable directly relies on the Python bin directory being added to the path, which is not reliably the case in different environments. I've created a new copy of the script since I'm iterating very actively right now in the new wheels workflow. We can get rid of the old script once I've verified that everything is working as expected.
- Loading branch information
Showing
1 changed file
with
57 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
#!/bin/bash | ||
# A utility script that wraps twine to upload all pip wheels of a workflow run | ||
# | ||
# Positional Arguments: | ||
# 1) wheel name | ||
set -exou pipefail | ||
source rapids-constants | ||
export RAPIDS_SCRIPT_NAME="rapids-twine-new" | ||
|
||
if [ -z "$1" ]; then | ||
rapids-echo-stderr "Must specify input arguments: WHEEL_NAME" | ||
exit 1 | ||
fi | ||
WHEEL_NAME="$1" | ||
|
||
WHEEL_SEARCH_KEY="wheel_python_${WHEEL_NAME}" | ||
|
||
WHEEL_DIR="./dist" | ||
mkdir -p "${WHEEL_DIR}" | ||
|
||
S3_PATH=$(rapids-s3-path) | ||
BUCKET_PREFIX=${S3_PATH/s3:\/\/${RAPIDS_DOWNLOADS_BUCKET}\//} # removes s3://rapids-downloads/ from s3://rapids-downloads/ci/rmm/... | ||
|
||
# shellcheck disable=SC2016 | ||
WHEEL_TARBALLS=$( | ||
set -eo pipefail; | ||
aws \ | ||
--output json \ | ||
s3api list-objects \ | ||
--bucket "${RAPIDS_DOWNLOADS_BUCKET}" \ | ||
--prefix "${BUCKET_PREFIX}" \ | ||
--page-size 100 \ | ||
--query "Contents[?contains(Key, '${WHEEL_SEARCH_KEY}')].Key" \ | ||
| jq -c | ||
) | ||
export WHEEL_TARBALLS | ||
|
||
# first untar them all | ||
for OBJ in $(jq -nr 'env.WHEEL_TARBALLS | fromjson | .[]'); do | ||
FILENAME=$(basename "${OBJ}") | ||
S3_URI="${S3_PATH}${FILENAME}" | ||
|
||
rapids-echo-stderr "Untarring ${S3_URI} into ${WHEEL_DIR}" | ||
aws s3 cp --only-show-errors "${S3_URI}" - | tar xzf - -C "${WHEEL_DIR}" | ||
done | ||
|
||
# then run twine on all wheels | ||
export RAPIDS_RETRY_SLEEP=180 | ||
# shellcheck disable=SC2086 | ||
rapids-retry python -m twine \ | ||
upload \ | ||
--disable-progress-bar \ | ||
--non-interactive \ | ||
--skip-existing \ | ||
"${WHEEL_DIR}"/*.whl | ||
|
||
echo "" |