From 6fa48b00ede5d38725909647057b02112817db9d Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 8 Jun 2023 14:11:45 -0700 Subject: [PATCH] smoke test poetry Add a little more tests test Test poetry test Test poetry on python 3.10 Add more poetry tests Test en us test test Try verboose testing testing try quiet install Code refactooring test move linux pipy validation to workflow test test Fix path try test pipy More torch installations test testing test test test new fix install 2 try poetry nightly test nightly test test Test poetry validation test test_new test --- .github/scripts/validate_binaries.sh | 12 --------- .github/scripts/validate_pipy.sh | 12 +++++++++ .github/scripts/validate_poetry.sh | 25 +++++++++++++++++++ .github/workflows/validate-linux-binaries.yml | 9 +++++++ .../workflows/validate-nightly-binaries.yml | 10 +------- 5 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 .github/scripts/validate_pipy.sh create mode 100644 .github/scripts/validate_poetry.sh diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index c4de395c8..88a76de86 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -2,18 +2,6 @@ if [[ ${MATRIX_PACKAGE_TYPE} == "libtorch" ]]; then curl ${MATRIX_INSTALLATION} -o libtorch.zip unzip libtorch.zip else - # Special case Pypi installation package, only applicable to linux nightly CUDA 11.7 builds, wheel package - if [[ ${TARGET_OS} == 'linux' && ${MATRIX_GPU_ARCH_VERSION} == '11.7' && ${MATRIX_PACKAGE_TYPE} == 'manywheel' && ${MATRIX_CHANNEL} != 'nightly' ]]; then - conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg - INSTALLATION_PYPI=${MATRIX_INSTALLATION/"cu117"/"cu117_pypi_cudnn"} - INSTALLATION_PYPI=${INSTALLATION_PYPI/"torchvision torchaudio"/""} - INSTALLATION_PYPI=${INSTALLATION_PYPI/"index-url"/"extra-index-url"} - conda run -p ${ENV_NAME}_pypi ${INSTALLATION_PYPI} - conda run -p ${ENV_NAME}_pypi python ./test/smoke_test/smoke_test.py --package torchonly - conda deactivate - conda env remove -p ${ENV_NAME}_pypi - fi - # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159 conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg conda activate ${ENV_NAME} diff --git a/.github/scripts/validate_pipy.sh b/.github/scripts/validate_pipy.sh new file mode 100644 index 000000000..85b788cf0 --- /dev/null +++ b/.github/scripts/validate_pipy.sh @@ -0,0 +1,12 @@ +conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg + +if [[ ${MATRIX_CHANNEL} != "release" ]]; then + conda run -p ${ENV_NAME}_pypi pip3 install --pre torch --index-url "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}_pypi_cudnn" + conda run -p ${ENV_NAME}_pypi pip3 install --pre torchvision torchaudio --index-url "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}" +else + conda run -p ${ENV_NAME}_pypi pip3 install torch torchvision torchaudio +fi + +conda run -p ${ENV_NAME}_pypi python ./test/smoke_test/smoke_test.py +conda deactivate +conda env remove -p ${ENV_NAME}_pypi diff --git a/.github/scripts/validate_poetry.sh b/.github/scripts/validate_poetry.sh new file mode 100644 index 000000000..344a7a85f --- /dev/null +++ b/.github/scripts/validate_poetry.sh @@ -0,0 +1,25 @@ + +conda create -y -n ${ENV_NAME}_poetry python=${MATRIX_PYTHON_VERSION} numpy ffmpeg +conda activate ${ENV_NAME}_poetry +curl -sSL https://install.python-poetry.org | python3 - --git https://github.com/python-poetry/poetry.git@master +export PATH="/root/.local/bin:$PATH" + +poetry --version +poetry new test_poetry +cd test_poetry + +if [[ ${MATRIX_CHANNEL} != "release" ]]; then + # Installing poetry from our custom repo. We need to configure it before use and disable authentication + export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring + poetry source add --priority=explicit domains "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}" + poetry source add --priority=explicit pytorch "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}_pypi_cudnn" + poetry --quiet add --source pytorch torch + poetry --quiet add --source domains torchvision torchaudio +else + poetry --quiet add torch torchaudio torchvision +fi + +python ../test/smoke_test/smoke_test.py +conda deactivate +conda env remove -p ${ENV_NAME}_poetry +cd .. diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml index 438062f91..b20b10ea1 100644 --- a/.github/workflows/validate-linux-binaries.yml +++ b/.github/workflows/validate-linux-binaries.yml @@ -55,4 +55,13 @@ jobs: export ENV_NAME="conda-env-${{ github.run_id }}" export TARGET_OS="linux" eval "$(conda shell.bash hook)" + + # Special case PyPi installation package. And Install of PyPi package via poetry + if [[ ${MATRIX_PACKAGE_TYPE} == "manywheel" ]] && \ + {[${MATRIX_GPU_ARCH_VERSION} == "12.1" && ${MATRIX_CHANNEL} != "release"] || \ + [${MATRIX_GPU_ARCH_VERSION} == "11.7" && ${MATRIX_CHANNEL} == "release"]}; then + source ./.github/scripts/validate_pipy.sh + source ./.github/scripts/validate_poetry.sh + fi + source ./.github/scripts/validate_binaries.sh diff --git a/.github/workflows/validate-nightly-binaries.yml b/.github/workflows/validate-nightly-binaries.yml index c252e0433..b8ecd23c8 100644 --- a/.github/workflows/validate-nightly-binaries.yml +++ b/.github/workflows/validate-nightly-binaries.yml @@ -17,15 +17,7 @@ on: - .github/workflows/validate-macos-binaries.yml - .github/workflows/validate-macos-arm64-binaries.yml - test/smoke_test/* - pull_request: - paths: - - .github/workflows/validate-nightly-binaries.yml - - .github/workflows/validate-linux-binaries.yml - - .github/workflows/validate-windows-binaries.yml - - .github/workflows/validate-macos-binaries.yml - - .github/workflows/validate-macos-arm64-binaries.yml - - .github/scripts/validate_binaries.sh - - test/smoke_test/* + jobs: nightly: uses: ./.github/workflows/validate-binaries.yml