From 6e2a07a639b80581c75b6fbc3d56b94c1e746900 Mon Sep 17 00:00:00 2001 From: peter Date: Wed, 27 May 2020 16:17:59 +0800 Subject: [PATCH 1/8] Reuse common logic and migrate CUDA >= 10.1 jobs to VS 2019 for Windows jobs --- .circleci/config.yml | 83 ++++++++++++------- .circleci/config.yml.in | 83 ++++++++++++------- packaging/build_wheel.bat | 22 +++++ packaging/build_wheel.sh | 6 +- packaging/pkg_helpers.bash | 5 +- packaging/vs2019/meta.yaml | 21 ----- packaging/windows/internal/build_conda.bat | 2 +- packaging/windows/internal/build_wheels.bat | 2 +- .../{vs_install.ps1 => vs2017_install.ps1} | 0 packaging/windows/internal/vs2019_install.ps1 | 21 +++++ 10 files changed, 158 insertions(+), 87 deletions(-) create mode 100644 packaging/build_wheel.bat rename packaging/windows/internal/{vs_install.ps1 => vs2017_install.ps1} (100%) create mode 100644 packaging/windows/internal/vs2019_install.ps1 diff --git a/.circleci/config.yml b/.circleci/config.yml index d4634a95f2f..0219d3f445c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,14 +6,17 @@ version: 2.1 # - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. # Job names are 'name:' key. -orbs: - win: circleci/windows@2.0.0 - executors: - windows-gpu-prototype: + windows-cpu: + machine: + resource_class: windows.xlarge + image: windows-server-2019-vs2019:stable + shell: bash.exe + + windows-gpu: machine: - resource_class: windows.gpu.small.prototype - image: windows-server-2019-nvidia:201908-28 + resource_class: windows.gpu.nvidia.medium + image: windows-server-2019-nvidia:stable shell: bash.exe commands: @@ -98,7 +101,6 @@ binary_windows: &binary_windows PYTORCH_VERSION: << parameters.pytorch_version >> CUDA_VERSION: << parameters.cu_version >> USE_SCCACHE: "1" - VC_YEAR: "2017" jobs: circleci_consistency: @@ -248,51 +250,63 @@ jobs: binary_win_conda: <<: *binary_common - executor: - name: win/default - shell: bash.exe + executor: windows-cpu steps: - checkout_merge - run: command: | - choco install miniconda3 - (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression + set -ex + export VC_YEAR=2019 + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" - bash packaging/build_conda.sh - shell: powershell.exe + packaging/build_conda.sh - store_test_results: path: build_results/ binary_win_conda_cuda: <<: *binary_common - executor: windows-gpu-prototype + executor: windows-gpu steps: - checkout_merge - run: command: | - choco install miniconda3 - (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression + set -ex + export VC_YEAR=2019 + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" - bash packaging/build_conda.sh + packaging/build_conda.sh shell: powershell.exe binary_win_conda_release: <<: *binary_windows - executor: - name: win/default - shell: cmd.exe + executor: windows-cpu steps: - checkout_merge - run: name: Build conda packages command: | - call packaging/windows/internal/build_conda.bat + set -ex + if [[ "$CUDA_VERSION" == "92" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="-vcvars_ver=14.11" + powershell packaging/windows/internal/vs2017_install.ps1 + elif [["$CUDA_VERSION" == "100" ]]; then + export VC_YEAR=2017 + powershell packaging/windows/internal/vs2017_install.ps1 + else + export VC_YEAR=2019 + fi + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + conda install -yq conda-build "conda-package-handling!=1.5.0" + packaging/build_conda.sh + rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2 - store_artifacts: - path: packaging/windows/output + path: C:/tools/miniconda3/conda-bld/win-64 - persist_to_workspace: - root: packaging/windows/output + root: C:/tools/miniconda3/conda-bld/win-64 paths: - "*" - store_test_results: @@ -300,19 +314,28 @@ jobs: binary_win_wheel_release: <<: *binary_windows - executor: - name: win/default - shell: cmd.exe + executor: windows-cpu steps: - checkout_merge - run: name: Build wheel packages command: | - call packaging/windows/internal/build_wheels.bat + set -ex + if [[ "$CUDA_VERSION" == "92" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="-vcvars_ver=14.11" + powershell packaging/windows/internal/vs2017_install.ps1 + elif [["$CUDA_VERSION" == "100" ]]; then + export VC_YEAR=2017 + powershell packaging/windows/internal/vs2017_install.ps1 + else + export VC_YEAR=2019 + fi + packaging/build_wheel.sh - store_artifacts: - path: packaging/windows/output + path: dist - persist_to_workspace: - root: packaging/windows/output + root: dist paths: - "*" - store_test_results: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 459cae94ac0..746969a2f06 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -6,14 +6,17 @@ version: 2.1 # - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. # Job names are 'name:' key. -orbs: - win: circleci/windows@2.0.0 - executors: - windows-gpu-prototype: + windows-cpu: + machine: + resource_class: windows.xlarge + image: windows-server-2019-vs2019:stable + shell: bash.exe + + windows-gpu: machine: - resource_class: windows.gpu.small.prototype - image: windows-server-2019-nvidia:201908-28 + resource_class: windows.gpu.nvidia.medium + image: windows-server-2019-nvidia:stable shell: bash.exe commands: @@ -98,7 +101,6 @@ binary_windows: &binary_windows PYTORCH_VERSION: << parameters.pytorch_version >> CUDA_VERSION: << parameters.cu_version >> USE_SCCACHE: "1" - VC_YEAR: "2017" jobs: circleci_consistency: @@ -248,51 +250,63 @@ jobs: binary_win_conda: <<: *binary_common - executor: - name: win/default - shell: bash.exe + executor: windows-cpu steps: - checkout_merge - run: command: | - choco install miniconda3 - (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression + set -ex + export VC_YEAR=2019 + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" - bash packaging/build_conda.sh - shell: powershell.exe + packaging/build_conda.sh - store_test_results: path: build_results/ binary_win_conda_cuda: <<: *binary_common - executor: windows-gpu-prototype + executor: windows-gpu steps: - checkout_merge - run: command: | - choco install miniconda3 - (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression + set -ex + export VC_YEAR=2019 + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" - bash packaging/build_conda.sh + packaging/build_conda.sh shell: powershell.exe binary_win_conda_release: <<: *binary_windows - executor: - name: win/default - shell: cmd.exe + executor: windows-cpu steps: - checkout_merge - run: name: Build conda packages command: | - call packaging/windows/internal/build_conda.bat + set -ex + if [[ "$CUDA_VERSION" == "92" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="-vcvars_ver=14.11" + powershell packaging/windows/internal/vs2017_install.ps1 + elif [["$CUDA_VERSION" == "100" ]]; then + export VC_YEAR=2017 + powershell packaging/windows/internal/vs2017_install.ps1 + else + export VC_YEAR=2019 + fi + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + conda install -yq conda-build "conda-package-handling!=1.5.0" + packaging/build_conda.sh + rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2 - store_artifacts: - path: packaging/windows/output + path: C:/tools/miniconda3/conda-bld/win-64 - persist_to_workspace: - root: packaging/windows/output + root: C:/tools/miniconda3/conda-bld/win-64 paths: - "*" - store_test_results: @@ -300,19 +314,28 @@ jobs: binary_win_wheel_release: <<: *binary_windows - executor: - name: win/default - shell: cmd.exe + executor: windows-cpu steps: - checkout_merge - run: name: Build wheel packages command: | - call packaging/windows/internal/build_wheels.bat + set -ex + if [[ "$CUDA_VERSION" == "92" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="-vcvars_ver=14.11" + powershell packaging/windows/internal/vs2017_install.ps1 + elif [["$CUDA_VERSION" == "100" ]]; then + export VC_YEAR=2017 + powershell packaging/windows/internal/vs2017_install.ps1 + else + export VC_YEAR=2019 + fi + packaging/build_wheel.sh - store_artifacts: - path: packaging/windows/output + path: dist - persist_to_workspace: - root: packaging/windows/output + root: dist paths: - "*" - store_test_results: diff --git a/packaging/build_wheel.bat b/packaging/build_wheel.bat new file mode 100644 index 00000000000..d9ba42ad046 --- /dev/null +++ b/packaging/build_wheel.bat @@ -0,0 +1,22 @@ +@echo on + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,17^) -property installationPath`) do ( + if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VS15INSTALLDIR=%%i" + set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" + goto vswhere + ) +) + +:vswhere +if "%VSDEVCMD_ARGS%" == "" ( + call "%VS15VCVARSALL%" x64 || exit /b 1 +) else ( + call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 +) + +@echo on + +set DISTUTILS_USE_SDK=1 + +python setup.py bdist_wheel || exit /b 1 diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index f4bacff91c0..d43f81694a2 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -10,4 +10,8 @@ setup_wheel_python pip_install numpy pyyaml future ninja setup_pip_pytorch_version python setup.py clean -IS_WHEEL=1 python setup.py bdist_wheel +if [[ "$OSTYPE" == "msys" ]]; then + IS_WHEEL=1 "$script_dir/build_wheel.bat" +else + IS_WHEEL=1 python setup.py bdist_wheel +fi diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index c306e4f05f9..91dfd89803a 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -165,7 +165,7 @@ retry () { # # Precondition: If Linux, you are in a soumith/manylinux-cuda* Docker image setup_wheel_python() { - if [[ "$(uname)" == Darwin ]]; then + if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then eval "$(conda shell.bash hook)" conda env remove -n "env$PYTHON_VERSION" || true conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" @@ -285,8 +285,7 @@ setup_conda_cudatoolkit_constraint() { # Build the proper compiler package before building the final package setup_visual_studio_constraint() { if [[ "$OSTYPE" == "msys" ]]; then - export VSTOOLCHAIN_PACKAGE=vs2019 - export VSDEVCMD_ARGS='' + export VSTOOLCHAIN_PACKAGE=vs$VC_YEAR conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload packaging/$VSTOOLCHAIN_PACKAGE cp packaging/$VSTOOLCHAIN_PACKAGE/conda_build_config.yaml packaging/torchvision/conda_build_config.yaml fi diff --git a/packaging/vs2019/meta.yaml b/packaging/vs2019/meta.yaml index e3f8b471481..94a0ed4db3e 100644 --- a/packaging/vs2019/meta.yaml +++ b/packaging/vs2019/meta.yaml @@ -19,27 +19,6 @@ outputs: # VS 2019 is binary-compatible with VS 2017/vc 14.1 and 2015/vc14. Tools are "v142". strong: - vc{{ vcfeature }} - run_exports: - - vc {{ vcver }} about: summary: Activation and version verification of MSVC {{ vcver }} (VS {{ vsyear }}) compiler license: BSD 3-clause - - name: vs{{ vsyear }}_runtime - script: install_runtime.bat - - name: vc - version: {{ vcver }} - track_features: - - vc{{ vcfeature }} - requirements: - run: - - {{ pin_subpackage('vs' ~ vsyear ~ '_runtime') }} - about: - home: https://github.com/conda/conda/wiki/VC-features - license: Modified BSD License (3-clause) - license_family: BSD - summary: A meta-package to track VC features. - description: | - This metapackage is used to activate vc features without - depending on Python. - doc_url: https://github.com/conda/conda/wiki/VC-features - dev_url: https://github.com/conda/conda/wiki/VC-features diff --git a/packaging/windows/internal/build_conda.bat b/packaging/windows/internal/build_conda.bat index ec6af1772b4..e66d5596298 100644 --- a/packaging/windows/internal/build_conda.bat +++ b/packaging/windows/internal/build_conda.bat @@ -1,5 +1,5 @@ if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.11 -if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs_install.ps1 +if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs2017_install.ps1 if errorlevel 1 exit /b 1 call packaging/windows/internal/cuda_install.bat diff --git a/packaging/windows/internal/build_wheels.bat b/packaging/windows/internal/build_wheels.bat index 405ddd977a2..869e594b395 100644 --- a/packaging/windows/internal/build_wheels.bat +++ b/packaging/windows/internal/build_wheels.bat @@ -1,5 +1,5 @@ if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.11 -if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs_install.ps1 +if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs2017_install.ps1 if errorlevel 1 exit /b 1 call packaging/windows/internal/cuda_install.bat diff --git a/packaging/windows/internal/vs_install.ps1 b/packaging/windows/internal/vs2017_install.ps1 similarity index 100% rename from packaging/windows/internal/vs_install.ps1 rename to packaging/windows/internal/vs2017_install.ps1 diff --git a/packaging/windows/internal/vs2019_install.ps1 b/packaging/windows/internal/vs2019_install.ps1 new file mode 100644 index 00000000000..e436051f0db --- /dev/null +++ b/packaging/windows/internal/vs2019_install.ps1 @@ -0,0 +1,21 @@ +$VS_DOWNLOAD_LINK = "https://aka.ms/vs/16/release/vs_buildtools.exe" +$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools", + "--add Microsoft.Component.MSBuild", + "--add Microsoft.VisualStudio.Component.Roslyn.Compiler", + "--add Microsoft.VisualStudio.Component.VC.CoreBuildTools", + "--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest", + "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64") + +curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe +if ($LASTEXITCODE -ne 0) { + echo "Download of the VS 2019 installer failed" + exit 1 +} + +$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru +Remove-Item -Path vs_installer.exe -Force +$exitCode = $process.ExitCode +if (($exitCode -ne 0) -and ($exitCode -ne 3010)) { + echo "VS 2019 installer exited with code $exitCode, which should be one of [0, 3010]." + exit 1 +} From 0ff0f0e326f7a450c3142d35c3e96081abaf8acf Mon Sep 17 00:00:00 2001 From: peter Date: Wed, 27 May 2020 16:47:23 +0800 Subject: [PATCH 2/8] Reuse variables --- .circleci/config.yml | 104 ++++++++++++++++------------------------ .circleci/config.yml.in | 32 ++----------- .circleci/regenerate.py | 2 +- 3 files changed, 45 insertions(+), 93 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0219d3f445c..3889837ab85 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,30 +78,6 @@ binary_common: &binary_common UNICODE_ABI: << parameters.unicode_abi >> CU_VERSION: << parameters.cu_version >> -binary_windows: &binary_windows - parameters: - # Edit these defaults to do a release` - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.7)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - environment: - DESIRED_PYTHON: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - CUDA_VERSION: << parameters.cu_version >> - USE_SCCACHE: "1" - jobs: circleci_consistency: docker: @@ -288,11 +264,11 @@ jobs: name: Build conda packages command: | set -ex - if [[ "$CUDA_VERSION" == "92" ]]; then + if [[ "$CU_VERSION" == "cu92" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CUDA_VERSION" == "100" ]]; then + elif [["$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 powershell packaging/windows/internal/vs2017_install.ps1 else @@ -321,11 +297,11 @@ jobs: name: Build wheel packages command: | set -ex - if [[ "$CUDA_VERSION" == "92" ]]; then + if [[ "$CU_VERSION" == "cu92" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CUDA_VERSION" == "100" ]]; then + elif [["$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 powershell packaging/windows/internal/vs2017_install.ps1 else @@ -514,21 +490,21 @@ workflows: name: binary_win_wheel_py3.6_cpu python_version: '3.6' - binary_win_wheel_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: master name: binary_win_wheel_py3.6_cu92 python_version: '3.6' - binary_win_wheel_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: master name: binary_win_wheel_py3.6_cu101 python_version: '3.6' - binary_win_wheel_release: - cu_version: '102' + cu_version: cu102 filters: branches: only: master @@ -542,21 +518,21 @@ workflows: name: binary_win_wheel_py3.7_cpu python_version: '3.7' - binary_win_wheel_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: master name: binary_win_wheel_py3.7_cu92 python_version: '3.7' - binary_win_wheel_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: master name: binary_win_wheel_py3.7_cu101 python_version: '3.7' - binary_win_wheel_release: - cu_version: '102' + cu_version: cu102 filters: branches: only: master @@ -567,21 +543,21 @@ workflows: name: binary_win_wheel_py3.8_cpu python_version: '3.8' - binary_win_wheel_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: master name: binary_win_wheel_py3.8_cu92 python_version: '3.8' - binary_win_wheel_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: master name: binary_win_wheel_py3.8_cu101 python_version: '3.8' - binary_win_wheel_release: - cu_version: '102' + cu_version: cu102 name: binary_win_wheel_py3.8_cu102 python_version: '3.8' - binary_linux_conda: @@ -667,21 +643,21 @@ workflows: name: binary_win_conda_py3.6_cpu python_version: '3.6' - binary_win_conda_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: master name: binary_win_conda_py3.6_cu92 python_version: '3.6' - binary_win_conda_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: master name: binary_win_conda_py3.6_cu101 python_version: '3.6' - binary_win_conda_release: - cu_version: '102' + cu_version: cu102 filters: branches: only: master @@ -695,21 +671,21 @@ workflows: name: binary_win_conda_py3.7_cpu python_version: '3.7' - binary_win_conda_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: master name: binary_win_conda_py3.7_cu92 python_version: '3.7' - binary_win_conda_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: master name: binary_win_conda_py3.7_cu101 python_version: '3.7' - binary_win_conda_release: - cu_version: '102' + cu_version: cu102 filters: branches: only: master @@ -720,21 +696,21 @@ workflows: name: binary_win_conda_py3.8_cpu python_version: '3.8' - binary_win_conda_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: master name: binary_win_conda_py3.8_cu92 python_version: '3.8' - binary_win_conda_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: master name: binary_win_conda_py3.8_cu101 python_version: '3.8' - binary_win_conda_release: - cu_version: '102' + cu_version: cu102 name: binary_win_conda_py3.8_cu102 python_version: '3.8' - binary_linux_conda_cuda: @@ -1063,7 +1039,7 @@ workflows: - nightly_binary_win_wheel_py3.6_cpu subfolder: cpu/ - binary_win_wheel_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: nightly @@ -1081,7 +1057,7 @@ workflows: - nightly_binary_win_wheel_py3.6_cu92 subfolder: cu92/ - binary_win_wheel_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: nightly @@ -1099,7 +1075,7 @@ workflows: - nightly_binary_win_wheel_py3.6_cu101 subfolder: cu101/ - binary_win_wheel_release: - cu_version: '102' + cu_version: cu102 filters: branches: only: nightly @@ -1135,7 +1111,7 @@ workflows: - nightly_binary_win_wheel_py3.7_cpu subfolder: cpu/ - binary_win_wheel_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: nightly @@ -1153,7 +1129,7 @@ workflows: - nightly_binary_win_wheel_py3.7_cu92 subfolder: cu92/ - binary_win_wheel_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: nightly @@ -1171,7 +1147,7 @@ workflows: - nightly_binary_win_wheel_py3.7_cu101 subfolder: cu101/ - binary_win_wheel_release: - cu_version: '102' + cu_version: cu102 filters: branches: only: nightly @@ -1207,7 +1183,7 @@ workflows: - nightly_binary_win_wheel_py3.8_cpu subfolder: cpu/ - binary_win_wheel_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: nightly @@ -1225,7 +1201,7 @@ workflows: - nightly_binary_win_wheel_py3.8_cu92 subfolder: cu92/ - binary_win_wheel_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: nightly @@ -1243,7 +1219,7 @@ workflows: - nightly_binary_win_wheel_py3.8_cu101 subfolder: cu101/ - binary_win_wheel_release: - cu_version: '102' + cu_version: cu102 filters: branches: only: nightly @@ -1548,7 +1524,7 @@ workflows: requires: - nightly_binary_win_conda_py3.6_cpu - binary_win_conda_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: nightly @@ -1565,7 +1541,7 @@ workflows: requires: - nightly_binary_win_conda_py3.6_cu92 - binary_win_conda_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: nightly @@ -1582,7 +1558,7 @@ workflows: requires: - nightly_binary_win_conda_py3.6_cu101 - binary_win_conda_release: - cu_version: '102' + cu_version: cu102 filters: branches: only: nightly @@ -1616,7 +1592,7 @@ workflows: requires: - nightly_binary_win_conda_py3.7_cpu - binary_win_conda_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: nightly @@ -1633,7 +1609,7 @@ workflows: requires: - nightly_binary_win_conda_py3.7_cu92 - binary_win_conda_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: nightly @@ -1650,7 +1626,7 @@ workflows: requires: - nightly_binary_win_conda_py3.7_cu101 - binary_win_conda_release: - cu_version: '102' + cu_version: cu102 filters: branches: only: nightly @@ -1684,7 +1660,7 @@ workflows: requires: - nightly_binary_win_conda_py3.8_cpu - binary_win_conda_release: - cu_version: '92' + cu_version: cu92 filters: branches: only: nightly @@ -1701,7 +1677,7 @@ workflows: requires: - nightly_binary_win_conda_py3.8_cu92 - binary_win_conda_release: - cu_version: '101' + cu_version: cu101 filters: branches: only: nightly @@ -1718,7 +1694,7 @@ workflows: requires: - nightly_binary_win_conda_py3.8_cu101 - binary_win_conda_release: - cu_version: '102' + cu_version: cu102 filters: branches: only: nightly diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 746969a2f06..0c404a1cc5f 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -78,30 +78,6 @@ binary_common: &binary_common UNICODE_ABI: << parameters.unicode_abi >> CU_VERSION: << parameters.cu_version >> -binary_windows: &binary_windows - parameters: - # Edit these defaults to do a release` - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.7)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - environment: - DESIRED_PYTHON: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - CUDA_VERSION: << parameters.cu_version >> - USE_SCCACHE: "1" - jobs: circleci_consistency: docker: @@ -288,11 +264,11 @@ jobs: name: Build conda packages command: | set -ex - if [[ "$CUDA_VERSION" == "92" ]]; then + if [[ "$CU_VERSION" == "cu92" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CUDA_VERSION" == "100" ]]; then + elif [["$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 powershell packaging/windows/internal/vs2017_install.ps1 else @@ -321,11 +297,11 @@ jobs: name: Build wheel packages command: | set -ex - if [[ "$CUDA_VERSION" == "92" ]]; then + if [[ "$CU_VERSION" == "cu92" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CUDA_VERSION" == "100" ]]; then + elif [["$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 powershell packaging/windows/internal/vs2017_install.ps1 else diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 4fb8116a72b..1e929242974 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -76,7 +76,7 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version, d = { "name": base_workflow_name, "python_version": python_version, - "cu_version": cu_version.replace("cu", "") if os_type == "win" else cu_version, + "cu_version": cu_version, } if os_type != "win" and unicode: From 799c23900178b7da95d38b9485d95c2c49026417 Mon Sep 17 00:00:00 2001 From: peter Date: Wed, 27 May 2020 16:50:43 +0800 Subject: [PATCH 3/8] Remove Windows-only parameter set --- .circleci/config.yml | 4 ++-- .circleci/config.yml.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3889837ab85..6322c5116ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -256,7 +256,7 @@ jobs: shell: powershell.exe binary_win_conda_release: - <<: *binary_windows + <<: *binary_common executor: windows-cpu steps: - checkout_merge @@ -289,7 +289,7 @@ jobs: path: build_results/ binary_win_wheel_release: - <<: *binary_windows + <<: *binary_common executor: windows-cpu steps: - checkout_merge diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 0c404a1cc5f..f6a331cf175 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -256,7 +256,7 @@ jobs: shell: powershell.exe binary_win_conda_release: - <<: *binary_windows + <<: *binary_common executor: windows-cpu steps: - checkout_merge @@ -289,7 +289,7 @@ jobs: path: build_results/ binary_win_wheel_release: - <<: *binary_windows + <<: *binary_common executor: windows-cpu steps: - checkout_merge From 56c08b2f271258fbaacdcb9bf4e2ce2d5e0a5c17 Mon Sep 17 00:00:00 2001 From: peter Date: Wed, 27 May 2020 16:53:18 +0800 Subject: [PATCH 4/8] Don't use powershell --- .circleci/config.yml | 1 - .circleci/config.yml.in | 1 - 2 files changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6322c5116ea..53b2dfddaa8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -253,7 +253,6 @@ jobs: conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" packaging/build_conda.sh - shell: powershell.exe binary_win_conda_release: <<: *binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index f6a331cf175..0103389b1c4 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -253,7 +253,6 @@ jobs: conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" packaging/build_conda.sh - shell: powershell.exe binary_win_conda_release: <<: *binary_common From 21c8e34815eb58d6e174aa1613c7d61b8a132e90 Mon Sep 17 00:00:00 2001 From: peter Date: Wed, 27 May 2020 17:05:58 +0800 Subject: [PATCH 5/8] Fix config --- .circleci/config.yml | 30 ++++++++++++++++++++++++++++-- .circleci/config.yml.in | 30 ++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 53b2dfddaa8..344d285aa8d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -232,7 +232,18 @@ jobs: - run: command: | set -ex - export VC_YEAR=2019 + if [[ "$CU_VERSION" == "cu92" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="-vcvars_ver=14.11" + powershell packaging/windows/internal/vs2017_install.ps1 + elif [["$CU_VERSION" == "cu100" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="" + powershell packaging/windows/internal/vs2017_install.ps1 + else + export VC_YEAR=2019 + export VSDEVCMD_ARGS="" + fi eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" @@ -248,7 +259,18 @@ jobs: - run: command: | set -ex - export VC_YEAR=2019 + if [[ "$CU_VERSION" == "cu92" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="-vcvars_ver=14.11" + powershell packaging/windows/internal/vs2017_install.ps1 + elif [["$CU_VERSION" == "cu100" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="" + powershell packaging/windows/internal/vs2017_install.ps1 + else + export VC_YEAR=2019 + export VSDEVCMD_ARGS="" + fi eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" @@ -269,9 +291,11 @@ jobs: powershell packaging/windows/internal/vs2017_install.ps1 elif [["$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 + export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 else export VC_YEAR=2019 + export VSDEVCMD_ARGS="" fi eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base @@ -302,9 +326,11 @@ jobs: powershell packaging/windows/internal/vs2017_install.ps1 elif [["$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 + export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 else export VC_YEAR=2019 + export VSDEVCMD_ARGS="" fi packaging/build_wheel.sh - store_artifacts: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 0103389b1c4..c3a7840d4e5 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -232,7 +232,18 @@ jobs: - run: command: | set -ex - export VC_YEAR=2019 + if [[ "$CU_VERSION" == "cu92" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="-vcvars_ver=14.11" + powershell packaging/windows/internal/vs2017_install.ps1 + elif [["$CU_VERSION" == "cu100" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="" + powershell packaging/windows/internal/vs2017_install.ps1 + else + export VC_YEAR=2019 + export VSDEVCMD_ARGS="" + fi eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" @@ -248,7 +259,18 @@ jobs: - run: command: | set -ex - export VC_YEAR=2019 + if [[ "$CU_VERSION" == "cu92" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="-vcvars_ver=14.11" + powershell packaging/windows/internal/vs2017_install.ps1 + elif [["$CU_VERSION" == "cu100" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="" + powershell packaging/windows/internal/vs2017_install.ps1 + else + export VC_YEAR=2019 + export VSDEVCMD_ARGS="" + fi eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" @@ -269,9 +291,11 @@ jobs: powershell packaging/windows/internal/vs2017_install.ps1 elif [["$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 + export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 else export VC_YEAR=2019 + export VSDEVCMD_ARGS="" fi eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base @@ -302,9 +326,11 @@ jobs: powershell packaging/windows/internal/vs2017_install.ps1 elif [["$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 + export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 else export VC_YEAR=2019 + export VSDEVCMD_ARGS="" fi packaging/build_wheel.sh - store_artifacts: From cd134d22cba10caea322364156ad741cc100d84b Mon Sep 17 00:00:00 2001 From: peter Date: Wed, 27 May 2020 17:22:33 +0800 Subject: [PATCH 6/8] Install CUDA --- .circleci/config.yml | 10 ++++++---- .circleci/config.yml.in | 10 ++++++---- packaging/windows/internal/cuda_install.bat | 8 ++++---- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 344d285aa8d..609a3efd6e8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -236,7 +236,7 @@ jobs: export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CU_VERSION" == "cu100" ]]; then + elif [[ "$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 @@ -263,7 +263,7 @@ jobs: export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CU_VERSION" == "cu100" ]]; then + elif [[ "$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 @@ -289,7 +289,7 @@ jobs: export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CU_VERSION" == "cu100" ]]; then + elif [[ "$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 @@ -297,6 +297,7 @@ jobs: export VC_YEAR=2019 export VSDEVCMD_ARGS="" fi + packaging/windows/internal/cuda_install.bat eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" @@ -324,7 +325,7 @@ jobs: export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CU_VERSION" == "cu100" ]]; then + elif [[ "$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 @@ -332,6 +333,7 @@ jobs: export VC_YEAR=2019 export VSDEVCMD_ARGS="" fi + packaging/windows/internal/cuda_install.bat packaging/build_wheel.sh - store_artifacts: path: dist diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index c3a7840d4e5..27621ee9975 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -236,7 +236,7 @@ jobs: export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CU_VERSION" == "cu100" ]]; then + elif [[ "$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 @@ -263,7 +263,7 @@ jobs: export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CU_VERSION" == "cu100" ]]; then + elif [[ "$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 @@ -289,7 +289,7 @@ jobs: export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CU_VERSION" == "cu100" ]]; then + elif [[ "$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 @@ -297,6 +297,7 @@ jobs: export VC_YEAR=2019 export VSDEVCMD_ARGS="" fi + packaging/windows/internal/cuda_install.bat eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" @@ -324,7 +325,7 @@ jobs: export VC_YEAR=2017 export VSDEVCMD_ARGS="-vcvars_ver=14.11" powershell packaging/windows/internal/vs2017_install.ps1 - elif [["$CU_VERSION" == "cu100" ]]; then + elif [[ "$CU_VERSION" == "cu100" ]]; then export VC_YEAR=2017 export VSDEVCMD_ARGS="" powershell packaging/windows/internal/vs2017_install.ps1 @@ -332,6 +333,7 @@ jobs: export VC_YEAR=2019 export VSDEVCMD_ARGS="" fi + packaging/windows/internal/cuda_install.bat packaging/build_wheel.sh - store_artifacts: path: dist diff --git a/packaging/windows/internal/cuda_install.bat b/packaging/windows/internal/cuda_install.bat index d118421825d..aff5f834a99 100644 --- a/packaging/windows/internal/cuda_install.bat +++ b/packaging/windows/internal/cuda_install.bat @@ -1,6 +1,6 @@ @echo on -if "%CUDA_VERSION%" == "cpu" ( +if "%CU_VERSION%" == "cpu" ( echo Skipping for CPU builds exit /b 0 ) @@ -9,9 +9,9 @@ set SRC_DIR=%~dp0\.. if not exist "%SRC_DIR%\temp_build" mkdir "%SRC_DIR%\temp_build" -set /a CUDA_VER=%CUDA_VERSION% -set CUDA_VER_MAJOR=%CUDA_VERSION:~0,-1% -set CUDA_VER_MINOR=%CUDA_VERSION:~-1,1% +set /a CUDA_VER=%CU_VERSION:cu=% +set CUDA_VER_MAJOR=%CUDA_VER:~0,-1% +set CUDA_VER_MINOR=%CUDA_VER:~-1,1% set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR% if %CUDA_VER% EQU 92 goto cuda92 From 3e4326e89a26b3feeaaffb9581157fb794ee0ee8 Mon Sep 17 00:00:00 2001 From: peter Date: Thu, 28 May 2020 00:00:32 +0800 Subject: [PATCH 7/8] Support VS2017 in build_wheel.bat --- packaging/build_wheel.bat | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packaging/build_wheel.bat b/packaging/build_wheel.bat index d9ba42ad046..425c8166b64 100644 --- a/packaging/build_wheel.bat +++ b/packaging/build_wheel.bat @@ -1,6 +1,13 @@ @echo on -for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,17^) -property installationPath`) do ( +set VC_VERSION_LOWER=16 +set VC_VERSION_UPPER=17 +if "%VC_YEAR%" == "2017" ( + set VC_VERSION_LOWER=15 + set VC_VERSION_UPPER=16 +) + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( set "VS15INSTALLDIR=%%i" set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" From 3d470bbd0d86bcce96bd135d02423e6579d5190f Mon Sep 17 00:00:00 2001 From: peter Date: Thu, 28 May 2020 13:10:41 +0800 Subject: [PATCH 8/8] Target comments --- .circleci/config.yml | 52 ++----------------- .circleci/config.yml.in | 52 ++----------------- packaging/build_wheel.sh | 2 +- .../internal/vc_env_helper.bat} | 16 +++++- .../windows/internal/vc_install_helper.sh | 16 ++++++ 5 files changed, 40 insertions(+), 98 deletions(-) rename packaging/{build_wheel.bat => windows/internal/vc_env_helper.bat} (76%) create mode 100644 packaging/windows/internal/vc_install_helper.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 609a3efd6e8..4ca5e4ec15d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -232,18 +232,7 @@ jobs: - run: command: | set -ex - if [[ "$CU_VERSION" == "cu92" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="-vcvars_ver=14.11" - powershell packaging/windows/internal/vs2017_install.ps1 - elif [[ "$CU_VERSION" == "cu100" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="" - powershell packaging/windows/internal/vs2017_install.ps1 - else - export VC_YEAR=2019 - export VSDEVCMD_ARGS="" - fi + source packaging/windows/internal/vc_install_helper.sh eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" @@ -259,18 +248,7 @@ jobs: - run: command: | set -ex - if [[ "$CU_VERSION" == "cu92" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="-vcvars_ver=14.11" - powershell packaging/windows/internal/vs2017_install.ps1 - elif [[ "$CU_VERSION" == "cu100" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="" - powershell packaging/windows/internal/vs2017_install.ps1 - else - export VC_YEAR=2019 - export VSDEVCMD_ARGS="" - fi + source packaging/windows/internal/vc_install_helper.sh eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" @@ -285,18 +263,7 @@ jobs: name: Build conda packages command: | set -ex - if [[ "$CU_VERSION" == "cu92" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="-vcvars_ver=14.11" - powershell packaging/windows/internal/vs2017_install.ps1 - elif [[ "$CU_VERSION" == "cu100" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="" - powershell packaging/windows/internal/vs2017_install.ps1 - else - export VC_YEAR=2019 - export VSDEVCMD_ARGS="" - fi + source packaging/windows/internal/vc_install_helper.sh packaging/windows/internal/cuda_install.bat eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base @@ -321,18 +288,7 @@ jobs: name: Build wheel packages command: | set -ex - if [[ "$CU_VERSION" == "cu92" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="-vcvars_ver=14.11" - powershell packaging/windows/internal/vs2017_install.ps1 - elif [[ "$CU_VERSION" == "cu100" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="" - powershell packaging/windows/internal/vs2017_install.ps1 - else - export VC_YEAR=2019 - export VSDEVCMD_ARGS="" - fi + source packaging/windows/internal/vc_install_helper.sh packaging/windows/internal/cuda_install.bat packaging/build_wheel.sh - store_artifacts: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 27621ee9975..cb815e6fc91 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -232,18 +232,7 @@ jobs: - run: command: | set -ex - if [[ "$CU_VERSION" == "cu92" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="-vcvars_ver=14.11" - powershell packaging/windows/internal/vs2017_install.ps1 - elif [[ "$CU_VERSION" == "cu100" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="" - powershell packaging/windows/internal/vs2017_install.ps1 - else - export VC_YEAR=2019 - export VSDEVCMD_ARGS="" - fi + source packaging/windows/internal/vc_install_helper.sh eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" @@ -259,18 +248,7 @@ jobs: - run: command: | set -ex - if [[ "$CU_VERSION" == "cu92" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="-vcvars_ver=14.11" - powershell packaging/windows/internal/vs2017_install.ps1 - elif [[ "$CU_VERSION" == "cu100" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="" - powershell packaging/windows/internal/vs2017_install.ps1 - else - export VC_YEAR=2019 - export VSDEVCMD_ARGS="" - fi + source packaging/windows/internal/vc_install_helper.sh eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" @@ -285,18 +263,7 @@ jobs: name: Build conda packages command: | set -ex - if [[ "$CU_VERSION" == "cu92" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="-vcvars_ver=14.11" - powershell packaging/windows/internal/vs2017_install.ps1 - elif [[ "$CU_VERSION" == "cu100" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="" - powershell packaging/windows/internal/vs2017_install.ps1 - else - export VC_YEAR=2019 - export VSDEVCMD_ARGS="" - fi + source packaging/windows/internal/vc_install_helper.sh packaging/windows/internal/cuda_install.bat eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base @@ -321,18 +288,7 @@ jobs: name: Build wheel packages command: | set -ex - if [[ "$CU_VERSION" == "cu92" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="-vcvars_ver=14.11" - powershell packaging/windows/internal/vs2017_install.ps1 - elif [[ "$CU_VERSION" == "cu100" ]]; then - export VC_YEAR=2017 - export VSDEVCMD_ARGS="" - powershell packaging/windows/internal/vs2017_install.ps1 - else - export VC_YEAR=2019 - export VSDEVCMD_ARGS="" - fi + source packaging/windows/internal/vc_install_helper.sh packaging/windows/internal/cuda_install.bat packaging/build_wheel.sh - store_artifacts: diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index d43f81694a2..d94c0f4c7c8 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -11,7 +11,7 @@ pip_install numpy pyyaml future ninja setup_pip_pytorch_version python setup.py clean if [[ "$OSTYPE" == "msys" ]]; then - IS_WHEEL=1 "$script_dir/build_wheel.bat" + IS_WHEEL=1 "$script_dir/windows/internal/vc_env_helper.bat" python setup.py bdist_wheel else IS_WHEEL=1 python setup.py bdist_wheel fi diff --git a/packaging/build_wheel.bat b/packaging/windows/internal/vc_env_helper.bat similarity index 76% rename from packaging/build_wheel.bat rename to packaging/windows/internal/vc_env_helper.bat index 425c8166b64..e85a372f93d 100644 --- a/packaging/build_wheel.bat +++ b/packaging/windows/internal/vc_env_helper.bat @@ -26,4 +26,18 @@ if "%VSDEVCMD_ARGS%" == "" ( set DISTUTILS_USE_SDK=1 -python setup.py bdist_wheel || exit /b 1 +set args=%1 +shift +:start +if [%1] == [] goto done +set args=%args% %1 +shift +goto start + +:done +if "%args%" == "" ( + echo Usage: vc_env_helper.bat [command] [args] + echo e.g. vc_env_helper.bat cl /c test.cpp +) + +%args% || exit /b 1 diff --git a/packaging/windows/internal/vc_install_helper.sh b/packaging/windows/internal/vc_install_helper.sh new file mode 100644 index 00000000000..9910677acac --- /dev/null +++ b/packaging/windows/internal/vc_install_helper.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -ex + +if [[ "$CU_VERSION" == "cu92" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="-vcvars_ver=14.11" + powershell packaging/windows/internal/vs2017_install.ps1 +elif [[ "$CU_VERSION" == "cu100" ]]; then + export VC_YEAR=2017 + export VSDEVCMD_ARGS="" + powershell packaging/windows/internal/vs2017_install.ps1 +else + export VC_YEAR=2019 + export VSDEVCMD_ARGS="" +fi