Add windows build steps and wheel build scripts (#998)

packaging/README.md

@@ -29,6 +29,18 @@ conda install -y conda-build anaconda-client
 # anaconda upload -u pytorch torchvision*.bz2
 ```
 
+### Windows
+
+```bash
+# Open `Git Bash` and change dir to `conda`
+./build_vision.sh 9.0
+./build_vision.sh 10.0
+./build_vision.sh cpu
+
+# copy packages to a output directory
+# anaconda upload -u pytorch torchvision*.bz2
+```
+
 ## Wheels
 
 ### Linux
@@ -62,6 +74,17 @@ pushd wheel
 ./osx_wheel.sh
 ```
 
+### Windows
+
+```cmd
+set PYTORCH_REPO=pytorch
+
+pushd windows
+call build_vision.bat 90 0.3.0 1
+call build_vision.bat 100 0.3.0 1
+call build_vision.bat cpu 0.3.0 1
+```
+
 wheels are in the current folder.
 
 You can upload them to twine with `twine upload *.whl`

packaging/conda/install_conda.bat

@@ -0,0 +1 @@
+start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%

packaging/windows/build_vision.bat

@@ -0,0 +1,94 @@
+@echo off
+
+:: This script parses args, installs required libraries (miniconda, MKL,
+:: Magma), and then delegates to cpu.bat, cuda80.bat, etc.
+
+IF NOT "%CUDA_VERSION%" == "" IF NOT "%TORCHVISION_BUILD_VERSION%" == "" if NOT "%TORCHVISION_BUILD_NUMBER%" == "" goto env_end
+if "%~1"=="" goto arg_error
+if "%~2"=="" goto arg_error
+if "%~3"=="" goto arg_error
+if NOT "%~4"=="" goto arg_error
+goto arg_end
+
+:arg_error
+
+echo Illegal number of parameters. Pass cuda version, pytorch version, build number
+echo CUDA version should be Mm with no dot, e.g. '80'
+echo DESIRED_PYTHON should be M.m, e.g. '2.7'
+exit /b 1
+
+:arg_end
+
+set CUDA_VERSION=%~1
+set TORCHVISION_BUILD_VERSION=%~2
+set TORCHVISION_BUILD_NUMBER=%~3
+
+:env_end
+
+if NOT "%CUDA_VERSION%" == "cpu" (
+    set CUDA_PREFIX=cuda%CUDA_VERSION%
+    set CUVER=cu%CUDA_VERSION%
+) else (
+    set CUDA_PREFIX=cpu
+    set CUVER=cpu
+)
+
+set BUILD_VISION=1
+set TORCH_WHEEL=torch -f https://download.pytorch.org/whl/%CUVER%/stable.html --no-index
+
+IF "%DESIRED_PYTHON%" == "" set DESIRED_PYTHON=3.5;3.6;3.7
+set DESIRED_PYTHON_PREFIX=%DESIRED_PYTHON:.=%
+set DESIRED_PYTHON_PREFIX=py%DESIRED_PYTHON_PREFIX:;=;py%
+
+set SRC_DIR=%~dp0
+pushd %SRC_DIR%
+
+:: Install Miniconda3
+set "CONDA_HOME=%CD%\conda"
+set "tmp_conda=%CONDA_HOME%"
+set "miniconda_exe=%CD%\miniconda.exe"
+rmdir /s /q conda
+del miniconda.exe
+curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%"
+call ..\conda\install_conda.bat
+IF ERRORLEVEL 1 exit /b 1
+set "ORIG_PATH=%PATH%"
+set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%"
+
+:: Create a new conda environment
+setlocal EnableDelayedExpansion
+FOR %%v IN (%DESIRED_PYTHON%) DO (
+    set PYTHON_VERSION_STR=%%v
+    set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
+    conda remove -n py!PYTHON_VERSION_STR! --all -y || rmdir %CONDA_HOME%\envs\py!PYTHON_VERSION_STR! /s
+    conda create -n py!PYTHON_VERSION_STR! -y -q numpy>=1.11 mkl>=2018 python=%%v
+)
+endlocal
+
+if "%DEBUG%" == "1" (
+    set BUILD_TYPE=debug
+) ELSE (
+    set BUILD_TYPE=release
+)
+
+for %%v in (%DESIRED_PYTHON_PREFIX%) do (
+    :: Activate Python Environment
+    set PYTHON_PREFIX=%%v
+    set "PATH=%CONDA_HOME%\envs\%%v;%CONDA_HOME%\envs\%%v\scripts;%CONDA_HOME%\envs\%%v\Library\bin;%ORIG_PATH%"
+    pip install %TORCH_WHEEL%
+    @setlocal
+    :: Set Flags
+    if NOT "%CUDA_VERSION%"=="cpu" (
+        set CUDNN_VERSION=7
+    )
+    call %CUDA_PREFIX%.bat
+    IF ERRORLEVEL 1 exit /b 1
+    call internal\test.bat
+    IF ERRORLEVEL 1 exit /b 1
+    @endlocal
+)
+
+set "PATH=%ORIG_PATH%"
+popd
+
+IF ERRORLEVEL 1 exit /b 1

packaging/windows/cpu.bat

@@ -0,0 +1,37 @@
+@echo off
+
+IF NOT "%BUILD_VISION%" == "" (
+    set MODULE_NAME=vision
+) ELSE (
+    set MODULE_NAME=pytorch
+)
+
+IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
+    call internal\clone.bat
+    cd ..
+    IF ERRORLEVEL 1 goto eof
+) ELSE (
+    call internal\clean.bat
+)
+
+call internal\check_deps.bat
+IF ERRORLEVEL 1 goto eof
+
+REM Check for optional components
+
+echo Disabling CUDA
+set NO_CUDA=1
+set USE_CUDA=0
+
+IF "%BUILD_VISION%" == "" (
+    call internal\check_opts.bat
+    IF ERRORLEVEL 1 goto eof
+
+    call internal\copy_cpu.bat
+    IF ERRORLEVEL 1 goto eof
+)
+
+call internal\setup.bat
+IF ERRORLEVEL 1 goto eof
+
+:eof

packaging/windows/cuda100.bat

@@ -0,0 +1,59 @@
+@echo off
+
+IF NOT "%BUILD_VISION%" == "" (
+    set MODULE_NAME=vision
+) ELSE (
+    set MODULE_NAME=pytorch
+)
+
+IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
+    call internal\clone.bat
+    cd ..
+    IF ERRORLEVEL 1 goto eof
+) ELSE (
+    call internal\clean.bat
+)
+
+call internal\check_deps.bat
+IF ERRORLEVEL 1 goto eof
+
+REM Check for optional components
+
+set NO_CUDA=
+set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+
+IF "%NVTOOLSEXT_PATH%"=="" (
+    echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
+    exit /b 1
+    goto optcheck
+)
+
+IF "%CUDA_PATH_V10_0%"=="" (
+    echo CUDA 10.0 not found, failing
+    exit /b 1
+) ELSE (
+    IF "%BUILD_VISION%" == "" (
+        set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0;7.5
+        set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
+    ) ELSE (
+        set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50
+    )
+
+    set "CUDA_PATH=%CUDA_PATH_V10_0%"
+    set "PATH=%CUDA_PATH_V10_0%\bin;%PATH%"
+)
+
+:optcheck
+
+IF "%BUILD_VISION%" == "" (
+    call internal\check_opts.bat
+    IF ERRORLEVEL 1 goto eof
+
+    call internal\copy.bat
+    IF ERRORLEVEL 1 goto eof
+)
+
+call internal\setup.bat
+IF ERRORLEVEL 1 goto eof
+
+:eof

packaging/windows/cuda90.bat

@@ -0,0 +1,59 @@
+@echo off
+
+IF NOT "%BUILD_VISION%" == "" (
+    set MODULE_NAME=vision
+) ELSE (
+    set MODULE_NAME=pytorch
+)
+
+IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
+    call internal\clone.bat
+    cd ..
+    IF ERRORLEVEL 1 goto eof
+) ELSE (
+    call internal\clean.bat
+)
+
+call internal\check_deps.bat
+IF ERRORLEVEL 1 goto eof
+
+REM Check for optional components
+
+set NO_CUDA=
+set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+
+IF "%NVTOOLSEXT_PATH%"=="" (
+    echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
+    exit /b 1
+    goto optcheck
+)
+
+IF "%CUDA_PATH_V9_0%"=="" (
+    echo CUDA 9 not found, failing
+    exit /b 1
+) ELSE (
+    IF "%BUILD_VISION%" == "" (
+        set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;7.0
+        set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
+    ) ELSE (
+        set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
+    )
+
+    set "CUDA_PATH=%CUDA_PATH_V9_0%"
+    set "PATH=%CUDA_PATH_V9_0%\bin;%PATH%"
+)
+
+:optcheck
+
+IF "%BUILD_VISION%" == "" (
+    call internal\check_opts.bat
+    IF ERRORLEVEL 1 goto eof
+
+    call internal\copy.bat
+    IF ERRORLEVEL 1 goto eof
+)
+
+call internal\setup.bat
+IF ERRORLEVEL 1 goto eof
+
+:eof

packaging/windows/internal/7z_install.bat

@@ -0,0 +1,9 @@
+@echo off
+
+curl -k https://www.7-zip.org/a/7z1805-x64.exe -O
+if errorlevel 1 exit /b 1
+
+start /wait 7z1805-x64.exe /S
+if errorlevel 1 exit /b 1
+
+set "PATH=%ProgramFiles%\7-Zip;%PATH%"

packaging/windows/internal/auth.bat

@@ -0,0 +1,46 @@
+@echo off
+
+: From the following doc, the build won't be triggered if the users don't sign in daily.
+: https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?tabs=yaml&view=vsts#my-build-didnt-run-what-happened
+: To avoid this problem, we can just go through the sign in process using the following command.
+
+:auth_start
+
+if "%RETRY_TIMES%" == "" (
+    set /a RETRY_TIMES=10
+    set /a SLEEP_TIME=2
+) else (
+    set /a RETRY_TIMES=%RETRY_TIMES%-1
+    set /a SLEEP_TIME=%SLEEP_TIME%*2
+)
+
+for /f "usebackq tokens=*" %%i in (`curl -so NUL -w "%%{http_code}" -u %VSTS_AUTH% https://dev.azure.com/pytorch`) do (
+    set STATUS_CODE=%%i
+)
+
+IF NOT "%STATUS_CODE%" == "200" (
+    echo Auth retry times remaining: %RETRY_TIMES%
+    echo Sleep time: %SLEEP_TIME% seconds
+    IF %RETRY_TIMES% EQU 0 (
+        echo Auth failed
+        goto err
+    )
+    waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul
+    goto auth_start
+) ELSE (
+    echo Login Attempt Succeeded
+    goto auth_end
+)
+
+:err
+
+: Throw a warning if it fails
+powershell -c "Write-Warning 'Login Attempt Failed'"
+
+:auth_end
+
+set RETRY_TIMES=
+set SLEEP_TIME=
+set STATUS_CODE=
+
+exit /b 0