From 0b0d091805675bee14c9323d2ca69ccc955b4815 Mon Sep 17 00:00:00 2001 From: ptrblck Date: Wed, 5 Apr 2023 13:02:22 -0700 Subject: [PATCH] Windows CUDA 12.1 changes (#1376) * Windows CUDA 12.1 changes * add CUDA version checks for Windows MAGMA builds * use magma branch without fermi arch --- .github/workflows/build-magma-windows.yml | 2 +- conda/pytorch-nightly/bld.bat | 7 ++- windows/cuda121.bat | 58 +++++++++++++++++++++++ windows/internal/build_magma.bat | 13 +++-- windows/internal/cuda_install.bat | 32 ++++++++++++- windows/internal/smoke_test.bat | 3 ++ 6 files changed, 108 insertions(+), 7 deletions(-) create mode 100644 windows/cuda121.bat diff --git a/.github/workflows/build-magma-windows.yml b/.github/workflows/build-magma-windows.yml index 5ad6ba29a..d6e6a6762 100644 --- a/.github/workflows/build-magma-windows.yml +++ b/.github/workflows/build-magma-windows.yml @@ -17,7 +17,7 @@ jobs: runs-on: windows-2019 strategy: matrix: - cuda_version: ["118", "117", "116"] + cuda_version: ["121", "118", "117"] config: ["Release", "Debug"] env: CUDA_VERSION: ${{ matrix.cuda_version }} diff --git a/conda/pytorch-nightly/bld.bat b/conda/pytorch-nightly/bld.bat index 1b0909f13..80da30807 100644 --- a/conda/pytorch-nightly/bld.bat +++ b/conda/pytorch-nightly/bld.bat @@ -20,11 +20,16 @@ if "%build_with_cuda%" == "" goto cuda_flags_end set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%desired_cuda% set CUDA_BIN_PATH=%CUDA_PATH%\bin set TORCH_NVCC_FLAGS=-Xfatbin -compress-all -set TORCH_CUDA_ARCH_LIST=3.7+PTX;5.0;6.0;6.1;7.0;7.5;8.0;8.6 +set TORCH_CUDA_ARCH_LIST=5.0;6.0;6.1;7.0;7.5;8.0;8.6 if "%desired_cuda%" == "11.7" ( + set TORCH_CUDA_ARCH_LIST=%TORCH_CUDA_ARCH_LIST%;3.7+PTX set TORCH_NVCC_FLAGS=-Xfatbin -compress-all --threads 2 ) if "%desired_cuda%" == "11.8" ( + set TORCH_CUDA_ARCH_LIST=%TORCH_CUDA_ARCH_LIST%;3.7+PTX;9.0 + set TORCH_NVCC_FLAGS=-Xfatbin -compress-all --threads 2 +) +if "%desired_cuda%" == "12.1" ( set TORCH_CUDA_ARCH_LIST=%TORCH_CUDA_ARCH_LIST%;9.0 set TORCH_NVCC_FLAGS=-Xfatbin -compress-all --threads 2 ) diff --git a/windows/cuda121.bat b/windows/cuda121.bat new file mode 100644 index 000000000..e8be06baa --- /dev/null +++ b/windows/cuda121.bat @@ -0,0 +1,58 @@ +@echo off + +set MODULE_NAME=pytorch + +IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( + call internal\clone.bat + cd .. +) ELSE ( + call internal\clean.bat +) +IF ERRORLEVEL 1 goto :eof + +call internal\check_deps.bat +IF ERRORLEVEL 1 goto :eof + +REM Check for optional components + +set USE_CUDA= +set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 + +IF "%NVTOOLSEXT_PATH%"=="" ( + IF EXIST "C:\Program Files\NVIDIA Corporation\NvToolsExt\lib\x64\nvToolsExt64_1.lib" ( + set NVTOOLSEXT_PATH=C:\Program Files\NVIDIA Corporation\NvToolsExt + ) ELSE ( + echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing + exit /b 1 + ) +) + +IF "%CUDA_PATH_V121%"=="" ( + IF EXIST "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\bin\nvcc.exe" ( + set "CUDA_PATH_V121=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1" + ) ELSE ( + echo CUDA 12.1 not found, failing + exit /b 1 + ) +) + +IF "%BUILD_VISION%" == "" ( + set TORCH_CUDA_ARCH_LIST=5.0;6.0;6.1;7.0;7.5;8.0;8.6;9.0 + set TORCH_NVCC_FLAGS=-Xfatbin -compress-all +) ELSE ( + set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_80,code=compute_80 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_90,code=compute_90 +) + +set "CUDA_PATH=%CUDA_PATH_V121%" +set "PATH=%CUDA_PATH_V121%\bin;%PATH%" + +:optcheck + +call internal\check_opts.bat +IF ERRORLEVEL 1 goto :eof + +call internal\copy.bat +IF ERRORLEVEL 1 goto :eof + +call internal\setup.bat +IF ERRORLEVEL 1 goto :eof diff --git a/windows/internal/build_magma.bat b/windows/internal/build_magma.bat index b8a2f3377..b2fcb976c 100644 --- a/windows/internal/build_magma.bat +++ b/windows/internal/build_magma.bat @@ -24,7 +24,7 @@ cd magma_cuda%CUVER_NODOT% if not exist magma ( :: MAGMA 2.5.4 from http://icl.utk.edu/projectsfiles/magma/downloads/ with applied patches from our magma folder - git clone https://github.com/peterjc123/magma.git magma + git clone https://github.com/ptrblck/magma_win.git magma if errorlevel 1 exit /b 1 ) else ( rmdir /S /Q magma\build @@ -35,8 +35,15 @@ cd magma mkdir build && cd build set GPU_TARGET=All -set CUDA_ARCH_LIST= -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -set "CUDA_ARCH_LIST=%CUDA_ARCH_LIST% -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86" +if "%CUVER_NODOT%" == "121" ( + set CUDA_ARCH_LIST=-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90 +) +if "%CUVER_NODOT%" == "118" ( + set CUDA_ARCH_LIST= -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90 +) +if "%CUVER_NODOT%" == "117" ( + set CUDA_ARCH_LIST= -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 +) set CC=cl.exe set CXX=cl.exe diff --git a/windows/internal/cuda_install.bat b/windows/internal/cuda_install.bat index b38821f60..cdcae5497 100644 --- a/windows/internal/cuda_install.bat +++ b/windows/internal/cuda_install.bat @@ -21,6 +21,7 @@ if exist "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR% if %CUDA_VER% EQU 117 goto cuda117 if %CUDA_VER% EQU 118 goto cuda118 +if %CUDA_VER% EQU 121 goto cuda121 echo CUDA %CUDA_VERSION_STR% is not supported exit /b 1 @@ -44,7 +45,7 @@ if not exist "%SRC_DIR%\temp_build\%CUDNN_INSTALL_ZIP%" ( set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\%CUDNN_INSTALL_ZIP%" ) -@REM Cuda 8.3+ required zlib to be installed on the path +@REM cuDNN 8.3+ required zlib to be installed on the path echo Installing ZLIB dlls curl -k -L "http://s3.amazonaws.com/ossci-windows/zlib123dllx64.zip" --output "%SRC_DIR%\temp_build\zlib123dllx64.zip" 7z x "%SRC_DIR%\temp_build\zlib123dllx64.zip" -o"%SRC_DIR%\temp_build\zlib" @@ -71,7 +72,34 @@ if not exist "%SRC_DIR%\temp_build\%CUDNN_INSTALL_ZIP%" ( set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\%CUDNN_INSTALL_ZIP%" ) -@REM Cuda 8.3+ required zlib to be installed on the path +@REM cuDNN 8.3+ required zlib to be installed on the path +echo Installing ZLIB dlls +curl -k -L "http://s3.amazonaws.com/ossci-windows/zlib123dllx64.zip" --output "%SRC_DIR%\temp_build\zlib123dllx64.zip" +7z x "%SRC_DIR%\temp_build\zlib123dllx64.zip" -o"%SRC_DIR%\temp_build\zlib" +xcopy /Y "%SRC_DIR%\temp_build\zlib\dll_x64\*.dll" "C:\Windows\System32" + +goto cuda_common + +:cuda121 + +set CUDA_INSTALL_EXE=cuda_12.1.0_531.14_windows.exe +if not exist "%SRC_DIR%\temp_build\%CUDA_INSTALL_EXE%" ( + curl -k -L "https://ossci-windows.s3.amazonaws.com/%CUDA_INSTALL_EXE%" --output "%SRC_DIR%\temp_build\%CUDA_INSTALL_EXE%" + if errorlevel 1 exit /b 1 + set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\%CUDA_INSTALL_EXE%" + set "ARGS=cuda_profiler_api_12.1 thrust_12.1 nvcc_12.1 cuobjdump_12.1 nvprune_12.1 nvprof_12.1 cupti_12.1 cublas_12.1 cublas_dev_12.1 cudart_12.1 cufft_12.1 cufft_dev_12.1 curand_12.1 curand_dev_12.1 cusolver_12.1 cusolver_dev_12.1 cusparse_12.1 cusparse_dev_12.1 npp_12.1 npp_dev_12.1 nvrtc_12.1 nvrtc_dev_12.1 nvml_dev_12.1" +) + +set CUDNN_FOLDER=cudnn-windows-x86_64-8.8.1.3_cuda12-archive +set CUDNN_LIB_FOLDER="lib" +set "CUDNN_INSTALL_ZIP=%CUDNN_FOLDER%.zip" +if not exist "%SRC_DIR%\temp_build\%CUDNN_INSTALL_ZIP%" ( + curl -k -L "http://s3.amazonaws.com/ossci-windows/%CUDNN_INSTALL_ZIP%" --output "%SRC_DIR%\temp_build\%CUDNN_INSTALL_ZIP%" + if errorlevel 1 exit /b 1 + set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\%CUDNN_INSTALL_ZIP%" +) + +@REM cuDNN 8.3+ required zlib to be installed on the path echo Installing ZLIB dlls curl -k -L "http://s3.amazonaws.com/ossci-windows/zlib123dllx64.zip" --output "%SRC_DIR%\temp_build\zlib123dllx64.zip" 7z x "%SRC_DIR%\temp_build\zlib123dllx64.zip" -o"%SRC_DIR%\temp_build\zlib" diff --git a/windows/internal/smoke_test.bat b/windows/internal/smoke_test.bat index afe285716..1be7d1744 100644 --- a/windows/internal/smoke_test.bat +++ b/windows/internal/smoke_test.bat @@ -71,6 +71,9 @@ if "%CUDA_VERSION%" == "117" ( if "%CUDA_VERSION%" == "118" ( set "CONDA_EXTRA_ARGS=pytorch-cuda=11.8 -c nvidia -c pytorch-nightly" ) +if "%CUDA_VERSION%" == "121" ( + set "CONDA_EXTRA_ARGS=pytorch-cuda=12.1 -c nvidia -c pytorch-nightly" +) rmdir /s /q conda del miniconda.exe