Skip to content

Commit

Permalink
Update to thrust 1.9.8 on Windows (apache#18218)
Browse files Browse the repository at this point in the history
* Update to thrust 1.9.8 on Windows

* Remove debug logic
  • Loading branch information
leezu authored and AntiZpvoh committed Jul 6, 2020
1 parent e04cde6 commit c1f890e
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 36 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ if(MSVC)
add_definitions(-DNNVM_EXPORTS)
add_definitions(-DNOMINMAX)
set(CMAKE_C_FLAGS "/MP")
set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} /bigobj")
set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS} /bigobj")
else()
include(CheckCXXCompilerFlag)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wno-sign-compare")
Expand Down
77 changes: 42 additions & 35 deletions ci/build_windows.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import tempfile
import time
import zipfile
import requests
from distutils.dir_util import copy_tree
from enum import Enum
from subprocess import check_call, call
Expand Down Expand Up @@ -127,7 +128,6 @@ class BuildFlavour(Enum):
'-DUSE_LAPACK=ON '
'-DUSE_DIST_KVSTORE=OFF '
'-DMXNET_CUDA_ARCH="5.2" '
'-DCMAKE_CXX_FLAGS="/FS /MD /O2 /Ob2" '
'-DUSE_MKL_IF_AVAILABLE=OFF '
'-DCMAKE_BUILD_TYPE=Release')

Expand All @@ -144,7 +144,6 @@ class BuildFlavour(Enum):
'-DUSE_DIST_KVSTORE=OFF '
'-DMXNET_CUDA_ARCH="5.2" '
'-DUSE_MKLDNN=ON '
'-DCMAKE_CXX_FLAGS="/FS /MD /O2 /Ob2" '
'-DCMAKE_BUILD_TYPE=Release')

}
Expand All @@ -155,44 +154,52 @@ def windows_build(args):

path = args.output

# cuda thrust + VS 2019 is flaky: try multiple times if fail
MAXIMUM_TRY = 5
build_try = 0

while build_try < MAXIMUM_TRY:
if os.path.exists(path):
shutil.rmtree(path)
os.makedirs(path, exist_ok=True)
if os.path.exists(path):
shutil.rmtree(path)
os.makedirs(path, exist_ok=True)

mxnet_root = get_mxnet_root()
logging.info("Found MXNet root: {}".format(mxnet_root))
mxnet_root = get_mxnet_root()
logging.info("Found MXNet root: {}".format(mxnet_root))

if 'GPU' in args.flavour:
# Get Thrust version to be shipped in Cuda 11, due to flakyness of
# older Thrust versions with MSVC 19 compiler
with remember_cwd():
os.chdir(path)
cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
CMAKE_FLAGS[args.flavour],
mxnet_root)
logging.info("Generating project with CMake:\n{}".format(cmd))
check_call(cmd, shell=True)

cmd = "\"{}\" && ninja".format(args.vcvars)
logging.info("Building:\n{}".format(cmd))

t0 = int(time.time())
ret = call(cmd, shell=True)

if ret != 0:
build_try += 1
logging.info("{} build(s) have failed".format(build_try))
else:
logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path)))
logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0))))
break
if ret == 0:
windows_package(args)
else:
tmpdirname = tempfile.mkdtemp()
os.chdir(tmpdirname)
r = requests.get('https://github.com/thrust/thrust/archive/1.9.8.zip', allow_redirects=True)
with open('thrust.zip', 'wb') as f:
f.write(r.content)
with zipfile.ZipFile('thrust.zip', 'r') as zip_ref:
zip_ref.extractall('.')
thrust_path = os.path.join(tmpdirname, "thrust-1.9.8")

with remember_cwd():
os.chdir(path)
env = os.environ.copy()
if 'GPU' in args.flavour:
env["CXXFLAGS"] = '/FS /MD /O2 /Ob2 /I {}'.format(thrust_path)
env["CUDAFLAGS"] = '-I {}'.format(thrust_path)
cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
CMAKE_FLAGS[args.flavour],
mxnet_root)
logging.info("Generating project with CMake:\n{}".format(cmd))
check_call(cmd, shell=True, env=env)

cmd = "\"{}\" && ninja".format(args.vcvars)
logging.info("Building:\n{}".format(cmd))

t0 = int(time.time())
ret = call(cmd, shell=True)

if ret != 0:
logging.info("Build failed")
sys.exit(1)

logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path)))
logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0))))
windows_package(args)


def windows_package(args):
pkgfile = 'windows_package.7z'
Expand Down

0 comments on commit c1f890e

Please sign in to comment.