From 3e5613767a3b957e05706c8e700a655606f21333 Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Fri, 21 Dec 2018 01:51:40 +0000 Subject: [PATCH 01/12] Jenkins build & test on Windows --- Jenkinsfile | 54 +++++++++++++++++++++++++++++ tests/scripts/task_example_test.bat | 28 +++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 tests/scripts/task_example_test.bat diff --git a/Jenkinsfile b/Jenkinsfile index ab0e24f8d421..a8c4e1b33e66 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,6 +23,23 @@ def build_dgl() { } } +def build_dgl_win64() { + /* Assuming that Windows slaves are already configured with MSBuild VS2017, + * CMake and Python/pip/setuptools etc. */ + bat "DEL /S /Q build" + bat "DEL /S /Q _download" + bat 'CALL "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat"' + dir ("build") { + bat 'cmake -DCMAKE_CXX_FLAGS="/DDGL_EXPORTS" -DCMAKE_BUILD_TYPE=Release .. -G "NMake Makefiles"' + bat "nmake" + } + dir ("python") { + bat "DEL /S /Q build *.egg-info dist" + bat "pip3 uninstall -y dgl" + bat "python setup.py install" + } +} + def pytorch_unit_test(dev) { withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) { sh "python3 -m nose -v --with-xunit tests" @@ -31,6 +48,14 @@ def pytorch_unit_test(dev) { } } +def pytorch_unit_test_win64(dev) { + withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python"]) { + bat "python -m nose -v --with-xunit tests" + bat "python -m nose -v --with-xunit tests\\pytorch" + bat "python -m nose -v --with-xunit tests\\graph_index" + } +} + def mxnet_unit_test(dev) { withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) { sh "python3 -m nose -v --with-xunit tests/mxnet" @@ -46,6 +71,14 @@ def example_test(dev) { } } +def example_test_win64(dev) { + withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python"]) { + dir ("tests\\scripts") { + sh "CALL task_example_test ${dev}" + } + } +} + def pytorch_tutorials() { withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) { dir ("tests/scripts") { @@ -99,6 +132,13 @@ pipeline { build_dgl() } } + stage("CPU Build (Win64/PyTorch)") { + agent { label "windows" } + steps { + setup() + build_dgl_win64() + } + } } } stage("Test") { @@ -117,6 +157,20 @@ pipeline { always { junit "*.xml" } } } + stage("Pytorch CPU (Windows)") { + agent { label "windows" } + stages { + stage("TH CPU Win64 unittest") { + steps { pytorch_unit_test_win64("CPU") } + } + stage("TH CPU Win64 example test") { + steps { example_test_win64("CPU") } + } + } + post { + always { junit "*.xml" } + } + } stage("Pytorch GPU") { agent { docker { diff --git a/tests/scripts/task_example_test.bat b/tests/scripts/task_example_test.bat new file mode 100644 index 000000000000..9a33e49a4ca1 --- /dev/null +++ b/tests/scripts/task_example_test.bat @@ -0,0 +1,28 @@ +@ECHO OFF +SETLOCAL EnableDelayedExpansion + +IF x%1x==xx ( + ECHO Must supply CPU or GPU + GOTO :FAIL +) ELSE IF x%1x==xCPUx ( + SET DEV=-1 +) ELSE IF x%1x==xGPUx ( + SET DEV=0 + SET CUDA_VISIBLE_DEVICES=0 +) ELSE ( + ECHO Must supply CPU or GPU + GOTO :FAIL +) + +PUSHD ..\..\examples\pytorch +python pagerank.py || GOTO :FAIL +python gcn\gcn.py --dataset cora --gpu !dev! || GOTO :FAIL +python gcn\gcn_spmv.py --dataset cora --gpu !dev! || GOTO :FAIL +POPD +ENDLOCAL +EXIT /B + +:FAIL +ECHO Example test failed +ENDLOCAL +EXIT /B 1 From 590bd9fcc7b4e591d259fd0041f97ec079d69295 Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Fri, 21 Dec 2018 02:10:33 +0000 Subject: [PATCH 02/12] oops --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a8c4e1b33e66..909ffb5c4cbc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -74,7 +74,7 @@ def example_test(dev) { def example_test_win64(dev) { withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python"]) { dir ("tests\\scripts") { - sh "CALL task_example_test ${dev}" + bat "CALL task_example_test ${dev}" } } } From 5ec80ec8786700ac371adaef391967e296d12560 Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Fri, 21 Dec 2018 02:14:37 +0000 Subject: [PATCH 03/12] still running nohup on Windows slaves --- Jenkinsfile | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 909ffb5c4cbc..68041eafef13 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -98,7 +98,10 @@ pipeline { agent none stages { stage("Lint Check") { - agent { docker { image "dgllib/dgl-ci-lint" } } + agent { + label "linux" + docker { image "dgllib/dgl-ci-lint" } + } steps { init_git_submodule() sh "bash tests/scripts/task_lint.sh" @@ -107,7 +110,10 @@ pipeline { stage("Build") { parallel { stage("CPU Build") { - agent { docker { image "dgllib/dgl-ci-cpu" } } + agent { + label "linux" + docker { image "dgllib/dgl-ci-cpu" } + } steps { setup() build_dgl() @@ -115,6 +121,7 @@ pipeline { } stage("GPU Build") { agent { + label "linux" docker { image "dgllib/dgl-ci-gpu" args "--runtime nvidia" @@ -126,14 +133,19 @@ pipeline { } } stage("MXNet CPU Build (temp)") { - agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } } + agent { + label "linux" + docker { image "dgllib/dgl-ci-mxnet-cpu" } + } steps { setup() build_dgl() } } stage("CPU Build (Win64/PyTorch)") { - agent { label "windows" } + agent { + label "windows" + } steps { setup() build_dgl_win64() @@ -144,7 +156,10 @@ pipeline { stage("Test") { parallel { stage("Pytorch CPU") { - agent { docker { image "dgllib/dgl-ci-cpu" } } + agent { + label "linux" + docker { image "dgllib/dgl-ci-cpu" } + } stages { stage("TH CPU unittest") { steps { pytorch_unit_test("CPU") } @@ -173,6 +188,7 @@ pipeline { } stage("Pytorch GPU") { agent { + label "linux" docker { image "dgllib/dgl-ci-gpu" args "--runtime nvidia" @@ -193,7 +209,10 @@ pipeline { //} } stage("MXNet CPU") { - agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } } + agent { + label "linux" + docker { image "dgllib/dgl-ci-mxnet-cpu" } + } stages { stage("MX Unittest") { steps { mxnet_unit_test("CPU") } @@ -208,13 +227,19 @@ pipeline { stage("Doc") { parallel { stage("TH Tutorial") { - agent { docker { image "dgllib/dgl-ci-cpu" } } + agent { + label "linux" + docker { image "dgllib/dgl-ci-cpu" } + } steps { pytorch_tutorials() } } stage("MX Tutorial") { - agent { docker { image "dgllib/dgl-ci-mxnet-cpu" } } + agent { + label "linux" + docker { image "dgllib/dgl-ci-mxnet-cpu" } + } steps { mxnet_tutorials() } From 34dcecaec1e806c64c9eb5b51c5bde95a94eeaa2 Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Fri, 21 Dec 2018 02:19:17 +0000 Subject: [PATCH 04/12] ooops again --- Jenkinsfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 68041eafef13..273ddd831ba6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,10 +5,19 @@ def init_git_submodule() { sh "git submodule update" } +def init_git_submodule_win64() { + bat "git submodule init" + bat "git submodule update" +} + def setup() { init_git_submodule() } +def setup_win64() { + init_git_submodule_win64() +} + def build_dgl() { sh "if [ -d build ]; then rm -rf build; fi; mkdir build" sh "rm -rf _download" @@ -99,7 +108,6 @@ pipeline { stages { stage("Lint Check") { agent { - label "linux" docker { image "dgllib/dgl-ci-lint" } } steps { @@ -111,7 +119,6 @@ pipeline { parallel { stage("CPU Build") { agent { - label "linux" docker { image "dgllib/dgl-ci-cpu" } } steps { @@ -121,7 +128,6 @@ pipeline { } stage("GPU Build") { agent { - label "linux" docker { image "dgllib/dgl-ci-gpu" args "--runtime nvidia" @@ -134,7 +140,6 @@ pipeline { } stage("MXNet CPU Build (temp)") { agent { - label "linux" docker { image "dgllib/dgl-ci-mxnet-cpu" } } steps { @@ -147,7 +152,7 @@ pipeline { label "windows" } steps { - setup() + setup_win64() build_dgl_win64() } } @@ -157,7 +162,6 @@ pipeline { parallel { stage("Pytorch CPU") { agent { - label "linux" docker { image "dgllib/dgl-ci-cpu" } } stages { @@ -188,7 +192,6 @@ pipeline { } stage("Pytorch GPU") { agent { - label "linux" docker { image "dgllib/dgl-ci-gpu" args "--runtime nvidia" @@ -210,7 +213,6 @@ pipeline { } stage("MXNet CPU") { agent { - label "linux" docker { image "dgllib/dgl-ci-mxnet-cpu" } } stages { @@ -228,7 +230,6 @@ pipeline { parallel { stage("TH Tutorial") { agent { - label "linux" docker { image "dgllib/dgl-ci-cpu" } } steps { @@ -237,7 +238,6 @@ pipeline { } stage("MX Tutorial") { agent { - label "linux" docker { image "dgllib/dgl-ci-mxnet-cpu" } } steps { From b666393e063e408114ac6740fcc7c0abac105cef Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Fri, 21 Dec 2018 02:31:10 +0000 Subject: [PATCH 05/12] squishing vcvars and cmake --- Jenkinsfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 273ddd831ba6..6a315a2abb36 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,10 +37,12 @@ def build_dgl_win64() { * CMake and Python/pip/setuptools etc. */ bat "DEL /S /Q build" bat "DEL /S /Q _download" - bat 'CALL "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat"' dir ("build") { - bat 'cmake -DCMAKE_CXX_FLAGS="/DDGL_EXPORTS" -DCMAKE_BUILD_TYPE=Release .. -G "NMake Makefiles"' - bat "nmake" + bat """ + CALL "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat"' + cmake -DCMAKE_CXX_FLAGS="/DDGL_EXPORTS" -DCMAKE_BUILD_TYPE=Release .. -G "NMake Makefiles" + nmake + """ } dir ("python") { bat "DEL /S /Q build *.egg-info dist" From dd0d8edcdb8db240511b613f92a07cabe5bdfb6c Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Fri, 21 Dec 2018 02:44:50 +0000 Subject: [PATCH 06/12] another try --- Jenkinsfile | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6a315a2abb36..1e88e6af95ee 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,7 +39,7 @@ def build_dgl_win64() { bat "DEL /S /Q _download" dir ("build") { bat """ - CALL "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat"' + CALL "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat" cmake -DCMAKE_CXX_FLAGS="/DDGL_EXPORTS" -DCMAKE_BUILD_TYPE=Release .. -G "NMake Makefiles" nmake """ @@ -149,6 +149,10 @@ pipeline { build_dgl() } } + } + } + stage("Build (Win64)") { + parallel { stage("CPU Build (Win64/PyTorch)") { agent { label "windows" @@ -178,20 +182,6 @@ pipeline { always { junit "*.xml" } } } - stage("Pytorch CPU (Windows)") { - agent { label "windows" } - stages { - stage("TH CPU Win64 unittest") { - steps { pytorch_unit_test_win64("CPU") } - } - stage("TH CPU Win64 example test") { - steps { example_test_win64("CPU") } - } - } - post { - always { junit "*.xml" } - } - } stage("Pytorch GPU") { agent { docker { @@ -228,6 +218,24 @@ pipeline { } } } + stage("Test (Win64)") { + parallel { + stage("Pytorch CPU (Windows)") { + agent { label "windows" } + stages { + stage("TH CPU Win64 unittest") { + steps { pytorch_unit_test_win64("CPU") } + } + stage("TH CPU Win64 example test") { + steps { example_test_win64("CPU") } + } + } + post { + always { junit "*.xml" } + } + } + } + } stage("Doc") { parallel { stage("TH Tutorial") { From 55b7ea6ad1165c710a7baa708733677141512090 Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Fri, 21 Dec 2018 05:39:19 +0000 Subject: [PATCH 07/12] reverting back --- Jenkinsfile | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1e88e6af95ee..426484875ece 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -149,10 +149,6 @@ pipeline { build_dgl() } } - } - } - stage("Build (Win64)") { - parallel { stage("CPU Build (Win64/PyTorch)") { agent { label "windows" @@ -182,6 +178,20 @@ pipeline { always { junit "*.xml" } } } + stage("Pytorch CPU (Windows)") { + agent { label "windows" } + stages { + stage("TH CPU Win64 unittest") { + steps { pytorch_unit_test_win64("CPU") } + } + stage("TH CPU Win64 example test") { + steps { example_test_win64("CPU") } + } + } + post { + always { junit "*.xml" } + } + } stage("Pytorch GPU") { agent { docker { @@ -218,24 +228,6 @@ pipeline { } } } - stage("Test (Win64)") { - parallel { - stage("Pytorch CPU (Windows)") { - agent { label "windows" } - stages { - stage("TH CPU Win64 unittest") { - steps { pytorch_unit_test_win64("CPU") } - } - stage("TH CPU Win64 example test") { - steps { example_test_win64("CPU") } - } - } - post { - always { junit "*.xml" } - } - } - } - } stage("Doc") { parallel { stage("TH Tutorial") { From 5bc4c9b45a898374b64bbd741328a6670e02eb11 Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Fri, 21 Dec 2018 05:47:49 +0000 Subject: [PATCH 08/12] --user --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 426484875ece..0126a4dbb1b6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,7 +47,7 @@ def build_dgl_win64() { dir ("python") { bat "DEL /S /Q build *.egg-info dist" bat "pip3 uninstall -y dgl" - bat "python setup.py install" + bat "python setup.py install --user" } } From d0d739e4eed7b446b2aa0b06d5b17ec3febcd016 Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Fri, 21 Dec 2018 06:05:02 +0000 Subject: [PATCH 09/12] switching to msbuild --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0126a4dbb1b6..da15bee5d29f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,13 +40,14 @@ def build_dgl_win64() { dir ("build") { bat """ CALL "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat" - cmake -DCMAKE_CXX_FLAGS="/DDGL_EXPORTS" -DCMAKE_BUILD_TYPE=Release .. -G "NMake Makefiles" - nmake + cmake -DCMAKE_CXX_FLAGS="/DDGL_EXPORTS" -DCMAKE_CONFIGURATION_TYPES="Release" .. -G "Visual Studio 15 2017 Win64" + msbuild dgl.sln + COPY Release\\dgl.dll . """ } dir ("python") { bat "DEL /S /Q build *.egg-info dist" - bat "pip3 uninstall -y dgl" + bat "pip3 uninstall -y dgl || (call )" // ignore error bat "python setup.py install --user" } } From 4ef7b1a03352358f6a09033d68c079d847d7eca9 Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Fri, 21 Dec 2018 06:20:12 +0000 Subject: [PATCH 10/12] made the graph size in cache testing bigger --- tests/pytorch/test_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytorch/test_graph.py b/tests/pytorch/test_graph.py index d334c258436d..302e6bec5b69 100644 --- a/tests/pytorch/test_graph.py +++ b/tests/pytorch/test_graph.py @@ -94,7 +94,7 @@ def test_incmat(): def test_incmat_cache(): n = 1000 - p = 2 * math.log(n) / n + p = 10 * math.log(n) / n a = sp.random(n, n, p, data_rvs=lambda n: np.ones(n)) g = dgl.DGLGraph(a) # the first call should contruct the inc From ddd1bafb7420257dab7a16073bc2421172ccd706 Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Sat, 22 Dec 2018 04:22:19 +0000 Subject: [PATCH 11/12] put commands into script files --- Jenkinsfile | 103 +++++++++---------------------- tests/scripts/build_dgl.bat | 21 +++++++ tests/scripts/build_dgl.sh | 19 ++++++ tests/scripts/task_unit_test.bat | 14 +++++ tests/scripts/task_unit_test.sh | 21 +++++++ 5 files changed, 105 insertions(+), 73 deletions(-) create mode 100644 tests/scripts/build_dgl.bat create mode 100644 tests/scripts/build_dgl.sh create mode 100644 tests/scripts/task_unit_test.bat create mode 100644 tests/scripts/task_unit_test.sh diff --git a/Jenkinsfile b/Jenkinsfile index da15bee5d29f..3474fda7a869 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,81 +10,38 @@ def init_git_submodule_win64() { bat "git submodule update" } -def setup() { - init_git_submodule() -} - -def setup_win64() { - init_git_submodule_win64() -} - def build_dgl() { - sh "if [ -d build ]; then rm -rf build; fi; mkdir build" - sh "rm -rf _download" - dir ("build") { - sh "cmake .." - sh "make -j4" - } - dir("python") { - sh "rm -rf build *.egg-info dist" - sh "pip3 uninstall -y dgl" - sh "python3 setup.py install" - } + sh "bash tests/scripts/build_dgl.sh" } def build_dgl_win64() { /* Assuming that Windows slaves are already configured with MSBuild VS2017, * CMake and Python/pip/setuptools etc. */ - bat "DEL /S /Q build" - bat "DEL /S /Q _download" - dir ("build") { - bat """ - CALL "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat" - cmake -DCMAKE_CXX_FLAGS="/DDGL_EXPORTS" -DCMAKE_CONFIGURATION_TYPES="Release" .. -G "Visual Studio 15 2017 Win64" - msbuild dgl.sln - COPY Release\\dgl.dll . - """ - } - dir ("python") { - bat "DEL /S /Q build *.egg-info dist" - bat "pip3 uninstall -y dgl || (call )" // ignore error - bat "python setup.py install --user" - } + bat "CALL tests\\scripts\\build_dgl.bat" } -def pytorch_unit_test(dev) { - withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) { - sh "python3 -m nose -v --with-xunit tests" - sh "python3 -m nose -v --with-xunit tests/pytorch" - sh "python3 -m nose -v --with-xunit tests/graph_index" +def unit_test(backend, dev) { + withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python", "DGLBACKEND=${backend}"]) { + sh "bash tests/scripts/task_unit_test.sh ${backend}" } } -def pytorch_unit_test_win64(dev) { - withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python"]) { - bat "python -m nose -v --with-xunit tests" - bat "python -m nose -v --with-xunit tests\\pytorch" - bat "python -m nose -v --with-xunit tests\\graph_index" +def unit_test_win64(backend, dev) { + withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python", "DGLBACKEND=${backend}"]) { + bat "CALL tests\\scripts\\task_unit_test.bat ${backend}" } } -def mxnet_unit_test(dev) { - withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) { - sh "python3 -m nose -v --with-xunit tests/mxnet" - sh "python3 -m nose -v --with-xunit tests/graph_index" - } -} - -def example_test(dev) { - withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python"]) { +def example_test(backend, dev) { + withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}/build", "PYTHONPATH=${env.WORKSPACE}/python", "DGLBACKEND=${backend}"]) { dir ("tests/scripts") { sh "bash task_example_test.sh ${dev}" } } } -def example_test_win64(dev) { - withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python"]) { +def example_test_win64(backend, dev) { + withEnv(["DGL_LIBRARY_PATH=${env.WORKSPACE}\\build", "PYTHONPATH=${env.WORKSPACE}\\python", "DGLBACKEND=${backend}"]) { dir ("tests\\scripts") { bat "CALL task_example_test ${dev}" } @@ -125,7 +82,7 @@ pipeline { docker { image "dgllib/dgl-ci-cpu" } } steps { - setup() + init_git_submodule() build_dgl() } } @@ -137,7 +94,7 @@ pipeline { } } steps { - setup() + init_git_submodule() build_dgl() } } @@ -146,19 +103,19 @@ pipeline { docker { image "dgllib/dgl-ci-mxnet-cpu" } } steps { - setup() + init_git_submodule() build_dgl() } } - stage("CPU Build (Win64/PyTorch)") { - agent { + stage("CPU Build (Win64/PyTorch)") { + agent { label "windows" } - steps { - setup_win64() - build_dgl_win64() - } - } + steps { + init_git_submodule_win64() + build_dgl_win64() + } + } } } stage("Test") { @@ -169,10 +126,10 @@ pipeline { } stages { stage("TH CPU unittest") { - steps { pytorch_unit_test("CPU") } + steps { unit_test("pytorch", "CPU") } } stage("TH CPU example test") { - steps { example_test("CPU") } + steps { example_test("pytorch", "CPU") } } } post { @@ -183,11 +140,11 @@ pipeline { agent { label "windows" } stages { stage("TH CPU Win64 unittest") { - steps { pytorch_unit_test_win64("CPU") } + steps { unit_test_win64("pytorch", "CPU") } + } + stage("TH CPU Win64 example test") { + steps { example_test_win64("pytorch", "CPU") } } - stage("TH CPU Win64 example test") { - steps { example_test_win64("CPU") } - } } post { always { junit "*.xml" } @@ -206,7 +163,7 @@ pipeline { // steps { pytorch_unit_test("GPU") } //} stage("TH GPU example test") { - steps { example_test("GPU") } + steps { example_test("pytorch", "GPU") } } } // TODO: have GPU unittest @@ -220,7 +177,7 @@ pipeline { } stages { stage("MX Unittest") { - steps { mxnet_unit_test("CPU") } + steps { unit_test("mxnet", "CPU") } } } post { diff --git a/tests/scripts/build_dgl.bat b/tests/scripts/build_dgl.bat new file mode 100644 index 000000000000..7ef8dd4db3d3 --- /dev/null +++ b/tests/scripts/build_dgl.bat @@ -0,0 +1,21 @@ +@ECHO OFF +SETLOCAL EnableDelayedExpansion + +DEL /S /Q build +DEL /S /Q _download +MD build + +PUSHD build +CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat" +cmake -DCMAKE_CXX_FLAGS="/DDGL_EXPORTS" -DCMAKE_CONFIGURATION_TYPES="Release" .. -G "Visual Studio 15 2017 Win64" || EXIT /B 1 +msbuild dgl.sln || EXIT /B 1 +COPY Release\dgl.dll . +POPD + +PUSHD python +DEL /S /Q build *.egg-info dist +pip install -e . --force-reinstall || EXIT /B 1 +POPD + +ENDLOCAL +EXIT /B diff --git a/tests/scripts/build_dgl.sh b/tests/scripts/build_dgl.sh new file mode 100644 index 000000000000..44282ad42ba9 --- /dev/null +++ b/tests/scripts/build_dgl.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ -d build ]; then + rm -rf build +fi +mkdir build + +rm -rf _download + +pushd build +cmake .. +make -j4 +popd + +pushd python +rm -rf build *.egg-info dist +pip3 uninstall -y dgl +python3 setup.py install +popd diff --git a/tests/scripts/task_unit_test.bat b/tests/scripts/task_unit_test.bat new file mode 100644 index 000000000000..0a10b7f6abd2 --- /dev/null +++ b/tests/scripts/task_unit_test.bat @@ -0,0 +1,14 @@ +@ECHO OFF +SETLOCAL EnableDelayedExpansion + +IF x%1x==xx ( + ECHO Specify backend + EXIT /B 1 +) ELSE ( + SET BACKEND=%1 +) + +python -m nose -v --with-xunit tests || EXIT /B 1 +python -m nose -v --with-xunit tests\!BACKEND! || EXIT /B 1 +python -m nose -v --with-xunit tests\graph_index || EXIT /B 1 +EXIT /B diff --git a/tests/scripts/task_unit_test.sh b/tests/scripts/task_unit_test.sh new file mode 100644 index 000000000000..54aab1645986 --- /dev/null +++ b/tests/scripts/task_unit_test.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +function fail { + echo FAIL: $@ + exit -1 +} + +function usage { + echo "Usage: $0 backend" +} + +if [ $# -ne 1 ]; then + usage + fail "Error: must specify backend" +fi + +BACKEND=$1 + +python3 -m nose -v --with-xunit tests || fail "tests" +python3 -m nose -v --with-xunit tests/$BACKEND || fail "backend" +python3 -m nose -v --with-xunit tests/graph_index || fail "graph_index" From a8a7d4bc84d5a9c7fa78c10737275f1a8a3698f8 Mon Sep 17 00:00:00 2001 From: Quan Gan Date: Sat, 22 Dec 2018 04:24:25 +0000 Subject: [PATCH 12/12] oooops --- tests/scripts/build_dgl.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/build_dgl.bat b/tests/scripts/build_dgl.bat index 7ef8dd4db3d3..42c1d2b11688 100644 --- a/tests/scripts/build_dgl.bat +++ b/tests/scripts/build_dgl.bat @@ -14,7 +14,7 @@ POPD PUSHD python DEL /S /Q build *.egg-info dist -pip install -e . --force-reinstall || EXIT /B 1 +pip install -e . --force-reinstall --user || EXIT /B 1 POPD ENDLOCAL