From 93e9e70419b2cfe04a9eec42fd66a02db81d88d5 Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Wed, 12 Dec 2018 12:15:48 -0800 Subject: [PATCH 1/4] Revert "Revert "Feature/mkldnn static 2 (#13503)" (#13540)" This reverts commit a3eca5f5c96eed0bc29bd4e58e470997091a1fb3. --- CMakeLists.txt | 1 + Makefile | 9 +++- ci/docker/runtime_functions.sh | 3 -- ci/jenkins/Jenkins_steps.groovy | 8 ++-- mkldnn.mk | 12 ++++-- tests/cpp/unittest.mk | 8 ++-- tests/python/mkl/test_mkldnn.py | 6 +-- tests/python/mkl/test_mkldnn_install.py | 56 ------------------------- 8 files changed, 26 insertions(+), 77 deletions(-) delete mode 100644 tests/python/mkl/test_mkldnn_install.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b8bbd2e0272..161705643194 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -227,6 +227,7 @@ if(USE_MKLDNN) include(cmake/DownloadMKLML.cmake) # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3). if(NOT MSVC) + set(MKLDNN_LIBRARY_TYPE "STATIC" CACHE INTERNAL "" FORCE) set(ARCH_OPT_FLAGS "-mtune=generic") else() set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /EHsc") diff --git a/Makefile b/Makefile index 822704e26752..5c2b066a8971 100644 --- a/Makefile +++ b/Makefile @@ -131,8 +131,13 @@ ifeq ($(USE_MKLDNN), 1) CFLAGS += -I$(MKLROOT)/include LDFLAGS += -L$(MKLROOT)/lib endif - CFLAGS += -I$(MKLDNNROOT)/include - LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}' + # MKLDNN but to needs to be dynamically linked for windows as not all VS compilers support static linking + ifneq ($(UNAME_S), Windows) + LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a + else + CFLAGS += -I$(MKLDNNROOT)/include + LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}' + endif endif # setup opencv diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 6dd5bb6f239d..82e6feb2a728 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -657,9 +657,6 @@ build_ubuntu_gpu_cmake_mkldnn() { /work/mxnet ninja -v - # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0. - cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp - mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0 } build_ubuntu_gpu_cmake() { diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 74bde1eee211..d5cbd97683ed 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -23,19 +23,19 @@ utils = load('ci/Jenkinsfile_utils.groovy') // mxnet libraries -mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' +mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // Python wheels mx_pip = 'build/*.whl' // for scala build, need to pass extra libs when run with dist_kvstore -mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' +mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmkldnn.a' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' -mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' -mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' +mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' +mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so' mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*' mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*' diff --git a/mkldnn.mk b/mkldnn.mk index d79bbe7d2a0e..5af3e9b1d741 100644 --- a/mkldnn.mk +++ b/mkldnn.mk @@ -19,14 +19,20 @@ ifeq ($(USE_MKLDNN), 1) MKLDNN_SUBMODDIR = $(ROOTDIR)/3rdparty/mkldnn MKLDNN_BUILDDIR = $(MKLDNN_SUBMODDIR)/build MXNET_LIBDIR = $(ROOTDIR)/lib + MKLDNN_LIBRARY_TYPE=STATIC ifeq ($(UNAME_S), Darwin) OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.dylib MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml.dylib - MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.0.dylib + MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.a +else ifeq ($(UNAME_S), Windows) + OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so + MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so + MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.so + MKLDNN_LIBRARY_TYPE=SHARED else OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so - MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.so.0 + MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.a endif endif @@ -37,7 +43,7 @@ mkldnn_build: $(MKLDNN_LIBFILE) $(MKLDNN_LIBFILE): mkdir -p $(MKLDNNROOT) cd $(MKLDNN_SUBMODDIR) && rm -rf external && cd scripts && ./prepare_mkl.sh && cd .. && cp -a external/*/* $(MKLDNNROOT)/. - cmake $(MKLDNN_SUBMODDIR) -DCMAKE_INSTALL_PREFIX=$(MKLDNNROOT) -B$(MKLDNN_BUILDDIR) -DARCH_OPT_FLAGS="-mtune=generic" -DWITH_TEST=OFF -DWITH_EXAMPLE=OFF + cmake $(MKLDNN_SUBMODDIR) -DCMAKE_INSTALL_PREFIX=$(MKLDNNROOT) -B$(MKLDNN_BUILDDIR) -DARCH_OPT_FLAGS="-mtune=generic" -DWITH_TEST=OFF -DWITH_EXAMPLE=OFF -DMKLDNN_LIBRARY_TYPE=$(MKLDNN_LIBRARY_TYPE) $(MAKE) -C $(MKLDNN_BUILDDIR) VERBOSE=1 $(MAKE) -C $(MKLDNN_BUILDDIR) install mkdir -p $(MXNET_LIBDIR) diff --git a/tests/cpp/unittest.mk b/tests/cpp/unittest.mk index 746ee2f096f1..665ce6982874 100644 --- a/tests/cpp/unittest.mk +++ b/tests/cpp/unittest.mk @@ -41,22 +41,22 @@ gtest-all.o : $(GTEST_SRCS_) gtest.a : gtest-all.o $(AR) $(ARFLAGS) $@ $^ -build/tests/cpp/%.o : tests/cpp/%.cc | mkldnn +build/tests/cpp/%.o : tests/cpp/%.cc @mkdir -p $(@D) $(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/$* $< > build/tests/cpp/$*.d $(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/$*.o $(filter %.cc %.a, $^) -build/tests/cpp/operator/%.o : tests/cpp/operator/%.cc | mkldnn +build/tests/cpp/operator/%.o : tests/cpp/operator/%.cc @mkdir -p $(@D) $(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/operator/$* $< > build/tests/cpp/operator/$*.d $(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/operator/$*.o $(filter %.cc %.a, $^) -build/tests/cpp/storage/%.o : tests/cpp/storage/%.cc | mkldnn +build/tests/cpp/storage/%.o : tests/cpp/storage/%.cc @mkdir -p $(@D) $(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/storage/$* $< > build/tests/cpp/storage/$*.d $(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/storage/$*.o $(filter %.cc %.a, $^) -build/tests/cpp/engine/%.o : tests/cpp/engine/%.cc | mkldnn +build/tests/cpp/engine/%.o : tests/cpp/engine/%.cc @mkdir -p $(@D) $(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/engine/$* $< > build/tests/cpp/engine/$*.d $(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/engine/$*.o $(filter %.cc %.a, $^) diff --git a/tests/python/mkl/test_mkldnn.py b/tests/python/mkl/test_mkldnn.py index c6c0a0832f1f..d9d3abfc3ced 100644 --- a/tests/python/mkl/test_mkldnn.py +++ b/tests/python/mkl/test_mkldnn.py @@ -27,7 +27,6 @@ from mxnet import gluon from mxnet.gluon import nn from mxnet.test_utils import * -import test_mkldnn_install as install curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) sys.path.append(os.path.join(curr_path, '../unittest/')) from common import with_seed @@ -441,7 +440,4 @@ def backward(self, req, out_grad, in_data, out_data, in_grad, aux): custom = mx.symbol.Custom(name='custom', data=conv, op_type='custom') exec1 = custom.bind(mx.cpu(), args={'data': mx.nd.ones([10,3,96,96]), 'conv_weight': mx.nd.ones([8,3,5,5])}) exec1.forward()[0].wait_to_read() - - -if __name__ == '__main__': - install.test_mkldnn_install() + diff --git a/tests/python/mkl/test_mkldnn_install.py b/tests/python/mkl/test_mkldnn_install.py deleted file mode 100644 index c2f26df72f2e..000000000000 --- a/tests/python/mkl/test_mkldnn_install.py +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -MKL-DNN related test cases -""" - -import sys -import os -import logging - - -def test_mkldnn_install(): - """ - This test will verify that MXNet is built/installed correctly when - compiled with Intel MKL-DNN library. The method will try to import - the mxnet module and see if the mkldnn library is mapped to this - process's address space. - """ - logging.basicConfig(level=logging.INFO) - - if not sys.platform.startswith('linux'): - logging.info("Bypass mkldnn install test for non-Linux OS") - return - - try: - #pylint: disable=unused-variable - import mxnet as mx - except (ImportError, OSError) as e: - assert 0, "Import mxnet error: %s. Please double check your build/" \ - "install steps or environment variable settings" % str(e) - - pid = os.getpid() - rc = os.system("cat /proc/" + str(pid) + - "/maps | grep libmkldnn > /dev/null") - - if rc == 0: - logging.info("MXNet is built/installed correctly with MKL-DNN") - else: - assert 0, "MXNet is built/installed incorrectly with MKL-DNN, please " \ - "double check your build/install steps or environment " \ - "variable settings" From 0ff3219b5d723918614e7d347475500cb2fbac0e Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Wed, 12 Dec 2018 13:02:51 -0800 Subject: [PATCH 2/4] include headers on mkldnn lib --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5c2b066a8971..a349f35bd97e 100644 --- a/Makefile +++ b/Makefile @@ -131,11 +131,11 @@ ifeq ($(USE_MKLDNN), 1) CFLAGS += -I$(MKLROOT)/include LDFLAGS += -L$(MKLROOT)/lib endif + CFLAGS += -I$(MKLDNNROOT)/include # MKLDNN but to needs to be dynamically linked for windows as not all VS compilers support static linking ifneq ($(UNAME_S), Windows) LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a else - CFLAGS += -I$(MKLDNNROOT)/include LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}' endif endif From 78f7a69289e21c26a690e5f95cd8324663d93b61 Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Wed, 12 Dec 2018 17:45:48 -0800 Subject: [PATCH 3/4] retrigger From ac8d55b1df05580dae082ec1ea1dc70a9ca1420f Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Wed, 12 Dec 2018 18:31:55 -0800 Subject: [PATCH 4/4] retrigger