diff --git a/ci/dev_menu.py b/ci/dev_menu.py index c471302c76d7..cd2aa8d46e1e 100644 --- a/ci/dev_menu.py +++ b/ci/dev_menu.py @@ -122,7 +122,7 @@ def provision_virtualenv(venv_path=DEFAULT_PYENV): "pytest -v tests/python/unittest/" ), ('[Docker] Build the MXNet binary - outputs to "lib/"', - "ci/build.py --platform ubuntu_cpu_lite /work/runtime_functions.sh build_ubuntu_cpu_docs"), + "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_docs"), ('[Docker] Build the Jekyll website - outputs to "docs/static_site/build/html/"', "ci/build.py --platform ubuntu_cpu_jekyll /work/runtime_functions.sh build_jekyll_docs"), ('[Docker] Build the Python API docs - outputs to "docs/python_docs/python/build/_build/html/"', diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_lite b/ci/docker/Dockerfile.build.ubuntu_cpu_lite deleted file mode 100644 index ca5618ac1cd7..000000000000 --- a/ci/docker/Dockerfile.build.ubuntu_cpu_lite +++ /dev/null @@ -1,45 +0,0 @@ -# -*- mode: dockerfile -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU - -FROM ubuntu:16.04 - -WORKDIR /work/deps - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh - -COPY install/deb_ubuntu_ccache.sh /work/ -RUN /work/deb_ubuntu_ccache.sh - -COPY install/ubuntu_clang.sh /work/ -RUN /work/ubuntu_clang.sh - -COPY install/ubuntu_gcc8.sh /work/ -RUN /work/ubuntu_gcc8.sh - -# Always last -ARG USER_ID=0 -ARG GROUP_ID=0 -COPY install/ubuntu_adduser.sh /work/ -RUN /work/ubuntu_adduser.sh - -COPY runtime_functions.sh /work/ - -WORKDIR /work/mxnet \ No newline at end of file diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 666c59e6e4aa..2e3a5b59b412 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -321,21 +321,6 @@ build_centos7_cpu() { ninja } -build_centos7_cpu_make() { - set -ex - cd /work/mxnet - source /opt/rh/devtoolset-7/enable - make \ - DEV=1 \ - USE_LAPACK=1 \ - USE_LAPACK_PATH=/usr/lib64/liblapack.so \ - USE_BLAS=openblas \ - USE_MKLDNN=0 \ - USE_DIST_KVSTORE=1 \ - USE_SIGNAL_HANDLER=1 \ - -j$(nproc) -} - build_centos7_mkldnn() { set -ex cd /work/build @@ -384,24 +369,6 @@ build_ubuntu_cpu_openblas() { ninja } -build_ubuntu_cpu_openblas_make() { - set -ex - export CC=gcc-7 - export CXX=g++-7 - build_ccache_wrappers - make \ - DEV=1 \ - USE_TVM_OP=1 \ - USE_CPP_PACKAGE=1 \ - USE_BLAS=openblas \ - USE_MKLDNN=0 \ - USE_DIST_KVSTORE=1 \ - USE_LIBJPEG_TURBO=1 \ - USE_SIGNAL_HANDLER=1 \ - -j$(nproc) - make cython PYTHON=python3 -} - build_ubuntu_cpu_mkl() { set -ex cd /work/build @@ -589,22 +556,6 @@ build_ubuntu_cpu_clang100_mkldnn() { ninja } -build_ubuntu_cpu_mkldnn_make() { - set -ex - - export CC=gcc-7 - export CXX=g++-7 - build_ccache_wrappers - - make \ - DEV=1 \ - USE_CPP_PACKAGE=1 \ - USE_TVM_OP=1 \ - USE_BLAS=openblas \ - USE_SIGNAL_HANDLER=1 \ - -j$(nproc) -} - build_ubuntu_cpu_mkldnn() { set -ex cd /work/build @@ -754,45 +705,6 @@ build_ubuntu_gpu_cuda101_cudnn7_debug() { ninja } -build_ubuntu_gpu_cuda101_cudnn7_make() { - set -ex - export CC=gcc-7 - export CXX=g++-7 - build_ccache_wrappers - make \ - USE_BLAS=openblas \ - USE_MKLDNN=0 \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_CPP_PACKAGE=1 \ - USE_DIST_KVSTORE=1 \ - CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ - USE_SIGNAL_HANDLER=1 \ - -j$(nproc) - make cython PYTHON=python3 -} - -build_ubuntu_gpu_cuda101_cudnn7_mkldnn_cpp_test() { - set -ex - export CC=gcc-7 - export CXX=g++-7 - build_ccache_wrappers - make \ - USE_BLAS=openblas \ - USE_MKLDNN=1 \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_CPP_PACKAGE=1 \ - USE_DIST_KVSTORE=1 \ - CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ - USE_SIGNAL_HANDLER=1 \ - -j$(nproc) - make test USE_CPP_PACKAGE=1 -j$(nproc) - make cython PYTHON=python3 -} - build_ubuntu_gpu_cmake() { set -ex cd /work/build @@ -878,9 +790,9 @@ build_ubuntu_blc() { sanity_check() { set -ex tools/license_header.py check - make cpplint - make pylint - pytest -n 4 tests/tutorials/test_sanity_tutorials.py + 3rdparty/dmlc-core/scripts/lint.py mxnet cpp include src plugin tests --exclude_path src/operator/contrib/ctc_include include/mkldnn + python3 -m pylint --rcfile=ci/other/pylintrc --ignore-patterns=".*\.so$$,.*\.dll$$,.*\.dylib$$" python/mxnet + OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/tutorials/test_sanity_tutorials.py } # Tests libmxnet @@ -1267,19 +1179,7 @@ build_docs_setup() { } build_ubuntu_cpu_docs() { - set -ex - export CC="gcc-7" - export CXX="g++-7" - build_ccache_wrappers - make \ - DEV=1 \ - USE_CPP_PACKAGE=1 \ - USE_BLAS=openblas \ - USE_MKLDNN=0 \ - USE_DIST_KVSTORE=1 \ - USE_LIBJPEG_TURBO=1 \ - USE_SIGNAL_HANDLER=1 \ - -j$(nproc) + build_ubuntu_cpu_openblas } @@ -1409,8 +1309,6 @@ build_static_libmxnet() { pushd . source /opt/rh/devtoolset-7/enable source /opt/rh/rh-python36/enable - export USE_SYSTEM_CUDA=1 - export CMAKE_STATICBUILD=1 local mxnet_variant=${1:?"This function requires a python command as the first argument"} source tools/staticbuild/build.sh ${mxnet_variant} popd @@ -1484,7 +1382,6 @@ build_static_python_cu92() { set -ex pushd . export mxnet_variant=cu92 - export USE_SYSTEM_CUDA=1 source /opt/rh/devtoolset-7/enable source /opt/rh/rh-python36/enable ./ci/publish/python/build.sh diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 923d41b4c383..f4cb84efe715 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -25,7 +25,6 @@ utils = load('ci/Jenkinsfile_utils.groovy') // mxnet libraries mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so' mx_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_ffi/_cy3/*.so' -mx_lib_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' @@ -34,11 +33,8 @@ mx_cmake_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests' mx_mkldnn_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so' -mx_mkldnn_lib_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' mx_tensorrt_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so' mx_lib_cpp_examples = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' -mx_lib_cpp_examples_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' -mx_lib_cpp_capi_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, libsample_lib.so, lib/libmkldnn.so.1, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so, build/tests/cpp/mxnet_unit_tests' mx_lib_cpp_examples_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so' mx_cd_lib = 'lib/libmxnet.so, licenses/*, lib/libgfortran.so.4, lib/libquadmath.so.0, lib/libopenblas.so.0, include/mkldnn/dnnl_version.h, include/mkldnn/dnnl_config.h' @@ -101,20 +97,6 @@ def compile_unix_cpu_openblas(lib_name) { }] } -def compile_unix_cpu_openblas_make(lib_name) { - return ['CPU: Openblas Makefile': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cpu-openblas') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas_make', false) - utils.pack_lib(lib_name, mx_lib_make) - } - } - } - }] -} - def compile_unix_openblas_debug_cpu(lib_name) { return ['CPU: Openblas, cmake, debug': { node(NODE_LINUX_CPU) { @@ -198,20 +180,6 @@ def compile_unix_mkldnn_cpu(lib_name) { }] } -def compile_unix_mkldnn_cpu_make(lib_name) { - return ['CPU: MKLDNN Makefile': { - node(NODE_LINUX_CPU) { - ws('workspace/build-mkldnn-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn_make', false) - utils.pack_lib(lib_name, mx_mkldnn_lib_make) - } - } - } - }] -} - def compile_unix_mkldnn_mkl_cpu(lib_name) { return ['CPU: MKLDNN_MKL': { node(NODE_LINUX_CPU) { @@ -268,21 +236,6 @@ def compile_unix_full_gpu(lib_name) { }] } -def compile_unix_full_gpu_make(lib_name) { - return ['GPU: CUDA10.1+cuDNN7 Makefile': { - node(NODE_LINUX_CPU) { - ws('workspace/build-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda101_cudnn7_make', false) - utils.pack_lib(lib_name, mx_lib_cpp_examples_make) - } - } - } - }] -} - - def compile_unix_full_gpu_debug(lib_name) { return ['GPU: CUDA10.1+cuDNN7, debug': { node(NODE_LINUX_CPU) { @@ -297,20 +250,6 @@ def compile_unix_full_gpu_debug(lib_name) { }] } -def compile_unix_full_gpu_mkldnn_cpp_test(lib_name) { - return ['GPU: CUDA10.1+cuDNN7+MKLDNN+CPPTEST Makefile': { - node(NODE_LINUX_CPU) { - ws('workspace/build-gpu-mkldnn-cpp') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda101_cudnn7_mkldnn_cpp_test', false) - utils.pack_lib(lib_name, mx_lib_cpp_capi_make) - } - } - } - }] -} - def compile_unix_cmake_gpu(lib_name) { return ['GPU: CMake': { node(NODE_LINUX_CPU) { @@ -367,20 +306,6 @@ def compile_centos7_cpu(lib_name) { }] } -def compile_centos7_cpu_make(lib_name) { - return ['CPU: CentOS 7 Makefile': { - node(NODE_LINUX_CPU) { - ws('workspace/build-centos7-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('centos7_cpu', 'build_centos7_cpu_make', false) - utils.pack_lib(lib_name, mx_lib_make) - } - } - } - }] -} - def compile_centos7_cpu_mkldnn() { return ['CPU: CentOS 7 MKLDNN': { node(NODE_LINUX_CPU) { @@ -903,7 +828,7 @@ def test_unix_onnx_cpu(lib_name) { node(NODE_LINUX_CPU) { ws('workspace/it-onnx-cpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init(lib_name, mx_lib_make) + utils.unpack_and_init(lib_name, mx_lib) utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false) utils.publish_test_coverage() } @@ -1132,8 +1057,8 @@ def compile_unix_lite(lib_name) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.docker_run('ubuntu_cpu_lite', 'build_ubuntu_cpu_docs', false) - utils.pack_lib(lib_name, 'lib/libmxnet.so', false) + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_docs', false) + utils.pack_lib(lib_name, mx_lib, false) } } } @@ -1161,7 +1086,7 @@ def docs_python(lib_name) { node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init(lib_name, 'lib/libmxnet.so', false) + utils.unpack_and_init(lib_name, mx_lib, false) utils.docker_run('ubuntu_cpu_python', 'build_python_docs', false) if (should_pack_website()) { utils.pack_lib('python-artifacts', 'docs/_build/python-artifacts.tgz', false) diff --git a/ci/jenkins/Jenkinsfile_centos_cpu b/ci/jenkins/Jenkinsfile_centos_cpu index f652478a6f32..cddb3c977a5d 100644 --- a/ci/jenkins/Jenkinsfile_centos_cpu +++ b/ci/jenkins/Jenkinsfile_centos_cpu @@ -35,7 +35,6 @@ utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ custom_steps.compile_centos7_cpu('centos7_cpu'), - custom_steps.compile_centos7_cpu_make('centos7_cpu_make'), custom_steps.compile_centos7_cpu_mkldnn(), custom_steps.compile_static_python_cpu(), custom_steps.compile_static_cd_cpu('centos7_cpu_cd') diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 0909fc5139fe..67d5afc4cc76 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -35,11 +35,9 @@ utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ custom_steps.compile_unix_cpu_openblas('cpu'), - custom_steps.compile_unix_cpu_openblas_make('cpu_make'), custom_steps.compile_unix_openblas_debug_cpu('cpu_debug'), custom_steps.compile_unix_mkl_cpu('cpu_mkl'), custom_steps.compile_unix_mkldnn_cpu('mkldnn_cpu'), - custom_steps.compile_unix_mkldnn_cpu_make('mkldnn_cpu_make'), custom_steps.compile_unix_mkldnn_mkl_cpu('mkldnn_mkl_cpu'), custom_steps.compile_unix_int64_cpu('ubuntu_cpu'), custom_steps.compile_unix_openblas_cpu_no_tvm_op('cpu_openblas_no_tvm_op'), @@ -50,7 +48,7 @@ core_logic: { custom_steps.test_unix_python3_mkl_cpu('cpu_mkl'), custom_steps.test_unix_python3_mkldnn_cpu('mkldnn_cpu'), custom_steps.test_unix_python3_mkldnn_mkl_cpu('mkldnn_mkl_cpu'), - custom_steps.test_unix_onnx_cpu('cpu_make'), + custom_steps.test_unix_onnx_cpu('cpu'), /* Disabled due to master build failure: * http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/ * https://github.com/apache/incubator-mxnet/issues/11801 diff --git a/ci/jenkins/Jenkinsfile_unix_gpu b/ci/jenkins/Jenkinsfile_unix_gpu index 1fe96bf690df..1fcdc96f46ac 100644 --- a/ci/jenkins/Jenkinsfile_unix_gpu +++ b/ci/jenkins/Jenkinsfile_unix_gpu @@ -37,13 +37,11 @@ core_logic: { custom_steps.compile_unix_mkldnn_gpu('mkldnn_gpu'), custom_steps.compile_unix_mkldnn_nocudnn_gpu('mkldnn_gpu_nocudnn'), custom_steps.compile_unix_full_gpu('gpu'), - custom_steps.compile_unix_full_gpu_make('gpu_make'), custom_steps.compile_unix_full_gpu_debug('gpu_debug'), custom_steps.compile_unix_cmake_gpu('cmake_gpu'), custom_steps.compile_unix_tensorrt_gpu('tensorrt'), custom_steps.compile_unix_int64_gpu('gpu_int64'), custom_steps.compile_unix_cmake_gpu_no_rtc('gpu_no_rtc'), - custom_steps.compile_unix_full_gpu_mkldnn_cpp_test('gpu_mkldnn_cpp_test_make') ]) utils.parallel_stage('Tests', [ diff --git a/ci/jenkins/Jenkinsfile_website_c_docs b/ci/jenkins/Jenkinsfile_website_c_docs deleted file mode 100644 index caddbae8c620..000000000000 --- a/ci/jenkins/Jenkinsfile_website_c_docs +++ /dev/null @@ -1,48 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 20 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite('libmxnet') - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_clojure_docs b/ci/jenkins/Jenkinsfile_website_clojure_docs deleted file mode 100644 index caddbae8c620..000000000000 --- a/ci/jenkins/Jenkinsfile_website_clojure_docs +++ /dev/null @@ -1,48 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 20 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite('libmxnet') - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_java_docs b/ci/jenkins/Jenkinsfile_website_java_docs deleted file mode 100644 index 03d160009740..000000000000 --- a/ci/jenkins/Jenkinsfile_website_java_docs +++ /dev/null @@ -1,47 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 20 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite('libmxnet') - ]) -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_julia_docs b/ci/jenkins/Jenkinsfile_website_julia_docs deleted file mode 100644 index 8a2528f89ce7..000000000000 --- a/ci/jenkins/Jenkinsfile_website_julia_docs +++ /dev/null @@ -1,48 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 60 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite('libmxnet') - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/docs/static_site/src/pages/api/cpp/index.md b/docs/static_site/src/pages/api/cpp/index.md index 3aff76b85337..2f834a763a12 100644 --- a/docs/static_site/src/pages/api/cpp/index.md +++ b/docs/static_site/src/pages/api/cpp/index.md @@ -41,12 +41,8 @@ The cpp-package directory contains the implementation of C++ API. As mentioned a ``` 3. Install the [prerequisites](), desired [BLAS libraries]() and optional [OpenCV, CUDA, and cuDNN]() for building MXNet from source. -4. There is a configuration file for make, [make/config.mk]() that contains all the compilation options. You can edit this file and set the appropriate options prior to running the **make** command. -5. Please refer to [platform specific build instructions]() and available [build configurations](https://mxnet.apache.org/get_started/build_from_source#build-configurations) for more details. -5. For enabling the build of C++ Package, set the **USE\_CPP\_PACKAGE = 1** in [make/config.mk](). Optionally, the compilation flag can also be specified on **make** command line as follows. - ``` - make -j USE_CPP_PACKAGE=1 - ``` +4. Please refer to [platform specific build instructions]() and available [build configurations](https://mxnet.apache.org/get_started/build_from_source#build-configurations) for more details. +5. For enabling the build of C++ Package, set the **USE\_CPP\_PACKAGE = 1** in the config file. ## Usage diff --git a/docs/static_site/src/pages/api/faq/cloud.md b/docs/static_site/src/pages/api/faq/cloud.md index 480773515c49..2a5837b017eb 100644 --- a/docs/static_site/src/pages/api/faq/cloud.md +++ b/docs/static_site/src/pages/api/faq/cloud.md @@ -66,15 +66,9 @@ unzip mnist.zip && s3cmd put t*-ubyte s3://dmlc/mnist/ ``` ### Use Pre-installed EC2 GPU Instance -The [Deep Learning AMI](https://aws.amazon.com/marketplace/pp/B01M0AXXQB?qid=1475211685369&sr=0-1&ref_=srh_res_product_title) is an Amazon Linux image -supported and maintained by Amazon Web Services for use on Amazon Elastic Compute Cloud (Amazon EC2). -It contains [MXNet-v0.9.3 tag](https://github.com/apache/incubator-mxnet) and the necessary components to get going with deep learning, -including Nvidia drivers, CUDA, cuDNN, Anaconda, Python2 and Python3. -The AMI IDs are the following: - -* us-east-1: ami-e7c96af1 -* us-west-2: ami-dfb13ebf -* eu-west-1: ami-6e5d6808 +The [Deep Learning AMIs](https://aws.amazon.com/marketplace/search/results?x=0&y=0&searchTerms=Deep+Learning+AMI) +are a series of images supported and maintained by Amazon Web Services for use +on Amazon Elastic Compute Cloud (Amazon EC2) and contain the latest MXNet release. Now you can launch _MXNet_ directly on an EC2 GPU instance. You can also use [Jupyter](https://jupyter.org) notebook on EC2 machine. @@ -83,69 +77,19 @@ on how to connect to a Jupyter notebook running on an EC2 instance. ### Set Up an EC2 GPU Instance from Scratch -_MXNet_ requires the following libraries: - -- C++ compiler with C++11 support, such as `gcc >= 4.8` -- `CUDA` (`CUDNN` in optional) for GPU linear algebra -- `BLAS` (cblas, open-blas, atblas, mkl, or others) for CPU linear algebra -- `opencv` for image augmentations -- `curl` and `openssl` for the ability to read/write to Amazon S3 - -Installing `CUDA` on EC2 instances requires some effort. Caffe has a good -[tutorial](https://github.com/BVLC/caffe/wiki/Install-Caffe-on-EC2-from-scratch-(Ubuntu,-CUDA-7,-cuDNN-3)) -on how to install CUDA 7.0 on Ubuntu 14.04. - -***Note:*** We tried CUDA 7.5 on Nov 7, 2015, but found it problematic. - -You can install the rest using the package manager. For example, on Ubuntu: - -``` -sudo apt-get update -sudo apt-get install -y build-essential git libcurl4-openssl-dev libatlas-base-dev libopencv-dev python-numpy -``` - -The Amazon Machine Image (AMI) [ami-12fd8178](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#LaunchInstanceWizard:ami=ami-12fd8178) has the packages listed above installed. - - -### Build and Run MXNet on a GPU Instance - -The following commands build _MXNet_ with CUDA/CUDNN, Amazon S3, and distributed -training. - -```bash -git clone --recursive https://github.com/dmlc/mxnet -cd mxnet; cp make/config.mk . -echo "USE_CUDA=1" >>config.mk -echo "USE_CUDA_PATH=/usr/local/cuda" >>config.mk -echo "USE_CUDNN=1" >>config.mk -echo "USE_BLAS=atlas" >> config.mk -echo "USE_DIST_KVSTORE = 1" >>config.mk -echo "USE_S3=1" >>config.mk -make -j$(nproc) -``` - -To test whether everything is installed properly, we can try training a convolutional neural network (CNN) on the MNIST dataset using a GPU: - -```bash -python example/image-classification/train_mnist.py -``` - -If you've placed the MNIST data on `s3://dmlc/mnist`, you can read the data stored on Amazon S3 directly with the following command: - -```bash -sed -i.bak "s!data_dir = 'data'!data_dir = 's3://dmlc/mnist'!" example/image-classification/train_mnist.py -``` - -***Note:*** You can use `sudo ln /dev/null /dev/raw1394` to fix the opencv error `libdc1394 error: Failed to initialize libdc1394`. +[Deep Learning Base AMIs](https://aws.amazon.com/marketplace/search/results?x=0&y=0&searchTerms=Deep+Learning+Base+AMI) +provide a foundational image with NVIDIA CUDA, cuDNN, GPU drivers, Intel +MKL-DNN, Docker and Nvidia-Docker, etc. for deploying your own custom deep +learning environment. You may follow the [MXNet Build From Source +instructions](> config.mk - ``` +2. Set `USE_S3=1` in the configuration file. ## Step 2: Configure S3 authentication tokens diff --git a/make/config.mk b/make/config.mk deleted file mode 100644 index 3d8e974a31c1..000000000000 --- a/make/config.mk +++ /dev/null @@ -1,248 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet -# -# If you want to change the configuration, please use the following steps. -# Assume you are on the root directory of mxnet. First copy this file so that -# any local changes will be ignored by git -# -# $ cp make/config.mk . -# -# Next modify the according entries, and then compile by -# -# $ make -# -# or build in parallel with 8 threads -# -# $ make -j8 -#------------------------------------------------------------------------------- - -#--------------------- -# choice of compiler -#-------------------- - -ifndef CC -export CC = gcc -endif -ifndef CXX -export CXX = g++ -endif -ifndef NVCC -export NVCC = nvcc -endif - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# whether to turn on segfault signal handler to log the stack trace -USE_SIGNAL_HANDLER = - -# the additional link flags you want to add -ADD_LDFLAGS = - -# the additional compile flags you want to add -ADD_CFLAGS = - -# whether to build operators written in TVM -USE_TVM_OP = 0 - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# whether use CUDA during compile -USE_CUDA = 0 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -USE_CUDA_PATH = NONE - -# whether to enable CUDA runtime compilation -ENABLE_CUDA_RTC = 1 - -# whether use CuDNN R3 library -USE_CUDNN = 0 - -# whether to use NVTX when profiling -USE_NVTX = 0 - -#whether to use NCCL library -USE_NCCL = 0 -#add the path to NCCL library -USE_NCCL_PATH = NONE - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -#whether use libjpeg-turbo for image decode without OpenCV wrapper -USE_LIBJPEG_TURBO = 0 -#add the path to libjpeg-turbo library -USE_LIBJPEG_TURBO_PATH = NONE - -# use openmp for parallelization -USE_OPENMP = 1 - -# whether use MKL-DNN library: 0 = disabled, 1 = enabled -# if USE_MKLDNN is not defined, MKL-DNN will be enabled by default on x86 Linux. -# you can disable it explicity with USE_MKLDNN = 0 -USE_MKLDNN = - -# whether use NNPACK library -USE_NNPACK = 0 - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -# in default use atlas for linux while apple for osx -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S), Darwin) -USE_BLAS = apple -else -USE_BLAS = atlas -endif - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# path to lapack library in case of a non-standard installation -USE_LAPACK_PATH = - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -# If use MKL only for BLAS, choose static link automatically to allow python wrapper -ifeq ($(USE_BLAS), mkl) -USE_STATIC_MKL = 1 -else -USE_STATIC_MKL = NONE -endif - -#---------------------------- -# Settings for power and arm arch -#---------------------------- -ARCH := $(shell uname -a) -ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) - USE_SSE=0 - USE_F16C=0 -else - USE_SSE=1 -endif - -#---------------------------- -# F16C instruction support for faster arithmetic of fp16 on CPU -#---------------------------- -# For distributed training with fp16, this helps even if training on GPUs -# If left empty, checks CPU support and turns it on. -# For cross compilation, please check support for F16C on target device and turn off if necessary. -USE_F16C = - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 0 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 0 - -#---------------------------- -# performance settings -#---------------------------- -# Use operator tuning -USE_OPERATOR_TUNING = 1 - -# Use gperftools if found -# Disable because of #8968 -USE_GPERFTOOLS = 0 - -# path to gperftools (tcmalloc) library in case of a non-standard installation -USE_GPERFTOOLS_PATH = - -# Link gperftools statically -USE_GPERFTOOLS_STATIC = - -# Use JEMalloc if found, and not using gperftools -USE_JEMALLOC = 0 - -# path to jemalloc library in case of a non-standard installation -USE_JEMALLOC_PATH = - -# Link jemalloc statically -USE_JEMALLOC_STATIC = - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - -#---------------------------- -# other features -#---------------------------- - -# Create C++ interface package -USE_CPP_PACKAGE = 0 - -# Use int64_t type to represent the total number of elements in a tensor -# This will cause performance degradation reported in issue #14496 -# Set to 1 for large tensor with tensor size greater than INT32_MAX i.e. 2147483647 -# Note: the size of each dimension is still bounded by INT32_MAX -USE_INT64_TENSOR_SIZE = 0 - -# Python executable. Needed for cython target -PYTHON = python - -#---------------------------- -# plugins -#---------------------------- - -# whether to use caffe integration. This requires installing caffe. -# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH -# CAFFE_PATH = $(HOME)/caffe -# MXNET_PLUGINS += plugin/caffe/caffe.mk - -# WARPCTC_PATH = $(HOME)/warp-ctc -# MXNET_PLUGINS += plugin/warpctc/warpctc.mk - -# whether to use sframe integration. This requires build sframe -# git@github.com:dato-code/SFrame.git -# SFRAME_PATH = $(HOME)/SFrame -# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/config/libmxnet.sym b/make/config/libmxnet.sym deleted file mode 100644 index df81f6861666..000000000000 --- a/make/config/libmxnet.sym +++ /dev/null @@ -1,10 +0,0 @@ -*MX* -*NN* -*mx* -*nn* -Java_org_apache_mxnet* -*NDArray* -*Engine*Get* -*Storage*Get* -*on_enter_api* -*on_exit_api* diff --git a/make/config/libmxnet.ver b/make/config/libmxnet.ver deleted file mode 100644 index 0a9f8e67db69..000000000000 --- a/make/config/libmxnet.ver +++ /dev/null @@ -1,14 +0,0 @@ -{ - global: - *NN*; - *MX*; - *nn*; - *mx*; - Java_org_apache_mxnet*; - *NDArray*; - *Engine*Get*; - *Storage*Get*; - *on_enter_api*; - *on_exit_api*; - local: *; -}; diff --git a/make/config_jetson.mk b/make/config_jetson.mk deleted file mode 100644 index 7de6eff7b6b5..000000000000 --- a/make/config_jetson.mk +++ /dev/null @@ -1,219 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet -# -# If you want to change the configuration, please use the following -# steps. Assume you are on the root directory of mxnet. First copy the this -# file so that any local changes will be ignored by git -# -# $ cp make/config.mk . -# -# Next modify the according entries, and then compile by -# -# $ make -# -# or build in parallel with 8 threads -# -# $ make -j8 -#------------------------------------------------------------------------------- - -#--------------------- -# For cross compilation we only explictily set a compiler when one is not already present. -#-------------------- - -ifndef CC -export CC = gcc -endif -ifndef CXX -export CXX = g++ -endif -ifndef NVCC -export NVCC = nvcc -endif - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# whether to turn on segfault signal handler to log the stack trace -USE_SIGNAL_HANDLER = 1 - -# the additional link flags you want to add -ADD_LDFLAGS = -L${CROSS_ROOT}/lib -L/usr/lib/aarch64-linux-gnu/ - -# the additional compile flags you want to add -ADD_CFLAGS = -I${CROSS_ROOT}/include -I/usr/include/aarch64-linux-gnu/ - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# whether use CUDA during compile -USE_CUDA = 1 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -USE_CUDA_PATH = /usr/local/cuda - -# CUDA_ARCH setting -CUDA_ARCH = -gencode arch=compute_53,code=sm_53 -gencode arch=compute_62,code=sm_62 -gencode arch=compute_72,code=sm_72 - -# whether to enable CUDA runtime compilation -ENABLE_CUDA_RTC = 0 - -# whether use CuDNN R3 library -USE_CUDNN = 1 - -#whether to use NCCL library -USE_NCCL = 0 -#add the path to NCCL library -USE_NCCL_PATH = NONE - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -#whether use libjpeg-turbo for image decode without OpenCV wrapper -USE_LIBJPEG_TURBO = 0 -#add the path to libjpeg-turbo library -USE_LIBJPEG_TURBO_PATH = NONE - -# use openmp for parallelization -USE_OPENMP = 1 - -# whether use MKL-DNN library -USE_MKLDNN = 0 - -# whether use NNPACK library -USE_NNPACK = 0 - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -# in default use atlas for linux while apple for osx -UNAME_S := $(shell uname -s) -USE_BLAS = openblas - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# path to lapack library in case of a non-standard installation -USE_LAPACK_PATH = - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -# If use MKL only for BLAS, choose static link automatically to allow python wrapper -ifeq ($(USE_BLAS), mkl) -USE_STATIC_MKL = 1 -else -USE_STATIC_MKL = NONE -endif - -#---------------------------- -# Settings for power and arm arch -#---------------------------- -USE_SSE=0 - -# Turn off F16C instruction set support -USE_F16C=0 - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 0 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 0 - -#---------------------------- -# performance settings -#---------------------------- -# Use operator tuning -USE_OPERATOR_TUNING = 1 - -# Use gperftools if found -# Disable because of #8968 -USE_GPERFTOOLS = 0 - -# path to gperftools (tcmalloc) library in case of a non-standard installation -USE_GPERFTOOLS_PATH = - -# Use JEMalloc if found, and not using gperftools -USE_JEMALLOC = 1 - -# path to jemalloc library in case of a non-standard installation -USE_JEMALLOC_PATH = - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - -#---------------------------- -# other features -#---------------------------- - -# Create C++ interface package -USE_CPP_PACKAGE = 0 - -# Use int64_t type to represent the total number of elements in the tensor -# This will cause performance degradation reported in issue #14496 -# Set to 1 for large tensor with tensor size greater than INT32_MAX i.e. 2147483647 -# Note: the size of each dimension is still bounded by INT32_MAX -USE_INT64_TENSOR_SIZE = 0 - -#---------------------------- -# plugins -#---------------------------- - -# whether to use caffe integration. This requires installing caffe. -# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH -# CAFFE_PATH = $(HOME)/caffe -# MXNET_PLUGINS += plugin/caffe/caffe.mk - -# WARPCTC_PATH = $(HOME)/warp-ctc -# MXNET_PLUGINS += plugin/warpctc/warpctc.mk - -# whether to use sframe integration. This requires build sframe -# git@github.com:dato-code/SFrame.git -# SFRAME_PATH = $(HOME)/SFrame -# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/osx.mk b/make/osx.mk deleted file mode 100644 index 25f3ba6df55b..000000000000 --- a/make/osx.mk +++ /dev/null @@ -1,153 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet -# -# If you want to change the configuration, please use the following -# steps. Assume you are on the root directory of mxnet. First copy the this -# file so that any local changes will be ignored by git -# -# $ cp make/config.mk . -# -# Next modify the according entries, and then compile by -# -# $ make -# -# or build in parallel with 8 threads -# -# $ make -j8 -#------------------------------------------------------------------------------- - -#--------------------- -# choice of compiler -#-------------------- - -export CC = gcc -export CXX = g++ -export NVCC = nvcc - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# the additional link flags you want to add -ADD_LDFLAGS = - -# the additional compile flags you want to add -ADD_CFLAGS = - -# whether to build operators written in TVM -USE_TVM_OP = 0 - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# whether use CUDA during compile -USE_CUDA = 0 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -USE_CUDA_PATH = NONE - -# whether to enable CUDA runtime compilation -ENABLE_CUDA_RTC = 1 - -# whether use CUDNN R3 library -USE_CUDNN = 0 - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -# use openmp for parallelization -# apple-clang by default does not have openmp built-in -USE_OPENMP = 0 - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas, apple -USE_BLAS = apple - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# by default, disable lapack when using MKL -# switch on when there is a full installation of MKL available (not just MKL_ML) -ifeq ($(USE_BLAS), mkl) -USE_LAPACK = 0 -endif - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 0 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 0 - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - -#---------------------------- -# other features -#---------------------------- - -# Create C++ interface package -USE_CPP_PACKAGE = 0 - -# Use int64_t type to represent the total number of elements in a tensor -# This will cause performance degradation reported in issue #14496 -# Set to 1 for large tensor with tensor size greater than INT32_MAX i.e. 2147483647 -# Note: the size of each dimension is still bounded by INT32_MAX -USE_INT64_TENSOR_SIZE = 0 - -#---------------------------- -# plugins -#---------------------------- - -# whether to use torch integration. This requires installing torch. -# TORCH_PATH = $(HOME)/torch -# MXNET_PLUGINS += plugin/torch/torch.mk diff --git a/make/readthedocs.mk b/make/readthedocs.mk deleted file mode 100644 index b33dd3c5d21f..000000000000 --- a/make/readthedocs.mk +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#-------------------------------------------------------- -# Configuration for document generation with less deps -# The library may not run, but doc generation could work -#-------------------------------------------------------- - -# choice of compiler -export CC = gcc -export CXX = g++ -export NVCC = nvcc - -# whether use CUDA during compile -USE_CUDA = 0 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -USE_CUDA_PATH = NONE - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 0 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -# whether use CUDNN R3 library -USE_CUDNN = 0 - - -# use openmp for parallelization -USE_OPENMP = 0 - -# -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -USE_STATIC_MKL = NONE -USE_BLAS = NONE -USE_LAPACK = 0 - -# -# add path to intel library, you may need it -# for MKL, if you did not add the path to environment variable -# -USE_INTEL_PATH = NONE - - -# the additional link flags you want to add -ADD_LDFLAGS = -lgomp - -# the additional compile flags you want to add -ADD_CFLAGS = -DMSHADOW_STAND_ALONE=1 -DMSHADOW_USE_SSE=0 -# -# If use MKL, choose static link automatically to fix python wrapper -# -ifeq ($(USE_BLAS), mkl) - USE_STATIC_MKL = 1 -endif - -#------------------------ -# configuration for DMLC -#------------------------ -# whether use HDFS support during compile -# this will allow cxxnet to directly save/load model from hdfs -USE_HDFS = 0 - -# whether use AWS S3 support during compile -# this will allow cxxnet to directly save/load model from s3 -USE_S3 = 0 - -# path to libjvm.so -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# uses O0 instead of O3 for better performance -DEBUG = 1 diff --git a/make/staticbuild/darwin_cpu.mk b/make/staticbuild/darwin_cpu.mk deleted file mode 100644 index 1859936f180e..000000000000 --- a/make/staticbuild/darwin_cpu.mk +++ /dev/null @@ -1,167 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet for making python wheel -#------------------------------------------------------------------------------- - -#--------------------- -# choice of compiler -#-------------------- - -export CC = gcc -export CXX = g++ -export NVCC = nvcc - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 1 - -# the additional link flags you want to add -ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -lz -framework CoreFoundation -framework Security -Wl,-exported_symbols_list,$(CURDIR)/make/config/libmxnet.sym,-rpath,'$${ORIGIN}',-dead_strip - -# the additional compile flags you want to add -ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -# in default use atlas for linux while apple for osx -USE_BLAS=apple - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -# whether use CUDA during compile -USE_CUDA = 0 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -USE_CUDA_PATH = NONE - -# whether use CuDNN R3 library -USE_CUDNN = 0 - -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -# CUDA_ARCH := - -# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -ENABLE_CUDA_RTC = 0 - -# use openmp for parallelization -USE_OPENMP = 0 -USE_OPERATOR_TUNING = 1 -USE_LIBJPEG_TURBO = 1 - -# whether use MKL-DNN library -USE_MKLDNN = 1 - -# whether use NNPACK library -USE_NNPACK = 0 - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# path to lapack library in case of a non-standard installation -USE_LAPACK_PATH = $(DEPS_PATH)/lib - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -# If use MKL, choose static link automatically to allow python wrapper -ifeq ($(USE_BLAS), mkl) -USE_STATIC_MKL = 1 -else -USE_STATIC_MKL = NONE -endif - -#---------------------------- -# Settings for power and arm arch -#---------------------------- -ARCH := $(shell uname -a) -ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) - USE_SSE=0 -else - USE_SSE=1 -endif - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 1 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 1 - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - - -#---------------------------- -# plugins -#---------------------------- - -# whether to use caffe integration. This requires installing caffe. -# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH -# CAFFE_PATH = $(HOME)/caffe -# MXNET_PLUGINS += plugin/caffe/caffe.mk - -# whether to use torch integration. This requires installing torch. -# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH -# TORCH_PATH = $(HOME)/torch -# MXNET_PLUGINS += plugin/torch/torch.mk - -# WARPCTC_PATH = $(HOME)/warp-ctc -# MXNET_PLUGINS += plugin/warpctc/warpctc.mk - -# whether to use sframe integration. This requires build sframe -# git@github.com:dato-code/SFrame.git -# SFRAME_PATH = $(HOME)/SFrame -# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/staticbuild/darwin_mkl.mk b/make/staticbuild/darwin_mkl.mk deleted file mode 100644 index 1859936f180e..000000000000 --- a/make/staticbuild/darwin_mkl.mk +++ /dev/null @@ -1,167 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet for making python wheel -#------------------------------------------------------------------------------- - -#--------------------- -# choice of compiler -#-------------------- - -export CC = gcc -export CXX = g++ -export NVCC = nvcc - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 1 - -# the additional link flags you want to add -ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -lz -framework CoreFoundation -framework Security -Wl,-exported_symbols_list,$(CURDIR)/make/config/libmxnet.sym,-rpath,'$${ORIGIN}',-dead_strip - -# the additional compile flags you want to add -ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -# in default use atlas for linux while apple for osx -USE_BLAS=apple - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -# whether use CUDA during compile -USE_CUDA = 0 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -USE_CUDA_PATH = NONE - -# whether use CuDNN R3 library -USE_CUDNN = 0 - -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -# CUDA_ARCH := - -# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -ENABLE_CUDA_RTC = 0 - -# use openmp for parallelization -USE_OPENMP = 0 -USE_OPERATOR_TUNING = 1 -USE_LIBJPEG_TURBO = 1 - -# whether use MKL-DNN library -USE_MKLDNN = 1 - -# whether use NNPACK library -USE_NNPACK = 0 - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# path to lapack library in case of a non-standard installation -USE_LAPACK_PATH = $(DEPS_PATH)/lib - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -# If use MKL, choose static link automatically to allow python wrapper -ifeq ($(USE_BLAS), mkl) -USE_STATIC_MKL = 1 -else -USE_STATIC_MKL = NONE -endif - -#---------------------------- -# Settings for power and arm arch -#---------------------------- -ARCH := $(shell uname -a) -ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) - USE_SSE=0 -else - USE_SSE=1 -endif - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 1 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 1 - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - - -#---------------------------- -# plugins -#---------------------------- - -# whether to use caffe integration. This requires installing caffe. -# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH -# CAFFE_PATH = $(HOME)/caffe -# MXNET_PLUGINS += plugin/caffe/caffe.mk - -# whether to use torch integration. This requires installing torch. -# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH -# TORCH_PATH = $(HOME)/torch -# MXNET_PLUGINS += plugin/torch/torch.mk - -# WARPCTC_PATH = $(HOME)/warp-ctc -# MXNET_PLUGINS += plugin/warpctc/warpctc.mk - -# whether to use sframe integration. This requires build sframe -# git@github.com:dato-code/SFrame.git -# SFRAME_PATH = $(HOME)/SFrame -# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/staticbuild/linux_cpu.mk b/make/staticbuild/linux_cpu.mk deleted file mode 100644 index 1cf389ae4a57..000000000000 --- a/make/staticbuild/linux_cpu.mk +++ /dev/null @@ -1,167 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet for making python wheel -#------------------------------------------------------------------------------- - -#--------------------- -# choice of compiler -#-------------------- - -export CC = gcc -export CXX = g++ -export NVCC = nvcc - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 1 - -# the additional link flags you want to add -ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections - -# the additional compile flags you want to add -ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -# in default use atlas for linux while apple for osx -USE_BLAS=openblas - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -# whether use CUDA during compile -USE_CUDA = 0 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -USE_CUDA_PATH = NONE - -# whether use CuDNN R3 library -USE_CUDNN = 0 - -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -# CUDA_ARCH := - -# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -ENABLE_CUDA_RTC = 0 - -# use openmp for parallelization -USE_OPENMP = 1 -USE_OPERATOR_TUNING = 1 -USE_LIBJPEG_TURBO = 1 - -# whether use MKL-DNN library -USE_MKLDNN = 1 - -# whether use NNPACK library -USE_NNPACK = 0 - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# path to lapack library in case of a non-standard installation -USE_LAPACK_PATH = $(DEPS_PATH)/lib - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -# If use MKL, choose static link automatically to allow python wrapper -ifeq ($(USE_BLAS), mkl) -USE_STATIC_MKL = 1 -else -USE_STATIC_MKL = NONE -endif - -#---------------------------- -# Settings for power and arm arch -#---------------------------- -ARCH := $(shell uname -a) -ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) - USE_SSE=0 -else - USE_SSE=1 -endif - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 1 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 1 - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - - -#---------------------------- -# plugins -#---------------------------- - -# whether to use caffe integration. This requires installing caffe. -# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH -# CAFFE_PATH = $(HOME)/caffe -# MXNET_PLUGINS += plugin/caffe/caffe.mk - -# whether to use torch integration. This requires installing torch. -# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH -# TORCH_PATH = $(HOME)/torch -# MXNET_PLUGINS += plugin/torch/torch.mk - -# WARPCTC_PATH = $(HOME)/warp-ctc -# MXNET_PLUGINS += plugin/warpctc/warpctc.mk - -# whether to use sframe integration. This requires build sframe -# git@github.com:dato-code/SFrame.git -# SFRAME_PATH = $(HOME)/SFrame -# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/staticbuild/linux_cu100.mk b/make/staticbuild/linux_cu100.mk deleted file mode 100644 index 855485c5b6df..000000000000 --- a/make/staticbuild/linux_cu100.mk +++ /dev/null @@ -1,180 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet for making python wheel -#------------------------------------------------------------------------------- - -#--------------------- -# choice of compiler -#-------------------- - -export CC = gcc -export CXX = g++ -export NVCC = nvcc - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 1 - -# the additional link flags you want to add -ifdef USE_SYSTEM_CUDA -ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections -else -ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections -endif - -# the additional compile flags you want to add -ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -# in default use atlas for linux while apple for osx -USE_BLAS=openblas - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -# whether use CUDA during compile -USE_CUDA = 1 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -ifdef USE_SYSTEM_CUDA -USE_CUDA_PATH = /usr/local/cuda-10.0 -else -USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-10.0 -endif - -# whether to use CuDNN library -USE_CUDNN = 1 - -# whether to use NCCL library -USE_NCCL = 1 - -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -# CUDA_ARCH := - -# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -ENABLE_CUDA_RTC = 1 - -USE_NVTX=1 - -# use openmp for parallelization -USE_OPENMP = 1 -USE_OPERATOR_TUNING = 1 -USE_LIBJPEG_TURBO = 1 - -# whether use MKL-DNN library -USE_MKLDNN = 1 - -# whether use NNPACK library -USE_NNPACK = 0 - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# path to lapack library in case of a non-standard installation -USE_LAPACK_PATH = $(DEPS_PATH)/lib - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -# If use MKL, choose static link automatically to allow python wrapper -ifeq ($(USE_BLAS), mkl) -USE_STATIC_MKL = 1 -else -USE_STATIC_MKL = NONE -endif - -#---------------------------- -# Settings for power and arm arch -#---------------------------- -ARCH := $(shell uname -a) -ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) - USE_SSE=0 -else - USE_SSE=1 -endif - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 1 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 1 - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - - -#---------------------------- -# plugins -#---------------------------- - -# whether to use caffe integration. This requires installing caffe. -# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH -# CAFFE_PATH = $(HOME)/caffe -# MXNET_PLUGINS += plugin/caffe/caffe.mk - -# whether to use torch integration. This requires installing torch. -# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH -# TORCH_PATH = $(HOME)/torch -# MXNET_PLUGINS += plugin/torch/torch.mk - -# WARPCTC_PATH = $(HOME)/warp-ctc -# MXNET_PLUGINS += plugin/warpctc/warpctc.mk - -# whether to use sframe integration. This requires build sframe -# git@github.com:dato-code/SFrame.git -# SFRAME_PATH = $(HOME)/SFrame -# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/staticbuild/linux_cu101.mk b/make/staticbuild/linux_cu101.mk deleted file mode 100644 index 7bbde85bee11..000000000000 --- a/make/staticbuild/linux_cu101.mk +++ /dev/null @@ -1,181 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet for making python wheel -#------------------------------------------------------------------------------- - -#--------------------- -# choice of compiler -#-------------------- - -export CC = gcc -export CXX = g++ -export NVCC = nvcc - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 1 - -# the additional link flags you want to add -ifdef USE_SYSTEM_CUDA -ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections -else -ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections -endif - -# the additional compile flags you want to add -ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -# in default use atlas for linux while apple for osx -USE_BLAS=openblas - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -# whether use CUDA during compile -USE_CUDA = 1 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -ifdef USE_SYSTEM_CUDA -USE_CUDA_PATH = /usr/local/cuda-10.1 -else -USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-10.1 -endif - -# whether to use CuDNN library -USE_CUDNN = 1 - -# whether to use NCCL library -USE_NCCL = 1 - -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -# CUDA_ARCH := - -# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -ENABLE_CUDA_RTC = 1 - -USE_NVTX=1 - -# use openmp for parallelization -USE_OPENMP = 1 -USE_OPERATOR_TUNING = 1 -USE_LIBJPEG_TURBO = 1 - -# whether use MKL-DNN library -USE_MKLDNN = 1 - -# whether use NNPACK library -USE_NNPACK = 0 - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# path to lapack library in case of a non-standard installation -USE_LAPACK_PATH = $(DEPS_PATH)/lib - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -# If use MKL, choose static link automatically to allow python wrapper -ifeq ($(USE_BLAS), mkl) -USE_STATIC_MKL = 1 -else -USE_STATIC_MKL = NONE -endif - -#---------------------------- -# Settings for power and arm arch -#---------------------------- -ARCH := $(shell uname -a) -ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) - USE_SSE=0 -else - USE_SSE=1 -endif - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 1 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 1 - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - - -#---------------------------- -# plugins -#---------------------------- - -# whether to use caffe integration. This requires installing caffe. -# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH -# CAFFE_PATH = $(HOME)/caffe -# MXNET_PLUGINS += plugin/caffe/caffe.mk - -# whether to use torch integration. This requires installing torch. -# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH -# TORCH_PATH = $(HOME)/torch -# MXNET_PLUGINS += plugin/torch/torch.mk - -# WARPCTC_PATH = $(HOME)/warp-ctc -# MXNET_PLUGINS += plugin/warpctc/warpctc.mk - -# whether to use sframe integration. This requires build sframe -# git@github.com:dato-code/SFrame.git -# SFRAME_PATH = $(HOME)/SFrame -# MXNET_PLUGINS += plugin/sframe/plugin.mk - diff --git a/make/staticbuild/linux_cu102.mk b/make/staticbuild/linux_cu102.mk deleted file mode 100644 index 963842a19cff..000000000000 --- a/make/staticbuild/linux_cu102.mk +++ /dev/null @@ -1,181 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet for making python wheel -#------------------------------------------------------------------------------- - -#--------------------- -# choice of compiler -#-------------------- - -export CC = gcc -export CXX = g++ -export NVCC = nvcc - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 1 - -# the additional link flags you want to add -ifdef USE_SYSTEM_CUDA -ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections -else -ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections -endif - -# the additional compile flags you want to add -ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -# in default use atlas for linux while apple for osx -USE_BLAS=openblas - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -# whether use CUDA during compile -USE_CUDA = 1 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -ifdef USE_SYSTEM_CUDA -USE_CUDA_PATH = /usr/local/cuda-10.2 -else -USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-10.2 -endif - -# whether to use CuDNN library -USE_CUDNN = 1 - -# whether to use NCCL library -USE_NCCL = 1 - -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -# CUDA_ARCH := - -# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -ENABLE_CUDA_RTC = 1 - -USE_NVTX=1 - -# use openmp for parallelization -USE_OPENMP = 1 -USE_OPERATOR_TUNING = 1 -USE_LIBJPEG_TURBO = 1 - -# whether use MKL-DNN library -USE_MKLDNN = 1 - -# whether use NNPACK library -USE_NNPACK = 0 - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# path to lapack library in case of a non-standard installation -USE_LAPACK_PATH = $(DEPS_PATH)/lib - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -# If use MKL, choose static link automatically to allow python wrapper -ifeq ($(USE_BLAS), mkl) -USE_STATIC_MKL = 1 -else -USE_STATIC_MKL = NONE -endif - -#---------------------------- -# Settings for power and arm arch -#---------------------------- -ARCH := $(shell uname -a) -ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) - USE_SSE=0 -else - USE_SSE=1 -endif - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 1 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 1 - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - - -#---------------------------- -# plugins -#---------------------------- - -# whether to use caffe integration. This requires installing caffe. -# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH -# CAFFE_PATH = $(HOME)/caffe -# MXNET_PLUGINS += plugin/caffe/caffe.mk - -# whether to use torch integration. This requires installing torch. -# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH -# TORCH_PATH = $(HOME)/torch -# MXNET_PLUGINS += plugin/torch/torch.mk - -# WARPCTC_PATH = $(HOME)/warp-ctc -# MXNET_PLUGINS += plugin/warpctc/warpctc.mk - -# whether to use sframe integration. This requires build sframe -# git@github.com:dato-code/SFrame.git -# SFRAME_PATH = $(HOME)/SFrame -# MXNET_PLUGINS += plugin/sframe/plugin.mk - diff --git a/make/staticbuild/linux_cu92.mk b/make/staticbuild/linux_cu92.mk deleted file mode 100644 index 2cbbdd25eeaf..000000000000 --- a/make/staticbuild/linux_cu92.mk +++ /dev/null @@ -1,180 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet for making python wheel -#------------------------------------------------------------------------------- - -#--------------------- -# choice of compiler -#-------------------- - -export CC = gcc -export CXX = g++ -export NVCC = nvcc - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 1 - -# the additional link flags you want to add -ifdef USE_SYSTEM_CUDA -ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections -else -ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections -endif - -# the additional compile flags you want to add -ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -# in default use atlas for linux while apple for osx -USE_BLAS=openblas - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -# whether use CUDA during compile -USE_CUDA = 1 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -ifdef USE_SYSTEM_CUDA -USE_CUDA_PATH = /usr/local/cuda-9.2 -else -USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.2 -endif - -# whether to use CuDNN library -USE_CUDNN = 1 - -# whether to use NCCL library -USE_NCCL = 1 - -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -# CUDA_ARCH := - -# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -ENABLE_CUDA_RTC = 1 - -USE_NVTX=1 - -# use openmp for parallelization -USE_OPENMP = 1 -USE_OPERATOR_TUNING = 1 -USE_LIBJPEG_TURBO = 1 - -# whether use MKL-DNN library -USE_MKLDNN = 1 - -# whether use NNPACK library -USE_NNPACK = 0 - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# path to lapack library in case of a non-standard installation -USE_LAPACK_PATH = $(DEPS_PATH)/lib - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -# If use MKL, choose static link automatically to allow python wrapper -ifeq ($(USE_BLAS), mkl) -USE_STATIC_MKL = 1 -else -USE_STATIC_MKL = NONE -endif - -#---------------------------- -# Settings for power and arm arch -#---------------------------- -ARCH := $(shell uname -a) -ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) - USE_SSE=0 -else - USE_SSE=1 -endif - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 1 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 1 - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - - -#---------------------------- -# plugins -#---------------------------- - -# whether to use caffe integration. This requires installing caffe. -# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH -# CAFFE_PATH = $(HOME)/caffe -# MXNET_PLUGINS += plugin/caffe/caffe.mk - -# whether to use torch integration. This requires installing torch. -# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH -# TORCH_PATH = $(HOME)/torch -# MXNET_PLUGINS += plugin/torch/torch.mk - -# WARPCTC_PATH = $(HOME)/warp-ctc -# MXNET_PLUGINS += plugin/warpctc/warpctc.mk - -# whether to use sframe integration. This requires build sframe -# git@github.com:dato-code/SFrame.git -# SFRAME_PATH = $(HOME)/SFrame -# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/make/staticbuild/linux_native.mk b/make/staticbuild/linux_native.mk deleted file mode 100644 index 348a659cd9e2..000000000000 --- a/make/staticbuild/linux_native.mk +++ /dev/null @@ -1,167 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------------------------- -# Template configuration for compiling mxnet for making python wheel -#------------------------------------------------------------------------------- - -#--------------------- -# choice of compiler -#-------------------- - -export CC = gcc -export CXX = g++ -export NVCC = nvcc - -# whether compile with options for MXNet developer -DEV = 0 - -# whether compile with debug -DEBUG = 0 - -# whether to turn on signal handler (e.g. segfault logger) -USE_SIGNAL_HANDLER = 1 - -# the additional link flags you want to add -ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections - -# the additional compile flags you want to add -ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections - -#--------------------------------------------- -# matrix computation libraries for CPU/GPU -#--------------------------------------------- - -# choose the version of blas you want to use -# can be: mkl, blas, atlas, openblas -# in default use atlas for linux while apple for osx -USE_BLAS=openblas - -# whether use opencv during compilation -# you can disable it, however, you will not able to use -# imbin iterator -USE_OPENCV = 1 -# Add OpenCV include path, in which the directory `opencv2` exists -USE_OPENCV_INC_PATH = NONE -# Add OpenCV shared library path, in which the shared library exists -USE_OPENCV_LIB_PATH = NONE - -# whether use CUDA during compile -USE_CUDA = 0 - -# add the path to CUDA library to link and compile flag -# if you have already add them to environment variable, leave it as NONE -# USE_CUDA_PATH = /usr/local/cuda -USE_CUDA_PATH = NONE - -# whether use CuDNN R3 library -USE_CUDNN = 0 - -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -# CUDA_ARCH := - -# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -ENABLE_CUDA_RTC = 0 - -# use openmp for parallelization -USE_OPENMP = 1 -USE_OPERATOR_TUNING = 1 -USE_LIBJPEG_TURBO = 1 - -# whether use MKL-DNN library -USE_MKLDNN = 0 - -# whether use NNPACK library -USE_NNPACK = 0 - -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 1 - -# path to lapack library in case of a non-standard installation -USE_LAPACK_PATH = $(DEPS_PATH)/lib - -# add path to intel library, you may need it for MKL, if you did not add the path -# to environment variable -USE_INTEL_PATH = NONE - -# If use MKL, choose static link automatically to allow python wrapper -ifeq ($(USE_BLAS), mkl) -USE_STATIC_MKL = 1 -else -USE_STATIC_MKL = NONE -endif - -#---------------------------- -# Settings for power and arm arch -#---------------------------- -ARCH := $(shell uname -a) -ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) - USE_SSE=0 -else - USE_SSE=1 -endif - -#---------------------------- -# distributed computing -#---------------------------- - -# whether or not to enable multi-machine supporting -USE_DIST_KVSTORE = 1 - -# whether or not allow to read and write HDFS directly. If yes, then hadoop is -# required -USE_HDFS = 0 - -# path to libjvm.so. required if USE_HDFS=1 -LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server - -# whether or not allow to read and write AWS S3 directly. If yes, then -# libcurl4-openssl-dev is required, it can be installed on Ubuntu by -# sudo apt-get install -y libcurl4-openssl-dev -USE_S3 = 1 - -#---------------------------- -# additional operators -#---------------------------- - -# path to folders containing projects specific operators that you don't want to put in src/operators -EXTRA_OPERATORS = - - -#---------------------------- -# plugins -#---------------------------- - -# whether to use caffe integration. This requires installing caffe. -# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH -# CAFFE_PATH = $(HOME)/caffe -# MXNET_PLUGINS += plugin/caffe/caffe.mk - -# whether to use torch integration. This requires installing torch. -# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH -# TORCH_PATH = $(HOME)/torch -# MXNET_PLUGINS += plugin/torch/torch.mk - -# WARPCTC_PATH = $(HOME)/warp-ctc -# MXNET_PLUGINS += plugin/warpctc/warpctc.mk - -# whether to use sframe integration. This requires build sframe -# git@github.com:dato-code/SFrame.git -# SFRAME_PATH = $(HOME)/SFrame -# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/mkldnn.mk b/mkldnn.mk deleted file mode 100644 index a22a64a018a8..000000000000 --- a/mkldnn.mk +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ifeq ($(USE_MKLDNN), 1) - MKLDNN_SUBMODDIR = $(ROOTDIR)/3rdparty/mkldnn - MKLDNN_BUILDDIR = $(MKLDNN_SUBMODDIR)/build - MXNET_LIBDIR = $(ROOTDIR)/lib - MXNET_INCLDIR = $(ROOTDIR)/include - MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libdnnl.a -endif - -mkldnn_FLAGS = -DCMAKE_INSTALL_PREFIX=$(MKLDNNROOT) -mkldnn_FLAGS += -DCMAKE_INSTALL_LIBDIR=lib -mkldnn_FLAGS += -B$(MKLDNN_BUILDDIR) -mkldnn_FLAGS += -DMKLDNN_ARCH_OPT_FLAGS="" -mkldnn_FLAGS += -DMKLDNN_BUILD_TESTS=OFF -mkldnn_FLAGS += -DMKLDNN_BUILD_EXAMPLES=OFF -mkldnn_FLAGS += -DMKLDNN_ENABLE_JIT_PROFILING=OFF -mkldnn_FLAGS += -DMKLDNN_LIBRARY_TYPE=STATIC -mkldnn_FLAGS += -DDNNL_ENABLE_CONCURRENT_EXEC=ON - -ifneq ($(USE_OPENMP), 1) - mkldnn_FLAGS += -DMKLDNN_CPU_RUNTIME=SEQ -endif - -ifeq ($(DEBUG), 1) - mkldnn_FLAGS += -DCMAKE_BUILD_TYPE=Debug -endif - -.PHONY: mkldnn mkldnn_clean - -mkldnn_build: $(MKLDNN_LIBFILE) - -$(MKLDNN_LIBFILE): - mkdir -p $(MKLDNNROOT)/lib - cmake $(MKLDNN_SUBMODDIR) $(mkldnn_FLAGS) - $(MAKE) -C $(MKLDNN_BUILDDIR) VERBOSE=1 - $(MAKE) -C $(MKLDNN_BUILDDIR) install - cp $(MKLDNN_BUILDDIR)/include/dnnl_version.h $(MXNET_INCLDIR)/mkldnn/. - cp $(MKLDNN_BUILDDIR)/include/dnnl_config.h $(MXNET_INCLDIR)/mkldnn/. - -mkldnn_clean: - $(RM) -r 3rdparty/mkldnn/build - $(RM) -r include/mkldnn/dnnl_version.h - $(RM) -r include/mkldnn/dnnl_config.h - -ifeq ($(USE_MKLDNN), 1) -mkldnn: mkldnn_build -else -mkldnn: -endif diff --git a/setup-utils/install-mxnet-amz-linux.sh b/setup-utils/install-mxnet-amz-linux.sh deleted file mode 100644 index 66788a984da6..000000000000 --- a/setup-utils/install-mxnet-amz-linux.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -###################################################################### -# This script installs MXNet for Python along with all required dependencies on a Amazon Linux Machine. -###################################################################### -set -e -# CMake is required for installing dependencies. -sudo yum install -y cmake - -# Set appropriate library path env variables -echo 'export PATH=/usr/local/bin:$PATH' >> ~/.profile -echo 'export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH' >> ~/.profile -echo 'export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH' >> ~/.profile -echo '. ~/.profile' >> ~/.bashrc -source ~/.profile - -# Install gcc-4.8/make and other development tools on Amazon Linux -# Reference: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/compile-software.html -# Install Python, Numpy, Scipy and set up tools. -sudo yum groupinstall -y "Development Tools" -sudo yum install -y python27 python27-setuptools python27-tools python-pip graphviz -sudo yum install -y python27-numpy python27-scipy python27-nose python27-matplotlib - -# Install OpenBLAS at /usr/local/openblas -git clone https://github.com/xianyi/OpenBLAS -cd OpenBLAS -make FC=gfortran -j $(($(nproc) + 1)) -sudo make PREFIX=/usr/local install -cd .. - -# Install OpenCV at /usr/local/opencv -git clone https://github.com/opencv/opencv -cd opencv -mkdir -p build -cd build -cmake -D BUILD_opencv_gpu=OFF -D WITH_EIGEN=ON -D WITH_TBB=ON -D WITH_CUDA=OFF -D WITH_1394=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local .. -sudo make PREFIX=/usr/local install - -# Export env variables for pkg config -export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH - -# Install MXNet Core without CUDA -MXNET_HOME="$HOME/mxnet/" -cd "$MXNET_HOME" -cp make/config.mk . -echo "USE_CUDA=0" >>config.mk -echo "USE_CUDNN=0" >>config.mk -echo "USE_BLAS=openblas" >>config.mk -echo "ADD_CFLAGS += -I/usr/include/openblas" >>config.mk -echo "ADD_LDFLAGS += -lopencv_core -lopencv_imgproc -lopencv_imgcodecs" >>config.mk -sudo make -j$(nproc) - -# Install MXNet Python package -cd python -sudo python setup.py install - -# Add MXNet path to ~/.bashrc file -echo "export PYTHONPATH=$MXNET_HOME/python:$PYTHONPATH" >> ~/.bashrc -source ~/.bashrc - -# Install graphviz for visualizing network graph and jupyter notebook to run tutorials and examples -sudo pip install graphviz -sudo pip install jupyter - -echo "Done! MXNet for Python installation is complete. Go ahead and explore MXNet with Python :-)" diff --git a/setup-utils/install-mxnet-fedora-python.sh b/setup-utils/install-mxnet-fedora-python.sh deleted file mode 100644 index 86116665db88..000000000000 --- a/setup-utils/install-mxnet-fedora-python.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -###################################################################### -# This script installs MXNet for Python along with all required dependencies on a Fedora Machine. -# Tested on Fedora 21.0 + distro. -###################################################################### -set -e - -MXNET_HOME="$HOME/mxnet/" -echo "MXNet root folder: $MXNET_HOME" - -echo "Installing basic development tools, atlas, opencv, pip, graphviz ..." -sudo yum update -sudo yum groupinstall -y "Development Tools" "Development Libraries" -sudo yum install -y atlas atlas-devel opencv opencv-devel graphviz graphviz-devel - -echo "Building MXNet core. This can take few minutes..." -cd "$MXNET_HOME" -cp make/config.mk . -make -j$(nproc) - -echo "Installing Numpy..." -sudo yum install numpy - -echo "Installing Python setuptools..." -sudo yum install -y python-setuptools python-pip - -echo "Adding MXNet path to your ~/.bashrc file" -echo "export PYTHONPATH=$MXNET_HOME/python:$PYTHONPATH" >> ~/.bashrc -source ~/.bashrc - -echo "Install Graphviz for plotting MXNet network graph..." -sudo pip install graphviz - -echo "Installing Jupyter notebook..." -sudo pip install jupyter - -echo "Done! MXNet for Python installation is complete. Go ahead and explore MXNet with Python :-)" diff --git a/setup-utils/install-mxnet-osx-python.sh b/setup-utils/install-mxnet-osx-python.sh deleted file mode 100755 index 0dde8096b315..000000000000 --- a/setup-utils/install-mxnet-osx-python.sh +++ /dev/null @@ -1,554 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# This scripts installs the dependencies and compiles -# MXNet source. -# -# The script also installs the MXNet package for Python. -# - -#set -ex - -export MXNET_GITPATH="https://github.com/apache/incubator-mxnet" - - -if [ -z ${MXNET_TAG} ]; -then - export MXNET_BRANCH_TAG="" -else - export MXNET_BRANCH_TAG="--branch $MXNET_TAG" -fi - -export TARIKH=`/bin/date +%Y-%m-%d-%H:%M:%S` -if [ -z ${MXNET_HOME} ]; -then - export MXNET_HOME="$HOME/mxnet" -fi -export MXNET_HOME_OLD="$HOME/mxnet_${TARIKH}" -export MXNET_LOG=${MXNET_HOME}/buildMXNet_mac.log - -# Insert the Homebrew directory at the top of your PATH environment variable -export PATH="$PATH:/usr/local/bin:/usr/local/sbin" # for brew -export PATH="$PATH:/usr/bin:/opt/local/bin" # for macports - -export MACPORTS_WEB="https://guide.macports.org/chunked/installing.macports.html" - -export BREW_PKGS="pkg-config python opencv graphviz homebrew/core/openblas" -export PORT_PKGS="pkgconfig python36 opencv graphviz openblas-devel" - -# graphviz, opencv-python skipped since already installed via brew/port -export PIP_PKGS_ALL="cython numpy" -export PIP_PKGS_USER="requests jupyter" - -export SLEEP_TIME=2 -LINE="########################################################################" - -print_intro_msg() { - # - # NOTE: Please test and ensure that the message does NOT scroll - # beyond the standard 80x25 format of a terminal shell. - # - echo $LINE - echo " " - echo "MXNet is a flexible, efficient and scalable library for Deep Learning." - echo " " - echo "This script installs MXNet on MacOS in \${MXNET_HOME}" - echo "If not set, the default value of \${MXNET_HOME} = ~/mxnet" - echo "The current value of \${MXNET_HOME} = ${MXNET_HOME}" - echo " " - echo "If this directory is already present, it is renamed to retain earlier contents." - echo "You may want to check and delete this directory if not required." - echo " " - echo "This script has been tested on: MacOS El Capitan (10.11) and Sierra (10.12)" - echo " " - echo "If you face any problems with this script, please let us know at:" - echo " https://discuss.mxnet.io/" - echo " " - echo "Typical run-time for this script is around 10 minutes." - echo "If your environment has never been setup for development (e.g. gcc), " - echo "it could take up to 30 minutes or longer." - echo " " - MACOS_VERSION=`/usr/bin/uname -r` - echo "Your macOS version is: $MACOS_VERSION" - echo " " - echo $LINE - read -p "Do you want to continue? (y/n): " response - echo " " - while true; do - case $response in - [Yy]* ) break;; - [Nn]* ) exit;; - * ) echo "Please answer yes or no.";; - esac - done - echo " " - echo " " - echo "MXNET GIT Path = ${MXNET_GITPATH}" - echo "MXNET Tag = ${MXNET_TAG}" - echo "You can set \$MXNET_TAG to the appropriate github repo tag" - echo "If not set, the default value used is the latest version on master" - read -p "Do you want to get a list of available tags? (y/n): " response - while true; do - case $response in - [Yy]* ) - echo "Available tags are:" - git ls-remote --tags ${MXNET_GITPATH} | sed 's/refs\/tags\///' | grep -v v | grep -v 201 \ - | grep -v "{}" | awk '{ print " ", $2 }'; - break;; - [Nn]* ) break;; - * ) echo "Please answer yes or no.";; - esac - done - read -p "Do you want to continue? (y/n): " response - echo " " - while true; do - case $response in - [Yy]* ) break;; - [Nn]* ) exit;; - * ) echo "Please answer yes or no.";; - esac - done -} # print_intro_msg() - -# wrapper routine to stop the script if the command invoked returns error -chkret() { - cmd=$* - echo "$cmd" - $cmd - ret=$? - if [[ ${ret} != 0 ]]; then - echo " " - echo "ERROR: Return value non-zero for: $cmd" - echo " " - exit 1 - fi -} # chkret() - -chk_mac_vers() { - export mac_vers=`sw_vers -productVersion | cut -d '.' -f 1,2` - if [[ $mac_vers != "10.11" && $mac_vers != "10.12" ]]; - then - echo " " - echo "ERROR: macOS version $mac_vers NOT supported." - echo " " - echo "Your macOS version is:" - sw_vers - echo " " - exit 1 - fi -} # chk_mac_vers() - -install_brew() { - echo " " - while true; do - echo "This script will install/update brew and " - echo "following dependent packages required for MXNet." - echo " Dependent brew packages: ${BREW_PKGS}" - echo " Dependent pip packages: ${PIP_PKGS_ALL} ${PIP_PKGS_USER}" - read -p "Do you want to continue? (y/n): " response - echo " " - case $response in - [Yy]* ) break;; - [Nn]* ) exit;; - * ) echo "Please answer yes or no.";; - esac - done - - echo " " - echo "BEGIN: Check/Install/Update Homebrew" - BREW_PATH=`which brew` - if [[ (-z ${BREW_PATH}) || (! -f ${BREW_PATH}) ]]; - then - yes '' | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - ret=$? - if [[ ${ret} != 0 ]]; then - echo " " - echo "ERROR: Return value non-zero for: homebrew installation using ruby" - echo " " - exit 1 - fi - else - chkret brew update - fi - echo "END: Check/Install/Update Homebrew" - echo $LINE - echo " " - - echo "BEGIN: Install dependent brew packages for MXNet: ${BREW_PKGS}" - - chkret brew tap homebrew/core - - # install each individually to see progress for each - for pkg in ${BREW_PKGS} - do - chkret brew_pkg_install ${pkg} - done - - echo "END: Install dependent brew packages for MXNet: ${BREW_PKGS}" - echo $LINE - echo " " -} # install_brew() - -brew_pkg_install () { - pkg=$1 - brew ls --versions $pkg > /dev/null - ret=$? - if [[ ${ret} != 0 ]]; then - echo "brew install $pkg" - brew install $pkg - else - echo "$pkg already installed" - fi -} # brew_pkg_install - -install_port () { - echo " " - while true; do - echo "This script will install/update port and " - echo "following dependent packages required for MXNet." - echo " Dependent port packages: ${PORT_PKGS}" - echo " Dependent pip packages: ${PIP_PKGS_ALL} ${PIP_PKGS_USER}" - read -p "Do you want to continue? (y/n): " response - echo " " - case $response in - [Yy]* ) break;; - [Nn]* ) exit;; - * ) echo "Please answer yes or no.";; - esac - done - - echo " " - echo "BEGIN: Check/Install/Update port" - MACPORTS_PATH=`which port` - if [[ (-z ${MACPORTS_PATH}) || (! -f ${MACPORTS_PATH}) ]]; - then - echo " " - echo "ERROR: Please install port for your macOS version from:" - echo " " - echo $MACPORTS_WEB - echo " " - exit 1 - else - echo "NOTE: Updating port if required" - export SLEEP_TIME=2 - sudo port upgrade outdated - echo " " - echo "port version is:" - port version - echo " " - fi - echo "END: Check/Install/Update port" - echo $LINE - echo " " - - echo "BEGIN: Install dependent port packages for MXNet: ${PORT_PKGS}" - echo " " - #sudo port install python36-readline - # install each individually to see progress for each - for pkg in ${PORT_PKGS} - do - chkret sudo port install ${pkg} - done - if [[ ! -f /opt/local/include/cblas.h ]]; - then - sudo ln -s /opt/local/include/cblas_openblas.h /opt/local/include/cblas.h - fi - #if [[ ! -f /usr/local/opt/openblas/lib/libopenblas.a ]]; - #then - # sudo mkdir -p /usr/local/opt/openblas/lib - # sudo ln -s /opt/local/lib/libopenblas.a /usr/local/opt/openblas/lib/libopenblas.a - #fi - - echo " " - echo "END: Install dependent port packages for MXNet: ${PORT_PKGS}" - echo $LINE - echo " " -} # install_port - -install_mac_pkg_manager() { - BREW_PATH=`which brew` - if [[ (-z ${BREW_PATH}) || (! -f ${BREW_PATH}) ]]; - then - echo "NOTE: brew NOT installed" - export MAC_BREW=0 - else - echo "NOTE: brew installed" - export MAC_BREW=1 - export PKG_MGR="brew" - fi - - MACPORTS_PATH=`which port` - if [[ (-z ${MACPORTS_PATH}) || (! -f ${MACPORTS_PATH}) ]]; - then - echo "NOTE: port NOT installed" - export MAC_PORT=0 - else - echo "NOTE: port installed" - export MAC_PORT=1 - export PKG_MGR="port" - fi - - if [[ $MAC_PORT -eq 1 && $MAC_BREW -eq 1 ]]; - then - echo "NOTE: Both port and brew installed" - export MAC_PKG_ASK=1 - export PKG_MGR="" - elif [[ $MAC_PORT -eq 0 && $MAC_BREW -eq 0 ]]; - then - echo "NOTE: Neither port and brew installed" - export MAC_PKG_ASK=1 - export PKG_MGR="" - else - export MAC_PKG_ASK=0 - - while true; do - echo "NOTE: Using the already installed package manager: $PKG_MGR" - read -p "Do you want to continue? (y/n): " response - echo " " - case $response in - [Yy]* ) break;; - [Nn]* ) exit;; - * ) echo "Please answer yes or no.";; - esac - done - fi - - if [[ $MAC_PKG_ASK -eq 1 ]]; - then - export MAC_BREW=0 - export MAC_PORT=0 - while true; do - echo " " - echo "NOTE: This script supports Homebrew OR Port package manager." - echo " " - read -p "Which package manager do you want to use? (b/p): " pkg_mgr - echo " " - case $pkg_mgr in - [Bb]* ) export MAC_BREW=1; break;; - [Pp]* ) export MAC_PORT=1; break;; - * ) echo "Please answer: b or p";; - esac - done - fi - - if [[ $MAC_PORT -eq 1 ]]; - then - install_port - else - install_brew - fi -} # install_mac_pkg_manager - -install_dep_pip_for_mxnet() { - echo " " - echo "BEGIN: Install dependent pip packages for MXNet: " - echo "${PIP_PKGS_ALL} ${PIP_PKGS_USER}" - echo " " - - # NOTE: sudo used here - chkret sudo easy_install pip - chkret sudo pip install --upgrade pip - for pkg in ${PIP_PKGS_ALL} - do - chkret sudo pip install ${pkg} - done - #chkret sudo pip install --upgrade numpy - - # NOTE: no sudo used here - for pkg in ${PIP_PKGS_USER} - do - chkret pip install --user ${pkg} - done - - echo "END: Install dependent pip packages for MXNet: ${PIP_PKGS_ALL} ${PIP_PKGS_USER}" - echo $LINE - echo " " -} # install_dep_pip_for_mxnet() - -# check if mxnet is already installed through other means -chk_mxnet_installed() { - mxnet_installed=`pip list --format=columns | grep mxnet` - if [ "$mxnet_installed" != "" ] - then - mxnet_version=`echo $mxnet_installed | awk '{print $2}'` - echo "MXNet ${mxnet_version} is already installed." - echo "This installation might interfere with current installation attempt." - read -p "Do you want to remove installed version? (y/n): " response - while true; do - case $response in - [Yy]* ) - sudo -H pip uninstall mxnet - chk_mxnet_installed - break - ;; - [Nn]* ) - while true; do - read -p "Do you want to continue? (y/n): " response1 - echo " " - case $response1 in - [Yy]* ) break 2;; # break out of nested loop - [Nn]* ) exit;; - * ) echo "Please answer yes or no.";; - esac - done - ;; - * ) echo "Please answer yes or no.";; - esac - done - fi -} # chk_mxnet - -download_mxnet() { - echo " " - echo "BEGIN: Download MXNet" - if [ -d ${MXNET_HOME} ]; then - mv ${MXNET_HOME} ${MXNET_HOME_OLD} - echo " " - echo "Renamed directory ${MXNET_HOME} to ${MXNET_HOME_OLD}" - echo "You may want to check and delete this directory if not required." - echo " " - sleep ${SLEEP_TIME} - fi - - - chkret git clone ${MXNET_BRANCH_TAG} ${MXNET_GITPATH}.git ${MXNET_HOME} --recursive - sleep ${SLEEP_TIME} - cd ${MXNET_HOME} - echo " " - sleep ${SLEEP_TIME} - echo "END: Download MXNet" - echo $LINE - echo " " -} # download_mxnet - -compile_mxnet() { - # Compile MXNet: It assumes MXNet source is in ${MXNET_HOME} - echo "BEGIN: Compile MXNet" - cd ${MXNET_HOME} - chkret cp make/osx.mk ./config.mk.tmp - - touch ./config.mk - # rm any old setting of USE_BLAS, if present in config file - egrep -v "^USE_BLAS" ./config.mk.tmp >> ./config.mk - # add the new setting of USE_BLAS to the config file - echo "USE_BLAS = openblas" >> ./config.mk - - if [[ $MAC_PORT -eq 1 ]]; - then - echo "ADD_CFLAGS += -I/opt/local/lib" >> ./config.mk - echo "ADD_LDFLAGS += -L/opt/local/lib" >> ./config.mk - echo "ADD_LDFLAGS += -L/opt/local/lib/graphviz/" >> ./config.mk - else - echo "ADD_CFLAGS += -I/usr/local/opt/openblas/include" >> ./config.mk - echo "ADD_LDFLAGS += -L/usr/local/opt/openblas/lib" >> ./config.mk - echo "ADD_LDFLAGS += -L/usr/local/lib/graphviz/" >> ./config.mk - fi - echo " " - - echo "NOTE: The following compile-time configurations will be used." - echo " If you want to change any of them, edit the following file" - echo " in another terminal window and then press enter to continue." - echo " " - echo " ${MXNET_HOME}/config.mk" - echo " " - echo $LINE - # remove commented and blank lines - egrep -v "^#" ${MXNET_HOME}/config.mk | egrep -v "^$" - echo $LINE - echo " " - read -p "Press enter to continue ..." - echo " " - echo "Running Make" - echo " " - chkret make -j$(sysctl -n hw.ncpu) - echo "END: Compile MXNet" - sleep ${SLEEP_TIME} - echo $LINE - echo " " -} # compile_mxnet - -install_mxnet_python() { - echo " " - echo "BEGIN: Install MXNet package for Python" - chkret cd ${MXNET_HOME}/python - chkret sudo python setup.py install - echo "END: Install MXNet package for Python" - sleep ${SLEEP_TIME} - echo $LINE - echo " " -} # install_mxnet_python - - -test_mxnet_python() { - echo "BEGIN: Test MXNet" - rm -f mxnet_test.log - python << END > mxnet_test.log -import mxnet as mx -a = mx.nd.ones((2, 3)); -print ((a*2).asnumpy()); -END - rm -f mxnet_test.expected - cat << END > mxnet_test.expected -[[2. 2. 2.] - [2. 2. 2.]] -END - diff mxnet_test.log mxnet_test.expected - if [[ $? = 0 ]]; then - echo " " - echo "SUCCESS: MXNet test passed" - echo "SUCCESS: MXNet is successfully installed and works fine!" - export MXNET_VERSION=`echo "import mxnet as mx; print(mx.__version__)" | python` - echo "SUCCESS: MXNet Version is: $MXNET_VERSION" - echo "END: Test MXNet" - echo ":-)" - echo " " - echo "FYI : You can fine-tune MXNet run-time behavior using environment variables described at:" - echo " https://mxnet.apache.org/api/faq/env_var" - echo " " - echo "NEXT: Try the tutorials at: https://mxnet.io/api" - echo " " - echo $LINE - echo " " - rm -f mxnet_test.log mxnet_test.expected - return 0 - else - echo " " - echo "ERROR: Following files differ: mxnet_test.log mxnet_test.expected" - echo "ERROR: MXNet test failed" - echo "END: Test MXNet" - echo " " - echo ":-(" - exit 1 - fi -} # test_mxnet_python() - -main() { - print_intro_msg - chk_mac_vers - install_mac_pkg_manager - install_dep_pip_for_mxnet - chk_mxnet_installed - download_mxnet - compile_mxnet - install_mxnet_python - test_mxnet_python -} # main - -main diff --git a/setup-utils/install-mxnet-ubuntu-python.sh b/setup-utils/install-mxnet-ubuntu-python.sh deleted file mode 100644 index 8aa0d0256a79..000000000000 --- a/setup-utils/install-mxnet-ubuntu-python.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -###################################################################### -# This script installs MXNet for Python along with all required dependencies on a Ubuntu Machine. -# Tested on Ubuntu 14.0 + distro. -###################################################################### -set -e - -MXNET_HOME="$HOME/mxnet/" -echo "MXNet root folder: $MXNET_HOME" - -echo "Installing build-essential, libatlas-base-dev, libopencv-dev, pip, graphviz ..." -sudo apt-get update -sudo apt-get install -y build-essential libatlas-base-dev libopencv-dev graphviz - -echo "Building MXNet core. This can take few minutes..." -cd "$MXNET_HOME" -make -j$(nproc) - -echo "Installing Numpy..." -sudo apt-get install python-numpy - -echo "Installing Python setuptools pip..." -sudo apt-get install -y python-setuptools python-pip - -echo "Updating pip..." -sudo pip install -U pip - -echo "Installing Python package for MXNet..." -cd python; sudo python setup.py install - -echo "Adding MXNet path to your ~/.bashrc file" -echo "export PYTHONPATH=$MXNET_HOME/python:$PYTHONPATH" >> ~/.bashrc -source ~/.bashrc - -echo "Install Graphviz for plotting MXNet network graph..." -sudo pip install graphviz - -echo "Installing Jupyter notebook..." -sudo pip install jupyter - -echo "Done! MXNet for Python installation is complete. Go ahead and explore MXNet with Python :-)" diff --git a/setup-utils/install-mxnet-ubuntu-r.sh b/setup-utils/install-mxnet-ubuntu-r.sh deleted file mode 100644 index a07d9d8ef10e..000000000000 --- a/setup-utils/install-mxnet-ubuntu-r.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -###################################################################### -# This script installs MXNet for R along with all required dependencies on a Ubuntu Machine. -# Tested on Ubuntu 14.04+ distro. -###################################################################### -set -e - -MXNET_HOME="$HOME/mxnet/" -echo "MXNet root folder: $MXNET_HOME" - -echo "Building MXNet core. This can take few minutes..." -cd "$MXNET_HOME" -make -j$(nproc) - -echo "Installing R dependencies. This can take few minutes..." - -# make sure we have essential tools installed -is_rscript_installed=$(which Rscript | wc -l) -if [ "$is_rscript_installed" = "0" ]; then - read -p "Seems like Rscript is not installed. Install Rscript? [Y/n]" - if [ x"$REPLY" = x"" -o x"$REPLY" = x"y" -o x"$REPLY" = x"Y" ]; then - sudo add-apt-repository -y "deb http://cran.rstudio.com/bin/linux/ubuntu `lsb_release -cs`/" - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9 - sudo apt-get -qq update - sudo apt-get install -y r-base-core - fi -fi - -# libcurl4-openssl-dev and libssl-dev are needed for devtools. -sudo apt-get -y install libcurl4-openssl-dev libssl-dev - -# Needed for R XML -sudo apt-get install libxml2-dev - -# Needed for R Cairo -sudo apt-get install libxt-dev - -sudo Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')" -cd R-package -sudo Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cran.rstudio.com')); install_deps(dependencies = TRUE)" -cd .. - -echo "Compiling R package. This can take few minutes..." -sudo make -f R-package/Makefile rpkg - -echo "Installing R package..." -sudo R CMD INSTALL mxnet_current_r.tar.gz - -echo "Done! MXNet for R installation is complete. Go ahead and explore MXNet with R :-)" diff --git a/setup-utils/install-mxnet-virtualenv.sh b/setup-utils/install-mxnet-virtualenv.sh deleted file mode 100755 index 5e00f79647ea..000000000000 --- a/setup-utils/install-mxnet-virtualenv.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -###################################################################### -# This script installs MXNet for Python in a virtualenv on OSX and ubuntu -###################################################################### -set -e -#set -x - -BUILDIR=build -VENV=mxnet_py3 - -setup_virtualenv() { - if [ ! -d $VENV ];then - virtualenv -p `which python3` $VENV - fi - source $VENV/bin/activate -} - -gpu_count() { - nvidia-smi -L | wc -l -} - -detect_platform() { - unameOut="$(uname -s)" - case "${unameOut}" in - Linux*) - distro=$(awk -F= '/^NAME/{gsub(/"/, "", $2); print $2}' /etc/os-release) - machine="Linux/$distro" - ;; - Darwin*) machine=Mac;; - CYGWIN*) machine=Cygwin;; - MINGW*) machine=MinGw;; - *) machine="UNKNOWN:${unameOut}" - esac - echo ${machine} -} - - -if [ $(gpu_count) -ge 1 ];then - USE_CUDA=ON -else - USE_CUDA=OFF -fi - -PLATFORM=$(detect_platform) -echo "Detected platform '$PLATFORM'" - -if [ $PLATFORM = "Mac" ];then - USE_OPENMP=OFF -else - USE_OPENMP=ON -fi - -if [ $PLATFORM = "Linux/Ubuntu" ];then - install_dependencies_ubuntu() { - sudo apt-get update - sudo apt-get install -y build-essential libatlas-base-dev libopencv-dev graphviz virtualenv cmake\ - ninja-build libopenblas-dev liblapack-dev python3 python3-dev - } - echo "Installing build dependencies in Ubuntu!" - install_dependencies_ubuntu -fi - -echo "Preparing a Python virtualenv in ${VENV}" -setup_virtualenv - -echo "Building MXNet core. This can take a few minutes..." -build_mxnet() { - pushd . - set -x - mkdir -p $BUILDIR && cd $BUILDIR - cmake -DUSE_CUDA=$USE_CUDA -DUSE_OPENCV=ON -DUSE_OPENMP=$USE_OPENMP -DUSE_SIGNAL_HANDLER=ON -DCMAKE_BUILD_TYPE=Release -GNinja .. - ninja - set +x - popd -} - - -build_mxnet - -echo "Installing mxnet under virtualenv ${VENV}" -install_mxnet() { - pushd . - cd python - pip3 install -e . - pip3 install opencv-python matplotlib graphviz jupyter ipython - popd -} - -install_mxnet - -echo " - -======================================================================================== -Done! MXNet for Python installation is complete. Go ahead and explore MXNet with Python. -======================================================================================== - -Use the following command to enter the virtualenv: -$ source ${VENV}/bin/activate -$ iptyhon - -You can then start using mxnet - -import mxnet as mx -x = mx.nd.ones((5,5)) -" diff --git a/setup-utils/install-mxnet-windows-python.bat b/setup-utils/install-mxnet-windows-python.bat deleted file mode 100644 index 021baaeff331..000000000000 --- a/setup-utils/install-mxnet-windows-python.bat +++ /dev/null @@ -1,295 +0,0 @@ -rem Licensed to the Apache Software Foundation (ASF) under one -rem or more contributor license agreements. See the NOTICE file -rem distributed with this work for additional information -rem regarding copyright ownership. The ASF licenses this file -rem to you under the Apache License, Version 2.0 (the -rem "License"); you may not use this file except in compliance -rem with the License. You may obtain a copy of the License at -rem -rem http://www.apache.org/licenses/LICENSE-2.0 -rem -rem Unless required by applicable law or agreed to in writing, -rem software distributed under the License is distributed on an -rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -rem KIND, either express or implied. See the License for the -rem specific language governing permissions and limitations -rem under the License. - -@echo off -setlocal -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -:::: This script setup directories, dependencies for MXNET :::: -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - - -:::: Customizable variables :::: - - -:: conda environment name for MXNET, default to MXNET-vcversion -REM set MXNET_CONDA_ENV=mxnet - -:: which to find MKL, default to openblas installed by conda -:: mkl: download from https://software.intel.com/intel-mkl, install and set following two variables -REM set INTEL_MKL_DIR=D:\\Intel\\SWTools\\compilers_and_libraries\\windows\\mkl\\ - -:: where to find cudnn library -REM set CUDNN_ROOT=D:\NVIDIA\CUDNN\v5.1\ - -:::: End of customization :::: - - -set ECHO_PREFIX=+++++++ - -set MXNET_SETUP_HAS_CUDA=0 -set MXNET_SETUP_HAS_CUDNN=0 -:::: validate msvc version :::: - -if "%VisualStudioVersion%" == "" ( - if not "%VS140COMNTOOLS%" == "" ( call "%VS140COMNTOOLS%..\..\VC\vcvarsall.bat" x64 && goto :VS_SETUP) -REM Not Supported yet due to dependencies -REM if not "%VS120COMNTOOLS%" == "" ( call "%VS120COMNTOOLS%..\..\VC\vcvarsall.bat" x64 && goto :VS_SETUP) -REM if not "%VS110COMNTOOLS%" == "" ( call "%VS110COMNTOOLS%..\..\VC\vcvarsall.bat" x64 && goto :VS_SETUP) -REM if not "%VS100COMNTOOLS%" == "" ( call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" x64 && goto :VS_SETUP) -REM if not "%VS90COMNTOOLS%" == "" ( call "%VS90COMNTOOLS%..\..\VC\vcvarsall.bat" x64 && goto :VS_SETUP) -) -:VS_SETUP - -if "%VisualStudioVersion%" == "" ( - echo %ECHO_PREFIX% Can not find environment variable VisualStudioVersion, msvc is not setup porperly - echo %ECHO_PREFIX% Please Install Visual Studio from: - echo %ECHO_PREFIX% https://go.microsoft.com/fwlink/?LinkId=691978&clcid=0x409 - goto :FAIL -) - -set MXNET_VS_VERSION=%VisualStudioVersion:.0=% - -if "%PreferredToolArchitecture%" == "x64" ( - if "%CommandPromptType%" == "Cross" ( - if "%Platform%" == "ARM" set MXNET_VS_PLATFORM=amd64_arm - if "%Platform%" == "X86" set MXNET_VS_PLATFORM=amd64_x86 - ) -) else ( - if "%CommandPromptType%" == "Cross" ( - if "%Platform%" == "ARM" set MXNET_VS_PLATFORM=x86_arm - if "%Platform%" == "x64" set MXNET_VS_PLATFORM=x86_amd64 - ) - if "%CommandPromptType%" == "Native" ( - if "%Platform%" == "X64" set MXNET_VS_PLATFORM=x64 - ) - if "%Platform%" == "" set MXNET_VS_PLATFORM=x86 -) - -if "%MXNET_VS_PLATFORM%" == "x86" set MXNET_VS_TARGET=x86 -if not "%MXNET_VS_PLATFORM%" == "%MXNET_VS_PLATFORM:_x86=%" set MXNET_VS_TARGET=x86 -if not "%MXNET_VS_PLATFORM%" == "%MXNET_VS_PLATFORM:_arm=%" set MXNET_VS_TARGET=arm -if "%MXNET_VS_TARGET%" == "" set MXNET_VS_TARGET=x64 - -:::: Setup directories :::: - -set MXNET_DISTRO=%~dp0. -set MXNET_DISTRO=%MXNET_DISTRO%\.. -if "%MXNET_INSTALL_DIR%" == "" set MXNET_INSTALL_DIR=%MXNET_DISTRO%\build - -echo %ECHO_PREFIX% MXNET will be installed under %MXNET_INSTALL_DIR%, vs%MXNET_VS_VERSION% %MXNET_VS_PLATFORM% - -:::: Setup dependencies :::: - -:: has blas/lapack? -if exist %INTEL_MKL_DIR% set MXNET_BLAS=MKL -if "%MXNET_BLAS%" == "" set MXNET_BLAS=Open - -:: has cuda? -for /f "delims=" %%i in ('where nvcc') do ( - set NVCC_CMD=%%i - goto :AFTER_NVCC -) -:AFTER_NVCC -if not "%NVCC_CMD%" == "" set MXNET_SETUP_HAS_CUDA=1 - -:: has cudnn -if exist %CUDNN_ROOT% set MXNET_SETUP_HAS_CUDNN=1 - -:: has conda? -for /f "delims=" %%i in ('where conda') do ( - set CONDA_CMD=%%i - goto :AFTER_CONDA -) -:AFTER_CONDA - -if "%CONDA_CMD%" == "" ( - echo %ECHO_PREFIX% Can not find conda, some dependencies can not be resolved - echo %ECHO_PREFIX% Please install Miniconda with Python 3.5 from here: - echo %ECHO_PREFIX% http://conda.pydata.org/miniconda.html - - goto :FAIL -) - -set MXNET_CONDA_INFO=%TEMP%\check_conda_info_for_MXNET.txt -conda info > %MXNET_CONDA_INFO% -if "%MXNET_VS_TARGET%" == "x64" set MXNET_CONDA_PLATFORM=win-64 -if "%MXNET_VS_TARGET%" == "arm" set MXNET_CONDA_PLATFORM=win-64 -if "%MXNET_VS_TARGET%" == "x86" set MXNET_CONDA_PLATFORM=win-32 - -findstr "%MXNET_CONDA_PLATFORM%" "%MXNET_CONDA_INFO%" >nul -if errorlevel 1 ( - echo %ECHO_PREFIX% %MXNET_VS_TARGET% MXNET requires %MXNET_CONDA_PLATFORM% conda, installation will continue without conda - goto :NO_CONDA -) - -if %MXNET_VS_VERSION% GEQ 14 ( set CONDA_VS_VERSION=14&& goto :CONDA_SETUP ) -if %MXNET_VS_VERSION% GEQ 10 ( set CONDA_VS_VERSION=10&& goto :CONDA_SETUP ) -set CONDA_VS_VERSION=9 - -:CONDA_SETUP - -if "%MXNET_CONDA_ENV%" == "" set MXNET_CONDA_ENV=mxnet-vc%CONDA_VS_VERSION% - -echo %ECHO_PREFIX% Createing conda environment '%MXNET_CONDA_ENV%' for MXNET dependencies -conda create -n %MXNET_CONDA_ENV% -c conda-forge vc=%CONDA_VS_VERSION% --yes - -set CONDA_DIR=%CONDA_CMD:\Scripts\conda.exe=% -set MXNET_CONDA_LIBRARY=%CONDA_DIR%\envs\%MXNET_CONDA_ENV%\Library -set MXNET_CONDA_LIBRARY=%MXNET_CONDA_LIBRARY:\=\\% -set PATH=%MXNET_CONDA_LIBRARY:\\=\%\bin;%PATH%; -set NEW_PATH=%CONDA_DIR%\Scripts;%MXNET_CONDA_LIBRARY%\bin;%NEW_PATH% - -set MXNET_CONDA_PKGS=%TEMP%\check_conda_packages_for_MXNET.txt -conda list -n %MXNET_CONDA_ENV% > %MXNET_CONDA_PKGS% - -:: has cmake? -for /f "delims=" %%i in ('where cmake') do ( - set CMAKE_CMD=%%i - goto :AFTER_CMAKE -) -if "%CMAKE_CMD%" == "" ( - echo %ECHO_PREFIX% Installing cmake by conda - conda install -n %MXNET_CONDA_ENV% -c conda-forge cmake --yes -) -:AFTER_CMAKE - -:: has patch? -for /f "delims=" %%i in ('where patch') do ( - set PATCH_CMD=%%i - goto :AFTER_PATCH -) -if "%PATCH_CMD%" == "" ( - echo %ECHO_PREFIX% Installing patch by conda - conda install -n %MXNET_CONDA_ENV% patch --yes -) -:AFTER_PATCH - -:: need openblas? -if "%MXNET_BLAS%" == "Open" goto :CONDA_INSTALL_OPENBLAS -goto :AFTER_OPENBLAS - -:CONDA_INSTALL_OPENBLAS -findstr "openblas" "%MXNET_CONDA_PKGS%" >nul -if errorlevel 1 ( - echo %ECHO_PREFIX% Installing openblas by conda - if "%MXNET_VS_TARGET%" == "x64" conda install -n %MXNET_CONDA_ENV% -c ukoethe openblas --yes || goto :Fail - CALL :PATCH_OPENBLAS -) - -if "%MXNET_VS_TARGET%" == "x64" ( - if "%OpenBLAS_INCLUDE_DIR%" == "" set OpenBLAS_INCLUDE_DIR=%MXNET_CONDA_LIBRARY%\\include\\ - if "%OpenBLAS_LIB%" == "" set OpenBLAS_LIB=%MXNET_CONDA_LIBRARY%\\lib\\libopenblas.lib -) -:AFTER_OPENBLAS - -if "%MXNET_BLAS%" == "MKL" goto:CONDA_INSTALL_MKL -goto :AFTER_MKL - -:CONDA_INSTALL_MKL -if "%MKL_INCLUDE_DIR%" == "" set MKL_INCLUDE_DIR=%INTEL_MKL_DIR%\\include\\ -if "%MKL_LIB%" == "" set MKL_LIB=%INTEL_MKL_DIR%\\lib\\ -:AFTER_MKL - -:: other dependencies -findstr "opencv" "%MXNET_CONDA_PKGS%" >nul -if errorlevel 1 set MXNET_DEPENDENCIES=%MXNET_DEPENDENCIES% opencv -findstr "numpy" "%MXNET_CONDA_PKGS%" >nul -if errorlevel 1 set MXNET_DEPENDENCIES=%MXNET_DEPENDENCIES% numpy -findstr "cython" "%MXNET_CONDA_PKGS%" >nul -if errorlevel 1 set MXNET_DEPENDENCIES=%MXNET_DEPENDENCIES% cython -findstr "numpy" "%MXNET_CONDA_PKGS%" >nul -if errorlevel 1 set MXNET_DEPENDENCIES=%MXNET_DEPENDENCIES% numpy - -if not "%MXNET_DEPENDENCIES%" == "" ( - echo %ECHO_PREFIX% Installing %MXNET_DEPENDENCIES% by conda for MXNET - conda install -n %MXNET_CONDA_ENV% -c conda-forge %MXNET_DEPENDENCIES% --yes -) - -:: make symbolic link to workaround opencv cmakefile bug -echo %ECHO_PREFIX% Install symblic link for opencv -mkdir %CONDA_DIR%\envs\x64\vc14\ -mklink /D %CONDA_DIR%\envs\x64\vc14\lib %MXNET_CONDA_LIBRARY:\\=\%\lib -mklink /D %CONDA_DIR%\envs\x64\vc14\bin %MXNET_CONDA_LIBRARY:\\=\%\bin - -:NO_CONDA -if exist "%MXNET_CONDA_INFO%" del /q %MXNET_CONDA_INFO% -if exist "%MXNET_CONDA_PKGS%" del /q %MXNET_CONDA_PKGS% - -:::: download graphviz :::: - -REM echo %ECHO_PREFIX% Downloading graphviz for graph package -REM cd %MXNET_DISTRO%\build -REM wget -nc https://github.com/mahkoCosmo/GraphViz_x64/raw/master/graphviz-2.38_x64.tar.gz --no-check-certificate -O graphviz-2.38_x64.tar.gz -REM 7z x graphviz-2.38_x64.tar.gz -y && 7z x graphviz-2.38_x64.tar -ographviz-2.38_x64 -y >NUL -REM if not exist %MXNET_INSTALL_BIN%\graphviz md %MXNET_INSTALL_BIN%\graphviz -REM copy /y %MXNET_DISTRO%\build\graphviz-2.38_x64\bin\ %MXNET_INSTALL_BIN%\graphviz\ - -REM set NEW_PATH=%NEW_PATH%;%MXNET_INSTALL_BIN%\graphviz - -echo %ECHO_PREFIX% Build libmxnet.dll -cd /D %MXNET_DISTRO% - -SET CMAKE_OPT=%CMAKE_OPT% -DUSE_CUDA=%MXNET_SETUP_HAS_CUDA% -SET CMAKE_OPT=%CMAKE_OPT% -DUSE_CUDNN=%MXNET_SETUP_HAS_CUDNN% -if %MXNET_SETUP_HAS_CUDNN%=="1" SET CMAKE_OPT=%CMAKE_OPT% -DCUDNN_ROOT="%CUDNN_ROOT%" -SET CMAKE_OPT=%CMAKE_OPT% -DBLAS="%MXNET_BLAS%" -if "%MXNET_BLAS%"=="Open" SET CMAKE_OPT=%CMAKE_OPT% -DOpenBLAS_HOME="%MXNET_CONDA_LIBRARY%" -DOpenBLAS_INCLUDE_DIR="%OpenBLAS_INCLUDE_DIR%" -DOpenBLAS_LIB="%OpenBLAS_LIB%" -if "%MXNET_BLAS%"=="MKL" SET CMAKE_OPT=%CMAKE_OPT% -DMKL_HOME="%INTEL_MKL_DIR%" -DMKL_INCLUDE_DIR="%MKL_INCLUDE_DIR%" -DMKL_LIB="%MKL_LIB%" - -SET CMAKE_OPT=%CMAKE_OPT% -DOpenCV_LIB_PATH="%MXNET_CONDA_LIBRARY%\\lib\\" -DOpenCV_INCLUDE_DIRS="%MXNET_CONDA_LIBRARY%\\include\\" -DOpenCV_CONFIG_PATH="%MXNET_CONDA_LIBRARY%" - -cmake -Wno-dev %CMAKE_OPT% -DCMAKE_PREFIX_PATH="%MXNET_CONDA_LIBRARY%" -G "Visual Studio 14 2015 Win64" -DUSE_PROFILER=1 -DCMAKE_BUILD_TYPE=Release -H. -Bbuild -if errorlevel 1 goto :FAIL -msbuild build\mxnet.sln /t:Build /p:Configuration=Release;Platform=x64 /m -if errorlevel 1 goto :FAIL - -echo %ECHO_PREFIX% Install libmxnet.dll -cd /D %MXNET_DISTRO%\python -python setup.py install -if errorlevel 1 goto :FAIL - -set NEW_PATH=%NEW_PATH:\\=\% -echo %ECHO_PREFIX% Setup succeed! -echo %ECHO_PREFIX% Add the following path to your system path or Run env.cmd per cmd window -echo %NEW_PATH% -echo @SET PATH=%%PATH%%;%NEW_PATH%>%MXNET_DISTRO%\env.cmd -goto :END - -:PATCH_OPENBLAS -echo %ECHO_PREFIX% Apply patch to fix openblas 2.15 - -echo --- openblas_config.h 2016-12-20 11:09:11.722445000 +0800>%TEMP%\openblas.diff -echo +++ openblas_config.h 2016-12-20 11:01:21.347244600 +0800>>%TEMP%\openblas.diff -echo @@ -109,7 +109,7 @@>>%TEMP%\openblas.diff -echo structure as fallback (see Clause 6.2.5.13 of the C99 standard). */>>%TEMP%\openblas.diff -echo #if (defined(__STDC_IEC_559_COMPLEX__) ^|^| __STDC_VERSION__ ^>= 199901L ^|^| \>>%TEMP%\openblas.diff -echo (__GNUC__ ^>= 3 ^&^& !defined(__cplusplus)) ^|^| \>>%TEMP%\openblas.diff -echo - _MSC_VER ^>= 1800) // Visual Studio 2013 supports complex>>%TEMP%\openblas.diff -echo + (_MSC_VER ^>= 1800 ^&^& !defined(__cplusplus))) // Visual Studio 2013 supports complex>>%TEMP%\openblas.diff -echo #define OPENBLAS_COMPLEX_C99>>%TEMP%\openblas.diff -echo #ifndef __cplusplus>>%TEMP%\openblas.diff -echo #include ^>>%TEMP%\openblas.diff -cd /D %MXNET_CONDA_LIBRARY:\\=\%\include -patch -p0 -i %TEMP%\openblas.diff -DEL %TEMP%\openblas.diff -GOTO :eof - -:FAIL -echo %ECHO_PREFIX% Setup fail! - -:END - diff --git a/tests/cpp/unittest.mk b/tests/cpp/unittest.mk deleted file mode 100644 index 704ee41fdc4c..000000000000 --- a/tests/cpp/unittest.mk +++ /dev/null @@ -1,81 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -TEST_SRC = $(shell find tests/cpp/ -name "*.cc") -TEST_OBJ = $(patsubst %.cc, build/%.o, $(TEST_SRC)) -TEST = build/tests/cpp/mxnet_unit_tests - -GTEST_DIR=3rdparty/googletest/googletest/ -GTEST_INC=3rdparty/googletest/googletest/include/ -GTEST_SRCS_ = $(GTEST_DIR)/src/*.cc $(GTEST_DIR)/src/*.h $(GTEST_HEADERS) -GTEST_HEADERS = $(GTEST_DIR)/include/gtest/*.h \ - $(GTEST_DIR)/include/gtest/internal/*.h - -TEST_CFLAGS = -Itests/cpp/include -Isrc $(CFLAGS) -TEST_LDFLAGS = $(LDFLAGS) -Llib -lmxnet - -ifeq ($(USE_BREAKPAD), 1) -TEST_CFLAGS += -I/usr/local/include/breakpad -TEST_LDFLAGS += -lbreakpad_client -lbreakpad -endif - -TEST_LIB_DEP = gtest.a -ifeq ($(USE_MKLDNN), 1) - TEST_LIB_DEP += $(MKLDNNROOT)/lib/libdnnl.a -endif - -.PHONY: runtest testclean - -gtest-all.o : $(GTEST_SRCS_) - $(CXX) -std=c++17 $(CPPFLAGS) -I$(GTEST_INC) -I$(GTEST_DIR) $(CXXFLAGS) -c $(GTEST_DIR)/src/gtest-all.cc - -gtest.a : gtest-all.o - $(AR) $(ARFLAGS) $@ $^ - -build/tests/cpp/%.o : tests/cpp/%.cc | mkldnn - @mkdir -p $(@D) - $(CXX) -std=c++17 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/$* $< > build/tests/cpp/$*.d - $(CXX) -c -std=c++17 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/$*.o $(filter %.cc %.a, $^) - -build/tests/cpp/operator/%.o : tests/cpp/operator/%.cc | mkldnn - @mkdir -p $(@D) - $(CXX) -std=c++17 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/operator/$* $< > build/tests/cpp/operator/$*.d - $(CXX) -c -std=c++17 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/operator/$*.o $(filter %.cc %.a, $^) - -build/tests/cpp/storage/%.o : tests/cpp/storage/%.cc | mkldnn - @mkdir -p $(@D) - $(CXX) -std=c++17 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/storage/$* $< > build/tests/cpp/storage/$*.d - $(CXX) -c -std=c++17 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/storage/$*.o $(filter %.cc %.a, $^) - -build/tests/cpp/engine/%.o : tests/cpp/engine/%.cc | mkldnn - @mkdir -p $(@D) - $(CXX) -std=c++17 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/engine/$* $< > build/tests/cpp/engine/$*.d - $(CXX) -c -std=c++17 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/engine/$*.o $(filter %.cc %.a, $^) - -$(TEST): $(TEST_OBJ) lib/libmxnet.so $(TEST_LIB_DEP) - $(CXX) -std=c++17 $(TEST_CFLAGS) -I$(GTEST_INC) -o $@ $^ $(TEST_LDFLAGS) - -runtest: $(TEST) - LD_LIBRARY_PATH=$(shell pwd)/lib:$(LD_LIBRARY_PATH) $(TEST) - -testclean: - rm -f $(TEST) $(TEST_OBJ) - --include build/tests/cpp/*.d --include build/tests/cpp/operator/*.d --include build/tests/cpp/storage/*.d --include build/tests/cpp/engine/*.d diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh index dad566ff877d..12cad3ee2037 100755 --- a/tools/dependencies/openblas.sh +++ b/tools/dependencies/openblas.sh @@ -19,39 +19,22 @@ # This script builds the static library of openblas that can be used as dependency of mxnet. set -ex -OPENBLAS_VERSION=4a4c50a7cef9fa91f14e508722f502d956ad5192 -if [[ ((! -e $DEPS_PATH/lib/libopenblas.a) && -z "$CMAKE_STATICBUILD") || - ((! -e $DEPS_PATH/lib/libopenblas.so) && -v CMAKE_STATICBUILD) ]]; then - # download and build openblas +OPENBLAS_VERSION=0.3.10 +if [[ (! -e $DEPS_PATH/lib/libopenblas.a) ]]; then >&2 echo "Building openblas..." download \ - https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.zip \ + https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.zip \ ${DEPS_PATH}/openblas.zip unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH pushd . cd $DEPS_PATH/OpenBLAS-${OPENBLAS_VERSION} # Adding NO_DYNAMIC=1 flag causes make install to fail - CXX="g++ -fPIC" CC="gcc -fPIC" $MAKE DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 USE_OPENMP=1 - - if [[ -v CMAKE_STATICBUILD ]]; then - # We link and redistribute libopenblas.so for cmake staticbuild - # cf https://gitlab.kitware.com/cmake/cmake/issues/16221#note_143330 - patchelf --set-rpath '$ORIGIN' --force-rpath libopenblas.so - fi + CFLAGS="-fPIC" CXXFLAGS="-fPIC" $MAKE DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 USE_OPENMP=1 + patchelf --set-rpath '$ORIGIN' --force-rpath libopenblas.so $MAKE PREFIX=$DEPS_PATH install - - if [[ -z "$CMAKE_STATICBUILD" ]]; then - # Manually removing .so to avoid linking against it - rm $DEPS_PATH/lib/libopenblas*.so - fi - popd - if [[ -z "$CMAKE_STATICBUILD" ]]; then - ln -s libopenblas.a $DEPS_PATH/lib/libcblas.a - ln -s libopenblas.a $DEPS_PATH/lib/liblapack.a - fi fi diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh index 139b68c5bef0..fce8c1547757 100755 --- a/tools/dependencies/opencv.sh +++ b/tools/dependencies/opencv.sh @@ -46,7 +46,7 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build pushd . cd $DEPS_PATH/opencv-$OPENCV_VERSION/build - CXX="g++ -fPIC" CC="gcc -fPIC" cmake \ + CFLAGS="-fPIC" CXXFLAGS="-fPIC" cmake \ -D OPENCV_ENABLE_NONFREE=OFF \ -D WITH_1394=OFF \ -D WITH_ARAVIS=OFF \ diff --git a/tools/setup_gpu_build_tools.sh b/tools/setup_gpu_build_tools.sh deleted file mode 100755 index e10d1f1570fb..000000000000 --- a/tools/setup_gpu_build_tools.sh +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This script installs the tools and libraries for CUDA GPU on Ubuntu. -# Usage: VARIANT=cu102mkl; DEPS_PATH=$HOME; setup_gpu_build_tools.sh $VARIANT $DEPS_PATH; -# It installs the tools into DEPS_PATH as specified by the second argument, and will set -# the following environment variables: -# PATH, CPLUS_INCLUDE_PATH, C_INCLUDE_PATH, LIBRARY_PATH, LD_LIBRARY_PATH, NVCC - -VARIANT=$1 -DEPS_PATH=$2 - ->&2 echo "Setting CUDA versions for $VARIANT" -if [[ $VARIANT == cu102* ]]; then - CUDA_VERSION='10.2.89-1' - CUDA_PATCH_VERSION='10.2.2.89-1' - LIBCUDA_VERSION='440.33.01-0ubuntu1' - LIBCUDNN_VERSION='7.6.5.32-1+cuda10.2' - LIBNCCL_VERSION='2.5.6-1+cuda10.2' -elif [[ $VARIANT == cu101* ]]; then - CUDA_VERSION='10.1.105-1' - CUDA_PATCH_VERSION='10.1.0.105-1' - LIBCUDA_VERSION='418.39-0ubuntu1' - LIBCUDNN_VERSION='7.6.5.32-1+cuda10.1' - LIBNCCL_VERSION='2.5.6-1+cuda10.1' -elif [[ $VARIANT == cu100* ]]; then - CUDA_VERSION='10.0.130-1' - CUDA_PATCH_VERSION='10.0.130-1' - LIBCUDA_VERSION='410.48-0ubuntu1' - LIBCUDNN_VERSION='7.6.5.32-1+cuda10.0' - LIBNCCL_VERSION='2.5.6-1+cuda10.0' -elif [[ $VARIANT == cu92* ]]; then - CUDA_VERSION='9.2.148-1' - CUDA_PATCH_VERSION='9.2.148.1-1' - LIBCUDA_VERSION='396.44-0ubuntu1' - LIBCUDNN_VERSION='7.6.5.32-1+cuda9.2' - LIBNCCL_VERSION='2.4.8-1+cuda9.2' -if [[ $VARIANT == cu* ]]; then - CUDA_MAJOR_VERSION=$(echo $CUDA_VERSION | tr '-' '.' | cut -d. -f1,2) - CUDA_MAJOR_DASH=$(echo $CUDA_VERSION | tr '-' '.' | cut -d. -f1,2 | tr '.' '-') - CUDA_PATCH_MAJOR_DASH=$(echo $CUDA_PATCH_VERSION | tr '-' '.' | cut -d. -f1,2 | tr '.' '-') - NVIDIA_MAJOR_VERSION=$(echo $LIBCUDA_VERSION | cut -d. -f1) - LIBCUDA_MAJOR=$(echo $LIBCUDA_VERSION | cut -d. -f1) - LIBCUDNN_MAJOR=$(echo $LIBCUDNN_VERSION | cut -d. -f1) - os_name=$(cat /etc/*release | grep '^ID=' | sed 's/^.*=//g') - os_version=$(cat /etc/*release | grep VERSION_ID | sed 's/^.*"\([0-9]*\)\.\([0-9]*\)"/\1\2/g') - os_id="${os_name}${os_version}" - if [[ $CUDA_MAJOR_DASH == 9-* ]] || [[ $CUDA_MAJOR_DASH == 10-* ]]; then - os_id="ubuntu1604" - fi - export PATH=/usr/lib/binutils-2.26/bin/:${PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/bin - export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/include:$DEPS_PATH/usr/include - export C_INCLUDE_PATH=${C_INCLUDE_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/include:$DEPS_PATH/usr/include - export LIBRARY_PATH=${LIBRARY_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/lib64:$DEPS_PATH/usr/lib/x86_64-linux-gnu:$DEPS_PATH/usr/lib/nvidia-$NVIDIA_MAJOR_VERSION - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/lib64:$DEPS_PATH/usr/lib/x86_64-linux-gnu:$DEPS_PATH/usr/lib/nvidia-$NVIDIA_MAJOR_VERSION - export NVCC=$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/bin/nvcc -fi - -# list of debs to download from nvidia -if [[ $VARIANT == cu102* ]]; then - cuda_files=( \ - "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "libcublas10_${CUDA_PATCH_VERSION}_amd64.deb" \ - "libcublas-dev_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvcc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvtx-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ - "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ - ) - ml_files=( \ - "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ - "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ - ) -elif [[ $VARIANT == cu101* ]]; then - cuda_files=( \ - "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "libcublas10_${CUDA_PATCH_VERSION}_amd64.deb" \ - "libcublas-dev_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvcc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvtx-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ - "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ - ) - ml_files=( \ - "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ - "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ - ) -elif [[ $VARIANT == cu100* ]]; then - cuda_files=( \ - "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvcc-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-nvtx-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ - "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ - ) - ml_files=( \ - "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ - "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ - ) -elif [[ $VARIANT == cu92* ]]; then - cuda_files=( \ - "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "cuda-nvcc-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \ - "cuda-nvtx-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \ - "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ - "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \ - ) - ml_files=( \ - "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \ - "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \ - ) -fi - - -if [[ ! -d $DEPS_PATH/usr/local/cuda-${CUDA_MAJOR_VERSION} ]]; then - prefix=$DEPS_PATH - - for item in ${cuda_files[*]} - do - echo "Installing $item" - curl -sL "http://developer.download.nvidia.com/compute/cuda/repos/${os_id}/x86_64/${item}" -o package.deb - dpkg -X package.deb ${prefix} - rm package.deb - done - for item in ${ml_files[*]} - do - echo "Installing $item" - curl -sL "http://developer.download.nvidia.com/compute/machine-learning/repos/${os_id}/x86_64/${item}" -o package.deb - dpkg -X package.deb ${prefix} - rm package.deb - done - - mkdir -p ${prefix}/include - mkdir -p ${prefix}/lib - cp ${prefix}/usr/include/x86_64-linux-gnu/cudnn_v${LIBCUDNN_MAJOR}.h ${prefix}/include/cudnn.h - ln -s libcudnn_static_v${LIBCUDNN_MAJOR}.a ${prefix}/usr/lib/x86_64-linux-gnu/libcudnn.a - cp ${prefix}/usr/local/cuda-${CUDA_MAJOR_VERSION}/lib64/*.a ${prefix}/lib/ - cp ${prefix}/usr/include/nccl.h ${prefix}/include/nccl.h - ln -s libnccl_static.a ${prefix}/usr/lib/x86_64-linux-gnu/libnccl.a -fi diff --git a/tools/staticbuild/README.md b/tools/staticbuild/README.md index 077150e11762..780cb6d39776 100644 --- a/tools/staticbuild/README.md +++ b/tools/staticbuild/README.md @@ -24,6 +24,9 @@ This script is a wrapper around `build_lib.sh. It simplifies the build by automatically identifing the system version, number of cores, and all environment variable settings. Here are examples you can run with this script: +You need to install `patchelf` first, for example via `apt install patchelf` on +Ubuntu systems. + ``` tools/staticbuild/build.sh cu102 ``` @@ -35,16 +38,6 @@ tools/staticbuild/build.sh cpu This would build the mxnet package based on MKL-DNN. -To use CMake to build the `libmxnet.so` instead of the deprecated Makefile based -build logic, set the `CMAKE_STATICBUILD` environment variable. For example - -``` -CMAKE_STATICBUILD=1 tools/staticbuild/build.sh cpu -``` - -For the CMake build, you need to install `patchelf` first, for example via `apt -install patchelf` on Ubuntu systems. - As the result, users would have a complete static dependencies in `/staticdeps` in the root folder as well as a static-linked `libmxnet.so` file lives in `lib`. You can build your language binding by using the `libmxnet.so`. ## `build_lib.sh` diff --git a/tools/staticbuild/build.sh b/tools/staticbuild/build.sh index e5fd24368ed3..c0621fd90e9f 100755 --- a/tools/staticbuild/build.sh +++ b/tools/staticbuild/build.sh @@ -48,16 +48,14 @@ else fi export MAKE="make $ADD_MAKE_FLAG" -export CC="gcc -fPIC -mno-avx" -export CXX="g++ -fPIC -mno-avx" +export CC="gcc" +export CXX="g++" +export CFLAGS="-fPIC -mno-avx" +export CXXFLAGS="-fPIC -mno-avx" export FC="gfortran" export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$DEPS_PATH/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH export CPATH=$DEPS_PATH/include:$CPATH -if [[ -z "$USE_SYSTEM_CUDA" && $PLATFORM == 'linux' && $VARIANT == cu* ]]; then - source tools/setup_gpu_build_tools.sh $VARIANT $DEPS_PATH -fi - mkdir -p $DEPS_PATH # Build Dependencies @@ -70,10 +68,5 @@ cp NOTICE licenses/ cp LICENSE licenses/ cp DISCLAIMER-WIP licenses/ - # Build mxnet -if [[ -z "$CMAKE_STATICBUILD" ]]; then - source tools/staticbuild/build_lib.sh -else - source tools/staticbuild/build_lib_cmake.sh -fi +source tools/staticbuild/build_lib.sh diff --git a/tools/staticbuild/build_lib.sh b/tools/staticbuild/build_lib.sh index 989070ac7078..5261b2a6942a 100755 --- a/tools/staticbuild/build_lib.sh +++ b/tools/staticbuild/build_lib.sh @@ -20,28 +20,29 @@ set -eo pipefail # This script builds the libraries of mxnet. -make_config=make/staticbuild/${PLATFORM}_${VARIANT}.mk -if [[ ! -f $make_config ]]; then - >&2 echo "Couldn't find make config $make_config for the current settings." +cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${VARIANT}.cmake +if [[ ! -f $cmake_config ]]; then + >&2 echo "Couldn't find cmake config $make_config for the current settings." exit 1 fi ->&2 echo "Now building mxnet modules..." -cp $make_config config.mk - git submodule update --init --recursive || true -$MAKE DEPS_PATH=$DEPS_PATH DMLCCORE -$MAKE DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a -$MAKE DEPS_PATH=$DEPS_PATH PSLITE -$MAKE DEPS_PATH=$DEPS_PATH mkldnn - ->&2 echo "Now building mxnet..." -$MAKE DEPS_PATH=$DEPS_PATH +# Build libmxnet.so +rm -rf build; mkdir build; cd build +cmake -GNinja -C $cmake_config -DCMAKE_PREFIX_PATH=${DEPS_PATH} -DCMAKE_FIND_ROOT_PATH=${DEPS_PATH} .. +ninja +cd - +# Move to lib +rm -rf lib; mkdir lib; if [[ $PLATFORM == 'linux' ]]; then + cp -L build/libmxnet.so lib/libmxnet.so + cp -L staticdeps/lib/libopenblas.so lib/libopenblas.so.0 cp -L $(ldd lib/libmxnet.so | grep libgfortran | awk '{print $3}') lib/ cp -L $(ldd lib/libmxnet.so | grep libquadmath | awk '{print $3}') lib/ +elif [[ $PLATFORM == 'darwin' ]]; then + cp -L build/libmxnet.dylib lib/libmxnet.dylib fi # Print the linked objects on libmxnet.so @@ -50,8 +51,8 @@ if [[ ! -z $(command -v readelf) ]]; then readelf -d lib/libmxnet.so strip --strip-unneeded lib/libmxnet.so elif [[ ! -z $(command -v otool) ]]; then - otool -L lib/libmxnet.so - strip -u -r -x lib/libmxnet.so + otool -L lib/libmxnet.dylib + strip -u -r -x lib/libmxnet.dylib else >&2 echo "Not available" fi diff --git a/tools/staticbuild/build_lib_cmake.sh b/tools/staticbuild/build_lib_cmake.sh deleted file mode 100755 index 5261b2a6942a..000000000000 --- a/tools/staticbuild/build_lib_cmake.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -# This script builds the libraries of mxnet. -cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${VARIANT}.cmake -if [[ ! -f $cmake_config ]]; then - >&2 echo "Couldn't find cmake config $make_config for the current settings." - exit 1 -fi - -git submodule update --init --recursive || true - -# Build libmxnet.so -rm -rf build; mkdir build; cd build -cmake -GNinja -C $cmake_config -DCMAKE_PREFIX_PATH=${DEPS_PATH} -DCMAKE_FIND_ROOT_PATH=${DEPS_PATH} .. -ninja -cd - - -# Move to lib -rm -rf lib; mkdir lib; -if [[ $PLATFORM == 'linux' ]]; then - cp -L build/libmxnet.so lib/libmxnet.so - cp -L staticdeps/lib/libopenblas.so lib/libopenblas.so.0 - cp -L $(ldd lib/libmxnet.so | grep libgfortran | awk '{print $3}') lib/ - cp -L $(ldd lib/libmxnet.so | grep libquadmath | awk '{print $3}') lib/ -elif [[ $PLATFORM == 'darwin' ]]; then - cp -L build/libmxnet.dylib lib/libmxnet.dylib -fi - -# Print the linked objects on libmxnet.so ->&2 echo "Checking linked objects on libmxnet.so..." -if [[ ! -z $(command -v readelf) ]]; then - readelf -d lib/libmxnet.so - strip --strip-unneeded lib/libmxnet.so -elif [[ ! -z $(command -v otool) ]]; then - otool -L lib/libmxnet.dylib - strip -u -r -x lib/libmxnet.dylib -else - >&2 echo "Not available" -fi - -if [[ ! -L deps ]]; then - ln -s staticdeps deps -fi