Merge branch 'master' into java-api

CONTRIBUTORS.md

@@ -190,6 +190,7 @@ List of Contributors
 * [Denisa Roberts](https://github.com/D-Roberts)
 * [Dick Carter](https://github.com/DickJC123)
 * [Rahul Padmanabhan](https://github.com/rahul3)
+* [Yuxi Hu](https://github.com/yuxihu)
 
 Label Bot
 ---------

ci/docker/install/ubuntu_caffe.sh

@@ -18,6 +18,7 @@
 # under the License.
 set -ex
 
+apt-get update || true
 apt-get install -y \
     libgflags-dev \
     libgoogle-glog-dev \

ci/docker/install/ubuntu_clang.sh

@@ -21,6 +21,8 @@
 # the whole docker cache for the image
 
 set -ex
+
+apt-get update || true
 # Install clang 3.9 (the same version as in XCode 8.*) and 6.0 (latest major release)
 wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
     apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main" && \

ci/docker/install/ubuntu_core.sh

@@ -21,7 +21,7 @@
 # the whole docker cache for the image
 
 set -ex
-apt-get update
+apt-get update || true
 apt-get install -y \
     apt-transport-https \
     build-essential \

ci/docker/install/ubuntu_docs.sh

@@ -23,6 +23,7 @@
 set -ex
 # Install dependencies
 echo 'Installing dependencies...'
+apt-get update || true
 apt-get install -y \
     doxygen \
     pandoc

ci/docker/install/ubuntu_emscripten.sh

@@ -25,6 +25,7 @@
 
 set -ex
 
+apt-get update || true
 apt-get -y install nodejs
 
 git clone -b 1.38.6 https://github.com/kripken/emscripten.git

ci/docker/install/ubuntu_gcc8.sh

@@ -19,5 +19,5 @@
 
 sudo add-apt-repository ppa:jonathonf/gcc-8.0
 sudo add-apt-repository ppa:jonathonf/gcc-7.3
-sudo apt-get update
+sudo apt-get update || true
 sudo apt-get install -y gcc-8 g++-8

ci/docker/install/ubuntu_llvm.sh

@@ -23,4 +23,5 @@ echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
      >> /etc/apt/sources.list.d/llvm.list
 
 wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get update && apt-get install -y --force-yes llvm-5.0
+apt-get update || true
+apt-get install -y --force-yes llvm-5.0

ci/docker/install/ubuntu_nightly_tests.sh

@@ -25,7 +25,7 @@ set -ex
 # Adding ppas frequently fails due to busy gpg servers, retry 5 times with 5 minute delays.
 for i in 1 2 3 4 5; do add-apt-repository -y ppa:ubuntu-toolchain-r/test && break || sleep 300; done
 
-apt-get update
+apt-get update || true
 apt-get -y install time
 
 # Install for RAT License Check Nightly Test

ci/docker/install/ubuntu_npm_blc.sh

@@ -22,7 +22,7 @@
 
 set -ex
 echo 'Installing npm...'
-apt-get update
+apt-get update || true
 apt-get install -y npm
 
 echo "Obtaining NodeJS version 8.x"

ci/docker/install/ubuntu_nvidia.sh

@@ -18,6 +18,7 @@
 # under the License.
 
 set -ex
+apt-get update || true
 apt install -y software-properties-common
 
 # Adding ppas frequently fails due to busy gpg servers, retry 5 times with 5 minute delays.

ci/docker/install/ubuntu_onnx.sh

@@ -27,6 +27,7 @@ set -e
 set -x
 
 echo "Installing libprotobuf-dev and protobuf-compiler ..."
+apt-get update || true
 apt-get install -y libprotobuf-dev protobuf-compiler
 
 echo "Installing pytest, pytest-cov, protobuf, Pillow, ONNX and tabulate ..."

ci/docker/install/ubuntu_perl.sh

@@ -22,5 +22,6 @@
 
 set -ex
 # install libraries for mxnet's perl package on ubuntu
+apt-get update || true
 apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl
 cpanm -q Function::Parameters Hash::Ordered PDL::CCS

ci/docker/install/ubuntu_python.sh

@@ -22,6 +22,7 @@
 
 set -ex
 # install libraries for mxnet's python package on ubuntu
+apt-get update || true
 apt-get install -y python-dev python3-dev virtualenv wget
 
 # the version of the pip shipped with ubuntu may be too lower, install a recent version here

ci/docker/install/ubuntu_r.sh

@@ -34,7 +34,7 @@ apt-key add r.gpg
 # Installing the latest version (3.3+) that is compatible with MXNet
 add-apt-repository 'deb [arch=amd64,i386] https://cran.rstudio.com/bin/linux/ubuntu xenial/'
 
-apt-get update
+apt-get update || true
 apt-get install -y --allow-unauthenticated \
     libcairo2-dev \
     libssl-dev \

ci/docker/install/ubuntu_rat.sh

@@ -20,7 +20,7 @@
 set -ex
 
 echo "Install dependencies"
-apt-get update
+apt-get update || true
 apt-get install -y subversion maven openjdk-8-jdk openjdk-8-jre
 
 echo "download RAT"

ci/docker/install/ubuntu_scala.sh

@@ -24,8 +24,9 @@ set -ex
 cd "$(dirname "$0")"
 # install libraries for mxnet's scala package on ubuntu
 echo 'Installing Scala...'
+apt-get update || true
 apt-get install -y software-properties-common
-apt-get update
+apt-get update || true
 apt-get install -y openjdk-8-jdk
 apt-get install -y openjdk-8-jre
 
@@ -34,7 +35,8 @@ echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/
 #apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
 #apt-key adv --keyserver keys.gnupg.net --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
 apt-key add sbt.gpg
-apt-get update && apt-get install -y \
+apt-get update || true
+apt-get install -y \
     maven \
     sbt \
     scala

ci/docker/install/ubuntu_tutorials.sh

@@ -21,6 +21,7 @@
 # the whole docker cache for the image
 
 set -ex
+apt-get update || true
 apt-get install graphviz python-opencv
 pip2 install jupyter matplotlib Pillow opencv-python scikit-learn graphviz tqdm
 pip3 install jupyter matplotlib Pillow opencv-python scikit-learn graphviz tqdm

ci/docker_cache.py

@@ -78,7 +78,7 @@ def _build_save_container(platform, registry, load_cache) -> Optional[str]:
     logging.debug('Building %s as %s', platform, docker_tag)
     try:
         # Increase the number of retries for building the cache.
-        image_id = build_util.build_docker(docker_binary='docker', platform=platform, registry=registry, num_retries=10, use_cache=True)
+        image_id = build_util.build_docker(docker_binary='docker', platform=platform, registry=registry, num_retries=10, no_cache=False)
         logging.info('Built %s as %s', docker_tag, image_id)
 
         # Push cache to registry

cmake/FirstClassLangCuda.cmake

@@ -185,6 +185,12 @@ function(mshadow_select_nvcc_arch_flags out_variable)
       elseif(${arch_name} STREQUAL "Pascal")
         set(arch_bin 6.0 6.1)
         set(arch_ptx 6.1)
+      elseif(${arch_name} STREQUAL "Volta")
+        set(arch_bin 7.0)
+        set(arch_ptx 7.0)
+      elseif(${arch_name} STREQUAL "Turing")
+        set(arch_bin 7.5)
+        set(arch_ptx 7.5)
       else()
         message(SEND_ERROR "Unknown CUDA Architecture Name ${arch_name} in CUDA_SELECT_NVCC_ARCH_FLAGS")
       endif()

docs/_static/js/auto_module_index.js

@@ -10,15 +10,17 @@ function auto_index(module) {
     var html = "<ul>";
 
     for (var i = 0; i < targets.length; ++i) {
-      var id = $(targets[i]).attr('id');
-      // remove 'mxnet.' prefix to make menus shorter
-      var id_simple = id.replace(/^mxnet\./, '');
-      html += "<li><a class='reference internal' href='#";
-      html += id;
-      html += "'>" + id_simple + "</a></li>";
+	var id = $(targets[i]).attr('id');
+	if ( id ) {
+	    // remove 'mxnet.' prefix to make menus shorter
+	    var id_simple = id.replace(/^mxnet\./, '');
+	    html += "<li><a class='reference internal' href='#";
+	    html += id;
+	    html += "'>" + id_simple + "</a></li>";
+	}
     }
 
     html += "</ul>";
     li_node.append(html);
   });
-}
+}

docs/api/python/ndarray/ndarray.md

@@ -656,6 +656,7 @@ The `ndarray` package provides several classes:
     log_softmax
     relu
     sigmoid
+    erf
 ```
 
 ### More
@@ -706,9 +707,14 @@ The `ndarray` package provides several classes:
     :members:
     :imported-members:
     :special-members:
-    :exclude-members: CachedOp, NDArray
+    :exclude-members: CachedOp, NDArray, save, load
+
+.. automodule:: mxnet.ndarray
+    :noindex:
+    :members: save, load
 
 .. automodule:: mxnet.random
+    :noindex:
     :members:
 
 ```

docs/api/python/ndarray/random.md

@@ -51,6 +51,7 @@ In the rest of this document, we list routines provided by the `ndarray.random`
 
 .. automodule:: mxnet.random
     :members:
+    :noindex:
 
 ```
 

docs/api/python/symbol/symbol.md

@@ -657,6 +657,7 @@ Composite multiple symbols into a new one by an operator.
     log_softmax
     relu
     sigmoid
+    erf
 ```
 
 ### More

docs/architecture/note_memory.md

@@ -312,18 +312,9 @@ that are already optimized for big operations,
 you can reduce memory consumption roughly *by half*.
 You can reduce memory usage even more
 if you are optimizing a fine-grained computation network
-used by symbolic libraries, such as Theano.
-
-Most of the ideas in this article inspired the design of _MXNet_.
-We've also provided a [Memory Cost Estimation Script](https://github.com/dmlc/mxnet/tree/master/example/memcost),
-which you can use to see how much memory you need under different scenarios.
-
-The script has an option called `forward_only`,
-which shows the cost of running only the forward pass.
-You will find that cost when using this option
-is extremely low compared to others.
-This is simply because there's  more memory reuse
-if you run only the forward pass.
+used by symbolic libraries, such as Theano. Most of the ideas in this article inspired the design of _MXNet_.
+
+Also, you will notice that memory cost, for forward pass only execution, is extremely low compared to running both forward and backward pass. This is simply because there's  more memory reuse if you run only the forward pass.
 
 So here are two takeaways:
 

docs/conf.py

@@ -107,7 +107,7 @@
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['3rdparty', 'build_version_doc', 'virtualenv', 'api/python/model.md', 'README.md']
+exclude_patterns = ['3rdparty', 'build_version_doc', 'virtualenv', 'api/python/model.md', 'README.md', 'tutorial_utils']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
 #default_role = None

docs/mxdoc.py

@@ -110,8 +110,13 @@ def build_scala(app):
 def build_scala_docs(app):
     """build scala doc and then move the outdir"""
     scala_path = app.builder.srcdir + '/../scala-package'
-    # scaldoc fails on some apis, so exit 0 to pass the check
-    _run_cmd('cd ' + scala_path + '; scaladoc `find . -type f -name "*.scala" | egrep \"\/core|\/infer\" | egrep -v \"Suite|javaapi\"`; exit 0')
+    scala_doc_sources = 'find . -type f -name "*.scala" | egrep \"\.\/core|\.\/infer\" | egrep -v \"Suite\"'
+    scala_doc_classpath = ':'.join([
+        '`find native -name "*.jar" | grep "target/lib/" | tr "\\n" ":" `',
+        '`find macros -name "*-SNAPSHOT.jar" | tr "\\n" ":" `'
+    ])
+    _run_cmd('cd {}; scaladoc `{}` -classpath {} -feature -deprecation'
+             .format(scala_path, scala_doc_sources, scala_doc_classpath))
     dest_path = app.builder.outdir + '/api/scala/docs'
     _run_cmd('rm -rf ' + dest_path)
     _run_cmd('mkdir -p ' + dest_path)

example/cnn_visualization/gradcam.py → docs/tutorial_utils/vision/cnn_visualization/gradcam.py

@@ -249,8 +249,8 @@ def visualize(net, preprocessed_img, orig_img, conv_layer_name):
     imggrad = get_image_grad(net, preprocessed_img)
     conv_out, conv_out_grad = get_conv_out_grad(net, preprocessed_img, conv_layer_name=conv_layer_name)
 
-    cam = get_cam(imggrad, conv_out)
-
+    cam = get_cam(conv_out_grad, conv_out)
+    cam = cv2.resize(cam, (imggrad.shape[1], imggrad.shape[2]))
     ggcam = get_guided_grad_cam(cam, imggrad)
     img_ggcam = grad_to_image(ggcam)