diff --git a/CMakeLists.txt b/CMakeLists.txt
index 688dd42c54fe..e2b665f50131 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -148,6 +148,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Distribution" AND UNIX AND NOT APPLE)
   # Enforce DT_PATH instead of DT_RUNPATH
   set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--disable-new-dtags")
   set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags")
+  set(Protobuf_USE_STATIC_LIBS ON)
 endif()
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/upstream;${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
diff --git a/config/distribution/darwin_cpu.cmake b/config/distribution/darwin_cpu.cmake
index 790e18320157..ed295efd33d2 100644
--- a/config/distribution/darwin_cpu.cmake
+++ b/config/distribution/darwin_cpu.cmake
@@ -31,3 +31,4 @@ set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
 set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
 set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
 set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
diff --git a/config/distribution/linux_cpu.cmake b/config/distribution/linux_cpu.cmake
index 15b4f5aa7e59..4579b42fce5c 100644
--- a/config/distribution/linux_cpu.cmake
+++ b/config/distribution/linux_cpu.cmake
@@ -29,3 +29,4 @@ set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
 set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
 set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
 set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
diff --git a/config/distribution/linux_cu100.cmake b/config/distribution/linux_cu100.cmake
index 457a14f89a32..357ccd457593 100644
--- a/config/distribution/linux_cu100.cmake
+++ b/config/distribution/linux_cu100.cmake
@@ -31,6 +31,7 @@ set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
 set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
 set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
 set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
 
 set(CUDACXX "/usr/local/cuda-10.0/bin/nvcc" CACHE STRING "Cuda compiler")
 set(MXNET_CUDA_ARCH "3.0;5.0;6.0;7.0" CACHE STRING "Cuda architectures")
diff --git a/config/distribution/linux_cu101.cmake b/config/distribution/linux_cu101.cmake
index 61089200f31f..29fdda2d6f0a 100644
--- a/config/distribution/linux_cu101.cmake
+++ b/config/distribution/linux_cu101.cmake
@@ -33,6 +33,7 @@ set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
 set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
 set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
 set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
 
 set(CUDACXX "/usr/local/cuda-10.1/bin/nvcc" CACHE STRING "Cuda compiler")
 set(MXNET_CUDA_ARCH "3.0;5.0;6.0;7.0" CACHE STRING "Cuda architectures")
diff --git a/config/distribution/linux_cu102.cmake b/config/distribution/linux_cu102.cmake
index 9701b99a6a1f..4e7e0509e4f9 100644
--- a/config/distribution/linux_cu102.cmake
+++ b/config/distribution/linux_cu102.cmake
@@ -31,6 +31,7 @@ set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
 set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
 set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
 set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
 
 set(CUDACXX "/usr/local/cuda-10.2/bin/nvcc" CACHE STRING "Cuda compiler")
 set(MXNET_CUDA_ARCH "3.0;5.0;6.0;7.0" CACHE STRING "Cuda architectures")
diff --git a/tools/dependencies/cityhash.sh b/tools/dependencies/cityhash.sh
index 6bc663e906fa..4e6c7fb776a6 100755
--- a/tools/dependencies/cityhash.sh
+++ b/tools/dependencies/cityhash.sh
@@ -20,6 +20,12 @@
 # This script builds the static library of cityhash that can be used as dependency of mxnet.
 set -ex
 CITYHASH_VERSION=1.1.1
+if [[ $PLATFORM == 'darwin' ]]; then
+    DY_EXT="dylib"
+else
+    DY_EXT="so"
+fi
+
 if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then
     # Download and build cityhash
     >&2 echo "Building cityhash..."
@@ -30,5 +36,6 @@ if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then
     ./configure -prefix=$DEPS_PATH --enable-sse4.2
     $MAKE CXXFLAGS="-g -O3 -msse4.2"
     $MAKE install
+    rm $DEPS_PATH/lib/*cityhash*$DY_EXT*
     popd
 fi
diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh
index 39fa24c87ecd..93f1823da641 100755
--- a/tools/dependencies/libpng.sh
+++ b/tools/dependencies/libpng.sh
@@ -20,8 +20,7 @@
 # This script builds the static library of libpng that can be used as dependency of mxnet/opencv.
 set -ex
 PNG_VERSION=1.6.35
-if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
-    # download and build libpng
+if [[ ! -f $DEPS_PATH/lib/libpng.a ]] && [[ ! -f $DEPS_PATH/lib64/libpng.a ]]; then    # download and build libpng
     >&2 echo "Building libpng..."
     download \
         https://github.com/glennrp/libpng/archive/v${PNG_VERSION}.zip \
diff --git a/tools/dependencies/libturbojpeg.sh b/tools/dependencies/libturbojpeg.sh
index 911827a16fcf..88b8ec45a193 100755
--- a/tools/dependencies/libturbojpeg.sh
+++ b/tools/dependencies/libturbojpeg.sh
@@ -25,8 +25,8 @@ if [[ $PLATFORM == 'darwin' ]]; then
     JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm"
 fi
 
-if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a ]]; then
-    # download and build libjpeg
+if [[ ( ! -f $DEPS_PATH/lib/libjpeg.a ) || ( ! -f $DEPS_PATH/lib/libturbojpeg.a ) ]] && \
+       [[ ( ! -f $DEPS_PATH/lib64/libjpeg.a ) || ( ! -f $DEPS_PATH/lib64/libturbojpeg.a ) ]]; then    # download and build libjpeg
     >&2 echo "Building libjpeg-turbo..."
     download \
         https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${TURBO_JPEG_VERSION}.zip \
diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh
index ce72908fdc2b..ec4b04576a86 100755
--- a/tools/dependencies/lz4.sh
+++ b/tools/dependencies/lz4.sh
@@ -20,6 +20,12 @@
 # This script builds the static library of lz4 that can be used as dependency of mxnet.
 set -ex
 LZ4_VERSION=r130
+if [[ $PLATFORM == 'darwin' ]]; then
+    DY_EXT="dylib"
+else
+    DY_EXT="so"
+fi
+
 if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then
     # Download and build lz4
     >&2 echo "Building lz4..."
@@ -31,5 +37,6 @@ if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then
     cd $DEPS_PATH/lz4-$LZ4_VERSION
     $MAKE
     $MAKE PREFIX=$DEPS_PATH install
+    rm $DEPS_PATH/lib/*lz4*$DY_EXT*
     popd
 fi
diff --git a/tools/dependencies/openssl.sh b/tools/dependencies/openssl.sh
index 78673a3ac84b..cc989a254057 100755
--- a/tools/dependencies/openssl.sh
+++ b/tools/dependencies/openssl.sh
@@ -20,8 +20,8 @@
 # This script builds the static library of openssl that can be used as dependency of mxnet.
 set -ex
 OPENSSL_VERSION=1.1.1b
-if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; then
-    # download and build openssl
+if [[ ( ! -f $DEPS_PATH/lib/libssl.a ) || ( ! -f $DEPS_PATH/lib/libcrypto.a ) ]] && \
+       [[ ( ! -f $DEPS_PATH/lib64/libssl.a ) || ( ! -f $DEPS_PATH/lib64/libcrypto.a ) ]]; then    # download and build openssl
     >&2 echo "Building openssl..."
     OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
     download \
diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh
index 7da4c2537b42..4b58a1cd31ff 100755
--- a/tools/dependencies/protobuf.sh
+++ b/tools/dependencies/protobuf.sh
@@ -20,11 +20,6 @@
 # This script builds the static library of protobuf along with protoc, that can be used as dependency of mxnet.
 set -ex
 PROTOBUF_VERSION=3.5.1
-if [[ $PLATFORM == 'darwin' ]]; then
-    DY_EXT="dylib"
-else
-    DY_EXT="so"
-fi
 
 LIBPROTOBUF="$DEPS_PATH/lib/libprotobuf.$DY_EXT"
 LIBPROTOC="$DEPS_PATH/lib/libprotoc.$DY_EXT"
diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh
index 33ea628d53bb..0ab5335e9045 100755
--- a/tools/dependencies/zmq.sh
+++ b/tools/dependencies/zmq.sh
@@ -20,6 +20,12 @@
 # This script builds the static library of zeroMQ that can be used as dependency of mxnet.
 set -ex
 ZEROMQ_VERSION=4.2.2
+if [[ $PLATFORM == 'darwin' ]]; then
+    DY_EXT="dylib"
+else
+    DY_EXT="so"
+fi
+
 if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
     # Download and build zmq
     >&2 echo "Building zmq..."
@@ -39,8 +45,11 @@ if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
     $MAKE install
 
     if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
+        rm $DEPS_PATH/lib64/*zmq*$DY_EXT*
         mkdir -p $DEPS_PATH/lib
         cp $DEPS_PATH/lib64/*zmq* $DEPS_PATH/lib
+    else
+        rm $DEPS_PATH/lib/*zmq*$DY_EXT*
     fi
 
     popd