update runtime setting default values

apache · Sep 6, 2020 · 1c85d2b · 1c85d2b
1 parent af2b4bc
commit 1c85d2b
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 8 deletions.
diff --git a/src/resource.cc b/src/resource.cc
@@ -96,9 +96,9 @@ class ResourceManagerImpl : public ResourceManager {
     cpu_temp_space_copy_ = dmlc::GetEnv("MXNET_CPU_TEMP_COPY", 4);
     gpu_temp_space_copy_ = dmlc::GetEnv("MXNET_GPU_TEMP_COPY", 1);
     cpu_native_rand_copy_ = dmlc::GetEnv("MXNET_CPU_PARALLEL_RAND_COPY", 1);
-    gpu_native_rand_copy_ = dmlc::GetEnv("MXNET_GPU_PARALLEL_RAND_COPY", 4);
+    gpu_native_rand_copy_ = dmlc::GetEnv("MXNET_GPU_PARALLEL_RAND_COPY", 1);
 #if MXNET_USE_CUDNN == 1
-    gpu_cudnn_dropout_state_copy_ = dmlc::GetEnv("MXNET_GPU_CUDNN_DROPOUT_STATE_COPY", 4);
+    gpu_cudnn_dropout_state_copy_ = dmlc::GetEnv("MXNET_GPU_CUDNN_DROPOUT_STATE_COPY", 1);
 #endif  // MXNET_USE_CUDNN == 1
     engine_ref_ = Engine::_GetSharedRef();
     storage_ref_ = Storage::_GetSharedRef();

diff --git a/src/storage/pooled_storage_manager.h b/src/storage/pooled_storage_manager.h
@@ -178,14 +178,19 @@ void PooledStorageManager<BucketingStrategy, StoringMethod>::Alloc(Storage::Hand
     void *ret = nullptr;
     auto e = contextHelper_->Malloc(&ret, roundSize);
     if (e) {
-      const std::string err(
+      // retry in case of fragmentation
+      ReleaseAllNoLock(false);
+      e = contextHelper_->Malloc(&ret, roundSize);
+      if (e) {
+        const std::string err(
 #if MXNET_USE_CUDA
-      dev_type_ == Context::kGPU?
-         cudaGetErrorString(static_cast<cudaError_t>(e)) :
+        dev_type_ == Context::kGPU?
+           cudaGetErrorString(static_cast<cudaError_t>(e)) :
 #endif
-         std::strerror(errno));
+           std::strerror(errno));
 
-      LOG(FATAL) << "Memory allocation failed " << err;
+        LOG(FATAL) << "Memory allocation failed " << err;
+      }
     }
 
     UNSET_DEVICE(device_store);

diff --git a/src/storage/storage.cc b/src/storage/storage.cc
@@ -67,8 +67,9 @@ StorageManager *CreateStorageManager(const Context &ctx, const char *context,
                                      int num_gpu_device, std::string *pStrategy) {
   const auto env_var = env_var_name(context, pool_type);
   const char *type = getenv(env_var.c_str());
-  if (type == nullptr)
+  if (type == nullptr) {
     type = "Naive";   // default pool
+  }
 
   *pStrategy = type;
   StorageManager *ptr = nullptr;