Updated network retry delay strategy to scale

This allows for network retries, to scale well with the
number of machines, and still retains the existing functionality
for cases with smaller num_machines ( 500 )

Fixes #3301

include/LightGBM/c_api.h

@@ -47,7 +47,7 @@ typedef void* FastConfigHandle; /*!< \brief Handle of FastConfig. */
 LIGHTGBM_C_EXPORT const char* LGBM_GetLastError();
 
 /*!
- * \brief Register a callback function for log redirecting. 
+ * \brief Register a callback function for log redirecting.
  * \param callback The callback function to register
  * \return 0 when succeed, -1 when failure happens
  */

src/network/linkers_socket.cpp

@@ -15,6 +15,7 @@
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
+#include <algorithm>
 
 #include "linkers.h"
 
@@ -186,7 +187,9 @@ void Linkers::Construct() {
   listener_->SetTimeout(socket_timeout_);
   listener_->Listen(incoming_cnt);
   std::thread listen_thread(&Linkers::ListenThread, this, incoming_cnt);
-  const int connect_fail_retry_cnt = 20;
+  const int connect_fail_retries_factor_machine = 25;
+  const int connect_fail_retries_scale_factor = static_cast<int>(num_machines_ / connect_fail_retries_factor_machine);
+  const int connect_fail_retry_cnt = max(20, connect_fail_retries_scale_factor);
   const int connect_fail_retry_first_delay_interval = 200;  // 0.2 s
   const float connect_fail_retry_delay_factor = 1.3f;
   // start connect