handle busy agent hb timeout

Netflix · Nov 7, 2024 · 418a0d0 · 418a0d0
1 parent f9d3a5c
commit 418a0d0
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 4 deletions.
diff --git a/...src/main/java/io/mantisrx/server/master/resourcecluster/ResourceClusterGatewayClient.java b/...src/main/java/io/mantisrx/server/master/resourcecluster/ResourceClusterGatewayClient.java
@@ -24,6 +24,7 @@
 import io.mantisrx.server.core.CoreConfiguration;
 import io.mantisrx.server.core.master.MasterDescription;
 import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper;
+import io.netty.util.concurrent.DefaultThreadFactory;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.concurrent.CompletableFuture;
@@ -129,6 +130,16 @@ private AsyncHttpClient buildCloseableHttpClient(CoreConfiguration configuration
             .setRequestTimeout(configuration.getAsyncHttpClientRequestTimeoutMs())
             .setReadTimeout(configuration.getAsyncHttpClientReadTimeoutMs())
             .setFollowRedirect(configuration.getAsyncHttpClientFollowRedirect())
+            // set the http client thread priority to max - 1 to ensure control plane signals can still be retrieved
+            // even when the worker is busy.
+            .setThreadFactory(new DefaultThreadFactory(generateThreadPoolName(), Thread.MAX_PRIORITY - 1))
             .build());
   }
+
+  private String generateThreadPoolName() {
+      return String.format(
+          "resourceClusterGatewayClient-httpclient-%s-%s",
+          this.masterDescription.getHostname(),
+          this.clusterID.getResourceID());
+  }
 }
diff --git a/...ne/mantis-control-plane-core/src/main/java/io/mantisrx/server/core/CoreConfiguration.java b/...ne/mantis-control-plane-core/src/main/java/io/mantisrx/server/core/CoreConfiguration.java
@@ -67,15 +67,15 @@ public interface CoreConfiguration {
     int getAsyncHttpClientMaxConnectionsPerHost();
 
     @Config("mantis.asyncHttpClient.connectionTimeoutMs")
-    @Default("10000")
+    @Default("90000")
     int getAsyncHttpClientConnectionTimeoutMs();
 
     @Config("mantis.asyncHttpClient.requestTimeoutMs")
-    @Default("10000")
+    @Default("90000")
     int getAsyncHttpClientRequestTimeoutMs();
 
     @Config("mantis.asyncHttpClient.readTimeoutMs")
-    @Default("10000")
+    @Default("90000")
     int getAsyncHttpClientReadTimeoutMs();
 
     @Config("mantis.asyncHttpClient.followRedirect")

diff --git a/...s-runtime-loader/src/main/java/io/mantisrx/runtime/loader/config/WorkerConfiguration.java b/...s-runtime-loader/src/main/java/io/mantisrx/runtime/loader/config/WorkerConfiguration.java
@@ -85,7 +85,7 @@ default String getTaskExecutorHostName() {
     int getTolerableConsecutiveHeartbeatFailures();
 
     @Config("mantis.taskexecutor.heartbeats.timeout.ms")
-    @Default("5000")
+    @Default("90000")
     int heartbeatTimeoutMs();
 
     @Config("mantis.taskexecutor.heartbeats.retry.initial-delay.ms")