(selectdb-cloud) Add grpc retry policy for MetaServiceClient (apache#…

…1891) * refer to https://github.com/grpc/proposal/blob/master/A6-client-retries.md#retry-policy-capabilities
SWJTU-ZhangLei · Jun 11, 2023 · e33f0cf · e33f0cf
1 parent 205efc2
commit e33f0cf
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 2 deletions.
diff --git a/fe/fe-core/src/main/java/com/selectdb/cloud/rpc/MetaServiceClient.java b/fe/fe-core/src/main/java/com/selectdb/cloud/rpc/MetaServiceClient.java
@@ -3,20 +3,24 @@
 import com.selectdb.cloud.proto.MetaServiceGrpc;
 import com.selectdb.cloud.proto.SelectdbCloud;
 
+import com.google.gson.Gson;
+import com.google.gson.stream.JsonReader;
 import io.grpc.ManagedChannel;
 import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
 import org.apache.doris.common.Config;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 public class MetaServiceClient {
     public static final Logger LOG = LogManager.getLogger(MetaServiceClient.class);
 
-    private static final int MAX_RETRY_NUM = 0;
     private final TNetworkAddress address;
     private final MetaServiceGrpc.MetaServiceFutureStub stub;
     private final MetaServiceGrpc.MetaServiceBlockingStub blockingStub;
@@ -26,12 +30,23 @@ public MetaServiceClient(TNetworkAddress address) {
         this.address = address;
         channel = NettyChannelBuilder.forAddress(address.getHostname(), address.getPort())
             .flowControlWindow(Config.grpc_max_message_size_bytes)
-            .maxInboundMessageSize(Config.grpc_max_message_size_bytes).enableRetry().maxRetryAttempts(MAX_RETRY_NUM)
+            .maxInboundMessageSize(Config.grpc_max_message_size_bytes)
+            .defaultServiceConfig(getRetryingServiceConfig())
+            .enableRetry()
             .usePlaintext().build();
         stub = MetaServiceGrpc.newFutureStub(channel);
         blockingStub = MetaServiceGrpc.newBlockingStub(channel);
     }
 
+    protected Map<String, ?> getRetryingServiceConfig() {
+        // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#retry-policy-capabilities
+        Map<String, ?> serviceConfig = new Gson().fromJson(new JsonReader(new InputStreamReader(
+                MetaServiceClient.class.getResourceAsStream("/retrying_service_config.json"),
+                        StandardCharsets.UTF_8)), Map.class);
+        LOG.info("serviceConfig:{}", serviceConfig);
+        return serviceConfig;
+    }
+
     public void shutdown() {
         if (!channel.isShutdown()) {
             channel.shutdown();

diff --git a/fe/fe-core/src/main/resources/retrying_service_config.json b/fe/fe-core/src/main/resources/retrying_service_config.json
@@ -0,0 +1,20 @@
+{
+    "methodConfig": [
+        {
+            "name": [
+                {
+                    "service": "selectdb.MetaService"
+                }
+            ],
+            "retryPolicy": {
+                "maxAttempts": 10,
+                "initialBackoff": "0.5s",
+                "maxBackoff": "30s",
+                "backoffMultiplier": 2,
+                "retryableStatusCodes": [
+                    "UNAVAILABLE"
+                ]
+            }
+        }
+    ]
+}