add check

apache · Jul 4, 2024 · c8c3162 · c8c3162
1 parent b2dc07d
commit c8c3162
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 10 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -20,7 +20,6 @@
 import org.apache.doris.common.credentials.CloudCredential;
 import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
 import org.apache.doris.common.util.LocationPath;
-import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.CatalogMgr;
 import org.apache.doris.datasource.InitCatalogLog.Type;
 import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
@@ -93,11 +92,6 @@ public static Map<String, String> convertToMetaProperties(Map<String, String> pr
             metaProperties = convertToDLFProperties(props, DLFProperties.getCredential(props));
         } else if (props.containsKey(S3Properties.Env.ENDPOINT)) {
             if (!hasS3Properties(props)) {
-                String type = props.get(CatalogMgr.CATALOG_TYPE_PROP);
-                // paimon do not need old s3 properties
-                if (type != null && type.equalsIgnoreCase(Type.PAIMON.toString())) {
-                    return props;
-                }
                 // checkout env in the end
                 // if meet AWS_XXX properties, convert to s3 properties
                 return convertToS3EnvProperties(props, S3Properties.getEnvironmentCredentialWithEndpoint(props), true);
@@ -262,13 +256,14 @@ private static Map<String, String> convertToS3Properties(Map<String, String> pro
         return s3Properties;
     }
 
-    private static String checkRegion(String endpoint, String region, String regionKey) {
+    public static String checkRegion(String endpoint, String region, String regionKey) {
         if (Strings.isNullOrEmpty(region)) {
             region = S3Properties.getRegionOfEndpoint(endpoint);
         }
         if (Strings.isNullOrEmpty(region)) {
-            String errorMsg = String.format("Required property '%s' when region is not in endpoint.", regionKey);
-            Util.logAndThrowRuntimeException(LOG, errorMsg, new IllegalArgumentException(errorMsg));
+            String errorMsg = String.format("No '%s' info found, using SDK default region: us-east-1", regionKey);
+            LOG.warn(errorMsg);
+            return "us-east-1";
         }
         return region;
     }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java
@@ -55,7 +55,8 @@ private static Map<String, String> getBeAWSPropertiesFromS3(Map<String, String>
         Map<String, String> beProperties = new HashMap<>();
         String endpoint = properties.get(S3Properties.ENDPOINT);
         beProperties.put(S3Properties.Env.ENDPOINT, endpoint);
-        String region = S3Properties.getRegionOfEndpoint(endpoint);
+        String region = PropertyConverter.checkRegion(endpoint, properties.get(S3Properties.Env.REGION),
+                S3Properties.Env.REGION);
         beProperties.put(S3Properties.Env.REGION, properties.getOrDefault(S3Properties.REGION, region));
         if (properties.containsKey(S3Properties.ACCESS_KEY)) {
             beProperties.put(S3Properties.Env.ACCESS_KEY, properties.get(S3Properties.ACCESS_KEY));

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
@@ -42,6 +42,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 public class S3Properties extends BaseProperties {
 
@@ -82,6 +83,8 @@ public class S3Properties extends BaseProperties {
             WebIdentityTokenCredentialsProvider.class.getName(),
             IAMInstanceCredentialsProvider.class.getName());
 
+    private static final Pattern IPV4_PORT_PATTERN = Pattern.compile("((?:\\d{1,3}\\.){3}\\d{1,3}:\\d{1,5})");
+
     public static Map<String, String> credentialToMap(CloudCredentialWithEndpoint credential) {
         Map<String, String> resMap = new HashMap<>();
         resMap.put(S3Properties.ENDPOINT, credential.getEndpoint());
@@ -135,6 +138,10 @@ public static CloudCredentialWithEndpoint getEnvironmentCredentialWithEndpoint(M
     }
 
     public static String getRegionOfEndpoint(String endpoint) {
+        if (IPV4_PORT_PATTERN.matcher(endpoint).find()) {
+            // if endpoint is like '192.168.0.1:8999', return null region
+            return null;
+        }
         String[] endpointSplit = endpoint.split("\\.");
         if (endpointSplit.length < 2) {
             return null;