[improvement](jdbc catalog) Optimize JdbcCatalog case mapping stabili…

…ty (#40891)

This PR makes the following changes to the uppercase and lowercase
mapping of JdbcCatalog
1. The identifierMapping is managed by JdbcExternalCatalog instead of
JdbcClient to better control its lifecycle
2. The identifierMapping no longer loads remoteName alone, but Catalog
controls the loading uniformly
3. The identifierMapping will be loaded when each FE performs
makeSureInitialized() to ensure that each FE has a mapping
4. The initialization of mapping will only be performed once in
makeSureInitialized(), which means that even if you use metaCache, if
your source data is updated when identifierMapping is enabled, you must
refresh the catalog to query normally.
5. The identifierMapping is only responsible for the properties of the
Catalog and is no longer affected by the fe config, simplifying the
processing logic
6. If lower_case_mete_names is false and meta_names_mapping is empty in
the catalog properties, the identifierMapping will no longer take
effect, further enhancing the stability of the default settings
7. The JdbcClient is no longer closed during onRefreshCache, reducing
the repeated creation of resources, improving reuse, and reducing the
leakage of some global shared threads

fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java

@@ -48,6 +48,7 @@
 import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase;
 import org.apache.doris.datasource.jdbc.JdbcExternalDatabase;
 import org.apache.doris.datasource.lakesoul.LakeSoulExternalDatabase;
+import org.apache.doris.datasource.mapping.IdentifierMapping;
 import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase;
 import org.apache.doris.datasource.metacache.MetaCache;
 import org.apache.doris.datasource.operations.ExternalMetadataOps;
@@ -149,6 +150,9 @@ public abstract class ExternalCatalog
     protected Optional<Boolean> useMetaCache = Optional.empty();
     protected MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache;
 
+    protected IdentifierMapping identifierMapping;
+    private boolean mappingsInitialized = false;
+
     public ExternalCatalog() {
     }
 
@@ -181,6 +185,10 @@ protected List<String> listDatabaseNames() {
         }
     }
 
+    // only for forward to master
+    protected void buildDatabaseMapping() {
+    }
+
     // Will be called when creating catalog(so when as replaying)
     // to add some default properties if missing.
     public void setDefaultPropsIfMissing(boolean isReplay) {
@@ -209,6 +217,10 @@ public void checkWhenCreating() throws DdlException {
      */
     public abstract List<String> listTableNames(SessionContext ctx, String dbName);
 
+    // only for forward to master
+    protected void buildTableMapping(SessionContext ctx, String dbName) {
+    }
+
     /**
      * check if the specified table exist.
      *
@@ -273,6 +285,10 @@ public final synchronized void makeSureInitialized() {
             }
             initialized = true;
         }
+        if (!mappingsInitialized) {
+            buildDatabaseMapping();
+            mappingsInitialized = true;
+        }
     }
 
     protected final void initLocalObjects() {
@@ -398,6 +414,7 @@ private List<String> getFilteredDatabaseNames() {
     public void onRefresh(boolean invalidCache) {
         this.objectCreated = false;
         this.initialized = false;
+        this.mappingsInitialized = false;
         synchronized (this.propLock) {
             this.convertedProperties = null;
         }
@@ -756,6 +773,7 @@ public void gsonPostProcess() throws IOException {
         }
         this.propLock = new byte[0];
         this.initialized = false;
+        this.mappingsInitialized = false;
         setDefaultPropsIfMissing(true);
         if (tableAutoAnalyzePolicy == null) {
             tableAutoAnalyzePolicy = Maps.newHashMap();

fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java

@@ -91,6 +91,8 @@ public abstract class ExternalDatabase<T extends ExternalTable>
 
     private MetaCache<T> metaCache;
 
+    private boolean mappingsInitialized = false;
+
     /**
      * Create external database.
      *
@@ -117,6 +119,7 @@ public void setTableExtCatalog(ExternalCatalog extCatalog) {
 
     public void setUnInitialized(boolean invalidCache) {
         this.initialized = false;
+        this.mappingsInitialized = false;
         this.invalidCacheInInit = invalidCache;
         if (extCatalog.getUseMetaCache().isPresent()) {
             if (extCatalog.getUseMetaCache().get() && metaCache != null) {
@@ -170,6 +173,10 @@ public final synchronized void makeSureInitialized() {
             }
             initialized = true;
         }
+        if (!mappingsInitialized) {
+            extCatalog.buildTableMapping(null, name);
+            mappingsInitialized = true;
+        }
     }
 
     public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) {

fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java

@@ -32,6 +32,7 @@
 import org.apache.doris.datasource.jdbc.client.JdbcClient;
 import org.apache.doris.datasource.jdbc.client.JdbcClientConfig;
 import org.apache.doris.datasource.jdbc.client.JdbcClientException;
+import org.apache.doris.datasource.mapping.DefaultIdentifierMapping;
 import org.apache.doris.proto.InternalService;
 import org.apache.doris.proto.InternalService.PJdbcTestConnectionRequest;
 import org.apache.doris.proto.InternalService.PJdbcTestConnectionResult;
@@ -119,19 +120,16 @@ public void onRefresh(boolean invalidCache) {
         super.onRefresh(invalidCache);
         if (jdbcClient != null) {
             jdbcClient.closeClient();
+            jdbcClient = null;
         }
     }
 
-    @Override
-    public void onRefreshCache(boolean invalidCache) {
-        onRefresh(invalidCache);
-    }
-
     @Override
     public void onClose() {
         super.onClose();
         if (jdbcClient != null) {
             jdbcClient.closeClient();
+            jdbcClient = null;
         }
     }
 
@@ -232,8 +230,6 @@ protected void initLocalObjectsImpl() {
                 .setDriverUrl(getDriverUrl())
                 .setDriverClass(getDriverClass())
                 .setOnlySpecifiedDatabase(getOnlySpecifiedDatabase())
-                .setIsLowerCaseMetaNames(getLowerCaseMetaNames())
-                .setMetaNamesMapping(getMetaNamesMapping())
                 .setIncludeDatabaseMap(getIncludeDatabaseMap())
                 .setExcludeDatabaseMap(getExcludeDatabaseMap())
                 .setConnectionPoolMinSize(getConnectionPoolMinSize())
@@ -243,22 +239,62 @@ protected void initLocalObjectsImpl() {
                 .setConnectionPoolKeepAlive(isConnectionPoolKeepAlive());
 
         jdbcClient = JdbcClient.createJdbcClient(jdbcClientConfig);
+        identifierMapping = new DefaultIdentifierMapping(Boolean.parseBoolean(getLowerCaseMetaNames()),
+                getMetaNamesMapping());
     }
 
+    @Override
     protected List<String> listDatabaseNames() {
-        return jdbcClient.getDatabaseNameList();
+        return identifierMapping.fromRemoteDatabaseName(jdbcClient.getDatabaseNameList());
+    }
+
+    @Override
+    protected void buildDatabaseMapping() {
+        identifierMapping.fromRemoteDatabaseName(jdbcClient.getDatabaseNameList());
+    }
+
+    protected String getRemoteDatabaseName(String dbName) {
+        return identifierMapping.toRemoteDatabaseName(dbName);
     }
 
     @Override
     public List<String> listTableNames(SessionContext ctx, String dbName) {
         makeSureInitialized();
-        return jdbcClient.getTablesNameList(dbName);
+        String remoteDbName = getRemoteDatabaseName(dbName);
+        return identifierMapping.fromRemoteTableName(remoteDbName, jdbcClient.getTablesNameList(remoteDbName));
+    }
+
+    @Override
+    protected void buildTableMapping(SessionContext ctx, String dbName) {
+        String remoteDbName = getRemoteDatabaseName(dbName);
+        identifierMapping.fromRemoteTableName(getRemoteDatabaseName(dbName),
+                jdbcClient.getTablesNameList(remoteDbName));
+    }
+
+    protected String getRemoteTableName(String dbName, String tblName) {
+        return identifierMapping.toRemoteTableName(getRemoteDatabaseName(dbName), tblName);
     }
 
     @Override
     public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
         makeSureInitialized();
-        return jdbcClient.isTableExist(dbName, tblName);
+        String remoteDbName = getRemoteDatabaseName(dbName);
+        String remoteTblName = getRemoteTableName(dbName, tblName);
+        return jdbcClient.isTableExist(remoteDbName, remoteTblName);
+    }
+
+    public List<Column> listColumns(String dbName, String tblName) {
+        makeSureInitialized();
+        String remoteDbName = getRemoteDatabaseName(dbName);
+        String remoteTblName = getRemoteTableName(dbName, tblName);
+        return identifierMapping.fromRemoteColumnName(remoteDbName, remoteTblName,
+                jdbcClient.getColumnsFromJdbc(remoteDbName,
+                        remoteTblName));
+    }
+
+    protected Map<String, String> getRemoteColumnNames(String dbName, String tblName) {
+        return identifierMapping.toRemoteColumnNames(getRemoteDatabaseName(dbName),
+                getRemoteTableName(dbName, tblName));
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java

@@ -32,6 +32,7 @@
 import org.apache.doris.statistics.util.StatisticsUtil;
 import org.apache.doris.thrift.TTableDescriptor;
 
+import com.google.common.collect.Maps;
 import org.apache.commons.text.StringSubstitutor;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -86,21 +87,29 @@ public TTableDescriptor toThrift() {
 
     @Override
     public Optional<SchemaCacheValue> initSchema() {
-        return Optional.of(new SchemaCacheValue(((JdbcExternalCatalog) catalog).getJdbcClient()
-                .getColumnsFromJdbc(dbName, name)));
+        return Optional.of(new SchemaCacheValue(((JdbcExternalCatalog) catalog).listColumns(dbName, name)));
     }
 
     private JdbcTable toJdbcTable() {
         List<Column> schema = getFullSchema();
         JdbcExternalCatalog jdbcCatalog = (JdbcExternalCatalog) catalog;
-        String fullDbName = this.dbName + "." + this.name;
-        JdbcTable jdbcTable = new JdbcTable(this.id, fullDbName, schema, TableType.JDBC_EXTERNAL_TABLE);
-        jdbcCatalog.configureJdbcTable(jdbcTable, fullDbName);
+        String fullTableName = this.dbName + "." + this.name;
+        JdbcTable jdbcTable = new JdbcTable(this.id, fullTableName, schema, TableType.JDBC_EXTERNAL_TABLE);
+        jdbcCatalog.configureJdbcTable(jdbcTable, fullTableName);
 
         // Set remote properties
-        jdbcTable.setRemoteDatabaseName(jdbcCatalog.getJdbcClient().getRemoteDatabaseName(this.dbName));
-        jdbcTable.setRemoteTableName(jdbcCatalog.getJdbcClient().getRemoteTableName(this.dbName, this.name));
-        jdbcTable.setRemoteColumnNames(jdbcCatalog.getJdbcClient().getRemoteColumnNames(this.dbName, this.name));
+        jdbcTable.setRemoteDatabaseName(jdbcCatalog.getRemoteDatabaseName(this.dbName));
+        jdbcTable.setRemoteTableName(jdbcCatalog.getRemoteTableName(this.dbName, this.name));
+        Map<String, String> remoteColumnNames = jdbcCatalog.getRemoteColumnNames(this.dbName, this.name);
+        if (!remoteColumnNames.isEmpty()) {
+            jdbcTable.setRemoteColumnNames(remoteColumnNames);
+        } else {
+            remoteColumnNames = Maps.newHashMap();
+            for (Column column : schema) {
+                remoteColumnNames.put(column.getName(), column.getName());
+            }
+            jdbcTable.setRemoteColumnNames(remoteColumnNames);
+        }
 
         return jdbcTable;
     }