apache · morningman · Sep 29, 2024 · Sep 17, 2024
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -47,6 +47,7 @@
 import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase;
 import org.apache.doris.datasource.jdbc.JdbcExternalDatabase;
 import org.apache.doris.datasource.lakesoul.LakeSoulExternalDatabase;
+import org.apache.doris.datasource.mapping.IdentifierMapping;
 import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase;
 import org.apache.doris.datasource.metacache.MetaCache;
 import org.apache.doris.datasource.operations.ExternalMetadataOps;
@@ -148,6 +149,9 @@ public abstract class ExternalCatalog
     protected Optional<Boolean> useMetaCache = Optional.empty();
     protected MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache;
 
+    protected IdentifierMapping identifierMapping;
+    private boolean mappingsInitialized = false;
+
     public ExternalCatalog() {
     }
 
@@ -180,6 +184,10 @@ protected List<String> listDatabaseNames() {
         }
     }
 
+    // only for forward to master
+    protected void buildDatabaseMapping() {
+    }
+
     // Will be called when creating catalog(so when as replaying)
     // to add some default properties if missing.
     public void setDefaultPropsIfMissing(boolean isReplay) {
@@ -208,6 +216,10 @@ public void checkWhenCreating() throws DdlException {
      */
     public abstract List<String> listTableNames(SessionContext ctx, String dbName);
 
+    // only for forward to master
+    protected void buildTableMapping(SessionContext ctx, String dbName) {
+    }
+
     /**
      * check if the specified table exist.
      *
@@ -272,6 +284,10 @@ public final synchronized void makeSureInitialized() {
             }
             initialized = true;
         }
+        if (!mappingsInitialized) {
+            buildDatabaseMapping();
+            mappingsInitialized = true;
+        }
     }
 
     protected final void initLocalObjects() {
@@ -397,6 +413,7 @@ private List<String> getFilteredDatabaseNames() {
     public void onRefresh(boolean invalidCache) {
         this.objectCreated = false;
         this.initialized = false;
+        this.mappingsInitialized = false;
         synchronized (this.propLock) {
             this.convertedProperties = null;
         }
@@ -726,6 +743,7 @@ public void gsonPostProcess() throws IOException {
         }
         this.propLock = new byte[0];
         this.initialized = false;
+        this.mappingsInitialized = false;
         setDefaultPropsIfMissing(true);
         if (tableAutoAnalyzePolicy == null) {
             tableAutoAnalyzePolicy = Maps.newHashMap();

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -91,6 +91,8 @@ public abstract class ExternalDatabase<T extends ExternalTable>
 
     private MetaCache<T> metaCache;
 
+    private boolean mappingsInitialized = false;
+
     /**
      * Create external database.
      *
@@ -117,6 +119,7 @@ public void setTableExtCatalog(ExternalCatalog extCatalog) {
 
     public void setUnInitialized(boolean invalidCache) {
         this.initialized = false;
+        this.mappingsInitialized = false;
         this.invalidCacheInInit = invalidCache;
         if (extCatalog.getUseMetaCache().isPresent()) {
             if (extCatalog.getUseMetaCache().get() && metaCache != null) {
@@ -170,6 +173,10 @@ public final synchronized void makeSureInitialized() {
             }
             initialized = true;
         }
+        if (!mappingsInitialized) {
+            extCatalog.buildTableMapping(null, name);
+            mappingsInitialized = true;
+        }
     }
 
     public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
@@ -32,6 +32,7 @@
 import org.apache.doris.datasource.jdbc.client.JdbcClient;
 import org.apache.doris.datasource.jdbc.client.JdbcClientConfig;
 import org.apache.doris.datasource.jdbc.client.JdbcClientException;
+import org.apache.doris.datasource.mapping.DefaultIdentifierMapping;
 import org.apache.doris.proto.InternalService;
 import org.apache.doris.proto.InternalService.PJdbcTestConnectionRequest;
 import org.apache.doris.proto.InternalService.PJdbcTestConnectionResult;
@@ -119,19 +120,16 @@ public void onRefresh(boolean invalidCache) {
         super.onRefresh(invalidCache);
         if (jdbcClient != null) {
             jdbcClient.closeClient();
+            jdbcClient = null;
         }
     }
 
-    @Override
-    public void onRefreshCache(boolean invalidCache) {
-        onRefresh(invalidCache);
-    }
-
     @Override
     public void onClose() {
         super.onClose();
         if (jdbcClient != null) {
             jdbcClient.closeClient();
+            jdbcClient = null;
         }
     }
 
@@ -232,8 +230,6 @@ protected void initLocalObjectsImpl() {
                 .setDriverUrl(getDriverUrl())
                 .setDriverClass(getDriverClass())
                 .setOnlySpecifiedDatabase(getOnlySpecifiedDatabase())
-                .setIsLowerCaseMetaNames(getLowerCaseMetaNames())
-                .setMetaNamesMapping(getMetaNamesMapping())
                 .setIncludeDatabaseMap(getIncludeDatabaseMap())
                 .setExcludeDatabaseMap(getExcludeDatabaseMap())
                 .setConnectionPoolMinSize(getConnectionPoolMinSize())
@@ -243,22 +239,62 @@ protected void initLocalObjectsImpl() {
                 .setConnectionPoolKeepAlive(isConnectionPoolKeepAlive());
 
         jdbcClient = JdbcClient.createJdbcClient(jdbcClientConfig);
+        identifierMapping = new DefaultIdentifierMapping(Boolean.parseBoolean(getLowerCaseMetaNames()),
+                getMetaNamesMapping());
     }
 
+    @Override
     protected List<String> listDatabaseNames() {
-        return jdbcClient.getDatabaseNameList();
+        return identifierMapping.fromRemoteDatabaseName(jdbcClient.getDatabaseNameList());
+    }
+
+    @Override
+    protected void buildDatabaseMapping() {
+        identifierMapping.fromRemoteDatabaseName(jdbcClient.getDatabaseNameList());
+    }
+
+    protected String getRemoteDatabaseName(String dbName) {
+        return identifierMapping.toRemoteDatabaseName(dbName);
     }
 
     @Override
     public List<String> listTableNames(SessionContext ctx, String dbName) {
         makeSureInitialized();
-        return jdbcClient.getTablesNameList(dbName);
+        String remoteDbName = getRemoteDatabaseName(dbName);
+        return identifierMapping.fromRemoteTableName(remoteDbName, jdbcClient.getTablesNameList(remoteDbName));
+    }
+
+    @Override
+    protected void buildTableMapping(SessionContext ctx, String dbName) {
+        String remoteDbName = getRemoteDatabaseName(dbName);
+        identifierMapping.fromRemoteTableName(getRemoteDatabaseName(dbName),
+                jdbcClient.getTablesNameList(remoteDbName));
+    }
+
+    protected String getRemoteTableName(String dbName, String tblName) {
+        return identifierMapping.toRemoteTableName(getRemoteDatabaseName(dbName), tblName);
     }
 
     @Override
     public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
         makeSureInitialized();
-        return jdbcClient.isTableExist(dbName, tblName);
+        String remoteDbName = getRemoteDatabaseName(dbName);
+        String remoteTblName = getRemoteTableName(dbName, tblName);
+        return jdbcClient.isTableExist(remoteDbName, remoteTblName);
+    }
+
+    public List<Column> listColumns(String dbName, String tblName) {
+        makeSureInitialized();
+        String remoteDbName = getRemoteDatabaseName(dbName);
+        String remoteTblName = getRemoteTableName(dbName, tblName);
+        return identifierMapping.fromRemoteColumnName(remoteDbName, remoteTblName,
+                jdbcClient.getColumnsFromJdbc(remoteDbName,
+                        remoteTblName));
+    }
+
+    protected Map<String, String> getRemoteColumnNames(String dbName, String tblName) {
+        return identifierMapping.toRemoteColumnNames(getRemoteDatabaseName(dbName),
+                getRemoteTableName(dbName, tblName));
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
@@ -32,6 +32,7 @@
 import org.apache.doris.statistics.util.StatisticsUtil;
 import org.apache.doris.thrift.TTableDescriptor;
 
+import com.google.common.collect.Maps;
 import org.apache.commons.text.StringSubstitutor;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -86,21 +87,29 @@ public TTableDescriptor toThrift() {
 
     @Override
     public Optional<SchemaCacheValue> initSchema() {
-        return Optional.of(new SchemaCacheValue(((JdbcExternalCatalog) catalog).getJdbcClient()
-                .getColumnsFromJdbc(dbName, name)));
+        return Optional.of(new SchemaCacheValue(((JdbcExternalCatalog) catalog).listColumns(dbName, name)));
     }
 
     private JdbcTable toJdbcTable() {
         List<Column> schema = getFullSchema();
         JdbcExternalCatalog jdbcCatalog = (JdbcExternalCatalog) catalog;
-        String fullDbName = this.dbName + "." + this.name;
-        JdbcTable jdbcTable = new JdbcTable(this.id, fullDbName, schema, TableType.JDBC_EXTERNAL_TABLE);
-        jdbcCatalog.configureJdbcTable(jdbcTable, fullDbName);
+        String fullTableName = this.dbName + "." + this.name;
+        JdbcTable jdbcTable = new JdbcTable(this.id, fullTableName, schema, TableType.JDBC_EXTERNAL_TABLE);
+        jdbcCatalog.configureJdbcTable(jdbcTable, fullTableName);
 
         // Set remote properties
-        jdbcTable.setRemoteDatabaseName(jdbcCatalog.getJdbcClient().getRemoteDatabaseName(this.dbName));
-        jdbcTable.setRemoteTableName(jdbcCatalog.getJdbcClient().getRemoteTableName(this.dbName, this.name));
-        jdbcTable.setRemoteColumnNames(jdbcCatalog.getJdbcClient().getRemoteColumnNames(this.dbName, this.name));
+        jdbcTable.setRemoteDatabaseName(jdbcCatalog.getRemoteDatabaseName(this.dbName));
+        jdbcTable.setRemoteTableName(jdbcCatalog.getRemoteTableName(this.dbName, this.name));
+        Map<String, String> remoteColumnNames = jdbcCatalog.getRemoteColumnNames(this.dbName, this.name);
+        if (!remoteColumnNames.isEmpty()) {
+            jdbcTable.setRemoteColumnNames(remoteColumnNames);
+        } else {
+            remoteColumnNames = Maps.newHashMap();
+            for (Column column : schema) {
+                remoteColumnNames.put(column.getName(), column.getName());
+            }
+            jdbcTable.setRemoteColumnNames(remoteColumnNames);
+        }
 
         return jdbcTable;
     }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java