Merge branch 'release-3.1' into cherry-pick-2675-to-release-3.1

.github/workflows/codeql-analysis.yml

@@ -60,8 +60,11 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+#    - name: Autobuild
+#      uses: github/codeql-action/autobuild@v1
+    - name: build
+      run: |
+          mvn clean package -Dmaven.test.skip=true -B
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl

assembly/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

core-test/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

core/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

core/src/main/scala/com/pingcap/tispark/TiConfigConst.scala

@@ -85,4 +85,7 @@ object TiConfigConst {
   val DEFAULT_GC_MAX_WAIT_TIME: Long = 24 * 60 * 60
   val DEFAULT_GC_SAFE_POINT_TTL: Int = 5 * 60
 
+  // cache load
+  val LOAD_TABLES: String = "spark.tispark.load_tables"
+  val DEFAULT_LOAD_TABLES: Boolean = true
 }

core/src/main/scala/com/pingcap/tispark/utils/TiUtil.scala

@@ -237,6 +237,8 @@ object TiUtil {
       tiConf.setNewCollationEnable(conf.get(TiConfigConst.NEW_COLLATION_ENABLE).toBoolean)
     }
 
+    tiConf.setLoadTables(
+      conf.get(TiConfigConst.LOAD_TABLES, TiConfigConst.DEFAULT_LOAD_TABLES.toString).toBoolean)
     tiConf
   }
 

db-random-test/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

docs/userguide_3.0.md

@@ -277,6 +277,7 @@ spark.sql("select t1.id,t2.id from spark_catalog.default.t t1 left join tidb_cat
 | `spark.tispark.replica_read.label`             | ""               | Only select TiKV store match specified labels. Format: label_x=value_x,label_y=value_y                                                                                                                                                                                                                                                                                                                                                            |
 | `spark.tispark.replica_read.address_whitelist` | ""               | Only select TiKV store with given ip addresses. Split mutil addresses by `,`                                                                                                                                                                                                                                                                                                                                                                      |
 | `spark.tispark.replica_read.address_blacklist` | ""               | Do not select TiKV store with given ip addresses. Split mutil addresses by `,`                                                                                                                                                                                                                                                                                                                                                                    |
+| `spark.tispark.load_tables`                     | true             | (experimental) Whether load all tables when we reload catalog cache. Disable it may cause table not find in scenarios where the table changes frequently.                                                                                                                                                                                                                                                                                           |
 
 ### TLS Configuration
 
@@ -385,13 +386,15 @@ TiSpark reads the range and hash partition table from TiDB.
 
 Currently, TiSpark doesn't support a MySQL/TiDB partition table syntax `select col_name from table_name partition(partition_name)`, but you can still use `where` condition to filter the partitions.
 
-TiSpark decides whether to apply partition pruning according to the partition type and the partition expression associated with the table. Currently, TiSpark partially apply partition pruning on range partition.
+TiSpark decides whether to apply partition pruning according to the partition type and the partition expression associated with the table.
+
+Currently, TiSpark partially apply partition pruning on range partition.
 
 The partition pruning is applied when the partition expression of the range partition is one of the following:
 
 + column expression
-+ `YEAR(col)` and its type is datetime/string/date literal that can be parsed as datetime.
-+ `TO_DAYS(col)` and its type is datetime/string/date literal that can be parsed as datetime.
++ `YEAR($argument)` where the argument is a column and its type is datetime or string literal
+  that can be parsed as datetime.
 
 If partition pruning is not applied, TiSpark's reading is equivalent to doing a table scan over all partitions.
 

pom.xml

@@ -10,7 +10,7 @@
 
     <groupId>com.pingcap.tispark</groupId>
     <artifactId>tispark-parent</artifactId>
-    <version>3.1.2</version>
+    <version>3.1.3</version>
     <packaging>pom</packaging>
     <name>TiSpark Project Parent POM</name>
     <url>http://github.copm/pingcap/tispark</url>

spark-wrapper/spark-3.0/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

spark-wrapper/spark-3.1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

spark-wrapper/spark-3.2/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

spark-wrapper/spark-3.3/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.pingcap.tispark</groupId>
     <artifactId>tispark-parent</artifactId>
-    <version>3.1.2</version>
+    <version>3.1.3</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

tikv-client/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

tikv-client/src/main/java/com/pingcap/tikv/TiConfiguration.java

@@ -48,6 +48,12 @@ public class TiConfiguration implements Serializable {
   private static final boolean DEF_IGNORE_TRUNCATE = true;
   private static final boolean DEF_TRUNCATE_AS_WARNING = false;
   private static final int DEF_MAX_FRAME_SIZE = 2147483647; // 2 GB
+  private boolean loadTables = true;
+
+  public boolean getLoadTables() {
+    return loadTables;
+  }
+
   private static final int DEF_INDEX_SCAN_BATCH_SIZE = 20000;
   private static final int DEF_REGION_SCAN_DOWNGRADE_THRESHOLD = 10000000;
   // if keyRange size per request exceeds this limit, the request might be too large to be accepted

tikv-client/src/main/java/com/pingcap/tikv/TiSession.java

@@ -197,9 +197,11 @@ public synchronized Catalog getOrCreateSnapShotCatalog(TiTimestamp ts) {
     if (snapshotCatalog == null) {
       snapshotCatalog =
           new Catalog(
-              this::createSnapshotWithSnapshotTimestamp, conf.isShowRowId(), conf.getDBPrefix());
+              this::createSnapshotWithSnapshotTimestamp,
+              conf.isShowRowId(),
+              conf.getDBPrefix(),
+              conf.getLoadTables());
     }
-    snapshotCatalog.reloadCache(true);
     return snapshotCatalog;
   }
 
@@ -208,7 +210,12 @@ public Catalog getCatalog() {
     if (res == null) {
       synchronized (this) {
         if (catalog == null) {
-          catalog = new Catalog(this::createSnapshot, conf.isShowRowId(), conf.getDBPrefix());
+          catalog =
+              new Catalog(
+                  this::createSnapshot,
+                  conf.isShowRowId(),
+                  conf.getDBPrefix(),
+                  conf.getLoadTables());
         }
         res = catalog;
       }

tikv-client/src/main/java/com/pingcap/tikv/catalog/Catalog.java

@@ -41,13 +41,16 @@ public class Catalog implements AutoCloseable {
   private final Supplier<Snapshot> snapshotProvider;
   private CatalogCache metaCache;
   private static final AtomicLong lastUpdateTime = new AtomicLong(0);
+  private final boolean loadTables;
 
-  public Catalog(Supplier<Snapshot> snapshotProvider, boolean showRowId, String dbPrefix) {
+  public Catalog(
+      Supplier<Snapshot> snapshotProvider, boolean showRowId, String dbPrefix, boolean loadTables) {
     this.snapshotProvider = Objects.requireNonNull(snapshotProvider, "Snapshot Provider is null");
     this.showRowId = showRowId;
     this.dbPrefix = dbPrefix;
-    metaCache = new CatalogCache(new CatalogTransaction(snapshotProvider.get()), dbPrefix, false);
-    reloadCache(true);
+    this.loadTables = loadTables;
+    metaCache =
+        new CatalogCache(new CatalogTransaction(snapshotProvider.get()), dbPrefix, loadTables);
   }
 
   @Override
@@ -67,18 +70,14 @@ public void reloadCache(boolean loadTables) {
     }
   }
 
-  private void reloadCache() {
-    reloadCache(false);
-  }
-
   public List<TiDBInfo> listDatabases() {
-    reloadCache();
+    reloadCache(false);
     return metaCache.listDatabases();
   }
 
   public List<TiTableInfo> listTables(TiDBInfo database) {
     Objects.requireNonNull(database, "database is null");
-    reloadCache(true);
+    reloadCache(loadTables);
     if (showRowId) {
       return metaCache
           .listTables(database)
@@ -133,7 +132,7 @@ public TiTableInfo getTableFromCache(TiDBInfo database, String tableName) {
 
   public TiDBInfo getDatabase(String dbName) {
     Objects.requireNonNull(dbName, "dbName is null");
-    reloadCache();
+    reloadCache(false);
     return metaCache.getDatabase(dbName);
   }
 
@@ -148,7 +147,7 @@ public TiTableInfo getTable(String dbName, String tableName) {
   public TiTableInfo getTable(TiDBInfo database, String tableName) {
     Objects.requireNonNull(database, "database is null");
     Objects.requireNonNull(tableName, "tableName is null");
-    reloadCache(true);
+    reloadCache(loadTables);
     TiTableInfo table = metaCache.getTable(database, tableName);
     if (showRowId && table != null) {
       return table.copyTableWithRowId();

tikv-client/src/test/java/com/pingcap/tikv/catalog/CatalogTest.java

@@ -70,8 +70,8 @@ public void listDatabasesTest() {
     helper.addDatabase(265, "other");
     helper.setSchemaVersion(667);
 
-    ReflectionWrapper wrapper = new ReflectionWrapper(cat);
-    wrapper.call("reloadCache");
+    ReflectionWrapper wrapper = new ReflectionWrapper(cat, boolean.class);
+    wrapper.call("reloadCache", false);
 
     dbs = cat.listDatabases();
     assertEquals(3, dbs.size());