diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index 900c9bafb4ff..725e90e2adfb 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -127,6 +127,12 @@ object CHBackendSettings extends BackendSettingsApi with Logging { val GLUTEN_AQE_PROPAGATEEMPTY: String = CHConf.prefixOf("aqe.propagate.empty.relation") + val GLUTEN_CLICKHOUSE_DELTA_SCAN_CACHE_SIZE: String = CHConf.prefixOf("deltascan.cache.size") + val GLUTEN_CLICKHOUSE_ADDFILES_TO_MTPS_CACHE_SIZE: String = + CHConf.prefixOf("addfiles.to.mtps.cache.size") + val GLUTEN_CLICKHOUSE_TABLE_PATH_TO_MTPS_CACHE_SIZE: String = + CHConf.prefixOf("table.path.to.mtps.cache.size") + def affinityMode: String = { SparkEnv.get.conf .get( diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala index c8d2d1c77ec7..f28e9e8fe468 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala @@ -16,6 +16,9 @@ */ package org.apache.spark.sql.delta +import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings + +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BindReferences, Expression, Predicate} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.stats.DeltaScan @@ -82,14 +85,20 @@ case class FilterExprsAsKey( object ClickhouseSnapshot { val deltaScanCache: Cache[FilterExprsAsKey, DeltaScan] = CacheBuilder.newBuilder - .maximumSize(100) - .expireAfterAccess(3600L, TimeUnit.SECONDS) + .maximumSize( + SparkSession.getActiveSession.get.conf + .get(CHBackendSettings.GLUTEN_CLICKHOUSE_DELTA_SCAN_CACHE_SIZE, "10000") + .toLong) + .expireAfterAccess(7200L, TimeUnit.SECONDS) .recordStats() .build() val addFileToAddMTPCache: LoadingCache[AddFileAsKey, AddMergeTreeParts] = CacheBuilder.newBuilder - .maximumSize(1000000) - .expireAfterAccess(3600L, TimeUnit.SECONDS) + .maximumSize( + SparkSession.getActiveSession.get.conf + .get(CHBackendSettings.GLUTEN_CLICKHOUSE_ADDFILES_TO_MTPS_CACHE_SIZE, "1000000") + .toLong) + .expireAfterAccess(7200L, TimeUnit.SECONDS) .recordStats .build[AddFileAsKey, AddMergeTreeParts](new CacheLoader[AddFileAsKey, AddMergeTreeParts]() { @throws[Exception] @@ -99,8 +108,11 @@ object ClickhouseSnapshot { }) val pathToAddMTPCache: Cache[String, AddMergeTreeParts] = CacheBuilder.newBuilder - .maximumSize(1000000) - .expireAfterAccess(3600L, TimeUnit.SECONDS) + .maximumSize( + SparkSession.getActiveSession.get.conf + .get(CHBackendSettings.GLUTEN_CLICKHOUSE_TABLE_PATH_TO_MTPS_CACHE_SIZE, "1000000") + .toLong) + .expireAfterAccess(7200L, TimeUnit.SECONDS) .recordStats() .build()