feat: table data cache for object storage

scripts/ci/deploy/config/databend-query-node-1.toml

@@ -33,13 +33,6 @@ cluster_id = "test_cluster"
 table_engine_memory_enabled = true
 database_engine_github_enabled = true
 
-table_meta_cache_enabled = true
-table_memory_cache_mb_size = 1024
-table_disk_cache_root = "_cache"
-table_disk_cache_mb_size = 10240
-table_cache_bloom_index_meta_count=3000
-table_cache_bloom_index_filter_count=1048576
-
 # [[query.users]]
 # name = "admin"
 # auth_type = "no_password"
@@ -110,3 +103,45 @@ data_path = "./.databend/stateless_test_data"
 # endpoint_url = "<your-endpoint>"
 # access_key_id = "<your-key-id>"
 # access_key_secret = "<your-account-key>"
+
+
+[cache]
+
+### table meta caches ###
+# Enable table meta cache. Default is true.
+# Set it to false wll disable all the table meta caches
+enable_table_meta_cache = true
+# Max number of cached table snapshot. Set it to 0 to disable it.
+table_meta_snapshot_count = 256
+# Max number of cached table segment. Set it to 0 to disable it.
+table_meta_segment_count = 10240
+# Max number of cached table statistic meta. Set it to 0 to disable it.
+table_meta_statistic_count = 256
+
+### table bloom index caches ###
+# Enable bloom index cache. Default is true
+# Set it to false will disable all the bloom index caches
+enable_table_bloom_index_cache = true
+# Max number of cached bloom index meta objects. Set it to 0 to disable it.
+table_bloom_index_meta_count = 3000
+# Max number of cached bloom index filters. Set it to 0 to disable it.
+table_bloom_index_filter_count = 1048576
+
+### table data caches ###
+
+# Type of storage to keep the table data cache
+#
+# available options: [none|disk]
+# default is "none", which disable table data cache
+# use "disk" to enabled disk cache
+data_cache_storage = "none"
+
+# Max size of external cache population queue length
+table_data_cache_population_queue_size = 65535
+
+
+[cache.disk]
+# cache path
+path = "./databend/_cache"
+# max bytes of cached data 20G
+max_bytes = 21474836480

scripts/ci/deploy/config/databend-query-node-2.toml

@@ -33,13 +33,6 @@ cluster_id = "test_cluster"
 table_engine_memory_enabled = true
 database_engine_github_enabled = true
 
-table_meta_cache_enabled = true
-table_memory_cache_mb_size = 1024
-table_disk_cache_root = "./.databend/cache"
-table_disk_cache_mb_size = 10240
-table_cache_bloom_index_meta_count=3000
-table_cache_bloom_index_filter_count=1048576
-
 [log]
 
 [log.file]
@@ -64,3 +57,44 @@ type = "fs"
 # Comment out this block if you're NOT using local file system as storage.
 [storage.fs]
 data_path = "./.databend/stateless_test_data"
+
+[cache]
+
+### table meta caches ###
+# Enable table meta cache. Default is true.
+# Set it to false wll disable all the table meta caches
+enable_table_meta_cache = true
+# Max number of cached table snapshot. Set it to 0 to disable it.
+table_meta_snapshot_count = 256
+# Max number of cached table segment. Set it to 0 to disable it.
+table_meta_segment_count = 10240
+# Max number of cached table statistic meta. Set it to 0 to disable it.
+table_meta_statistic_count = 256
+
+### table bloom index caches ###
+# Enable bloom index cache. Default is true
+# Set it to false will disable all the bloom index caches
+enable_table_bloom_index_cache = true
+# Max number of cached bloom index meta objects. Set it to 0 to disable it.
+table_bloom_index_meta_count = 3000
+# Max number of cached bloom index filters. Set it to 0 to disable it.
+table_bloom_index_filter_count = 1048576
+
+### table data caches ###
+
+# Type of storage to keep the table data cache
+#
+# available options: [none|disk]
+# default is "none", which disable table data cache
+# use "disk" to enabled disk cache
+data_cache_storage = "none"
+
+# Max size of external cache population queue length
+table_data_cache_population_queue_size = 65535
+
+
+[cache.disk]
+# cache path
+path = "./databend/_cache"
+# max bytes of cached data 20G
+max_bytes = 21474836480

scripts/ci/deploy/config/databend-query-node-3.toml

@@ -34,13 +34,6 @@ cluster_id = "test_cluster"
 table_engine_memory_enabled = true
 database_engine_github_enabled = true
 
-table_meta_cache_enabled = true
-table_memory_cache_mb_size = 1024
-table_disk_cache_root = "./.databend/cache"
-table_disk_cache_mb_size = 10240
-table_cache_bloom_index_meta_count=3000
-table_cache_bloom_index_filter_count=1048576
-
 [log]
 
 [log.file]
@@ -65,3 +58,44 @@ type = "fs"
 # Comment out this block if you're NOT using local file system as storage.
 [storage.fs]
 data_path = "./.databend/stateless_test_data"
+
+[cache]
+
+### table meta caches ###
+# Enable table meta cache. Default is true.
+# Set it to false wll disable all the table meta caches
+enable_table_meta_cache = true
+# Max number of cached table snapshot. Set it to 0 to disable it.
+table_meta_snapshot_count = 256
+# Max number of cached table segment. Set it to 0 to disable it.
+table_meta_segment_count = 10240
+# Max number of cached table statistic meta. Set it to 0 to disable it.
+table_meta_statistic_count = 256
+
+### table bloom index caches ###
+# Enable bloom index cache. Default is true
+# Set it to false will disable all the bloom index caches
+enable_table_bloom_index_cache = true
+# Max number of cached bloom index meta objects. Set it to 0 to disable it.
+table_bloom_index_meta_count = 3000
+# Max number of cached bloom index filters. Set it to 0 to disable it.
+table_bloom_index_filter_count = 1048576
+
+### table data caches ###
+
+# Type of storage to keep the table data cache
+#
+# available options: [none|disk]
+# default is "none", which disable table data cache
+# use "disk" to enabled disk cache
+data_cache_storage = "none"
+
+# Max size of external cache population queue length
+table_data_cache_population_queue_size = 65535
+
+
+[cache.disk]
+# cache path
+path = "./databend/_cache"
+# max bytes of cached data 20G
+max_bytes = 21474836480

scripts/ci/deploy/config/databend-query-node-shared.toml

@@ -33,13 +33,6 @@ cluster_id = "test_cluster"
 table_engine_memory_enabled = true
 database_engine_github_enabled = true
 
-table_meta_cache_enabled = true
-table_memory_cache_mb_size = 1024
-table_disk_cache_root = "_cache"
-table_disk_cache_mb_size = 10240
-table_cache_bloom_index_meta_count=3000
-table_cache_bloom_index_filter_count=1048576
-
 share_endpoint_address = "127.0.0.1:33003" # receive shared information from open sharing
 # [[query.users]]
 # name = "admin"
@@ -110,3 +103,44 @@ data_path = "./.databend/stateless_test_data"
 # endpoint_url = "<your-endpoint>"
 # access_key_id = "<your-key-id>"
 # access_key_secret = "<your-account-key>"
+
+[cache]
+
+### table meta caches ###
+# Enable table meta cache. Default is true.
+# Set it to false wll disable all the table meta caches
+enable_table_meta_cache = true
+# Max number of cached table snapshot. Set it to 0 to disable it.
+table_meta_snapshot_count = 256
+# Max number of cached table segment. Set it to 0 to disable it.
+table_meta_segment_count = 10240
+# Max number of cached table statistic meta. Set it to 0 to disable it.
+table_meta_statistic_count = 256
+
+### table bloom index caches ###
+# Enable bloom index cache. Default is true
+# Set it to false will disable all the bloom index caches
+enable_table_bloom_index_cache = true
+# Max number of cached bloom index meta objects. Set it to 0 to disable it.
+table_bloom_index_meta_count = 3000
+# Max number of cached bloom index filters. Set it to 0 to disable it.
+table_bloom_index_filter_count = 1048576
+
+### table data caches ###
+
+# Type of storage to keep the table data cache
+#
+# available options: [none|disk]
+# default is "none", which disable table data cache
+# use "disk" to enabled disk cache
+data_cache_storage = "none"
+
+# Max size of external cache population queue length
+table_data_cache_population_queue_size = 65535
+
+
+[cache.disk]
+# cache path
+path = "./databend/_cache"
+# max bytes of cached data 20G
+max_bytes = 21474836480

src/binaries/query/local.rs

@@ -16,7 +16,7 @@ use std::time::Instant;
 
 use comfy_table::Cell;
 use comfy_table::Table;
-use common_config::Config;
+use common_config::InnerConfig;
 use common_exception::ErrorCode;
 use common_exception::Result;
 use common_expression::DataBlock;
@@ -28,7 +28,7 @@ use databend_query::sql::Planner;
 use databend_query::GlobalServices;
 use tokio_stream::StreamExt;
 
-pub async fn query_local(conf: &Config) -> Result<()> {
+pub async fn query_local(conf: &InnerConfig) -> Result<()> {
     let mut conf = conf.clone();
     conf.storage.allow_insecure = true;
     let local_conf = conf.local.clone();

src/binaries/query/main.rs

@@ -23,7 +23,7 @@ use common_base::mem_allocator::GlobalAllocator;
 use common_base::runtime::Runtime;
 use common_base::runtime::GLOBAL_MEM_STAT;
 use common_base::set_alloc_error_hook;
-use common_config::Config;
+use common_config::InnerConfig;
 use common_config::DATABEND_COMMIT_VERSION;
 use common_config::QUERY_SEMVER;
 use common_exception::Result;
@@ -62,7 +62,7 @@ fn main() {
 }
 
 async fn main_entrypoint() -> Result<()> {
-    let conf: Config = Config::load()?;
+    let conf: InnerConfig = InnerConfig::load()?;
 
     if run_cmd(&conf).await? {
         return Ok(());
@@ -310,7 +310,7 @@ async fn main_entrypoint() -> Result<()> {
     Ok(())
 }
 
-async fn run_cmd(conf: &Config) -> Result<bool> {
+async fn run_cmd(conf: &InnerConfig) -> Result<bool> {
     if conf.cmd.is_empty() {
         return Ok(false);
     }

src/common/cache/Cargo.toml

@@ -14,14 +14,17 @@ test = false
 heapsize = ["heapsize_"]
 amortized = ["ritelinked/ahash-amortized", "ritelinked/inline-more-amortized"]
 
-[dependencies] # In alphabetical order
-# Github dependencies
+[dependencies]
 
 # Crates.io dependencies
+crc32fast = "1.3.2"
+hex = "0.4.3"
 ritelinked = { version = "0.3.2", default-features = false, features = ["ahash", "inline-more"] }
+siphasher = "0.3.10"
+tracing = "0.1.36"
+walkdir = "2.3.2"
 
 [target.'cfg(not(target_os = "macos"))'.dependencies]
 heapsize_ = { package = "heapsize", version = "0.4.2", optional = true }
 
 [dev-dependencies]
-tempfile = "3.3.0"

src/common/cache/src/lib.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![feature(write_all_vectored)]
 #![allow(clippy::uninlined_format_args)]
 #[cfg(feature = "heapsize")]
 #[cfg(not(target_os = "macos"))]