diff --git a/Cargo.lock b/Cargo.lock index 33cb55bbf6f7..18a1a3d21522 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6488,6 +6488,7 @@ dependencies = [ "mur3", "object-store", "prometheus", + "serde", "serde_json", "snafu 0.8.5", "store-api", diff --git a/config/config.md b/config/config.md index f14148bfcc9c..4dc9dee00f53 100644 --- a/config/config.md +++ b/config/config.md @@ -170,6 +170,8 @@ | `region_engine.mito.memtable.data_freeze_threshold` | Integer | `32768` | The max rows of data inside the actively writing buffer in one shard.
Only available for `partition_tree` memtable. | | `region_engine.mito.memtable.fork_dictionary_bytes` | String | `1GiB` | Max dictionary bytes.
Only available for `partition_tree` memtable. | | `region_engine.file` | -- | -- | Enable the file engine. | +| `region_engine.metric` | -- | -- | Metric engine options. | +| `region_engine.metric.experimental_sparse_primary_key_encoding` | Bool | `false` | Whether to enable the experimental sparse primary key encoding. | | `logging` | -- | -- | The logging options. | | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. | | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. | @@ -506,6 +508,8 @@ | `region_engine.mito.memtable.data_freeze_threshold` | Integer | `32768` | The max rows of data inside the actively writing buffer in one shard.
Only available for `partition_tree` memtable. | | `region_engine.mito.memtable.fork_dictionary_bytes` | String | `1GiB` | Max dictionary bytes.
Only available for `partition_tree` memtable. | | `region_engine.file` | -- | -- | Enable the file engine. | +| `region_engine.metric` | -- | -- | Metric engine options. | +| `region_engine.metric.experimental_sparse_primary_key_encoding` | Bool | `false` | Whether to enable the experimental sparse primary key encoding. | | `logging` | -- | -- | The logging options. | | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. | | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 9d9ff5925dc5..5c8afd757ceb 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -622,6 +622,12 @@ fork_dictionary_bytes = "1GiB" ## Enable the file engine. [region_engine.file] +[[region_engine]] +## Metric engine options. +[region_engine.metric] +## Whether to enable the experimental sparse primary key encoding. +experimental_sparse_primary_key_encoding = false + ## The logging options. [logging] ## The directory to store the log files. If set to empty, logs will not be written to files. diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 275abf40e4a5..8e66122916ec 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -671,6 +671,12 @@ fork_dictionary_bytes = "1GiB" ## Enable the file engine. [region_engine.file] +[[region_engine]] +## Metric engine options. +[region_engine.metric] +## Whether to enable the experimental sparse primary key encoding. +experimental_sparse_primary_key_encoding = false + ## The logging options. [logging] ## The directory to store the log files. If set to empty, logs will not be written to files. diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs index 4e1b4f3195a7..2ac2ec0e44af 100644 --- a/src/datanode/src/config.rs +++ b/src/datanode/src/config.rs @@ -24,6 +24,7 @@ use common_telemetry::logging::{LoggingOptions, TracingOptions}; use common_wal::config::DatanodeWalConfig; use file_engine::config::EngineConfig as FileEngineConfig; use meta_client::MetaClientOptions; +use metric_engine::config::EngineConfig as MetricEngineConfig; use mito2::config::MitoConfig; use serde::{Deserialize, Serialize}; use servers::export_metrics::ExportMetricsOption; @@ -432,6 +433,8 @@ pub enum RegionEngineConfig { Mito(MitoConfig), #[serde(rename = "file")] File(FileEngineConfig), + #[serde(rename = "metric")] + Metric(MetricEngineConfig), } #[cfg(test)] diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs index 01e1e072c066..82c5f61d143b 100644 --- a/src/datanode/src/datanode.rs +++ b/src/datanode/src/datanode.rs @@ -395,6 +395,11 @@ impl DatanodeBuilder { plugins: Plugins, ) -> Result> { let mut engines = vec![]; + let mut metric_engine_config = opts.region_engine.iter().find_map(|c| match c { + RegionEngineConfig::Metric(config) => Some(config.clone()), + _ => None, + }); + for engine in &opts.region_engine { match engine { RegionEngineConfig::Mito(config) => { @@ -407,7 +412,10 @@ impl DatanodeBuilder { ) .await?; - let metric_engine = MetricEngine::new(mito_engine.clone()); + let metric_engine = MetricEngine::new( + mito_engine.clone(), + metric_engine_config.take().unwrap_or_default(), + ); engines.push(Arc::new(mito_engine) as _); engines.push(Arc::new(metric_engine) as _); } @@ -418,6 +426,9 @@ impl DatanodeBuilder { ); engines.push(Arc::new(engine) as _); } + RegionEngineConfig::Metric(_) => { + // Already handled in `build_mito_engine`. + } } } Ok(engines) diff --git a/src/metric-engine/Cargo.toml b/src/metric-engine/Cargo.toml index 85aa371594e8..64cbb51b38e8 100644 --- a/src/metric-engine/Cargo.toml +++ b/src/metric-engine/Cargo.toml @@ -27,6 +27,7 @@ mito2.workspace = true mur3 = "0.1" object-store.workspace = true prometheus.workspace = true +serde.workspace = true serde_json.workspace = true snafu.workspace = true store-api.workspace = true diff --git a/src/metric-engine/src/config.rs b/src/metric-engine/src/config.rs new file mode 100644 index 000000000000..b35b4121885b --- /dev/null +++ b/src/metric-engine/src/config.rs @@ -0,0 +1,20 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct EngineConfig { + pub experimental_sparse_primary_key_encoding: bool, +} diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs index 15b94701139b..cc6f7bd4955a 100644 --- a/src/metric-engine/src/engine.rs +++ b/src/metric-engine/src/engine.rs @@ -45,6 +45,7 @@ use store_api::region_request::RegionRequest; use store_api::storage::{RegionId, ScanRequest}; use self::state::MetricEngineState; +use crate::config::EngineConfig; use crate::data_region::DataRegion; use crate::error::{self, Result, UnsupportedRegionRequestSnafu}; use crate::metadata_region::MetadataRegion; @@ -255,7 +256,7 @@ impl RegionEngine for MetricEngine { } impl MetricEngine { - pub fn new(mito: MitoEngine) -> Self { + pub fn new(mito: MitoEngine, config: EngineConfig) -> Self { let metadata_region = MetadataRegion::new(mito.clone()); let data_region = DataRegion::new(mito.clone()); Self { @@ -264,6 +265,7 @@ impl MetricEngine { metadata_region, data_region, state: RwLock::default(), + config, }), } } @@ -304,6 +306,9 @@ struct MetricEngineInner { metadata_region: MetadataRegion, data_region: DataRegion, state: RwLock, + /// TODO(weny): remove it after the config is used. + #[allow(unused)] + config: EngineConfig, } #[cfg(test)] diff --git a/src/metric-engine/src/engine/create.rs b/src/metric-engine/src/engine/create.rs index b87682d4599e..fb7031e9daf8 100644 --- a/src/metric-engine/src/engine/create.rs +++ b/src/metric-engine/src/engine/create.rs @@ -549,6 +549,7 @@ mod test { use store_api::metric_engine_consts::{METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY}; use super::*; + use crate::config::EngineConfig; use crate::engine::MetricEngine; use crate::test_util::TestEnv; @@ -707,7 +708,7 @@ mod test { // set up let env = TestEnv::new().await; - let engine = MetricEngine::new(env.mito()); + let engine = MetricEngine::new(env.mito(), EngineConfig::default()); let engine_inner = engine.inner; // check create data region request diff --git a/src/metric-engine/src/lib.rs b/src/metric-engine/src/lib.rs index 3fe4640da4aa..89b98351282c 100644 --- a/src/metric-engine/src/lib.rs +++ b/src/metric-engine/src/lib.rs @@ -52,6 +52,7 @@ #![feature(let_chains)] +pub mod config; mod data_region; pub mod engine; pub mod error; diff --git a/src/metric-engine/src/test_util.rs b/src/metric-engine/src/test_util.rs index d0f8cf5028e6..5e081982c835 100644 --- a/src/metric-engine/src/test_util.rs +++ b/src/metric-engine/src/test_util.rs @@ -32,6 +32,7 @@ use store_api::region_request::{ }; use store_api::storage::{ColumnId, RegionId}; +use crate::config::EngineConfig; use crate::data_region::DataRegion; use crate::engine::MetricEngine; use crate::metadata_region::MetadataRegion; @@ -54,7 +55,7 @@ impl TestEnv { pub async fn with_prefix(prefix: &str) -> Self { let mut mito_env = MitoTestEnv::with_prefix(prefix); let mito = mito_env.create_engine(MitoConfig::default()).await; - let metric = MetricEngine::new(mito.clone()); + let metric = MetricEngine::new(mito.clone(), EngineConfig::default()); Self { mito_env, mito,