diff --git a/Cargo.lock b/Cargo.lock
index 33cb55bbf6f7..18a1a3d21522 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6488,6 +6488,7 @@ dependencies = [
"mur3",
"object-store",
"prometheus",
+ "serde",
"serde_json",
"snafu 0.8.5",
"store-api",
diff --git a/config/config.md b/config/config.md
index f14148bfcc9c..4dc9dee00f53 100644
--- a/config/config.md
+++ b/config/config.md
@@ -170,6 +170,8 @@
| `region_engine.mito.memtable.data_freeze_threshold` | Integer | `32768` | The max rows of data inside the actively writing buffer in one shard.
Only available for `partition_tree` memtable. |
| `region_engine.mito.memtable.fork_dictionary_bytes` | String | `1GiB` | Max dictionary bytes.
Only available for `partition_tree` memtable. |
| `region_engine.file` | -- | -- | Enable the file engine. |
+| `region_engine.metric` | -- | -- | Metric engine options. |
+| `region_engine.metric.experimental_sparse_primary_key_encoding` | Bool | `false` | Whether to enable the experimental sparse primary key encoding. |
| `logging` | -- | -- | The logging options. |
| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
| `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
@@ -506,6 +508,8 @@
| `region_engine.mito.memtable.data_freeze_threshold` | Integer | `32768` | The max rows of data inside the actively writing buffer in one shard.
Only available for `partition_tree` memtable. |
| `region_engine.mito.memtable.fork_dictionary_bytes` | String | `1GiB` | Max dictionary bytes.
Only available for `partition_tree` memtable. |
| `region_engine.file` | -- | -- | Enable the file engine. |
+| `region_engine.metric` | -- | -- | Metric engine options. |
+| `region_engine.metric.experimental_sparse_primary_key_encoding` | Bool | `false` | Whether to enable the experimental sparse primary key encoding. |
| `logging` | -- | -- | The logging options. |
| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
| `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 9d9ff5925dc5..5c8afd757ceb 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -622,6 +622,12 @@ fork_dictionary_bytes = "1GiB"
## Enable the file engine.
[region_engine.file]
+[[region_engine]]
+## Metric engine options.
+[region_engine.metric]
+## Whether to enable the experimental sparse primary key encoding.
+experimental_sparse_primary_key_encoding = false
+
## The logging options.
[logging]
## The directory to store the log files. If set to empty, logs will not be written to files.
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 275abf40e4a5..8e66122916ec 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -671,6 +671,12 @@ fork_dictionary_bytes = "1GiB"
## Enable the file engine.
[region_engine.file]
+[[region_engine]]
+## Metric engine options.
+[region_engine.metric]
+## Whether to enable the experimental sparse primary key encoding.
+experimental_sparse_primary_key_encoding = false
+
## The logging options.
[logging]
## The directory to store the log files. If set to empty, logs will not be written to files.
diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs
index 4e1b4f3195a7..2ac2ec0e44af 100644
--- a/src/datanode/src/config.rs
+++ b/src/datanode/src/config.rs
@@ -24,6 +24,7 @@ use common_telemetry::logging::{LoggingOptions, TracingOptions};
use common_wal::config::DatanodeWalConfig;
use file_engine::config::EngineConfig as FileEngineConfig;
use meta_client::MetaClientOptions;
+use metric_engine::config::EngineConfig as MetricEngineConfig;
use mito2::config::MitoConfig;
use serde::{Deserialize, Serialize};
use servers::export_metrics::ExportMetricsOption;
@@ -432,6 +433,8 @@ pub enum RegionEngineConfig {
Mito(MitoConfig),
#[serde(rename = "file")]
File(FileEngineConfig),
+ #[serde(rename = "metric")]
+ Metric(MetricEngineConfig),
}
#[cfg(test)]
diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs
index 01e1e072c066..82c5f61d143b 100644
--- a/src/datanode/src/datanode.rs
+++ b/src/datanode/src/datanode.rs
@@ -395,6 +395,11 @@ impl DatanodeBuilder {
plugins: Plugins,
) -> Result> {
let mut engines = vec![];
+ let mut metric_engine_config = opts.region_engine.iter().find_map(|c| match c {
+ RegionEngineConfig::Metric(config) => Some(config.clone()),
+ _ => None,
+ });
+
for engine in &opts.region_engine {
match engine {
RegionEngineConfig::Mito(config) => {
@@ -407,7 +412,10 @@ impl DatanodeBuilder {
)
.await?;
- let metric_engine = MetricEngine::new(mito_engine.clone());
+ let metric_engine = MetricEngine::new(
+ mito_engine.clone(),
+ metric_engine_config.take().unwrap_or_default(),
+ );
engines.push(Arc::new(mito_engine) as _);
engines.push(Arc::new(metric_engine) as _);
}
@@ -418,6 +426,9 @@ impl DatanodeBuilder {
);
engines.push(Arc::new(engine) as _);
}
+ RegionEngineConfig::Metric(_) => {
+ // Already handled in `build_mito_engine`.
+ }
}
}
Ok(engines)
diff --git a/src/metric-engine/Cargo.toml b/src/metric-engine/Cargo.toml
index 85aa371594e8..64cbb51b38e8 100644
--- a/src/metric-engine/Cargo.toml
+++ b/src/metric-engine/Cargo.toml
@@ -27,6 +27,7 @@ mito2.workspace = true
mur3 = "0.1"
object-store.workspace = true
prometheus.workspace = true
+serde.workspace = true
serde_json.workspace = true
snafu.workspace = true
store-api.workspace = true
diff --git a/src/metric-engine/src/config.rs b/src/metric-engine/src/config.rs
new file mode 100644
index 000000000000..b35b4121885b
--- /dev/null
+++ b/src/metric-engine/src/config.rs
@@ -0,0 +1,20 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct EngineConfig {
+ pub experimental_sparse_primary_key_encoding: bool,
+}
diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs
index 15b94701139b..cc6f7bd4955a 100644
--- a/src/metric-engine/src/engine.rs
+++ b/src/metric-engine/src/engine.rs
@@ -45,6 +45,7 @@ use store_api::region_request::RegionRequest;
use store_api::storage::{RegionId, ScanRequest};
use self::state::MetricEngineState;
+use crate::config::EngineConfig;
use crate::data_region::DataRegion;
use crate::error::{self, Result, UnsupportedRegionRequestSnafu};
use crate::metadata_region::MetadataRegion;
@@ -255,7 +256,7 @@ impl RegionEngine for MetricEngine {
}
impl MetricEngine {
- pub fn new(mito: MitoEngine) -> Self {
+ pub fn new(mito: MitoEngine, config: EngineConfig) -> Self {
let metadata_region = MetadataRegion::new(mito.clone());
let data_region = DataRegion::new(mito.clone());
Self {
@@ -264,6 +265,7 @@ impl MetricEngine {
metadata_region,
data_region,
state: RwLock::default(),
+ config,
}),
}
}
@@ -304,6 +306,9 @@ struct MetricEngineInner {
metadata_region: MetadataRegion,
data_region: DataRegion,
state: RwLock,
+ /// TODO(weny): remove it after the config is used.
+ #[allow(unused)]
+ config: EngineConfig,
}
#[cfg(test)]
diff --git a/src/metric-engine/src/engine/create.rs b/src/metric-engine/src/engine/create.rs
index b87682d4599e..fb7031e9daf8 100644
--- a/src/metric-engine/src/engine/create.rs
+++ b/src/metric-engine/src/engine/create.rs
@@ -549,6 +549,7 @@ mod test {
use store_api::metric_engine_consts::{METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY};
use super::*;
+ use crate::config::EngineConfig;
use crate::engine::MetricEngine;
use crate::test_util::TestEnv;
@@ -707,7 +708,7 @@ mod test {
// set up
let env = TestEnv::new().await;
- let engine = MetricEngine::new(env.mito());
+ let engine = MetricEngine::new(env.mito(), EngineConfig::default());
let engine_inner = engine.inner;
// check create data region request
diff --git a/src/metric-engine/src/lib.rs b/src/metric-engine/src/lib.rs
index 3fe4640da4aa..89b98351282c 100644
--- a/src/metric-engine/src/lib.rs
+++ b/src/metric-engine/src/lib.rs
@@ -52,6 +52,7 @@
#![feature(let_chains)]
+pub mod config;
mod data_region;
pub mod engine;
pub mod error;
diff --git a/src/metric-engine/src/test_util.rs b/src/metric-engine/src/test_util.rs
index d0f8cf5028e6..5e081982c835 100644
--- a/src/metric-engine/src/test_util.rs
+++ b/src/metric-engine/src/test_util.rs
@@ -32,6 +32,7 @@ use store_api::region_request::{
};
use store_api::storage::{ColumnId, RegionId};
+use crate::config::EngineConfig;
use crate::data_region::DataRegion;
use crate::engine::MetricEngine;
use crate::metadata_region::MetadataRegion;
@@ -54,7 +55,7 @@ impl TestEnv {
pub async fn with_prefix(prefix: &str) -> Self {
let mut mito_env = MitoTestEnv::with_prefix(prefix);
let mito = mito_env.create_engine(MitoConfig::default()).await;
- let metric = MetricEngine::new(mito.clone());
+ let metric = MetricEngine::new(mito.clone(), EngineConfig::default());
Self {
mito_env,
mito,