From c8a51312d11c309437089c96cac1a8152df5d548 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Tue, 23 Jan 2024 23:03:16 +0800 Subject: [PATCH 1/3] chore: adjust metrics to metric engine and mito engine Signed-off-by: Ruihang Xia --- src/metric-engine/src/engine/put.rs | 4 ++++ src/metric-engine/src/engine/read.rs | 9 +++++++++ src/metric-engine/src/metrics.rs | 12 ++++++++++++ src/mito2/src/metrics.rs | 6 ++++-- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/metric-engine/src/engine/put.rs b/src/metric-engine/src/engine/put.rs index 10ffd39fee4a..80fbbf0e74ed 100644 --- a/src/metric-engine/src/engine/put.rs +++ b/src/metric-engine/src/engine/put.rs @@ -65,6 +65,10 @@ impl MetricEngineInner { logical_region_id: RegionId, mut request: RegionPutRequest, ) -> Result { + let _timer = MITO_OPERATION_ELAPSED + .with_label_values(&["put"]) + .start_timer(); + let physical_region_id = *self .state .read() diff --git a/src/metric-engine/src/engine/read.rs b/src/metric-engine/src/engine/read.rs index 3d8a9952612f..cba9fbd343b4 100644 --- a/src/metric-engine/src/engine/read.rs +++ b/src/metric-engine/src/engine/read.rs @@ -30,6 +30,7 @@ use crate::engine::MetricEngineInner; use crate::error::{ InvalidMetadataSnafu, LogicalRegionNotFoundSnafu, MitoReadOperationSnafu, Result, }; +use crate::metrics::MITO_OPERATION_ELAPSED; use crate::utils; impl MetricEngineInner { @@ -57,6 +58,10 @@ impl MetricEngineInner { region_id: RegionId, request: ScanRequest, ) -> Result { + let _timer = MITO_OPERATION_ELAPSED + .with_label_values(&["read_physical"]) + .start_timer(); + self.mito .handle_query(region_id, request) .await @@ -68,6 +73,10 @@ impl MetricEngineInner { logical_region_id: RegionId, request: ScanRequest, ) -> Result { + let _timer = MITO_OPERATION_ELAPSED + .with_label_values(&["read"]) + .start_timer(); + let physical_region_id = self.get_physical_region_id(logical_region_id).await?; let data_region_id = utils::to_data_region_id(physical_region_id); let request = self diff --git a/src/metric-engine/src/metrics.rs b/src/metric-engine/src/metrics.rs index 2804c72dbb81..4309097533f2 100644 --- a/src/metric-engine/src/metrics.rs +++ b/src/metric-engine/src/metrics.rs @@ -17,6 +17,9 @@ use lazy_static::lazy_static; use prometheus::*; +/// Stage label. +pub const OPERATION_LABEL: &str = "operation"; + lazy_static! { /// Gauge for opened regions pub static ref PHYSICAL_REGION_COUNT: IntGauge = @@ -37,4 +40,13 @@ lazy_static! { /// Counter for forbidden operations pub static ref FORBIDDEN_OPERATION_COUNT: IntCounter = register_int_counter!("greptime_metric_engine_forbidden_request", "metric forbidden request").unwrap(); + + /// Histogram for underlying mito operations + pub static ref MITO_OPERATION_ELAPSED: HistogramVec = register_histogram_vec!( + "greptime_metric_engine_mito_op_elapsed", + "metric engine's mito operation elapsed", + &[OPERATION_LABEL], + vec![0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 60.0, 300.0] + ) + .unwrap(); } diff --git a/src/mito2/src/metrics.rs b/src/mito2/src/metrics.rs index 27e2c3961317..75a44cdcaf00 100644 --- a/src/mito2/src/metrics.rs +++ b/src/mito2/src/metrics.rs @@ -35,7 +35,8 @@ lazy_static! { pub static ref HANDLE_REQUEST_ELAPSED: HistogramVec = register_histogram_vec!( "greptime_mito_handle_request_elapsed", "mito handle request elapsed", - &[TYPE_LABEL] + &[TYPE_LABEL], + vec![0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.5, 5.0, 10.0, 60.0, 300.0] ) .unwrap(); @@ -75,7 +76,8 @@ lazy_static! { pub static ref WRITE_STAGE_ELAPSED: HistogramVec = register_histogram_vec!( "greptime_mito_write_stage_elapsed", "mito write stage elapsed", - &[STAGE_LABEL] + &[STAGE_LABEL], + vec![0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 60.0, 300.0] ) .unwrap(); /// Counter of rows to write. From d309a4bfeb91ab4dad0968740e5395e868a7745d Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Mon, 29 Jan 2024 17:00:58 +0800 Subject: [PATCH 2/3] adjust more mito bucket Signed-off-by: Ruihang Xia --- src/mito2/src/metrics.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/mito2/src/metrics.rs b/src/mito2/src/metrics.rs index 75a44cdcaf00..90a6d5b5b0c5 100644 --- a/src/mito2/src/metrics.rs +++ b/src/mito2/src/metrics.rs @@ -56,7 +56,8 @@ lazy_static! { pub static ref FLUSH_ELAPSED: HistogramVec = register_histogram_vec!( "greptime_mito_flush_elapsed", "mito flush elapsed", - &[TYPE_LABEL] + &[TYPE_LABEL], + vec![0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 60.0, 300.0] ) .unwrap(); /// Histogram of flushed bytes. @@ -95,7 +96,8 @@ lazy_static! { pub static ref COMPACTION_STAGE_ELAPSED: HistogramVec = register_histogram_vec!( "greptime_mito_compaction_stage_elapsed", "mito compaction stage elapsed", - &[STAGE_LABEL] + &[STAGE_LABEL], + vec![0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 60.0, 300.0] ) .unwrap(); /// Timer of whole compaction task. @@ -114,7 +116,8 @@ lazy_static! { pub static ref READ_STAGE_ELAPSED: HistogramVec = register_histogram_vec!( "greptime_mito_read_stage_elapsed", "mito read stage elapsed", - &[STAGE_LABEL] + &[STAGE_LABEL], + vec![0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 60.0, 300.0] ) .unwrap(); /// Counter of rows read. From 4fa80bff95605e7069d6d9dbbaf676bb3ef15a1e Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Mon, 29 Jan 2024 19:18:26 +0800 Subject: [PATCH 3/3] fix compile Signed-off-by: Ruihang Xia --- src/metric-engine/src/engine/put.rs | 2 +- src/servers/src/http/prom_store.rs | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/metric-engine/src/engine/put.rs b/src/metric-engine/src/engine/put.rs index 80fbbf0e74ed..0f5d84551bb1 100644 --- a/src/metric-engine/src/engine/put.rs +++ b/src/metric-engine/src/engine/put.rs @@ -28,7 +28,7 @@ use crate::engine::MetricEngineInner; use crate::error::{ ColumnNotFoundSnafu, ForbiddenPhysicalAlterSnafu, LogicalRegionNotFoundSnafu, Result, }; -use crate::metrics::FORBIDDEN_OPERATION_COUNT; +use crate::metrics::{FORBIDDEN_OPERATION_COUNT, MITO_OPERATION_ELAPSED}; use crate::utils::{to_data_region_id, to_metadata_region_id}; // A random number diff --git a/src/servers/src/http/prom_store.rs b/src/servers/src/http/prom_store.rs index af1f2a5261da..d70e341600f3 100644 --- a/src/servers/src/http/prom_store.rs +++ b/src/servers/src/http/prom_store.rs @@ -49,13 +49,13 @@ pub async fn remote_write( Extension(query_ctx): Extension, RawBody(body): RawBody, ) -> Result<(StatusCode, ())> { - let request = decode_remote_write_request(body).await?; let db = params.db.clone().unwrap_or_default(); - let _timer = crate::metrics::METRIC_HTTP_PROM_STORE_WRITE_ELAPSED .with_label_values(&[db.as_str()]) .start_timer(); + let request = decode_remote_write_request(body).await?; + handler.write(request, query_ctx).await?; Ok((StatusCode::NO_CONTENT, ())) } @@ -80,12 +80,13 @@ pub async fn remote_read( Extension(query_ctx): Extension, RawBody(body): RawBody, ) -> Result { - let request = decode_remote_read_request(body).await?; let db = params.db.clone().unwrap_or_default(); - let _timer = crate::metrics::METRIC_HTTP_PROM_STORE_READ_ELAPSED .with_label_values(&[db.as_str()]) .start_timer(); + + let request = decode_remote_read_request(body).await?; + handler.read(request, query_ctx).await }