Add cortex_ingester_tsdb_data_replay_duration_seconds metric

Signed-off-by: Xiaochao Dong (@damnever) <the.xcdong@gmail.com>

CHANGELOG.md

@@ -24,6 +24,7 @@
 * [FEATURE] Store Gateway: Add `-store-gateway.sharding-ring.keep-instance-in-the-ring-on-shutdown` to skip unregistering instance from the ring in shutdown. #5421
 * [FEATURE] Ruler: Support for filtering rules in the API. #5417
 * [FEATURE] Compactor: Add `-compactor.ring.tokens-file-path` to store generated tokens locally. #5432
+* [FEATURE] Ingester: Add per-tenant new metric `cortex_ingester_tsdb_data_replay_duration_seconds`. #5477
 * [ENHANCEMENT] Distributor/Ingester: Add span on push path #5319
 * [ENHANCEMENT] Support object storage backends for runtime configuration file. #5292
 * [ENHANCEMENT] Query Frontend: Reject subquery with too small step size. #5323

pkg/ingester/metrics.go

@@ -262,6 +262,7 @@ type tsdbMetrics struct {
 	tsdbChunksRemovedTotal             *prometheus.Desc
 	tsdbMmapChunkCorruptionTotal       *prometheus.Desc
 	tsdbChunkwriteQueueOperationsTotal *prometheus.Desc
+	tsdbDataTotalReplayDuration        *prometheus.Desc
 
 	tsdbExemplarsTotal          *prometheus.Desc
 	tsdbExemplarsInStorage      *prometheus.Desc
@@ -389,6 +390,10 @@ func newTSDBMetrics(r prometheus.Registerer) *tsdbMetrics {
 			"cortex_ingester_tsdb_chunk_write_queue_operations_total",
 			"Number of currently tsdb chunk write queues.",
 			[]string{"user", "operation"}, nil),
+		tsdbDataTotalReplayDuration: prometheus.NewDesc(
+			"cortex_ingester_tsdb_data_replay_duration_seconds",
+			"Time taken to replay the tsdb data on disk.",
+			[]string{"user"}, nil),
 		tsdbLoadedBlocks: prometheus.NewDesc(
 			"cortex_ingester_tsdb_blocks_loaded",
 			"Number of currently loaded data blocks",
@@ -491,6 +496,7 @@ func (sm *tsdbMetrics) Describe(out chan<- *prometheus.Desc) {
 	out <- sm.tsdbChunksRemovedTotal
 	out <- sm.tsdbMmapChunkCorruptionTotal
 	out <- sm.tsdbChunkwriteQueueOperationsTotal
+	out <- sm.tsdbDataTotalReplayDuration
 	out <- sm.tsdbLoadedBlocks
 	out <- sm.tsdbSymbolTableSize
 	out <- sm.tsdbReloads
@@ -541,6 +547,7 @@ func (sm *tsdbMetrics) Collect(out chan<- prometheus.Metric) {
 	data.SendSumOfCountersPerUser(out, sm.tsdbChunksRemovedTotal, "prometheus_tsdb_head_chunks_removed_total")
 	data.SendSumOfCounters(out, sm.tsdbMmapChunkCorruptionTotal, "prometheus_tsdb_mmap_chunk_corruptions_total")
 	data.SendSumOfCountersPerUserWithLabels(out, sm.tsdbChunkwriteQueueOperationsTotal, "prometheus_tsdb_chunk_write_queue_operations_total", "operation")
+	data.SendSumOfGaugesPerUser(out, sm.tsdbDataTotalReplayDuration, "prometheus_tsdb_data_replay_duration_seconds")
 	data.SendSumOfGauges(out, sm.tsdbLoadedBlocks, "prometheus_tsdb_blocks_loaded")
 	data.SendSumOfGaugesPerUser(out, sm.tsdbSymbolTableSize, "prometheus_tsdb_symbol_table_size_bytes")
 	data.SendSumOfCounters(out, sm.tsdbReloads, "prometheus_tsdb_reloads_total")