From 752ec179cc538ae81ad8d8a4b1db2d650f9e1f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 25 May 2020 12:30:56 +0200 Subject: [PATCH 1/9] Removed Cortex-specific metrics for index cache, reuse Thanos metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (with name="index-cache", and appropriate component) Signed-off-by: Peter Štibraný --- CHANGELOG.md | 15 + pkg/storage/tsdb/index_cache_metrics.go | 175 ----------- pkg/storage/tsdb/index_cache_metrics_test.go | 306 ------------------- pkg/storegateway/bucket_stores.go | 8 +- 4 files changed, 17 insertions(+), 487 deletions(-) delete mode 100644 pkg/storage/tsdb/index_cache_metrics.go delete mode 100644 pkg/storage/tsdb/index_cache_metrics_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 685592ccdf..796241ac0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ * [CHANGE] Query Frontend now uses Round Robin to choose a tenant queue to service next. #2553 * [CHANGE] `-promql.lookback-delta` is now deprecated and has been replaced by `-querier.lookback-delta` along with `lookback_delta` entry under `querier` in the config file. `-promql.lookback-delta` will be removed in v1.4.0. #2604 +* [CHANGE] Experimental TSDB: Renamed metrics in block stores: + * `cortex__blocks_index_cache_items_evicted_total` => `thanos_store_index_cache_items_evicted_total{name="index-cache"}` + * `cortex__blocks_index_cache_items_added_total` => `thanos_store_index_cache_items_added_total{name="index-cache"}` + * `cortex__blocks_index_cache_requests_total` => `thanos_store_index_cache_requests_total{name="index-cache"}` + * `cortex__blocks_index_cache_items_overflowed_total` => `thanos_store_index_cache_items_overflowed_total{name="index-cache"}` + * `cortex__blocks_index_cache_hits_total` => `thanos_store_index_cache_hits_total{name="index-cache"}` + * `cortex__blocks_index_cache_items` => `thanos_store_index_cache_items{name="index-cache"}` + * `cortex__blocks_index_cache_items_size_bytes` => `thanos_store_index_cache_items_size_bytes{name="index-cache"}` + * `cortex__blocks_index_cache_total_size_bytes` => `thanos_store_index_cache_total_size_bytes{name="index-cache"}` + * `cortex__blocks_index_cache_requests_total` => `thanos_store_index_cache_requests_total{name="index-cache"}` + * `cortex__blocks_index_cache_hits_total` => `thanos_store_index_cache_hits_total{name="index-cache"}` + * `cortex__blocks_index_cache_memcached_operations_total` => `thanos_memcached_operations_total{name="index-cache"}` + * `cortex__blocks_index_cache_memcached_operation_failures_total` => `thanos_memcached_operation_failures_total{name="index-cache"}` + * `cortex__blocks_index_cache_memcached_operation_duration_seconds` => `thanos_memcached_operation_duration_seconds{name="index-cache"}` + * `cortex__blocks_index_cache_memcached_operation_skipped_total` => `thanos_memcached_operation_skipped_total{name="index-cache"}` * [FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client & Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. #2502 * [FEATURE] The flag `-frontend.max-cache-freshness` is now supported within the limits overrides, to specify per-tenant max cache freshness values. The corresponding YAML config parameter has been changed from `results_cache.max_freshness` to `limits_config.max_cache_freshness`. The legacy YAML config parameter (`results_cache.max_freshness`) will continue to be supported till Cortex release `v1.4.0`. #2609 * [ENHANCEMENT] Experimental TSDB: added the following metrics to the ingester: #2580 #2583 #2589 diff --git a/pkg/storage/tsdb/index_cache_metrics.go b/pkg/storage/tsdb/index_cache_metrics.go deleted file mode 100644 index 8cf75cffcb..0000000000 --- a/pkg/storage/tsdb/index_cache_metrics.go +++ /dev/null @@ -1,175 +0,0 @@ -package tsdb - -import ( - "github.com/prometheus/client_golang/prometheus" - - "github.com/cortexproject/cortex/pkg/util" -) - -func MustNewIndexCacheMetrics(backend string, reg *prometheus.Registry) prometheus.Collector { - switch backend { - case IndexCacheBackendInMemory: - return NewInMemoryIndexCacheMetrics(reg) - case IndexCacheBackendMemcached: - return NewMemcachedIndexCacheMetrics(reg) - default: - panic(errUnsupportedIndexCacheBackend.Error()) - } -} - -// InMemoryIndexCacheMetrics aggregates metrics exported by Thanos in-memory index cache -// and re-exports them as Cortex metrics. -type InMemoryIndexCacheMetrics struct { - reg *prometheus.Registry - - // Metrics gathered from Thanos InMemoryIndexCache - cacheItemsEvicted *prometheus.Desc - cacheItemsAdded *prometheus.Desc - cacheRequests *prometheus.Desc - cacheItemsOverflow *prometheus.Desc - cacheHits *prometheus.Desc - cacheItemsCurrentCount *prometheus.Desc - cacheItemsCurrentSize *prometheus.Desc - cacheItemsTotalCurrentSize *prometheus.Desc - - // Ignored: - // thanos_store_index_cache_max_size_bytes - // thanos_store_index_cache_max_item_size_bytes -} - -// NewInMemoryIndexCacheMetrics makes InMemoryIndexCacheMetrics. -func NewInMemoryIndexCacheMetrics(reg *prometheus.Registry) *InMemoryIndexCacheMetrics { - return &InMemoryIndexCacheMetrics{ - reg: reg, - - // Cache - cacheItemsEvicted: prometheus.NewDesc( - "blocks_index_cache_items_evicted_total", - "Total number of items that were evicted from the index cache.", - []string{"item_type"}, nil), - cacheItemsAdded: prometheus.NewDesc( - "blocks_index_cache_items_added_total", - "Total number of items that were added to the index cache.", - []string{"item_type"}, nil), - cacheRequests: prometheus.NewDesc( - "blocks_index_cache_requests_total", - "Total number of requests to the cache.", - []string{"item_type"}, nil), - cacheItemsOverflow: prometheus.NewDesc( - "blocks_index_cache_items_overflowed_total", - "Total number of items that could not be added to the cache due to being too big.", - []string{"item_type"}, nil), - cacheHits: prometheus.NewDesc( - "blocks_index_cache_hits_total", - "Total number of requests to the cache that were a hit.", - []string{"item_type"}, nil), - cacheItemsCurrentCount: prometheus.NewDesc( - "blocks_index_cache_items", - "Current number of items in the index cache.", - []string{"item_type"}, nil), - cacheItemsCurrentSize: prometheus.NewDesc( - "blocks_index_cache_items_size_bytes", - "Current byte size of items in the index cache.", - []string{"item_type"}, nil), - cacheItemsTotalCurrentSize: prometheus.NewDesc( - "blocks_index_cache_total_size_bytes", - "Current byte size of items (both value and key) in the index cache.", - []string{"item_type"}, nil), - } -} - -func (m *InMemoryIndexCacheMetrics) Describe(out chan<- *prometheus.Desc) { - out <- m.cacheItemsEvicted - out <- m.cacheItemsAdded - out <- m.cacheRequests - out <- m.cacheItemsOverflow - out <- m.cacheHits - out <- m.cacheItemsCurrentCount - out <- m.cacheItemsCurrentSize - out <- m.cacheItemsTotalCurrentSize -} - -func (m *InMemoryIndexCacheMetrics) Collect(out chan<- prometheus.Metric) { - data := util.BuildMetricFamiliesPerUserFromUserRegistries(map[string]*prometheus.Registry{ - "": m.reg, - }) - - data.SendSumOfCountersWithLabels(out, m.cacheItemsEvicted, "thanos_store_index_cache_items_evicted_total", "item_type") - data.SendSumOfCountersWithLabels(out, m.cacheItemsAdded, "thanos_store_index_cache_items_added_total", "item_type") - data.SendSumOfCountersWithLabels(out, m.cacheRequests, "thanos_store_index_cache_requests_total", "item_type") - data.SendSumOfCountersWithLabels(out, m.cacheItemsOverflow, "thanos_store_index_cache_items_overflowed_total", "item_type") - data.SendSumOfCountersWithLabels(out, m.cacheHits, "thanos_store_index_cache_hits_total", "item_type") - - data.SendSumOfGaugesWithLabels(out, m.cacheItemsCurrentCount, "thanos_store_index_cache_items", "item_type") - data.SendSumOfGaugesWithLabels(out, m.cacheItemsCurrentSize, "thanos_store_index_cache_items_size_bytes", "item_type") - data.SendSumOfGaugesWithLabels(out, m.cacheItemsTotalCurrentSize, "thanos_store_index_cache_total_size_bytes", "item_type") -} - -// MemcachedIndexCacheMetrics aggregates metrics exported by Thanos memcached index cache -// and re-exports them as Cortex metrics. -type MemcachedIndexCacheMetrics struct { - reg *prometheus.Registry - - // Metrics gathered from Thanos MemcachedIndexCache (and client). - cacheRequests *prometheus.Desc - cacheHits *prometheus.Desc - memcachedOperations *prometheus.Desc - memcachedFailures *prometheus.Desc - memcachedDuration *prometheus.Desc - memcachedSkipped *prometheus.Desc -} - -// NewMemcachedIndexCacheMetrics makes MemcachedIndexCacheMetrics. -func NewMemcachedIndexCacheMetrics(reg *prometheus.Registry) *MemcachedIndexCacheMetrics { - return &MemcachedIndexCacheMetrics{ - reg: reg, - - cacheRequests: prometheus.NewDesc( - "blocks_index_cache_requests_total", - "Total number of requests to the cache.", - []string{"item_type"}, nil), - cacheHits: prometheus.NewDesc( - "blocks_index_cache_hits_total", - "Total number of requests to the cache that were a hit.", - []string{"item_type"}, nil), - memcachedOperations: prometheus.NewDesc( - "blocks_index_cache_memcached_operations_total", - "Total number of operations against memcached.", - []string{"operation"}, nil), - memcachedFailures: prometheus.NewDesc( - "blocks_index_cache_memcached_operation_failures_total", - "Total number of operations against memcached that failed.", - []string{"operation"}, nil), - memcachedDuration: prometheus.NewDesc( - "blocks_index_cache_memcached_operation_duration_seconds", - "Duration of operations against memcached.", - []string{"operation"}, nil), - memcachedSkipped: prometheus.NewDesc( - "blocks_index_cache_memcached_operation_skipped_total", - "Total number of operations against memcached that have been skipped.", - []string{"operation", "reason"}, nil), - } -} - -func (m *MemcachedIndexCacheMetrics) Describe(out chan<- *prometheus.Desc) { - out <- m.cacheRequests - out <- m.cacheHits - out <- m.memcachedOperations - out <- m.memcachedFailures - out <- m.memcachedDuration - out <- m.memcachedSkipped -} - -func (m *MemcachedIndexCacheMetrics) Collect(out chan<- prometheus.Metric) { - data := util.BuildMetricFamiliesPerUserFromUserRegistries(map[string]*prometheus.Registry{ - "": m.reg, - }) - - data.SendSumOfCountersWithLabels(out, m.cacheRequests, "thanos_store_index_cache_requests_total", "item_type") - data.SendSumOfCountersWithLabels(out, m.cacheHits, "thanos_store_index_cache_hits_total", "item_type") - - data.SendSumOfCountersWithLabels(out, m.memcachedOperations, "thanos_memcached_operations_total", "operation") - data.SendSumOfCountersWithLabels(out, m.memcachedFailures, "thanos_memcached_operation_failures_total", "operation") - data.SendSumOfHistogramsWithLabels(out, m.memcachedDuration, "thanos_memcached_operation_duration_seconds", "operation") - data.SendSumOfCountersWithLabels(out, m.memcachedSkipped, "thanos_memcached_operation_skipped_total", "operation", "reason") -} diff --git a/pkg/storage/tsdb/index_cache_metrics_test.go b/pkg/storage/tsdb/index_cache_metrics_test.go deleted file mode 100644 index f8e1cad98d..0000000000 --- a/pkg/storage/tsdb/index_cache_metrics_test.go +++ /dev/null @@ -1,306 +0,0 @@ -package tsdb - -import ( - "bytes" - "testing" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/stretchr/testify/require" -) - -const ( - cacheTypePostings string = "Postings" - cacheTypeSeries string = "Series" - - cacheOpSet string = "set" - cacheOpGetMulti string = "getmulti" -) - -func TestInMemoryIndexCacheMetrics(t *testing.T) { - mainReg := prometheus.NewPedanticRegistry() - cacheMetrics := NewInMemoryIndexCacheMetrics(populateInMemoryIndexCacheMetrics(5328)) - mainReg.MustRegister(cacheMetrics) - - //noinspection ALL - err := testutil.GatherAndCompare(mainReg, bytes.NewBufferString(` - # HELP blocks_index_cache_items_evicted_total Total number of items that were evicted from the index cache. - # TYPE blocks_index_cache_items_evicted_total counter - blocks_index_cache_items_evicted_total{item_type="Postings"} 5328 - blocks_index_cache_items_evicted_total{item_type="Series"} 10656 - - # HELP blocks_index_cache_requests_total Total number of requests to the cache. - # TYPE blocks_index_cache_requests_total counter - blocks_index_cache_requests_total{item_type="Postings"} 15984 - blocks_index_cache_requests_total{item_type="Series"} 21312 - - # HELP blocks_index_cache_hits_total Total number of requests to the cache that were a hit. - # TYPE blocks_index_cache_hits_total counter - blocks_index_cache_hits_total{item_type="Postings"} 26640 - blocks_index_cache_hits_total{item_type="Series"} 31968 - - # HELP blocks_index_cache_items_added_total Total number of items that were added to the index cache. - # TYPE blocks_index_cache_items_added_total counter - blocks_index_cache_items_added_total{item_type="Postings"} 37296 - blocks_index_cache_items_added_total{item_type="Series"} 42624 - - # HELP blocks_index_cache_items Current number of items in the index cache. - # TYPE blocks_index_cache_items gauge - blocks_index_cache_items{item_type="Postings"} 47952 - blocks_index_cache_items{item_type="Series"} 53280 - - # HELP blocks_index_cache_items_size_bytes Current byte size of items in the index cache. - # TYPE blocks_index_cache_items_size_bytes gauge - blocks_index_cache_items_size_bytes{item_type="Postings"} 58608 - blocks_index_cache_items_size_bytes{item_type="Series"} 63936 - - # HELP blocks_index_cache_total_size_bytes Current byte size of items (both value and key) in the index cache. - # TYPE blocks_index_cache_total_size_bytes gauge - blocks_index_cache_total_size_bytes{item_type="Postings"} 69264 - blocks_index_cache_total_size_bytes{item_type="Series"} 74592 - - # HELP blocks_index_cache_items_overflowed_total Total number of items that could not be added to the cache due to being too big. - # TYPE blocks_index_cache_items_overflowed_total counter - blocks_index_cache_items_overflowed_total{item_type="Postings"} 79920 - blocks_index_cache_items_overflowed_total{item_type="Series"} 85248 -`)) - require.NoError(t, err) -} - -// Copied from Thanos, pkg/store/cache/inmemory.go, InMemoryIndexCache struct -type inMemoryIndexStoreCacheMetrics struct { - evicted *prometheus.CounterVec - requests *prometheus.CounterVec - hits *prometheus.CounterVec - added *prometheus.CounterVec - current *prometheus.GaugeVec - currentSize *prometheus.GaugeVec - totalCurrentSize *prometheus.GaugeVec - overflow *prometheus.CounterVec -} - -func newInMemoryIndexStoreCacheMetrics(reg prometheus.Registerer) *inMemoryIndexStoreCacheMetrics { - c := inMemoryIndexStoreCacheMetrics{} - c.evicted = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_items_evicted_total", - Help: "Total number of items that were evicted from the index cache.", - }, []string{"item_type"}) - c.evicted.WithLabelValues(cacheTypePostings) - c.evicted.WithLabelValues(cacheTypeSeries) - - c.added = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_items_added_total", - Help: "Total number of items that were added to the index cache.", - }, []string{"item_type"}) - c.added.WithLabelValues(cacheTypePostings) - c.added.WithLabelValues(cacheTypeSeries) - - c.requests = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_requests_total", - Help: "Total number of requests to the cache.", - }, []string{"item_type"}) - c.requests.WithLabelValues(cacheTypePostings) - c.requests.WithLabelValues(cacheTypeSeries) - - c.overflow = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_items_overflowed_total", - Help: "Total number of items that could not be added to the cache due to being too big.", - }, []string{"item_type"}) - c.overflow.WithLabelValues(cacheTypePostings) - c.overflow.WithLabelValues(cacheTypeSeries) - - c.hits = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_hits_total", - Help: "Total number of requests to the cache that were a hit.", - }, []string{"item_type"}) - c.hits.WithLabelValues(cacheTypePostings) - c.hits.WithLabelValues(cacheTypeSeries) - - c.current = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_items", - Help: "Current number of items in the index cache.", - }, []string{"item_type"}) - c.current.WithLabelValues(cacheTypePostings) - c.current.WithLabelValues(cacheTypeSeries) - - c.currentSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_items_size_bytes", - Help: "Current byte size of items in the index cache.", - }, []string{"item_type"}) - c.currentSize.WithLabelValues(cacheTypePostings) - c.currentSize.WithLabelValues(cacheTypeSeries) - - c.totalCurrentSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_total_size_bytes", - Help: "Current byte size of items (both value and key) in the index cache.", - }, []string{"item_type"}) - c.totalCurrentSize.WithLabelValues(cacheTypePostings) - c.totalCurrentSize.WithLabelValues(cacheTypeSeries) - - if reg != nil { - reg.MustRegister(c.requests, c.hits, c.added, c.evicted, c.current, c.currentSize, c.totalCurrentSize, c.overflow) - } - - return &c -} - -func populateInMemoryIndexCacheMetrics(base float64) *prometheus.Registry { - reg := prometheus.NewRegistry() - c := newInMemoryIndexStoreCacheMetrics(reg) - - c.evicted.WithLabelValues(cacheTypePostings).Add(base * 1) - c.evicted.WithLabelValues(cacheTypeSeries).Add(base * 2) - c.requests.WithLabelValues(cacheTypePostings).Add(base * 3) - c.requests.WithLabelValues(cacheTypeSeries).Add(base * 4) - c.hits.WithLabelValues(cacheTypePostings).Add(base * 5) - c.hits.WithLabelValues(cacheTypeSeries).Add(base * 6) - c.added.WithLabelValues(cacheTypePostings).Add(base * 7) - c.added.WithLabelValues(cacheTypeSeries).Add(base * 8) - c.current.WithLabelValues(cacheTypePostings).Set(base * 9) - c.current.WithLabelValues(cacheTypeSeries).Set(base * 10) - c.currentSize.WithLabelValues(cacheTypePostings).Set(base * 11) - c.currentSize.WithLabelValues(cacheTypeSeries).Set(base * 12) - c.totalCurrentSize.WithLabelValues(cacheTypePostings).Set(base * 13) - c.totalCurrentSize.WithLabelValues(cacheTypeSeries).Set(base * 14) - c.overflow.WithLabelValues(cacheTypePostings).Add(base * 15) - c.overflow.WithLabelValues(cacheTypeSeries).Add(base * 16) - - return reg -} - -func TestMemcachedIndexCacheMetrics(t *testing.T) { - mainReg := prometheus.NewPedanticRegistry() - cacheMetrics := NewMemcachedIndexCacheMetrics(populateMemcachedIndexCacheMetrics(1)) - mainReg.MustRegister(cacheMetrics) - - //noinspection ALL - err := testutil.GatherAndCompare(mainReg, bytes.NewBufferString(` - # HELP blocks_index_cache_requests_total Total number of requests to the cache. - # TYPE blocks_index_cache_requests_total counter - blocks_index_cache_requests_total{item_type="Postings"} 1 - blocks_index_cache_requests_total{item_type="Series"} 2 - - # HELP blocks_index_cache_hits_total Total number of requests to the cache that were a hit. - # TYPE blocks_index_cache_hits_total counter - blocks_index_cache_hits_total{item_type="Postings"} 3 - blocks_index_cache_hits_total{item_type="Series"} 4 - - # HELP blocks_index_cache_memcached_operations_total Total number of operations against memcached. - # TYPE blocks_index_cache_memcached_operations_total counter - blocks_index_cache_memcached_operations_total{operation="set"} 5 - blocks_index_cache_memcached_operations_total{operation="getmulti"} 6 - - # HELP blocks_index_cache_memcached_operation_failures_total Total number of operations against memcached that failed. - # TYPE blocks_index_cache_memcached_operation_failures_total counter - blocks_index_cache_memcached_operation_failures_total{operation="set"} 7 - blocks_index_cache_memcached_operation_failures_total{operation="getmulti"} 8 - - # HELP blocks_index_cache_memcached_operation_duration_seconds Duration of operations against memcached. - # TYPE blocks_index_cache_memcached_operation_duration_seconds histogram - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="set",le="0.001"} 0 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="set",le="0.005"} 0 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="set",le="0.01"} 0 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="set",le="0.025"} 0 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="set",le="0.05"} 0 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="set",le="0.1"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="set",le="0.2"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="set",le="0.5"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="set",le="1"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="set",le="+Inf"} 1 - blocks_index_cache_memcached_operation_duration_seconds_sum{operation="set"} 0.1 - blocks_index_cache_memcached_operation_duration_seconds_count{operation="set"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="getmulti",le="0.001"} 0 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="getmulti",le="0.005"} 0 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="getmulti",le="0.01"} 0 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="getmulti",le="0.025"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="getmulti",le="0.05"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="getmulti",le="0.1"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="getmulti",le="0.2"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="getmulti",le="0.5"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="getmulti",le="1"} 1 - blocks_index_cache_memcached_operation_duration_seconds_bucket{operation="getmulti",le="+Inf"} 1 - blocks_index_cache_memcached_operation_duration_seconds_sum{operation="getmulti"} 0.025 - blocks_index_cache_memcached_operation_duration_seconds_count{operation="getmulti"} 1 - - # HELP blocks_index_cache_memcached_operation_skipped_total Total number of operations against memcached that have been skipped. - # TYPE blocks_index_cache_memcached_operation_skipped_total counter - blocks_index_cache_memcached_operation_skipped_total{operation="getmulti",reason="spoiled"} 10 - blocks_index_cache_memcached_operation_skipped_total{operation="set",reason="too_big"} 9 -`)) - require.NoError(t, err) -} - -type memcachedIndexStoreCacheMetrics struct { - requests *prometheus.CounterVec - hits *prometheus.CounterVec - operations *prometheus.CounterVec - failures *prometheus.CounterVec - skipped *prometheus.CounterVec - duration *prometheus.HistogramVec -} - -func newMemcachedIndexStoreCacheMetrics(reg prometheus.Registerer) *memcachedIndexStoreCacheMetrics { - c := memcachedIndexStoreCacheMetrics{} - - c.requests = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_requests_total", - Help: "Total number of requests to the cache.", - }, []string{"item_type"}) - c.requests.WithLabelValues(cacheTypePostings) - c.requests.WithLabelValues(cacheTypeSeries) - - c.hits = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_hits_total", - Help: "Total number of requests to the cache that were a hit.", - }, []string{"item_type"}) - c.hits.WithLabelValues(cacheTypePostings) - c.hits.WithLabelValues(cacheTypeSeries) - - c.operations = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_memcached_operations_total", - Help: "Total number of operations against memcached.", - }, []string{"operation"}) - - c.failures = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_memcached_operation_failures_total", - Help: "Total number of operations against memcached that failed.", - }, []string{"operation"}) - - c.skipped = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_memcached_operation_skipped_total", - Help: "Total number of operations against memcached that have been skipped.", - }, []string{"operation", "reason"}) - - c.duration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Name: "thanos_memcached_operation_duration_seconds", - Help: "Duration of operations against memcached.", - Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1}, - }, []string{"operation"}) - - if reg != nil { - reg.MustRegister(c.requests, c.hits, c.operations, c.failures, c.skipped, c.duration) - } - - return &c -} - -func populateMemcachedIndexCacheMetrics(base float64) *prometheus.Registry { - reg := prometheus.NewRegistry() - c := newMemcachedIndexStoreCacheMetrics(reg) - - c.requests.WithLabelValues(cacheTypePostings).Add(base * 1) - c.requests.WithLabelValues(cacheTypeSeries).Add(base * 2) - c.hits.WithLabelValues(cacheTypePostings).Add(base * 3) - c.hits.WithLabelValues(cacheTypeSeries).Add(base * 4) - - c.operations.WithLabelValues(cacheOpSet).Add(base * 5) - c.operations.WithLabelValues(cacheOpGetMulti).Add(base * 6) - c.failures.WithLabelValues(cacheOpSet).Add(base * 7) - c.failures.WithLabelValues(cacheOpGetMulti).Add(base * 8) - c.duration.WithLabelValues(cacheOpSet).Observe(0.1) - c.duration.WithLabelValues(cacheOpGetMulti).Observe(0.025) - c.skipped.WithLabelValues(cacheOpSet, "too_big").Add(base * 9) - c.skipped.WithLabelValues(cacheOpGetMulti, "spoiled").Add(base * 10) - - return reg -} diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go index c3568b1550..a520eb5f60 100644 --- a/pkg/storegateway/bucket_stores.go +++ b/pkg/storegateway/bucket_stores.go @@ -38,7 +38,6 @@ type BucketStores struct { logLevel logging.Level bucketStoreMetrics *BucketStoreMetrics metaFetcherMetrics *MetadataFetcherMetrics - indexCacheMetrics prometheus.Collector filters []block.MetadataFilter // Index cache shared across all tenants. @@ -60,8 +59,6 @@ func NewBucketStores(cfg tsdb.Config, filters []block.MetadataFilter, bucketClie return nil, errors.Wrapf(err, "create caching bucket") } - indexCacheRegistry := prometheus.NewRegistry() - u := &BucketStores{ logger: logger, cfg: cfg, @@ -71,7 +68,6 @@ func NewBucketStores(cfg tsdb.Config, filters []block.MetadataFilter, bucketClie logLevel: logLevel, bucketStoreMetrics: NewBucketStoreMetrics(), metaFetcherMetrics: NewMetadataFetcherMetrics(), - indexCacheMetrics: tsdb.MustNewIndexCacheMetrics(cfg.BucketStore.IndexCache.Backend, indexCacheRegistry), syncTimes: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ Name: "blocks_sync_seconds", Help: "The total time it takes to perform a sync stores", @@ -84,12 +80,12 @@ func NewBucketStores(cfg tsdb.Config, filters []block.MetadataFilter, bucketClie } // Init the index cache. - if u.indexCache, err = tsdb.NewIndexCache(cfg.BucketStore.IndexCache, logger, indexCacheRegistry); err != nil { + if u.indexCache, err = tsdb.NewIndexCache(cfg.BucketStore.IndexCache, logger, reg); err != nil { return nil, errors.Wrap(err, "create index cache") } if reg != nil { - reg.MustRegister(u.bucketStoreMetrics, u.metaFetcherMetrics, u.indexCacheMetrics) + reg.MustRegister(u.bucketStoreMetrics, u.metaFetcherMetrics) } return u, nil From ad55d7e9b52cc3027c758d49b9fb7bfa185a961e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 25 May 2020 12:41:13 +0200 Subject: [PATCH 2/9] Don't prefix metrics, but use new label instead. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- CHANGELOG.md | 29 ++++++++++++++++++++- pkg/querier/blocks_bucket_stores_service.go | 2 +- pkg/storegateway/bucket_stores.go | 4 +-- pkg/storegateway/gateway.go | 2 +- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 796241ac0d..efa99555df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * [CHANGE] Query Frontend now uses Round Robin to choose a tenant queue to service next. #2553 * [CHANGE] `-promql.lookback-delta` is now deprecated and has been replaced by `-querier.lookback-delta` along with `lookback_delta` entry under `querier` in the config file. `-promql.lookback-delta` will be removed in v1.4.0. #2604 -* [CHANGE] Experimental TSDB: Renamed metrics in block stores: +* [CHANGE] Experimental TSDB: Renamed index-cache metrics to use original metric names from Thanos, as Cortex is not aggregating them in any way: * `cortex__blocks_index_cache_items_evicted_total` => `thanos_store_index_cache_items_evicted_total{name="index-cache"}` * `cortex__blocks_index_cache_items_added_total` => `thanos_store_index_cache_items_added_total{name="index-cache"}` * `cortex__blocks_index_cache_requests_total` => `thanos_store_index_cache_requests_total{name="index-cache"}` @@ -19,6 +19,33 @@ * `cortex__blocks_index_cache_memcached_operation_failures_total` => `thanos_memcached_operation_failures_total{name="index-cache"}` * `cortex__blocks_index_cache_memcached_operation_duration_seconds` => `thanos_memcached_operation_duration_seconds{name="index-cache"}` * `cortex__blocks_index_cache_memcached_operation_skipped_total` => `thanos_memcached_operation_skipped_total{name="index-cache"}` +* [CHANGE] Experimental TSDB: Renamed metrics in bucket stores: + * `cortex__blocks_meta_syncs_total` => `blocks_meta_syncs_total{component=""}` + * `cortex__blocks_meta_sync_failures_total` => `blocks_meta_sync_failures_total{component=""}` + * `cortex__blocks_meta_sync_duration_seconds` => `blocks_meta_sync_duration_seconds{component=""}` + * `cortex__blocks_meta_sync_consistency_delay_seconds` => `blocks_meta_sync_consistency_delay_seconds{component=""}` + * `cortex__blocks_meta_synced` => `blocks_meta_synced{component=""}` + * `cortex__bucket_store_block_loads_total` => `bucket_store_block_loads_total{component=""}` + * `cortex__bucket_store_block_load_failures_total` => `bucket_store_block_load_failures_total{component=""}` + * `cortex__bucket_store_block_drops_total` => `bucket_store_block_drops_total{component=""}` + * `cortex__bucket_store_block_drop_failures_total` => `bucket_store_block_drop_failures_total{component=""}` + * `cortex__bucket_store_blocks_loaded` => `bucket_store_blocks_loaded{component=""}` + * `cortex__bucket_store_series_data_touched` => `bucket_store_series_data_touched{component=""}` + * `cortex__bucket_store_series_data_fetched` => `bucket_store_series_data_fetched{component=""}` + * `cortex__bucket_store_series_data_size_touched_bytes` => `bucket_store_series_data_size_touched_bytes{component=""}` + * `cortex__bucket_store_series_data_size_fetched_bytes` => `bucket_store_series_data_size_fetched_bytes{component=""}` + * `cortex__bucket_store_series_blocks_queried` => `bucket_store_series_blocks_queried{component=""}` + * `cortex__bucket_store_series_get_all_duration_seconds` => `bucket_store_series_get_all_duration_seconds{component=""}` + * `cortex__bucket_store_series_merge_duration_seconds` => `bucket_store_series_merge_duration_seconds{component=""}` + * `cortex__bucket_store_series_refetches_total` => `bucket_store_series_refetches_total{component=""}` + * `cortex__bucket_store_series_result_series` => `bucket_store_series_result_series{component=""}` + * `cortex__bucket_store_cached_postings_compressions_total` => `bucket_store_cached_postings_compressions_total{component=""}` + * `cortex__bucket_store_cached_postings_compression_errors_total` => `bucket_store_cached_postings_compression_errors_total{component=""}` + * `cortex__bucket_store_cached_postings_compression_time_seconds` => `bucket_store_cached_postings_compression_time_seconds{component=""}` + * `cortex__bucket_store_cached_postings_original_size_bytes_total` => `bucket_store_cached_postings_original_size_bytes_total{component=""}` + * `cortex__bucket_store_cached_postings_compressed_size_bytes_total` => `bucket_store_cached_postings_compressed_size_bytes_total{component=""}` + * `cortex__blocks_sync_seconds` => `bucket_stores_blocks_sync_seconds{component=""}` + * `cortex__blocks_last_successful_sync_timestamp_seconds` => `bucket_stores_blocks_last_successful_sync_timestamp_seconds{component=""}` * [FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client & Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. #2502 * [FEATURE] The flag `-frontend.max-cache-freshness` is now supported within the limits overrides, to specify per-tenant max cache freshness values. The corresponding YAML config parameter has been changed from `results_cache.max_freshness` to `limits_config.max_cache_freshness`. The legacy YAML config parameter (`results_cache.max_freshness`) will continue to be supported till Cortex release `v1.4.0`. #2609 * [ENHANCEMENT] Experimental TSDB: added the following metrics to the ingester: #2580 #2583 #2589 diff --git a/pkg/querier/blocks_bucket_stores_service.go b/pkg/querier/blocks_bucket_stores_service.go index 31fa7b1ada..0941806d2b 100644 --- a/pkg/querier/blocks_bucket_stores_service.go +++ b/pkg/querier/blocks_bucket_stores_service.go @@ -33,7 +33,7 @@ type BucketStoresService struct { func NewBucketStoresService(cfg tsdb.Config, bucketClient objstore.Bucket, logLevel logging.Level, logger log.Logger, registerer prometheus.Registerer) (*BucketStoresService, error) { var storesReg prometheus.Registerer if registerer != nil { - storesReg = prometheus.WrapRegistererWithPrefix("cortex_querier_", registerer) + storesReg = prometheus.WrapRegistererWith(prometheus.Labels{"component": "querier"}, registerer) } stores, err := storegateway.NewBucketStores(cfg, nil, bucketClient, logLevel, logger, storesReg) diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go index a520eb5f60..91fa34b06f 100644 --- a/pkg/storegateway/bucket_stores.go +++ b/pkg/storegateway/bucket_stores.go @@ -69,12 +69,12 @@ func NewBucketStores(cfg tsdb.Config, filters []block.MetadataFilter, bucketClie bucketStoreMetrics: NewBucketStoreMetrics(), metaFetcherMetrics: NewMetadataFetcherMetrics(), syncTimes: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ - Name: "blocks_sync_seconds", + Name: "bucket_stores_blocks_sync_seconds", Help: "The total time it takes to perform a sync stores", Buckets: []float64{0.1, 1, 10, 30, 60, 120, 300, 600, 900}, }), syncLastSuccess: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "blocks_last_successful_sync_timestamp_seconds", + Name: "bucket_stores_blocks_last_successful_sync_timestamp_seconds", Help: "Unix timestamp of the last successful blocks sync.", }), } diff --git a/pkg/storegateway/gateway.go b/pkg/storegateway/gateway.go index d454106667..cd7563df40 100644 --- a/pkg/storegateway/gateway.go +++ b/pkg/storegateway/gateway.go @@ -143,7 +143,7 @@ func newStoreGateway(gatewayCfg Config, storageCfg cortex_tsdb.Config, bucketCli var storesReg prometheus.Registerer if reg != nil { - storesReg = prometheus.WrapRegistererWithPrefix("cortex_storegateway_", reg) + storesReg = prometheus.WrapRegistererWith(prometheus.Labels{"component": "storegateway"}, reg) } g.stores, err = NewBucketStores(storageCfg, filters, bucketClient, logLevel, logger, storesReg) From 7202b0fe10a29c87852d44fbdd633c40e4c0ccce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 25 May 2020 12:51:18 +0200 Subject: [PATCH 3/9] Fix tests. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/storegateway/gateway_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/storegateway/gateway_test.go b/pkg/storegateway/gateway_test.go index 136f280ced..777aab2452 100644 --- a/pkg/storegateway/gateway_test.go +++ b/pkg/storegateway/gateway_test.go @@ -271,11 +271,11 @@ func TestStoreGateway_BlocksSharding(t *testing.T) { // Assert on the number of blocks loaded extracting this information from metrics. metrics := util.BuildMetricFamiliesPerUserFromUserRegistries(registries) - assert.Equal(t, float64(testData.expectedBlocksLoaded), metrics.GetSumOfGauges("cortex_storegateway_bucket_store_blocks_loaded")) + assert.Equal(t, float64(testData.expectedBlocksLoaded), metrics.GetSumOfGauges("bucket_store_blocks_loaded")) // The total number of blocks synced (before filtering) is always equal to the total // number of blocks for each instance. - assert.Equal(t, float64(testData.numGateways*numBlocks), metrics.GetSumOfGauges("cortex_storegateway_blocks_meta_synced")) + assert.Equal(t, float64(testData.numGateways*numBlocks), metrics.GetSumOfGauges("blocks_meta_synced")) }) } } From 7708c652b85d896f25ed0e606bb71c5632b81a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 25 May 2020 14:03:19 +0200 Subject: [PATCH 4/9] Fixed tests after renaming metrics. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- CHANGELOG.md | 2 -- integration/querier_test.go | 72 ++++++++++++++++++------------------- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efa99555df..a0c9bb43b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,6 @@ * `cortex__blocks_index_cache_items` => `thanos_store_index_cache_items{name="index-cache"}` * `cortex__blocks_index_cache_items_size_bytes` => `thanos_store_index_cache_items_size_bytes{name="index-cache"}` * `cortex__blocks_index_cache_total_size_bytes` => `thanos_store_index_cache_total_size_bytes{name="index-cache"}` - * `cortex__blocks_index_cache_requests_total` => `thanos_store_index_cache_requests_total{name="index-cache"}` - * `cortex__blocks_index_cache_hits_total` => `thanos_store_index_cache_hits_total{name="index-cache"}` * `cortex__blocks_index_cache_memcached_operations_total` => `thanos_memcached_operations_total{name="index-cache"}` * `cortex__blocks_index_cache_memcached_operation_failures_total` => `thanos_memcached_operation_failures_total{name="index-cache"}` * `cortex__blocks_index_cache_memcached_operation_duration_seconds` => `thanos_memcached_operation_duration_seconds{name="index-cache"}` diff --git a/integration/querier_test.go b/integration/querier_test.go index 2e208185a8..59f2b0773d 100644 --- a/integration/querier_test.go +++ b/integration/querier_test.go @@ -128,7 +128,7 @@ func TestQuerierWithBlocksStorageWithoutStoreGateway(t *testing.T) { require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(2), "cortex_ingester_memory_series_removed_total")) // Wait until the querier has synched the new uploaded blocks. - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "cortex_querier_bucket_store_blocks_loaded")) + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "bucket_store_blocks_loaded")) // Query back the series (1 only in the storage, 1 only in the ingesters, 1 on both). result, err := c.Query("series_1", series1Timestamp) @@ -147,14 +147,14 @@ func TestQuerierWithBlocksStorageWithoutStoreGateway(t *testing.T) { assert.Equal(t, expectedVector3, result.(model.Vector)) // Check the in-memory index cache metrics (in the querier). - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(7), "cortex_querier_blocks_index_cache_requests_total")) - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(0), "cortex_querier_blocks_index_cache_hits_total")) // no cache hit cause the cache was empty + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(7), "thanos_store_index_cache_requests_total")) + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(0), "thanos_store_index_cache_hits_total")) // no cache hit cause the cache was empty if indexCacheBackend == tsdb.IndexCacheBackendInMemory { - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*2), "cortex_querier_blocks_index_cache_items")) // 2 series both for postings and series cache - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*2), "cortex_querier_blocks_index_cache_items_added_total")) // 2 series both for postings and series cache + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items")) // 2 series both for postings and series cache + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items_added_total")) // 2 series both for postings and series cache } else if indexCacheBackend == tsdb.IndexCacheBackendMemcached { - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(11), "cortex_querier_blocks_index_cache_memcached_operations_total")) // 7 gets + 4 sets + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(11), "thanos_memcached_operations_total")) // 7 gets + 4 sets } // Query back again the 1st series from storage. This time it should use the index cache. @@ -163,14 +163,14 @@ func TestQuerierWithBlocksStorageWithoutStoreGateway(t *testing.T) { require.Equal(t, model.ValVector, result.Type()) assert.Equal(t, expectedVector1, result.(model.Vector)) - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(7+2), "cortex_querier_blocks_index_cache_requests_total")) - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "cortex_querier_blocks_index_cache_hits_total")) // this time has used the index cache + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(7+2), "thanos_store_index_cache_requests_total")) + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "thanos_store_index_cache_hits_total")) // this time has used the index cache if indexCacheBackend == tsdb.IndexCacheBackendInMemory { - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*2), "cortex_querier_blocks_index_cache_items")) // as before - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*2), "cortex_querier_blocks_index_cache_items_added_total")) // as before + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items")) // as before + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items_added_total")) // as before } else if indexCacheBackend == tsdb.IndexCacheBackendMemcached { - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(11+2), "cortex_querier_blocks_index_cache_memcached_operations_total")) // as before + 2 gets + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(11+2), "thanos_memcached_operations_total")) // as before + 2 gets } // Ensure no service-specific metrics prefix is used by the wrong service. @@ -307,15 +307,15 @@ func TestQuerierWithBlocksStorageAndStoreGatewayRunningInMicroservicesMode(t *te require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(2), "cortex_ingester_memory_series_removed_total")) // Wait until the querier has discovered the uploaded blocks. - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "cortex_querier_blocks_meta_synced")) + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "blocks_meta_synced")) // Wait until the store-gateway has synched the new uploaded blocks. When sharding is enabled // we don't known which store-gateway instance will synch the blocks, so we need to wait on // metrics extracted from all instances. if testCfg.blocksShardingEnabled { - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2), "cortex_storegateway_bucket_store_blocks_loaded")) + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2), "bucket_store_blocks_loaded")) } else { - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(float64(2*storeGateways.NumInstances())), "cortex_storegateway_bucket_store_blocks_loaded")) + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(float64(2*storeGateways.NumInstances())), "bucket_store_blocks_loaded")) } // Query back the series (1 only in the storage, 1 only in the ingesters, 1 on both). @@ -335,14 +335,14 @@ func TestQuerierWithBlocksStorageAndStoreGatewayRunningInMicroservicesMode(t *te assert.Equal(t, expectedVector3, result.(model.Vector)) // Check the in-memory index cache metrics (in the store-gateway). - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(7), "cortex_storegateway_blocks_index_cache_requests_total")) - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(0), "cortex_storegateway_blocks_index_cache_hits_total")) // no cache hit cause the cache was empty + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(7), "thanos_store_index_cache_requests_total")) + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(0), "thanos_store_index_cache_hits_total")) // no cache hit cause the cache was empty if testCfg.indexCacheBackend == tsdb.IndexCacheBackendInMemory { - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "cortex_storegateway_blocks_index_cache_items")) // 2 series both for postings and series cache - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "cortex_storegateway_blocks_index_cache_items_added_total")) // 2 series both for postings and series cache + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items")) // 2 series both for postings and series cache + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items_added_total")) // 2 series both for postings and series cache } else if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached { - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(11), "cortex_storegateway_blocks_index_cache_memcached_operations_total")) // 7 gets + 4 sets + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(11), "thanos_memcached_operations_total")) // 7 gets + 4 sets } // Query back again the 1st series from storage. This time it should use the index cache. @@ -351,14 +351,14 @@ func TestQuerierWithBlocksStorageAndStoreGatewayRunningInMicroservicesMode(t *te require.Equal(t, model.ValVector, result.Type()) assert.Equal(t, expectedVector1, result.(model.Vector)) - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(7+2), "cortex_storegateway_blocks_index_cache_requests_total")) - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2), "cortex_storegateway_blocks_index_cache_hits_total")) // this time has used the index cache + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(7+2), "thanos_store_index_cache_requests_total")) + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2), "thanos_store_index_cache_hits_total")) // this time has used the index cache if testCfg.indexCacheBackend == tsdb.IndexCacheBackendInMemory { - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "cortex_storegateway_blocks_index_cache_items")) // as before - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "cortex_storegateway_blocks_index_cache_items_added_total")) // as before + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items")) // as before + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items_added_total")) // as before } else if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached { - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(11+2), "cortex_storegateway_blocks_index_cache_memcached_operations_total")) // as before + 2 gets + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(11+2), "thanos_memcached_operations_total")) // as before + 2 gets } // Ensure no service-specific metrics prefix is used by the wrong service. @@ -505,9 +505,9 @@ func TestQuerierWithBlocksStorageAndStoreGatewayRunningInSingleBinaryMode(t *tes const shippedBlocks = 2 if testCfg.blocksShardingEnabled { - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(shippedBlocks*seriesReplicationFactor)), "cortex_storegateway_bucket_store_blocks_loaded")) + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(shippedBlocks*seriesReplicationFactor)), "bucket_store_blocks_loaded")) } else { - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(shippedBlocks*seriesReplicationFactor*cluster.NumInstances())), "cortex_storegateway_bucket_store_blocks_loaded")) + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(shippedBlocks*seriesReplicationFactor*cluster.NumInstances())), "bucket_store_blocks_loaded")) } // Query back the series (1 only in the storage, 1 only in the ingesters, 1 on both). @@ -527,14 +527,14 @@ func TestQuerierWithBlocksStorageAndStoreGatewayRunningInSingleBinaryMode(t *tes assert.Equal(t, expectedVector3, result.(model.Vector)) // Check the in-memory index cache metrics (in the store-gateway). - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(7*seriesReplicationFactor)), "cortex_storegateway_blocks_index_cache_requests_total")) - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(0), "cortex_storegateway_blocks_index_cache_hits_total")) // no cache hit cause the cache was empty + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(7*seriesReplicationFactor)), "thanos_store_index_cache_requests_total")) + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(0), "thanos_store_index_cache_hits_total")) // no cache hit cause the cache was empty if testCfg.indexCacheBackend == tsdb.IndexCacheBackendInMemory { - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "cortex_storegateway_blocks_index_cache_items")) // 2 series both for postings and series cache - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "cortex_storegateway_blocks_index_cache_items_added_total")) // 2 series both for postings and series cache + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "thanos_store_index_cache_items")) // 2 series both for postings and series cache + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "thanos_store_index_cache_items_added_total")) // 2 series both for postings and series cache } else if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached { - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(11*seriesReplicationFactor)), "cortex_storegateway_blocks_index_cache_memcached_operations_total")) // 7 gets + 4 sets + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(11*seriesReplicationFactor)), "thanos_memcached_operations_total")) // 7 gets + 4 sets } // Query back again the 1st series from storage. This time it should use the index cache. @@ -543,14 +543,14 @@ func TestQuerierWithBlocksStorageAndStoreGatewayRunningInSingleBinaryMode(t *tes require.Equal(t, model.ValVector, result.Type()) assert.Equal(t, expectedVector1, result.(model.Vector)) - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64((7+2)*seriesReplicationFactor)), "cortex_storegateway_blocks_index_cache_requests_total")) - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*seriesReplicationFactor)), "cortex_storegateway_blocks_index_cache_hits_total")) // this time has used the index cache + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64((7+2)*seriesReplicationFactor)), "thanos_store_index_cache_requests_total")) + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*seriesReplicationFactor)), "thanos_store_index_cache_hits_total")) // this time has used the index cache if testCfg.indexCacheBackend == tsdb.IndexCacheBackendInMemory { - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "cortex_storegateway_blocks_index_cache_items")) // as before - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "cortex_storegateway_blocks_index_cache_items_added_total")) // as before + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "thanos_store_index_cache_items")) // as before + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "thanos_store_index_cache_items_added_total")) // as before } else if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached { - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64((11+2)*seriesReplicationFactor)), "cortex_storegateway_blocks_index_cache_memcached_operations_total")) // as before + 2 gets + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64((11+2)*seriesReplicationFactor)), "thanos_memcached_operations_total")) // as before + 2 gets } }) } From 4289de82b04a0493af2fc2b92c435c2ba29bb878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 25 May 2020 16:11:58 +0200 Subject: [PATCH 5/9] Add cortex_ prefix to metrics (re-)defined in Cortex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- CHANGELOG.md | 52 ++-- pkg/querier/blocks_bucket_stores_service.go | 8 +- pkg/querier/blocks_scanner.go | 2 +- pkg/querier/blocks_scanner_test.go | 48 ++-- pkg/storegateway/bucket_store_metrics.go | 38 +-- pkg/storegateway/bucket_store_metrics_test.go | 264 +++++++++--------- pkg/storegateway/bucket_stores.go | 4 +- pkg/storegateway/bucket_stores_test.go | 44 +-- pkg/storegateway/gateway.go | 8 +- pkg/storegateway/gateway_test.go | 4 +- pkg/storegateway/metadata_fetcher_metrics.go | 10 +- .../metadata_fetcher_metrics_test.go | 56 ++-- 12 files changed, 265 insertions(+), 273 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0c9bb43b2..3f2698138e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,32 +18,32 @@ * `cortex__blocks_index_cache_memcached_operation_duration_seconds` => `thanos_memcached_operation_duration_seconds{name="index-cache"}` * `cortex__blocks_index_cache_memcached_operation_skipped_total` => `thanos_memcached_operation_skipped_total{name="index-cache"}` * [CHANGE] Experimental TSDB: Renamed metrics in bucket stores: - * `cortex__blocks_meta_syncs_total` => `blocks_meta_syncs_total{component=""}` - * `cortex__blocks_meta_sync_failures_total` => `blocks_meta_sync_failures_total{component=""}` - * `cortex__blocks_meta_sync_duration_seconds` => `blocks_meta_sync_duration_seconds{component=""}` - * `cortex__blocks_meta_sync_consistency_delay_seconds` => `blocks_meta_sync_consistency_delay_seconds{component=""}` - * `cortex__blocks_meta_synced` => `blocks_meta_synced{component=""}` - * `cortex__bucket_store_block_loads_total` => `bucket_store_block_loads_total{component=""}` - * `cortex__bucket_store_block_load_failures_total` => `bucket_store_block_load_failures_total{component=""}` - * `cortex__bucket_store_block_drops_total` => `bucket_store_block_drops_total{component=""}` - * `cortex__bucket_store_block_drop_failures_total` => `bucket_store_block_drop_failures_total{component=""}` - * `cortex__bucket_store_blocks_loaded` => `bucket_store_blocks_loaded{component=""}` - * `cortex__bucket_store_series_data_touched` => `bucket_store_series_data_touched{component=""}` - * `cortex__bucket_store_series_data_fetched` => `bucket_store_series_data_fetched{component=""}` - * `cortex__bucket_store_series_data_size_touched_bytes` => `bucket_store_series_data_size_touched_bytes{component=""}` - * `cortex__bucket_store_series_data_size_fetched_bytes` => `bucket_store_series_data_size_fetched_bytes{component=""}` - * `cortex__bucket_store_series_blocks_queried` => `bucket_store_series_blocks_queried{component=""}` - * `cortex__bucket_store_series_get_all_duration_seconds` => `bucket_store_series_get_all_duration_seconds{component=""}` - * `cortex__bucket_store_series_merge_duration_seconds` => `bucket_store_series_merge_duration_seconds{component=""}` - * `cortex__bucket_store_series_refetches_total` => `bucket_store_series_refetches_total{component=""}` - * `cortex__bucket_store_series_result_series` => `bucket_store_series_result_series{component=""}` - * `cortex__bucket_store_cached_postings_compressions_total` => `bucket_store_cached_postings_compressions_total{component=""}` - * `cortex__bucket_store_cached_postings_compression_errors_total` => `bucket_store_cached_postings_compression_errors_total{component=""}` - * `cortex__bucket_store_cached_postings_compression_time_seconds` => `bucket_store_cached_postings_compression_time_seconds{component=""}` - * `cortex__bucket_store_cached_postings_original_size_bytes_total` => `bucket_store_cached_postings_original_size_bytes_total{component=""}` - * `cortex__bucket_store_cached_postings_compressed_size_bytes_total` => `bucket_store_cached_postings_compressed_size_bytes_total{component=""}` - * `cortex__blocks_sync_seconds` => `bucket_stores_blocks_sync_seconds{component=""}` - * `cortex__blocks_last_successful_sync_timestamp_seconds` => `bucket_stores_blocks_last_successful_sync_timestamp_seconds{component=""}` + * `cortex__blocks_meta_syncs_total` => `cortex_blocks_meta_syncs_total{component=""}` + * `cortex__blocks_meta_sync_failures_total` => `cortex_blocks_meta_sync_failures_total{component=""}` + * `cortex__blocks_meta_sync_duration_seconds` => `cortex_blocks_meta_sync_duration_seconds{component=""}` + * `cortex__blocks_meta_sync_consistency_delay_seconds` => `cortex_blocks_meta_sync_consistency_delay_seconds{component=""}` + * `cortex__blocks_meta_synced` => `cortex_blocks_meta_synced{component=""}` + * `cortex__bucket_store_block_loads_total` => `cortex_block_loads_total{component=""}` + * `cortex__bucket_store_block_load_failures_total` => `cortex_block_load_failures_total{component=""}` + * `cortex__bucket_store_block_drops_total` => `cortex_block_drops_total{component=""}` + * `cortex__bucket_store_block_drop_failures_total` => `cortex_bucket_store_block_drop_failures_total{component=""}` + * `cortex__bucket_store_blocks_loaded` => `cortex_bucket_store_blocks_loaded{component=""}` + * `cortex__bucket_store_series_data_touched` => `cortex_bucket_store_series_data_touched{component=""}` + * `cortex__bucket_store_series_data_fetched` => `cortex_bucket_store_series_data_fetched{component=""}` + * `cortex__bucket_store_series_data_size_touched_bytes` => `cortex_bucket_store_series_data_size_touched_bytes{component=""}` + * `cortex__bucket_store_series_data_size_fetched_bytes` => `cortex_bucket_store_series_data_size_fetched_bytes{component=""}` + * `cortex__bucket_store_series_blocks_queried` => `cortex_bucket_store_series_blocks_queried{component=""}` + * `cortex__bucket_store_series_get_all_duration_seconds` => `cortex_bucket_store_series_get_all_duration_seconds{component=""}` + * `cortex__bucket_store_series_merge_duration_seconds` => `cortex_bucket_store_series_merge_duration_seconds{component=""}` + * `cortex__bucket_store_series_refetches_total` => `cortex_bucket_store_series_refetches_total{component=""}` + * `cortex__bucket_store_series_result_series` => `cortex_bucket_store_series_result_series{component=""}` + * `cortex__bucket_store_cached_postings_compressions_total` => `cortex_bucket_store_cached_postings_compressions_total{component=""}` + * `cortex__bucket_store_cached_postings_compression_errors_total` => `cortex_bucket_store_cached_postings_compression_errors_total{component=""}` + * `cortex__bucket_store_cached_postings_compression_time_seconds` => `cortex_bucket_store_cached_postings_compression_time_seconds{component=""}` + * `cortex__bucket_store_cached_postings_original_size_bytes_total` => `cortex_bucket_store_cached_postings_original_size_bytes_total{component=""}` + * `cortex__bucket_store_cached_postings_compressed_size_bytes_total` => `cortex_bucket_store_cached_postings_compressed_size_bytes_total{component=""}` + * `cortex__blocks_sync_seconds` => `cortex_bucket_stores_blocks_sync_seconds{component=""}` + * `cortex__blocks_last_successful_sync_timestamp_seconds` => `cortex_bucket_stores_blocks_last_successful_sync_timestamp_seconds{component=""}` * [FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client & Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. #2502 * [FEATURE] The flag `-frontend.max-cache-freshness` is now supported within the limits overrides, to specify per-tenant max cache freshness values. The corresponding YAML config parameter has been changed from `results_cache.max_freshness` to `limits_config.max_cache_freshness`. The legacy YAML config parameter (`results_cache.max_freshness`) will continue to be supported till Cortex release `v1.4.0`. #2609 * [ENHANCEMENT] Experimental TSDB: added the following metrics to the ingester: #2580 #2583 #2589 diff --git a/pkg/querier/blocks_bucket_stores_service.go b/pkg/querier/blocks_bucket_stores_service.go index 0941806d2b..e40c388e0b 100644 --- a/pkg/querier/blocks_bucket_stores_service.go +++ b/pkg/querier/blocks_bucket_stores_service.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/storage" + "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" @@ -31,12 +32,7 @@ type BucketStoresService struct { } func NewBucketStoresService(cfg tsdb.Config, bucketClient objstore.Bucket, logLevel logging.Level, logger log.Logger, registerer prometheus.Registerer) (*BucketStoresService, error) { - var storesReg prometheus.Registerer - if registerer != nil { - storesReg = prometheus.WrapRegistererWith(prometheus.Labels{"component": "querier"}, registerer) - } - - stores, err := storegateway.NewBucketStores(cfg, nil, bucketClient, logLevel, logger, storesReg) + stores, err := storegateway.NewBucketStores(cfg, nil, bucketClient, logLevel, logger, extprom.WrapRegistererWith(prometheus.Labels{"component": "querier"}, registerer)) if err != nil { return nil, err } diff --git a/pkg/querier/blocks_scanner.go b/pkg/querier/blocks_scanner.go index 1ab7596ec9..1c18de9d4d 100644 --- a/pkg/querier/blocks_scanner.go +++ b/pkg/querier/blocks_scanner.go @@ -82,7 +82,7 @@ func NewBlocksScanner(cfg BlocksScannerConfig, bucketClient objstore.Bucket, log } if reg != nil { - prometheus.WrapRegistererWithPrefix("cortex_querier_", reg).MustRegister(d.fetchersMetrics) + prometheus.WrapRegistererWith(prometheus.Labels{"component": "querier"}, reg).MustRegister(d.fetchersMetrics) } d.Service = services.NewTimerService(cfg.ScanInterval, d.starting, d.scan, nil) diff --git a/pkg/querier/blocks_scanner_test.go b/pkg/querier/blocks_scanner_test.go index eefed633e8..abda9339a9 100644 --- a/pkg/querier/blocks_scanner_test.go +++ b/pkg/querier/blocks_scanner_test.go @@ -55,21 +55,21 @@ func TestBlocksScanner_InitialScan(t *testing.T) { }, deletionMarks) assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_querier_blocks_meta_syncs_total Total blocks metadata synchronization attempts - # TYPE cortex_querier_blocks_meta_syncs_total counter - cortex_querier_blocks_meta_syncs_total 2 + # HELP cortex_blocks_meta_syncs_total Total blocks metadata synchronization attempts + # TYPE cortex_blocks_meta_syncs_total counter + cortex_blocks_meta_syncs_total{component="querier"} 2 - # HELP cortex_querier_blocks_meta_sync_failures_total Total blocks metadata synchronization failures - # TYPE cortex_querier_blocks_meta_sync_failures_total counter - cortex_querier_blocks_meta_sync_failures_total 0 + # HELP cortex_blocks_meta_sync_failures_total Total blocks metadata synchronization failures + # TYPE cortex_blocks_meta_sync_failures_total counter + cortex_blocks_meta_sync_failures_total{component="querier"} 0 - # HELP cortex_querier_blocks_meta_sync_consistency_delay_seconds Configured consistency delay in seconds. - # TYPE cortex_querier_blocks_meta_sync_consistency_delay_seconds gauge - cortex_querier_blocks_meta_sync_consistency_delay_seconds 0 + # HELP cortex_blocks_meta_sync_consistency_delay_seconds Configured consistency delay in seconds. + # TYPE cortex_blocks_meta_sync_consistency_delay_seconds gauge + cortex_blocks_meta_sync_consistency_delay_seconds{component="querier"} 0 `), - "cortex_querier_blocks_meta_syncs_total", - "cortex_querier_blocks_meta_sync_failures_total", - "cortex_querier_blocks_meta_sync_consistency_delay_seconds", + "cortex_blocks_meta_syncs_total", + "cortex_blocks_meta_sync_failures_total", + "cortex_blocks_meta_sync_consistency_delay_seconds", )) assert.Greater(t, testutil.ToFloat64(s.scanLastSuccess), float64(0)) @@ -107,25 +107,25 @@ func TestBlocksScanner_InitialScanFailure(t *testing.T) { assert.Nil(t, deletionMarks) assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_querier_blocks_meta_syncs_total Total blocks metadata synchronization attempts - # TYPE cortex_querier_blocks_meta_syncs_total counter - cortex_querier_blocks_meta_syncs_total 3 + # HELP cortex_blocks_meta_syncs_total Total blocks metadata synchronization attempts + # TYPE cortex_blocks_meta_syncs_total counter + cortex_blocks_meta_syncs_total{component="querier"} 3 - # HELP cortex_querier_blocks_meta_sync_failures_total Total blocks metadata synchronization failures - # TYPE cortex_querier_blocks_meta_sync_failures_total counter - cortex_querier_blocks_meta_sync_failures_total 3 + # HELP cortex_blocks_meta_sync_failures_total Total blocks metadata synchronization failures + # TYPE cortex_blocks_meta_sync_failures_total counter + cortex_blocks_meta_sync_failures_total{component="querier"} 3 - # HELP cortex_querier_blocks_meta_sync_consistency_delay_seconds Configured consistency delay in seconds. - # TYPE cortex_querier_blocks_meta_sync_consistency_delay_seconds gauge - cortex_querier_blocks_meta_sync_consistency_delay_seconds 0 + # HELP cortex_blocks_meta_sync_consistency_delay_seconds Configured consistency delay in seconds. + # TYPE cortex_blocks_meta_sync_consistency_delay_seconds gauge + cortex_blocks_meta_sync_consistency_delay_seconds{component="querier"} 0 # HELP cortex_querier_blocks_last_successful_scan_timestamp_seconds Unix timestamp of the last successful blocks scan. # TYPE cortex_querier_blocks_last_successful_scan_timestamp_seconds gauge cortex_querier_blocks_last_successful_scan_timestamp_seconds 0 `), - "cortex_querier_blocks_meta_syncs_total", - "cortex_querier_blocks_meta_sync_failures_total", - "cortex_querier_blocks_meta_sync_consistency_delay_seconds", + "cortex_blocks_meta_syncs_total", + "cortex_blocks_meta_sync_failures_total", + "cortex_blocks_meta_sync_consistency_delay_seconds", "cortex_querier_blocks_last_successful_scan_timestamp_seconds", )) } diff --git a/pkg/storegateway/bucket_store_metrics.go b/pkg/storegateway/bucket_store_metrics.go index 764c86740e..853478c251 100644 --- a/pkg/storegateway/bucket_store_metrics.go +++ b/pkg/storegateway/bucket_store_metrics.go @@ -43,81 +43,81 @@ func NewBucketStoreMetrics() *BucketStoreMetrics { regs: map[string]*prometheus.Registry{}, blockLoads: prometheus.NewDesc( - "bucket_store_block_loads_total", + "cortex_bucket_store_block_loads_total", "Total number of remote block loading attempts.", nil, nil), blockLoadFailures: prometheus.NewDesc( - "bucket_store_block_load_failures_total", + "cortex_bucket_store_block_load_failures_total", "Total number of failed remote block loading attempts.", nil, nil), blockDrops: prometheus.NewDesc( - "bucket_store_block_drops_total", + "cortex_bucket_store_block_drops_total", "Total number of local blocks that were dropped.", nil, nil), blockDropFailures: prometheus.NewDesc( - "bucket_store_block_drop_failures_total", + "cortex_bucket_store_block_drop_failures_total", "Total number of local blocks that failed to be dropped.", nil, nil), blocksLoaded: prometheus.NewDesc( - "bucket_store_blocks_loaded", + "cortex_bucket_store_blocks_loaded", "Number of currently loaded blocks.", nil, nil), seriesDataTouched: prometheus.NewDesc( - "bucket_store_series_data_touched", + "cortex_bucket_store_series_data_touched", "How many items of a data type in a block were touched for a single series request.", []string{"data_type"}, nil), seriesDataFetched: prometheus.NewDesc( - "bucket_store_series_data_fetched", + "cortex_bucket_store_series_data_fetched", "How many items of a data type in a block were fetched for a single series request.", []string{"data_type"}, nil), seriesDataSizeTouched: prometheus.NewDesc( - "bucket_store_series_data_size_touched_bytes", + "cortex_bucket_store_series_data_size_touched_bytes", "Size of all items of a data type in a block were touched for a single series request.", []string{"data_type"}, nil), seriesDataSizeFetched: prometheus.NewDesc( - "bucket_store_series_data_size_fetched_bytes", + "cortex_bucket_store_series_data_size_fetched_bytes", "Size of all items of a data type in a block were fetched for a single series request.", []string{"data_type"}, nil), seriesBlocksQueried: prometheus.NewDesc( - "bucket_store_series_blocks_queried", + "cortex_bucket_store_series_blocks_queried", "Number of blocks in a bucket store that were touched to satisfy a query.", nil, nil), seriesGetAllDuration: prometheus.NewDesc( - "bucket_store_series_get_all_duration_seconds", + "cortex_bucket_store_series_get_all_duration_seconds", "Time it takes until all per-block prepares and preloads for a query are finished.", nil, nil), seriesMergeDuration: prometheus.NewDesc( - "bucket_store_series_merge_duration_seconds", + "cortex_bucket_store_series_merge_duration_seconds", "Time it takes to merge sub-results from all queried blocks into a single result.", nil, nil), seriesRefetches: prometheus.NewDesc( - "bucket_store_series_refetches_total", + "cortex_bucket_store_series_refetches_total", "Total number of cases where the built-in max series size was not enough to fetch series from index, resulting in refetch.", nil, nil), resultSeriesCount: prometheus.NewDesc( - "bucket_store_series_result_series", + "cortex_bucket_store_series_result_series", "Number of series observed in the final result of a query.", nil, nil), cachedPostingsCompressions: prometheus.NewDesc( - "bucket_store_cached_postings_compressions_total", + "cortex_bucket_store_cached_postings_compressions_total", "Number of postings compressions and decompressions when storing to index cache.", []string{"op"}, nil), cachedPostingsCompressionErrors: prometheus.NewDesc( - "bucket_store_cached_postings_compression_errors_total", + "cortex_bucket_store_cached_postings_compression_errors_total", "Number of postings compression and decompression errors.", []string{"op"}, nil), cachedPostingsCompressionTimeSeconds: prometheus.NewDesc( - "bucket_store_cached_postings_compression_time_seconds", + "cortex_bucket_store_cached_postings_compression_time_seconds", "Time spent compressing and decompressing postings when storing to / reading from postings cache.", []string{"op"}, nil), cachedPostingsOriginalSizeBytes: prometheus.NewDesc( - "bucket_store_cached_postings_original_size_bytes_total", + "cortex_bucket_store_cached_postings_original_size_bytes_total", "Original size of postings stored into cache.", nil, nil), cachedPostingsCompressedSizeBytes: prometheus.NewDesc( - "bucket_store_cached_postings_compressed_size_bytes_total", + "cortex_bucket_store_cached_postings_compressed_size_bytes_total", "Compressed size of postings stored into cache.", nil, nil), } diff --git a/pkg/storegateway/bucket_store_metrics_test.go b/pkg/storegateway/bucket_store_metrics_test.go index b83fa86940..fc4a712afe 100644 --- a/pkg/storegateway/bucket_store_metrics_test.go +++ b/pkg/storegateway/bucket_store_metrics_test.go @@ -23,138 +23,138 @@ func TestBucketStoreMetrics(t *testing.T) { //noinspection ALL err := testutil.GatherAndCompare(mainReg, bytes.NewBufferString(` - # HELP bucket_store_blocks_loaded Number of currently loaded blocks. - # TYPE bucket_store_blocks_loaded gauge - bucket_store_blocks_loaded 22519 - - # HELP bucket_store_block_loads_total Total number of remote block loading attempts. - # TYPE bucket_store_block_loads_total counter - bucket_store_block_loads_total 45038 - - # HELP bucket_store_block_load_failures_total Total number of failed remote block loading attempts. - # TYPE bucket_store_block_load_failures_total counter - bucket_store_block_load_failures_total 67557 - - # HELP bucket_store_block_drops_total Total number of local blocks that were dropped. - # TYPE bucket_store_block_drops_total counter - bucket_store_block_drops_total 90076 - - # HELP bucket_store_block_drop_failures_total Total number of local blocks that failed to be dropped. - # TYPE bucket_store_block_drop_failures_total counter - bucket_store_block_drop_failures_total 112595 - - # HELP bucket_store_series_blocks_queried Number of blocks in a bucket store that were touched to satisfy a query. - # TYPE bucket_store_series_blocks_queried summary - bucket_store_series_blocks_queried_sum 1.283583e+06 - bucket_store_series_blocks_queried_count 9 - - # HELP bucket_store_series_data_fetched How many items of a data type in a block were fetched for a single series request. - # TYPE bucket_store_series_data_fetched summary - bucket_store_series_data_fetched_sum{data_type="fetched-a"} 202671 - bucket_store_series_data_fetched_count{data_type="fetched-a"} 3 - bucket_store_series_data_fetched_sum{data_type="fetched-b"} 225190 - bucket_store_series_data_fetched_count{data_type="fetched-b"} 3 - bucket_store_series_data_fetched_sum{data_type="fetched-c"} 247709 - bucket_store_series_data_fetched_count{data_type="fetched-c"} 3 - - # HELP bucket_store_series_data_size_fetched_bytes Size of all items of a data type in a block were fetched for a single series request. - # TYPE bucket_store_series_data_size_fetched_bytes summary - bucket_store_series_data_size_fetched_bytes_sum{data_type="size-fetched-a"} 337785 - bucket_store_series_data_size_fetched_bytes_count{data_type="size-fetched-a"} 3 - bucket_store_series_data_size_fetched_bytes_sum{data_type="size-fetched-b"} 360304 - bucket_store_series_data_size_fetched_bytes_count{data_type="size-fetched-b"} 3 - bucket_store_series_data_size_fetched_bytes_sum{data_type="size-fetched-c"} 382823 - bucket_store_series_data_size_fetched_bytes_count{data_type="size-fetched-c"} 3 - - # HELP bucket_store_series_data_size_touched_bytes Size of all items of a data type in a block were touched for a single series request. - # TYPE bucket_store_series_data_size_touched_bytes summary - bucket_store_series_data_size_touched_bytes_sum{data_type="size-touched-a"} 270228 - bucket_store_series_data_size_touched_bytes_count{data_type="size-touched-a"} 3 - bucket_store_series_data_size_touched_bytes_sum{data_type="size-touched-b"} 292747 - bucket_store_series_data_size_touched_bytes_count{data_type="size-touched-b"} 3 - bucket_store_series_data_size_touched_bytes_sum{data_type="size-touched-c"} 315266 - bucket_store_series_data_size_touched_bytes_count{data_type="size-touched-c"} 3 - - # HELP bucket_store_series_data_touched How many items of a data type in a block were touched for a single series request. - # TYPE bucket_store_series_data_touched summary - bucket_store_series_data_touched_sum{data_type="touched-a"} 135114 - bucket_store_series_data_touched_count{data_type="touched-a"} 3 - bucket_store_series_data_touched_sum{data_type="touched-b"} 157633 - bucket_store_series_data_touched_count{data_type="touched-b"} 3 - bucket_store_series_data_touched_sum{data_type="touched-c"} 180152 - bucket_store_series_data_touched_count{data_type="touched-c"} 3 - - # HELP bucket_store_series_get_all_duration_seconds Time it takes until all per-block prepares and preloads for a query are finished. - # TYPE bucket_store_series_get_all_duration_seconds histogram - bucket_store_series_get_all_duration_seconds_bucket{le="0.001"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="0.01"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="0.1"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="0.3"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="0.6"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="1"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="3"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="6"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="9"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="20"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="30"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="60"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="90"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="120"} 0 - bucket_store_series_get_all_duration_seconds_bucket{le="+Inf"} 9 - bucket_store_series_get_all_duration_seconds_sum 1.486254e+06 - bucket_store_series_get_all_duration_seconds_count 9 - - # HELP bucket_store_series_merge_duration_seconds Time it takes to merge sub-results from all queried blocks into a single result. - # TYPE bucket_store_series_merge_duration_seconds histogram - bucket_store_series_merge_duration_seconds_bucket{le="0.001"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="0.01"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="0.1"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="0.3"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="0.6"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="1"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="3"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="6"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="9"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="20"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="30"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="60"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="90"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="120"} 0 - bucket_store_series_merge_duration_seconds_bucket{le="+Inf"} 9 - bucket_store_series_merge_duration_seconds_sum 1.688925e+06 - bucket_store_series_merge_duration_seconds_count 9 - - # HELP bucket_store_series_refetches_total Total number of cases where the built-in max series size was not enough to fetch series from index, resulting in refetch. - # TYPE bucket_store_series_refetches_total counter - bucket_store_series_refetches_total 743127 - - # HELP bucket_store_series_result_series Number of series observed in the final result of a query. - # TYPE bucket_store_series_result_series summary - bucket_store_series_result_series_sum 1.238545e+06 - bucket_store_series_result_series_count 6 - - # HELP bucket_store_cached_postings_compressions_total Number of postings compressions and decompressions when storing to index cache. - # TYPE bucket_store_cached_postings_compressions_total counter - bucket_store_cached_postings_compressions_total{op="encode"} 1125950 - bucket_store_cached_postings_compressions_total{op="decode"} 1148469 - - # HELP bucket_store_cached_postings_compression_errors_total Number of postings compression and decompression errors. - # TYPE bucket_store_cached_postings_compression_errors_total counter - bucket_store_cached_postings_compression_errors_total{op="encode"} 1170988 - bucket_store_cached_postings_compression_errors_total{op="decode"} 1193507 - - # HELP bucket_store_cached_postings_compression_time_seconds Time spent compressing and decompressing postings when storing to / reading from postings cache. - # TYPE bucket_store_cached_postings_compression_time_seconds counter - bucket_store_cached_postings_compression_time_seconds{op="encode"} 1216026 - bucket_store_cached_postings_compression_time_seconds{op="decode"} 1238545 - - # HELP bucket_store_cached_postings_original_size_bytes_total Original size of postings stored into cache. - # TYPE bucket_store_cached_postings_original_size_bytes_total counter - bucket_store_cached_postings_original_size_bytes_total 1261064 - - # HELP bucket_store_cached_postings_compressed_size_bytes_total Compressed size of postings stored into cache. - # TYPE bucket_store_cached_postings_compressed_size_bytes_total counter - bucket_store_cached_postings_compressed_size_bytes_total 1283583 + # HELP cortex_bucket_store_blocks_loaded Number of currently loaded blocks. + # TYPE cortex_bucket_store_blocks_loaded gauge + cortex_bucket_store_blocks_loaded 22519 + + # HELP cortex_bucket_store_block_loads_total Total number of remote block loading attempts. + # TYPE cortex_bucket_store_block_loads_total counter + cortex_bucket_store_block_loads_total 45038 + + # HELP cortex_bucket_store_block_load_failures_total Total number of failed remote block loading attempts. + # TYPE cortex_bucket_store_block_load_failures_total counter + cortex_bucket_store_block_load_failures_total 67557 + + # HELP cortex_bucket_store_block_drops_total Total number of local blocks that were dropped. + # TYPE cortex_bucket_store_block_drops_total counter + cortex_bucket_store_block_drops_total 90076 + + # HELP cortex_bucket_store_block_drop_failures_total Total number of local blocks that failed to be dropped. + # TYPE cortex_bucket_store_block_drop_failures_total counter + cortex_bucket_store_block_drop_failures_total 112595 + + # HELP cortex_bucket_store_series_blocks_queried Number of blocks in a bucket store that were touched to satisfy a query. + # TYPE cortex_bucket_store_series_blocks_queried summary + cortex_bucket_store_series_blocks_queried_sum 1.283583e+06 + cortex_bucket_store_series_blocks_queried_count 9 + + # HELP cortex_bucket_store_series_data_fetched How many items of a data type in a block were fetched for a single series request. + # TYPE cortex_bucket_store_series_data_fetched summary + cortex_bucket_store_series_data_fetched_sum{data_type="fetched-a"} 202671 + cortex_bucket_store_series_data_fetched_count{data_type="fetched-a"} 3 + cortex_bucket_store_series_data_fetched_sum{data_type="fetched-b"} 225190 + cortex_bucket_store_series_data_fetched_count{data_type="fetched-b"} 3 + cortex_bucket_store_series_data_fetched_sum{data_type="fetched-c"} 247709 + cortex_bucket_store_series_data_fetched_count{data_type="fetched-c"} 3 + + # HELP cortex_bucket_store_series_data_size_fetched_bytes Size of all items of a data type in a block were fetched for a single series request. + # TYPE cortex_bucket_store_series_data_size_fetched_bytes summary + cortex_bucket_store_series_data_size_fetched_bytes_sum{data_type="size-fetched-a"} 337785 + cortex_bucket_store_series_data_size_fetched_bytes_count{data_type="size-fetched-a"} 3 + cortex_bucket_store_series_data_size_fetched_bytes_sum{data_type="size-fetched-b"} 360304 + cortex_bucket_store_series_data_size_fetched_bytes_count{data_type="size-fetched-b"} 3 + cortex_bucket_store_series_data_size_fetched_bytes_sum{data_type="size-fetched-c"} 382823 + cortex_bucket_store_series_data_size_fetched_bytes_count{data_type="size-fetched-c"} 3 + + # HELP cortex_bucket_store_series_data_size_touched_bytes Size of all items of a data type in a block were touched for a single series request. + # TYPE cortex_bucket_store_series_data_size_touched_bytes summary + cortex_bucket_store_series_data_size_touched_bytes_sum{data_type="size-touched-a"} 270228 + cortex_bucket_store_series_data_size_touched_bytes_count{data_type="size-touched-a"} 3 + cortex_bucket_store_series_data_size_touched_bytes_sum{data_type="size-touched-b"} 292747 + cortex_bucket_store_series_data_size_touched_bytes_count{data_type="size-touched-b"} 3 + cortex_bucket_store_series_data_size_touched_bytes_sum{data_type="size-touched-c"} 315266 + cortex_bucket_store_series_data_size_touched_bytes_count{data_type="size-touched-c"} 3 + + # HELP cortex_bucket_store_series_data_touched How many items of a data type in a block were touched for a single series request. + # TYPE cortex_bucket_store_series_data_touched summary + cortex_bucket_store_series_data_touched_sum{data_type="touched-a"} 135114 + cortex_bucket_store_series_data_touched_count{data_type="touched-a"} 3 + cortex_bucket_store_series_data_touched_sum{data_type="touched-b"} 157633 + cortex_bucket_store_series_data_touched_count{data_type="touched-b"} 3 + cortex_bucket_store_series_data_touched_sum{data_type="touched-c"} 180152 + cortex_bucket_store_series_data_touched_count{data_type="touched-c"} 3 + + # HELP cortex_bucket_store_series_get_all_duration_seconds Time it takes until all per-block prepares and preloads for a query are finished. + # TYPE cortex_bucket_store_series_get_all_duration_seconds histogram + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="0.001"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="0.01"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="0.1"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="0.3"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="0.6"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="1"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="3"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="6"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="9"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="20"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="30"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="60"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="90"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="120"} 0 + cortex_bucket_store_series_get_all_duration_seconds_bucket{le="+Inf"} 9 + cortex_bucket_store_series_get_all_duration_seconds_sum 1.486254e+06 + cortex_bucket_store_series_get_all_duration_seconds_count 9 + + # HELP cortex_bucket_store_series_merge_duration_seconds Time it takes to merge sub-results from all queried blocks into a single result. + # TYPE cortex_bucket_store_series_merge_duration_seconds histogram + cortex_bucket_store_series_merge_duration_seconds_bucket{le="0.001"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="0.01"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="0.1"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="0.3"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="0.6"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="1"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="3"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="6"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="9"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="20"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="30"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="60"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="90"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="120"} 0 + cortex_bucket_store_series_merge_duration_seconds_bucket{le="+Inf"} 9 + cortex_bucket_store_series_merge_duration_seconds_sum 1.688925e+06 + cortex_bucket_store_series_merge_duration_seconds_count 9 + + # HELP cortex_bucket_store_series_refetches_total Total number of cases where the built-in max series size was not enough to fetch series from index, resulting in refetch. + # TYPE cortex_bucket_store_series_refetches_total counter + cortex_bucket_store_series_refetches_total 743127 + + # HELP cortex_bucket_store_series_result_series Number of series observed in the final result of a query. + # TYPE cortex_bucket_store_series_result_series summary + cortex_bucket_store_series_result_series_sum 1.238545e+06 + cortex_bucket_store_series_result_series_count 6 + + # HELP cortex_bucket_store_cached_postings_compressions_total Number of postings compressions and decompressions when storing to index cache. + # TYPE cortex_bucket_store_cached_postings_compressions_total counter + cortex_bucket_store_cached_postings_compressions_total{op="encode"} 1125950 + cortex_bucket_store_cached_postings_compressions_total{op="decode"} 1148469 + + # HELP cortex_bucket_store_cached_postings_compression_errors_total Number of postings compression and decompression errors. + # TYPE cortex_bucket_store_cached_postings_compression_errors_total counter + cortex_bucket_store_cached_postings_compression_errors_total{op="encode"} 1170988 + cortex_bucket_store_cached_postings_compression_errors_total{op="decode"} 1193507 + + # HELP cortex_bucket_store_cached_postings_compression_time_seconds Time spent compressing and decompressing postings when storing to / reading from postings cache. + # TYPE cortex_bucket_store_cached_postings_compression_time_seconds counter + cortex_bucket_store_cached_postings_compression_time_seconds{op="encode"} 1216026 + cortex_bucket_store_cached_postings_compression_time_seconds{op="decode"} 1238545 + + # HELP cortex_bucket_store_cached_postings_original_size_bytes_total Original size of postings stored into cache. + # TYPE cortex_bucket_store_cached_postings_original_size_bytes_total counter + cortex_bucket_store_cached_postings_original_size_bytes_total 1261064 + + # HELP cortex_bucket_store_cached_postings_compressed_size_bytes_total Compressed size of postings stored into cache. + # TYPE cortex_bucket_store_cached_postings_compressed_size_bytes_total counter + cortex_bucket_store_cached_postings_compressed_size_bytes_total 1283583 `)) require.NoError(t, err) } diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go index 91fa34b06f..d540d08611 100644 --- a/pkg/storegateway/bucket_stores.go +++ b/pkg/storegateway/bucket_stores.go @@ -69,12 +69,12 @@ func NewBucketStores(cfg tsdb.Config, filters []block.MetadataFilter, bucketClie bucketStoreMetrics: NewBucketStoreMetrics(), metaFetcherMetrics: NewMetadataFetcherMetrics(), syncTimes: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ - Name: "bucket_stores_blocks_sync_seconds", + Name: "cortex_bucket_stores_blocks_sync_seconds", Help: "The total time it takes to perform a sync stores", Buckets: []float64{0.1, 1, 10, 30, 60, 120, 300, 600, 900}, }), syncLastSuccess: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "bucket_stores_blocks_last_successful_sync_timestamp_seconds", + Name: "cortex_bucket_stores_blocks_last_successful_sync_timestamp_seconds", Help: "Unix timestamp of the last successful blocks sync.", }), } diff --git a/pkg/storegateway/bucket_stores_test.go b/pkg/storegateway/bucket_stores_test.go index 660bc00f8c..44921d7c70 100644 --- a/pkg/storegateway/bucket_stores_test.go +++ b/pkg/storegateway/bucket_stores_test.go @@ -77,18 +77,18 @@ func TestBucketStores_InitialSync(t *testing.T) { assert.Empty(t, seriesSet) assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP bucket_store_blocks_loaded Number of currently loaded blocks. - # TYPE bucket_store_blocks_loaded gauge - bucket_store_blocks_loaded 2 + # HELP cortex_bucket_store_blocks_loaded Number of currently loaded blocks. + # TYPE cortex_bucket_store_blocks_loaded gauge + cortex_bucket_store_blocks_loaded 2 - # HELP bucket_store_block_loads_total Total number of remote block loading attempts. - # TYPE bucket_store_block_loads_total counter - bucket_store_block_loads_total 2 + # HELP cortex_bucket_store_block_loads_total Total number of remote block loading attempts. + # TYPE cortex_bucket_store_block_loads_total counter + cortex_bucket_store_block_loads_total 2 - # HELP bucket_store_block_load_failures_total Total number of failed remote block loading attempts. - # TYPE bucket_store_block_load_failures_total counter - bucket_store_block_load_failures_total 0 - `), "bucket_store_blocks_loaded", "bucket_store_block_loads_total", "bucket_store_block_load_failures_total")) + # HELP cortex_bucket_store_block_load_failures_total Total number of failed remote block loading attempts. + # TYPE cortex_bucket_store_block_load_failures_total counter + cortex_bucket_store_block_load_failures_total 0 + `), "cortex_bucket_store_blocks_loaded", "cortex_bucket_store_block_loads_total", "cortex_bucket_store_block_load_failures_total")) assert.Greater(t, testutil.ToFloat64(stores.syncLastSuccess), float64(0)) } @@ -134,18 +134,18 @@ func TestBucketStores_SyncBlocks(t *testing.T) { assert.Equal(t, []storepb.Label{{Name: labels.MetricName, Value: metricName}}, seriesSet[0].Labels) assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP bucket_store_blocks_loaded Number of currently loaded blocks. - # TYPE bucket_store_blocks_loaded gauge - bucket_store_blocks_loaded 2 - - # HELP bucket_store_block_loads_total Total number of remote block loading attempts. - # TYPE bucket_store_block_loads_total counter - bucket_store_block_loads_total 2 - - # HELP bucket_store_block_load_failures_total Total number of failed remote block loading attempts. - # TYPE bucket_store_block_load_failures_total counter - bucket_store_block_load_failures_total 0 - `), "bucket_store_blocks_loaded", "bucket_store_block_loads_total", "bucket_store_block_load_failures_total")) + # HELP cortex_bucket_store_blocks_loaded Number of currently loaded blocks. + # TYPE cortex_bucket_store_blocks_loaded gauge + cortex_bucket_store_blocks_loaded 2 + + # HELP cortex_bucket_store_block_loads_total Total number of remote block loading attempts. + # TYPE cortex_bucket_store_block_loads_total counter + cortex_bucket_store_block_loads_total 2 + + # HELP cortex_bucket_store_block_load_failures_total Total number of failed remote block loading attempts. + # TYPE cortex_bucket_store_block_load_failures_total counter + cortex_bucket_store_block_load_failures_total 0 + `), "cortex_bucket_store_blocks_loaded", "cortex_bucket_store_block_loads_total", "cortex_bucket_store_block_load_failures_total")) assert.Greater(t, testutil.ToFloat64(stores.syncLastSuccess), float64(0)) } diff --git a/pkg/storegateway/gateway.go b/pkg/storegateway/gateway.go index cd7563df40..7d7620a39e 100644 --- a/pkg/storegateway/gateway.go +++ b/pkg/storegateway/gateway.go @@ -11,6 +11,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/weaveworks/common/logging" @@ -141,12 +142,7 @@ func newStoreGateway(gatewayCfg Config, storageCfg cortex_tsdb.Config, bucketCli filters = append(filters, NewShardingMetadataFilter(g.ring, lifecyclerCfg.Addr, logger)) } - var storesReg prometheus.Registerer - if reg != nil { - storesReg = prometheus.WrapRegistererWith(prometheus.Labels{"component": "storegateway"}, reg) - } - - g.stores, err = NewBucketStores(storageCfg, filters, bucketClient, logLevel, logger, storesReg) + g.stores, err = NewBucketStores(storageCfg, filters, bucketClient, logLevel, logger, extprom.WrapRegistererWith(prometheus.Labels{"component": "storegateway"}, reg)) if err != nil { return nil, errors.Wrap(err, "create bucket stores") } diff --git a/pkg/storegateway/gateway_test.go b/pkg/storegateway/gateway_test.go index 777aab2452..f559ff6086 100644 --- a/pkg/storegateway/gateway_test.go +++ b/pkg/storegateway/gateway_test.go @@ -271,11 +271,11 @@ func TestStoreGateway_BlocksSharding(t *testing.T) { // Assert on the number of blocks loaded extracting this information from metrics. metrics := util.BuildMetricFamiliesPerUserFromUserRegistries(registries) - assert.Equal(t, float64(testData.expectedBlocksLoaded), metrics.GetSumOfGauges("bucket_store_blocks_loaded")) + assert.Equal(t, float64(testData.expectedBlocksLoaded), metrics.GetSumOfGauges("cortex_bucket_store_blocks_loaded")) // The total number of blocks synced (before filtering) is always equal to the total // number of blocks for each instance. - assert.Equal(t, float64(testData.numGateways*numBlocks), metrics.GetSumOfGauges("blocks_meta_synced")) + assert.Equal(t, float64(testData.numGateways*numBlocks), metrics.GetSumOfGauges("cortex_blocks_meta_synced")) }) } } diff --git a/pkg/storegateway/metadata_fetcher_metrics.go b/pkg/storegateway/metadata_fetcher_metrics.go index a8e8f6b40c..1d02d1e996 100644 --- a/pkg/storegateway/metadata_fetcher_metrics.go +++ b/pkg/storegateway/metadata_fetcher_metrics.go @@ -32,23 +32,23 @@ func NewMetadataFetcherMetrics() *MetadataFetcherMetrics { regs: map[string]*prometheus.Registry{}, syncs: prometheus.NewDesc( - "blocks_meta_syncs_total", + "cortex_blocks_meta_syncs_total", "Total blocks metadata synchronization attempts", nil, nil), syncFailures: prometheus.NewDesc( - "blocks_meta_sync_failures_total", + "cortex_blocks_meta_sync_failures_total", "Total blocks metadata synchronization failures", nil, nil), syncDuration: prometheus.NewDesc( - "blocks_meta_sync_duration_seconds", + "cortex_blocks_meta_sync_duration_seconds", "Duration of the blocks metadata synchronization in seconds", nil, nil), syncConsistencyDelay: prometheus.NewDesc( - "blocks_meta_sync_consistency_delay_seconds", + "cortex_blocks_meta_sync_consistency_delay_seconds", "Configured consistency delay in seconds.", nil, nil), synced: prometheus.NewDesc( - "blocks_meta_synced", + "cortex_blocks_meta_synced", "Reflects current state of synced blocks (over all tenants).", []string{"state"}, nil), } diff --git a/pkg/storegateway/metadata_fetcher_metrics_test.go b/pkg/storegateway/metadata_fetcher_metrics_test.go index 6f2e0aed54..ea028cb07c 100644 --- a/pkg/storegateway/metadata_fetcher_metrics_test.go +++ b/pkg/storegateway/metadata_fetcher_metrics_test.go @@ -22,34 +22,34 @@ func TestMetadataFetcherMetrics(t *testing.T) { //noinspection ALL err := testutil.GatherAndCompare(mainReg, bytes.NewBufferString(` - # HELP blocks_meta_sync_duration_seconds Duration of the blocks metadata synchronization in seconds - # TYPE blocks_meta_sync_duration_seconds histogram - blocks_meta_sync_duration_seconds_bucket{le="0.01"} 0 - blocks_meta_sync_duration_seconds_bucket{le="1"} 0 - blocks_meta_sync_duration_seconds_bucket{le="10"} 3 - blocks_meta_sync_duration_seconds_bucket{le="100"} 3 - blocks_meta_sync_duration_seconds_bucket{le="1000"} 3 - blocks_meta_sync_duration_seconds_bucket{le="+Inf"} 3 - blocks_meta_sync_duration_seconds_sum 9 - blocks_meta_sync_duration_seconds_count 3 - - # HELP blocks_meta_sync_failures_total Total blocks metadata synchronization failures - # TYPE blocks_meta_sync_failures_total counter - blocks_meta_sync_failures_total 30 - - # HELP blocks_meta_syncs_total Total blocks metadata synchronization attempts - # TYPE blocks_meta_syncs_total counter - blocks_meta_syncs_total 15 - - # HELP blocks_meta_sync_consistency_delay_seconds Configured consistency delay in seconds. - # TYPE blocks_meta_sync_consistency_delay_seconds gauge - blocks_meta_sync_consistency_delay_seconds 300 - - # HELP blocks_meta_synced Reflects current state of synced blocks (over all tenants). - # TYPE blocks_meta_synced gauge - blocks_meta_synced{state="corrupted-meta-json"} 75 - blocks_meta_synced{state="loaded"} 90 - blocks_meta_synced{state="too-fresh"} 105 + # HELP cortex_blocks_meta_sync_duration_seconds Duration of the blocks metadata synchronization in seconds + # TYPE cortex_blocks_meta_sync_duration_seconds histogram + cortex_blocks_meta_sync_duration_seconds_bucket{le="0.01"} 0 + cortex_blocks_meta_sync_duration_seconds_bucket{le="1"} 0 + cortex_blocks_meta_sync_duration_seconds_bucket{le="10"} 3 + cortex_blocks_meta_sync_duration_seconds_bucket{le="100"} 3 + cortex_blocks_meta_sync_duration_seconds_bucket{le="1000"} 3 + cortex_blocks_meta_sync_duration_seconds_bucket{le="+Inf"} 3 + cortex_blocks_meta_sync_duration_seconds_sum 9 + cortex_blocks_meta_sync_duration_seconds_count 3 + + # HELP cortex_blocks_meta_sync_failures_total Total blocks metadata synchronization failures + # TYPE cortex_blocks_meta_sync_failures_total counter + cortex_blocks_meta_sync_failures_total 30 + + # HELP cortex_blocks_meta_syncs_total Total blocks metadata synchronization attempts + # TYPE cortex_blocks_meta_syncs_total counter + cortex_blocks_meta_syncs_total 15 + + # HELP cortex_blocks_meta_sync_consistency_delay_seconds Configured consistency delay in seconds. + # TYPE cortex_blocks_meta_sync_consistency_delay_seconds gauge + cortex_blocks_meta_sync_consistency_delay_seconds 300 + + # HELP cortex_blocks_meta_synced Reflects current state of synced blocks (over all tenants). + # TYPE cortex_blocks_meta_synced gauge + cortex_blocks_meta_synced{state="corrupted-meta-json"} 75 + cortex_blocks_meta_synced{state="loaded"} 90 + cortex_blocks_meta_synced{state="too-fresh"} 105 `)) require.NoError(t, err) } From 63775ad6f04421f93995b4bf88c33c0d422e1356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 25 May 2020 17:56:59 +0200 Subject: [PATCH 6/9] Fix integration tests. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- integration/e2e/composite_service.go | 35 ++++++++++++ integration/e2e/service.go | 85 +++++++++++++++++++--------- integration/querier_test.go | 14 ++--- 3 files changed, 100 insertions(+), 34 deletions(-) diff --git a/integration/e2e/composite_service.go b/integration/e2e/composite_service.go index b3a061a4ed..08f26b7916 100644 --- a/integration/e2e/composite_service.go +++ b/integration/e2e/composite_service.go @@ -60,6 +60,25 @@ func (s *CompositeHTTPService) WaitSumMetrics(isExpected func(sums ...float64) b return fmt.Errorf("unable to find metrics %s with expected values. Last values: %v", metricNames, sums) } +func (s *CompositeHTTPService) WaitSumMetricWithLabels(isExpected func(sums float64) bool, metricName string, expectedLabels map[string]string) error { + lastSum := 0.0 + + for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { + lastSum, err := s.SumMetricWithLabels(metricName, expectedLabels) + if err != nil { + return err + } + + if isExpected(lastSum) { + return nil + } + + s.retryBackoff.Wait() + } + + return fmt.Errorf("unable to find metric %s with labels %v with expected value. Last value: %v", metricName, expectedLabels, lastSum) +} + // SumMetrics returns the sum of the values of each given metric names. func (s *CompositeHTTPService) SumMetrics(metricNames ...string) ([]float64, error) { sums := make([]float64, len(metricNames)) @@ -81,3 +100,19 @@ func (s *CompositeHTTPService) SumMetrics(metricNames ...string) ([]float64, err return sums, nil } + +// SumMetrics returns the sum of the values of each given metric names. +func (s *CompositeHTTPService) SumMetricWithLabels(metricName string, expectedLabels map[string]string) (float64, error) { + sum := 0.0 + + for _, service := range s.services { + s, err := service.SumMetricWithLabels(metricName, expectedLabels) + if err != nil { + return 0, err + } + + sum += s + } + + return sum, nil +} diff --git a/integration/e2e/service.go b/integration/e2e/service.go index e42fea0f6b..c5184de4c8 100644 --- a/integration/e2e/service.go +++ b/integration/e2e/service.go @@ -14,6 +14,7 @@ import ( "github.com/go-kit/kit/log" "github.com/pkg/errors" + dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/thanos-io/thanos/pkg/runutil" @@ -575,44 +576,74 @@ func (s *HTTPService) SumMetrics(metricNames ...string) ([]float64, error) { // wait continues. If no such matching metric can be found or wait times out, function returns error. func (s *HTTPService) WaitForMetricWithLabels(okFn func(v float64) bool, metricName string, expectedLabels map[string]string) error { for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { - metrics, err := s.Metrics() + ms, err := s.getMetricsMatchingLabels(metricName, expectedLabels) if err != nil { return err } - var tp expfmt.TextParser - families, err := tp.TextToMetricFamilies(strings.NewReader(metrics)) - if err != nil { - return err + for _, m := range ms { + if okFn(getValue(m)) { + return nil + } } - mf, ok := families[metricName] - if !ok { - return errors.Errorf("metric %s not found in %s metric page", metricName, s.name) - } + s.retryBackoff.Wait() + } - for _, m := range mf.GetMetric() { - // check if some metric has all required labels - metricLabels := map[string]string{} - for _, lp := range m.GetLabel() { - metricLabels[lp.GetName()] = lp.GetValue() - } + return fmt.Errorf("unable to find metric %s with labels %v with expected value", metricName, expectedLabels) +} - matches := true - for k, v := range expectedLabels { - if mv, ok := metricLabels[k]; !ok || mv != v { - matches = false - break - } - } +// Returns sum of all metrics matching given labels. +func (s *HTTPService) SumMetricWithLabels(metricName string, expectedLabels map[string]string) (float64, error) { + sum := 0.0 + ms, err := s.getMetricsMatchingLabels(metricName, expectedLabels) + if err != nil { + return 0, err + } - if matches && okFn(getValue(m)) { - return nil + for _, m := range ms { + sum += getValue(m) + } + return sum, nil +} + +func (s *HTTPService) getMetricsMatchingLabels(metricName string, expectedLabels map[string]string) ([]*dto.Metric, error) { + metrics, err := s.Metrics() + if err != nil { + return nil, err + } + + var tp expfmt.TextParser + families, err := tp.TextToMetricFamilies(strings.NewReader(metrics)) + if err != nil { + return nil, err + } + + mf, ok := families[metricName] + if !ok { + return nil, errors.Errorf("metric %s not found in %s metric page", metricName, s.name) + } + + result := []*dto.Metric(nil) + + for _, m := range mf.GetMetric() { + // check if some metric has all required labels + metricLabels := map[string]string{} + for _, lp := range m.GetLabel() { + metricLabels[lp.GetName()] = lp.GetValue() + } + + matches := true + for k, v := range expectedLabels { + if mv, ok := metricLabels[k]; !ok || mv != v { + matches = false + break } } - s.retryBackoff.Wait() + if matches { + result = append(result, m) + } } - - return fmt.Errorf("unable to find metric %s with labels %v with expected value", metricName, expectedLabels) + return result, nil } diff --git a/integration/querier_test.go b/integration/querier_test.go index 59f2b0773d..d7660573a2 100644 --- a/integration/querier_test.go +++ b/integration/querier_test.go @@ -128,7 +128,7 @@ func TestQuerierWithBlocksStorageWithoutStoreGateway(t *testing.T) { require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(2), "cortex_ingester_memory_series_removed_total")) // Wait until the querier has synched the new uploaded blocks. - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "bucket_store_blocks_loaded")) + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "cortex_bucket_store_blocks_loaded")) // Query back the series (1 only in the storage, 1 only in the ingesters, 1 on both). result, err := c.Query("series_1", series1Timestamp) @@ -307,15 +307,15 @@ func TestQuerierWithBlocksStorageAndStoreGatewayRunningInMicroservicesMode(t *te require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(2), "cortex_ingester_memory_series_removed_total")) // Wait until the querier has discovered the uploaded blocks. - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "blocks_meta_synced")) + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "cortex_blocks_meta_synced")) // Wait until the store-gateway has synched the new uploaded blocks. When sharding is enabled // we don't known which store-gateway instance will synch the blocks, so we need to wait on // metrics extracted from all instances. if testCfg.blocksShardingEnabled { - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2), "bucket_store_blocks_loaded")) + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2), "cortex_bucket_store_blocks_loaded")) } else { - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(float64(2*storeGateways.NumInstances())), "bucket_store_blocks_loaded")) + require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(float64(2*storeGateways.NumInstances())), "cortex_bucket_store_blocks_loaded")) } // Query back the series (1 only in the storage, 1 only in the ingesters, 1 on both). @@ -499,15 +499,15 @@ func TestQuerierWithBlocksStorageAndStoreGatewayRunningInSingleBinaryMode(t *tes require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*cluster.NumInstances())), "cortex_ingester_memory_series_removed_total")) // Wait until the querier has discovered the uploaded blocks (discovered both by the querier and store-gateway). - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*cluster.NumInstances()*2)), "cortex_querier_blocks_meta_synced")) + require.NoError(t, cluster.WaitSumMetricWithLabels(e2e.EqualsSingle(float64(2*cluster.NumInstances()*2)), "cortex_blocks_meta_synced", map[string]string{"component": "querier"})) // Wait until the store-gateway has synched the new uploaded blocks. const shippedBlocks = 2 if testCfg.blocksShardingEnabled { - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(shippedBlocks*seriesReplicationFactor)), "bucket_store_blocks_loaded")) + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(shippedBlocks*seriesReplicationFactor)), "cortex_bucket_store_blocks_loaded")) } else { - require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(shippedBlocks*seriesReplicationFactor*cluster.NumInstances())), "bucket_store_blocks_loaded")) + require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(shippedBlocks*seriesReplicationFactor*cluster.NumInstances())), "cortex_bucket_store_blocks_loaded")) } // Query back the series (1 only in the storage, 1 only in the ingesters, 1 on both). From db45ff7d4742ef4941b0e29ab989b526aa108465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 25 May 2020 17:58:11 +0200 Subject: [PATCH 7/9] Fix comment. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- integration/e2e/composite_service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/e2e/composite_service.go b/integration/e2e/composite_service.go index 08f26b7916..d1a5e9b082 100644 --- a/integration/e2e/composite_service.go +++ b/integration/e2e/composite_service.go @@ -101,7 +101,7 @@ func (s *CompositeHTTPService) SumMetrics(metricNames ...string) ([]float64, err return sums, nil } -// SumMetrics returns the sum of the values of each given metric names. +// SumMetricWithLabels returns the sum of the values of metric with matching labels across all services. func (s *CompositeHTTPService) SumMetricWithLabels(metricName string, expectedLabels map[string]string) (float64, error) { sum := 0.0 From 7e9b35df06cef408cf18507716072a93b48f8365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Wed, 27 May 2020 21:38:06 +0200 Subject: [PATCH 8/9] Fix component name. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/storegateway/gateway.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/storegateway/gateway.go b/pkg/storegateway/gateway.go index 7d7620a39e..02ec9be63d 100644 --- a/pkg/storegateway/gateway.go +++ b/pkg/storegateway/gateway.go @@ -142,7 +142,7 @@ func newStoreGateway(gatewayCfg Config, storageCfg cortex_tsdb.Config, bucketCli filters = append(filters, NewShardingMetadataFilter(g.ring, lifecyclerCfg.Addr, logger)) } - g.stores, err = NewBucketStores(storageCfg, filters, bucketClient, logLevel, logger, extprom.WrapRegistererWith(prometheus.Labels{"component": "storegateway"}, reg)) + g.stores, err = NewBucketStores(storageCfg, filters, bucketClient, logLevel, logger, extprom.WrapRegistererWith(prometheus.Labels{"component": "store-gateway"}, reg)) if err != nil { return nil, errors.Wrap(err, "create bucket stores") } From 27c874c804d9d23b71061849cca5dc908ec4fea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 1 Jun 2020 15:56:08 +0200 Subject: [PATCH 9/9] Added PR number to CHANGELOG.md entries. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4710d0fb2..88c358de3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * [CHANGE] Query Frontend now uses Round Robin to choose a tenant queue to service next. #2553 * [CHANGE] `-promql.lookback-delta` is now deprecated and has been replaced by `-querier.lookback-delta` along with `lookback_delta` entry under `querier` in the config file. `-promql.lookback-delta` will be removed in v1.4.0. #2604 * [CHANGE] Experimental TSDB: removed `-experimental.tsdb.bucket-store.binary-index-header-enabled` flag. Now the binary index-header is always enabled. -* [CHANGE] Experimental TSDB: Renamed index-cache metrics to use original metric names from Thanos, as Cortex is not aggregating them in any way: +* [CHANGE] Experimental TSDB: Renamed index-cache metrics to use original metric names from Thanos, as Cortex is not aggregating them in any way: #2627 * `cortex__blocks_index_cache_items_evicted_total` => `thanos_store_index_cache_items_evicted_total{name="index-cache"}` * `cortex__blocks_index_cache_items_added_total` => `thanos_store_index_cache_items_added_total{name="index-cache"}` * `cortex__blocks_index_cache_requests_total` => `thanos_store_index_cache_requests_total{name="index-cache"}` @@ -18,7 +18,7 @@ * `cortex__blocks_index_cache_memcached_operation_failures_total` => `thanos_memcached_operation_failures_total{name="index-cache"}` * `cortex__blocks_index_cache_memcached_operation_duration_seconds` => `thanos_memcached_operation_duration_seconds{name="index-cache"}` * `cortex__blocks_index_cache_memcached_operation_skipped_total` => `thanos_memcached_operation_skipped_total{name="index-cache"}` -* [CHANGE] Experimental TSDB: Renamed metrics in bucket stores: +* [CHANGE] Experimental TSDB: Renamed metrics in bucket stores: #2627 * `cortex__blocks_meta_syncs_total` => `cortex_blocks_meta_syncs_total{component=""}` * `cortex__blocks_meta_sync_failures_total` => `cortex_blocks_meta_sync_failures_total{component=""}` * `cortex__blocks_meta_sync_duration_seconds` => `cortex_blocks_meta_sync_duration_seconds{component=""}`