diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index d4d05fef9a..ad84c57be4 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5553,22 +5553,22 @@ func TestExpendedPostingsCache(t *testing.T) { if c.expectedHeadPostingCall > 0 || c.expectedBlockPostingCall > 0 { metric := ` - # HELP cortex_ingester_expanded_postings_cache_requests Total number of requests to the cache. - # TYPE cortex_ingester_expanded_postings_cache_requests counter + # HELP cortex_ingester_expanded_postings_cache_requests_total Total number of requests to the cache. + # TYPE cortex_ingester_expanded_postings_cache_requests_total counter ` if c.expectedBlockPostingCall > 0 { metric += ` - cortex_ingester_expanded_postings_cache_requests{cache="block"} 4 + cortex_ingester_expanded_postings_cache_requests_total{cache="block"} 4 ` } if c.expectedHeadPostingCall > 0 { metric += ` - cortex_ingester_expanded_postings_cache_requests{cache="head"} 4 + cortex_ingester_expanded_postings_cache_requests_total{cache="head"} 4 ` } - err = testutil.GatherAndCompare(r, bytes.NewBufferString(metric), "cortex_ingester_expanded_postings_cache_requests") + err = testutil.GatherAndCompare(r, bytes.NewBufferString(metric), "cortex_ingester_expanded_postings_cache_requests_total") require.NoError(t, err) } @@ -5583,22 +5583,22 @@ func TestExpendedPostingsCache(t *testing.T) { if c.expectedHeadPostingCall > 0 || c.expectedBlockPostingCall > 0 { metric := ` - # HELP cortex_ingester_expanded_postings_cache_hits Total number of hit requests to the cache. - # TYPE cortex_ingester_expanded_postings_cache_hits counter + # HELP cortex_ingester_expanded_postings_cache_hits_total Total number of hit requests to the cache. + # TYPE cortex_ingester_expanded_postings_cache_hits_total counter ` if c.expectedBlockPostingCall > 0 { metric += ` - cortex_ingester_expanded_postings_cache_hits{cache="block"} 4 + cortex_ingester_expanded_postings_cache_hits_total{cache="block"} 4 ` } if c.expectedHeadPostingCall > 0 { metric += ` - cortex_ingester_expanded_postings_cache_hits{cache="head"} 4 + cortex_ingester_expanded_postings_cache_hits_total{cache="head"} 4 ` } - err = testutil.GatherAndCompare(r, bytes.NewBufferString(metric), "cortex_ingester_expanded_postings_cache_hits") + err = testutil.GatherAndCompare(r, bytes.NewBufferString(metric), "cortex_ingester_expanded_postings_cache_hits_total") require.NoError(t, err) } @@ -5644,10 +5644,10 @@ func TestExpendedPostingsCache(t *testing.T) { require.Equal(t, postingsForMatchersCalls.Load(), int64(c.expectedBlockPostingCall)) if c.cacheConfig.Head.Enabled { err = testutil.GatherAndCompare(r, bytes.NewBufferString(` - # HELP cortex_ingester_expanded_postings_non_cacheable_queries Total number of non cacheable queries. - # TYPE cortex_ingester_expanded_postings_non_cacheable_queries counter - cortex_ingester_expanded_postings_non_cacheable_queries{cache="head"} 1 -`), "cortex_ingester_expanded_postings_non_cacheable_queries") + # HELP cortex_ingester_expanded_postings_non_cacheable_queries_total Total number of non cacheable queries. + # TYPE cortex_ingester_expanded_postings_non_cacheable_queries_total counter + cortex_ingester_expanded_postings_non_cacheable_queries_total{cache="head"} 1 +`), "cortex_ingester_expanded_postings_non_cacheable_queries_total") require.NoError(t, err) } diff --git a/pkg/storage/tsdb/expanded_postings_cache.go b/pkg/storage/tsdb/expanded_postings_cache.go index 57027cf2a4..3ea8da709e 100644 --- a/pkg/storage/tsdb/expanded_postings_cache.go +++ b/pkg/storage/tsdb/expanded_postings_cache.go @@ -40,25 +40,30 @@ type ExpandedPostingsCacheMetrics struct { CacheRequests *prometheus.CounterVec CacheHits *prometheus.CounterVec CacheEvicts *prometheus.CounterVec + CacheMiss *prometheus.CounterVec NonCacheableQueries *prometheus.CounterVec } func NewPostingCacheMetrics(r prometheus.Registerer) *ExpandedPostingsCacheMetrics { return &ExpandedPostingsCacheMetrics{ CacheRequests: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_ingester_expanded_postings_cache_requests", + Name: "cortex_ingester_expanded_postings_cache_requests_total", Help: "Total number of requests to the cache.", }, []string{"cache"}), CacheHits: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_ingester_expanded_postings_cache_hits", + Name: "cortex_ingester_expanded_postings_cache_hits_total", Help: "Total number of hit requests to the cache.", }, []string{"cache"}), + CacheMiss: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_ingester_expanded_postings_cache_miss_total", + Help: "Total number of miss requests to the cache.", + }, []string{"cache", "reason"}), CacheEvicts: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_ingester_expanded_postings_cache_evicts", + Name: "cortex_ingester_expanded_postings_cache_evicts_total", Help: "Total number of evictions in the cache, excluding items that got evicted due to TTL.", }, []string{"cache", "reason"}), NonCacheableQueries: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_ingester_expanded_postings_non_cacheable_queries", + Name: "cortex_ingester_expanded_postings_non_cacheable_queries_total", Help: "Total number of non cacheable queries.", }, []string{"cache"}), } @@ -374,6 +379,7 @@ func (c *fifoCache[V]) getPromiseForKey(k string, fetch func() (V, int64, error) loaded, ok := c.cachedValues.LoadOrStore(k, r) if !ok { + c.metrics.CacheMiss.WithLabelValues(c.name, "miss").Inc() r.v, r.sizeBytes, r.err = fetch() r.sizeBytes += int64(len(k)) r.ts = c.timeNow() @@ -387,7 +393,7 @@ func (c *fifoCache[V]) getPromiseForKey(k string, fetch func() (V, int64, error) // If is cached but is expired, lets try to replace the cache value. if loaded.(*cacheEntryPromise[V]).isExpired(c.cfg.Ttl, c.timeNow()) && c.cachedValues.CompareAndSwap(k, loaded, r) { - c.metrics.CacheEvicts.WithLabelValues(c.name, "expired").Inc() + c.metrics.CacheMiss.WithLabelValues(c.name, "expired").Inc() r.v, r.sizeBytes, r.err = fetch() r.sizeBytes += int64(len(k)) c.updateSize(loaded.(*cacheEntryPromise[V]).sizeBytes, r.sizeBytes) diff --git a/pkg/storage/tsdb/expanded_postings_cache_test.go b/pkg/storage/tsdb/expanded_postings_cache_test.go index 3517ec6e9c..19272c00e7 100644 --- a/pkg/storage/tsdb/expanded_postings_cache_test.go +++ b/pkg/storage/tsdb/expanded_postings_cache_test.go @@ -154,10 +154,10 @@ func TestFifoCacheExpire(t *testing.T) { if c.expectedFinalItems != numberOfKeys { err := testutil.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` - # HELP cortex_ingester_expanded_postings_cache_evicts Total number of evictions in the cache, excluding items that got evicted due to TTL. - # TYPE cortex_ingester_expanded_postings_cache_evicts counter - cortex_ingester_expanded_postings_cache_evicts{cache="test",reason="full"} %v -`, numberOfKeys-c.expectedFinalItems)), "cortex_ingester_expanded_postings_cache_evicts") + # HELP cortex_ingester_expanded_postings_cache_evicts_total Total number of evictions in the cache, excluding items that got evicted due to TTL. + # TYPE cortex_ingester_expanded_postings_cache_evicts_total counter + cortex_ingester_expanded_postings_cache_evicts_total{cache="test",reason="full"} %v +`, numberOfKeys-c.expectedFinalItems)), "cortex_ingester_expanded_postings_cache_evicts_total") require.NoError(t, err) } @@ -181,10 +181,11 @@ func TestFifoCacheExpire(t *testing.T) { } err := testutil.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` - # HELP cortex_ingester_expanded_postings_cache_evicts Total number of evictions in the cache, excluding items that got evicted due to TTL. - # TYPE cortex_ingester_expanded_postings_cache_evicts counter - cortex_ingester_expanded_postings_cache_evicts{cache="test",reason="expired"} %v -`, numberOfKeys)), "cortex_ingester_expanded_postings_cache_evicts") + # HELP cortex_ingester_expanded_postings_cache_miss_total Total number of miss requests to the cache. + # TYPE cortex_ingester_expanded_postings_cache_miss_total counter + cortex_ingester_expanded_postings_cache_miss_total{cache="test",reason="expired"} %v + cortex_ingester_expanded_postings_cache_miss_total{cache="test",reason="miss"} %v +`, numberOfKeys, numberOfKeys)), "cortex_ingester_expanded_postings_cache_miss_total") require.NoError(t, err) cache.timeNow = func() time.Time { @@ -195,12 +196,12 @@ func TestFifoCacheExpire(t *testing.T) { return 2, 18, nil }) - // Should expire all keys again as ttl is expired + // Should expire all keys expired keys err = testutil.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` - # HELP cortex_ingester_expanded_postings_cache_evicts Total number of evictions in the cache, excluding items that got evicted due to TTL. - # TYPE cortex_ingester_expanded_postings_cache_evicts counter - cortex_ingester_expanded_postings_cache_evicts{cache="test",reason="expired"} %v -`, numberOfKeys*2)), "cortex_ingester_expanded_postings_cache_evicts") + # HELP cortex_ingester_expanded_postings_cache_evicts_total Total number of evictions in the cache, excluding items that got evicted due to TTL. + # TYPE cortex_ingester_expanded_postings_cache_evicts_total counter + cortex_ingester_expanded_postings_cache_evicts_total{cache="test",reason="expired"} %v +`, numberOfKeys)), "cortex_ingester_expanded_postings_cache_evicts_total") require.NoError(t, err) } })