From 8fc1576073a222e885c0ad15f805c36ef5990ca4 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Thu, 2 Apr 2020 17:34:46 +0300 Subject: [PATCH] Aggmetrics: track how long a GC() run takes and how many series deleted --- mdata/aggmetrics.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mdata/aggmetrics.go b/mdata/aggmetrics.go index bd0fde423e..0c6f2fda8f 100644 --- a/mdata/aggmetrics.go +++ b/mdata/aggmetrics.go @@ -49,11 +49,14 @@ func NewAggMetrics(store Store, cachePusher cache.CachePusher, dropFirstChunk bo // periodically scan chunks and close any that have not received data in a while func (ms *AggMetrics) GC() { for { + var purged int + unix := time.Duration(time.Now().UnixNano()) diff := ms.gcInterval - (unix % ms.gcInterval) time.Sleep(diff + time.Minute) - log.Info("checking for stale chunks that need persisting.") - now := uint32(time.Now().Unix()) + log.Info("Aggmetrics: checking for stale chunks that need persisting.") + nowTime := time.Now() + now := uint32(nowTime.Unix()) chunkMinTs := now - uint32(ms.chunkMaxStale) metricMinTs := now - uint32(ms.metricMaxStale) @@ -84,6 +87,7 @@ func (ms *AggMetrics) GC() { if stale { log.Debugf("metric %s is stale. Purging data from memory.", key) ms.Lock() + purged++ delete(ms.Metrics[org], key) orgActiveMetrics.Set(float64(len(ms.Metrics[org]))) // note: this is racey. if a metric has just become unstale, it may have created a new chunk, @@ -118,6 +122,8 @@ func (ms *AggMetrics) GC() { } ms.RUnlock() metricsActive.Set(totalActive) + + log.Infof("Aggmetrics: finished GC %s. number of purged (deleted) metrics from tank: %d", time.Since(nowTime), purged) } }