Don't prefix Thanos index-cache metrics (cortexproject/cortex#2627)

* Removed Cortex-specific metrics for index cache, reuse Thanos metrics (with name="index-cache", and appropriate component) Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com> * Don't prefix metrics, but use new label instead. Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com> * Fix tests. Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com> * Fixed tests after renaming metrics. Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com> * Add cortex_ prefix to metrics (re-)defined in Cortex Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com> * Fix integration tests. Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com> * Fix comment. Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com> * Fix component name. Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com> * Added PR number to CHANGELOG.md entries. Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>
grafana · Jun 1, 2020 · 90af158 · 90af158
1 parent e7400e1
commit 90af158
Show file tree

Hide file tree

Showing 2 changed files with 93 additions and 27 deletions.
diff --git a/composite_service.go b/composite_service.go
@@ -60,6 +60,25 @@ func (s *CompositeHTTPService) WaitSumMetrics(isExpected func(sums ...float64) b
 	return fmt.Errorf("unable to find metrics %s with expected values. Last values: %v", metricNames, sums)
 }
 
+func (s *CompositeHTTPService) WaitSumMetricWithLabels(isExpected func(sums float64) bool, metricName string, expectedLabels map[string]string) error {
+	lastSum := 0.0
+
+	for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); {
+		lastSum, err := s.SumMetricWithLabels(metricName, expectedLabels)
+		if err != nil {
+			return err
+		}
+
+		if isExpected(lastSum) {
+			return nil
+		}
+
+		s.retryBackoff.Wait()
+	}
+
+	return fmt.Errorf("unable to find metric %s with labels %v with expected value. Last value: %v", metricName, expectedLabels, lastSum)
+}
+
 // SumMetrics returns the sum of the values of each given metric names.
 func (s *CompositeHTTPService) SumMetrics(metricNames ...string) ([]float64, error) {
 	sums := make([]float64, len(metricNames))
@@ -81,3 +100,19 @@ func (s *CompositeHTTPService) SumMetrics(metricNames ...string) ([]float64, err
 
 	return sums, nil
 }
+
+// SumMetricWithLabels returns the sum of the values of metric with matching labels across all services.
+func (s *CompositeHTTPService) SumMetricWithLabels(metricName string, expectedLabels map[string]string) (float64, error) {
+	sum := 0.0
+
+	for _, service := range s.services {
+		s, err := service.SumMetricWithLabels(metricName, expectedLabels)
+		if err != nil {
+			return 0, err
+		}
+
+		sum += s
+	}
+
+	return sum, nil
+}
diff --git a/service.go b/service.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/pkg/errors"
+	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
 	"github.com/thanos-io/thanos/pkg/runutil"
 
@@ -575,44 +576,74 @@ func (s *HTTPService) SumMetrics(metricNames ...string) ([]float64, error) {
 // wait continues. If no such matching metric can be found or wait times out, function returns error.
 func (s *HTTPService) WaitForMetricWithLabels(okFn func(v float64) bool, metricName string, expectedLabels map[string]string) error {
 	for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); {
-		metrics, err := s.Metrics()
+		ms, err := s.getMetricsMatchingLabels(metricName, expectedLabels)
 		if err != nil {
 			return err
 		}
 
-		var tp expfmt.TextParser
-		families, err := tp.TextToMetricFamilies(strings.NewReader(metrics))
-		if err != nil {
-			return err
+		for _, m := range ms {
+			if okFn(getValue(m)) {
+				return nil
+			}
 		}
 
-		mf, ok := families[metricName]
-		if !ok {
-			return errors.Errorf("metric %s not found in %s metric page", metricName, s.name)
-		}
+		s.retryBackoff.Wait()
+	}
 
-		for _, m := range mf.GetMetric() {
-			// check if some metric has all required labels
-			metricLabels := map[string]string{}
-			for _, lp := range m.GetLabel() {
-				metricLabels[lp.GetName()] = lp.GetValue()
-			}
+	return fmt.Errorf("unable to find metric %s with labels %v with expected value", metricName, expectedLabels)
+}
 
-			matches := true
-			for k, v := range expectedLabels {
-				if mv, ok := metricLabels[k]; !ok || mv != v {
-					matches = false
-					break
-				}
-			}
+// Returns sum of all metrics matching given labels.
+func (s *HTTPService) SumMetricWithLabels(metricName string, expectedLabels map[string]string) (float64, error) {
+	sum := 0.0
+	ms, err := s.getMetricsMatchingLabels(metricName, expectedLabels)
+	if err != nil {
+		return 0, err
+	}
 
-			if matches && okFn(getValue(m)) {
-				return nil
+	for _, m := range ms {
+		sum += getValue(m)
+	}
+	return sum, nil
+}
+
+func (s *HTTPService) getMetricsMatchingLabels(metricName string, expectedLabels map[string]string) ([]*dto.Metric, error) {
+	metrics, err := s.Metrics()
+	if err != nil {
+		return nil, err
+	}
+
+	var tp expfmt.TextParser
+	families, err := tp.TextToMetricFamilies(strings.NewReader(metrics))
+	if err != nil {
+		return nil, err
+	}
+
+	mf, ok := families[metricName]
+	if !ok {
+		return nil, errors.Errorf("metric %s not found in %s metric page", metricName, s.name)
+	}
+
+	result := []*dto.Metric(nil)
+
+	for _, m := range mf.GetMetric() {
+		// check if some metric has all required labels
+		metricLabels := map[string]string{}
+		for _, lp := range m.GetLabel() {
+			metricLabels[lp.GetName()] = lp.GetValue()
+		}
+
+		matches := true
+		for k, v := range expectedLabels {
+			if mv, ok := metricLabels[k]; !ok || mv != v {
+				matches = false
+				break
 			}
 		}
 
-		s.retryBackoff.Wait()
+		if matches {
+			result = append(result, m)
+		}
 	}
-
-	return fmt.Errorf("unable to find metric %s with labels %v with expected value", metricName, expectedLabels)
+	return result, nil
 }