From 0e5b9e45beb6ee8d902ba3f2ba83808522c698f4 Mon Sep 17 00:00:00 2001
From: Daniel Mai <daniel@dgraph.io>
Date: Wed, 18 Mar 2020 21:08:38 -0700
Subject: [PATCH] Fix point-in-time Prometheus metrics. (#4948)

Fixes #4532.

The metrics for pending queries and active mutations would
report "1" or "-1" because the metrics view was set to LastValue.
This change fixes this by changing the view to a Sum so
measurements of "1" and "-1" would accumulate the metrics as
expected.

There were also metrics whose values were separated by tags when
they should be treated as a single metric. This made metrics that
were incremented and decremented show up as seperate metrics,
e.g.:

dgraph_pending_queries_total{method="Server.Query",status=""} 100
dgraph_pending_queries_total{method="Server.Query",status="ok"} -100

For these metrics, the tags are excluded from the view so the
metrics are shown correctly:

dgraph_pending_queries_total 0

(cherry picked from commit 6538fcebceb27f25377990fb9486ef9b10597e41)
---
 x/metrics.go | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/x/metrics.go b/x/metrics.go
index 07ba01e5eb9..f66d11ace1a 100644
--- a/x/metrics.go
+++ b/x/metrics.go
@@ -105,6 +105,8 @@ var (
 		20, 25, 30, 40, 50, 65, 80, 100, 130, 160, 200, 250, 300, 400, 500,
 		650, 800, 1000, 2000, 5000, 10000, 20000, 50000, 100000)
 
+	// Use this tag for the metric view if it needs status or method granularity.
+	// Metrics would be viewed separately for different tag values.
 	allTagKeys = []tag.Key{
 		KeyStatus, KeyMethod,
 	}
@@ -151,15 +153,15 @@ var (
 			Name:        PendingQueries.Name(),
 			Measure:     PendingQueries,
 			Description: PendingQueries.Description(),
-			Aggregation: view.LastValue(),
-			TagKeys:     allTagKeys,
+			Aggregation: view.Sum(),
+			TagKeys:     nil,
 		},
 		{
 			Name:        PendingProposals.Name(),
 			Measure:     PendingProposals,
 			Description: PendingProposals.Description(),
 			Aggregation: view.LastValue(),
-			TagKeys:     allTagKeys,
+			TagKeys:     nil,
 		},
 		{
 			Name:        MemoryInUse.Name(),
@@ -186,15 +188,15 @@ var (
 			Name:        ActiveMutations.Name(),
 			Measure:     ActiveMutations,
 			Description: ActiveMutations.Description(),
-			Aggregation: view.LastValue(),
-			TagKeys:     allTagKeys,
+			Aggregation: view.Sum(),
+			TagKeys:     nil,
 		},
 		{
 			Name:        AlphaHealth.Name(),
 			Measure:     AlphaHealth,
 			Description: AlphaHealth.Description(),
 			Aggregation: view.LastValue(),
-			TagKeys:     allTagKeys,
+			TagKeys:     nil,
 		},
 	}
 )