Start on hedged metrics

grafana · Jun 28, 2021 · 1d6d7ed · 1d6d7ed
1 parent 8e32ee8
commit 1d6d7ed
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 20 deletions.
diff --git a/tempodb/backend/azure/azure_helpers.go b/tempodb/backend/azure/azure_helpers.go
@@ -38,11 +38,12 @@ func GetContainerURL(ctx context.Context, cfg *Config, hedge bool) (blob.Contain
 
 	// add instrumentation
 	transport := instrumentation.NewAzureTransport(customTransport)
-	var _ *hedgedhttp.Stats
+	var stats *hedgedhttp.Stats
 
 	// hedge if desired (0 means disabled)
 	if hedge && cfg.HedgeRequestsAt != 0 {
-		transport, _ = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport)
+		transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport)
+		instrumentation.PublishHedgedMetrics(stats)
 	}
 
 	client := http.Client{Transport: transport}

diff --git a/tempodb/backend/gcs/gcs.go b/tempodb/backend/gcs/gcs.go
@@ -322,11 +322,12 @@ func createBucket(ctx context.Context, cfg *Config, hedge bool) (*storage.Bucket
 
 	// add instrumentation
 	transport = instrumentation.NewGCSTransport(transport)
-	var _ *hedgedhttp.Stats
+	var stats *hedgedhttp.Stats
 
 	// hedge if desired (0 means disabled)
 	if hedge && cfg.HedgeRequestsAt != 0 {
-		transport, _ = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport)
+		transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport)
+		instrumentation.PublishHedgedMetrics(stats)
 	}
 
 	// build client

diff --git a/tempodb/backend/instrumentation/backend_transports.go b/tempodb/backend/instrumentation/backend_transports.go
@@ -31,44 +31,49 @@ var (
 		Buckets:   prometheus.ExponentialBuckets(0.005, 4, 6),
 	}, []string{"operation", "status_code"})
 
-	//hedgedRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
-	//	Namespace: "tempodb",
-	//	Name:      "request_duration_seconds",
-	//	Help:      "Time spent doing backend requests.",
-	//	Buckets:   prometheus.ExponentialBuckets(0.005, 4, 6),
-	//}, []string{"operation", "status_code", "storage_type"})
+	requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "tempodb",
+		Name:      "backend_request_duration_seconds",
+		Help:      "Time spent doing requests.",
+		Buckets:   prometheus.ExponentialBuckets(0.005, 4, 6),
+	}, []string{"operation", "status_code"})
 )
 
 type instrumentedTransport struct {
-	observer prometheus.ObserverVec
-	next     http.RoundTripper
+	legacyObserver prometheus.ObserverVec
+	observer       prometheus.ObserverVec
+	next           http.RoundTripper
 }
 
 func NewGCSTransport(next http.RoundTripper) http.RoundTripper {
 	return instrumentedTransport{
-		next:     next,
-		observer: gcsRequestDuration,
+		next:           next,
+		observer:       requestDuration,
+		legacyObserver: gcsRequestDuration,
 	}
 }
 
 func NewS3Transport(next http.RoundTripper) http.RoundTripper {
 	return instrumentedTransport{
-		next:     next,
-		observer: s3RequestDuration,
+		next:           next,
+		observer:       requestDuration,
+		legacyObserver: s3RequestDuration,
 	}
 }
 
 func NewAzureTransport(next http.RoundTripper) http.RoundTripper {
 	return instrumentedTransport{
-		next:     next,
-		observer: azureRequestDuration,
+		next:           next,
+		observer:       requestDuration,
+		legacyObserver: azureRequestDuration,
 	}
 }
 
 func (i instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	start := time.Now()
 	resp, err := i.next.RoundTrip(req)
 	if err == nil {
+		i.legacyObserver.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds())
 		i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds())
 	}
 	return resp, err

diff --git a/tempodb/backend/instrumentation/hedged_requests.go b/tempodb/backend/instrumentation/hedged_requests.go
@@ -0,0 +1,42 @@
+package instrumentation
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cristalhq/hedgedhttp"
+)
+
+var (
+	//hedgedRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	//	Namespace: "tempodb",
+	//	Name:      "request_duration_seconds",
+	//	Help:      "Time spent doing backend requests.",
+	//	Buckets:   prometheus.ExponentialBuckets(0.005, 4, 6),
+	//}, []string{"operation", "status_code", "storage_type"})
+
+	hedgedRequestsMetrics = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: "tempodb",
+			Name:      "request_duration_seconds",
+			Help:      "Time spent doing backend requests.",
+		},
+	)
+)
+
+// https://github.com/grafana/tempo/issues/760
+
+// PublishHedgedMetrics flushes metrics from hedged requests every 10 seconds
+func PublishHedgedMetrics(s *hedgedhttp.Stats) {
+	ticker := time.NewTicker(time.Second * 10)
+	go func() {
+		for {
+			<-ticker.C
+
+			_ = s.Snapshot().RequestedRoundTrips
+			hedgedRequestsMetrics.Inc()
+			// TODO: Map from h.stats to Prometheus metrics
+		}
+	}()
+}
diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go
@@ -403,10 +403,11 @@ func createCore(cfg *Config, hedge bool) (*minio.Core, error) {
 
 	// add instrumentation
 	transport := instrumentation.NewS3Transport(customTransport)
-	var _ *hedgedhttp.Stats
+	var stats *hedgedhttp.Stats
 
 	if hedge && cfg.HedgeRequestsAt != 0 {
-		transport, _ = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport)
+		transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport)
+		instrumentation.PublishHedgedMetrics(stats)
 	}
 
 	opts := &minio.Options{