From 1d6d7ed96395f31bd306306d1969a3d62b29bb63 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Mon, 28 Jun 2021 09:45:52 +0100 Subject: [PATCH] Start on hedged metrics --- tempodb/backend/azure/azure_helpers.go | 5 ++- tempodb/backend/gcs/gcs.go | 5 ++- .../instrumentation/backend_transports.go | 33 ++++++++------- .../instrumentation/hedged_requests.go | 42 +++++++++++++++++++ tempodb/backend/s3/s3.go | 5 ++- 5 files changed, 70 insertions(+), 20 deletions(-) create mode 100644 tempodb/backend/instrumentation/hedged_requests.go diff --git a/tempodb/backend/azure/azure_helpers.go b/tempodb/backend/azure/azure_helpers.go index c9a7a07a82a..39b989395dd 100644 --- a/tempodb/backend/azure/azure_helpers.go +++ b/tempodb/backend/azure/azure_helpers.go @@ -38,11 +38,12 @@ func GetContainerURL(ctx context.Context, cfg *Config, hedge bool) (blob.Contain // add instrumentation transport := instrumentation.NewAzureTransport(customTransport) - var _ *hedgedhttp.Stats + var stats *hedgedhttp.Stats // hedge if desired (0 means disabled) if hedge && cfg.HedgeRequestsAt != 0 { - transport, _ = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + instrumentation.PublishHedgedMetrics(stats) } client := http.Client{Transport: transport} diff --git a/tempodb/backend/gcs/gcs.go b/tempodb/backend/gcs/gcs.go index ffa1432642e..0d1ed13a47d 100644 --- a/tempodb/backend/gcs/gcs.go +++ b/tempodb/backend/gcs/gcs.go @@ -322,11 +322,12 @@ func createBucket(ctx context.Context, cfg *Config, hedge bool) (*storage.Bucket // add instrumentation transport = instrumentation.NewGCSTransport(transport) - var _ *hedgedhttp.Stats + var stats *hedgedhttp.Stats // hedge if desired (0 means disabled) if hedge && cfg.HedgeRequestsAt != 0 { - transport, _ = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + instrumentation.PublishHedgedMetrics(stats) } // build client diff --git a/tempodb/backend/instrumentation/backend_transports.go b/tempodb/backend/instrumentation/backend_transports.go index ac279e25159..9a93ab480b2 100644 --- a/tempodb/backend/instrumentation/backend_transports.go +++ b/tempodb/backend/instrumentation/backend_transports.go @@ -31,37 +31,41 @@ var ( Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), }, []string{"operation", "status_code"}) - //hedgedRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - // Namespace: "tempodb", - // Name: "request_duration_seconds", - // Help: "Time spent doing backend requests.", - // Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), - //}, []string{"operation", "status_code", "storage_type"}) + requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "tempodb", + Name: "backend_request_duration_seconds", + Help: "Time spent doing requests.", + Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), + }, []string{"operation", "status_code"}) ) type instrumentedTransport struct { - observer prometheus.ObserverVec - next http.RoundTripper + legacyObserver prometheus.ObserverVec + observer prometheus.ObserverVec + next http.RoundTripper } func NewGCSTransport(next http.RoundTripper) http.RoundTripper { return instrumentedTransport{ - next: next, - observer: gcsRequestDuration, + next: next, + observer: requestDuration, + legacyObserver: gcsRequestDuration, } } func NewS3Transport(next http.RoundTripper) http.RoundTripper { return instrumentedTransport{ - next: next, - observer: s3RequestDuration, + next: next, + observer: requestDuration, + legacyObserver: s3RequestDuration, } } func NewAzureTransport(next http.RoundTripper) http.RoundTripper { return instrumentedTransport{ - next: next, - observer: azureRequestDuration, + next: next, + observer: requestDuration, + legacyObserver: azureRequestDuration, } } @@ -69,6 +73,7 @@ func (i instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, err start := time.Now() resp, err := i.next.RoundTrip(req) if err == nil { + i.legacyObserver.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds()) i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds()) } return resp, err diff --git a/tempodb/backend/instrumentation/hedged_requests.go b/tempodb/backend/instrumentation/hedged_requests.go new file mode 100644 index 00000000000..ed014beabe9 --- /dev/null +++ b/tempodb/backend/instrumentation/hedged_requests.go @@ -0,0 +1,42 @@ +package instrumentation + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/cristalhq/hedgedhttp" +) + +var ( + //hedgedRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + // Namespace: "tempodb", + // Name: "request_duration_seconds", + // Help: "Time spent doing backend requests.", + // Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), + //}, []string{"operation", "status_code", "storage_type"}) + + hedgedRequestsMetrics = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "tempodb", + Name: "request_duration_seconds", + Help: "Time spent doing backend requests.", + }, + ) +) + +// https://github.com/grafana/tempo/issues/760 + +// PublishHedgedMetrics flushes metrics from hedged requests every 10 seconds +func PublishHedgedMetrics(s *hedgedhttp.Stats) { + ticker := time.NewTicker(time.Second * 10) + go func() { + for { + <-ticker.C + + _ = s.Snapshot().RequestedRoundTrips + hedgedRequestsMetrics.Inc() + // TODO: Map from h.stats to Prometheus metrics + } + }() +} diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go index abbb507a7bd..247d8b73292 100644 --- a/tempodb/backend/s3/s3.go +++ b/tempodb/backend/s3/s3.go @@ -403,10 +403,11 @@ func createCore(cfg *Config, hedge bool) (*minio.Core, error) { // add instrumentation transport := instrumentation.NewS3Transport(customTransport) - var _ *hedgedhttp.Stats + var stats *hedgedhttp.Stats if hedge && cfg.HedgeRequestsAt != 0 { - transport, _ = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + instrumentation.PublishHedgedMetrics(stats) } opts := &minio.Options{