From dcf861afaacf9caa6f972b62ad23ac6d628010f2 Mon Sep 17 00:00:00 2001 From: Ian Billett Date: Wed, 2 Jun 2021 21:17:22 +0100 Subject: [PATCH 1/5] add tracing to exemplar APIs Signed-off-by: Ian Billett --- pkg/api/query/v1.go | 21 +++++++++++++++++---- pkg/exemplars/exemplars.go | 4 ++++ pkg/exemplars/prometheus.go | 9 ++++++++- pkg/exemplars/proxy.go | 11 +++++++++-- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index f0aec67ff6..5fc733a63c 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -791,6 +791,15 @@ func NewExemplarsHandler(client exemplars.UnaryClient, enablePartialResponse boo } return func(r *http.Request) (interface{}, []error, *api.ApiError) { + span, ctx := tracing.StartSpan(r.Context(), "exemplar_query_request") + defer span.Finish() + + var ( + exemplarsData []*exemplarspb.ExemplarData + exemplarWarnings storage.Warnings + exemplarError error + ) + start, err := cortexutil.ParseTime(r.FormValue("start")) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} @@ -806,11 +815,15 @@ func NewExemplarsHandler(client exemplars.UnaryClient, enablePartialResponse boo Query: r.FormValue("query"), PartialResponseStrategy: ps, } - exemplarsData, warnings, err := client.Exemplars(r.Context(), req) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorInternal, Err: errors.Wrap(err, "retrieving exemplars")} + + tracing.DoInSpan(ctx, "retrieve_exemplars", func(ctx context.Context) { + exemplarsData, exemplarWarnings, exemplarError = client.Exemplars(r.Context(), req) + }) + + if exemplarError != nil { + return nil, nil, &api.ApiError{Typ: api.ErrorInternal, Err: errors.Wrap(exemplarError, "retrieving exemplars")} } - return exemplarsData, warnings, nil + return exemplarsData, exemplarWarnings, nil } } diff --git a/pkg/exemplars/exemplars.go b/pkg/exemplars/exemplars.go index a5145a9b0f..bd372942ad 100644 --- a/pkg/exemplars/exemplars.go +++ b/pkg/exemplars/exemplars.go @@ -5,6 +5,7 @@ package exemplars import ( "context" + "github.com/thanos-io/thanos/pkg/tracing" "sort" "github.com/pkg/errors" @@ -73,6 +74,9 @@ func NewGRPCClientWithDedup(es exemplarspb.ExemplarsServer, replicaLabels []stri } func (rr *GRPCClient) Exemplars(ctx context.Context, req *exemplarspb.ExemplarsRequest) ([]*exemplarspb.ExemplarData, storage.Warnings, error) { + span, ctx := tracing.StartSpan(ctx, "exemplar_grpc_request") + defer span.Finish() + resp := &exemplarsServer{ctx: ctx} if err := rr.proxy.Exemplars(req, resp); err != nil { diff --git a/pkg/exemplars/prometheus.go b/pkg/exemplars/prometheus.go index 2ab3b5f648..c681fd2b16 100644 --- a/pkg/exemplars/prometheus.go +++ b/pkg/exemplars/prometheus.go @@ -4,12 +4,14 @@ package exemplars import ( + "context" "net/url" "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/store/labelpb" + "github.com/thanos-io/thanos/pkg/tracing" ) // Prometheus implements exemplarspb.Exemplars gRPC that allows to fetch exemplars from Prometheus. @@ -41,7 +43,12 @@ func (p *Prometheus) Exemplars(r *exemplarspb.ExemplarsRequest, s exemplarspb.Ex for _, e := range exemplars { // Make sure the returned series labels are sorted. e.SetSeriesLabels(labelpb.ExtendSortedLabels(e.SeriesLabels.PromLabels(), extLset)) - if err := s.Send(&exemplarspb.ExemplarsResponse{Result: &exemplarspb.ExemplarsResponse_Data{Data: e}}); err != nil { + + var err error + tracing.DoInSpan(s.Context(), "send_exemplars_response", func(_ context.Context) { + err = s.Send(&exemplarspb.ExemplarsResponse{Result: &exemplarspb.ExemplarsResponse_Data{Data: e}}) + }) + if err != nil { return err } } diff --git a/pkg/exemplars/proxy.go b/pkg/exemplars/proxy.go index 0386aa816c..d8e2df178e 100644 --- a/pkg/exemplars/proxy.go +++ b/pkg/exemplars/proxy.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/prometheus/promql/parser" "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/thanos-io/thanos/pkg/tracing" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -52,6 +53,9 @@ type exemplarsStream struct { } func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exemplars_ExemplarsServer) error { + span, ctx := tracing.StartSpan(srv.Context(), "proxy_exemplars") + defer span.Finish() + expr, err := parser.ParseExpr(req.Query) if err != nil { return err @@ -72,7 +76,7 @@ func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exe } var ( - g, gctx = errgroup.WithContext(srv.Context()) + g, gctx = errgroup.WithContext(ctx) respChan = make(chan *exemplarspb.ExemplarData, 10) exemplars []*exemplarspb.ExemplarData ) @@ -147,7 +151,10 @@ func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exe } for _, e := range exemplars { - if err := srv.Send(exemplarspb.NewExemplarsResponse(e)); err != nil { + tracing.DoInSpan(srv.Context(), "send_exemplars_response", func(_ context.Context) { + err = srv.Send(exemplarspb.NewExemplarsResponse(e)) + }) + if err != nil { return status.Error(codes.Unknown, errors.Wrap(err, "send exemplars response").Error()) } } From 0b9390a0e1784fc6871e59393de9de8fc7c466d1 Mon Sep 17 00:00:00 2001 From: Ian Billett Date: Thu, 3 Jun 2021 09:35:55 +0100 Subject: [PATCH 2/5] adds CHANGELOG Signed-off-by: Ian Billett --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce256defea..62c152b1d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re ### Added +- [#4299](https://github.com/thanos-io/thanos/pull/4299) Tracing: Add tracing to exemplar APIs. - [#4117](https://github.com/thanos-io/thanos/pull/4117) Mixin: new alert ThanosReceiveTrafficBelowThreshold to flag if the ingestion average of the last hour dips below 50% of the ingestion average for the last 12 hours. - [#4107](https://github.com/thanos-io/thanos/pull/4107) Store: `LabelNames` and `LabelValues` now support label matchers. - [#3940](https://github.com/thanos-io/thanos/pull/3940) Sidecar: Added matchers support to `LabelValues` From 8d5119bc41d7f5f52822ca665d0fd4059fc2874f Mon Sep 17 00:00:00 2001 From: Ian Billett Date: Thu, 3 Jun 2021 09:45:20 +0100 Subject: [PATCH 3/5] linter Signed-off-by: Ian Billett --- pkg/exemplars/exemplars.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/exemplars/exemplars.go b/pkg/exemplars/exemplars.go index bd372942ad..fab73d4814 100644 --- a/pkg/exemplars/exemplars.go +++ b/pkg/exemplars/exemplars.go @@ -5,13 +5,13 @@ package exemplars import ( "context" - "github.com/thanos-io/thanos/pkg/tracing" "sort" "github.com/pkg/errors" "github.com/prometheus/prometheus/storage" "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" "github.com/thanos-io/thanos/pkg/store/labelpb" + "github.com/thanos-io/thanos/pkg/tracing" ) var _ UnaryClient = &GRPCClient{} From fb293b6a4cefae1279a34b5134cfd2fd81654ab4 Mon Sep 17 00:00:00 2001 From: Ian Billett Date: Thu, 3 Jun 2021 10:46:30 +0100 Subject: [PATCH 4/5] Update pkg/api/query/v1.go Co-authored-by: Kemal Akkoyun Signed-off-by: Ian Billett --- pkg/api/query/v1.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index 5fc733a63c..9b8e11731f 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -817,7 +817,7 @@ func NewExemplarsHandler(client exemplars.UnaryClient, enablePartialResponse boo } tracing.DoInSpan(ctx, "retrieve_exemplars", func(ctx context.Context) { - exemplarsData, exemplarWarnings, exemplarError = client.Exemplars(r.Context(), req) + exemplarsData, exemplarWarnings, exemplarError = client.Exemplars(ctx, req) }) if exemplarError != nil { From deaaa315dc6b5644c06a7d948824867f5a77eea4 Mon Sep 17 00:00:00 2001 From: Ian Billett Date: Thu, 3 Jun 2021 10:57:22 +0100 Subject: [PATCH 5/5] rename Signed-off-by: Ian Billett --- pkg/api/query/v1.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index 9b8e11731f..b74dcf864a 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -795,9 +795,9 @@ func NewExemplarsHandler(client exemplars.UnaryClient, enablePartialResponse boo defer span.Finish() var ( - exemplarsData []*exemplarspb.ExemplarData - exemplarWarnings storage.Warnings - exemplarError error + data []*exemplarspb.ExemplarData + warnings storage.Warnings + err error ) start, err := cortexutil.ParseTime(r.FormValue("start")) @@ -817,13 +817,13 @@ func NewExemplarsHandler(client exemplars.UnaryClient, enablePartialResponse boo } tracing.DoInSpan(ctx, "retrieve_exemplars", func(ctx context.Context) { - exemplarsData, exemplarWarnings, exemplarError = client.Exemplars(ctx, req) + data, warnings, err = client.Exemplars(ctx, req) }) - if exemplarError != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorInternal, Err: errors.Wrap(exemplarError, "retrieving exemplars")} + if err != nil { + return nil, nil, &api.ApiError{Typ: api.ErrorInternal, Err: errors.Wrap(err, "retrieving exemplars")} } - return exemplarsData, exemplarWarnings, nil + return data, warnings, nil } }