Add tracing to exemplar APIs

CHANGELOG.md

@@ -28,6 +28,7 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 
 ### Added
 
+- [#4299](https://github.com/thanos-io/thanos/pull/4299) Tracing: Add tracing to exemplar APIs.
 - [#4117](https://github.com/thanos-io/thanos/pull/4117) Mixin:  new alert ThanosReceiveTrafficBelowThreshold to flag if the ingestion average of the last hour dips below 50% of the ingestion average for the last 12 hours.
 - [#4107](https://github.com/thanos-io/thanos/pull/4107) Store: `LabelNames` and `LabelValues` now support label matchers.
 - [#3940](https://github.com/thanos-io/thanos/pull/3940) Sidecar: Added matchers support to `LabelValues`

pkg/api/query/v1.go

@@ -791,6 +791,15 @@ func NewExemplarsHandler(client exemplars.UnaryClient, enablePartialResponse boo
 	}
 
 	return func(r *http.Request) (interface{}, []error, *api.ApiError) {
+		span, ctx := tracing.StartSpan(r.Context(), "exemplar_query_request")
+		defer span.Finish()
+
+		var (
+			data     []*exemplarspb.ExemplarData
+			warnings storage.Warnings
+			err      error
+		)
+
 		start, err := cortexutil.ParseTime(r.FormValue("start"))
 		if err != nil {
 			return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}
@@ -806,11 +815,15 @@ func NewExemplarsHandler(client exemplars.UnaryClient, enablePartialResponse boo
 			Query:                   r.FormValue("query"),
 			PartialResponseStrategy: ps,
 		}
-		exemplarsData, warnings, err := client.Exemplars(r.Context(), req)
+
+		tracing.DoInSpan(ctx, "retrieve_exemplars", func(ctx context.Context) {
+			data, warnings, err = client.Exemplars(ctx, req)
+		})
+
 		if err != nil {
 			return nil, nil, &api.ApiError{Typ: api.ErrorInternal, Err: errors.Wrap(err, "retrieving exemplars")}
 		}
-		return exemplarsData, warnings, nil
+		return data, warnings, nil
 	}
 }
 

pkg/exemplars/exemplars.go

@@ -11,6 +11,7 @@ import (
 	"github.com/prometheus/prometheus/storage"
 	"github.com/thanos-io/thanos/pkg/exemplars/exemplarspb"
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
+	"github.com/thanos-io/thanos/pkg/tracing"
 )
 
 var _ UnaryClient = &GRPCClient{}
@@ -73,6 +74,9 @@ func NewGRPCClientWithDedup(es exemplarspb.ExemplarsServer, replicaLabels []stri
 }
 
 func (rr *GRPCClient) Exemplars(ctx context.Context, req *exemplarspb.ExemplarsRequest) ([]*exemplarspb.ExemplarData, storage.Warnings, error) {
+	span, ctx := tracing.StartSpan(ctx, "exemplar_grpc_request")
+	defer span.Finish()
+
 	resp := &exemplarsServer{ctx: ctx}
 
 	if err := rr.proxy.Exemplars(req, resp); err != nil {

pkg/exemplars/prometheus.go

@@ -4,12 +4,14 @@
 package exemplars
 
 import (
+	"context"
 	"net/url"
 
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/thanos-io/thanos/pkg/exemplars/exemplarspb"
 	"github.com/thanos-io/thanos/pkg/promclient"
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
+	"github.com/thanos-io/thanos/pkg/tracing"
 )
 
 // Prometheus implements exemplarspb.Exemplars gRPC that allows to fetch exemplars from Prometheus.
@@ -41,7 +43,12 @@ func (p *Prometheus) Exemplars(r *exemplarspb.ExemplarsRequest, s exemplarspb.Ex
 	for _, e := range exemplars {
 		// Make sure the returned series labels are sorted.
 		e.SetSeriesLabels(labelpb.ExtendSortedLabels(e.SeriesLabels.PromLabels(), extLset))
-		if err := s.Send(&exemplarspb.ExemplarsResponse{Result: &exemplarspb.ExemplarsResponse_Data{Data: e}}); err != nil {
+
+		var err error
+		tracing.DoInSpan(s.Context(), "send_exemplars_response", func(_ context.Context) {
+			err = s.Send(&exemplarspb.ExemplarsResponse{Result: &exemplarspb.ExemplarsResponse_Data{Data: e}})
+		})
+		if err != nil {
 			return err
 		}
 	}

pkg/exemplars/proxy.go

@@ -14,6 +14,7 @@ import (
 	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/thanos-io/thanos/pkg/exemplars/exemplarspb"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
+	"github.com/thanos-io/thanos/pkg/tracing"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -52,6 +53,9 @@ type exemplarsStream struct {
 }
 
 func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exemplars_ExemplarsServer) error {
+	span, ctx := tracing.StartSpan(srv.Context(), "proxy_exemplars")
+	defer span.Finish()
+
 	expr, err := parser.ParseExpr(req.Query)
 	if err != nil {
 		return err
@@ -72,7 +76,7 @@ func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exe
 	}
 
 	var (
-		g, gctx   = errgroup.WithContext(srv.Context())
+		g, gctx   = errgroup.WithContext(ctx)
 		respChan  = make(chan *exemplarspb.ExemplarData, 10)
 		exemplars []*exemplarspb.ExemplarData
 	)
@@ -147,7 +151,10 @@ func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exe
 	}
 
 	for _, e := range exemplars {
-		if err := srv.Send(exemplarspb.NewExemplarsResponse(e)); err != nil {
+		tracing.DoInSpan(srv.Context(), "send_exemplars_response", func(_ context.Context) {
+			err = srv.Send(exemplarspb.NewExemplarsResponse(e))
+		})
+		if err != nil {
 			return status.Error(codes.Unknown, errors.Wrap(err, "send exemplars response").Error())
 		}
 	}