-
Notifications
You must be signed in to change notification settings - Fork 1.2k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
🌱 Add distributed tracing framework #1211
Closed
+623
−2
Closed
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit
Hold shift + click to select a range
6148b41
Add tracing framework
bboreham 1371ba8
Connect logging to tracing
bboreham e3408a7
Try to get GroupVersionKind to log Create()
bboreham d4bc91a
Convert from OpenTracing to OpenTelemetry
bboreham ec80672
Update go.mod after conversion to OpenTelemetry
bboreham File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package tracing | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| ot "github.com/opentracing/opentracing-go" | ||
| "k8s.io/apimachinery/pkg/api/meta" | ||
| "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| ) | ||
|
|
||
| // AddTraceAnnotationToUnstructured adds an annotation encoding span's context to all objects | ||
| // Objects are modified in-place. | ||
| func AddTraceAnnotationToUnstructured(span ot.Span, objs []unstructured.Unstructured) error { | ||
| spanContext, err := GenerateEmbeddableSpanContext(span) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for _, o := range objs { | ||
| a := o.GetAnnotations() | ||
| if a == nil { | ||
| a = make(map[string]string) | ||
| } | ||
| a[TraceAnnotationKey] = spanContext | ||
| o.SetAnnotations(a) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // AddTraceAnnotationToObject - if there is a span for the current context, and | ||
| // the object doesn't already have one set, adds it as an annotation | ||
| func AddTraceAnnotationToObject(ctx context.Context, obj runtime.Object) error { | ||
| sp := ot.SpanFromContext(ctx) | ||
| if sp == nil { | ||
| return nil | ||
| } | ||
| spanContext, err := GenerateEmbeddableSpanContext(sp) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| m, err := meta.Accessor(obj) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| annotations := m.GetAnnotations() | ||
| if annotations == nil { | ||
| annotations = map[string]string{TraceAnnotationKey: spanContext} | ||
| } else { | ||
| // Don't overwrite if the caller already set one. | ||
| if _, exists := annotations[TraceAnnotationKey]; !exists { | ||
| annotations[TraceAnnotationKey] = spanContext | ||
| } | ||
| } | ||
| m.SetAnnotations(annotations) | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| package tracing | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
|
|
||
| ot "github.com/opentracing/opentracing-go" | ||
| "github.com/uber/jaeger-client-go" | ||
| jaegercfg "github.com/uber/jaeger-client-go/config" | ||
| ) | ||
|
|
||
| // SetupJaeger sets up Jaeger with some defaults and config taken from the environment | ||
| func SetupJaeger(serviceName string) (io.Closer, error) { | ||
| jcfg := &jaegercfg.Configuration{ | ||
| ServiceName: serviceName, | ||
| Sampler: &jaegercfg.SamplerConfig{ | ||
| Type: jaeger.SamplerTypeConst, | ||
| Param: 1, | ||
| }, | ||
| Reporter: &jaegercfg.ReporterConfig{ | ||
| // This name chosen so you can create a Service in your cluster | ||
| // pointing to wherever Jager is running; override by env JAEGER_AGENT_HOST | ||
| LocalAgentHostPort: "jaeger-agent.default:6831", | ||
| }, | ||
| } | ||
| jcfg, err := jcfg.FromEnv() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed read config from environment: %w", err) | ||
| } | ||
|
|
||
| // Initialize tracer with a logger and a metrics factory | ||
| tracer, closer, err := jcfg.NewTracer() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| // Set the singleton opentracing.Tracer with the Jaeger tracer. | ||
| ot.SetGlobalTracer(tracer) | ||
| return closer, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| package tracing | ||
|
|
||
| import ( | ||
| "github.com/go-logr/logr" | ||
| ot "github.com/opentracing/opentracing-go" | ||
| "github.com/opentracing/opentracing-go/ext" | ||
| ) | ||
|
|
||
| type tracingLogger struct { | ||
| logr.Logger | ||
| ot.Span | ||
| } | ||
|
|
||
| func (t tracingLogger) Enabled() bool { | ||
| return t.Logger.Enabled() | ||
| } | ||
|
|
||
| func (t tracingLogger) Info(msg string, keysAndValues ...interface{}) { | ||
| t.Logger.Info(msg, keysAndValues...) | ||
| t.Span.LogKV(append([]interface{}{"info", msg}, keysAndValues...)...) | ||
| } | ||
|
|
||
| func (t tracingLogger) Error(err error, msg string, keysAndValues ...interface{}) { | ||
| t.Logger.Error(err, msg, keysAndValues...) | ||
| t.Span.LogKV(append([]interface{}{"error", err, "message", msg}, keysAndValues...)...) | ||
| ext.Error.Set(t.Span, true) | ||
| } | ||
|
|
||
| func (t tracingLogger) V(level int) logr.Logger { | ||
| return tracingLogger{Logger: t.Logger.V(level), Span: t.Span} | ||
| } | ||
|
|
||
| func (t tracingLogger) WithValues(keysAndValues ...interface{}) logr.Logger { | ||
| for i := 0; i+1 < len(keysAndValues); i += 2 { | ||
| key, ok := keysAndValues[i].(string) | ||
| if !ok { | ||
| continue | ||
| } | ||
| t.Span.SetTag(key, keysAndValues[i+1]) | ||
| } | ||
| return tracingLogger{Logger: t.Logger.WithValues(keysAndValues...), Span: t.Span} | ||
| } | ||
|
|
||
| func (t tracingLogger) WithName(name string) logr.Logger { | ||
| t.Span.SetTag("name", name) | ||
| return tracingLogger{Logger: t.Logger.WithName(name), Span: t.Span} | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| package tracing | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| ot "github.com/opentracing/opentracing-go" | ||
| otlog "github.com/opentracing/opentracing-go/log" | ||
| "k8s.io/apimachinery/pkg/api/meta" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| "k8s.io/client-go/rest" | ||
| "sigs.k8s.io/controller-runtime/pkg/cache" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| "sigs.k8s.io/controller-runtime/pkg/manager" | ||
| ) | ||
|
|
||
| // WrapRuntimeClient wraps a NewRuntimeClient function with one that does tracing | ||
| func WrapRuntimeClient(upstreamNew manager.NewClientFunc) manager.NewClientFunc { | ||
| return func(cache cache.Cache, config *rest.Config, options client.Options) (client.Client, error) { | ||
| delegatingClient, err := upstreamNew(cache, config, options) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return &tracingClient{Client: delegatingClient, scheme: options.Scheme}, nil | ||
| } | ||
| } | ||
|
|
||
| func objectFields(obj runtime.Object) (fields []otlog.Field) { | ||
| if gvk := obj.GetObjectKind().GroupVersionKind(); !gvk.Empty() { | ||
| fields = append(fields, otlog.String("objectKind", gvk.String())) | ||
| } | ||
| if m, err := meta.Accessor(obj); err == nil { | ||
| fields = append(fields, otlog.String("objectKey", m.GetNamespace()+"/"+m.GetName())) | ||
| } | ||
| return | ||
| } | ||
|
|
||
| func logStart(ctx context.Context, op string, fields ...otlog.Field) ot.Span { | ||
| sp := ot.SpanFromContext(ctx) | ||
| if sp != nil { | ||
| sp.LogFields(append([]otlog.Field{otlog.Event(op)}, fields...)...) | ||
| } | ||
| return sp | ||
| } | ||
|
|
||
| func logError(sp ot.Span, err error) error { | ||
| if sp != nil && err != nil { | ||
| sp.LogFields(otlog.Error(err)) | ||
| } | ||
| return err | ||
| } | ||
|
|
||
| // wrapper for Client which emits spans on each call | ||
| type tracingClient struct { | ||
| client.Client | ||
| scheme *runtime.Scheme | ||
| } | ||
|
|
||
| func (c *tracingClient) blankObjectFields(obj runtime.Object) (fields []otlog.Field) { | ||
| if c.scheme != nil { | ||
| gvks, _, _ := c.scheme.ObjectKinds(obj) | ||
| for _, gvk := range gvks { | ||
| fields = append(fields, otlog.String("objectKind", gvk.String())) | ||
| } | ||
| } | ||
| return | ||
| } | ||
|
|
||
| func (c *tracingClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { | ||
| sp := logStart(ctx, "k8s.Get", append([]otlog.Field{otlog.String("objectKey", key.String())}, c.blankObjectFields(obj)...)...) | ||
| return logError(sp, c.Client.Get(ctx, key, obj)) | ||
| } | ||
|
|
||
| func (c *tracingClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { | ||
| sp := logStart(ctx, "k8s.List", c.blankObjectFields(list)...) | ||
| return logError(sp, c.Client.List(ctx, list, opts...)) | ||
| } | ||
|
|
||
| func (c *tracingClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { | ||
| AddTraceAnnotationToObject(ctx, obj) | ||
| sp := logStart(ctx, "k8s.Create", c.blankObjectFields(obj)...) | ||
| return logError(sp, c.Client.Create(ctx, obj, opts...)) | ||
| } | ||
|
|
||
| func (c *tracingClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { | ||
| sp := logStart(ctx, "k8s.Delete", objectFields(obj)...) | ||
| return logError(sp, c.Client.Delete(ctx, obj, opts...)) | ||
| } | ||
|
|
||
| func (c *tracingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { | ||
| sp := logStart(ctx, "k8s.Update", objectFields(obj)...) | ||
| return logError(sp, c.Client.Update(ctx, obj, opts...)) | ||
| } | ||
|
|
||
| func (c *tracingClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { | ||
| fields := objectFields(obj) | ||
| if data, err := patch.Data(obj); err == nil { | ||
| fields = append(fields, otlog.String("patch", string(data))) | ||
| } | ||
| sp := logStart(ctx, "k8s.Patch", fields...) | ||
| return logError(sp, c.Client.Patch(ctx, obj, patch, opts...)) | ||
| } | ||
|
|
||
| func (c *tracingClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { | ||
| sp := logStart(ctx, "k8s.DeleteAllOf", c.blankObjectFields(obj)...) | ||
| return logError(sp, c.Client.DeleteAllOf(ctx, obj, opts...)) | ||
| } | ||
|
|
||
| func (c *tracingClient) Status() client.StatusWriter { | ||
| return &tracingStatusWriter{StatusWriter: c.Client.Status()} | ||
| } | ||
|
|
||
| type tracingStatusWriter struct { | ||
| client.StatusWriter | ||
| } | ||
|
|
||
| func (s *tracingStatusWriter) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { | ||
| sp := logStart(ctx, "k8s.Status.Update", objectFields(obj)...) | ||
| return logError(sp, s.StatusWriter.Update(ctx, obj, opts...)) | ||
| } | ||
|
|
||
| func (s *tracingStatusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { | ||
| fields := objectFields(obj) | ||
| if data, err := patch.Data(obj); err == nil { | ||
| fields = append(fields, otlog.String("patch", string(data))) | ||
| } | ||
| sp := logStart(ctx, "k8s.Status.Patch", fields...) | ||
| return logError(sp, s.StatusWriter.Patch(ctx, obj, patch, opts...)) | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
is there a way to avoid needing an extra allocation here?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
also, is there a semantic difference between
WithTagsand the key-value pairs specified here?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That
LogKV()call is through an interface so the memory will escape; it's unlikely to be possible to avoid an allocation. This particular line is only hit under error conditions so probably not performance-sensitive, but there are similar ones.Tags apply to the whole span so it wouldn't be right to put details from one error as a tag, however the next line sets an
errortag on the span so you can pick it out as one that had at least one error.