From 69746c7addc5bb4721eaf1561aba78740fad9cb7 Mon Sep 17 00:00:00 2001 From: Mike Beaumont Date: Fri, 30 Jun 2023 13:49:41 +0200 Subject: [PATCH 1/4] feat(kuma-cp): introduce OpenTelemetry tracing Signed-off-by: Mike Beaumont --- go.mod | 11 +- go.sum | 24 ++++- pkg/config/app/kuma-cp/config.go | 3 + pkg/config/tracing/config.go | 11 ++ pkg/core/bootstrap/plugins.go | 1 + pkg/plugins/runtime/opentelemetry/plugin.go | 110 ++++++++++++++++++++ 6 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 pkg/config/tracing/config.go create mode 100644 pkg/plugins/runtime/opentelemetry/plugin.go diff --git a/go.mod b/go.mod index 6c768e37bd0a..fba58cbb9454 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,9 @@ require ( github.com/testcontainers/testcontainers-go v0.21.0 github.com/vishvananda/netlink v1.2.1-beta.2 github.com/vishvananda/netns v0.0.4 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 + go.opentelemetry.io/otel/sdk v1.16.0 go.opentelemetry.io/proto/otlp v0.20.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.24.0 @@ -96,7 +99,7 @@ require ( github.com/aws/aws-sdk-go v1.44.187 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boombuler/barcode v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect @@ -115,6 +118,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -131,6 +135,7 @@ require ( github.com/google/s2a-go v0.1.3 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.8.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/gruntwork-io/go-commons v0.8.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -186,6 +191,10 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/trace v1.16.0 // indirect go.uber.org/atomic v1.10.0 // indirect golang.org/x/crypto v0.10.0 // indirect golang.org/x/mod v0.10.0 // indirect diff --git a/go.sum b/go.sum index 85a1058ae73a..1db417dc52a6 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= @@ -180,8 +180,11 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -210,6 +213,7 @@ github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8qQDT2iKfuoA= github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -295,6 +299,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gruntwork-io/go-commons v0.8.0 h1:k/yypwrPqSeYHevLlEDmvmgQzcyTwrlZGRaxEM6G0ro= github.com/gruntwork-io/go-commons v0.8.0/go.mod h1:gtp0yTtIBExIZp7vyIV9I0XQkVwiQZze678hvDXof78= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -568,6 +574,20 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.20.0 h1:BLOA1cZBAGSbRiNuGCCKiFrCdYB7deeHDeD1SueyOfA= go.opentelemetry.io/proto/otlp v0.20.0/go.mod h1:3QgjzPALBIv9pcknj2EXGPXjYPFdUh/RQfF8Lz3+Vnw= diff --git a/pkg/config/app/kuma-cp/config.go b/pkg/config/app/kuma-cp/config.go index e3d74837e98a..4212b5d9306b 100644 --- a/pkg/config/app/kuma-cp/config.go +++ b/pkg/config/app/kuma-cp/config.go @@ -18,6 +18,7 @@ import ( "github.com/kumahq/kuma/pkg/config/mads" "github.com/kumahq/kuma/pkg/config/multizone" "github.com/kumahq/kuma/pkg/config/plugins/runtime" + "github.com/kumahq/kuma/pkg/config/tracing" config_types "github.com/kumahq/kuma/pkg/config/types" "github.com/kumahq/kuma/pkg/config/xds" "github.com/kumahq/kuma/pkg/config/xds/bootstrap" @@ -150,6 +151,8 @@ type Config struct { Proxy xds.Proxy `json:"proxy"` // Intercommunication CP configuration InterCp intercp.InterCpConfig `json:"interCp"` + // Tracing + Tracing *tracing.Config `json:"tracing"` } func (c *Config) Sanitize() { diff --git a/pkg/config/tracing/config.go b/pkg/config/tracing/config.go new file mode 100644 index 000000000000..ccf7b51a6ac1 --- /dev/null +++ b/pkg/config/tracing/config.go @@ -0,0 +1,11 @@ +package tracing + +type Config struct { + OpenTelemetry OpenTelemetry `json:"openTelemetry"` +} + +type OpenTelemetry struct { + // Address of OpenTelemetry collector. + // E.g. otel-collector:4317 + Endpoint string `json:"endpoint"` +} diff --git a/pkg/core/bootstrap/plugins.go b/pkg/core/bootstrap/plugins.go index 7da4c2aa9a3d..ae50812792b3 100644 --- a/pkg/core/bootstrap/plugins.go +++ b/pkg/core/bootstrap/plugins.go @@ -15,6 +15,7 @@ import ( _ "github.com/kumahq/kuma/pkg/plugins/resources/memory" _ "github.com/kumahq/kuma/pkg/plugins/resources/postgres" _ "github.com/kumahq/kuma/pkg/plugins/runtime/gateway" + _ "github.com/kumahq/kuma/pkg/plugins/runtime/opentelemetry" _ "github.com/kumahq/kuma/pkg/plugins/runtime/k8s" _ "github.com/kumahq/kuma/pkg/plugins/runtime/universal" _ "github.com/kumahq/kuma/pkg/plugins/secrets/k8s" diff --git a/pkg/plugins/runtime/opentelemetry/plugin.go b/pkg/plugins/runtime/opentelemetry/plugin.go new file mode 100644 index 000000000000..0471700ec548 --- /dev/null +++ b/pkg/plugins/runtime/opentelemetry/plugin.go @@ -0,0 +1,110 @@ +package opentelemetry + +import ( + "context" + "fmt" + "time" + + "github.com/kumahq/kuma/pkg/config/tracing" + "github.com/kumahq/kuma/pkg/core" + core_plugins "github.com/kumahq/kuma/pkg/core/plugins" + core_runtime "github.com/kumahq/kuma/pkg/core/runtime" + "github.com/kumahq/kuma/pkg/core/runtime/component" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func init() { + core_plugins.Register("tracing.opentelemetry", &plugin{}) +} + +var log = core.Log.WithName("tracing").WithName("opentelemetry") + +type plugin struct{} + +var _ core_plugins.RuntimePlugin = &plugin{} + +type tracer struct { + config tracing.OpenTelemetry + provider *trace.TracerProvider +} + +var _ component.Component = &tracer{} + +func (t *tracer) Start(stop <-chan struct{}) error { + shutdown, err := initOtel(context.Background(), t.config) + if err != nil { + return err + } + + go func() { + <-stop + shutdown(context.Background()) + }() + + return nil +} + +func (t *tracer) NeedLeaderElection() bool { + return false +} + +func (p *plugin) Customize(rt core_runtime.Runtime) error { + tracing := rt.Config().Tracing + if tracing == nil { + return nil + } + + t := tracer{ + config: tracing.OpenTelemetry, + } + if err := rt.Add(component.NewResilientComponent(core.Log.WithName("otel-tracer"), &t)); err != nil { + return err + } + + return nil +} + +func initOtel(ctx context.Context, otelConfig tracing.OpenTelemetry) (func(context.Context) error, error) { + res, err := resource.New(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create resource: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + conn, err := grpc.DialContext( + ctx, + otelConfig.Endpoint, + grpc.WithTransportCredentials( + insecure.NewCredentials(), + ), + ) + if err != nil { + return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err) + } + + traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn)) + if err != nil { + return nil, fmt.Errorf("failed to create trace exporter: %w", err) + } + + bsp := trace.NewBatchSpanProcessor(traceExporter) + + tracerProvider := trace.NewTracerProvider( + trace.WithSampler(trace.AlwaysSample()), + trace.WithResource(res), + trace.WithSpanProcessor(bsp), + ) + otel.SetTracerProvider(tracerProvider) + + otel.SetTextMapPropagator(propagation.TraceContext{}) + + return tracerProvider.Shutdown, nil +} From 2351c8d8c041364475ecfc980f2841d8f038d798 Mon Sep 17 00:00:00 2001 From: Mike Beaumont Date: Fri, 30 Jun 2023 14:46:55 +0200 Subject: [PATCH 2/4] feat(kuma-cp): api server traces Signed-off-by: Mike Beaumont --- go.mod | 1 + go.sum | 3 +++ pkg/api-server/server.go | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/go.mod b/go.mod index fba58cbb9454..395f09b00c90 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( github.com/testcontainers/testcontainers-go v0.21.0 github.com/vishvananda/netlink v1.2.1-beta.2 github.com/vishvananda/netns v0.0.4 + go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.42.0 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 go.opentelemetry.io/otel/sdk v1.16.0 diff --git a/go.sum b/go.sum index 1db417dc52a6..997649cda22e 100644 --- a/go.sum +++ b/go.sum @@ -574,6 +574,9 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.42.0 h1:Z6SbqeRZAl2OczfkFOqLx1BeYBDYehNjEnqluD7581Y= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.42.0/go.mod h1:XiglO+8SPMqM3Mqh5/rtxR1VHc63o8tb38QrU6tm4mU= +go.opentelemetry.io/contrib/propagators/b3 v1.17.0 h1:ImOVvHnku8jijXqkwCSyYKRDt2YrnGXD4BbhcpfbfJo= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= diff --git a/pkg/api-server/server.go b/pkg/api-server/server.go index 634cfb4a4906..655d11d54755 100644 --- a/pkg/api-server/server.go +++ b/pkg/api-server/server.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" http_prometheus "github.com/slok/go-http-metrics/metrics/prometheus" "github.com/slok/go-http-metrics/middleware" + "go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful" "github.com/kumahq/kuma/pkg/api-server/authn" "github.com/kumahq/kuma/pkg/api-server/customization" @@ -121,6 +122,10 @@ func NewApiServer( } container.Filter(cors.Filter) + if cfg.Tracing != nil { + container.Filter(otelrestful.OTelFilter("api-server")) + } + // We create a WebService and set up resources endpoints and index endpoint instead of creating WebService // for every resource like /meshes/{mesh}/traffic-permissions, /meshes/{mesh}/traffic-log etc. // because go-restful detects it as a clash (you cannot register 2 WebServices with path /meshes/) From 1b23e5c67213e5acf3cbefbcdf13f36ba0f07de8 Mon Sep 17 00:00:00 2001 From: Mike Beaumont Date: Fri, 30 Jun 2023 15:23:36 +0200 Subject: [PATCH 3/4] refactor: remove pointer, add envconfig and test Signed-off-by: Mike Beaumont --- docs/generated/kuma-cp.md | 4 ++++ pkg/api-server/server.go | 4 +--- pkg/config/app/kuma-cp/config.go | 5 ++++- pkg/config/app/kuma-cp/kuma-cp.defaults.yaml | 4 ++++ pkg/config/loader_test.go | 1 + pkg/config/tracing/config.go | 16 ++++++++++++++-- pkg/plugins/runtime/opentelemetry/plugin.go | 6 +++--- 7 files changed, 31 insertions(+), 9 deletions(-) diff --git a/docs/generated/kuma-cp.md b/docs/generated/kuma-cp.md index 6da9a3b5849f..65e986798c74 100644 --- a/docs/generated/kuma-cp.md +++ b/docs/generated/kuma-cp.md @@ -700,4 +700,8 @@ proxy: # Sets the envoy runtime value to limit maximum number of incoming # connections to a builtin gateway data plane proxy globalDownstreamMaxConnections: 50000 # ENV: KUMA_PROXY_GATEWAY_GLOBAL_DOWNSTREAM_MAX_CONNECTIONS + +tracing: + openTelemetry: + endpoint: "" # e.g. otel-collector:4317 ``` diff --git a/pkg/api-server/server.go b/pkg/api-server/server.go index 655d11d54755..7fdb5f688f77 100644 --- a/pkg/api-server/server.go +++ b/pkg/api-server/server.go @@ -122,9 +122,7 @@ func NewApiServer( } container.Filter(cors.Filter) - if cfg.Tracing != nil { - container.Filter(otelrestful.OTelFilter("api-server")) - } + container.Filter(otelrestful.OTelFilter("api-server")) // We create a WebService and set up resources endpoints and index endpoint instead of creating WebService // for every resource like /meshes/{mesh}/traffic-permissions, /meshes/{mesh}/traffic-log etc. diff --git a/pkg/config/app/kuma-cp/config.go b/pkg/config/app/kuma-cp/config.go index 4212b5d9306b..7ff6d5df7f01 100644 --- a/pkg/config/app/kuma-cp/config.go +++ b/pkg/config/app/kuma-cp/config.go @@ -152,7 +152,7 @@ type Config struct { // Intercommunication CP configuration InterCp intercp.InterCpConfig `json:"interCp"` // Tracing - Tracing *tracing.Config `json:"tracing"` + Tracing tracing.Config `json:"tracing"` } func (c *Config) Sanitize() { @@ -290,6 +290,9 @@ func (c *Config) Validate() error { if err := c.InterCp.Validate(); err != nil { return errors.Wrap(err, "InterCp validation failed") } + if err := c.Tracing.Validate(); err != nil { + return errors.Wrap(err, "Tracing validation failed") + } return nil } diff --git a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml index bbbff700228a..e9cbca54d359 100644 --- a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml +++ b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml @@ -697,3 +697,7 @@ proxy: # Sets the envoy runtime value to limit maximum number of incoming # connections to a builtin gateway data plane proxy globalDownstreamMaxConnections: 50000 # ENV: KUMA_PROXY_GATEWAY_GLOBAL_DOWNSTREAM_MAX_CONNECTIONS + +tracing: + openTelemetry: + endpoint: "" # e.g. otel-collector:4317 diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go index 9c00005c6277..70d0b70e36e2 100644 --- a/pkg/config/loader_test.go +++ b/pkg/config/loader_test.go @@ -888,6 +888,7 @@ proxy: "KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL": "false", "KUMA_EXPERIMENTAL_INGRESS_TAG_FILTERS": "kuma.io/service", "KUMA_PROXY_GATEWAY_GLOBAL_DOWNSTREAM_MAX_CONNECTIONS": "1", + "KUMA_TRACING_OPENTELEMETRY_ENDPOINT": "otel-collector:4317", }, yamlFileConfig: "", }), diff --git a/pkg/config/tracing/config.go b/pkg/config/tracing/config.go index ccf7b51a6ac1..14b0c9b169c6 100644 --- a/pkg/config/tracing/config.go +++ b/pkg/config/tracing/config.go @@ -1,11 +1,23 @@ package tracing type Config struct { - OpenTelemetry OpenTelemetry `json:"openTelemetry"` + OpenTelemetry OpenTelemetry `json:"openTelemetry,omitempty"` +} + +func (c Config) Validate() error { + if err := c.OpenTelemetry.Validate(); err != nil { + return err + } + + return nil } type OpenTelemetry struct { // Address of OpenTelemetry collector. // E.g. otel-collector:4317 - Endpoint string `json:"endpoint"` + Endpoint string `json:"endpoint,omitempty" envconfig:"kuma_tracing_opentelemetry_endpoint"` +} + +func (c OpenTelemetry) Validate() error { + return nil } diff --git a/pkg/plugins/runtime/opentelemetry/plugin.go b/pkg/plugins/runtime/opentelemetry/plugin.go index 0471700ec548..45c5781056e1 100644 --- a/pkg/plugins/runtime/opentelemetry/plugin.go +++ b/pkg/plugins/runtime/opentelemetry/plugin.go @@ -55,13 +55,13 @@ func (t *tracer) NeedLeaderElection() bool { } func (p *plugin) Customize(rt core_runtime.Runtime) error { - tracing := rt.Config().Tracing - if tracing == nil { + otel := rt.Config().Tracing.OpenTelemetry + if otel.Endpoint == "" { return nil } t := tracer{ - config: tracing.OpenTelemetry, + config: otel, } if err := rt.Add(component.NewResilientComponent(core.Log.WithName("otel-tracer"), &t)); err != nil { return err From ff09c87841cdddcc5b9775711089319b52f659ec Mon Sep 17 00:00:00 2001 From: Mike Beaumont Date: Fri, 30 Jun 2023 15:50:30 +0200 Subject: [PATCH 4/4] chore: cleanup Signed-off-by: Mike Beaumont --- pkg/core/bootstrap/plugins.go | 2 +- pkg/plugins/runtime/opentelemetry/plugin.go | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/core/bootstrap/plugins.go b/pkg/core/bootstrap/plugins.go index ae50812792b3..352b903b8561 100644 --- a/pkg/core/bootstrap/plugins.go +++ b/pkg/core/bootstrap/plugins.go @@ -15,8 +15,8 @@ import ( _ "github.com/kumahq/kuma/pkg/plugins/resources/memory" _ "github.com/kumahq/kuma/pkg/plugins/resources/postgres" _ "github.com/kumahq/kuma/pkg/plugins/runtime/gateway" - _ "github.com/kumahq/kuma/pkg/plugins/runtime/opentelemetry" _ "github.com/kumahq/kuma/pkg/plugins/runtime/k8s" + _ "github.com/kumahq/kuma/pkg/plugins/runtime/opentelemetry" _ "github.com/kumahq/kuma/pkg/plugins/runtime/universal" _ "github.com/kumahq/kuma/pkg/plugins/secrets/k8s" _ "github.com/kumahq/kuma/pkg/plugins/secrets/universal" diff --git a/pkg/plugins/runtime/opentelemetry/plugin.go b/pkg/plugins/runtime/opentelemetry/plugin.go index 45c5781056e1..c416c91bb710 100644 --- a/pkg/plugins/runtime/opentelemetry/plugin.go +++ b/pkg/plugins/runtime/opentelemetry/plugin.go @@ -5,11 +5,6 @@ import ( "fmt" "time" - "github.com/kumahq/kuma/pkg/config/tracing" - "github.com/kumahq/kuma/pkg/core" - core_plugins "github.com/kumahq/kuma/pkg/core/plugins" - core_runtime "github.com/kumahq/kuma/pkg/core/runtime" - "github.com/kumahq/kuma/pkg/core/runtime/component" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/propagation" @@ -17,6 +12,12 @@ import ( "go.opentelemetry.io/otel/sdk/trace" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + + "github.com/kumahq/kuma/pkg/config/tracing" + "github.com/kumahq/kuma/pkg/core" + core_plugins "github.com/kumahq/kuma/pkg/core/plugins" + core_runtime "github.com/kumahq/kuma/pkg/core/runtime" + "github.com/kumahq/kuma/pkg/core/runtime/component" ) func init() { @@ -30,8 +31,7 @@ type plugin struct{} var _ core_plugins.RuntimePlugin = &plugin{} type tracer struct { - config tracing.OpenTelemetry - provider *trace.TracerProvider + config tracing.OpenTelemetry } var _ component.Component = &tracer{} @@ -44,7 +44,10 @@ func (t *tracer) Start(stop <-chan struct{}) error { go func() { <-stop - shutdown(context.Background()) + log.Info("stopping") + if err := shutdown(context.Background()); err != nil { + log.Error(err, "shutting down") + } }() return nil