diff --git a/.gitignore b/.gitignore index 273faa93c2..ae04781b0d 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ dockerize* .idea/ .DS_Store + +# redis dump file +*.rdb diff --git a/cmd/refinery/main.go b/cmd/refinery/main.go index 506e663fc8..fb5acd5409 100644 --- a/cmd/refinery/main.go +++ b/cmd/refinery/main.go @@ -11,6 +11,8 @@ import ( "syscall" "time" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" _ "go.uber.org/automaxprocs" "golang.org/x/exp/slices" @@ -23,6 +25,7 @@ import ( "github.com/honeycombio/refinery/app" "github.com/honeycombio/refinery/collect" "github.com/honeycombio/refinery/config" + "github.com/honeycombio/refinery/internal/otelutil" "github.com/honeycombio/refinery/internal/peer" "github.com/honeycombio/refinery/logger" "github.com/honeycombio/refinery/metrics" @@ -198,6 +201,18 @@ func main() { oTelMetrics = &metrics.OTelMetrics{} } + resourceLib := "refinery" + resourceVer := version + tracer := trace.Tracer(noop.Tracer{}) + shutdown := func() {} + + if c.GetOTelTracingConfig().Enabled { + // let's set up some OTel tracing + tracer, shutdown = otelutil.SetupTracing(c.GetOTelTracingConfig(), resourceLib, resourceVer) + } + + defer shutdown() + // we need to include all the metrics types so we can inject them in case they're needed var g inject.Graph if opts.Debug { @@ -216,6 +231,8 @@ func main() { {Value: legacyMetrics, Name: "legacyMetrics"}, {Value: promMetrics, Name: "promMetrics"}, {Value: oTelMetrics, Name: "otelMetrics"}, + {Value: tracer, Name: "tracer"}, + {Value: metricsSingleton, Name: "metrics"}, {Value: genericMetricsRecorder, Name: "genericMetrics"}, {Value: upstreamMetricsRecorder, Name: "upstreamMetrics"}, diff --git a/config/cmdenv.go b/config/cmdenv.go index df45f32ec8..e39a3ed8c5 100644 --- a/config/cmdenv.go +++ b/config/cmdenv.go @@ -40,6 +40,7 @@ type CmdEnv struct { HoneycombLoggerAPIKey string `long:"logger-api-key" env:"REFINERY_HONEYCOMB_LOGGER_API_KEY" description:"Honeycomb logger API key"` LegacyMetricsAPIKey string `long:"legacy-metrics-api-key" env:"REFINERY_HONEYCOMB_METRICS_API_KEY" description:"API key for legacy Honeycomb metrics"` OTelMetricsAPIKey string `long:"otel-metrics-api-key" env:"REFINERY_OTEL_METRICS_API_KEY" description:"API key for OTel metrics if being sent to Honeycomb"` + OTelTracesAPIKey string `long:"otel-traces-api-key" env:"REFINERY_OTEL_TRACES_API_KEY" description:"API key for OTel traces if being sent to Honeycomb"` QueryAuthToken string `long:"query-auth-token" env:"REFINERY_QUERY_AUTH_TOKEN" description:"Token for debug/management queries"` AvailableMemory MemorySize `long:"available-memory" env:"REFINERY_AVAILABLE_MEMORY" description:"The maximum memory available for Refinery to use (ex: 4GiB)."` Debug bool `short:"d" long:"debug" description:"Runs debug service (on the first open port between localhost:6060 and :6069 by default)"` diff --git a/config/config.go b/config/config.go index 7fe1b0971a..db7685ccb4 100644 --- a/config/config.go +++ b/config/config.go @@ -146,6 +146,8 @@ type Config interface { GetIdentifierInterfaceName() (string, error) + GetOTelTracingConfig() OTelTracingConfig + GetUseIPV6Identifier() (bool, error) GetRedisIdentifier() (string, error) diff --git a/config/configLoadHelpers.go b/config/configLoadHelpers.go index b4e39b4dd7..365a6e1e73 100644 --- a/config/configLoadHelpers.go +++ b/config/configLoadHelpers.go @@ -167,6 +167,10 @@ func validateConfig(opts *CmdEnv) ([]string, error) { config.OTelMetrics.APIKey = "InvalidHoneycombAPIKey" } + if config.OTelTracing.APIKey == "" { + config.OTelTracing.APIKey = "InvalidHoneycombAPIKey" + } + // write it out to a YAML buffer buf := new(bytes.Buffer) encoder := yaml.NewEncoder(buf) diff --git a/config/file_config.go b/config/file_config.go index c687e030a5..97ef864f1d 100644 --- a/config/file_config.go +++ b/config/file_config.go @@ -59,6 +59,7 @@ type configContents struct { PrometheusMetrics PrometheusMetricsConfig `yaml:"PrometheusMetrics"` LegacyMetrics LegacyMetricsConfig `yaml:"LegacyMetrics"` OTelMetrics OTelMetricsConfig `yaml:"OTelMetrics"` + OTelTracing OTelTracingConfig `yaml:"OTelTracing"` PeerManagement PeerManagementConfig `yaml:"PeerManagement"` RedisPeerManagement RedisPeerManagementConfig `yaml:"RedisPeerManagement"` Collection CollectionConfig `yaml:"Collection"` @@ -183,6 +184,14 @@ type OTelMetricsConfig struct { ReportingInterval Duration `yaml:"ReportingInterval" default:"30s"` } +type OTelTracingConfig struct { + Enabled bool `yaml:"Enabled" default:"false"` + APIHost string `yaml:"APIHost" default:"https://api.honeycomb.io"` + APIKey string `yaml:"APIKey" cmdenv:"OTelTracesAPIKey,HoneycombAPIKey"` + Dataset string `yaml:"Dataset" default:"Refinery Traces"` + SampleRate uint64 `yaml:"SampleRate" default:"100"` +} + type PeerManagementConfig struct { Type string `yaml:"Type" default:"file"` Identifier string `yaml:"Identifier"` @@ -832,6 +841,13 @@ func (f *fileConfig) GetEnvironmentCacheTTL() time.Duration { return time.Duration(f.mainConfig.Specialized.EnvironmentCacheTTL) } +func (f *fileConfig) GetOTelTracingConfig() OTelTracingConfig { + f.mux.RLock() + defer f.mux.RUnlock() + + return f.mainConfig.OTelTracing +} + func (f *fileConfig) GetDatasetPrefix() string { f.mux.RLock() defer f.mux.RUnlock() diff --git a/config/metadata/configMeta.yaml b/config/metadata/configMeta.yaml index f4b175b264..728566436b 100644 --- a/config/metadata/configMeta.yaml +++ b/config/metadata/configMeta.yaml @@ -740,6 +740,79 @@ groups: compression costs may outweigh the benefits, in which case `none` may be used. + - name: OTelTracing + title: "OpenTelemetry Tracing" + description: contains configuration for Refinery's own tracing. + fields: + - name: Enabled + type: bool + valuetype: nondefault + default: false + reload: false + firstversion: v2.6 + summary: controls whether to send Refinery's own otel traces. + description: > + The setting specifies if Refinery sends traces. + + - name: APIHost + type: url + valuetype: nondefault + default: "https://api.honeycomb.io" + reload: false + firstversion: v2.6 + summary: is the URL of the OpenTelemetry API to which traces will be sent. + description: > + Refinery's internal traces will be sent to the `/v1/traces` + endpoint on this host. + + - name: APIKey + type: string + pattern: apikey + valuetype: nondefault + default: "" + example: "SetThisToAHoneycombKey" + reload: false + firstversion: v2.6 + envvar: REFINERY_HONEYCOMB_TRACES_API_KEY, REFINERY_HONEYCOMB_API_KEY + commandline: otel-traces-api-key + validations: + - type: format + arg: apikey + summary: is the API key used to send Refinery's traces to Honeycomb. + description: > + It is recommended that you create a separate team and key for + Refinery telemetry. + + If this is blank, then Refinery will not set the Honeycomb-specific + headers for OpenTelemetry, and your `APIHost` must be set to a + valid OpenTelemetry endpoint. + + - name: Dataset + type: string + valuetype: nondefault + default: "Refinery Traces" + reload: false + validations: + - type: notempty + firstversion: v2.6 + summary: is the Honeycomb dataset to which Refinery sends its OpenTelemetry metrics. + description: > + Only used if `APIKey` is specified. + + - name: SampleRate + type: int + valuetype: nondefault + default: 100 + validations: + - type: minimum + arg: 1 + reload: true + summary: is the rate at which Refinery samples its own traces. + description: > + This is the Honeycomb sample rate used to sample traces sent by Refinery. Since each + incoming span generates multiple outgoing spans, a sample rate of at least 100 is + strongly advised. + - name: PeerManagement title: "Peer Management" description: controls how the Refinery cluster communicates between peers. diff --git a/config/mock.go b/config/mock.go index ec3c72660c..07ffe70b47 100644 --- a/config/mock.go +++ b/config/mock.go @@ -58,6 +58,7 @@ type MockConfig struct { GetLegacyMetricsConfigVal LegacyMetricsConfig GetPrometheusMetricsConfigVal PrometheusMetricsConfig GetOTelMetricsConfigVal OTelMetricsConfig + GetOTelTracingConfigVal OTelTracingConfig GetSendDelayErr error GetSendDelayVal time.Duration GetBatchTimeoutVal time.Duration @@ -295,6 +296,13 @@ func (m *MockConfig) GetOTelMetricsConfig() OTelMetricsConfig { return m.GetOTelMetricsConfigVal } +func (m *MockConfig) GetOTelTracingConfig() OTelTracingConfig { + m.Mux.RLock() + defer m.Mux.RUnlock() + + return m.GetOTelTracingConfigVal +} + func (m *MockConfig) GetSendDelay() (time.Duration, error) { m.Mux.RLock() defer m.Mux.RUnlock() diff --git a/go.mod b/go.mod index 65f65fb447..4492395043 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/honeycombio/dynsampler-go v0.6.0 github.com/honeycombio/husky v0.30.0 github.com/honeycombio/libhoney-go v1.23.1 + github.com/honeycombio/otel-config-go v1.16.0 github.com/jessevdk/go-flags v1.5.0 github.com/jonboulle/clockwork v0.4.0 github.com/json-iterator/go v1.1.12 @@ -38,7 +39,8 @@ require ( go.opentelemetry.io/otel/metric v1.27.0 go.opentelemetry.io/otel/sdk v1.27.0 go.opentelemetry.io/otel/sdk/metric v1.27.0 - go.opentelemetry.io/proto/otlp v1.2.0 + go.opentelemetry.io/otel/trace v1.27.0 + go.opentelemetry.io/proto/otlp v1.3.1 go.uber.org/automaxprocs v1.5.3 golang.org/x/exp v0.0.0-20231127185646-65229373498e google.golang.org/grpc v1.64.0 @@ -57,28 +59,43 @@ require ( github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 // indirect github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 // indirect github.com/facebookgo/structtag v0.0.0-20150214074306-217e25fb9691 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/sethvargo/go-envconfig v1.0.3 // indirect + github.com/shirou/gopsutil/v3 v3.24.5 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.8.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - go.opentelemetry.io/otel/trace v1.27.0 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.9.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/contrib/instrumentation/host v0.52.0 // indirect + go.opentelemetry.io/contrib/instrumentation/runtime v0.52.0 // indirect + go.opentelemetry.io/contrib/propagators/b3 v1.27.0 // indirect + go.opentelemetry.io/contrib/propagators/ot v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect ) -replace go.opentelemetry.io/proto/otlp => github.com/honeycombio/opentelemetry-proto-go/otlp v0.19.0-compat +replace go.opentelemetry.io/proto/otlp => github.com/honeycombio/opentelemetry-proto-go/otlp v1.3.1-compat replace github.com/panmari/cuckoofilter => github.com/honeycombio/cuckoofilter v0.0.0-20230630225016-cf48793fb7c1 diff --git a/go.sum b/go.sum index 1408c7d7e1..6c0ee1e92b 100644 --- a/go.sum +++ b/go.sum @@ -46,10 +46,13 @@ github.com/facebookgo/structtag v0.0.0-20150214074306-217e25fb9691/go.mod h1:sKL github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -72,8 +75,10 @@ github.com/honeycombio/husky v0.30.0 h1:eCISdKgFq2zwmB0d5miJnBgUV6As5QCKNtEXW94M github.com/honeycombio/husky v0.30.0/go.mod h1:amJNyAKYHWGWrgz+hrLl2OCodbOD2bjR5arceKyh3qw= github.com/honeycombio/libhoney-go v1.23.1 h1:dsZrY7wfnKyBnpQJeW9B+eawDYCZBGtmP06QEcE+YDM= github.com/honeycombio/libhoney-go v1.23.1/go.mod h1:mbaBmUkuGwrVa9NdsskMaOzvkYMRbknsfIvavWq+5kA= -github.com/honeycombio/opentelemetry-proto-go/otlp v0.19.0-compat h1:fMpIzVAl5C260HisnRWV//vfckZIC4qvn656M3VLLOY= -github.com/honeycombio/opentelemetry-proto-go/otlp v0.19.0-compat/go.mod h1:mC2aK20Z/exugKpqCgcpwEadiS0im8K6mZsD4Is/hCY= +github.com/honeycombio/opentelemetry-proto-go/otlp v1.3.1-compat h1:i9CAIguM5tMQC9xSRihqdFBoh40OBOhuhfR8OrXsZ9o= +github.com/honeycombio/opentelemetry-proto-go/otlp v1.3.1-compat/go.mod h1:ZyEcAltAA7tCBVo5o+5klmG2l+43E1fjpxGxvOIskic= +github.com/honeycombio/otel-config-go v1.16.0 h1:uBkQYJf6rcBksA9+/STZszMeLUb7UcWVokt59arF2OQ= +github.com/honeycombio/otel-config-go v1.16.0/go.mod h1:vmk9lKh45nmCm0rebihgRsB76TPRc/BQEkSh92Tj2Qs= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= @@ -87,6 +92,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= +github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -98,6 +105,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= @@ -114,6 +123,14 @@ github.com/redis/go-redis/v9 v9.5.3 h1:fOAp1/uJG+ZtcITgZOfYFmTKPE7n4Vclj1wZFgRci github.com/redis/go-redis/v9 v9.5.3/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/sethvargo/go-envconfig v1.0.3 h1:ZDxFGT1M7RPX0wgDOCdZMidrEB+NrayYr6fL0/+pk4I= +github.com/sethvargo/go-envconfig v1.0.3/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -135,14 +152,38 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/contrib/detectors/aws/lambda v0.52.0 h1:aZAZUtzXd7dklbWDqf0s8slF2KmGR2scZAtshZ3IGTI= +go.opentelemetry.io/contrib/detectors/aws/lambda v0.52.0/go.mod h1:0oIsMo4DjOJNB/mn/bssl3ba7TEZhSgqe26vRMoxHv4= +go.opentelemetry.io/contrib/instrumentation/host v0.52.0 h1:cluW2+yXY/RbUDcMMPFoAFrIHrb+Guu6Hjc3nGG8QLA= +go.opentelemetry.io/contrib/instrumentation/host v0.52.0/go.mod h1:QgaWu0LRrkD9KTV3Nf1wdItC4FA1x5OV2dCmiMwOgTE= +go.opentelemetry.io/contrib/instrumentation/runtime v0.52.0 h1:UaQVCH34fQsyDjlgS0L070Kjs9uCrLKoQfzn2Nl7XTY= +go.opentelemetry.io/contrib/instrumentation/runtime v0.52.0/go.mod h1:Ks4aHdMgu1vAfEY0cIBHcGx2l1S0+PwFm2BE/HRzqSk= +go.opentelemetry.io/contrib/propagators/b3 v1.27.0 h1:IjgxbomVrV9za6bRi8fWCNXENs0co37SZedQilP2hm0= +go.opentelemetry.io/contrib/propagators/b3 v1.27.0/go.mod h1:Dv9obQz25lCisDvvs4dy28UPh974CxkahRDUPsY7y9E= +go.opentelemetry.io/contrib/propagators/ot v1.27.0 h1:xFPqk7ntRR87dqvl6RfeHiq9UlE8mPSuL6Dtr/zysL8= +go.opentelemetry.io/contrib/propagators/ot v1.27.0/go.mod h1:nVLTPrDlSZPoVdeWRmpWBwxA73TYL6XLkC4bj72jvmg= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 h1:bFgvUr3/O4PHj3VQcFEuYKvRZJX1SJDQ+11JXuSB3/w= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0/go.mod h1:xJntEd2KL6Qdg5lwp97HMLQDVeAhrYxmzFseAMDPQ8I= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 h1:CIHWikMsN3wO+wq1Tp5VGdVRTcON+DmOJSfDjXypKOc= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0/go.mod h1:TNupZ6cxqyFEpLXAZW7On+mLFL0/g0TE3unIYL91xWc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= @@ -151,26 +192,29 @@ go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2N go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= diff --git a/internal/otelutil/otel_tracing.go b/internal/otelutil/otel_tracing.go new file mode 100644 index 0000000000..e4282a5678 --- /dev/null +++ b/internal/otelutil/otel_tracing.go @@ -0,0 +1,117 @@ +package otelutil + +import ( + "context" + "fmt" + "log" + "strings" + + "github.com/honeycombio/otel-config-go/otelconfig" + "github.com/honeycombio/refinery/config" + "github.com/honeycombio/refinery/types" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + samplers "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" +) + +// telemetry helpers + +func AddException(span trace.Span, err error) { + span.AddEvent("exception", trace.WithAttributes( + attribute.KeyValue{Key: "exception.type", Value: attribute.StringValue("error")}, + attribute.KeyValue{Key: "exception.message", Value: attribute.StringValue(err.Error())}, + attribute.KeyValue{Key: "exception.stacktrace", Value: attribute.StringValue("stacktrace")}, + attribute.KeyValue{Key: "exception.escaped", Value: attribute.BoolValue(false)}, + )) +} + +// addSpanField adds a field to a span, using the appropriate method for the type of the value. +func AddSpanField(span trace.Span, key string, value interface{}) { + span.SetAttributes(Attributes(map[string]interface{}{key: value})...) +} + +// AddSpanFields adds multiple fields to a span, using the appropriate method for the type of each value. +func AddSpanFields(span trace.Span, fields map[string]interface{}) { + span.SetAttributes(Attributes(fields)...) +} + +// Attributes converts a map of fields to a slice of attribute.KeyValue, setting types appropriately. +func Attributes(fields map[string]interface{}) []attribute.KeyValue { + attrs := make([]attribute.KeyValue, 0, len(fields)) + for k, v := range fields { + kv := attribute.KeyValue{Key: attribute.Key(k)} + switch val := v.(type) { + case string: + kv.Value = attribute.StringValue(val) + case int: + kv.Value = attribute.IntValue(val) + case int64: + kv.Value = attribute.Int64Value(val) + case float64: + kv.Value = attribute.Float64Value(val) + case bool: + kv.Value = attribute.BoolValue(val) + default: + kv.Value = attribute.StringValue(fmt.Sprintf("%v", val)) + } + attrs = append(attrs, kv) + } + return attrs +} + +// Starts a span with no extra fields. +func StartSpan(ctx context.Context, tracer trace.Tracer, name string) (context.Context, trace.Span) { + return tracer.Start(ctx, name) +} + +// Starts a span with a single field. +func StartSpanWith(ctx context.Context, tracer trace.Tracer, name string, field string, value interface{}) (context.Context, trace.Span) { + return tracer.Start(ctx, name, trace.WithAttributes(Attributes(map[string]interface{}{field: value})...)) +} + +// Starts a span with multiple fields. +func StartSpanMulti(ctx context.Context, tracer trace.Tracer, name string, fields map[string]interface{}) (context.Context, trace.Span) { + return tracer.Start(ctx, name, trace.WithAttributes(Attributes(fields)...)) +} + +func SetupTracing(cfg config.OTelTracingConfig, resourceLibrary string, resourceVersion string) (tracer trace.Tracer, shutdown func()) { + if cfg.APIKey != "" { + var protocol otelconfig.Protocol = otelconfig.ProtocolHTTPProto + + cfg.APIHost = strings.TrimSuffix(cfg.APIHost, "/") + apihost := fmt.Sprintf("%s:443", cfg.APIHost) + + sampleRate := cfg.SampleRate + if sampleRate < 1 { + sampleRate = 1 + } + + var sampleRatio float64 = 1.0 / float64(sampleRate) + + headers := map[string]string{ + types.APIKeyHeader: cfg.APIKey, + } + + if types.IsLegacyAPIKey(cfg.APIKey) { + headers[types.DatasetHeader] = cfg.Dataset + } + + otelshutdown, err := otelconfig.ConfigureOpenTelemetry( + otelconfig.WithExporterProtocol(protocol), + otelconfig.WithServiceName(cfg.Dataset), + otelconfig.WithTracesExporterEndpoint(apihost), + otelconfig.WithMetricsEnabled(false), + otelconfig.WithTracesEnabled(true), + otelconfig.WithSampler(samplers.TraceIDRatioBased(sampleRatio)), + otelconfig.WithHeaders(headers), + ) + if err != nil { + log.Fatalf("failure configuring otel: %v", err) + } + return otel.Tracer(resourceLibrary, trace.WithInstrumentationVersion(resourceVersion)), otelshutdown + } + pr := noop.NewTracerProvider() + return pr.Tracer(resourceLibrary, trace.WithInstrumentationVersion(resourceVersion)), func() {} +}