From e2c2d87b2c98d26c1b0403c5a5ad14797f758b89 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Sat, 18 Nov 2023 09:24:26 -0500 Subject: [PATCH 01/19] wip scaffolding --- exporter/rabbitmqexporter/Makefile | 6 +++ exporter/rabbitmqexporter/README.md | 0 exporter/rabbitmqexporter/config.go | 4 ++ exporter/rabbitmqexporter/factory.go | 42 +++++++++++++++++++ .../internal/metadata/generated_status.go | 14 +++++++ exporter/rabbitmqexporter/metadata.yaml | 9 ++++ .../rabbitmqexporter/rabbitmq_exporter.go | 22 ++++++++++ 7 files changed, 97 insertions(+) create mode 100644 exporter/rabbitmqexporter/Makefile create mode 100644 exporter/rabbitmqexporter/README.md create mode 100644 exporter/rabbitmqexporter/config.go create mode 100644 exporter/rabbitmqexporter/factory.go create mode 100644 exporter/rabbitmqexporter/internal/metadata/generated_status.go create mode 100644 exporter/rabbitmqexporter/metadata.yaml create mode 100644 exporter/rabbitmqexporter/rabbitmq_exporter.go diff --git a/exporter/rabbitmqexporter/Makefile b/exporter/rabbitmqexporter/Makefile new file mode 100644 index 000000000000..fb455d4977dd --- /dev/null +++ b/exporter/rabbitmqexporter/Makefile @@ -0,0 +1,6 @@ +include ../../Makefile.Common + +# Remove "-race" from the default set of test arguments. +# exporter/pulsarexporter tests are failing with the -race check. +# See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/12518 +GOTEST_OPT = -v -timeout 300s --tags=$(GO_BUILD_TAGS) diff --git a/exporter/rabbitmqexporter/README.md b/exporter/rabbitmqexporter/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/exporter/rabbitmqexporter/config.go b/exporter/rabbitmqexporter/config.go new file mode 100644 index 000000000000..8a0ff9da8540 --- /dev/null +++ b/exporter/rabbitmqexporter/config.go @@ -0,0 +1,4 @@ +package rabbitmqexporter + +type config struct { +} diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go new file mode 100644 index 000000000000..13ce51fb37f3 --- /dev/null +++ b/exporter/rabbitmqexporter/factory.go @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package rabbitmqexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/pulsarexporter" +import ( + "context" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter/internal/metadata" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/exporter/exporterhelper" +) + +type rabbitmqExporterFactory struct { +} + +// NewFactory creates Pulsar exporter factory. +func NewFactory() exporter.Factory { + f := &rabbitmqExporterFactory{} + return exporter.NewFactory( + metadata.Type, + createDefaultConfig, + exporter.WithLogs(f.createLogsExporter, metadata.LogsStability), + ) +} + +func createDefaultConfig() component.Config { + return &config{} +} + +func (f *rabbitmqExporterFactory) createLogsExporter( + ctx context.Context, + set exporter.CreateSettings, + cfg component.Config, +) (exporter.Logs, error) { + customConfig := *(cfg.(*config)) + exp, err := newLogsExporter(customConfig, set) + if err != nil { + return nil, err + } + + return exporterhelper.NewLogsExporter(ctx, set, cfg, exp.logsDataPusher) +} diff --git a/exporter/rabbitmqexporter/internal/metadata/generated_status.go b/exporter/rabbitmqexporter/internal/metadata/generated_status.go new file mode 100644 index 000000000000..8c5420db4dd0 --- /dev/null +++ b/exporter/rabbitmqexporter/internal/metadata/generated_status.go @@ -0,0 +1,14 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +const ( + Type = "rabbitmq" + TracesStability = component.StabilityLevelDevelopment + MetricsStability = component.StabilityLevelDevelopment + LogsStability = component.StabilityLevelDevelopment +) diff --git a/exporter/rabbitmqexporter/metadata.yaml b/exporter/rabbitmqexporter/metadata.yaml new file mode 100644 index 000000000000..c6c9e6ae996d --- /dev/null +++ b/exporter/rabbitmqexporter/metadata.yaml @@ -0,0 +1,9 @@ +type: rabbitmq + +status: + class: exporter + stability: + alpha: [traces, metrics, logs] + distributions: [contrib] + codeowners: + active: [] diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go new file mode 100644 index 000000000000..7fd48e6aafb2 --- /dev/null +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -0,0 +1,22 @@ +package rabbitmqexporter + +import ( + "context" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/pdata/plog" +) + +type rabbitMqLogsProducer struct{} + +func newLogsExporter(config config, set exporter.CreateSettings) (*rabbitMqLogsProducer, error) { + return &rabbitMqLogsProducer{}, nil +} + +func (e *rabbitMqLogsProducer) logsDataPusher(ctx context.Context, ld plog.Logs) error { + var errs error + return errs +} + +//func (e *rabbitMqLogsProducer) Close(context.Context) error { +// return nil +//} From bbd3c830e9d808bbcc67c36f21985730395bd82a Mon Sep 17 00:00:00 2001 From: swar8080 Date: Thu, 23 Nov 2023 11:43:55 -0500 Subject: [PATCH 02/19] initial plumbing between factory and amqp channel cacher --- exporter/rabbitmqexporter/channel_caching.go | 198 ++++++++++++++++++ exporter/rabbitmqexporter/config.go | 7 + exporter/rabbitmqexporter/factory.go | 4 +- .../rabbitmqexporter/rabbitmq_exporter.go | 35 +++- go.mod | 7 +- go.sum | 3 + 6 files changed, 244 insertions(+), 10 deletions(-) create mode 100644 exporter/rabbitmqexporter/channel_caching.go diff --git a/exporter/rabbitmqexporter/channel_caching.go b/exporter/rabbitmqexporter/channel_caching.go new file mode 100644 index 000000000000..b6b4c94a27ba --- /dev/null +++ b/exporter/rabbitmqexporter/channel_caching.go @@ -0,0 +1,198 @@ +package rabbitmqexporter + +import ( + amqp "github.com/rabbitmq/amqp091-go" + "go.uber.org/zap" + "sync" + "time" +) + +type connectionConfig struct { + logger *zap.Logger + connectionUrl string + connectionName string + channelPoolSize int + heartbeatInterval time.Duration + connectionTimeout time.Duration + confirmationMode bool + //durable bool // TODO figure out what to do with this +} + +type amqpChannelCacher struct { + logger *zap.Logger + config *connectionConfig + connection *amqp.Connection + connLock *sync.Mutex + cachedChannelPool chan *amqpChannelWrapper + connectionErrors chan *amqp.Error +} + +type amqpChannelWrapper struct { + id int + channel *amqp.Channel + wasHealthy bool + lock *sync.Mutex + logger *zap.Logger +} + +func newAmqpChannelCacher(config *connectionConfig) (*amqpChannelCacher, error) { + acc := &amqpChannelCacher{ + logger: config.logger, + config: config, + connLock: &sync.Mutex{}, + cachedChannelPool: make(chan *amqpChannelWrapper, config.channelPoolSize), + } + + err := acc.connect() + if err != nil { + return nil, err + } + + // Synchronously create and connect to channels + for i := 0; i < config.channelPoolSize; i++ { + acc.cachedChannelPool <- acc.createChannelWrapper(i, config.logger) + } + + return acc, nil +} + +func (acc *amqpChannelCacher) connect() error { + + // Compare, Lock, Recompare Strategy + if acc.connection != nil && !acc.connection.IsClosed() /* <- atomic */ { + acc.logger.Debug("Already connected before acquiring lock") + return nil + } + + acc.connLock.Lock() // Block all but one. + defer acc.connLock.Unlock() + + // Recompare, check if an operation is still necessary after acquiring lock. + if acc.connection != nil && !acc.connection.IsClosed() /* <- atomic */ { + acc.logger.Debug("Already connected after acquiring lock") + return nil + } + + // Proceed with reconnectivity + var amqpConn *amqp.Connection + var err error + + // TODO TLS config + amqpConn, err = amqp.DialConfig(acc.config.connectionUrl, amqp.Config{ + Heartbeat: acc.config.heartbeatInterval, + Dial: amqp.DefaultDial(acc.config.connectionTimeout), + Properties: amqp.Table{ + "connection_name": acc.config.connectionName, + }, + }) + if err != nil { + return err + } + + acc.connection = amqpConn + + // Goal is to lazily restore the connection so this needs to be buffered to avoid blocking on asynchronous amqp errors. + // Also re-create this channel each time because apparently the amqp library can close it + acc.connectionErrors = make(chan *amqp.Error, 1) + acc.connection.NotifyClose(acc.connectionErrors) + + // TODO flow control callback + //acc.Blockers = make(chan amqp.Blocking, 10) + //acc.connection.NotifyBlocked(acc.Blockers) + + return nil +} + +func (acc *amqpChannelCacher) restoreUnhealthyConnection() { + healthy := true + select { + case err := <-acc.connectionErrors: + healthy = false + acc.logger.Debug("Received connection error, will retry restoring unhealthy connection", zap.Error(err)) + default: + break + } + + if !healthy || acc.connection.IsClosed() { + // TODO, consider retrying multiple times with some timeout + if err := acc.connect(); err != nil { + acc.logger.Warn("Failed attempt at restoring unhealthy connection", zap.Error(err)) + } else { + acc.logger.Info("Restored unhealthy connection") + } + } +} + +func (acc *amqpChannelCacher) createChannelWrapper(id int, logger *zap.Logger) *amqpChannelWrapper { + channelWrapper := &amqpChannelWrapper{id: id, logger: logger, lock: &sync.Mutex{}} + channelWrapper.tryReplacingChannel(acc.connection, acc.config.confirmationMode) + return channelWrapper +} + +func (acw *amqpChannelWrapper) tryReplacingChannel(connection *amqp.Connection, confirmAcks bool) { + // TODO consider confirmation mode + + acw.lock.Lock() + defer acw.lock.Unlock() + + if acw.channel != nil { + err := acw.channel.Close() + if err != nil { + acw.logger.Debug("Error closing existing channel", zap.Error(err)) + acw.wasHealthy = false + return + } + } + + var err error + acw.channel, err = connection.Channel() + + if err != nil { + acw.logger.Warn("Channel creation error", zap.Error(err)) + acw.wasHealthy = false + return + } + + if confirmAcks { + err := acw.channel.Confirm(false) + if err != nil { + acw.logger.Debug("Error entering confirm mode", zap.Error(err)) + acw.wasHealthy = false + return + } + } + + // TODO consider error callbacks + //ch.Errors = make(chan *amqp.Error, 100) + //ch.Channel.NotifyClose(ch.Errors) + + acw.wasHealthy = true +} + +func (acc *amqpChannelCacher) getChannelFromPool() *amqpChannelWrapper { + return <-acc.cachedChannelPool +} + +func (acc *amqpChannelCacher) returnChannelToPool(channel *amqpChannelWrapper, isUnhealthy bool) { + if isUnhealthy { + acc.reconnectChannel(channel) + } + acc.cachedChannelPool <- channel + return +} + +func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelWrapper) { + acc.restoreUnhealthyConnection() + channel.tryReplacingChannel(acc.connection, acc.config.confirmationMode) +} + +func (acc *amqpChannelCacher) close() error { + err := acc.connection.Close() + if err != nil { + acc.logger.Debug("Received error from connection.Close()", zap.Error(err)) + if err != amqp.ErrClosed { + return err + } + } + return nil +} diff --git a/exporter/rabbitmqexporter/config.go b/exporter/rabbitmqexporter/config.go index 8a0ff9da8540..25134037b3cc 100644 --- a/exporter/rabbitmqexporter/config.go +++ b/exporter/rabbitmqexporter/config.go @@ -1,4 +1,11 @@ package rabbitmqexporter +import "time" + type config struct { + connectionUrl string + channelPoolSize int + connectionTimeout time.Duration + connectionHeartbeatInterval time.Duration + confirmMode bool } diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index 13ce51fb37f3..b5f01f9cf3d4 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -13,7 +13,9 @@ import ( type rabbitmqExporterFactory struct { } -// NewFactory creates Pulsar exporter factory. +// One connection per exporter definition since the same definition may be used for different telemetry types, resulting in different factory instances +// Channels need to be thread safe + func NewFactory() exporter.Factory { f := &rabbitmqExporterFactory{} return exporter.NewFactory( diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index 7fd48e6aafb2..b4e01ff37e00 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -2,14 +2,37 @@ package rabbitmqexporter import ( "context" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/pdata/plog" ) -type rabbitMqLogsProducer struct{} +type rabbitMqLogsProducer struct { + set component.TelemetrySettings + channelCacher *amqpChannelCacher +} + +func newLogsExporter(conf config, set exporter.CreateSettings) (*rabbitMqLogsProducer, error) { + amqpChannelCacher, err := newExporterChannelCacher(conf, set, "otel-logs") + if err != nil { + return nil, err + } -func newLogsExporter(config config, set exporter.CreateSettings) (*rabbitMqLogsProducer, error) { - return &rabbitMqLogsProducer{}, nil + logsProducer := &rabbitMqLogsProducer{set: set.TelemetrySettings, channelCacher: amqpChannelCacher} + return logsProducer, nil +} + +func newExporterChannelCacher(conf config, set exporter.CreateSettings, connectionName string) (*amqpChannelCacher, error) { + connectionConfig := &connectionConfig{ + logger: set.Logger, + connectionUrl: conf.connectionUrl, + connectionName: connectionName, + channelPoolSize: conf.channelPoolSize, + connectionTimeout: conf.connectionTimeout, + heartbeatInterval: conf.connectionHeartbeatInterval, + confirmationMode: conf.confirmMode, + } + return newAmqpChannelCacher(connectionConfig) } func (e *rabbitMqLogsProducer) logsDataPusher(ctx context.Context, ld plog.Logs) error { @@ -17,6 +40,6 @@ func (e *rabbitMqLogsProducer) logsDataPusher(ctx context.Context, ld plog.Logs) return errs } -//func (e *rabbitMqLogsProducer) Close(context.Context) error { -// return nil -//} +func (e *rabbitMqLogsProducer) Close(context.Context) error { + return e.channelCacher.close() +} diff --git a/go.mod b/go.mod index 9a0120e76d84..1af72cb88d59 100644 --- a/go.mod +++ b/go.mod @@ -164,6 +164,8 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/windowsperfcountersreceiver v0.88.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.88.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zookeeperreceiver v0.88.0 + github.com/rabbitmq/amqp091-go v1.8.1 + go.opentelemetry.io/collector/component v0.88.1-0.20231026220224-6405e152a2d9 go.opentelemetry.io/collector/exporter v0.88.1-0.20231026220224-6405e152a2d9 go.opentelemetry.io/collector/exporter/debugexporter v0.88.1-0.20231026220224-6405e152a2d9 go.opentelemetry.io/collector/exporter/loggingexporter v0.88.1-0.20231026220224-6405e152a2d9 @@ -173,11 +175,13 @@ require ( go.opentelemetry.io/collector/extension/ballastextension v0.88.1-0.20231026220224-6405e152a2d9 go.opentelemetry.io/collector/extension/zpagesextension v0.88.1-0.20231026220224-6405e152a2d9 go.opentelemetry.io/collector/otelcol v0.88.1-0.20231026220224-6405e152a2d9 + go.opentelemetry.io/collector/pdata v1.0.0-rcv0017.0.20231026220224-6405e152a2d9 go.opentelemetry.io/collector/processor v0.88.1-0.20231026220224-6405e152a2d9 go.opentelemetry.io/collector/processor/batchprocessor v0.88.1-0.20231026220224-6405e152a2d9 go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.88.1-0.20231026220224-6405e152a2d9 go.opentelemetry.io/collector/receiver v0.88.1-0.20231026220224-6405e152a2d9 go.opentelemetry.io/collector/receiver/otlpreceiver v0.88.1-0.20231026220224-6405e152a2d9 + go.uber.org/zap v1.26.0 ) require ( @@ -615,7 +619,6 @@ require ( go.mongodb.org/mongo-driver v1.12.1 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/collector v0.88.1-0.20231026220224-6405e152a2d9 // indirect - go.opentelemetry.io/collector/component v0.88.1-0.20231026220224-6405e152a2d9 // indirect go.opentelemetry.io/collector/config/configauth v0.88.1-0.20231026220224-6405e152a2d9 // indirect go.opentelemetry.io/collector/config/configcompression v0.88.1-0.20231026220224-6405e152a2d9 // indirect go.opentelemetry.io/collector/config/configgrpc v0.88.1-0.20231026220224-6405e152a2d9 // indirect @@ -630,7 +633,6 @@ require ( go.opentelemetry.io/collector/consumer v0.88.1-0.20231026220224-6405e152a2d9 // indirect go.opentelemetry.io/collector/extension/auth v0.88.1-0.20231026220224-6405e152a2d9 // indirect go.opentelemetry.io/collector/featuregate v1.0.0-rcv0017.0.20231026220224-6405e152a2d9 // indirect - go.opentelemetry.io/collector/pdata v1.0.0-rcv0017.0.20231026220224-6405e152a2d9 // indirect go.opentelemetry.io/collector/semconv v0.88.1-0.20231026220224-6405e152a2d9 // indirect go.opentelemetry.io/collector/service v0.88.1-0.20231026220224-6405e152a2d9 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect @@ -655,7 +657,6 @@ require ( go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect golang.org/x/mod v0.14.0 // indirect diff --git a/go.sum b/go.sum index 85995804cff0..d9604a3c4d6c 100644 --- a/go.sum +++ b/go.sum @@ -1350,6 +1350,8 @@ github.com/prometheus/prometheus v0.47.2 h1:jWcnuQHz1o1Wu3MZ6nMJDuTI0kU5yJp9pkxh github.com/prometheus/prometheus v0.47.2/go.mod h1:J/bmOSjgH7lFxz2gZhrWEZs2i64vMS+HIuZfmYNhJ/M= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= +github.com/rabbitmq/amqp091-go v1.8.1 h1:RejT1SBUim5doqcL6s7iN6SBmsQqyTgXb1xMlH0h1hA= +github.com/rabbitmq/amqp091-go v1.8.1/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1693,6 +1695,7 @@ go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= From 2a9e75eaacf7eac5626c31e8e413683249f3fc93 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Thu, 23 Nov 2023 12:02:51 -0500 Subject: [PATCH 03/19] adds default config --- exporter/rabbitmqexporter/factory.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index b5f01f9cf3d4..ed864691bf8d 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -8,6 +8,8 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/exporterhelper" + "runtime" + "time" ) type rabbitmqExporterFactory struct { @@ -26,7 +28,13 @@ func NewFactory() exporter.Factory { } func createDefaultConfig() component.Config { - return &config{} + return &config{ + connectionUrl: "amqp://swar8080amqp:swar8080amqp@localhost:5672/", + connectionTimeout: time.Second * 10, + connectionHeartbeatInterval: time.Second * 5, + channelPoolSize: runtime.NumCPU(), + confirmMode: true, + } } func (f *rabbitmqExporterFactory) createLogsExporter( From fbc66cfb6381c674905d50244392b61d6073a334 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Thu, 23 Nov 2023 13:17:15 -0500 Subject: [PATCH 04/19] publishing rough control flow --- exporter/rabbitmqexporter/channel_caching.go | 45 +++++++------ exporter/rabbitmqexporter/config.go | 22 ++++++- exporter/rabbitmqexporter/factory.go | 12 ---- exporter/rabbitmqexporter/marshaler.go | 17 +++++ .../rabbitmqexporter/rabbitmq_exporter.go | 65 +++++++++++++++++-- 5 files changed, 125 insertions(+), 36 deletions(-) create mode 100644 exporter/rabbitmqexporter/marshaler.go diff --git a/exporter/rabbitmqexporter/channel_caching.go b/exporter/rabbitmqexporter/channel_caching.go index b6b4c94a27ba..86adad3d4426 100644 --- a/exporter/rabbitmqexporter/channel_caching.go +++ b/exporter/rabbitmqexporter/channel_caching.go @@ -49,8 +49,8 @@ func newAmqpChannelCacher(config *connectionConfig) (*amqpChannelCacher, error) } // Synchronously create and connect to channels - for i := 0; i < config.channelPoolSize; i++ { - acc.cachedChannelPool <- acc.createChannelWrapper(i, config.logger) + for i := 0; i < acc.config.channelPoolSize; i++ { + acc.cachedChannelPool <- acc.createChannelWrapper(i) } return acc, nil @@ -103,7 +103,7 @@ func (acc *amqpChannelCacher) connect() error { return nil } -func (acc *amqpChannelCacher) restoreUnhealthyConnection() { +func (acc *amqpChannelCacher) restoreConnectionIfUnhealthy() { healthy := true select { case err := <-acc.connectionErrors: @@ -123,13 +123,13 @@ func (acc *amqpChannelCacher) restoreUnhealthyConnection() { } } -func (acc *amqpChannelCacher) createChannelWrapper(id int, logger *zap.Logger) *amqpChannelWrapper { - channelWrapper := &amqpChannelWrapper{id: id, logger: logger, lock: &sync.Mutex{}} +func (acc *amqpChannelCacher) createChannelWrapper(id int) *amqpChannelWrapper { + channelWrapper := &amqpChannelWrapper{id: id, logger: acc.logger, lock: &sync.Mutex{}} channelWrapper.tryReplacingChannel(acc.connection, acc.config.confirmationMode) return channelWrapper } -func (acw *amqpChannelWrapper) tryReplacingChannel(connection *amqp.Connection, confirmAcks bool) { +func (acw *amqpChannelWrapper) tryReplacingChannel(connection *amqp.Connection, confirmAcks bool) error { // TODO consider confirmation mode acw.lock.Lock() @@ -140,7 +140,7 @@ func (acw *amqpChannelWrapper) tryReplacingChannel(connection *amqp.Connection, if err != nil { acw.logger.Debug("Error closing existing channel", zap.Error(err)) acw.wasHealthy = false - return + return err } } @@ -150,7 +150,7 @@ func (acw *amqpChannelWrapper) tryReplacingChannel(connection *amqp.Connection, if err != nil { acw.logger.Warn("Channel creation error", zap.Error(err)) acw.wasHealthy = false - return + return err } if confirmAcks { @@ -158,7 +158,7 @@ func (acw *amqpChannelWrapper) tryReplacingChannel(connection *amqp.Connection, if err != nil { acw.logger.Debug("Error entering confirm mode", zap.Error(err)) acw.wasHealthy = false - return + return err } } @@ -167,23 +167,30 @@ func (acw *amqpChannelWrapper) tryReplacingChannel(connection *amqp.Connection, //ch.Channel.NotifyClose(ch.Errors) acw.wasHealthy = true + return nil } -func (acc *amqpChannelCacher) getChannelFromPool() *amqpChannelWrapper { - return <-acc.cachedChannelPool +func (acc *amqpChannelCacher) requestHealthyChannelFromPool() (*amqpChannelWrapper, error) { + channelWrapper := <-acc.cachedChannelPool + if !channelWrapper.wasHealthy { + err := acc.reconnectChannel(channelWrapper) + if err != nil { + acc.returnChannelToPool(channelWrapper, false) + } + return nil, err + } + return channelWrapper, nil } -func (acc *amqpChannelCacher) returnChannelToPool(channel *amqpChannelWrapper, isUnhealthy bool) { - if isUnhealthy { - acc.reconnectChannel(channel) - } - acc.cachedChannelPool <- channel +func (acc *amqpChannelCacher) returnChannelToPool(channelWrapper *amqpChannelWrapper, wasHealthy bool) { + channelWrapper.wasHealthy = wasHealthy + acc.cachedChannelPool <- channelWrapper return } -func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelWrapper) { - acc.restoreUnhealthyConnection() - channel.tryReplacingChannel(acc.connection, acc.config.confirmationMode) +func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelWrapper) error { + acc.restoreConnectionIfUnhealthy() + return channel.tryReplacingChannel(acc.connection, acc.config.confirmationMode) } func (acc *amqpChannelCacher) close() error { diff --git a/exporter/rabbitmqexporter/config.go b/exporter/rabbitmqexporter/config.go index 25134037b3cc..5d806a40fd8e 100644 --- a/exporter/rabbitmqexporter/config.go +++ b/exporter/rabbitmqexporter/config.go @@ -1,11 +1,31 @@ package rabbitmqexporter -import "time" +import ( + "go.opentelemetry.io/collector/component" + "runtime" + "time" +) type config struct { connectionUrl string channelPoolSize int connectionTimeout time.Duration + publishConfirmationTimeout time.Duration connectionHeartbeatInterval time.Duration confirmMode bool + durable bool + routingKey string +} + +func createDefaultConfig() component.Config { + return &config{ + connectionUrl: "amqp://swar8080amqp:swar8080amqp@localhost:5672/", + connectionTimeout: time.Second * 10, + publishConfirmationTimeout: time.Second * 5, + connectionHeartbeatInterval: time.Second * 3, + channelPoolSize: runtime.NumCPU(), + confirmMode: true, + durable: true, + routingKey: "otel", + } } diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index ed864691bf8d..5289c6360717 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -8,8 +8,6 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/exporterhelper" - "runtime" - "time" ) type rabbitmqExporterFactory struct { @@ -27,16 +25,6 @@ func NewFactory() exporter.Factory { ) } -func createDefaultConfig() component.Config { - return &config{ - connectionUrl: "amqp://swar8080amqp:swar8080amqp@localhost:5672/", - connectionTimeout: time.Second * 10, - connectionHeartbeatInterval: time.Second * 5, - channelPoolSize: runtime.NumCPU(), - confirmMode: true, - } -} - func (f *rabbitmqExporterFactory) createLogsExporter( ctx context.Context, set exporter.CreateSettings, diff --git a/exporter/rabbitmqexporter/marshaler.go b/exporter/rabbitmqexporter/marshaler.go new file mode 100644 index 000000000000..8083e1e9c5bf --- /dev/null +++ b/exporter/rabbitmqexporter/marshaler.go @@ -0,0 +1,17 @@ +package rabbitmqexporter + +import "go.opentelemetry.io/collector/pdata/plog" + +type publishingData struct { + ContentType string + ContentEncoding string + Body []byte +} + +func marshalLogs(logs plog.Logs) (*publishingData, error) { + return &publishingData{ + ContentType: "text/plain", + ContentEncoding: "", + Body: []byte("foo"), + }, nil +} diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index b4e01ff37e00..3051d0500c78 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -2,14 +2,21 @@ package rabbitmqexporter import ( "context" + "errors" + "fmt" + amqp "github.com/rabbitmq/amqp091-go" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" + "time" ) type rabbitMqLogsProducer struct { set component.TelemetrySettings + config config channelCacher *amqpChannelCacher + marshaller func(plog.Logs) (*publishingData, error) } func newLogsExporter(conf config, set exporter.CreateSettings) (*rabbitMqLogsProducer, error) { @@ -18,7 +25,12 @@ func newLogsExporter(conf config, set exporter.CreateSettings) (*rabbitMqLogsPro return nil, err } - logsProducer := &rabbitMqLogsProducer{set: set.TelemetrySettings, channelCacher: amqpChannelCacher} + logsProducer := &rabbitMqLogsProducer{ + set: set.TelemetrySettings, + config: conf, + channelCacher: amqpChannelCacher, + marshaller: marshalLogs, + } return logsProducer, nil } @@ -35,9 +47,54 @@ func newExporterChannelCacher(conf config, set exporter.CreateSettings, connecti return newAmqpChannelCacher(connectionConfig) } -func (e *rabbitMqLogsProducer) logsDataPusher(ctx context.Context, ld plog.Logs) error { - var errs error - return errs +func (e *rabbitMqLogsProducer) logsDataPusher(ctx context.Context, data plog.Logs) error { + e.channelCacher.restoreConnectionIfUnhealthy() + channelWrapper, err := e.channelCacher.requestHealthyChannelFromPool() + + if err != nil { + return err + } + + err, healthyChannel := e.pushData(ctx, data, channelWrapper) + e.channelCacher.returnChannelToPool(channelWrapper, healthyChannel) + return err +} + +func (e *rabbitMqLogsProducer) pushData(ctx context.Context, data plog.Logs, wrapper *amqpChannelWrapper) (err error, healthyChannel bool) { + publishingData, err := e.marshaller(data) + + if err != nil { + return err, true + } + + deliveryMode := amqp.Transient + if e.config.durable { + deliveryMode = amqp.Persistent + } + + confirmation, err := wrapper.channel.PublishWithDeferredConfirmWithContext(ctx, "", e.config.routingKey, false, false, amqp.Publishing{ + Headers: amqp.Table{}, + ContentType: publishingData.ContentType, + ContentEncoding: publishingData.ContentEncoding, + DeliveryMode: deliveryMode, + Body: publishingData.Body, + }) + + select { + case <-confirmation.Done(): + if confirmation.Acked() { + e.set.Logger.Debug("Received ack", zap.Int("channelId", wrapper.id), zap.Uint64("deliveryTag", confirmation.DeliveryTag)) + return nil, true + } + e.set.Logger.Warn("Received nack from rabbitmq publishing confirmation", zap.Uint64("deliveryTag", confirmation.DeliveryTag)) + err := errors.New("received nack from rabbitmq publishing confirmation") + return err, true + + case <-time.After(e.config.publishConfirmationTimeout): + e.set.Logger.Warn("Timeout waiting for publish confirmation", zap.Duration("timeout", e.config.publishConfirmationTimeout), zap.Uint64("deliveryTag", confirmation.DeliveryTag)) + err := fmt.Errorf("timeout waiting for publish confirmation after %s", e.config.publishConfirmationTimeout) + return err, false + } } func (e *rabbitMqLogsProducer) Close(context.Context) error { From 37065f3cd18ae7c1302f6acc0cb3947690d975cf Mon Sep 17 00:00:00 2001 From: swar8080 Date: Fri, 24 Nov 2023 12:17:17 -0500 Subject: [PATCH 05/19] add retry and timeout settings --- exporter/rabbitmqexporter/config.go | 6 ++++++ exporter/rabbitmqexporter/factory.go | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/exporter/rabbitmqexporter/config.go b/exporter/rabbitmqexporter/config.go index 5d806a40fd8e..6a29c4422690 100644 --- a/exporter/rabbitmqexporter/config.go +++ b/exporter/rabbitmqexporter/config.go @@ -2,6 +2,7 @@ package rabbitmqexporter import ( "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/exporter/exporterhelper" "runtime" "time" ) @@ -15,9 +16,13 @@ type config struct { confirmMode bool durable bool routingKey string + retrySettings exporterhelper.RetrySettings } func createDefaultConfig() component.Config { + retrySettings := exporterhelper.RetrySettings{ + Enabled: false, + } return &config{ connectionUrl: "amqp://swar8080amqp:swar8080amqp@localhost:5672/", connectionTimeout: time.Second * 10, @@ -27,5 +32,6 @@ func createDefaultConfig() component.Config { confirmMode: true, durable: true, routingKey: "otel", + retrySettings: retrySettings, } } diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index 5289c6360717..f8d38be7d811 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -36,5 +36,10 @@ func (f *rabbitmqExporterFactory) createLogsExporter( return nil, err } - return exporterhelper.NewLogsExporter(ctx, set, cfg, exp.logsDataPusher) + return exporterhelper.NewLogsExporter( + ctx, + set, + cfg, + exp.logsDataPusher, + exporterhelper.WithRetry(customConfig.retrySettings)) } From bc97fa4eb2ae8dc382bf7732cfde5e3801555ada Mon Sep 17 00:00:00 2001 From: swar8080 Date: Fri, 24 Nov 2023 12:45:16 -0500 Subject: [PATCH 06/19] add actual marshaling --- exporter/rabbitmqexporter/marshaler.go | 28 +++++++++++++++++-- .../rabbitmqexporter/rabbitmq_exporter.go | 6 ++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/exporter/rabbitmqexporter/marshaler.go b/exporter/rabbitmqexporter/marshaler.go index 8083e1e9c5bf..480e96c24195 100644 --- a/exporter/rabbitmqexporter/marshaler.go +++ b/exporter/rabbitmqexporter/marshaler.go @@ -1,6 +1,8 @@ package rabbitmqexporter -import "go.opentelemetry.io/collector/pdata/plog" +import ( + "go.opentelemetry.io/collector/pdata/plog" +) type publishingData struct { ContentType string @@ -8,10 +10,30 @@ type publishingData struct { Body []byte } -func marshalLogs(logs plog.Logs) (*publishingData, error) { +type LogsMarshaler interface { + // Marshal serializes logs into sarama's ProducerMessages + Marshal(logs plog.Logs) (*publishingData, error) +} + +type defaultLogsMarshaler struct { + impl *plog.JSONMarshaler +} + +func newLogMarshaler() LogsMarshaler { + return &defaultLogsMarshaler{ + impl: &plog.JSONMarshaler{}, + } +} + +func (m *defaultLogsMarshaler) Marshal(logs plog.Logs) (*publishingData, error) { + body, err := m.impl.MarshalLogs(logs) + if err != nil { + return nil, err + } + return &publishingData{ ContentType: "text/plain", ContentEncoding: "", - Body: []byte("foo"), + Body: body, }, nil } diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index 3051d0500c78..98ff5317e20b 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -16,7 +16,7 @@ type rabbitMqLogsProducer struct { set component.TelemetrySettings config config channelCacher *amqpChannelCacher - marshaller func(plog.Logs) (*publishingData, error) + marshaller LogsMarshaler } func newLogsExporter(conf config, set exporter.CreateSettings) (*rabbitMqLogsProducer, error) { @@ -29,7 +29,7 @@ func newLogsExporter(conf config, set exporter.CreateSettings) (*rabbitMqLogsPro set: set.TelemetrySettings, config: conf, channelCacher: amqpChannelCacher, - marshaller: marshalLogs, + marshaller: newLogMarshaler(), } return logsProducer, nil } @@ -61,7 +61,7 @@ func (e *rabbitMqLogsProducer) logsDataPusher(ctx context.Context, data plog.Log } func (e *rabbitMqLogsProducer) pushData(ctx context.Context, data plog.Logs, wrapper *amqpChannelWrapper) (err error, healthyChannel bool) { - publishingData, err := e.marshaller(data) + publishingData, err := e.marshaller.Marshal(data) if err != nil { return err, true From f27509c40b8e2591d4d3f736b0293e4a01a2ceb1 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Sat, 25 Nov 2023 10:45:00 -0500 Subject: [PATCH 07/19] handle publishing error --- exporter/rabbitmqexporter/rabbitmq_exporter.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index 98ff5317e20b..73b34f989cbd 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -80,6 +80,10 @@ func (e *rabbitMqLogsProducer) pushData(ctx context.Context, data plog.Logs, wra Body: publishingData.Body, }) + if err != nil { + return err, false + } + select { case <-confirmation.Done(): if confirmation.Acked() { From 8035f7e421d7ff0bb1594c4d24264a09daad4c39 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Sat, 25 Nov 2023 10:54:02 -0500 Subject: [PATCH 08/19] remove placeholder channel error channel code --- exporter/rabbitmqexporter/channel_caching.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/exporter/rabbitmqexporter/channel_caching.go b/exporter/rabbitmqexporter/channel_caching.go index 86adad3d4426..5be12b448263 100644 --- a/exporter/rabbitmqexporter/channel_caching.go +++ b/exporter/rabbitmqexporter/channel_caching.go @@ -162,10 +162,6 @@ func (acw *amqpChannelWrapper) tryReplacingChannel(connection *amqp.Connection, } } - // TODO consider error callbacks - //ch.Errors = make(chan *amqp.Error, 100) - //ch.Channel.NotifyClose(ch.Errors) - acw.wasHealthy = true return nil } From 19cf91ecb0506c8f73ee64367c0efde7d4ac927f Mon Sep 17 00:00:00 2001 From: swar8080 Date: Sat, 25 Nov 2023 10:59:00 -0500 Subject: [PATCH 09/19] rename channel wrapper to channel manager because introducing another layer of wrapping on-top of the amqp client library --- exporter/rabbitmqexporter/channel_caching.go | 40 +++++++++---------- .../rabbitmqexporter/rabbitmq_exporter.go | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/exporter/rabbitmqexporter/channel_caching.go b/exporter/rabbitmqexporter/channel_caching.go index 5be12b448263..928d1a4a6b37 100644 --- a/exporter/rabbitmqexporter/channel_caching.go +++ b/exporter/rabbitmqexporter/channel_caching.go @@ -19,15 +19,15 @@ type connectionConfig struct { } type amqpChannelCacher struct { - logger *zap.Logger - config *connectionConfig - connection *amqp.Connection - connLock *sync.Mutex - cachedChannelPool chan *amqpChannelWrapper - connectionErrors chan *amqp.Error + logger *zap.Logger + config *connectionConfig + connection *amqp.Connection + connLock *sync.Mutex + channelManagerPool chan *amqpChannelManager + connectionErrors chan *amqp.Error } -type amqpChannelWrapper struct { +type amqpChannelManager struct { id int channel *amqp.Channel wasHealthy bool @@ -37,10 +37,10 @@ type amqpChannelWrapper struct { func newAmqpChannelCacher(config *connectionConfig) (*amqpChannelCacher, error) { acc := &amqpChannelCacher{ - logger: config.logger, - config: config, - connLock: &sync.Mutex{}, - cachedChannelPool: make(chan *amqpChannelWrapper, config.channelPoolSize), + logger: config.logger, + config: config, + connLock: &sync.Mutex{}, + channelManagerPool: make(chan *amqpChannelManager, config.channelPoolSize), } err := acc.connect() @@ -50,7 +50,7 @@ func newAmqpChannelCacher(config *connectionConfig) (*amqpChannelCacher, error) // Synchronously create and connect to channels for i := 0; i < acc.config.channelPoolSize; i++ { - acc.cachedChannelPool <- acc.createChannelWrapper(i) + acc.channelManagerPool <- acc.createChannelWrapper(i) } return acc, nil @@ -123,13 +123,13 @@ func (acc *amqpChannelCacher) restoreConnectionIfUnhealthy() { } } -func (acc *amqpChannelCacher) createChannelWrapper(id int) *amqpChannelWrapper { - channelWrapper := &amqpChannelWrapper{id: id, logger: acc.logger, lock: &sync.Mutex{}} +func (acc *amqpChannelCacher) createChannelWrapper(id int) *amqpChannelManager { + channelWrapper := &amqpChannelManager{id: id, logger: acc.logger, lock: &sync.Mutex{}} channelWrapper.tryReplacingChannel(acc.connection, acc.config.confirmationMode) return channelWrapper } -func (acw *amqpChannelWrapper) tryReplacingChannel(connection *amqp.Connection, confirmAcks bool) error { +func (acw *amqpChannelManager) tryReplacingChannel(connection *amqp.Connection, confirmAcks bool) error { // TODO consider confirmation mode acw.lock.Lock() @@ -166,8 +166,8 @@ func (acw *amqpChannelWrapper) tryReplacingChannel(connection *amqp.Connection, return nil } -func (acc *amqpChannelCacher) requestHealthyChannelFromPool() (*amqpChannelWrapper, error) { - channelWrapper := <-acc.cachedChannelPool +func (acc *amqpChannelCacher) requestHealthyChannelFromPool() (*amqpChannelManager, error) { + channelWrapper := <-acc.channelManagerPool if !channelWrapper.wasHealthy { err := acc.reconnectChannel(channelWrapper) if err != nil { @@ -178,13 +178,13 @@ func (acc *amqpChannelCacher) requestHealthyChannelFromPool() (*amqpChannelWrapp return channelWrapper, nil } -func (acc *amqpChannelCacher) returnChannelToPool(channelWrapper *amqpChannelWrapper, wasHealthy bool) { +func (acc *amqpChannelCacher) returnChannelToPool(channelWrapper *amqpChannelManager, wasHealthy bool) { channelWrapper.wasHealthy = wasHealthy - acc.cachedChannelPool <- channelWrapper + acc.channelManagerPool <- channelWrapper return } -func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelWrapper) error { +func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelManager) error { acc.restoreConnectionIfUnhealthy() return channel.tryReplacingChannel(acc.connection, acc.config.confirmationMode) } diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index 73b34f989cbd..eda330503218 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -60,7 +60,7 @@ func (e *rabbitMqLogsProducer) logsDataPusher(ctx context.Context, data plog.Log return err } -func (e *rabbitMqLogsProducer) pushData(ctx context.Context, data plog.Logs, wrapper *amqpChannelWrapper) (err error, healthyChannel bool) { +func (e *rabbitMqLogsProducer) pushData(ctx context.Context, data plog.Logs, wrapper *amqpChannelManager) (err error, healthyChannel bool) { publishingData, err := e.marshaller.Marshal(data) if err != nil { From 6344ac19f0bd04ee28d10c450a7fa8cba7f31895 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Sat, 25 Nov 2023 11:42:06 -0500 Subject: [PATCH 10/19] finish wrapping amqp libray --- exporter/rabbitmqexporter/amqp_client.go | 75 ++++++++++++++++++++ exporter/rabbitmqexporter/channel_caching.go | 1 + 2 files changed, 76 insertions(+) create mode 100644 exporter/rabbitmqexporter/amqp_client.go diff --git a/exporter/rabbitmqexporter/amqp_client.go b/exporter/rabbitmqexporter/amqp_client.go new file mode 100644 index 000000000000..0c28d6337ae9 --- /dev/null +++ b/exporter/rabbitmqexporter/amqp_client.go @@ -0,0 +1,75 @@ +package rabbitmqexporter + +import ( + "context" + amqp "github.com/rabbitmq/amqp091-go" +) + +type AmqpDialer interface { + DialConfig(url string, config amqp.Config) (WrappedConnection, error) +} + +type WrappedConnection interface { + Channel() (WrappedChannel, error) + IsClosed() bool + NotifyClose(receiver chan *amqp.Error) chan *amqp.Error + Close() error +} + +type WrappedChannel interface { + Confirm(noWait bool) error + PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (*amqp.DeferredConfirmation, error) +} + +type amqpDialer struct{} + +func newAmqpDialer() AmqpDialer { + return &amqpDialer{} +} + +type wrappedConnection struct { + connection *amqp.Connection +} + +type wrappedChannel struct { + channel *amqp.Channel +} + +func (*amqpDialer) DialConfig(url string, config amqp.Config) (WrappedConnection, error) { + connection, err := amqp.DialConfig(url, config) + if err != nil { + return nil, err + } + + return &wrappedConnection{ + connection: connection, + }, nil +} + +func (c *wrappedConnection) Channel() (WrappedChannel, error) { + channel, err := c.connection.Channel() + if err != nil { + return nil, err + } + return &wrappedChannel{channel: channel}, nil +} + +func (c *wrappedConnection) IsClosed() bool { + return c.connection.IsClosed() +} + +func (c *wrappedConnection) NotifyClose(receiver chan *amqp.Error) chan *amqp.Error { + return c.connection.NotifyClose(receiver) +} + +func (c *wrappedConnection) Close() error { + return c.connection.Close() +} + +func (c *wrappedChannel) Confirm(noWait bool) error { + return c.channel.Confirm(noWait) +} + +func (c *wrappedChannel) PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (*amqp.DeferredConfirmation, error) { + return c.channel.PublishWithDeferredConfirmWithContext(ctx, exchange, key, mandatory, immediate, msg) +} diff --git a/exporter/rabbitmqexporter/channel_caching.go b/exporter/rabbitmqexporter/channel_caching.go index 928d1a4a6b37..7b620812e63b 100644 --- a/exporter/rabbitmqexporter/channel_caching.go +++ b/exporter/rabbitmqexporter/channel_caching.go @@ -21,6 +21,7 @@ type connectionConfig struct { type amqpChannelCacher struct { logger *zap.Logger config *connectionConfig + client *AmqpDialer connection *amqp.Connection connLock *sync.Mutex channelManagerPool chan *amqpChannelManager From df32ab194cc1b9fb9d4bea1bdee4566e917561c0 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Sat, 25 Nov 2023 12:05:06 -0500 Subject: [PATCH 11/19] wires up amqp client --- exporter/rabbitmqexporter/amqp_client.go | 22 ++++++++++++++----- exporter/rabbitmqexporter/channel_caching.go | 17 +++++++------- exporter/rabbitmqexporter/factory.go | 2 +- .../rabbitmqexporter/rabbitmq_exporter.go | 9 ++++---- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/exporter/rabbitmqexporter/amqp_client.go b/exporter/rabbitmqexporter/amqp_client.go index 0c28d6337ae9..823912da539d 100644 --- a/exporter/rabbitmqexporter/amqp_client.go +++ b/exporter/rabbitmqexporter/amqp_client.go @@ -3,10 +3,13 @@ package rabbitmqexporter import ( "context" amqp "github.com/rabbitmq/amqp091-go" + "net" + "time" ) -type AmqpDialer interface { +type AmqpClient interface { DialConfig(url string, config amqp.Config) (WrappedConnection, error) + DefaultDial(connectionTimeout time.Duration) func(network, addr string) (net.Conn, error) } type WrappedConnection interface { @@ -19,12 +22,13 @@ type WrappedConnection interface { type WrappedChannel interface { Confirm(noWait bool) error PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (*amqp.DeferredConfirmation, error) + Close() error } -type amqpDialer struct{} +type amqpClient struct{} -func newAmqpDialer() AmqpDialer { - return &amqpDialer{} +func newAmqpClient() AmqpClient { + return &amqpClient{} } type wrappedConnection struct { @@ -35,7 +39,7 @@ type wrappedChannel struct { channel *amqp.Channel } -func (*amqpDialer) DialConfig(url string, config amqp.Config) (WrappedConnection, error) { +func (*amqpClient) DialConfig(url string, config amqp.Config) (WrappedConnection, error) { connection, err := amqp.DialConfig(url, config) if err != nil { return nil, err @@ -46,6 +50,10 @@ func (*amqpDialer) DialConfig(url string, config amqp.Config) (WrappedConnection }, nil } +func (*amqpClient) DefaultDial(connectionTimeout time.Duration) func(network, addr string) (net.Conn, error) { + return amqp.DefaultDial(connectionTimeout) +} + func (c *wrappedConnection) Channel() (WrappedChannel, error) { channel, err := c.connection.Channel() if err != nil { @@ -73,3 +81,7 @@ func (c *wrappedChannel) Confirm(noWait bool) error { func (c *wrappedChannel) PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (*amqp.DeferredConfirmation, error) { return c.channel.PublishWithDeferredConfirmWithContext(ctx, exchange, key, mandatory, immediate, msg) } + +func (c *wrappedChannel) Close() error { + return c.channel.Close() +} diff --git a/exporter/rabbitmqexporter/channel_caching.go b/exporter/rabbitmqexporter/channel_caching.go index 7b620812e63b..a2f073f0ad31 100644 --- a/exporter/rabbitmqexporter/channel_caching.go +++ b/exporter/rabbitmqexporter/channel_caching.go @@ -21,8 +21,8 @@ type connectionConfig struct { type amqpChannelCacher struct { logger *zap.Logger config *connectionConfig - client *AmqpDialer - connection *amqp.Connection + amqpClient AmqpClient + connection WrappedConnection connLock *sync.Mutex channelManagerPool chan *amqpChannelManager connectionErrors chan *amqp.Error @@ -30,16 +30,17 @@ type amqpChannelCacher struct { type amqpChannelManager struct { id int - channel *amqp.Channel + channel WrappedChannel wasHealthy bool lock *sync.Mutex logger *zap.Logger } -func newAmqpChannelCacher(config *connectionConfig) (*amqpChannelCacher, error) { +func newAmqpChannelCacher(config *connectionConfig, amqpClient AmqpClient) (*amqpChannelCacher, error) { acc := &amqpChannelCacher{ logger: config.logger, config: config, + amqpClient: amqpClient, connLock: &sync.Mutex{}, channelManagerPool: make(chan *amqpChannelManager, config.channelPoolSize), } @@ -75,13 +76,13 @@ func (acc *amqpChannelCacher) connect() error { } // Proceed with reconnectivity - var amqpConn *amqp.Connection + var amqpConn WrappedConnection var err error // TODO TLS config - amqpConn, err = amqp.DialConfig(acc.config.connectionUrl, amqp.Config{ + amqpConn, err = acc.amqpClient.DialConfig(acc.config.connectionUrl, amqp.Config{ Heartbeat: acc.config.heartbeatInterval, - Dial: amqp.DefaultDial(acc.config.connectionTimeout), + Dial: acc.amqpClient.DefaultDial(acc.config.connectionTimeout), Properties: amqp.Table{ "connection_name": acc.config.connectionName, }, @@ -130,7 +131,7 @@ func (acc *amqpChannelCacher) createChannelWrapper(id int) *amqpChannelManager { return channelWrapper } -func (acw *amqpChannelManager) tryReplacingChannel(connection *amqp.Connection, confirmAcks bool) error { +func (acw *amqpChannelManager) tryReplacingChannel(connection WrappedConnection, confirmAcks bool) error { // TODO consider confirmation mode acw.lock.Lock() diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index f8d38be7d811..15cec65ca581 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -31,7 +31,7 @@ func (f *rabbitmqExporterFactory) createLogsExporter( cfg component.Config, ) (exporter.Logs, error) { customConfig := *(cfg.(*config)) - exp, err := newLogsExporter(customConfig, set) + exp, err := newLogsExporter(customConfig, set, newAmqpClient()) if err != nil { return nil, err } diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index eda330503218..3e9381739589 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -15,12 +15,13 @@ import ( type rabbitMqLogsProducer struct { set component.TelemetrySettings config config + amqpClient *AmqpClient channelCacher *amqpChannelCacher marshaller LogsMarshaler } -func newLogsExporter(conf config, set exporter.CreateSettings) (*rabbitMqLogsProducer, error) { - amqpChannelCacher, err := newExporterChannelCacher(conf, set, "otel-logs") +func newLogsExporter(conf config, set exporter.CreateSettings, amqpClient AmqpClient) (*rabbitMqLogsProducer, error) { + amqpChannelCacher, err := newExporterChannelCacher(conf, set, "otel-logs", amqpClient) if err != nil { return nil, err } @@ -34,7 +35,7 @@ func newLogsExporter(conf config, set exporter.CreateSettings) (*rabbitMqLogsPro return logsProducer, nil } -func newExporterChannelCacher(conf config, set exporter.CreateSettings, connectionName string) (*amqpChannelCacher, error) { +func newExporterChannelCacher(conf config, set exporter.CreateSettings, connectionName string, client AmqpClient) (*amqpChannelCacher, error) { connectionConfig := &connectionConfig{ logger: set.Logger, connectionUrl: conf.connectionUrl, @@ -44,7 +45,7 @@ func newExporterChannelCacher(conf config, set exporter.CreateSettings, connecti heartbeatInterval: conf.connectionHeartbeatInterval, confirmationMode: conf.confirmMode, } - return newAmqpChannelCacher(connectionConfig) + return newAmqpChannelCacher(connectionConfig, client) } func (e *rabbitMqLogsProducer) logsDataPusher(ctx context.Context, data plog.Logs) error { From 9f91eac8632c7fc8e9298f275cbc8e5f41ab1532 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Sat, 25 Nov 2023 18:21:30 -0500 Subject: [PATCH 12/19] mock plumbing iteration 1 compiles --- exporter/rabbitmqexporter/channel_caching.go | 12 +-- exporter/rabbitmqexporter/factory.go | 2 +- .../rabbitmq_exporter_test.go | 74 +++++++++++++++++++ 3 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 exporter/rabbitmqexporter/rabbitmq_exporter_test.go diff --git a/exporter/rabbitmqexporter/channel_caching.go b/exporter/rabbitmqexporter/channel_caching.go index a2f073f0ad31..749089ced1a4 100644 --- a/exporter/rabbitmqexporter/channel_caching.go +++ b/exporter/rabbitmqexporter/channel_caching.go @@ -93,14 +93,14 @@ func (acc *amqpChannelCacher) connect() error { acc.connection = amqpConn - // Goal is to lazily restore the connection so this needs to be buffered to avoid blocking on asynchronous amqp errors. + // Goal is to lazily restore the mockConnection so this needs to be buffered to avoid blocking on asynchronous amqp errors. // Also re-create this channel each time because apparently the amqp library can close it acc.connectionErrors = make(chan *amqp.Error, 1) acc.connection.NotifyClose(acc.connectionErrors) // TODO flow control callback //acc.Blockers = make(chan amqp.Blocking, 10) - //acc.connection.NotifyBlocked(acc.Blockers) + //acc.mockConnection.NotifyBlocked(acc.Blockers) return nil } @@ -110,7 +110,7 @@ func (acc *amqpChannelCacher) restoreConnectionIfUnhealthy() { select { case err := <-acc.connectionErrors: healthy = false - acc.logger.Debug("Received connection error, will retry restoring unhealthy connection", zap.Error(err)) + acc.logger.Debug("Received mockConnection error, will retry restoring unhealthy mockConnection", zap.Error(err)) default: break } @@ -118,9 +118,9 @@ func (acc *amqpChannelCacher) restoreConnectionIfUnhealthy() { if !healthy || acc.connection.IsClosed() { // TODO, consider retrying multiple times with some timeout if err := acc.connect(); err != nil { - acc.logger.Warn("Failed attempt at restoring unhealthy connection", zap.Error(err)) + acc.logger.Warn("Failed attempt at restoring unhealthy mockConnection", zap.Error(err)) } else { - acc.logger.Info("Restored unhealthy connection") + acc.logger.Info("Restored unhealthy mockConnection") } } } @@ -194,7 +194,7 @@ func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelManager) erro func (acc *amqpChannelCacher) close() error { err := acc.connection.Close() if err != nil { - acc.logger.Debug("Received error from connection.Close()", zap.Error(err)) + acc.logger.Debug("Received error from mockConnection.Close()", zap.Error(err)) if err != amqp.ErrClosed { return err } diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index 15cec65ca581..07d3b18d321b 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -13,7 +13,7 @@ import ( type rabbitmqExporterFactory struct { } -// One connection per exporter definition since the same definition may be used for different telemetry types, resulting in different factory instances +// One mockConnection per exporter definition since the same definition may be used for different telemetry types, resulting in different factory instances // Channels need to be thread safe func NewFactory() exporter.Factory { diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go new file mode 100644 index 000000000000..4510895ee24e --- /dev/null +++ b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go @@ -0,0 +1,74 @@ +package rabbitmqexporter + +import ( + "context" + "fmt" + amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/collector/exporter/exportertest" + "net" + "time" +) + +type dialFunc func() (WrappedConnection, error) +type mockClient struct { + dialImpl dialFunc +} + +type channelFunc func() (WrappedChannel, error) +type mockConnection struct { + channelImpl channelFunc + isClosed bool +} + +type publishFunc func() (*amqp.DeferredConfirmation, error) +type mockChannel struct { + published []amqp.Publishing + publishImpl publishFunc +} + +func (c *mockClient) DialConfig(url string, config amqp.Config) (WrappedConnection, error) { + return c.dialImpl() +} + +func (*mockClient) DefaultDial(connectionTimeout time.Duration) func(network, addr string) (net.Conn, error) { + return nil +} + +func (c *mockConnection) Channel() (WrappedChannel, error) { + return c.channelImpl() +} + +func (c *mockConnection) IsClosed() bool { + return c.isClosed +} + +func (c *mockConnection) NotifyClose(receiver chan *amqp.Error) chan *amqp.Error { + return make(chan *amqp.Error) +} + +func (c *mockConnection) Close() error { + return nil +} + +func (c *mockChannel) Confirm(noWait bool) error { + return nil +} + +func (c *mockChannel) PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (*amqp.DeferredConfirmation, error) { + c.published = append(c.published, msg) + return c.publishImpl() +} + +func (c *mockChannel) Close() error { + return nil +} + +func test() { + var dialImpl dialFunc = func() (WrappedConnection, error) { return nil, nil } + mockClient := &mockClient{dialImpl: dialImpl} + cfg := createDefaultConfig() + customConfig := *(cfg.(*config)) + + rabbitMqExporter, err := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), mockClient) + fmt.Println(rabbitMqExporter, err) +} From 11fb933de2e5976692846954497fd1da55f79540 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Mon, 27 Nov 2023 21:20:35 -0500 Subject: [PATCH 13/19] unit tests with mocked client --- exporter/fileexporter/config.go | 2 + exporter/rabbitmqexporter/amqp_client.go | 33 ++- exporter/rabbitmqexporter/channel_caching.go | 2 +- .../rabbitmqexporter/rabbitmq_exporter.go | 6 +- .../rabbitmq_exporter_test.go | 273 +++++++++++++++++- 5 files changed, 301 insertions(+), 15 deletions(-) diff --git a/exporter/fileexporter/config.go b/exporter/fileexporter/config.go index 587055a9f274..2b29ec92e2de 100644 --- a/exporter/fileexporter/config.go +++ b/exporter/fileexporter/config.go @@ -5,6 +5,7 @@ package fileexporter // import "github.com/open-telemetry/opentelemetry-collecto import ( "errors" + "fmt" "time" "go.opentelemetry.io/collector/component" @@ -67,6 +68,7 @@ var _ component.Config = (*Config)(nil) // Validate checks if the exporter configuration is valid func (cfg *Config) Validate() error { + fmt.Println("this is a test", cfg) if cfg.Path == "" { return errors.New("path must be non-empty") } diff --git a/exporter/rabbitmqexporter/amqp_client.go b/exporter/rabbitmqexporter/amqp_client.go index 823912da539d..a1b1905637cb 100644 --- a/exporter/rabbitmqexporter/amqp_client.go +++ b/exporter/rabbitmqexporter/amqp_client.go @@ -21,10 +21,16 @@ type WrappedConnection interface { type WrappedChannel interface { Confirm(noWait bool) error - PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (*amqp.DeferredConfirmation, error) + PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (WrappedDeferredConfirmation, error) Close() error } +type WrappedDeferredConfirmation interface { + DeliveryTag() uint64 + Done() chan struct{} + Acked() bool +} + type amqpClient struct{} func newAmqpClient() AmqpClient { @@ -39,6 +45,10 @@ type wrappedChannel struct { channel *amqp.Channel } +type wrappedDeferredConfirmation struct { + confirmation *amqp.DeferredConfirmation +} + func (*amqpClient) DialConfig(url string, config amqp.Config) (WrappedConnection, error) { connection, err := amqp.DialConfig(url, config) if err != nil { @@ -78,10 +88,27 @@ func (c *wrappedChannel) Confirm(noWait bool) error { return c.channel.Confirm(noWait) } -func (c *wrappedChannel) PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (*amqp.DeferredConfirmation, error) { - return c.channel.PublishWithDeferredConfirmWithContext(ctx, exchange, key, mandatory, immediate, msg) +func (c *wrappedChannel) PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (WrappedDeferredConfirmation, error) { + confirmation, err := c.channel.PublishWithDeferredConfirmWithContext(ctx, exchange, key, mandatory, immediate, msg) + if err != nil { + return nil, err + } + + return &wrappedDeferredConfirmation{confirmation: confirmation}, nil } func (c *wrappedChannel) Close() error { return c.channel.Close() } + +func (c *wrappedDeferredConfirmation) DeliveryTag() uint64 { + return c.confirmation.DeliveryTag +} + +func (c *wrappedDeferredConfirmation) Done() chan struct{} { + return c.Done() +} + +func (c *wrappedDeferredConfirmation) Acked() bool { + return c.Acked() +} diff --git a/exporter/rabbitmqexporter/channel_caching.go b/exporter/rabbitmqexporter/channel_caching.go index 749089ced1a4..28b194fab476 100644 --- a/exporter/rabbitmqexporter/channel_caching.go +++ b/exporter/rabbitmqexporter/channel_caching.go @@ -174,8 +174,8 @@ func (acc *amqpChannelCacher) requestHealthyChannelFromPool() (*amqpChannelManag err := acc.reconnectChannel(channelWrapper) if err != nil { acc.returnChannelToPool(channelWrapper, false) + return nil, err } - return nil, err } return channelWrapper, nil } diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index 3e9381739589..e0131b6e0890 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -88,15 +88,15 @@ func (e *rabbitMqLogsProducer) pushData(ctx context.Context, data plog.Logs, wra select { case <-confirmation.Done(): if confirmation.Acked() { - e.set.Logger.Debug("Received ack", zap.Int("channelId", wrapper.id), zap.Uint64("deliveryTag", confirmation.DeliveryTag)) + e.set.Logger.Debug("Received ack", zap.Int("channelId", wrapper.id), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) return nil, true } - e.set.Logger.Warn("Received nack from rabbitmq publishing confirmation", zap.Uint64("deliveryTag", confirmation.DeliveryTag)) + e.set.Logger.Warn("Received nack from rabbitmq publishing confirmation", zap.Uint64("deliveryTag", confirmation.DeliveryTag())) err := errors.New("received nack from rabbitmq publishing confirmation") return err, true case <-time.After(e.config.publishConfirmationTimeout): - e.set.Logger.Warn("Timeout waiting for publish confirmation", zap.Duration("timeout", e.config.publishConfirmationTimeout), zap.Uint64("deliveryTag", confirmation.DeliveryTag)) + e.set.Logger.Warn("Timeout waiting for publish confirmation", zap.Duration("timeout", e.config.publishConfirmationTimeout), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) err := fmt.Errorf("timeout waiting for publish confirmation after %s", e.config.publishConfirmationTimeout) return err, false } diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go index 4510895ee24e..7d69a3d5223d 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go @@ -2,10 +2,16 @@ package rabbitmqexporter import ( "context" - "fmt" + "errors" amqp "github.com/rabbitmq/amqp091-go" + "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/exporter/exportertest" + "go.opentelemetry.io/collector/pdata/plog" "net" + "runtime" + "sync" + "sync/atomic" + "testing" "time" ) @@ -20,10 +26,17 @@ type mockConnection struct { isClosed bool } -type publishFunc func() (*amqp.DeferredConfirmation, error) +type publishFunc func() (WrappedDeferredConfirmation, error) type mockChannel struct { published []amqp.Publishing publishImpl publishFunc + confirmMode bool +} + +type mockDeferredConfirmation struct { + deliveryTag uint64 + done chan struct{} + acked bool } func (c *mockClient) DialConfig(url string, config amqp.Config) (WrappedConnection, error) { @@ -51,10 +64,14 @@ func (c *mockConnection) Close() error { } func (c *mockChannel) Confirm(noWait bool) error { + c.confirmMode = true return nil } -func (c *mockChannel) PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (*amqp.DeferredConfirmation, error) { +func (c *mockChannel) PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (WrappedDeferredConfirmation, error) { + if c.published == nil { + c.published = make([]amqp.Publishing, 0) + } c.published = append(c.published, msg) return c.publishImpl() } @@ -63,12 +80,252 @@ func (c *mockChannel) Close() error { return nil } -func test() { - var dialImpl dialFunc = func() (WrappedConnection, error) { return nil, nil } - mockClient := &mockClient{dialImpl: dialImpl} +func (c *mockDeferredConfirmation) DeliveryTag() uint64 { + return c.deliveryTag +} + +func (c *mockDeferredConfirmation) Done() chan struct{} { + return c.done +} + +func (c *mockDeferredConfirmation) Acked() bool { + return c.acked +} + +func TestPublishLogsHappyPath(t *testing.T) { + client, _, channel, confirmation := buildMocks() + + cfg := createDefaultConfig() + customConfig := *(cfg.(*config)) + customConfig.publishConfirmationTimeout = time.Second + + rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) + + confirmAsynchronously(confirmation, true, time.Millisecond*50) + err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + + assert.NoError(t, err) + assert.Equal(t, 1, len(channel.published)) +} + +func TestPublishLogsConcurrently(t *testing.T) { + if runtime.NumCPU() <= 1 { + t.Skip("Requires multiple CPUs") + } + + client, connection, channel, confirmation := buildMocks() + _, _, channel2, confirmation2 := buildMocks() + confirmations := []*mockDeferredConfirmation{confirmation, confirmation2} + + channelIndex := atomic.Uint32{} + channels := []*mockChannel{channel, channel2} + connection.channelImpl = func() (WrappedChannel, error) { + res := channels[channelIndex.Load()] + channelIndex.Add(1) + return res, nil + } + + cfg := createDefaultConfig() + customConfig := *(cfg.(*config)) + customConfig.channelPoolSize = 2 + customConfig.publishConfirmationTimeout = time.Second + + rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) + wg := sync.WaitGroup{} + wg.Add(2) + + for i := 0; i < 2; i++ { + go func(ii int) { + confirmAsynchronously(confirmations[ii], true, time.Millisecond*100) + err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + assert.NoError(t, err) + wg.Done() + }(i) + } + + start := time.Now() + wg.Wait() + duration := time.Since(start) + + assert.Equal(t, 1, len(channel.published)) + assert.Equal(t, 1, len(channel2.published)) + assert.GreaterOrEqual(t, duration, 100*time.Millisecond) + assert.Less(t, duration, 200*time.Millisecond) +} + +func TestPublishLogsWithTimeout(t *testing.T) { + client, _, channel, _ := buildMocks() + cfg := createDefaultConfig() + customConfig := *(cfg.(*config)) + customConfig.publishConfirmationTimeout = time.Millisecond * 100 + + rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) + err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + + assert.ErrorContains(t, err, "timeout waiting for publish confirmation after 100ms") + assert.Equal(t, 1, len(channel.published)) +} + +func TestPublishLogsWithError(t *testing.T) { + client, _, channel, _ := buildMocks() + cfg := createDefaultConfig() + customConfig := *(cfg.(*config)) + + channel.publishImpl = func() (WrappedDeferredConfirmation, error) { + return nil, errors.New("expected publish error") + } + + rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) + err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + + assert.ErrorContains(t, err, "expected publish error") +} + +func TestRecoveryFromBadConnectionAfterPublishError(t *testing.T) { + client, badConnection, badChannel, _ := buildMocks() + _, goodConnection, goodChannel, goodConfirm := buildMocks() + + badChannel.publishImpl = func() (WrappedDeferredConfirmation, error) { + badConnection.isClosed = true + return nil, errors.New("expected publish error") + } + connectionQueue := []*mockConnection{badConnection, goodConnection} + index := atomic.Uint32{} + client.dialImpl = func() (WrappedConnection, error) { + res := connectionQueue[index.Load()] + index.Add(1) + return res, nil + } + + cfg := createDefaultConfig() + customConfig := *(cfg.(*config)) + customConfig.channelPoolSize = 1 + + rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) + + err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + assert.ErrorContains(t, err, "expected publish error") + assert.Equal(t, 1, len(badChannel.published)) + + confirmAsynchronously(goodConfirm, true, time.Millisecond*50) + err = rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + assert.NoError(t, err) + assert.Equal(t, 1, len(goodChannel.published)) +} + +func TestRecoveryFromBadChannelAfterPublishError(t *testing.T) { + client, connection, badChannel, _ := buildMocks() + _, _, goodChannel, goodConfirm := buildMocks() + + badChannel.publishImpl = func() (WrappedDeferredConfirmation, error) { + return nil, errors.New("expected publish error") + } + channelQueue := []*mockChannel{badChannel, goodChannel} + index := atomic.Uint32{} + connection.channelImpl = func() (WrappedChannel, error) { + res := channelQueue[index.Load()] + index.Add(1) + return res, nil + } + + cfg := createDefaultConfig() + customConfig := *(cfg.(*config)) + customConfig.channelPoolSize = 1 + + rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) + + err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + assert.ErrorContains(t, err, "expected publish error") + + confirmAsynchronously(goodConfirm, true, time.Millisecond*50) + err = rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + assert.NoError(t, err) + assert.Equal(t, 1, len(goodChannel.published)) +} + +func TestPublishLogsWithNack(t *testing.T) { + client, _, channel, confirmation := buildMocks() cfg := createDefaultConfig() customConfig := *(cfg.(*config)) + customConfig.publishConfirmationTimeout = time.Second + + confirmAsynchronously(confirmation, false, time.Millisecond*50) + rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) + err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + + assert.ErrorContains(t, err, "received nack from rabbitmq publishing confirmation") + assert.Equal(t, 1, len(channel.published)) +} + +func TestPublishConfirmationFlag(t *testing.T) { + tests := []struct { + confirmMode bool + }{ + {confirmMode: true}, + {confirmMode: false}, + } + + for _, tt := range tests { + cfg := createDefaultConfig() + customConfig := *(cfg.(*config)) + customConfig.confirmMode = tt.confirmMode + + client, _, channel, _ := buildMocks() + + _, _ = newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) + + assert.Equal(t, tt.confirmMode, channel.confirmMode) + } +} + +func TestPublishDurableFlag(t *testing.T) { + tests := []struct { + durable bool + deliveryMode uint8 + }{ + {durable: true, deliveryMode: amqp.Persistent}, + {durable: false, deliveryMode: amqp.Transient}, + } + + for _, tt := range tests { + cfg := createDefaultConfig() + customConfig := *(cfg.(*config)) + customConfig.durable = tt.durable + + client, _, channel, confirmation := buildMocks() + + exporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) + confirmAsynchronously(confirmation, true, time.Millisecond*50) + err := exporter.logsDataPusher(context.Background(), plog.NewLogs()) + + assert.NoError(t, err) + assert.Equal(t, 1, len(channel.published)) + assert.Equal(t, tt.deliveryMode, channel.published[0].DeliveryMode) + } +} + +func buildMocks() (*mockClient, *mockConnection, *mockChannel, *mockDeferredConfirmation) { + confirmation := mockDeferredConfirmation{ + deliveryTag: 1, + done: make(chan struct{}), + acked: false, + } + mockChannel := mockChannel{publishImpl: func() (WrappedDeferredConfirmation, error) { + return &confirmation, nil + }} + mockConnection := mockConnection{isClosed: false, channelImpl: func() (WrappedChannel, error) { + return &mockChannel, nil + }} + mockClient := mockClient{dialImpl: func() (WrappedConnection, error) { + return &mockConnection, nil + }} + + return &mockClient, &mockConnection, &mockChannel, &confirmation +} - rabbitMqExporter, err := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), mockClient) - fmt.Println(rabbitMqExporter, err) +func confirmAsynchronously(confirmation *mockDeferredConfirmation, acked bool, delay time.Duration) { + time.AfterFunc(delay, func() { + confirmation.acked = acked + confirmation.done <- struct{}{} + }) } From 0f7aabada574cefadfcc2267e22b2eba6300c747 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Wed, 29 Nov 2023 10:03:22 -0500 Subject: [PATCH 14/19] working when run with ocb builder --- exporter/fileexporter/config.go | 2 - exporter/rabbitmqexporter/Makefile | 5 - exporter/rabbitmqexporter/amqp_client.go | 8 +- exporter/rabbitmqexporter/channel_caching.go | 20 +- exporter/rabbitmqexporter/factory.go | 2 +- exporter/rabbitmqexporter/go.mod | 49 ++++ exporter/rabbitmqexporter/go.sum | 221 ++++++++++++++++++ .../internal/metadata/generated_status.go | 2 - .../rabbitmqexporter/rabbitmq_exporter.go | 3 +- .../rabbitmq_exporter_test.go | 2 +- 10 files changed, 289 insertions(+), 25 deletions(-) create mode 100644 exporter/rabbitmqexporter/go.mod create mode 100644 exporter/rabbitmqexporter/go.sum diff --git a/exporter/fileexporter/config.go b/exporter/fileexporter/config.go index 2b29ec92e2de..587055a9f274 100644 --- a/exporter/fileexporter/config.go +++ b/exporter/fileexporter/config.go @@ -5,7 +5,6 @@ package fileexporter // import "github.com/open-telemetry/opentelemetry-collecto import ( "errors" - "fmt" "time" "go.opentelemetry.io/collector/component" @@ -68,7 +67,6 @@ var _ component.Config = (*Config)(nil) // Validate checks if the exporter configuration is valid func (cfg *Config) Validate() error { - fmt.Println("this is a test", cfg) if cfg.Path == "" { return errors.New("path must be non-empty") } diff --git a/exporter/rabbitmqexporter/Makefile b/exporter/rabbitmqexporter/Makefile index fb455d4977dd..ded7a36092dc 100644 --- a/exporter/rabbitmqexporter/Makefile +++ b/exporter/rabbitmqexporter/Makefile @@ -1,6 +1 @@ include ../../Makefile.Common - -# Remove "-race" from the default set of test arguments. -# exporter/pulsarexporter tests are failing with the -race check. -# See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/12518 -GOTEST_OPT = -v -timeout 300s --tags=$(GO_BUILD_TAGS) diff --git a/exporter/rabbitmqexporter/amqp_client.go b/exporter/rabbitmqexporter/amqp_client.go index a1b1905637cb..0c1012cfedab 100644 --- a/exporter/rabbitmqexporter/amqp_client.go +++ b/exporter/rabbitmqexporter/amqp_client.go @@ -27,7 +27,7 @@ type WrappedChannel interface { type WrappedDeferredConfirmation interface { DeliveryTag() uint64 - Done() chan struct{} + Done() <-chan struct{} Acked() bool } @@ -105,10 +105,10 @@ func (c *wrappedDeferredConfirmation) DeliveryTag() uint64 { return c.confirmation.DeliveryTag } -func (c *wrappedDeferredConfirmation) Done() chan struct{} { - return c.Done() +func (c *wrappedDeferredConfirmation) Done() <-chan struct{} { + return c.confirmation.Done() } func (c *wrappedDeferredConfirmation) Acked() bool { - return c.Acked() + return c.confirmation.Acked() } diff --git a/exporter/rabbitmqexporter/channel_caching.go b/exporter/rabbitmqexporter/channel_caching.go index 28b194fab476..d3280c6b9d84 100644 --- a/exporter/rabbitmqexporter/channel_caching.go +++ b/exporter/rabbitmqexporter/channel_caching.go @@ -59,6 +59,7 @@ func newAmqpChannelCacher(config *connectionConfig, amqpClient AmqpClient) (*amq } func (acc *amqpChannelCacher) connect() error { + acc.logger.Debug("Connecting to rabbitmq") // Compare, Lock, Recompare Strategy if acc.connection != nil && !acc.connection.IsClosed() /* <- atomic */ { @@ -93,14 +94,14 @@ func (acc *amqpChannelCacher) connect() error { acc.connection = amqpConn - // Goal is to lazily restore the mockConnection so this needs to be buffered to avoid blocking on asynchronous amqp errors. + // Goal is to lazily restore the connection so this needs to be buffered to avoid blocking on asynchronous amqp errors. // Also re-create this channel each time because apparently the amqp library can close it acc.connectionErrors = make(chan *amqp.Error, 1) acc.connection.NotifyClose(acc.connectionErrors) // TODO flow control callback //acc.Blockers = make(chan amqp.Blocking, 10) - //acc.mockConnection.NotifyBlocked(acc.Blockers) + //acc.connection.NotifyBlocked(acc.Blockers) return nil } @@ -110,7 +111,7 @@ func (acc *amqpChannelCacher) restoreConnectionIfUnhealthy() { select { case err := <-acc.connectionErrors: healthy = false - acc.logger.Debug("Received mockConnection error, will retry restoring unhealthy mockConnection", zap.Error(err)) + acc.logger.Debug("Received connection error, will retry restoring unhealthy connection", zap.Error(err)) default: break } @@ -118,22 +119,23 @@ func (acc *amqpChannelCacher) restoreConnectionIfUnhealthy() { if !healthy || acc.connection.IsClosed() { // TODO, consider retrying multiple times with some timeout if err := acc.connect(); err != nil { - acc.logger.Warn("Failed attempt at restoring unhealthy mockConnection", zap.Error(err)) + acc.logger.Warn("Failed attempt at restoring unhealthy connection", zap.Error(err)) } else { - acc.logger.Info("Restored unhealthy mockConnection") + acc.logger.Info("Restored unhealthy connection") } } } func (acc *amqpChannelCacher) createChannelWrapper(id int) *amqpChannelManager { channelWrapper := &amqpChannelManager{id: id, logger: acc.logger, lock: &sync.Mutex{}} - channelWrapper.tryReplacingChannel(acc.connection, acc.config.confirmationMode) + err := channelWrapper.tryReplacingChannel(acc.connection, acc.config.confirmationMode) + if err != nil { + acc.logger.Warn("Error creating channel wrapper's channel", zap.Error(err)) + } return channelWrapper } func (acw *amqpChannelManager) tryReplacingChannel(connection WrappedConnection, confirmAcks bool) error { - // TODO consider confirmation mode - acw.lock.Lock() defer acw.lock.Unlock() @@ -194,7 +196,7 @@ func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelManager) erro func (acc *amqpChannelCacher) close() error { err := acc.connection.Close() if err != nil { - acc.logger.Debug("Received error from mockConnection.Close()", zap.Error(err)) + acc.logger.Warn("Error closing connection", zap.Error(err)) if err != amqp.ErrClosed { return err } diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index 07d3b18d321b..7a101f77a318 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package rabbitmqexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/pulsarexporter" +package rabbitmqexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter" import ( "context" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter/internal/metadata" diff --git a/exporter/rabbitmqexporter/go.mod b/exporter/rabbitmqexporter/go.mod new file mode 100644 index 000000000000..7c4d4daf589d --- /dev/null +++ b/exporter/rabbitmqexporter/go.mod @@ -0,0 +1,49 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter + +go 1.20 + +require ( + github.com/rabbitmq/amqp091-go v1.9.0 + github.com/stretchr/testify v1.8.4 + go.opentelemetry.io/collector/component v0.90.0 + go.opentelemetry.io/collector/exporter v0.90.0 + go.opentelemetry.io/collector/pdata v1.0.0 + go.uber.org/zap v1.26.0 +) + +require ( + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.0.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/collector v0.90.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.90.0 // indirect + go.opentelemetry.io/collector/confmap v0.90.0 // indirect + go.opentelemetry.io/collector/consumer v0.90.0 // indirect + go.opentelemetry.io/collector/extension v0.90.0 // indirect + go.opentelemetry.io/collector/featuregate v1.0.0 // indirect + go.opentelemetry.io/collector/receiver v0.90.0 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.59.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/exporter/rabbitmqexporter/go.sum b/exporter/rabbitmqexporter/go.sum new file mode 100644 index 000000000000..068421135ed5 --- /dev/null +++ b/exporter/rabbitmqexporter/go.sum @@ -0,0 +1,221 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= +github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY= +github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= +github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo= +github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/collector v0.90.0 h1:Wyiiu+78tV5zZDvza9hvZu6FgOkFqURNzPHkKcI+asw= +go.opentelemetry.io/collector v0.90.0/go.mod h1:qRhpGBXozKMn+7SiniobhcZ0AbCSWdYqL+XM3gnwejQ= +go.opentelemetry.io/collector/component v0.90.0 h1:rufHQfFpZQ4mc30GAsW6JSm1DvJWCGjoyw+dNXpgTV8= +go.opentelemetry.io/collector/component v0.90.0/go.mod h1:+WX5h5I98AwL256AdFvn8EpPZ02Q+UrKo9AdI8LLfuQ= +go.opentelemetry.io/collector/config/configtelemetry v0.90.0 h1:1exyNLDVSSkdDLUoVTLiy5pfzB7ak802JhOaOTOe2Zo= +go.opentelemetry.io/collector/config/configtelemetry v0.90.0/go.mod h1:+LAXM5WFMW/UbTlAuSs6L/W72WC+q8TBJt/6z39FPOU= +go.opentelemetry.io/collector/confmap v0.90.0 h1:vU+759p/4zLeet8yeI8uVq4+xCm73/5K8t2Tx0MzX/8= +go.opentelemetry.io/collector/confmap v0.90.0/go.mod h1:uxV+fZ85kG31oovL6Cl3fAMQ3RRPwUvfAbbA9WT1Yhk= +go.opentelemetry.io/collector/consumer v0.90.0 h1:5cScUTbv9PIvI/bKTa2GbAn/LAMwcg2znAb0UKfhVy4= +go.opentelemetry.io/collector/consumer v0.90.0/go.mod h1:mh/eEA0UClEtgQMDICQVL7oSylgbskFfueBO0i5HkSQ= +go.opentelemetry.io/collector/exporter v0.90.0 h1:XMpOprVtAG3yryRQ8fw6a9TZsL7t9jzCrYCvhHrtBw4= +go.opentelemetry.io/collector/exporter v0.90.0/go.mod h1:QNhT4FZ/698dDybYM2FbfguNvh2S7M7jKiDvFLntWOw= +go.opentelemetry.io/collector/extension v0.90.0 h1:NDvZneZEapDeOD195kDZiEW8IUb2SimmkI/CrKfy+WA= +go.opentelemetry.io/collector/extension v0.90.0/go.mod h1:vUiLcJQuM04CuyCf6AbjW8OCSeINSU4242GPVzTzX9w= +go.opentelemetry.io/collector/featuregate v1.0.0 h1:5MGqe2v5zxaoo73BUOvUTunftX5J8RGrbFsC2Ha7N3g= +go.opentelemetry.io/collector/featuregate v1.0.0/go.mod h1:xGbRuw+GbutRtVVSEy3YR2yuOlEyiUMhN2M9DJljgqY= +go.opentelemetry.io/collector/pdata v1.0.0 h1:ECP2jnLztewsHmL1opL8BeMtWVc7/oSlKNhfY9jP8ec= +go.opentelemetry.io/collector/pdata v1.0.0/go.mod h1:TsDFgs4JLNG7t6x9D8kGswXUz4mme+MyNChHx8zSF6k= +go.opentelemetry.io/collector/receiver v0.90.0 h1:cVp1s9c9kSfn5ZTXb9o8nlZnLEgs2gutEYzty5+eUEI= +go.opentelemetry.io/collector/receiver v0.90.0/go.mod h1:oRmH7WKmkJo7tgc7odoArLXjrz2TZdcw7pco0KRZjWo= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/exporters/prometheus v0.44.0 h1:08qeJgaPC0YEBu2PQMbqU3rogTlyzpjhCI2b58Yn00w= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/exporter/rabbitmqexporter/internal/metadata/generated_status.go b/exporter/rabbitmqexporter/internal/metadata/generated_status.go index 8c5420db4dd0..dc0918efd9a7 100644 --- a/exporter/rabbitmqexporter/internal/metadata/generated_status.go +++ b/exporter/rabbitmqexporter/internal/metadata/generated_status.go @@ -8,7 +8,5 @@ import ( const ( Type = "rabbitmq" - TracesStability = component.StabilityLevelDevelopment - MetricsStability = component.StabilityLevelDevelopment LogsStability = component.StabilityLevelDevelopment ) diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index e0131b6e0890..067d3111c37e 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -73,7 +73,8 @@ func (e *rabbitMqLogsProducer) pushData(ctx context.Context, data plog.Logs, wra deliveryMode = amqp.Persistent } - confirmation, err := wrapper.channel.PublishWithDeferredConfirmWithContext(ctx, "", e.config.routingKey, false, false, amqp.Publishing{ + // TODO handle case where message doesn't get routed to any queues and is returned (when configured with mandatory = true) + confirmation, err := wrapper.channel.PublishWithDeferredConfirmWithContext(ctx, "amq.direct", e.config.routingKey, false, false, amqp.Publishing{ Headers: amqp.Table{}, ContentType: publishingData.ContentType, ContentEncoding: publishingData.ContentEncoding, diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go index 7d69a3d5223d..81438aaf4cea 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go @@ -84,7 +84,7 @@ func (c *mockDeferredConfirmation) DeliveryTag() uint64 { return c.deliveryTag } -func (c *mockDeferredConfirmation) Done() chan struct{} { +func (c *mockDeferredConfirmation) Done() <-chan struct{} { return c.done } From df4c063ad322f1990b889293da640e4f01d9219c Mon Sep 17 00:00:00 2001 From: swar8080 Date: Wed, 29 Nov 2023 16:50:57 -0500 Subject: [PATCH 15/19] code clean-up and comments --- exporter/rabbitmqexporter/amqp_client.go | 6 + .../{channel_caching.go => channel_cacher.go} | 107 ++++++++++-------- exporter/rabbitmqexporter/config.go | 1 + exporter/rabbitmqexporter/factory.go | 3 - exporter/rabbitmqexporter/marshaler.go | 2 +- .../rabbitmqexporter/rabbitmq_exporter.go | 1 + .../rabbitmq_exporter_test.go | 8 +- 7 files changed, 75 insertions(+), 53 deletions(-) rename exporter/rabbitmqexporter/{channel_caching.go => channel_cacher.go} (77%) diff --git a/exporter/rabbitmqexporter/amqp_client.go b/exporter/rabbitmqexporter/amqp_client.go index 0c1012cfedab..b40dccbafa9c 100644 --- a/exporter/rabbitmqexporter/amqp_client.go +++ b/exporter/rabbitmqexporter/amqp_client.go @@ -7,6 +7,7 @@ import ( "time" ) +// AmqpClient Wrapper around the AMQP client library calls to allow mocking in tests type AmqpClient interface { DialConfig(url string, config amqp.Config) (WrappedConnection, error) DefaultDial(connectionTimeout time.Duration) func(network, addr string) (net.Conn, error) @@ -22,6 +23,7 @@ type WrappedConnection interface { type WrappedChannel interface { Confirm(noWait bool) error PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (WrappedDeferredConfirmation, error) + IsClosed() bool Close() error } @@ -97,6 +99,10 @@ func (c *wrappedChannel) PublishWithDeferredConfirmWithContext(ctx context.Conte return &wrappedDeferredConfirmation{confirmation: confirmation}, nil } +func (c *wrappedChannel) IsClosed() bool { + return c.channel.IsClosed() +} + func (c *wrappedChannel) Close() error { return c.channel.Close() } diff --git a/exporter/rabbitmqexporter/channel_caching.go b/exporter/rabbitmqexporter/channel_cacher.go similarity index 77% rename from exporter/rabbitmqexporter/channel_caching.go rename to exporter/rabbitmqexporter/channel_cacher.go index d3280c6b9d84..2453f3d7a484 100644 --- a/exporter/rabbitmqexporter/channel_caching.go +++ b/exporter/rabbitmqexporter/channel_cacher.go @@ -7,17 +7,14 @@ import ( "time" ) -type connectionConfig struct { - logger *zap.Logger - connectionUrl string - connectionName string - channelPoolSize int - heartbeatInterval time.Duration - connectionTimeout time.Duration - confirmationMode bool - //durable bool // TODO figure out what to do with this -} - +// The channel cacher is used to gain exclusive access to an AMQP 0.9.1 channel for publishing queue messages, since they are not thread safe. +// AMQP channels are like logical connections that share a single physical connection to reduce rabbitmq resource usage. +// +// This struct wraps a single connection and a fixed number of AMQP channels. +// Re-using channels between batches avoids a few network calls for closing/recreating the channel (which took ~50ms in local testing with the nearest AWS region). +// It also handles lazily recreating unhealthy connections/channels when a new batch comes in. + +// Much of this implementation is adapted from https://github.com/houseofcat/turbocookedrabbit's connection pool. type amqpChannelCacher struct { logger *zap.Logger config *connectionConfig @@ -28,6 +25,16 @@ type amqpChannelCacher struct { connectionErrors chan *amqp.Error } +type connectionConfig struct { + logger *zap.Logger + connectionUrl string + connectionName string + channelPoolSize int + heartbeatInterval time.Duration + connectionTimeout time.Duration + confirmationMode bool +} + type amqpChannelManager struct { id int channel WrappedChannel @@ -50,9 +57,9 @@ func newAmqpChannelCacher(config *connectionConfig, amqpClient AmqpClient) (*amq return nil, err } - // Synchronously create and connect to channels + // Synchronously try creating and connecting to channels for i := 0; i < acc.config.channelPoolSize; i++ { - acc.channelManagerPool <- acc.createChannelWrapper(i) + acc.channelManagerPool <- acc.createChannelManager(i) } return acc, nil @@ -61,32 +68,31 @@ func newAmqpChannelCacher(config *connectionConfig, amqpClient AmqpClient) (*amq func (acc *amqpChannelCacher) connect() error { acc.logger.Debug("Connecting to rabbitmq") - // Compare, Lock, Recompare Strategy - if acc.connection != nil && !acc.connection.IsClosed() /* <- atomic */ { + if acc.connection != nil && !acc.connection.IsClosed() { acc.logger.Debug("Already connected before acquiring lock") return nil } - acc.connLock.Lock() // Block all but one. + acc.connLock.Lock() defer acc.connLock.Unlock() // Recompare, check if an operation is still necessary after acquiring lock. - if acc.connection != nil && !acc.connection.IsClosed() /* <- atomic */ { + if acc.connection != nil && !acc.connection.IsClosed() { acc.logger.Debug("Already connected after acquiring lock") return nil } - // Proceed with reconnectivity + // Proceed with re-connecting var amqpConn WrappedConnection var err error - // TODO TLS config amqpConn, err = acc.amqpClient.DialConfig(acc.config.connectionUrl, amqp.Config{ Heartbeat: acc.config.heartbeatInterval, Dial: acc.amqpClient.DefaultDial(acc.config.connectionTimeout), Properties: amqp.Table{ "connection_name": acc.config.connectionName, }, + // TODO TLS config }) if err != nil { return err @@ -99,7 +105,7 @@ func (acc *amqpChannelCacher) connect() error { acc.connectionErrors = make(chan *amqp.Error, 1) acc.connection.NotifyClose(acc.connectionErrors) - // TODO flow control callback + // TODO handle upstream flow control throttling publishing //acc.Blockers = make(chan amqp.Blocking, 10) //acc.connection.NotifyBlocked(acc.Blockers) @@ -117,7 +123,13 @@ func (acc *amqpChannelCacher) restoreConnectionIfUnhealthy() { } if !healthy || acc.connection.IsClosed() { - // TODO, consider retrying multiple times with some timeout + if !acc.connection.IsClosed() { + err := acc.close() + if err != nil { + acc.logger.Warn("Error closing unhealthy connection", zap.Error(err)) + } + } + if err := acc.connect(); err != nil { acc.logger.Warn("Failed attempt at restoring unhealthy connection", zap.Error(err)) } else { @@ -126,20 +138,43 @@ func (acc *amqpChannelCacher) restoreConnectionIfUnhealthy() { } } -func (acc *amqpChannelCacher) createChannelWrapper(id int) *amqpChannelManager { +func (acc *amqpChannelCacher) createChannelManager(id int) *amqpChannelManager { channelWrapper := &amqpChannelManager{id: id, logger: acc.logger, lock: &sync.Mutex{}} err := channelWrapper.tryReplacingChannel(acc.connection, acc.config.confirmationMode) if err != nil { - acc.logger.Warn("Error creating channel wrapper's channel", zap.Error(err)) + acc.logger.Warn("Error creating channel manager's channel", zap.Error(err)) } return channelWrapper } +func (acc *amqpChannelCacher) requestHealthyChannelFromPool() (*amqpChannelManager, error) { + channelWrapper := <-acc.channelManagerPool + if !channelWrapper.wasHealthy { + err := acc.reconnectChannel(channelWrapper) + if err != nil { + acc.returnChannelToPool(channelWrapper, false) + return nil, err + } + } + return channelWrapper, nil +} + +func (acc *amqpChannelCacher) returnChannelToPool(channelWrapper *amqpChannelManager, wasHealthy bool) { + channelWrapper.wasHealthy = wasHealthy + acc.channelManagerPool <- channelWrapper + return +} + +func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelManager) error { + acc.restoreConnectionIfUnhealthy() + return channel.tryReplacingChannel(acc.connection, acc.config.confirmationMode) +} + func (acw *amqpChannelManager) tryReplacingChannel(connection WrappedConnection, confirmAcks bool) error { acw.lock.Lock() defer acw.lock.Unlock() - if acw.channel != nil { + if acw.channel != nil && !acw.channel.IsClosed() { err := acw.channel.Close() if err != nil { acw.logger.Debug("Error closing existing channel", zap.Error(err)) @@ -150,7 +185,6 @@ func (acw *amqpChannelManager) tryReplacingChannel(connection WrappedConnection, var err error acw.channel, err = connection.Channel() - if err != nil { acw.logger.Warn("Channel creation error", zap.Error(err)) acw.wasHealthy = false @@ -170,29 +204,6 @@ func (acw *amqpChannelManager) tryReplacingChannel(connection WrappedConnection, return nil } -func (acc *amqpChannelCacher) requestHealthyChannelFromPool() (*amqpChannelManager, error) { - channelWrapper := <-acc.channelManagerPool - if !channelWrapper.wasHealthy { - err := acc.reconnectChannel(channelWrapper) - if err != nil { - acc.returnChannelToPool(channelWrapper, false) - return nil, err - } - } - return channelWrapper, nil -} - -func (acc *amqpChannelCacher) returnChannelToPool(channelWrapper *amqpChannelManager, wasHealthy bool) { - channelWrapper.wasHealthy = wasHealthy - acc.channelManagerPool <- channelWrapper - return -} - -func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelManager) error { - acc.restoreConnectionIfUnhealthy() - return channel.tryReplacingChannel(acc.connection, acc.config.confirmationMode) -} - func (acc *amqpChannelCacher) close() error { err := acc.connection.Close() if err != nil { diff --git a/exporter/rabbitmqexporter/config.go b/exporter/rabbitmqexporter/config.go index 6a29c4422690..3e889d216da5 100644 --- a/exporter/rabbitmqexporter/config.go +++ b/exporter/rabbitmqexporter/config.go @@ -8,6 +8,7 @@ import ( ) type config struct { + //TODO add json struct tags connectionUrl string channelPoolSize int connectionTimeout time.Duration diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index 7a101f77a318..6e76d3a77db2 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -13,9 +13,6 @@ import ( type rabbitmqExporterFactory struct { } -// One mockConnection per exporter definition since the same definition may be used for different telemetry types, resulting in different factory instances -// Channels need to be thread safe - func NewFactory() exporter.Factory { f := &rabbitmqExporterFactory{} return exporter.NewFactory( diff --git a/exporter/rabbitmqexporter/marshaler.go b/exporter/rabbitmqexporter/marshaler.go index 480e96c24195..1f0d2093eb69 100644 --- a/exporter/rabbitmqexporter/marshaler.go +++ b/exporter/rabbitmqexporter/marshaler.go @@ -11,7 +11,6 @@ type publishingData struct { } type LogsMarshaler interface { - // Marshal serializes logs into sarama's ProducerMessages Marshal(logs plog.Logs) (*publishingData, error) } @@ -20,6 +19,7 @@ type defaultLogsMarshaler struct { } func newLogMarshaler() LogsMarshaler { + // TODO revisit which encoding(s) to use return &defaultLogsMarshaler{ impl: &plog.JSONMarshaler{}, } diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index 067d3111c37e..0de22becdfb3 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -48,6 +48,7 @@ func newExporterChannelCacher(conf config, set exporter.CreateSettings, connecti return newAmqpChannelCacher(connectionConfig, client) } +// TODO implement and add code re-use for other types of telemetry func (e *rabbitMqLogsProducer) logsDataPusher(ctx context.Context, data plog.Logs) error { e.channelCacher.restoreConnectionIfUnhealthy() channelWrapper, err := e.channelCacher.requestHealthyChannelFromPool() diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go index 81438aaf4cea..2d4553ceb2b2 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go @@ -30,6 +30,7 @@ type publishFunc func() (WrappedDeferredConfirmation, error) type mockChannel struct { published []amqp.Publishing publishImpl publishFunc + isClosed bool confirmMode bool } @@ -76,7 +77,12 @@ func (c *mockChannel) PublishWithDeferredConfirmWithContext(ctx context.Context, return c.publishImpl() } +func (c *mockChannel) IsClosed() bool { + return c.isClosed +} + func (c *mockChannel) Close() error { + c.isClosed = true return nil } @@ -310,7 +316,7 @@ func buildMocks() (*mockClient, *mockConnection, *mockChannel, *mockDeferredConf done: make(chan struct{}), acked: false, } - mockChannel := mockChannel{publishImpl: func() (WrappedDeferredConfirmation, error) { + mockChannel := mockChannel{isClosed: false, publishImpl: func() (WrappedDeferredConfirmation, error) { return &confirmation, nil }} mockConnection := mockConnection{isClosed: false, channelImpl: func() (WrappedChannel, error) { From ba24dad6f27f6d0c65491ee2d98d99a0e2a4cc34 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Fri, 1 Dec 2023 15:58:34 -0500 Subject: [PATCH 16/19] move amqp_client to internal directory for compliance with the checkapi rules --- exporter/rabbitmqexporter/channel_cacher.go | 13 ++++---- exporter/rabbitmqexporter/factory.go | 3 +- .../{ => internal}/amqp_client.go | 2 +- .../rabbitmqexporter/rabbitmq_exporter.go | 7 +++-- .../rabbitmq_exporter_test.go | 31 ++++++++++--------- 5 files changed, 30 insertions(+), 26 deletions(-) rename exporter/rabbitmqexporter/{ => internal}/amqp_client.go (99%) diff --git a/exporter/rabbitmqexporter/channel_cacher.go b/exporter/rabbitmqexporter/channel_cacher.go index 2453f3d7a484..4f4d7c4178d6 100644 --- a/exporter/rabbitmqexporter/channel_cacher.go +++ b/exporter/rabbitmqexporter/channel_cacher.go @@ -1,6 +1,7 @@ package rabbitmqexporter import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter/internal" amqp "github.com/rabbitmq/amqp091-go" "go.uber.org/zap" "sync" @@ -18,8 +19,8 @@ import ( type amqpChannelCacher struct { logger *zap.Logger config *connectionConfig - amqpClient AmqpClient - connection WrappedConnection + amqpClient internal.AmqpClient + connection internal.WrappedConnection connLock *sync.Mutex channelManagerPool chan *amqpChannelManager connectionErrors chan *amqp.Error @@ -37,13 +38,13 @@ type connectionConfig struct { type amqpChannelManager struct { id int - channel WrappedChannel + channel internal.WrappedChannel wasHealthy bool lock *sync.Mutex logger *zap.Logger } -func newAmqpChannelCacher(config *connectionConfig, amqpClient AmqpClient) (*amqpChannelCacher, error) { +func newAmqpChannelCacher(config *connectionConfig, amqpClient internal.AmqpClient) (*amqpChannelCacher, error) { acc := &amqpChannelCacher{ logger: config.logger, config: config, @@ -83,7 +84,7 @@ func (acc *amqpChannelCacher) connect() error { } // Proceed with re-connecting - var amqpConn WrappedConnection + var amqpConn internal.WrappedConnection var err error amqpConn, err = acc.amqpClient.DialConfig(acc.config.connectionUrl, amqp.Config{ @@ -170,7 +171,7 @@ func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelManager) erro return channel.tryReplacingChannel(acc.connection, acc.config.confirmationMode) } -func (acw *amqpChannelManager) tryReplacingChannel(connection WrappedConnection, confirmAcks bool) error { +func (acw *amqpChannelManager) tryReplacingChannel(connection internal.WrappedConnection, confirmAcks bool) error { acw.lock.Lock() defer acw.lock.Unlock() diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index 6e76d3a77db2..ac6dfb243c19 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -4,6 +4,7 @@ package rabbitmqexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter" import ( "context" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter/internal" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter/internal/metadata" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter" @@ -28,7 +29,7 @@ func (f *rabbitmqExporterFactory) createLogsExporter( cfg component.Config, ) (exporter.Logs, error) { customConfig := *(cfg.(*config)) - exp, err := newLogsExporter(customConfig, set, newAmqpClient()) + exp, err := newLogsExporter(customConfig, set, internal.newAmqpClient()) if err != nil { return nil, err } diff --git a/exporter/rabbitmqexporter/amqp_client.go b/exporter/rabbitmqexporter/internal/amqp_client.go similarity index 99% rename from exporter/rabbitmqexporter/amqp_client.go rename to exporter/rabbitmqexporter/internal/amqp_client.go index b40dccbafa9c..e22dbdb257da 100644 --- a/exporter/rabbitmqexporter/amqp_client.go +++ b/exporter/rabbitmqexporter/internal/amqp_client.go @@ -1,4 +1,4 @@ -package rabbitmqexporter +package internal import ( "context" diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index 0de22becdfb3..757d6b3feeaa 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter/internal" amqp "github.com/rabbitmq/amqp091-go" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter" @@ -15,12 +16,12 @@ import ( type rabbitMqLogsProducer struct { set component.TelemetrySettings config config - amqpClient *AmqpClient + amqpClient *internal.AmqpClient channelCacher *amqpChannelCacher marshaller LogsMarshaler } -func newLogsExporter(conf config, set exporter.CreateSettings, amqpClient AmqpClient) (*rabbitMqLogsProducer, error) { +func newLogsExporter(conf config, set exporter.CreateSettings, amqpClient internal.AmqpClient) (*rabbitMqLogsProducer, error) { amqpChannelCacher, err := newExporterChannelCacher(conf, set, "otel-logs", amqpClient) if err != nil { return nil, err @@ -35,7 +36,7 @@ func newLogsExporter(conf config, set exporter.CreateSettings, amqpClient AmqpCl return logsProducer, nil } -func newExporterChannelCacher(conf config, set exporter.CreateSettings, connectionName string, client AmqpClient) (*amqpChannelCacher, error) { +func newExporterChannelCacher(conf config, set exporter.CreateSettings, connectionName string, client internal.AmqpClient) (*amqpChannelCacher, error) { connectionConfig := &connectionConfig{ logger: set.Logger, connectionUrl: conf.connectionUrl, diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go index 2d4553ceb2b2..8e6fe6a53e47 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go @@ -3,6 +3,7 @@ package rabbitmqexporter import ( "context" "errors" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter/internal" amqp "github.com/rabbitmq/amqp091-go" "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/exporter/exportertest" @@ -15,18 +16,18 @@ import ( "time" ) -type dialFunc func() (WrappedConnection, error) +type dialFunc func() (internal.WrappedConnection, error) type mockClient struct { dialImpl dialFunc } -type channelFunc func() (WrappedChannel, error) +type channelFunc func() (internal.WrappedChannel, error) type mockConnection struct { channelImpl channelFunc isClosed bool } -type publishFunc func() (WrappedDeferredConfirmation, error) +type publishFunc func() (internal.WrappedDeferredConfirmation, error) type mockChannel struct { published []amqp.Publishing publishImpl publishFunc @@ -40,7 +41,7 @@ type mockDeferredConfirmation struct { acked bool } -func (c *mockClient) DialConfig(url string, config amqp.Config) (WrappedConnection, error) { +func (c *mockClient) DialConfig(url string, config amqp.Config) (internal.WrappedConnection, error) { return c.dialImpl() } @@ -48,7 +49,7 @@ func (*mockClient) DefaultDial(connectionTimeout time.Duration) func(network, ad return nil } -func (c *mockConnection) Channel() (WrappedChannel, error) { +func (c *mockConnection) Channel() (internal.WrappedChannel, error) { return c.channelImpl() } @@ -69,7 +70,7 @@ func (c *mockChannel) Confirm(noWait bool) error { return nil } -func (c *mockChannel) PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (WrappedDeferredConfirmation, error) { +func (c *mockChannel) PublishWithDeferredConfirmWithContext(ctx context.Context, exchange, key string, mandatory, immediate bool, msg amqp.Publishing) (internal.WrappedDeferredConfirmation, error) { if c.published == nil { c.published = make([]amqp.Publishing, 0) } @@ -125,7 +126,7 @@ func TestPublishLogsConcurrently(t *testing.T) { channelIndex := atomic.Uint32{} channels := []*mockChannel{channel, channel2} - connection.channelImpl = func() (WrappedChannel, error) { + connection.channelImpl = func() (internal.WrappedChannel, error) { res := channels[channelIndex.Load()] channelIndex.Add(1) return res, nil @@ -177,7 +178,7 @@ func TestPublishLogsWithError(t *testing.T) { cfg := createDefaultConfig() customConfig := *(cfg.(*config)) - channel.publishImpl = func() (WrappedDeferredConfirmation, error) { + channel.publishImpl = func() (internal.WrappedDeferredConfirmation, error) { return nil, errors.New("expected publish error") } @@ -191,13 +192,13 @@ func TestRecoveryFromBadConnectionAfterPublishError(t *testing.T) { client, badConnection, badChannel, _ := buildMocks() _, goodConnection, goodChannel, goodConfirm := buildMocks() - badChannel.publishImpl = func() (WrappedDeferredConfirmation, error) { + badChannel.publishImpl = func() (internal.WrappedDeferredConfirmation, error) { badConnection.isClosed = true return nil, errors.New("expected publish error") } connectionQueue := []*mockConnection{badConnection, goodConnection} index := atomic.Uint32{} - client.dialImpl = func() (WrappedConnection, error) { + client.dialImpl = func() (internal.WrappedConnection, error) { res := connectionQueue[index.Load()] index.Add(1) return res, nil @@ -223,12 +224,12 @@ func TestRecoveryFromBadChannelAfterPublishError(t *testing.T) { client, connection, badChannel, _ := buildMocks() _, _, goodChannel, goodConfirm := buildMocks() - badChannel.publishImpl = func() (WrappedDeferredConfirmation, error) { + badChannel.publishImpl = func() (internal.WrappedDeferredConfirmation, error) { return nil, errors.New("expected publish error") } channelQueue := []*mockChannel{badChannel, goodChannel} index := atomic.Uint32{} - connection.channelImpl = func() (WrappedChannel, error) { + connection.channelImpl = func() (internal.WrappedChannel, error) { res := channelQueue[index.Load()] index.Add(1) return res, nil @@ -316,13 +317,13 @@ func buildMocks() (*mockClient, *mockConnection, *mockChannel, *mockDeferredConf done: make(chan struct{}), acked: false, } - mockChannel := mockChannel{isClosed: false, publishImpl: func() (WrappedDeferredConfirmation, error) { + mockChannel := mockChannel{isClosed: false, publishImpl: func() (internal.WrappedDeferredConfirmation, error) { return &confirmation, nil }} - mockConnection := mockConnection{isClosed: false, channelImpl: func() (WrappedChannel, error) { + mockConnection := mockConnection{isClosed: false, channelImpl: func() (internal.WrappedChannel, error) { return &mockChannel, nil }} - mockClient := mockClient{dialImpl: func() (WrappedConnection, error) { + mockClient := mockClient{dialImpl: func() (internal.WrappedConnection, error) { return &mockConnection, nil }} From 904d87fa0db16e29602addee3559dcbc1216d4c2 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Fri, 1 Dec 2023 16:22:26 -0500 Subject: [PATCH 17/19] create re-usable publishing struct for the different exporter types --- exporter/rabbitmqexporter/factory.go | 4 +- .../rabbitmqexporter/internal/amqp_client.go | 2 +- .../rabbitmqexporter/rabbitmq_exporter.go | 86 +++++++++++-------- .../rabbitmq_exporter_test.go | 20 ++--- 4 files changed, 63 insertions(+), 49 deletions(-) diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index ac6dfb243c19..3042bd8e0987 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -29,7 +29,7 @@ func (f *rabbitmqExporterFactory) createLogsExporter( cfg component.Config, ) (exporter.Logs, error) { customConfig := *(cfg.(*config)) - exp, err := newLogsExporter(customConfig, set, internal.newAmqpClient()) + exp, err := newLogsExporter(customConfig, set, internal.NewAmqpClient()) if err != nil { return nil, err } @@ -38,6 +38,6 @@ func (f *rabbitmqExporterFactory) createLogsExporter( ctx, set, cfg, - exp.logsDataPusher, + exp.exportLogs, exporterhelper.WithRetry(customConfig.retrySettings)) } diff --git a/exporter/rabbitmqexporter/internal/amqp_client.go b/exporter/rabbitmqexporter/internal/amqp_client.go index e22dbdb257da..964d67012c54 100644 --- a/exporter/rabbitmqexporter/internal/amqp_client.go +++ b/exporter/rabbitmqexporter/internal/amqp_client.go @@ -35,7 +35,7 @@ type WrappedDeferredConfirmation interface { type amqpClient struct{} -func newAmqpClient() AmqpClient { +func NewAmqpClient() AmqpClient { return &amqpClient{} } diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index 757d6b3feeaa..895d781873ef 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -13,30 +13,31 @@ import ( "time" ) -type rabbitMqLogsProducer struct { +type rabbitMqPublisher struct { set component.TelemetrySettings config config amqpClient *internal.AmqpClient channelCacher *amqpChannelCacher - marshaller LogsMarshaler } -func newLogsExporter(conf config, set exporter.CreateSettings, amqpClient internal.AmqpClient) (*rabbitMqLogsProducer, error) { - amqpChannelCacher, err := newExporterChannelCacher(conf, set, "otel-logs", amqpClient) +type rabbitMqLogsExporter struct { + publisher *rabbitMqPublisher + marshaller LogsMarshaler +} + +func newLogsExporter(conf config, set exporter.CreateSettings, amqpClient internal.AmqpClient) (*rabbitMqLogsExporter, error) { + publisher, err := newPublisher(conf, set, amqpClient, "otel-logs") if err != nil { return nil, err } - logsProducer := &rabbitMqLogsProducer{ - set: set.TelemetrySettings, - config: conf, - channelCacher: amqpChannelCacher, - marshaller: newLogMarshaler(), - } - return logsProducer, nil + return &rabbitMqLogsExporter{ + publisher: publisher, + marshaller: newLogMarshaler(), + }, nil } -func newExporterChannelCacher(conf config, set exporter.CreateSettings, connectionName string, client internal.AmqpClient) (*amqpChannelCacher, error) { +func newPublisher(conf config, set exporter.CreateSettings, client internal.AmqpClient, connectionName string) (*rabbitMqPublisher, error) { connectionConfig := &connectionConfig{ logger: set.Logger, connectionUrl: conf.connectionUrl, @@ -46,42 +47,55 @@ func newExporterChannelCacher(conf config, set exporter.CreateSettings, connecti heartbeatInterval: conf.connectionHeartbeatInterval, confirmationMode: conf.confirmMode, } - return newAmqpChannelCacher(connectionConfig, client) -} + channelCacher, err := newAmqpChannelCacher(connectionConfig, client) + if err != nil { + return nil, err + } -// TODO implement and add code re-use for other types of telemetry -func (e *rabbitMqLogsProducer) logsDataPusher(ctx context.Context, data plog.Logs) error { - e.channelCacher.restoreConnectionIfUnhealthy() - channelWrapper, err := e.channelCacher.requestHealthyChannelFromPool() + publisher := rabbitMqPublisher{ + set: set.TelemetrySettings, + config: conf, + channelCacher: channelCacher, + } + return &publisher, nil +} +func (e *rabbitMqLogsExporter) exportLogs(ctx context.Context, data plog.Logs) error { + publishingData, err := e.marshaller.Marshal(data) if err != nil { return err } - err, healthyChannel := e.pushData(ctx, data, channelWrapper) - e.channelCacher.returnChannelToPool(channelWrapper, healthyChannel) - return err + return e.publisher.publish(ctx, publishingData) } -func (e *rabbitMqLogsProducer) pushData(ctx context.Context, data plog.Logs, wrapper *amqpChannelManager) (err error, healthyChannel bool) { - publishingData, err := e.marshaller.Marshal(data) +// TODO implement and add code re-use for other types of telemetry +func (p *rabbitMqPublisher) publish(ctx context.Context, data *publishingData) error { + p.channelCacher.restoreConnectionIfUnhealthy() + channelWrapper, err := p.channelCacher.requestHealthyChannelFromPool() if err != nil { - return err, true + return err } + err, healthyChannel := p.publishWithWrapper(ctx, data, channelWrapper) + p.channelCacher.returnChannelToPool(channelWrapper, healthyChannel) + return err +} + +func (p *rabbitMqPublisher) publishWithWrapper(ctx context.Context, data *publishingData, wrapper *amqpChannelManager) (err error, healthyChannel bool) { deliveryMode := amqp.Transient - if e.config.durable { + if p.config.durable { deliveryMode = amqp.Persistent } // TODO handle case where message doesn't get routed to any queues and is returned (when configured with mandatory = true) - confirmation, err := wrapper.channel.PublishWithDeferredConfirmWithContext(ctx, "amq.direct", e.config.routingKey, false, false, amqp.Publishing{ + confirmation, err := wrapper.channel.PublishWithDeferredConfirmWithContext(ctx, "amq.direct", p.config.routingKey, false, false, amqp.Publishing{ Headers: amqp.Table{}, - ContentType: publishingData.ContentType, - ContentEncoding: publishingData.ContentEncoding, + ContentType: data.ContentType, + ContentEncoding: data.ContentEncoding, DeliveryMode: deliveryMode, - Body: publishingData.Body, + Body: data.Body, }) if err != nil { @@ -91,20 +105,20 @@ func (e *rabbitMqLogsProducer) pushData(ctx context.Context, data plog.Logs, wra select { case <-confirmation.Done(): if confirmation.Acked() { - e.set.Logger.Debug("Received ack", zap.Int("channelId", wrapper.id), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) + p.set.Logger.Debug("Received ack", zap.Int("channelId", wrapper.id), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) return nil, true } - e.set.Logger.Warn("Received nack from rabbitmq publishing confirmation", zap.Uint64("deliveryTag", confirmation.DeliveryTag())) + p.set.Logger.Warn("Received nack from rabbitmq publishing confirmation", zap.Uint64("deliveryTag", confirmation.DeliveryTag())) err := errors.New("received nack from rabbitmq publishing confirmation") return err, true - case <-time.After(e.config.publishConfirmationTimeout): - e.set.Logger.Warn("Timeout waiting for publish confirmation", zap.Duration("timeout", e.config.publishConfirmationTimeout), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) - err := fmt.Errorf("timeout waiting for publish confirmation after %s", e.config.publishConfirmationTimeout) + case <-time.After(p.config.publishConfirmationTimeout): + p.set.Logger.Warn("Timeout waiting for publish confirmation", zap.Duration("timeout", p.config.publishConfirmationTimeout), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) + err := fmt.Errorf("timeout waiting for publish confirmation after %s", p.config.publishConfirmationTimeout) return err, false } } -func (e *rabbitMqLogsProducer) Close(context.Context) error { - return e.channelCacher.close() +func (e *rabbitMqLogsExporter) Close(context.Context) error { + return e.publisher.channelCacher.close() } diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go index 8e6fe6a53e47..d6c294a9d6c6 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter_test.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter_test.go @@ -109,7 +109,7 @@ func TestPublishLogsHappyPath(t *testing.T) { rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) confirmAsynchronously(confirmation, true, time.Millisecond*50) - err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + err := rabbitMqExporter.exportLogs(context.Background(), plog.NewLogs()) assert.NoError(t, err) assert.Equal(t, 1, len(channel.published)) @@ -144,7 +144,7 @@ func TestPublishLogsConcurrently(t *testing.T) { for i := 0; i < 2; i++ { go func(ii int) { confirmAsynchronously(confirmations[ii], true, time.Millisecond*100) - err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + err := rabbitMqExporter.exportLogs(context.Background(), plog.NewLogs()) assert.NoError(t, err) wg.Done() }(i) @@ -167,7 +167,7 @@ func TestPublishLogsWithTimeout(t *testing.T) { customConfig.publishConfirmationTimeout = time.Millisecond * 100 rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) - err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + err := rabbitMqExporter.exportLogs(context.Background(), plog.NewLogs()) assert.ErrorContains(t, err, "timeout waiting for publish confirmation after 100ms") assert.Equal(t, 1, len(channel.published)) @@ -183,7 +183,7 @@ func TestPublishLogsWithError(t *testing.T) { } rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) - err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + err := rabbitMqExporter.exportLogs(context.Background(), plog.NewLogs()) assert.ErrorContains(t, err, "expected publish error") } @@ -210,12 +210,12 @@ func TestRecoveryFromBadConnectionAfterPublishError(t *testing.T) { rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) - err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + err := rabbitMqExporter.exportLogs(context.Background(), plog.NewLogs()) assert.ErrorContains(t, err, "expected publish error") assert.Equal(t, 1, len(badChannel.published)) confirmAsynchronously(goodConfirm, true, time.Millisecond*50) - err = rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + err = rabbitMqExporter.exportLogs(context.Background(), plog.NewLogs()) assert.NoError(t, err) assert.Equal(t, 1, len(goodChannel.published)) } @@ -241,11 +241,11 @@ func TestRecoveryFromBadChannelAfterPublishError(t *testing.T) { rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) - err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + err := rabbitMqExporter.exportLogs(context.Background(), plog.NewLogs()) assert.ErrorContains(t, err, "expected publish error") confirmAsynchronously(goodConfirm, true, time.Millisecond*50) - err = rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + err = rabbitMqExporter.exportLogs(context.Background(), plog.NewLogs()) assert.NoError(t, err) assert.Equal(t, 1, len(goodChannel.published)) } @@ -258,7 +258,7 @@ func TestPublishLogsWithNack(t *testing.T) { confirmAsynchronously(confirmation, false, time.Millisecond*50) rabbitMqExporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) - err := rabbitMqExporter.logsDataPusher(context.Background(), plog.NewLogs()) + err := rabbitMqExporter.exportLogs(context.Background(), plog.NewLogs()) assert.ErrorContains(t, err, "received nack from rabbitmq publishing confirmation") assert.Equal(t, 1, len(channel.published)) @@ -303,7 +303,7 @@ func TestPublishDurableFlag(t *testing.T) { exporter, _ := newLogsExporter(customConfig, exportertest.NewNopCreateSettings(), client) confirmAsynchronously(confirmation, true, time.Millisecond*50) - err := exporter.logsDataPusher(context.Background(), plog.NewLogs()) + err := exporter.exportLogs(context.Background(), plog.NewLogs()) assert.NoError(t, err) assert.Equal(t, 1, len(channel.published)) From 06b693129b2bb927ad43925b573f47bfb3862fb2 Mon Sep 17 00:00:00 2001 From: swar8080 Date: Fri, 1 Dec 2023 16:39:25 -0500 Subject: [PATCH 18/19] add trace and metric exporter --- exporter/rabbitmqexporter/factory.go | 99 +++++++++++++++++++ .../internal/metadata/generated_status.go | 2 + exporter/rabbitmqexporter/marshaler.go | 58 +++++++++++ .../rabbitmqexporter/rabbitmq_exporter.go | 50 +++++----- 4 files changed, 181 insertions(+), 28 deletions(-) diff --git a/exporter/rabbitmqexporter/factory.go b/exporter/rabbitmqexporter/factory.go index 3042bd8e0987..cedaee4d3bc4 100644 --- a/exporter/rabbitmqexporter/factory.go +++ b/exporter/rabbitmqexporter/factory.go @@ -19,10 +19,50 @@ func NewFactory() exporter.Factory { return exporter.NewFactory( metadata.Type, createDefaultConfig, + exporter.WithTraces(f.createTracesExporter, metadata.TracesStability), + exporter.WithMetrics(f.createMetricsExporter, metadata.MetricsStability), exporter.WithLogs(f.createLogsExporter, metadata.LogsStability), ) } +func (f *rabbitmqExporterFactory) createTracesExporter( + ctx context.Context, + set exporter.CreateSettings, + cfg component.Config, +) (exporter.Traces, error) { + customConfig := *(cfg.(*config)) + exp, err := newTracesExporter(customConfig, set, internal.NewAmqpClient()) + if err != nil { + return nil, err + } + + return exporterhelper.NewTracesExporter( + ctx, + set, + cfg, + exp.exportTraces, + exporterhelper.WithRetry(customConfig.retrySettings)) +} + +func (f *rabbitmqExporterFactory) createMetricsExporter( + ctx context.Context, + set exporter.CreateSettings, + cfg component.Config, +) (exporter.Metrics, error) { + customConfig := *(cfg.(*config)) + exp, err := newMetricsExporter(customConfig, set, internal.NewAmqpClient()) + if err != nil { + return nil, err + } + + return exporterhelper.NewMetricsExporter( + ctx, + set, + cfg, + exp.exportMetrics, + exporterhelper.WithRetry(customConfig.retrySettings)) +} + func (f *rabbitmqExporterFactory) createLogsExporter( ctx context.Context, set exporter.CreateSettings, @@ -41,3 +81,62 @@ func (f *rabbitmqExporterFactory) createLogsExporter( exp.exportLogs, exporterhelper.WithRetry(customConfig.retrySettings)) } + +func newTracesExporter(conf config, set exporter.CreateSettings, amqpClient internal.AmqpClient) (*rabbitMqTracesExporter, error) { + publisher, err := newPublisher(conf, set, amqpClient, "otel-traces") + if err != nil { + return nil, err + } + + return &rabbitMqTracesExporter{ + publisher: publisher, + marshaller: newTracesMarshaler(), + }, nil +} + +func newMetricsExporter(conf config, set exporter.CreateSettings, amqpClient internal.AmqpClient) (*rabbitMqMetricsExporter, error) { + publisher, err := newPublisher(conf, set, amqpClient, "otel-metrics") + if err != nil { + return nil, err + } + + return &rabbitMqMetricsExporter{ + publisher: publisher, + marshaller: newMetricsMarshaler(), + }, nil +} + +func newLogsExporter(conf config, set exporter.CreateSettings, amqpClient internal.AmqpClient) (*rabbitMqLogsExporter, error) { + publisher, err := newPublisher(conf, set, amqpClient, "otel-logs") + if err != nil { + return nil, err + } + + return &rabbitMqLogsExporter{ + publisher: publisher, + marshaller: newLogMarshaler(), + }, nil +} + +func newPublisher(conf config, set exporter.CreateSettings, client internal.AmqpClient, connectionName string) (*rabbitMqPublisher, error) { + connectionConfig := &connectionConfig{ + logger: set.Logger, + connectionUrl: conf.connectionUrl, + connectionName: connectionName, + channelPoolSize: conf.channelPoolSize, + connectionTimeout: conf.connectionTimeout, + heartbeatInterval: conf.connectionHeartbeatInterval, + confirmationMode: conf.confirmMode, + } + channelCacher, err := newAmqpChannelCacher(connectionConfig, client) + if err != nil { + return nil, err + } + + publisher := rabbitMqPublisher{ + set: set.TelemetrySettings, + config: conf, + channelCacher: channelCacher, + } + return &publisher, nil +} diff --git a/exporter/rabbitmqexporter/internal/metadata/generated_status.go b/exporter/rabbitmqexporter/internal/metadata/generated_status.go index dc0918efd9a7..2e3ba0726850 100644 --- a/exporter/rabbitmqexporter/internal/metadata/generated_status.go +++ b/exporter/rabbitmqexporter/internal/metadata/generated_status.go @@ -8,5 +8,7 @@ import ( const ( Type = "rabbitmq" + TracesStability = component.StabilityLevelDevelopment LogsStability = component.StabilityLevelDevelopment + MetricsStability = component.StabilityLevelDevelopment ) diff --git a/exporter/rabbitmqexporter/marshaler.go b/exporter/rabbitmqexporter/marshaler.go index 1f0d2093eb69..0424bc2085af 100644 --- a/exporter/rabbitmqexporter/marshaler.go +++ b/exporter/rabbitmqexporter/marshaler.go @@ -2,6 +2,8 @@ package rabbitmqexporter import ( "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/ptrace" ) type publishingData struct { @@ -10,10 +12,66 @@ type publishingData struct { Body []byte } +type TracesMarshaler interface { + Marshal(traces ptrace.Traces) (*publishingData, error) +} + +type MetricsMarshaler interface { + Marshal(metrics pmetric.Metrics) (*publishingData, error) +} + type LogsMarshaler interface { Marshal(logs plog.Logs) (*publishingData, error) } +type defaultTracesMarshaler struct { + impl *ptrace.JSONMarshaler +} + +func newTracesMarshaler() TracesMarshaler { + // TODO revisit which encoding(s) to use + return &defaultTracesMarshaler{ + impl: &ptrace.JSONMarshaler{}, + } +} + +func (m *defaultTracesMarshaler) Marshal(traces ptrace.Traces) (*publishingData, error) { + body, err := m.impl.MarshalTraces(traces) + if err != nil { + return nil, err + } + + return &publishingData{ + ContentType: "text/plain", + ContentEncoding: "", + Body: body, + }, nil +} + +type defaultMetricsMarshaler struct { + impl *pmetric.JSONMarshaler +} + +func newMetricsMarshaler() MetricsMarshaler { + // TODO revisit which encoding(s) to use + return &defaultMetricsMarshaler{ + impl: &pmetric.JSONMarshaler{}, + } +} + +func (m *defaultMetricsMarshaler) Marshal(Metrics pmetric.Metrics) (*publishingData, error) { + body, err := m.impl.MarshalMetrics(Metrics) + if err != nil { + return nil, err + } + + return &publishingData{ + ContentType: "text/plain", + ContentEncoding: "", + Body: body, + }, nil +} + type defaultLogsMarshaler struct { impl *plog.JSONMarshaler } diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index 895d781873ef..b2f0cfa141db 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -7,8 +7,9 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter/internal" amqp "github.com/rabbitmq/amqp091-go" "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap" "time" ) @@ -20,44 +21,37 @@ type rabbitMqPublisher struct { channelCacher *amqpChannelCacher } +type rabbitMqTracesExporter struct { + publisher *rabbitMqPublisher + marshaller TracesMarshaler +} + +type rabbitMqMetricsExporter struct { + publisher *rabbitMqPublisher + marshaller MetricsMarshaler +} + type rabbitMqLogsExporter struct { publisher *rabbitMqPublisher marshaller LogsMarshaler } -func newLogsExporter(conf config, set exporter.CreateSettings, amqpClient internal.AmqpClient) (*rabbitMqLogsExporter, error) { - publisher, err := newPublisher(conf, set, amqpClient, "otel-logs") +func (e *rabbitMqTracesExporter) exportTraces(ctx context.Context, data ptrace.Traces) error { + publishingData, err := e.marshaller.Marshal(data) if err != nil { - return nil, err + return err } - return &rabbitMqLogsExporter{ - publisher: publisher, - marshaller: newLogMarshaler(), - }, nil + return e.publisher.publish(ctx, publishingData) } -func newPublisher(conf config, set exporter.CreateSettings, client internal.AmqpClient, connectionName string) (*rabbitMqPublisher, error) { - connectionConfig := &connectionConfig{ - logger: set.Logger, - connectionUrl: conf.connectionUrl, - connectionName: connectionName, - channelPoolSize: conf.channelPoolSize, - connectionTimeout: conf.connectionTimeout, - heartbeatInterval: conf.connectionHeartbeatInterval, - confirmationMode: conf.confirmMode, - } - channelCacher, err := newAmqpChannelCacher(connectionConfig, client) +func (e *rabbitMqMetricsExporter) exportMetrics(ctx context.Context, data pmetric.Metrics) error { + publishingData, err := e.marshaller.Marshal(data) if err != nil { - return nil, err + return err } - publisher := rabbitMqPublisher{ - set: set.TelemetrySettings, - config: conf, - channelCacher: channelCacher, - } - return &publisher, nil + return e.publisher.publish(ctx, publishingData) } func (e *rabbitMqLogsExporter) exportLogs(ctx context.Context, data plog.Logs) error { @@ -78,12 +72,12 @@ func (p *rabbitMqPublisher) publish(ctx context.Context, data *publishingData) e return err } - err, healthyChannel := p.publishWithWrapper(ctx, data, channelWrapper) + err, healthyChannel := p.publishWithChannelManager(ctx, data, channelWrapper) p.channelCacher.returnChannelToPool(channelWrapper, healthyChannel) return err } -func (p *rabbitMqPublisher) publishWithWrapper(ctx context.Context, data *publishingData, wrapper *amqpChannelManager) (err error, healthyChannel bool) { +func (p *rabbitMqPublisher) publishWithChannelManager(ctx context.Context, data *publishingData, wrapper *amqpChannelManager) (err error, healthyChannel bool) { deliveryMode := amqp.Transient if p.config.durable { deliveryMode = amqp.Persistent From 01006ca69749f15f6a01bcd5767e80a6fd47832c Mon Sep 17 00:00:00 2001 From: swar8080 Date: Fri, 29 Dec 2023 17:13:11 -0500 Subject: [PATCH 19/19] wip --- exporter/rabbitmqexporter/channel_cacher.go | 27 ++++++++++++------- .../rabbitmqexporter/rabbitmq_exporter.go | 6 ++--- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/exporter/rabbitmqexporter/channel_cacher.go b/exporter/rabbitmqexporter/channel_cacher.go index 4f4d7c4178d6..c0e554367894 100644 --- a/exporter/rabbitmqexporter/channel_cacher.go +++ b/exporter/rabbitmqexporter/channel_cacher.go @@ -5,6 +5,7 @@ import ( amqp "github.com/rabbitmq/amqp091-go" "go.uber.org/zap" "sync" + "sync/atomic" "time" ) @@ -24,6 +25,7 @@ type amqpChannelCacher struct { connLock *sync.Mutex channelManagerPool chan *amqpChannelManager connectionErrors chan *amqp.Error + channelIdCounter *atomic.Uint64 } type connectionConfig struct { @@ -37,7 +39,7 @@ type connectionConfig struct { } type amqpChannelManager struct { - id int + id uint64 channel internal.WrappedChannel wasHealthy bool lock *sync.Mutex @@ -51,6 +53,7 @@ func newAmqpChannelCacher(config *connectionConfig, amqpClient internal.AmqpClie amqpClient: amqpClient, connLock: &sync.Mutex{}, channelManagerPool: make(chan *amqpChannelManager, config.channelPoolSize), + channelIdCounter: &atomic.Uint64{}, } err := acc.connect() @@ -60,7 +63,7 @@ func newAmqpChannelCacher(config *connectionConfig, amqpClient internal.AmqpClie // Synchronously try creating and connecting to channels for i := 0; i < acc.config.channelPoolSize; i++ { - acc.channelManagerPool <- acc.createChannelManager(i) + acc.channelManagerPool <- acc.createChannelManager() } return acc, nil @@ -139,9 +142,11 @@ func (acc *amqpChannelCacher) restoreConnectionIfUnhealthy() { } } -func (acc *amqpChannelCacher) createChannelManager(id int) *amqpChannelManager { - channelWrapper := &amqpChannelManager{id: id, logger: acc.logger, lock: &sync.Mutex{}} - err := channelWrapper.tryReplacingChannel(acc.connection, acc.config.confirmationMode) +func (acc *amqpChannelCacher) createChannelManager() *amqpChannelManager { + channelId := acc.channelIdCounter.Add(1) + channelWrapper := &amqpChannelManager{id: channelId, logger: acc.logger, lock: &sync.Mutex{}} + + err := channelWrapper.tryReplacingChannel(acc.connection, channelId, acc.config.confirmationMode) if err != nil { acc.logger.Warn("Error creating channel manager's channel", zap.Error(err)) } @@ -150,7 +155,8 @@ func (acc *amqpChannelCacher) createChannelManager(id int) *amqpChannelManager { func (acc *amqpChannelCacher) requestHealthyChannelFromPool() (*amqpChannelManager, error) { channelWrapper := <-acc.channelManagerPool - if !channelWrapper.wasHealthy { + if !channelWrapper.wasHealthy || channelWrapper.channel.IsClosed() { + acc.logger.Warn("Attempting to replace closed or unhealthy AMQP channel", zap.Uint64("channelId", channelWrapper.id)) err := acc.reconnectChannel(channelWrapper) if err != nil { acc.returnChannelToPool(channelWrapper, false) @@ -168,10 +174,10 @@ func (acc *amqpChannelCacher) returnChannelToPool(channelWrapper *amqpChannelMan func (acc *amqpChannelCacher) reconnectChannel(channel *amqpChannelManager) error { acc.restoreConnectionIfUnhealthy() - return channel.tryReplacingChannel(acc.connection, acc.config.confirmationMode) + return channel.tryReplacingChannel(acc.connection, acc.channelIdCounter.Add(1), acc.config.confirmationMode) } -func (acw *amqpChannelManager) tryReplacingChannel(connection internal.WrappedConnection, confirmAcks bool) error { +func (acw *amqpChannelManager) tryReplacingChannel(connection internal.WrappedConnection, channelId uint64, confirmAcks bool) error { acw.lock.Lock() defer acw.lock.Unlock() @@ -185,12 +191,15 @@ func (acw *amqpChannelManager) tryReplacingChannel(connection internal.WrappedCo } var err error - acw.channel, err = connection.Channel() + channel, err := connection.Channel() if err != nil { acw.logger.Warn("Channel creation error", zap.Error(err)) acw.wasHealthy = false return err } + acw.channel = channel + acw.id = channelId + acw.logger.Debug("Created new channel", zap.Uint64("channelId", channelId)) if confirmAcks { err := acw.channel.Confirm(false) diff --git a/exporter/rabbitmqexporter/rabbitmq_exporter.go b/exporter/rabbitmqexporter/rabbitmq_exporter.go index b2f0cfa141db..24428800ae17 100644 --- a/exporter/rabbitmqexporter/rabbitmq_exporter.go +++ b/exporter/rabbitmqexporter/rabbitmq_exporter.go @@ -99,15 +99,15 @@ func (p *rabbitMqPublisher) publishWithChannelManager(ctx context.Context, data select { case <-confirmation.Done(): if confirmation.Acked() { - p.set.Logger.Debug("Received ack", zap.Int("channelId", wrapper.id), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) + p.set.Logger.Debug("Received ack", zap.Uint64("channelId", wrapper.id), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) return nil, true } - p.set.Logger.Warn("Received nack from rabbitmq publishing confirmation", zap.Uint64("deliveryTag", confirmation.DeliveryTag())) + p.set.Logger.Warn("Received nack from rabbitmq publishing confirmation", zap.Uint64("channelId", wrapper.id), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) err := errors.New("received nack from rabbitmq publishing confirmation") return err, true case <-time.After(p.config.publishConfirmationTimeout): - p.set.Logger.Warn("Timeout waiting for publish confirmation", zap.Duration("timeout", p.config.publishConfirmationTimeout), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) + p.set.Logger.Warn("Timeout waiting for publish confirmation", zap.Duration("timeout", p.config.publishConfirmationTimeout), zap.Uint64("channelId", wrapper.id), zap.Uint64("deliveryTag", confirmation.DeliveryTag())) err := fmt.Errorf("timeout waiting for publish confirmation after %s", p.config.publishConfirmationTimeout) return err, false }