feat: [v2] Metrics cleanup

README.md

@@ -88,10 +88,10 @@ Refinery supports the following environment variables.  Environment variables ta
 | `REFINERY_REDIS_USERNAME`                                         | `PeerManagement.RedisUsername`   |
 | `REFINERY_REDIS_PASSWORD`                                         | `PeerManagement.RedisPassword`   |
 | `REFINERY_HONEYCOMB_API_KEY`                                      | `HoneycombLogger.LoggerAPIKey`   |
-| `REFINERY_HONEYCOMB_METRICS_API_KEY` `REFINERY_HONEYCOMB_API_KEY` | `HoneycombMetrics.MetricsAPIKey` |
+| `REFINERY_LEGACY_METRICS_API_KEY` `REFINERY_HONEYCOMB_API_KEY`    | `LegacyMetrics.APIKey`           |
 | `REFINERY_QUERY_AUTH_TOKEN`                                       | `QueryAuthToken`                 |
 
-Note, `REFINERY_HONEYCOMB_METRICS_API_KEY` takes precedence over `REFINERY_HONEYCOMB_API_KEY` for the `HoneycombMetrics.MetricsAPIKey` configuration.
+Note, `REFINERY_HONEYCOMB_METRICS_API_KEY` takes precedence over `REFINERY_HONEYCOMB_API_KEY` for the `LegacyMetrics.APIKey` configuration.
 
 ### Mixing Classic and Environment & Services Rule Definitions
 

cmd/refinery/main.go

@@ -181,25 +181,30 @@ func main() {
 	if opts.Debug {
 		g.Logger = graphLogger{}
 	}
-	err = g.Provide(
-		&inject.Object{Value: c},
-		&inject.Object{Value: peers},
-		&inject.Object{Value: lgr},
-		&inject.Object{Value: upstreamTransport, Name: "upstreamTransport"},
-		&inject.Object{Value: peerTransport, Name: "peerTransport"},
-		&inject.Object{Value: transmit.NewDefaultTransmission(upstreamClient, upstreamMetricsRecorder, "upstream"), Name: "upstreamTransmission"},
-		&inject.Object{Value: transmit.NewDefaultTransmission(peerClient, peerMetricsRecorder, "peer"), Name: "peerTransmission"},
-		&inject.Object{Value: shrdr},
-		&inject.Object{Value: collector},
-		&inject.Object{Value: metricsSingleton, Name: "metrics"},
-		&inject.Object{Value: genericMetricsRecorder, Name: "genericMetrics"},
-		&inject.Object{Value: upstreamMetricsRecorder, Name: "upstreamMetrics"},
-		&inject.Object{Value: peerMetricsRecorder, Name: "peerMetrics"},
-		&inject.Object{Value: version, Name: "version"},
-		&inject.Object{Value: samplerFactory},
-		&inject.Object{Value: stressRelief, Name: "stressRelief"},
-		&inject.Object{Value: &a},
-	)
+	objects := []*inject.Object{
+		{Value: c},
+		{Value: peers},
+		{Value: lgr},
+		{Value: upstreamTransport, Name: "upstreamTransport"},
+		{Value: peerTransport, Name: "peerTransport"},
+		{Value: transmit.NewDefaultTransmission(upstreamClient, upstreamMetricsRecorder, "upstream"), Name: "upstreamTransmission"},
+		{Value: transmit.NewDefaultTransmission(peerClient, peerMetricsRecorder, "peer"), Name: "peerTransmission"},
+		{Value: shrdr},
+		{Value: collector},
+		{Value: metricsSingleton, Name: "metrics"},
+		{Value: genericMetricsRecorder, Name: "genericMetrics"},
+		{Value: upstreamMetricsRecorder, Name: "upstreamMetrics"},
+		{Value: peerMetricsRecorder, Name: "peerMetrics"},
+		{Value: version, Name: "version"},
+		{Value: samplerFactory},
+		{Value: stressRelief, Name: "stressRelief"},
+		{Value: &a},
+	}
+	// we need to add the multimetrics children to the graph as well
+	for _, obj := range metricsSingleton.Children() {
+		objects = append(objects, &inject.Object{Value: obj})
+	}
+	err = g.Provide(objects...)
 	if err != nil {
 		fmt.Printf("failed to provide injection graph. error: %+v\n", err)
 		os.Exit(1)

collect/stressRelief.go

@@ -81,6 +81,7 @@ func (s *StressRelief) Start() error {
 		"sigmoid": s.sigmoid, // don't worry about small stuff, but if we cross the midline, start worrying quickly
 	}
 
+	// All of the numerator metrics are gauges. The denominator metrics are constants.
 	s.calcs = []StressReliefCalculation{
 		{Numerator: "collector_peer_queue_length", Denominator: "PEER_CAP", Algorithm: "sqrt", Reason: "CacheCapacity (peer)"},
 		{Numerator: "collector_incoming_queue_length", Denominator: "INCOMING_CAP", Algorithm: "sqrt", Reason: "CacheCapacity (incoming)"},

config/cmdenv.go

@@ -38,6 +38,7 @@ type CmdEnv struct {
 	HoneycombAPIKey       string `long:"honeycomb-api-key" env:"REFINERY_HONEYCOMB_API_KEY" description:"Honeycomb API key (for logger and metrics)"`
 	HoneycombLoggerAPIKey string `long:"logger-api-key" env:"REFINERY_HONEYCOMB_LOGGER_API_KEY" description:"Honeycomb logger API key"`
 	LegacyMetricsAPIKey   string `long:"legacy-metrics-api-key" env:"REFINERY_LEGACY_METRICS_API_KEY" description:"API key for legacy Honeycomb metrics"`
+	OTelMetricsAPIKey     string `long:"otel-metrics-api-key" env:"REFINERY_OTEL_METRICS_API_KEY" description:"API key for OTel metrics if being sent to Honeycomb"`
 	QueryAuthToken        string `long:"query-auth-token" env:"REFINERY_QUERY_AUTH_TOKEN" description:"Token for debug/management queries"`
 	Debug                 bool   `short:"d" long:"debug" description:"Runs debug service (on the first open port between localhost:6060 and :6069 by default)"`
 	Version               bool   `short:"v" long:"version" description:"Print version number and exit"`

config/config.go

@@ -116,11 +116,14 @@ type Config interface {
 	// metrics package
 	GetMetricsType() (string, error)
 
-	// GetHoneycombMetricsConfig returns the config specific to HoneycombMetrics
-	GetHoneycombMetricsConfig() (LegacyMetricsConfig, error)
+	// GetLegacyMetricsConfig returns the config specific to LegacyMetrics
+	GetLegacyMetricsConfig() LegacyMetricsConfig
 
 	// GetPrometheusMetricsConfig returns the config specific to PrometheusMetrics
-	GetPrometheusMetricsConfig() (PrometheusMetricsConfig, error)
+	GetPrometheusMetricsConfig() PrometheusMetricsConfig
+
+	// GetOTelMetricsConfig returns the config specific to OTelMetrics
+	GetOTelMetricsConfig() OTelMetricsConfig
 
 	// GetUpstreamBufferSize returns the size of the libhoney buffer to use for the upstream
 	// libhoney client

config/config_test.go

@@ -149,7 +149,7 @@ func TestMetricsAPIKeyEnvVar(t *testing.T) {
 				t.Error(err)
 			}
 
-			if d, _ := c.GetHoneycombMetricsConfig(); d.APIKey != tc.key {
+			if d := c.GetLegacyMetricsConfig(); d.APIKey != tc.key {
 				t.Error("received", d, "expected", tc.key)
 			}
 		})
@@ -170,7 +170,7 @@ func TestMetricsAPIKeyMultipleEnvVar(t *testing.T) {
 	c, err := getConfig([]string{"--config", "../config.yaml", "--rules_config", "../rules.yaml"})
 	assert.NoError(t, err)
 
-	if d, _ := c.GetHoneycombMetricsConfig(); d.APIKey != specificKey {
+	if d := c.GetLegacyMetricsConfig(); d.APIKey != specificKey {
 		t.Error("received", d, "expected", specificKey)
 	}
 }
@@ -184,7 +184,7 @@ func TestMetricsAPIKeyFallbackEnvVar(t *testing.T) {
 	c, err := getConfig([]string{"--config", "../config.yaml", "--rules_config", "../rules.yaml"})
 	assert.NoError(t, err)
 
-	if d, _ := c.GetHoneycombMetricsConfig(); d.APIKey != key {
+	if d := c.GetLegacyMetricsConfig(); d.APIKey != key {
 		t.Error("received", d, "expected", key)
 	}
 }

config/file_config.go

@@ -50,6 +50,7 @@ type configContents struct {
 	StdoutLogger         StdoutLoggerConfig        `yaml:"StdoutLogger"`
 	PrometheusMetrics    PrometheusMetricsConfig   `yaml:"PrometheusMetrics"`
 	LegacyMetrics        LegacyMetricsConfig       `yaml:"LegacyMetrics"`
+	OTelMetrics          OTelMetricsConfig         `yaml:"OTelMetrics"`
 	PeerManagement       PeerManagementConfig      `yaml:"PeerManagement"`
 	RedisPeerManagement  RedisPeerManagementConfig `yaml:"RedisPeerManagement"`
 	Collection           CollectionConfig          `yaml:"Collection"`
@@ -130,6 +131,15 @@ type LegacyMetricsConfig struct {
 	ReportingInterval Duration `yaml:"ReportingInterval" default:"30s" validate:"dmin=1s"`
 }
 
+type OTelMetricsConfig struct {
+	Enabled           bool     `yaml:"Enabled" default:"false"`
+	APIHost           string   `yaml:"APIHost" default:"https://api.honeycomb.io" validate:"url"`
+	APIKey            string   `yaml:"APIKey" cmdenv:"OTelMetricsAPIKey,HoneycombAPIKey"`
+	Dataset           string   `yaml:"Dataset"`
+	Compression       string   `yaml:"Compression" default:"gzip" validate:"oneof=gzip none"`
+	ReportingInterval Duration `yaml:"ReportingInterval" default:"30s" validate:"dmin=1s"`
+}
+
 type PeerManagementConfig struct {
 	Type                    string   `yaml:"Type" default:"file" validate:"required,oneof=file redis"`
 	Identifier              string   `yaml:"Identifier"`
@@ -511,18 +521,25 @@ func (f *fileConfig) GetMetricsType() (string, error) {
 	return "", nil
 }
 
-func (f *fileConfig) GetHoneycombMetricsConfig() (LegacyMetricsConfig, error) {
+func (f *fileConfig) GetLegacyMetricsConfig() LegacyMetricsConfig {
+	f.mux.RLock()
+	defer f.mux.RUnlock()
+
+	return f.mainConfig.LegacyMetrics
+}
+
+func (f *fileConfig) GetPrometheusMetricsConfig() PrometheusMetricsConfig {
 	f.mux.RLock()
 	defer f.mux.RUnlock()
 
-	return f.mainConfig.LegacyMetrics, nil
+	return f.mainConfig.PrometheusMetrics
 }
 
-func (f *fileConfig) GetPrometheusMetricsConfig() (PrometheusMetricsConfig, error) {
+func (f *fileConfig) GetOTelMetricsConfig() OTelMetricsConfig {
 	f.mux.RLock()
 	defer f.mux.RUnlock()
 
-	return f.mainConfig.PrometheusMetrics, nil
+	return f.mainConfig.OTelMetrics
 }
 
 func (f *fileConfig) GetSendDelay() (time.Duration, error) {

config/mock.go

@@ -49,10 +49,9 @@ type MockConfig struct {
 	GetSamplerTypeVal                    interface{}
 	GetMetricsTypeErr                    error
 	GetMetricsTypeVal                    string
-	GetHoneycombMetricsConfigErr         error
-	GetHoneycombMetricsConfigVal         LegacyMetricsConfig
-	GetPrometheusMetricsConfigErr        error
+	GetLegacyMetricsConfigVal            LegacyMetricsConfig
 	GetPrometheusMetricsConfigVal        PrometheusMetricsConfig
+	GetOTelMetricsConfigVal              OTelMetricsConfig
 	GetSendDelayErr                      error
 	GetSendDelayVal                      time.Duration
 	GetBatchTimeoutVal                   time.Duration
@@ -248,18 +247,25 @@ func (m *MockConfig) GetMetricsType() (string, error) {
 	return m.GetMetricsTypeVal, m.GetMetricsTypeErr
 }
 
-func (m *MockConfig) GetHoneycombMetricsConfig() (LegacyMetricsConfig, error) {
+func (m *MockConfig) GetLegacyMetricsConfig() LegacyMetricsConfig {
 	m.Mux.RLock()
 	defer m.Mux.RUnlock()
 
-	return m.GetHoneycombMetricsConfigVal, m.GetHoneycombMetricsConfigErr
+	return m.GetLegacyMetricsConfigVal
 }
 
-func (m *MockConfig) GetPrometheusMetricsConfig() (PrometheusMetricsConfig, error) {
+func (m *MockConfig) GetPrometheusMetricsConfig() PrometheusMetricsConfig {
 	m.Mux.RLock()
 	defer m.Mux.RUnlock()
 
-	return m.GetPrometheusMetricsConfigVal, m.GetPrometheusMetricsConfigErr
+	return m.GetPrometheusMetricsConfigVal
+}
+
+func (m *MockConfig) GetOTelMetricsConfig() OTelMetricsConfig {
+	m.Mux.RLock()
+	defer m.Mux.RUnlock()
+
+	return m.GetOTelMetricsConfigVal
 }
 
 func (m *MockConfig) GetSendDelay() (time.Duration, error) {

go.mod

@@ -24,27 +24,35 @@ require (
 	github.com/prometheus/client_golang v1.15.1
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0
 	github.com/sirupsen/logrus v1.9.0
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.3
 	github.com/tidwall/gjson v1.14.4
 	github.com/vmihailenco/msgpack/v4 v4.3.11
+	go.opentelemetry.io/otel v1.16.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.39.0
+	go.opentelemetry.io/otel/metric v1.16.0
+	go.opentelemetry.io/otel/sdk v1.16.0
+	go.opentelemetry.io/otel/sdk/metric v0.39.0
 	go.opentelemetry.io/proto/otlp v0.19.0
 	go.uber.org/automaxprocs v1.5.2
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
-	google.golang.org/grpc v1.54.0
+	google.golang.org/grpc v1.55.0
 	google.golang.org/protobuf v1.30.0
 	gopkg.in/alexcesaro/statsd.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
 	github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 // indirect
 	github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 // indirect
 	github.com/facebookgo/structtag v0.0.0-20150214074306-217e25fb9691 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -63,12 +71,15 @@ require (
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
 	github.com/vmihailenco/tagparser v0.1.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
 	golang.org/x/crypto v0.7.0 // indirect
 	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.8.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
+	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
 )
 
 replace go.opentelemetry.io/proto/otlp => github.com/honeycombio/opentelemetry-proto-go/otlp v0.19.0-compat