fix: set 0 for otel metrics during registration

honeycombio · Sep 25, 2024 · dd8efd7 · dd8efd7
1 parent 398f615
commit dd8efd7
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 6 deletions.
diff --git a/cmd/refinery/main.go b/cmd/refinery/main.go
@@ -316,11 +316,16 @@ func main() {
 		"messages_sent":          "counter",
 		"response_decode_errors": "counter",
 	}
+
 	for name, typ := range libhoneyMetricsName {
 		upstreamMetricsRecorder.Register(name, typ)
 		peerMetricsRecorder.Register(name, typ)
 	}
 
+	// Register metrics after the metrics object has been created
+	peerTransmission.RegisterMetrics()
+	upstreamTransmission.RegisterMetrics()
+
 	metricsSingleton.Store("UPSTREAM_BUFFER_SIZE", float64(c.GetUpstreamBufferSize()))
 	metricsSingleton.Store("PEER_BUFFER_SIZE", float64(c.GetPeerBufferSize()))
 

diff --git a/metrics/otel_metrics.go b/metrics/otel_metrics.go
@@ -150,6 +150,7 @@ func (o *OTelMetrics) Start() error {
 	if err != nil {
 		return err
 	}
+
 	o.gauges[name] = g
 
 	name = "memory_inuse"
@@ -185,13 +186,17 @@ func (o *OTelMetrics) Register(name string, metricType string) {
 	o.lock.Lock()
 	defer o.lock.Unlock()
 
+	ctx := context.Background()
 	switch metricType {
 	case "counter":
 		ctr, err := o.meter.Int64Counter(name)
 		if err != nil {
 			o.Logger.Error().WithString("name", name).Logf("failed to create counter")
 			return
 		}
+		// Initialize the counter to 0 so that it will be reported
+		ctr.Add(ctx, 0)
+
 		o.counters[name] = ctr
 	case "gauge":
 		var f metric.Float64Callback = func(_ context.Context, result metric.Float64Observer) error {
@@ -213,20 +218,25 @@ func (o *OTelMetrics) Register(name string, metricType string) {
 			o.Logger.Error().WithString("name", name).Logf("failed to create gauge")
 			return
 		}
+
 		o.gauges[name] = g
 	case "histogram":
 		h, err := o.meter.Float64Histogram(name)
 		if err != nil {
 			o.Logger.Error().WithString("name", name).Logf("failed to create histogram")
 			return
 		}
+
+		h.Record(ctx, 0)
 		o.histograms[name] = h
 	case "updown":
 		ud, err := o.meter.Int64UpDownCounter(name)
 		if err != nil {
 			o.Logger.Error().WithString("name", name).Logf("failed to create updown counter")
 			return
 		}
+
+		ud.Add(ctx, 0)
 		o.updowns[name] = ud
 	default:
 		o.Logger.Error().WithString("type", metricType).Logf("unknown metric type")

diff --git a/transmit/mock.go b/transmit/mock.go
@@ -31,3 +31,5 @@ func (m *MockTransmission) Flush() {
 	defer m.Mux.Unlock()
 	m.Events = m.Events[:0]
 }
+
+func (m *MockTransmission) RegisterMetrics() {}
diff --git a/transmit/transmit.go b/transmit/transmit.go
@@ -20,6 +20,8 @@ type Transmission interface {
 	EnqueueSpan(ev *types.Span)
 	// Flush flushes the in-flight queue of all events and spans
 	Flush()
+
+	RegisterMetrics()
 }
 
 const (
@@ -65,12 +67,6 @@ func (d *DefaultTransmission) Start() error {
 		libhoney.UserAgentAddition = "refinery/" + d.Version
 	})
 
-	d.Metrics.Register(counterEnqueueErrors, "counter")
-	d.Metrics.Register(counterResponse20x, "counter")
-	d.Metrics.Register(counterResponseErrors, "counter")
-	d.Metrics.Register(updownQueuedItems, "updown")
-	d.Metrics.Register(histogramQueueTime, "histogram")
-
 	processCtx, canceler := context.WithCancel(context.Background())
 	d.responseCanceler = canceler
 	go d.processResponses(processCtx, d.LibhClient.TxResponses())
@@ -141,6 +137,16 @@ func (d *DefaultTransmission) Flush() {
 	d.LibhClient.Flush()
 }
 
+// RegisterMetrics registers the metrics used by the DefaultTransmission.
+// it should be called after the metrics object has been created.
+func (d *DefaultTransmission) RegisterMetrics() {
+	d.Metrics.Register(counterEnqueueErrors, "counter")
+	d.Metrics.Register(counterResponse20x, "counter")
+	d.Metrics.Register(counterResponseErrors, "counter")
+	d.Metrics.Register(updownQueuedItems, "updown")
+	d.Metrics.Register(histogramQueueTime, "histogram")
+}
+
 func (d *DefaultTransmission) Stop() error {
 	// signal processResponses to stop
 	if d.responseCanceler != nil {