feat: add incoming/peer_router_otlp metrics for otlp traffic (#1474)

## Which problem is this PR solving?

We have don't have metrics for otlp traffic in Refinery. It's hard to
see traffic load from otlp requests

## Short description of the changes

- add incoming/peer_router_otlp metrics
- also fixes missing registration for some existing metrics

collect/cache/cuckoo.go

@@ -48,6 +48,8 @@ var cuckooTraceCheckerMetrics = []metrics.Metadata{
 	{Name: CurrentCapacity, Type: metrics.Gauge, Unit: metrics.Dimensionless, Description: "current capacity of the cuckoo filter"},
 	{Name: FutureLoadFactor, Type: metrics.Gauge, Unit: metrics.Percent, Description: "the fraction of slots occupied in the future cuckoo filter"},
 	{Name: CurrentLoadFactor, Type: metrics.Gauge, Unit: metrics.Percent, Description: "the fraction of slots occupied in the current cuckoo filter"},
+	{Name: AddQueueFull, Type: metrics.Counter, Unit: metrics.Dimensionless, Description: "the number of times the add queue was full and a drop decision was dropped"},
+	{Name: AddQueueLockTime, Type: metrics.Histogram, Unit: metrics.Microseconds, Description: "the time spent holding the add queue lock"},
 }
 
 func NewCuckooTraceChecker(capacity uint, m metrics.Metrics) *CuckooTraceChecker {

collect/collect.go

@@ -154,6 +154,7 @@ var inMemCollectorMetrics = []metrics.Metadata{
 	{Name: "trace_aggregate_sample_rate", Type: metrics.Histogram, Unit: metrics.Dimensionless, Description: "aggregate sample rate of both kept and dropped traces"},
 	{Name: "collector_redistribute_traces_duration_ms", Type: metrics.Histogram, Unit: metrics.Milliseconds, Description: "duration of redistributing traces to peers"},
 	{Name: "collector_collect_loop_duration_ms", Type: metrics.Histogram, Unit: metrics.Milliseconds, Description: "duration of the collect loop, the primary event processing goroutine"},
+	{Name: "collector_send_expired_traces_in_cache_dur_ms", Type: metrics.Histogram, Unit: metrics.Milliseconds, Description: "duration of sending expired traces in cache"},
 	{Name: "collector_outgoing_queue", Type: metrics.Histogram, Unit: metrics.Dimensionless, Description: "number of traces waiting to be send to upstream"},
 	{Name: "collector_drop_decision_batch_count", Type: metrics.Histogram, Unit: metrics.Dimensionless, Description: "number of drop decisions sent in a batch"},
 	{Name: "collector_expired_traces_missing_decisions", Type: metrics.Gauge, Unit: metrics.Dimensionless, Description: "number of decision spans forwarded for expired traces missing trace decision"},

route/route.go

@@ -132,6 +132,8 @@ var routerMetrics = []metrics.Metadata{
 	{Name: "_router_dropped", Type: metrics.Counter, Unit: metrics.Dimensionless, Description: "the number of events dropped because the channel was full"},
 	{Name: "_router_nonspan", Type: metrics.Counter, Unit: metrics.Dimensionless, Description: "the number of non-span events received"},
 	{Name: "_router_peer", Type: metrics.Counter, Unit: metrics.Dimensionless, Description: "the number of spans proxied to a peer"},
+	{Name: "_router_batch", Type: metrics.Counter, Unit: metrics.Dimensionless, Description: "the number of batches of events received"},
+	{Name: "_router_otlp", Type: metrics.Counter, Unit: metrics.Dimensionless, Description: "the number of batches of otlp requests received"},
 }
 
 // LnS spins up the Listen and Serve portion of the router. A router is
@@ -536,6 +538,8 @@ func (router *Router) processOTLPRequest(
 	apiKey string,
 	incomingUserAgent string) error {
 
+	router.Metrics.Increment(router.incomingOrPeer + "_router_otlp")
+
 	var requestID types.RequestIDContextKey
 	apiHost := router.Config.GetHoneycombAPI()