feat: Add sampler metrics (#714)

## Which problem is this PR solving? - Adds sampler-specific metrics now that dynsampler-go supports them. ## Short description of the changes - Add getMetricType function to get the type from a metric by its name. This is required because Metrics can only increment counters, not set them to a value, so we need to treat metrics ending in _count specially by keeping track of their previous value so we can get a delta. - Register all metrics provided by the sampler - Record them on every call to GetSampleRate - Add metrics to DeterministicSampler following the same pattern - Update rules to do the same (and add support for new downstream sampler types) Closes #704. Closes #554.
honeycombio · Jun 12, 2023 · a943a8d · a943a8d
1 parent caa95a0
commit a943a8d
Show file tree

Hide file tree

Showing 8 changed files with 145 additions and 44 deletions.
diff --git a/sample/deterministic.go b/sample/deterministic.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/honeycombio/refinery/config"
 	"github.com/honeycombio/refinery/logger"
+	"github.com/honeycombio/refinery/metrics"
 	"github.com/honeycombio/refinery/types"
 )
 
@@ -15,17 +16,23 @@ import (
 const shardingSalt = "5VQ8l2jE5aJLPVqk"
 
 type DeterministicSampler struct {
-	Config *config.DeterministicSamplerConfig
-	Logger logger.Logger
+	Config  *config.DeterministicSamplerConfig
+	Logger  logger.Logger
+	Metrics metrics.Metrics
 
 	sampleRate int
 	upperBound uint32
+	prefix     string
 }
 
 func (d *DeterministicSampler) Start() error {
 	d.Logger.Debug().Logf("Starting DeterministicSampler")
 	defer func() { d.Logger.Debug().Logf("Finished starting DeterministicSampler") }()
 	d.sampleRate = d.Config.SampleRate
+	d.prefix = "deterministic_"
+	if d.Metrics == nil {
+		d.Metrics = &metrics.NullMetrics{}
+	}
 
 	// Get the actual upper bound - the largest possible value divided by
 	// the sample rate. In the case where the sample rate is 1, this should
@@ -41,5 +48,12 @@ func (d *DeterministicSampler) GetSampleRate(trace *types.Trace) (rate uint, kee
 	}
 	sum := sha1.Sum([]byte(trace.TraceID + shardingSalt))
 	v := binary.BigEndian.Uint32(sum[:4])
-	return uint(d.sampleRate), v <= d.upperBound, "deterministic/chance", ""
+	shouldKeep := v <= d.upperBound
+	if shouldKeep {
+		d.Metrics.Increment(d.prefix + "num_kept")
+	} else {
+		d.Metrics.Increment(d.prefix + "num_dropped")
+	}
+
+	return uint(d.sampleRate), shouldKeep, "deterministic/chance", ""
 }
diff --git a/sample/dynamic.go b/sample/dynamic.go
@@ -20,6 +20,8 @@ type DynamicSampler struct {
 	sampleRate     int64
 	clearFrequency config.Duration
 	maxKeys        int
+	prefix         string
+	lastMetrics    map[string]int64
 
 	key *traceKey
 
@@ -39,6 +41,7 @@ func (d *DynamicSampler) Start() error {
 	if d.maxKeys == 0 {
 		d.maxKeys = 500
 	}
+	d.prefix = "dynamic_"
 
 	// spin up the actual dynamic sampler
 	d.dynsampler = &dynsampler.AvgSampleRate{
@@ -49,9 +52,13 @@ func (d *DynamicSampler) Start() error {
 	d.dynsampler.Start()
 
 	// Register statistics this package will produce
-	d.Metrics.Register("dynsampler_num_dropped", "counter")
-	d.Metrics.Register("dynsampler_num_kept", "counter")
-	d.Metrics.Register("dynsampler_sample_rate", "histogram")
+	d.lastMetrics = d.dynsampler.GetMetrics(d.prefix)
+	for name := range d.lastMetrics {
+		d.Metrics.Register(name, getMetricType(name))
+	}
+	d.Metrics.Register(d.prefix+"num_dropped", "counter")
+	d.Metrics.Register(d.prefix+"num_kept", "counter")
+	d.Metrics.Register(d.prefix+"sample_rate", "histogram")
 
 	return nil
 }
@@ -70,10 +77,19 @@ func (d *DynamicSampler) GetSampleRate(trace *types.Trace) (rate uint, keep bool
 		"trace_id":    trace.TraceID,
 	}).Logf("got sample rate and decision")
 	if shouldKeep {
-		d.Metrics.Increment("dynsampler_num_kept")
+		d.Metrics.Increment(d.prefix + "num_kept")
 	} else {
-		d.Metrics.Increment("dynsampler_num_dropped")
+		d.Metrics.Increment(d.prefix + "num_dropped")
+	}
+	d.Metrics.Histogram(d.prefix+"sample_rate", float64(rate))
+	for name, val := range d.dynsampler.GetMetrics(d.prefix) {
+		switch getMetricType(name) {
+		case "counter":
+			delta := val - d.lastMetrics[name]
+			d.Metrics.Count(name, delta)
+		case "gauge":
+			d.Metrics.Gauge(name, val)
+		}
 	}
-	d.Metrics.Histogram("dynsampler_sample_rate", float64(rate))
 	return rate, shouldKeep, "dynamic", key
 }
diff --git a/sample/dynamic_ema.go b/sample/dynamic_ema.go
@@ -24,6 +24,8 @@ type EMADynamicSampler struct {
 	burstMultiple       float64
 	burstDetectionDelay uint
 	maxKeys             int
+	prefix              string
+	lastMetrics         map[string]int64
 
 	key *traceKey
 
@@ -44,6 +46,7 @@ func (d *EMADynamicSampler) Start() error {
 	if d.maxKeys == 0 {
 		d.maxKeys = 500
 	}
+	d.prefix = "emadynamic_"
 
 	// spin up the actual dynamic sampler
 	d.dynsampler = &dynsampler.EMASampleRate{
@@ -58,10 +61,13 @@ func (d *EMADynamicSampler) Start() error {
 	d.dynsampler.Start()
 
 	// Register statistics this package will produce
-	d.Metrics.Register("dynsampler_num_dropped", "counter")
-	d.Metrics.Register("dynsampler_num_kept", "counter")
-	d.Metrics.Register("dynsampler_sample_rate", "histogram")
-
+	d.lastMetrics = d.dynsampler.GetMetrics(d.prefix)
+	for name := range d.lastMetrics {
+		d.Metrics.Register(name, getMetricType(name))
+	}
+	d.Metrics.Register(d.prefix+"num_dropped", "counter")
+	d.Metrics.Register(d.prefix+"num_kept", "counter")
+	d.Metrics.Register(d.prefix+"sample_rate", "histogram")
 	return nil
 }
 
@@ -79,10 +85,19 @@ func (d *EMADynamicSampler) GetSampleRate(trace *types.Trace) (rate uint, keep b
 		"trace_id":    trace.TraceID,
 	}).Logf("got sample rate and decision")
 	if shouldKeep {
-		d.Metrics.Increment("dynsampler_num_kept")
+		d.Metrics.Increment(d.prefix + "num_kept")
 	} else {
-		d.Metrics.Increment("dynsampler_num_dropped")
+		d.Metrics.Increment(d.prefix + "num_dropped")
+	}
+	d.Metrics.Histogram(d.prefix+"sample_rate", float64(rate))
+	for name, val := range d.dynsampler.GetMetrics(d.prefix) {
+		switch getMetricType(name) {
+		case "counter":
+			delta := val - d.lastMetrics[name]
+			d.Metrics.Count(name, delta)
+		case "gauge":
+			d.Metrics.Gauge(name, val)
+		}
 	}
-	d.Metrics.Histogram("dynsampler_sample_rate", float64(rate))
 	return rate, shouldKeep, "emadynamic", key
 }
diff --git a/sample/ema_throughput.go b/sample/ema_throughput.go
@@ -25,6 +25,8 @@ type EMAThroughputSampler struct {
 	burstMultiple        float64
 	burstDetectionDelay  uint
 	maxKeys              int
+	prefix               string
+	lastMetrics          map[string]int64
 
 	key *traceKey
 
@@ -46,6 +48,7 @@ func (d *EMAThroughputSampler) Start() error {
 	if d.maxKeys == 0 {
 		d.maxKeys = 500
 	}
+	d.prefix = "emathroughput_"
 
 	// spin up the actual dynamic sampler
 	d.dynsampler = &dynsampler.EMAThroughput{
@@ -61,6 +64,10 @@ func (d *EMAThroughputSampler) Start() error {
 	d.dynsampler.Start()
 
 	// Register statistics this package will produce
+	d.lastMetrics = d.dynsampler.GetMetrics(d.prefix)
+	for name := range d.lastMetrics {
+		d.Metrics.Register(name, getMetricType(name))
+	}
 	d.Metrics.Register("dynsampler_num_dropped", "counter")
 	d.Metrics.Register("dynsampler_num_kept", "counter")
 	d.Metrics.Register("dynsampler_sample_rate", "histogram")
@@ -82,10 +89,19 @@ func (d *EMAThroughputSampler) GetSampleRate(trace *types.Trace) (rate uint, kee
 		"trace_id":    trace.TraceID,
 	}).Logf("got sample rate and decision")
 	if shouldKeep {
-		d.Metrics.Increment("dynsampler_num_kept")
+		d.Metrics.Increment(d.prefix + "num_kept")
 	} else {
-		d.Metrics.Increment("dynsampler_num_dropped")
+		d.Metrics.Increment(d.prefix + "num_dropped")
+	}
+	d.Metrics.Histogram(d.prefix+"sample_rate", float64(rate))
+	for name, val := range d.dynsampler.GetMetrics(d.prefix) {
+		switch getMetricType(name) {
+		case "counter":
+			delta := val - d.lastMetrics[name]
+			d.Metrics.Count(name, delta)
+		case "gauge":
+			d.Metrics.Gauge(name, val)
+		}
 	}
-	d.Metrics.Histogram("dynsampler_sample_rate", float64(rate))
 	return rate, shouldKeep, "emathroughput", key
 }
diff --git a/sample/rules.go b/sample/rules.go
@@ -17,15 +17,17 @@ type RulesBasedSampler struct {
 	Logger   logger.Logger
 	Metrics  metrics.Metrics
 	samplers map[string]Sampler
+	prefix   string
 }
 
 func (s *RulesBasedSampler) Start() error {
 	s.Logger.Debug().Logf("Starting RulesBasedSampler")
 	defer func() { s.Logger.Debug().Logf("Finished starting RulesBasedSampler") }()
+	s.prefix = "rulesbased_"
 
-	s.Metrics.Register("rulessampler_num_dropped", "counter")
-	s.Metrics.Register("rulessampler_num_kept", "counter")
-	s.Metrics.Register("rulessampler_sample_rate", "histogram")
+	s.Metrics.Register(s.prefix+"num_dropped", "counter")
+	s.Metrics.Register(s.prefix+"num_kept", "counter")
+	s.Metrics.Register(s.prefix+"sample_rate", "histogram")
 
 	s.samplers = make(map[string]Sampler)
 
@@ -48,6 +50,10 @@ func (s *RulesBasedSampler) Start() error {
 				sampler = &EMADynamicSampler{Config: rule.Sampler.EMADynamicSampler, Logger: s.Logger, Metrics: s.Metrics}
 			} else if rule.Sampler.TotalThroughputSampler != nil {
 				sampler = &TotalThroughputSampler{Config: rule.Sampler.TotalThroughputSampler, Logger: s.Logger, Metrics: s.Metrics}
+			} else if rule.Sampler.EMAThroughputSampler != nil {
+				sampler = &EMAThroughputSampler{Config: rule.Sampler.EMAThroughputSampler, Logger: s.Logger, Metrics: s.Metrics}
+			} else if rule.Sampler.WindowedThroughputSampler != nil {
+				sampler = &WindowedThroughputSampler{Config: rule.Sampler.WindowedThroughputSampler, Logger: s.Logger, Metrics: s.Metrics}
 			} else {
 				s.Logger.Debug().WithFields(map[string]interface{}{
 					"rule_name": rule.Name,
@@ -116,11 +122,11 @@ func (s *RulesBasedSampler) GetSampleRate(trace *types.Trace) (rate uint, keep b
 				reason += rule.Name
 			}
 
-			s.Metrics.Histogram("rulessampler_sample_rate", float64(rule.SampleRate))
+			s.Metrics.Histogram(s.prefix+"sample_rate", float64(rule.SampleRate))
 			if keep {
-				s.Metrics.Increment("rulessampler_num_kept")
+				s.Metrics.Increment(s.prefix + "num_kept")
 			} else {
-				s.Metrics.Increment("rulessampler_num_dropped")
+				s.Metrics.Increment(s.prefix + "num_dropped")
 			}
 			logger.WithFields(map[string]interface{}{
 				"rate":      rate,

diff --git a/sample/sample.go b/sample/sample.go
@@ -3,6 +3,7 @@ package sample
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/honeycombio/refinery/config"
 	"github.com/honeycombio/refinery/logger"
@@ -40,7 +41,7 @@ func (s *SamplerFactory) GetSamplerImplementationForKey(samplerKey string, isLeg
 
 	switch c := c.(type) {
 	case *config.DeterministicSamplerConfig:
-		sampler = &DeterministicSampler{Config: c, Logger: s.Logger}
+		sampler = &DeterministicSampler{Config: c, Logger: s.Logger, Metrics: s.Metrics}
 	case *config.DynamicSamplerConfig:
 		sampler = &DynamicSampler{Config: c, Logger: s.Logger, Metrics: s.Metrics}
 	case *config.EMADynamicSamplerConfig:
@@ -64,3 +65,10 @@ func (s *SamplerFactory) GetSamplerImplementationForKey(samplerKey string, isLeg
 
 	return sampler
 }
+
+func getMetricType(name string) string {
+	if strings.HasSuffix(name, "_count") {
+		return "counter"
+	}
+	return "gauge"
+}
diff --git a/sample/totalthroughput.go b/sample/totalthroughput.go
@@ -20,6 +20,8 @@ type TotalThroughputSampler struct {
 	goalThroughputPerSec int64
 	clearFrequency       config.Duration
 	maxKeys              int
+	prefix               string
+	lastMetrics          map[string]int64
 
 	key *traceKey
 
@@ -43,6 +45,7 @@ func (d *TotalThroughputSampler) Start() error {
 	if d.maxKeys == 0 {
 		d.maxKeys = 500
 	}
+	d.prefix = "totalthroughput_"
 
 	// spin up the actual dynamic sampler
 	d.dynsampler = &dynsampler.TotalThroughput{
@@ -53,9 +56,13 @@ func (d *TotalThroughputSampler) Start() error {
 	d.dynsampler.Start()
 
 	// Register statistics this package will produce
-	d.Metrics.Register("dynsampler_num_dropped", "counter")
-	d.Metrics.Register("dynsampler_num_kept", "counter")
-	d.Metrics.Register("dynsampler_sample_rate", "histogram")
+	d.lastMetrics = d.dynsampler.GetMetrics(d.prefix)
+	for name := range d.lastMetrics {
+		d.Metrics.Register(name, getMetricType(name))
+	}
+	d.Metrics.Register(d.prefix+"num_dropped", "counter")
+	d.Metrics.Register(d.prefix+"num_kept", "counter")
+	d.Metrics.Register(d.prefix+"sample_rate", "histogram")
 
 	return nil
 }
@@ -74,10 +81,19 @@ func (d *TotalThroughputSampler) GetSampleRate(trace *types.Trace) (rate uint, k
 		"trace_id":    trace.TraceID,
 	}).Logf("got sample rate and decision")
 	if shouldKeep {
-		d.Metrics.Increment("dynsampler_num_kept")
+		d.Metrics.Increment(d.prefix + "num_kept")
 	} else {
-		d.Metrics.Increment("dynsampler_num_dropped")
+		d.Metrics.Increment(d.prefix + "num_dropped")
+	}
+	d.Metrics.Histogram(d.prefix+"sample_rate", float64(rate))
+	for name, val := range d.dynsampler.GetMetrics(d.prefix) {
+		switch getMetricType(name) {
+		case "counter":
+			delta := val - d.lastMetrics[name]
+			d.Metrics.Count(name, delta)
+		case "gauge":
+			d.Metrics.Gauge(name, val)
+		}
 	}
-	d.Metrics.Histogram("dynsampler_sample_rate", float64(rate))
 	return rate, shouldKeep, "totalthroughput", key
 }