feat: enable sampling for stdout logger

config.md

@@ -1,7 +1,7 @@
 # Honeycomb Refinery Configuration Documentation
 
 This is the documentation for the configuration file for Honeycomb's Refinery.
-It was automatically generated on 2023-10-31 at 20:19:42 UTC.
+It was automatically generated on 2023-11-02 at 19:50:49 UTC.
 
 ## The Config file
 
@@ -403,7 +403,7 @@ The sampling algorithm attempts to make sure that the average throughput approxi
 
 - Not eligible for live reload.
 - Type: `float`
-- Default: `10`
+- Default: `5`
 - Example: `10`
 
 ## Stdout Logger
@@ -420,6 +420,26 @@ Structured controls whether to use structured logging.
 - Not eligible for live reload.
 - Type: `bool`
 
+### `SamplerEnabled`
+
+SamplerEnabled controls whether logs are sampled before sending to stdout.
+
+The sample rate is controlled by the `SamplerThroughput` setting.
+
+- Not eligible for live reload.
+- Type: `bool`
+
+### `SamplerThroughput`
+
+SamplerThroughput is the sampling throughput for logs in events per second.
+
+The sampling algorithm attempts to make sure that the average throughput approximates this value, while also ensuring that all unique logs arrive at stdout at least once per sampling period.
+
+- Not eligible for live reload.
+- Type: `float`
+- Default: `5`
+- Example: `10`
+
 ## Prometheus Metrics
 
 `PrometheusMetrics` contains configuration for Refinery's internally-generated metrics as made available through Prometheus.

config/config_test.go

@@ -595,6 +595,8 @@ func TestStdoutLoggerConfig(t *testing.T) {
 		"General.ConfigurationVersion", 2,
 		"Logger.Type", "stdout",
 		"StdoutLogger.Structured", true,
+		"StdoutLogger.SamplerThroughput", 10,
+		"StdoutLogger.SamplerEnabled", true,
 	)
 	rm := makeYAML("ConfigVersion", 2)
 	config, rules := createTempConfigs(t, cm, rm)
@@ -609,6 +611,8 @@ func TestStdoutLoggerConfig(t *testing.T) {
 	assert.NoError(t, err)
 
 	assert.True(t, loggerConfig.Structured)
+	assert.True(t, loggerConfig.SamplerEnabled)
+	assert.Equal(t, 10, loggerConfig.SamplerThroughput)
 }
 
 func TestStdoutLoggerConfigDefaults(t *testing.T) {
@@ -627,6 +631,8 @@ func TestStdoutLoggerConfigDefaults(t *testing.T) {
 	assert.NoError(t, err)
 
 	assert.False(t, loggerConfig.Structured)
+	assert.False(t, loggerConfig.SamplerEnabled)
+	assert.Equal(t, 5, loggerConfig.SamplerThroughput)
 }
 func TestDatasetPrefix(t *testing.T) {
 	cm := makeYAML(

config/file_config.go

@@ -276,6 +276,8 @@ type HoneycombLoggerConfig struct {
 
 type StdoutLoggerConfig struct {
 	Structured bool `yaml:"Structured" default:"false"`
+	SamplerEnabled	bool   `yaml:"SamplerEnabled" `
+	SamplerThroughput int    `yaml:"SamplerThroughput" default:"5"`
 }
 
 type PrometheusMetricsConfig struct {

config/metadata/configMeta.yaml

@@ -460,7 +460,7 @@ groups:
       - name: SamplerThroughput
         valuetype: showexample
         type: float
-        default: 10
+        default: 5
         example: 10
         reload: false
         summary: is the sampling throughput for logs in events per second.
@@ -484,6 +484,27 @@ groups:
           `true` generates structured logs (JSON).
           `false` generates plain text logs.
 
+      - name: SamplerEnabled
+        type: bool
+        valuetype: nondefault
+        default: false
+        reload: false
+        summary: controls whether logs are sampled before sending to stdout.
+        description: >
+          The sample rate is controlled by the `SamplerThroughput` setting.
+
+      - name: SamplerThroughput
+        valuetype: showexample
+        type: float
+        default: 5
+        example: 10
+        reload: false
+        summary: is the sampling throughput for logs in events per second.
+        description: >
+          The sampling algorithm attempts to make sure that the average
+          throughput approximates this value, while also ensuring that all
+          unique logs arrive at stdout at least once per sampling
+          period.
   - name: PrometheusMetrics
     title: "Prometheus Metrics"
     description: contains configuration for Refinery's internally-generated metrics as made available through Prometheus.

logger/logrus.go

@@ -1,8 +1,13 @@
 package logger
 
 import (
+	"fmt"
+	"math/rand"
+	"time"
+
 	"github.com/sirupsen/logrus"
 
+	"github.com/honeycombio/dynsampler-go"
 	"github.com/honeycombio/refinery/config"
 )
 
@@ -13,13 +18,16 @@ type StdoutLogger struct {
 
 	logger *logrus.Logger
 	level  logrus.Level
+
+	sampler dynsampler.Sampler
 }
 
 var _ = Logger((*StdoutLogger)(nil))
 
 type LogrusEntry struct {
 	entry *logrus.Entry
 	level logrus.Level
+	sampler dynsampler.Sampler
 }
 
 func (l *StdoutLogger) Start() error {
@@ -34,6 +42,18 @@ func (l *StdoutLogger) Start() error {
 		l.logger.SetFormatter(&logrus.JSONFormatter{})
 	}
 
+	if cfg.SamplerEnabled {
+		l.sampler = &dynsampler.PerKeyThroughput{
+			ClearFrequencyDuration: 10*time.Second,
+			PerKeyThroughputPerSec: cfg.SamplerThroughput,
+			MaxKeys: 1000,
+		}
+		err = l.sampler.Start()
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -45,6 +65,7 @@ func (l *StdoutLogger) Debug() Entry {
 	return &LogrusEntry{
 		entry: logrus.NewEntry(l.logger),
 		level: logrus.DebugLevel,
+		sampler: l.sampler,
 	}
 }
 
@@ -56,6 +77,7 @@ func (l *StdoutLogger) Info() Entry {
 	return &LogrusEntry{
 		entry: logrus.NewEntry(l.logger),
 		level: logrus.InfoLevel,
+		sampler: l.sampler,
 	}
 }
 
@@ -67,6 +89,7 @@ func (l *StdoutLogger) Warn() Entry {
 	return &LogrusEntry{
 		entry: logrus.NewEntry(l.logger),
 		level: logrus.WarnLevel,
+		sampler: l.sampler,
 	}
 }
 
@@ -78,6 +101,7 @@ func (l *StdoutLogger) Error() Entry {
 	return &LogrusEntry{
 		entry: logrus.NewEntry(l.logger),
 		level: logrus.ErrorLevel,
+		sampler: l.sampler,
 	}
 }
 
@@ -98,30 +122,54 @@ func (l *LogrusEntry) WithField(key string, value interface{}) Entry {
 	return &LogrusEntry{
 		entry: l.entry.WithField(key, value),
 		level: l.level,
+		sampler: l.sampler,
 	}
 }
 
 func (l *LogrusEntry) WithString(key string, value string) Entry {
 	return &LogrusEntry{
 		entry: l.entry.WithField(key, value),
 		level: l.level,
+		sampler: l.sampler,
 	}
 }
 
 func (l *LogrusEntry) WithFields(fields map[string]interface{}) Entry {
 	return &LogrusEntry{
 		entry: l.entry.WithFields(fields),
 		level: l.level,
+		sampler: l.sampler,
 	}
 }
 
 func (l *LogrusEntry) Logf(f string, args ...interface{}) {
+	if l.sampler != nil {
+		// use the level and format string as the key to sample on
+		// this will give us a different sample rate for each level and format string
+		// and avoid high cardinality args making the throughput sampler less effective
+		rate := l.sampler.GetSampleRate(fmt.Sprintf("%s:%s", l.level, f))
+		if shouldDrop(uint(rate)){
+			return 
+		}
+		l.entry.WithField("SampleRate", rate)
+	}
+
 	switch l.level {
 	case logrus.DebugLevel:
 		l.entry.Debugf(f, args...)
 	case logrus.InfoLevel:
 		l.entry.Infof(f, args...)
+	case logrus.WarnLevel:
+		l.entry.Warnf(f, args...)
 	default:
 		l.entry.Errorf(f, args...)
 	}
 }
+
+func shouldDrop(rate uint) bool {
+	if rate <= 1 {
+		return false
+	}
+
+	return rand.Intn(int(rate)) != 0
+}

refinery_config.md

@@ -385,7 +385,7 @@ The sampling algorithm attempts to make sure that the average throughput approxi
 
 - Not eligible for live reload.
 - Type: `float`
-- Default: `10`
+- Default: `5`
 - Example: `10`
 
 ## Stdout Logger
@@ -403,6 +403,26 @@ Only used if `Logger.Type` is "stdout".
 - Not eligible for live reload.
 - Type: `bool`
 
+### `SamplerEnabled`
+
+`SamplerEnabled` controls whether logs are sampled before sending to stdout.
+
+The sample rate is controlled by the `SamplerThroughput` setting.
+
+- Not eligible for live reload.
+- Type: `bool`
+
+### `SamplerThroughput`
+
+`SamplerThroughput` is the sampling throughput for logs in events per second.
+
+The sampling algorithm attempts to make sure that the average throughput approximates this value, while also ensuring that all unique logs arrive at stdout at least once per sampling period.
+
+- Not eligible for live reload.
+- Type: `float`
+- Default: `5`
+- Example: `10`
+
 ## Prometheus Metrics
 
 `PrometheusMetrics` contains configuration for Refinery's internally-generated metrics as made available through Prometheus.

rules.md

@@ -1,7 +1,7 @@
 # Honeycomb Refinery Rules Documentation
 
 This is the documentation for the rules configuration for Honeycomb's Refinery.
-It was automatically generated on 2023-10-31 at 20:19:43 UTC.
+It was automatically generated on 2023-11-02 at 19:50:50 UTC.
 
 ## The Rules file
 

tools/convert/templates/configV2.tmpl

@@ -2,7 +2,7 @@
 ## Honeycomb Refinery Configuration ##
 ######################################
 #
-# created {{ now }} from {{ .Input }} using a template generated on 2023-10-31 at 20:19:40 UTC
+# created {{ now }} from {{ .Input }} using a template generated on 2023-11-02 at 19:50:45 UTC
 
 # This file contains a configuration for the Honeycomb Refinery. It is in YAML
 # format, organized into named groups, each of which contains a set of
@@ -404,7 +404,7 @@ HoneycombLogger:
     ## throughput approximates this value, while also ensuring that all
     ## unique logs arrive at Honeycomb at least once per sampling period.
     ##
-    ## default: 10
+    ## default: 5
     ## Not eligible for live reload.
     # SamplerThroughput: 10
 
@@ -423,6 +423,25 @@ StdoutLogger:
     ## Not eligible for live reload.
     {{ nonDefaultOnly .Data "Structured" "Structured" false }}
 
+    ## SamplerEnabled controls whether logs are sampled before sending to
+    ## stdout.
+    ##
+    ## The sample rate is controlled by the `SamplerThroughput` setting.
+    ##
+    ## Not eligible for live reload.
+    {{ nonDefaultOnly .Data "SamplerEnabled" "SamplerEnabled" false }}
+
+    ## SamplerThroughput is the sampling throughput for logs in events per
+    ## second.
+    ##
+    ## The sampling algorithm attempts to make sure that the average
+    ## throughput approximates this value, while also ensuring that all
+    ## unique logs arrive at stdout at least once per sampling period.
+    ##
+    ## default: 5
+    ## Not eligible for live reload.
+    # SamplerThroughput: 10
+
 ########################
 ## Prometheus Metrics ##
 ########################