Add support for environment and dataset rules with same names

README.md

@@ -70,6 +70,36 @@ By default, a Refinery process will register itself in Redis using its local hos
 In environments where domain name resolution is slow or unreliable, override the reliance on name lookups by specifying the name of the peering network interface with the `IdentifierInterfaceName` configuration option.
 See the [Refinery documentation](https://docs.honeycomb.io/manage-data-volume/refinery/) for more details on tuning a cluster.
 
+
+### Mixing Legacy and Environment & Services Rule Definitions
+
+With the change to support environemt and services in Honeycomb, some users will want to support both sending telemetry to a legacy dataset and a new environment called the same thing (eg `production`).
+
+This can be accomplished by leveraging the new `DatasetPrefix` configuration property and then using that prefix in the rules definitions for the legacy datasets.
+
+When Refinery receives telemetry using an API key associated to a legacy dataset, it will then use the prefix in the form `{prefix}.{dataset}` when trying to resolve the rules definition.
+
+For example
+config.toml
+```toml
+DatasetPrefix = "legacy"
+```
+
+rules.toml
+```toml
+# default rules
+Sampler = "DeterministicSampler"
+SampleRate = 1
+
+    [production] # environment called "production"
+    Sampler = "DeterministicSampler"
+    SampleRate = 5
+
+    [legacy.production] # dataset called "production"
+    Sampler = "DeterministicSampler"
+    SampleRate = 10
+```
+
 ## How sampling decisions are made
 
 In the configuration file, you can choose from a few sampling methods and specify options for each. The `DynamicSampler` is the most interesting and most commonly used. It uses the `AvgSampleRate` algorithm from the [`dynsampler-go`](https://github.com/honeycombio/dynsampler-go) package. Briefly described, you configure Refinery to examine the trace for a set of fields (for example, `request.status_code` and `request.method`). It collects all the values found in those fields anywhere in the trace (eg "200" and "GET") together into a key it hands to the dynsampler. The dynsampler code will look at the frequency that key appears during the previous 30 seconds (or other value set by the `ClearFrequencySec` setting) and use that to hand back a desired sample rate. More frequent keys are sampled more heavily, so that an even distribution of traffic across the keyspace is represented in Honeycomb.

collect/collect.go

@@ -449,9 +449,9 @@ func (i *InMemCollector) send(trace *types.Trace) {
 		logFields["environment"] = samplerKey
 	}
 
-	// use sampler key to find sampler, crete and cache if not found
+	// use sampler key to find sampler; create and cache if not found
 	if sampler, found = i.datasetSamplers[samplerKey]; !found {
-		sampler = i.SamplerFactory.GetSamplerImplementationForDataset(samplerKey)
+		sampler = i.SamplerFactory.GetSamplerImplementationForKey(samplerKey, isLegacyKey)
 		i.datasetSamplers[samplerKey] = sampler
 	}
 

collect/collect_test.go

@@ -186,7 +186,7 @@ func TestDryRunMode(t *testing.T) {
 		Config: conf,
 		Logger: &logger.NullLogger{},
 	}
-	sampler := samplerFactory.GetSamplerImplementationForDataset("test")
+	sampler := samplerFactory.GetSamplerImplementationForKey("test", true)
 	coll := &InMemCollector{
 		Config:         conf,
 		Logger:         &logger.NullLogger{},

config/config.go

@@ -139,4 +139,6 @@ type Config interface {
 	GetAddHostMetadataToTrace() bool
 
 	GetEnvironmentCacheTTL() time.Duration
+
+	GetDatasetPrefix() string
 }

config/config_test.go

@@ -652,3 +652,40 @@ func TestHoneycombLoggerConfigDefaults(t *testing.T) {
 	assert.Equal(t, false, loggerConfig.LoggerSamplerEnabled)
 	assert.Equal(t, 5, loggerConfig.LoggerSamplerThroughput)
 }
+
+func TestDatasetPrefix(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "")
+	assert.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	configFile, err := ioutil.TempFile(tmpDir, "*.toml")
+	assert.NoError(t, err)
+
+	_, err = configFile.Write([]byte(`
+	DatasetPrefix = "dataset"
+
+	[InMemCollector]
+		CacheCapacity=1000
+
+	[HoneycombMetrics]
+		MetricsHoneycombAPI="http://honeycomb.io"
+		MetricsAPIKey="1234"
+		MetricsDataset="testDatasetName"
+		MetricsReportingInterval=3
+
+	[HoneycombLogger]
+		LoggerHoneycombAPI="http://honeycomb.io"
+		LoggerAPIKey="1234"
+		LoggerDataset="loggerDataset"
+	`))
+	assert.NoError(t, err)
+	configFile.Close()
+
+	rulesFile, err := ioutil.TempFile(tmpDir, "*.toml")
+	assert.NoError(t, err)
+
+	c, err := NewConfig(configFile.Name(), rulesFile.Name(), func(err error) {})
+	assert.NoError(t, err)
+
+	assert.Equal(t, "dataset", c.GetDatasetPrefix())
+}

config/file_config.go

@@ -49,6 +49,7 @@ type configContents struct {
 	InMemCollector            InMemoryCollectorCacheCapacity `validate:"required"`
 	AddHostMetadataToTrace    bool
 	EnvironmentCacheTTL       time.Duration
+	DatasetPrefix             string
 }
 
 type InMemoryCollectorCacheCapacity struct {
@@ -736,3 +737,10 @@ func (f *fileConfig) GetEnvironmentCacheTTL() time.Duration {
 
 	return f.conf.EnvironmentCacheTTL
 }
+
+func (f *fileConfig) GetDatasetPrefix() string {
+	f.mux.RLock()
+	defer f.mux.RUnlock()
+
+	return f.conf.DatasetPrefix
+}

config/mock.go

@@ -71,6 +71,7 @@ type MockConfig struct {
 	DryRunFieldName               string
 	AddHostMetadataToTrace        bool
 	EnvironmentCacheTTL           time.Duration
+	DatasetPrefix                 string
 
 	Mux sync.RWMutex
 }
@@ -328,3 +329,10 @@ func (f *MockConfig) GetEnvironmentCacheTTL() time.Duration {
 
 	return f.EnvironmentCacheTTL
 }
+
+func (f *MockConfig) GetDatasetPrefix() string {
+	f.Mux.RLock()
+	defer f.Mux.RUnlock()
+
+	return f.DatasetPrefix
+}

sample/sample.go

@@ -1,6 +1,7 @@
 package sample
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/honeycombio/refinery/config"
@@ -21,10 +22,16 @@ type SamplerFactory struct {
 	Metrics metrics.Metrics `inject:"metrics"`
 }
 
-// GetSamplerImplementationForDataset returns the sampler implementation for the dataset,
-// or nil if it is not defined
-func (s *SamplerFactory) GetSamplerImplementationForDataset(dataset string) Sampler {
-	c, err := s.Config.GetSamplerConfigForDataset(dataset)
+// GetSamplerImplementationForKey returns the sampler implementation for the given
+// samplerKey (dataset for legacy keys, environment otherwise), or nil if it is not defined
+func (s *SamplerFactory) GetSamplerImplementationForKey(samplerKey string, isLegacyKey bool) Sampler {
+	if isLegacyKey {
+		if prefix := s.Config.GetDatasetPrefix(); prefix != "" {
+			samplerKey = fmt.Sprintf("%s.%s", prefix, samplerKey)
+		}
+	}
+
+	c, err := s.Config.GetSamplerConfigForDataset(samplerKey)
 	if err != nil {
 		return nil
 	}
@@ -49,11 +56,11 @@ func (s *SamplerFactory) GetSamplerImplementationForDataset(dataset string) Samp
 
 	err = sampler.Start()
 	if err != nil {
-		s.Logger.Debug().WithField("dataset", dataset).Logf("failed to start sampler")
+		s.Logger.Debug().WithField("dataset", samplerKey).Logf("failed to start sampler")
 		return nil
 	}
 
-	s.Logger.Debug().WithField("dataset", dataset).Logf("created implementation for sampler type %T", c)
+	s.Logger.Debug().WithField("dataset", samplerKey).Logf("created implementation for sampler type %T", c)
 
 	return sampler
 }

sample/sample_test.go

@@ -3,12 +3,15 @@
 package sample
 
 import (
+	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/facebookgo/inject"
 	"github.com/honeycombio/refinery/config"
 	"github.com/honeycombio/refinery/logger"
 	"github.com/honeycombio/refinery/metrics"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestDependencyInjection(t *testing.T) {
@@ -27,3 +30,80 @@ func TestDependencyInjection(t *testing.T) {
 		t.Error(err)
 	}
 }
+
+func TestDatasetPrefix(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "")
+	assert.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	configFile, err := ioutil.TempFile(tmpDir, "*.toml")
+	assert.NoError(t, err)
+
+	_, err = configFile.Write([]byte(`
+	DatasetPrefix = "dataset"
+
+	[InMemCollector]
+		CacheCapacity=1000
+
+	[HoneycombMetrics]
+		MetricsHoneycombAPI="http://honeycomb.io"
+		MetricsAPIKey="1234"
+		MetricsDataset="testDatasetName"
+		MetricsReportingInterval=3
+
+	[HoneycombLogger]
+		LoggerHoneycombAPI="http://honeycomb.io"
+		LoggerAPIKey="1234"
+		LoggerDataset="loggerDataset"
+	`))
+	assert.NoError(t, err)
+	configFile.Close()
+
+	rulesFile, err := ioutil.TempFile(tmpDir, "*.toml")
+	assert.NoError(t, err)
+
+	_, err = rulesFile.Write([]byte(`
+	Sampler = "DeterministicSampler"
+	SampleRate = 1
+
+	[production]
+		Sampler = "DeterministicSampler"
+		SampleRate = 10
+
+	[dataset.production]
+		Sampler = "DeterministicSampler"
+		SampleRate = 20
+	`))
+	assert.NoError(t, err)
+	rulesFile.Close()
+
+	c, err := config.NewConfig(configFile.Name(), rulesFile.Name(), func(err error) {})
+	assert.NoError(t, err)
+
+	assert.Equal(t, "dataset", c.GetDatasetPrefix())
+
+	factory := SamplerFactory{Config: c, Logger: &logger.NullLogger{}, Metrics: &metrics.NullMetrics{}}
+
+	defaultSampler := &DeterministicSampler{
+		Config: &config.DeterministicSamplerConfig{SampleRate: 1},
+		Logger: &logger.NullLogger{},
+	}
+	defaultSampler.Start()
+
+	envSampler := &DeterministicSampler{
+		Config: &config.DeterministicSamplerConfig{SampleRate: 10},
+		Logger: &logger.NullLogger{},
+	}
+	envSampler.Start()
+
+	datasetSampler := &DeterministicSampler{
+		Config: &config.DeterministicSamplerConfig{SampleRate: 20},
+		Logger: &logger.NullLogger{},
+	}
+	datasetSampler.Start()
+
+	assert.Equal(t, defaultSampler, factory.GetSamplerImplementationForKey("unknown", false))
+	assert.Equal(t, defaultSampler, factory.GetSamplerImplementationForKey("unknown", true))
+	assert.Equal(t, envSampler, factory.GetSamplerImplementationForKey("production", false))
+	assert.Equal(t, datasetSampler, factory.GetSamplerImplementationForKey("production", true))
+}