Skip to content

Commit

Permalink
add blank rules file to build-pkg, use old rules file as example (#196)
Browse files Browse the repository at this point in the history 
In adding more detail to the getting started docs, we noticed that the rules file had been left out of the build package. Since the existing rules file was more of an example file, and so as not to pollute the config with those examples, we renamed rules.toml to rules_complete.toml. Since defaults for the rules configuration are set in file_config.go, rules.toml is now blank until folks start adding their own rules.

Drive-by cleanup of a few things in file_config.go and regarding our config tests.
  • Loading branch information
@asdvalenzuela
asdvalenzuela authored Nov 19, 2020
1 parent 398c477 commit dfcec54
Show file tree
Hide file tree
Showing 6 changed files with 280 additions and 262 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ For more detail on how this algorithm works, please refer to the `dynsampler` pa

## Dry Run Mode

When getting started with Refinery or when updating sampling rules, it may be helpful to verify that the rules are working as expected before you start dropping traffic. By enabling dry run mode, all spans in each trace will be marked with the sampling decision in a field called `refinery_kept`. All traces will be sent to Honeycomb regardless of the sampling decision. You can then run queries in Honeycomb on this field to check your results and verify that the rules are working as intended. Enable dry run mode by adding `DryRun = true` in your configuration, as noted in `rules.toml`.
When getting started with Refinery or when updating sampling rules, it may be helpful to verify that the rules are working as expected before you start dropping traffic. By enabling dry run mode, all spans in each trace will be marked with the sampling decision in a field called `refinery_kept`. All traces will be sent to Honeycomb regardless of the sampling decision. You can then run queries in Honeycomb on this field to check your results and verify that the rules are working as intended. Enable dry run mode by adding `DryRun = true` in your configuration, as noted in `rules_complete.toml`.

When dry run mode is enabled, the metric `trace_send_kept` will increment for each trace, and the metric for `trace_send_dropped` will remain 0, reflecting that we are sending all traces to Honeycomb.

Expand Down
3 changes: 2 additions & 1 deletion build-pkg.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,5 @@ fpm -s dir -n refinery \
$GOPATH/bin/refinery-linux-${arch}=/usr/bin/refinery \
./refinery.upstart=/etc/init/refinery.conf \
./refinery.service=/lib/systemd/system/refinery.service \
./config.toml=/etc/refinery/refinery.toml
./config.toml=/etc/refinery/refinery.toml \
./rules.toml=/etc/refinery/rules.toml
72 changes: 42 additions & 30 deletions config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,36 +132,6 @@ func TestReadDefaults(t *testing.T) {
t.Error("received", d, "expected", 100*time.Millisecond)
}

if d, err := c.GetSamplerConfigForDataset("dataset-doesnt-exist"); err != nil {
assert.IsType(t, &DeterministicSamplerConfig{}, d)
}

if d, err := c.GetSamplerConfigForDataset("dataset1"); err != nil {
assert.IsType(t, &DynamicSamplerConfig{}, d)
}

if d, err := c.GetSamplerConfigForDataset("dataset4"); err != nil {
switch r := d.(type) {
case RulesBasedSamplerConfig:
assert.Len(t, r.Rule, 3)

var rule *RulesBasedSamplerRule

rule = r.Rule[0]
assert.Equal(t, 1, rule.SampleRate)
assert.Equal(t, "500 errors", rule.Name)
assert.Len(t, rule.Condition, 2)

rule = r.Rule[1]
assert.True(t, rule.Drop)
assert.Equal(t, 0, rule.SampleRate)
assert.Len(t, rule.Condition, 1)

default:
assert.Fail(t, "dataset4 should have a rules based sampler", d)
}
}

if d, _ := c.GetPeers(); !(len(d) == 1 && d[0] == "http://127.0.0.1:8081") {
t.Error("received", d, "expected", "[http://127.0.0.1:8081]")
}
Expand All @@ -182,6 +152,10 @@ func TestReadDefaults(t *testing.T) {
t.Error("received", d, "expected", "refinery_kept")
}

d, err := c.GetSamplerConfigForDataset("dataset-doesnt-exist")
assert.NoError(t, err)
assert.IsType(t, &DeterministicSamplerConfig{}, d)

type imcConfig struct {
CacheCapacity int
}
Expand All @@ -193,6 +167,44 @@ func TestReadDefaults(t *testing.T) {
assert.Equal(t, collectorConfig.CacheCapacity, 1000)
}

func TestReadRulesConfig(t *testing.T) {
c, err := NewConfig("../config.toml", "../rules_complete.toml", func(err error) {})

if err != nil {
t.Error(err)
}

d, err := c.GetSamplerConfigForDataset("dataset-doesnt-exist")
assert.NoError(t, err)
assert.IsType(t, &DeterministicSamplerConfig{}, d)

d, err = c.GetSamplerConfigForDataset("dataset1")
assert.NoError(t, err)
assert.IsType(t, &DynamicSamplerConfig{}, d)

d, err = c.GetSamplerConfigForDataset("dataset4")
assert.NoError(t, err)
switch r := d.(type) {
case *RulesBasedSamplerConfig:
assert.Len(t, r.Rule, 3)

var rule *RulesBasedSamplerRule

rule = r.Rule[0]
assert.Equal(t, 1, rule.SampleRate)
assert.Equal(t, "500 errors", rule.Name)
assert.Len(t, rule.Condition, 2)

rule = r.Rule[1]
assert.True(t, rule.Drop)
assert.Equal(t, 0, rule.SampleRate)
assert.Len(t, rule.Condition, 1)

default:
assert.Fail(t, "dataset4 should have a rules based sampler", d)
}
}

func TestPeerManagementType(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "")
assert.NoError(t, err)
Expand Down
5 changes: 2 additions & 3 deletions config/file_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,15 +124,12 @@ func NewConfig(config, rules string, errorCallback func(error)) (Config, error)
c.SetDefault("Logger", "logrus")
c.SetDefault("LoggingLevel", "debug")
c.SetDefault("Collector", "InMemCollector")
c.SetDefault("Sampler", "DynamicSampler")
c.SetDefault("Metrics", "honeycomb")
c.SetDefault("SendDelay", 2*time.Second)
c.SetDefault("TraceTimeout", 60*time.Second)
c.SetDefault("SendTicker", 100*time.Millisecond)
c.SetDefault("UpstreamBufferSize", libhoney.DefaultPendingWorkCapacity)
c.SetDefault("PeerBufferSize", libhoney.DefaultPendingWorkCapacity)
c.SetDefault("DryRun", false)
c.SetDefault("DryRunFieldName", "refinery_kept")
c.SetDefault("MaxAlloc", uint64(0))

c.SetConfigFile(config)
Expand All @@ -146,6 +143,8 @@ func NewConfig(config, rules string, errorCallback func(error)) (Config, error)

r.SetDefault("Sampler", "DeterministicSampler")
r.SetDefault("SampleRate", 1)
r.SetDefault("DryRun", false)
r.SetDefault("DryRunFieldName", "refinery_kept")

r.SetConfigFile(rules)
err = r.ReadInConfig()
Expand Down
229 changes: 2 additions & 227 deletions rules.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,230 +2,5 @@
## Sampling Rules Config ##
############################

# DryRun - If enabled, marks traces that would be dropped given current sampling rules,
# and sends all traces regardless
# DryRun = false

# DryRunFieldName - the key to add to use to add to event data when using DryRun mode above, defaults to refinery_kept
# DryRunFieldName = "refinery_kept"

# DeterministicSampler is a section of the config for manipulating the
# Deterministic Sampler implementation. This is the simplest sampling algorithm
# - it is a static sample rate, choosing traces randomly to either keep or send
# (at the appropriate rate). It is not influenced by the contents of the trace.
Sampler = "DeterministicSampler"

# SampleRate is the rate at which to sample. It indicates a ratio, where one
# sample trace is kept for every n traces seen. For example, a SampleRate of 30
# will keep 1 out of every 30 traces. The choice on whether to keep any specific
# trace is random, so the rate is approximate.
# Eligible for live reload.
SampleRate = 1

[dataset1]

# Note: If your dataset name contains a space, you will have to escape the dataset name
# using single quotes, such as ['dataset 1']

# DynamicSampler is a section of the config for manipulating the simple Dynamic Sampler
# implementation. This sampler collects the values of a number of fields from a
# trace and uses them to form a key. This key is handed to the standard dynamic
# sampler algorithm which generates a sample rate based on the frequency with
# which that key has appeared in the previous ClearFrequencySec seconds. See
# https://github.com/honeycombio/dynsampler-go for more detail on the mechanics
# of the dynamic sampler. This sampler uses the AvgSampleRate algorithm from
# that package.
Sampler = "DynamicSampler"

# SampleRate is the goal rate at which to sample. It indicates a ratio, where
# one sample trace is kept for every n traces seen. For example, a SampleRate of
# 30 will keep 1 out of every 30 traces. This rate is handed to the dynamic
# sampler, who assigns a sample rate for each trace based on the fields selected
# from that trace.
# Eligible for live reload.
SampleRate = 2

# FieldList is a list of all the field names to use to form the key that will be
# handed to the dynamic sampler. The cardinality of the combination of values
# from all of these keys should be reasonable in the face of the frequency of
# those keys. If the combination of fields in these keys essentially makes them
# unique, the dynamic sampler will do no sampling. If the keys have too few
# values, you won't get samples of the most interesting traces. A good key
# selection will have consistent values for high frequency boring traffic and
# unique values for outliers and interesting traffic. Including an error field
# (or something like HTTP status code) is an excellent choice. As an example,
# assuming 30 or so endpoints, a combination of HTTP endpoint and status code
# would be a good set of keys in order to let you see accurately use of all
# endpoints and call out when there is failing traffic to any endpoint. Field
# names may come from any span in the trace.
# Eligible for live reload.
FieldList = ["request.method","response.status_code"]

# UseTraceLength will add the number of spans in the trace in to the dynamic
# sampler as part of the key. The number of spans is exact, so if there are
# normally small variations in trace length you may want to leave this off. If
# traces are consistent lengths and changes in trace length is a useful
# indicator of traces you'd like to see in Honeycomb, set this to true.
# Eligible for live reload.
UseTraceLength = true

# AddSampleRateKeyToTrace when this is set to true, the sampler will add a field
# to the root span of the trace containing the key used by the sampler to decide
# the sample rate. This can be helpful in understanding why the sampler is
# making certain decisions about sample rate and help you understand how to
# better choose the sample rate key (aka the FieldList setting above) to use.
AddSampleRateKeyToTrace = true

# AddSampleRateKeyToTraceField is the name of the field the sampler will use
# when adding the sample rate key to the trace. This setting is only used when
# AddSampleRateKeyToTrace is true.
AddSampleRateKeyToTraceField = "meta.refinery.dynsampler_key"

# ClearFrequencySec is the name of the field the sampler will use to determine
# the period over which it will calculate the sample rate. This setting defaults
# to 30.
# Eligible for live reload.
ClearFrequencySec = 60

[dataset2]

# EMADynamicSampler is a section of the config for manipulating the Exponential
# Moving Average (EMA) Dynamic Sampler implementation. Like the simple DynamicSampler,
# it attempts to average a given sample rate, weighting rare traffic and frequent
# traffic differently so as to end up with the correct average.
#
# EMADynamicSampler is an improvement upon the simple DynamicSampler and is recommended
# for most use cases. Based on the DynamicSampler implementation, EMADynamicSampler differs
# in that rather than compute rate based on a periodic sample of traffic, it maintains an Exponential
# Moving Average of counts seen per key, and adjusts this average at regular intervals.
# The weight applied to more recent intervals is defined by `weight`, a number between
# (0, 1) - larger values weight the average more toward recent observations. In other words,
# a larger weight will cause sample rates more quickly adapt to traffic patterns,
# while a smaller weight will result in sample rates that are less sensitive to bursts or drops
# in traffic and thus more consistent over time.
#
# Keys that are not found in the EMA will always have a sample
# rate of 1. Keys that occur more frequently will be sampled on a logarithmic
# curve. In other words, every key will be represented at least once in any
# given window and more frequent keys will have their sample rate
# increased proportionally to wind up with the goal sample rate.
Sampler = "EMADynamicSampler"

# GoalSampleRate is the goal rate at which to sample. It indicates a ratio, where
# one sample trace is kept for every n traces seen. For example, a SampleRate of
# 30 will keep 1 out of every 30 traces. This rate is handed to the dynamic
# sampler, who assigns a sample rate for each trace based on the fields selected
# from that trace.
# Eligible for live reload.
GoalSampleRate = 2

# FieldList is a list of all the field names to use to form the key that will be
# handed to the dynamic sampler. The cardinality of the combination of values
# from all of these keys should be reasonable in the face of the frequency of
# those keys. If the combination of fields in these keys essentially makes them
# unique, the dynamic sampler will do no sampling. If the keys have too few
# values, you won't get samples of the most interesting traces. A good key
# selection will have consistent values for high frequency boring traffic and
# unique values for outliers and interesting traffic. Including an error field
# (or something like HTTP status code) is an excellent choice. As an example,
# assuming 30 or so endpoints, a combination of HTTP endpoint and status code
# would be a good set of keys in order to let you see accurately use of all
# endpoints and call out when there is failing traffic to any endpoint. Field
# names may come from any span in the trace.
# Eligible for live reload.
FieldList = ["request.method","response.status_code"]

# UseTraceLength will add the number of spans in the trace in to the dynamic
# sampler as part of the key. The number of spans is exact, so if there are
# normally small variations in trace length you may want to leave this off. If
# traces are consistent lengths and changes in trace length is a useful
# indicator of traces you'd like to see in Honeycomb, set this to true.
# Eligible for live reload.
UseTraceLength = true

# AddSampleRateKeyToTrace when this is set to true, the sampler will add a field
# to the root span of the trace containing the key used by the sampler to decide
# the sample rate. This can be helpful in understanding why the sampler is
# making certain decisions about sample rate and help you understand how to
# better choose the sample rate key (aka the FieldList setting above) to use.
AddSampleRateKeyToTrace = true

# AddSampleRateKeyToTraceField is the name of the field the sampler will use
# when adding the sample rate key to the trace. This setting is only used when
# AddSampleRateKeyToTrace is true.
AddSampleRateKeyToTraceField = "meta.refinery.dynsampler_key"

# AdjustmentInterval defines how often (in seconds) we adjust the moving average from
# recent observations. Default 15s
# Eligible for live reload.
AdjustmentInterval = 15

# Weight is a value between (0, 1) indicating the weighting factor used to adjust
# the EMA. With larger values, newer data will influence the average more, and older
# values will be factored out more quickly. In mathematical literature concerning EMA,
# this is referred to as the `alpha` constant.
# Default is 0.5
# Eligible for live reload.
Weight = 0.5

# MaxKeys, if greater than 0, limits the number of distinct keys tracked in EMA.
# Once MaxKeys is reached, new keys will not be included in the sample rate map, but
# existing keys will continue to be be counted. You can use this to keep the sample rate
# map size under control.
# Eligible for live reload
MaxKeys = 0

# AgeOutValue indicates the threshold for removing keys from the EMA. The EMA of any key
# will approach 0 if it is not repeatedly observed, but will never truly reach it, so we have to
# decide what constitutes "zero". Keys with averages below this threshold will be removed
# from the EMA. Default is the same as Weight, as this prevents a key with the smallest
# integer value (1) from being aged out immediately. This value should generally be <= Weight,
# unless you have very specific reasons to set it higher.
# Eligible for live reload
AgeOutValue = 0.5

# BurstMultiple, if set, is multiplied by the sum of the running average of counts to define
# the burst detection threshold. If total counts observed for a given interval exceed the threshold
# EMA is updated immediately, rather than waiting on the AdjustmentInterval.
# Defaults to 2; negative value disables. With a default of 2, if your traffic suddenly doubles,
# burst detection will kick in.
# Eligible for live reload
BurstMultiple = 2.0

# BurstDetectionDelay indicates the number of intervals to run after Start is called before
# burst detection kicks in.
# Defaults to 3
# Eligible for live reload
BurstDetectionDelay = 3

[dataset3]

Sampler = "DeterministicSampler"
SampleRate = 10

[dataset4]

Sampler = "RulesBasedSampler"

[[dataset4.rule]]
name = "500 errors"
SampleRate = 1
[[dataset4.rule.condition]]
field = "status_code"
operator = "="
value = 500
[[dataset4.rule.condition]]
field = "duration_ms"
operator = ">="
value = 1000.789

[[dataset4.rule]]
name = "drop 200 responses"
drop = true
[[dataset4.rule.condition]]
field = "status_code"
operator = "="
value = 200

[[dataset4.rule]]
SampleRate = 10 # default when no rules match, if missing defaults to 1
# Defaults for the rules configuration are set in file_config.go.
# For an example file with various sampling methods and their configurations, see rules_complete.toml.
Loading

0 comments on commit dfcec54

Please sign in to comment.