From a38cd8a0fd789ef4bde95aba4669068b3cb0c441 Mon Sep 17 00:00:00 2001 From: Sean Hanson Date: Tue, 5 Mar 2019 15:53:33 -0500 Subject: [PATCH 1/2] Add configuration for tracing sample rate --- cmd/metrictank/metrictank.go | 9 +++++---- cmd/mt-store-cat/main.go | 2 +- conf/tracing.go | 16 ++++++++++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cmd/metrictank/metrictank.go b/cmd/metrictank/metrictank.go index 0b2eef8539..2b917d332b 100644 --- a/cmd/metrictank/metrictank.go +++ b/cmd/metrictank/metrictank.go @@ -76,9 +76,10 @@ var ( proftrigMinDiffStr = flag.String("proftrigger-min-diff", "1h", "minimum time between triggered profiles") proftrigHeapThresh = flag.Int("proftrigger-heap-thresh", 25000000000, "if this many bytes allocated, trigger a profile") - tracingEnabled = flag.Bool("tracing-enabled", false, "enable/disable distributed opentracing via jaeger") - tracingAddr = flag.String("tracing-addr", "localhost:6831", "address of the jaeger agent to send data to") - tracingAddTags = flag.String("tracing-add-tags", "", "tracer/process-level tags to include, specified as comma-separated key:value pairs") + tracingEnabled = flag.Bool("tracing-enabled", false, "enable/disable distributed opentracing via jaeger") + tracingAddr = flag.String("tracing-addr", "localhost:6831", "address of the jaeger agent to send data to") + tracingAddTags = flag.String("tracing-add-tags", "", "tracer/process-level tags to include, specified as comma-separated key:value pairs") + tracingSampleRatio = flag.Float64("tracing-sample-ratio", 1.0, "Ratio of traces to sample between 0 (none) and 1 (all)") ) func main() { @@ -252,7 +253,7 @@ func main() { tags[split[0]] = split[1] } } - tracer, traceCloser, err := conf.GetTracer(*tracingEnabled, *tracingAddr, tags) + tracer, traceCloser, err := conf.GetTracer(*tracingEnabled, *tracingAddr, tags, *tracingSampleRatio) if err != nil { log.Fatalf("Could not initialize jaeger tracer: %s", err.Error()) } diff --git a/cmd/mt-store-cat/main.go b/cmd/mt-store-cat/main.go index cc27a5198a..f30d5fb256 100644 --- a/cmd/mt-store-cat/main.go +++ b/cmd/mt-store-cat/main.go @@ -178,7 +178,7 @@ func main() { if err != nil { log.Fatalf("failed to initialize cassandra. %s", err.Error()) } - tracer, traceCloser, err := conf.GetTracer(false, "", nil) + tracer, traceCloser, err := conf.GetTracer(false, "", nil, 0.0) if err != nil { log.Fatalf("Could not initialize jaeger tracer: %s", err.Error()) } diff --git a/conf/tracing.go b/conf/tracing.go index 6611325b23..e711c3e82e 100644 --- a/conf/tracing.go +++ b/conf/tracing.go @@ -11,14 +11,22 @@ import ( // GetTracer returns a jaeger tracer // any tags specified will be added as process/tracer-level tags -func GetTracer(enabled bool, addr string, tags map[string]string) (opentracing.Tracer, io.Closer, error) { - // Sample configuration for testing. Use constant sampling to sample every trace +func GetTracer(enabled bool, addr string, tags map[string]string, sampleRatio float64) (opentracing.Tracer, io.Closer, error) { + // Sample configuration for testing. Use sampleRatio >= 1.0 to sample every trace // and enable LogSpan to log every span via configured Logger. + + // Must be between 0 and 1 (inclusive) + if sampleRatio < 0.0 { + sampleRatio = 0.0 + } else if sampleRatio > 1.0 { + sampleRatio = 1.0 + } + cfg := jaegercfg.Configuration{ Disabled: !enabled, Sampler: &jaegercfg.SamplerConfig{ - Type: jaeger.SamplerTypeConst, - Param: 1, + Type: jaeger.SamplerTypeProbabilistic, + Param: sampleRatio, }, Reporter: &jaegercfg.ReporterConfig{ LogSpans: false, From edc405f2a78811b2121e8b853b343ae3f6654e1a Mon Sep 17 00:00:00 2001 From: Sean Hanson Date: Tue, 5 Mar 2019 15:54:23 -0500 Subject: [PATCH 2/2] Update docs --- docker/docker-chaos/metrictank.ini | 2 ++ docker/docker-cluster/metrictank.ini | 2 ++ docker/docker-dev-custom-cfg-kafka/metrictank.ini | 2 ++ docs/config.md | 2 ++ metrictank-sample.ini | 2 ++ scripts/config/metrictank-docker.ini | 2 ++ scripts/config/metrictank-package.ini | 2 ++ 7 files changed, 14 insertions(+) diff --git a/docker/docker-chaos/metrictank.ini b/docker/docker-chaos/metrictank.ini index c2fece8dbc..8ef8775650 100644 --- a/docker/docker-chaos/metrictank.ini +++ b/docker/docker-chaos/metrictank.ini @@ -51,6 +51,8 @@ tracing-enabled = true tracing-addr = jaeger:6831 # tracer/process-level tags to include, specified as comma-separated key:value pairs tracing-add-tags = +# Ratio of traces to sample between 0 (none) and 1 (all) +tracing-sample-ratio = 1.0 ## metric data storage in cassandra ## [cassandra] diff --git a/docker/docker-cluster/metrictank.ini b/docker/docker-cluster/metrictank.ini index 41d59628ba..193bb99447 100644 --- a/docker/docker-cluster/metrictank.ini +++ b/docker/docker-cluster/metrictank.ini @@ -51,6 +51,8 @@ tracing-enabled = true tracing-addr = jaeger:6831 # tracer/process-level tags to include, specified as comma-separated key:value pairs tracing-add-tags = +# Ratio of traces to sample between 0 (none) and 1 (all) +tracing-sample-ratio = 1.0 ## metric data storage in cassandra ## [cassandra] diff --git a/docker/docker-dev-custom-cfg-kafka/metrictank.ini b/docker/docker-dev-custom-cfg-kafka/metrictank.ini index f2c572e0ef..d752a7b6b8 100644 --- a/docker/docker-dev-custom-cfg-kafka/metrictank.ini +++ b/docker/docker-dev-custom-cfg-kafka/metrictank.ini @@ -51,6 +51,8 @@ tracing-enabled = true tracing-addr = jaeger:6831 # tracer/process-level tags to include, specified as comma-separated key:value pairs tracing-add-tags = +# Ratio of traces to sample between 0 (none) and 1 (all) +tracing-sample-ratio = 1.0 ## metric data storage in cassandra ## [cassandra] diff --git a/docs/config.md b/docs/config.md index f48b24b591..5747d5b5bf 100644 --- a/docs/config.md +++ b/docs/config.md @@ -79,6 +79,8 @@ tracing-enabled = false tracing-addr = localhost:6831 # tracer/process-level tags to include, specified as comma-separated key:value pairs tracing-add-tags = +# Ratio of traces to sample between 0 (none) and 1 (all) +tracing-sample-ratio = 1.0 ``` ## metric data storage in cassandra ## diff --git a/metrictank-sample.ini b/metrictank-sample.ini index 238f176cac..d77fc2a773 100644 --- a/metrictank-sample.ini +++ b/metrictank-sample.ini @@ -54,6 +54,8 @@ tracing-enabled = false tracing-addr = localhost:6831 # tracer/process-level tags to include, specified as comma-separated key:value pairs tracing-add-tags = +# Ratio of traces to sample between 0 (none) and 1 (all) +tracing-sample-ratio = 1.0 ## metric data storage in cassandra ## [cassandra] diff --git a/scripts/config/metrictank-docker.ini b/scripts/config/metrictank-docker.ini index b5fa28c064..ebd60f629e 100644 --- a/scripts/config/metrictank-docker.ini +++ b/scripts/config/metrictank-docker.ini @@ -51,6 +51,8 @@ tracing-enabled = false tracing-addr = localhost:6831 # tracer/process-level tags to include, specified as comma-separated key:value pairs tracing-add-tags = +# Ratio of traces to sample between 0 (none) and 1 (all) +tracing-sample-ratio = 1.0 ## metric data storage in cassandra ## [cassandra] diff --git a/scripts/config/metrictank-package.ini b/scripts/config/metrictank-package.ini index e7433101e7..d6d3aa1bf8 100644 --- a/scripts/config/metrictank-package.ini +++ b/scripts/config/metrictank-package.ini @@ -51,6 +51,8 @@ tracing-enabled = false tracing-addr = localhost:6831 # tracer/process-level tags to include, specified as comma-separated key:value pairs tracing-add-tags = +# Ratio of traces to sample between 0 (none) and 1 (all) +tracing-sample-ratio = 1.0 ## metric data storage in cassandra ## [cassandra]