Sample trace probabilistically

cmd/metrictank/metrictank.go

@@ -76,9 +76,10 @@ var (
 	proftrigMinDiffStr = flag.String("proftrigger-min-diff", "1h", "minimum time between triggered profiles")
 	proftrigHeapThresh = flag.Int("proftrigger-heap-thresh", 25000000000, "if this many bytes allocated, trigger a profile")
 
-	tracingEnabled = flag.Bool("tracing-enabled", false, "enable/disable distributed opentracing via jaeger")
-	tracingAddr    = flag.String("tracing-addr", "localhost:6831", "address of the jaeger agent to send data to")
-	tracingAddTags = flag.String("tracing-add-tags", "", "tracer/process-level tags to include, specified as comma-separated key:value pairs")
+	tracingEnabled     = flag.Bool("tracing-enabled", false, "enable/disable distributed opentracing via jaeger")
+	tracingAddr        = flag.String("tracing-addr", "localhost:6831", "address of the jaeger agent to send data to")
+	tracingAddTags     = flag.String("tracing-add-tags", "", "tracer/process-level tags to include, specified as comma-separated key:value pairs")
+	tracingSampleRatio = flag.Float64("tracing-sample-ratio", 1.0, "Ratio of traces to sample between 0 (none) and 1 (all)")
 )
 
 func main() {
@@ -252,7 +253,7 @@ func main() {
 			tags[split[0]] = split[1]
 		}
 	}
-	tracer, traceCloser, err := conf.GetTracer(*tracingEnabled, *tracingAddr, tags)
+	tracer, traceCloser, err := conf.GetTracer(*tracingEnabled, *tracingAddr, tags, *tracingSampleRatio)
 	if err != nil {
 		log.Fatalf("Could not initialize jaeger tracer: %s", err.Error())
 	}

cmd/mt-store-cat/main.go

@@ -178,7 +178,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("failed to initialize cassandra. %s", err.Error())
 	}
-	tracer, traceCloser, err := conf.GetTracer(false, "", nil)
+	tracer, traceCloser, err := conf.GetTracer(false, "", nil, 0.0)
 	if err != nil {
 		log.Fatalf("Could not initialize jaeger tracer: %s", err.Error())
 	}

conf/tracing.go

@@ -11,14 +11,22 @@ import (
 
 // GetTracer returns a jaeger tracer
 // any tags specified will be added as process/tracer-level tags
-func GetTracer(enabled bool, addr string, tags map[string]string) (opentracing.Tracer, io.Closer, error) {
-	// Sample configuration for testing. Use constant sampling to sample every trace
+func GetTracer(enabled bool, addr string, tags map[string]string, sampleRatio float64) (opentracing.Tracer, io.Closer, error) {
+	// Sample configuration for testing. Use sampleRatio >= 1.0 to sample every trace
 	// and enable LogSpan to log every span via configured Logger.
+
+	// Must be between 0 and 1 (inclusive)
+	if sampleRatio < 0.0 {
+		sampleRatio = 0.0
+	} else if sampleRatio > 1.0 {
+		sampleRatio = 1.0
+	}
+
 	cfg := jaegercfg.Configuration{
 		Disabled: !enabled,
 		Sampler: &jaegercfg.SamplerConfig{
-			Type:  jaeger.SamplerTypeConst,
-			Param: 1,
+			Type:  jaeger.SamplerTypeProbabilistic,
+			Param: sampleRatio,
 		},
 		Reporter: &jaegercfg.ReporterConfig{
 			LogSpans:           false,

docker/docker-chaos/metrictank.ini

@@ -51,6 +51,8 @@ tracing-enabled = true
 tracing-addr = jaeger:6831
 # tracer/process-level tags to include, specified as comma-separated key:value pairs
 tracing-add-tags =
+# Ratio of traces to sample between 0 (none) and 1 (all)
+tracing-sample-ratio = 1.0
 
 ## metric data storage in cassandra ##
 [cassandra]

docker/docker-cluster/metrictank.ini

@@ -51,6 +51,8 @@ tracing-enabled = true
 tracing-addr = jaeger:6831
 # tracer/process-level tags to include, specified as comma-separated key:value pairs
 tracing-add-tags =
+# Ratio of traces to sample between 0 (none) and 1 (all)
+tracing-sample-ratio = 1.0
 
 ## metric data storage in cassandra ##
 [cassandra]

docker/docker-dev-custom-cfg-kafka/metrictank.ini

@@ -51,6 +51,8 @@ tracing-enabled = true
 tracing-addr = jaeger:6831
 # tracer/process-level tags to include, specified as comma-separated key:value pairs
 tracing-add-tags =
+# Ratio of traces to sample between 0 (none) and 1 (all)
+tracing-sample-ratio = 1.0
 
 ## metric data storage in cassandra ##
 [cassandra]

docs/config.md

@@ -79,6 +79,8 @@ tracing-enabled = false
 tracing-addr = localhost:6831
 # tracer/process-level tags to include, specified as comma-separated key:value pairs
 tracing-add-tags =
+# Ratio of traces to sample between 0 (none) and 1 (all)
+tracing-sample-ratio = 1.0
 ```
 
 ## metric data storage in cassandra ##

metrictank-sample.ini

@@ -54,6 +54,8 @@ tracing-enabled = false
 tracing-addr = localhost:6831
 # tracer/process-level tags to include, specified as comma-separated key:value pairs
 tracing-add-tags =
+# Ratio of traces to sample between 0 (none) and 1 (all)
+tracing-sample-ratio = 1.0
 
 ## metric data storage in cassandra ##
 [cassandra]

scripts/config/metrictank-docker.ini

@@ -51,6 +51,8 @@ tracing-enabled = false
 tracing-addr = localhost:6831
 # tracer/process-level tags to include, specified as comma-separated key:value pairs
 tracing-add-tags =
+# Ratio of traces to sample between 0 (none) and 1 (all)
+tracing-sample-ratio = 1.0
 
 ## metric data storage in cassandra ##
 [cassandra]

scripts/config/metrictank-package.ini

@@ -51,6 +51,8 @@ tracing-enabled = false
 tracing-addr = localhost:6831
 # tracer/process-level tags to include, specified as comma-separated key:value pairs
 tracing-add-tags =
+# Ratio of traces to sample between 0 (none) and 1 (all)
+tracing-sample-ratio = 1.0
 
 ## metric data storage in cassandra ##
 [cassandra]