From d7ef432be2554a029daee409cdf38577ed47a0d7 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Thu, 10 Aug 2017 16:05:25 +0200 Subject: [PATCH] jaeger config & docs --- conf/tracing.go | 5 +++-- docker/docker-cluster/metrictank.ini | 5 +++++ docker/docker-dev-custom-cfg-kafka/metrictank.ini | 5 +++++ docs/config.md | 4 ++++ docs/operations.md | 6 ++++++ metrictank-sample.ini | 5 +++++ metrictank.go | 5 ++++- scripts/config/metrictank-docker.ini | 5 +++++ scripts/config/metrictank-package.ini | 5 +++++ 9 files changed, 42 insertions(+), 3 deletions(-) diff --git a/conf/tracing.go b/conf/tracing.go index a976fc4222..8b31c58060 100644 --- a/conf/tracing.go +++ b/conf/tracing.go @@ -9,17 +9,18 @@ import ( jaegerlog "github.com/uber/jaeger-client-go/log" ) -func GetTracer() (opentracing.Tracer, io.Closer, error) { +func GetTracer(enabled bool, addr string) (opentracing.Tracer, io.Closer, error) { // Sample configuration for testing. Use constant sampling to sample every trace // and enable LogSpan to log every span via configured Logger. cfg := jaegercfg.Configuration{ + Disabled: !enabled, Sampler: &jaegercfg.SamplerConfig{ Type: jaeger.SamplerTypeConst, Param: 1, }, Reporter: &jaegercfg.ReporterConfig{ LogSpans: true, - LocalAgentHostPort: "jaeger:6831", + LocalAgentHostPort: addr, }, } diff --git a/docker/docker-cluster/metrictank.ini b/docker/docker-cluster/metrictank.ini index 31d42173ee..c744b2acbb 100644 --- a/docker/docker-cluster/metrictank.ini +++ b/docker/docker-cluster/metrictank.ini @@ -86,6 +86,11 @@ proftrigger-heap-thresh = 25000000000 # only log log-level and higher. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL log-level = 2 +# enable/disable distributed opentracing via jaeger +tracing-enabled = false +# address of the jaeger agent to send data to +tracing-addr = localhost:6831 + ## Retention settings ## [retention] # path to storage-schemas.conf file diff --git a/docker/docker-dev-custom-cfg-kafka/metrictank.ini b/docker/docker-dev-custom-cfg-kafka/metrictank.ini index 2c1d05933f..bc0046b468 100644 --- a/docker/docker-dev-custom-cfg-kafka/metrictank.ini +++ b/docker/docker-dev-custom-cfg-kafka/metrictank.ini @@ -86,6 +86,11 @@ proftrigger-heap-thresh = 25000000000 # only log log-level and higher. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL log-level = 2 +# enable/disable distributed opentracing via jaeger +tracing-enabled = false +# address of the jaeger agent to send data to +tracing-addr = localhost:6831 + ## Retention settings ## [retention] # path to storage-schemas.conf file diff --git a/docs/config.md b/docs/config.md index 2741f9b313..57f38fc7ca 100644 --- a/docs/config.md +++ b/docs/config.md @@ -115,6 +115,10 @@ proftrigger-min-diff = 1h proftrigger-heap-thresh = 25000000000 # only log log-level and higher. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL log-level = 2 +# enable/disable distributed opentracing via jaeger +tracing-enabled = false +# address of the jaeger agent to send data to +tracing-addr = localhost:6831 ``` ## Retention settings ## diff --git a/docs/operations.md b/docs/operations.md index 00ebeba2cd..c2cb07435b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -151,3 +151,9 @@ For more information on profiling see the excellent [Profiling Go Programs](http * can use debug logging to trace data throughout the pipeline. mt-store-cat to see what's in cassandra, mt-kafka-mdm-sniff, etc. * if it's old data, make sure you have a primary that can save data to cassandra, that the write queue can drain * check `metric-max-stale` and `chunk-max-stale` settings, make sure chunks are not being prematurely sealed (happens in some rare cases if you send data very infrequently. see `tank.add_to_closed_chunk` metric) + +## Opentracing + +Metrictank supports opentracing via [Jaeger](http://jaeger.readthedocs.io/en/latest/) +It can give good insights into why certain requests are slow, and is easy to run. +To use, enable in the config and point it at a Jaeger collector. diff --git a/metrictank-sample.ini b/metrictank-sample.ini index a0ee35efe7..e9baa7cb63 100644 --- a/metrictank-sample.ini +++ b/metrictank-sample.ini @@ -89,6 +89,11 @@ proftrigger-heap-thresh = 25000000000 # only log log-level and higher. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL log-level = 2 +# enable/disable distributed opentracing via jaeger +tracing-enabled = false +# address of the jaeger agent to send data to +tracing-addr = localhost:6831 + ## Retention settings ## [retention] # path to storage-schemas.conf file diff --git a/metrictank.go b/metrictank.go index d1cbb95c9e..3463a056d6 100644 --- a/metrictank.go +++ b/metrictank.go @@ -90,6 +90,9 @@ var ( proftrigFreqStr = flag.String("proftrigger-freq", "60s", "inspect status frequency. set to 0 to disable") proftrigMinDiffStr = flag.String("proftrigger-min-diff", "1h", "minimum time between triggered profiles") proftrigHeapThresh = flag.Int("proftrigger-heap-thresh", 25000000000, "if this many bytes allocated, trigger a profile") + + tracingEnabled = flag.Bool("tracing-enabled", false, "enable/disable distributed opentracing via jaeger") + tracingAddr = flag.String("tracing-addr", "localhost:6831", "address of the jaeger agent to send data to") ) func init() { @@ -315,7 +318,7 @@ func main() { /*********************************** Initialize tracer ***********************************/ - tracer, traceCloser, err := conf.GetTracer() + tracer, traceCloser, err := conf.GetTracer(*tracingEnabled, *tracingAddr) if err != nil { log.Fatal(4, "Could not initialize jaeger tracer: %s", err.Error()) } diff --git a/scripts/config/metrictank-docker.ini b/scripts/config/metrictank-docker.ini index 231efb3de4..d2a9290caf 100644 --- a/scripts/config/metrictank-docker.ini +++ b/scripts/config/metrictank-docker.ini @@ -86,6 +86,11 @@ proftrigger-heap-thresh = 25000000000 # only log log-level and higher. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL log-level = 2 +# enable/disable distributed opentracing via jaeger +tracing-enabled = false +# address of the jaeger agent to send data to +tracing-addr = localhost:6831 + ## Retention settings ## [retention] # path to storage-schemas.conf file diff --git a/scripts/config/metrictank-package.ini b/scripts/config/metrictank-package.ini index c5507a0af6..0af0d207be 100644 --- a/scripts/config/metrictank-package.ini +++ b/scripts/config/metrictank-package.ini @@ -86,6 +86,11 @@ proftrigger-heap-thresh = 25000000000 # only log log-level and higher. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL log-level = 2 +# enable/disable distributed opentracing via jaeger +tracing-enabled = false +# address of the jaeger agent to send data to +tracing-addr = localhost:6831 + ## Retention settings ## [retention] # path to storage-schemas.conf file