[Tempo] Automatic Logging (#551)

* garbage POC, let's goooooooooooOOO! Signed-off-by: Joe Elliott <number101010@gmail.com> * Added basic configuration Signed-off-by: Joe Elliott <number101010@gmail.com> * Added skeleton processor Signed-off-by: Joe Elliott <number101010@gmail.com> * first pass config wiring Signed-off-by: Joe Elliott <number101010@gmail.com> * got things to work? Signed-off-by: Joe Elliott <number101010@gmail.com> * kinda sorta works Signed-off-by: Joe Elliott <number101010@gmail.com> * made things work Signed-off-by: Joe Elliott <number101010@gmail.com> * fixed-ish dur. upped loki and grafana Signed-off-by: Joe Elliott <number101010@gmail.com> * fixed duration, added svc name, did processes correctly Signed-off-by: Joe Elliott <number101010@gmail.com> * Added tempo Signed-off-by: Joe Elliott <number101010@gmail.com> * configurable tags Signed-off-by: Joe Elliott <number101010@gmail.com> * config Signed-off-by: Joe Elliott <number101010@gmail.com> * Addd stopping flag Signed-off-by: Joe Elliott <number101010@gmail.com> * Added tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Added configurable stuff Signed-off-by: Joe Elliott <number101010@gmail.com> * Added automatic logging to docs Signed-off-by: Joe Elliott <number101010@gmail.com> * added otel config test Signed-off-by: Joe Elliott <number101010@gmail.com> * Made tid configurable Signed-off-by: Joe Elliott <number101010@gmail.com> * changelog Signed-off-by: Joe Elliott <number101010@gmail.com> * Removed zero valued defaults Signed-off-by: Joe Elliott <number101010@gmail.com> * Put changelog entry in the proper spot Signed-off-by: Joe Elliott <number101010@gmail.com> * - _processor Signed-off-by: Joe Elliott <number101010@gmail.com> * fixed tests Signed-off-by: Joe Elliott <number101010@gmail.com> * Added cross config validation Signed-off-by: Joe Elliott <number101010@gmail.com> * review suggestions Signed-off-by: Joe Elliott <number101010@gmail.com> * First pass SendEntry Signed-off-by: Joe Elliott <number101010@gmail.com> * lint Signed-off-by: Joe Elliott <number101010@gmail.com> * Added configurable timeout Signed-off-by: Joe Elliott <number101010@gmail.com> * Addd logging message when bad things happen Signed-off-by: Joe Elliott <number101010@gmail.com> * Actually check sent :P Signed-off-by: Joe Elliott <number101010@gmail.com> * review cleanup Signed-off-by: Joe Elliott <number101010@gmail.com> * config updates Signed-off-by: Joe Elliott <number101010@gmail.com> * duh Signed-off-by: Joe Elliott <number101010@gmail.com>
grafana · Apr 22, 2021 · 238dc73 · 238dc73
1 parent e874ac4
commit 238dc73
Show file tree

Hide file tree

Showing 21 changed files with 728 additions and 67 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,8 @@ cross-compilation issue, but will return in v0.13.0.
 
 # Main (unreleased)
 
+- [FEATURE] Added Automatic Logging feature for Tempo (@joe-elliott)
+
 - [BUGFIX] Ensure defaults are applied to undefined sections in config file.
   This fixes a problem where integrations didn't work if `prometheus:` wasn't
   configured. (@rfratto)

diff --git a/cmd/agent/entrypoint.go b/cmd/agent/entrypoint.go
@@ -79,7 +79,7 @@ func NewEntrypoint(logger *util.Logger, cfg *config.Config, reloader Reloader) (
 		return nil, err
 	}
 
-	ep.tempoTraces, err = tempo.New(prometheus.DefaultRegisterer, cfg.Tempo, cfg.Server.LogLevel.Logrus)
+	ep.tempoTraces, err = tempo.New(ep.lokiLogs, prometheus.DefaultRegisterer, cfg.Tempo, cfg.Server.LogLevel.Logrus)
 	if err != nil {
 		return nil, err
 	}
@@ -125,7 +125,7 @@ func (ep *Entrypoint) ApplyConfig(cfg config.Config) error {
 		failed = true
 	}
 
-	if err := ep.tempoTraces.ApplyConfig(cfg.Tempo, cfg.Server.LogLevel.Logrus); err != nil {
+	if err := ep.tempoTraces.ApplyConfig(ep.lokiLogs, cfg.Tempo, cfg.Server.LogLevel.Logrus); err != nil {
 		level.Error(ep.log).Log("msg", "failed to update tempo", "err", err)
 		failed = true
 	}

diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md
@@ -2045,6 +2045,32 @@ remote_write:
     [ sending_queue: <otlpexporter.sending_queue> ]
     [ retry_on_failure: <otlpexporter.retry_on_failure> ]
 
+    # This processor writes a well formatted log line to a Loki instance for each span, root, or process 
+    # that passes through the Agent. This allows for automatically building a mechanism for trace
+    # discovery and building metrics from traces using Loki.
+    automatic_logging:
+      # indicates the Loki instance to write logs to.
+      loki_name: <string>
+      # log one line per span. Warning! possibly very high volume
+      [ spans: <boolean> ]
+      # log one line for every root span of a trace.
+      [ roots: <boolean> ]
+      # log one line for every process
+      [ processes: <boolean> ]
+      # additional span attributes to log
+      [ span_attributes: <string array> ]
+      # additional process attributes to log
+      [ process_attributes: <string array> ]
+      # timeout on sending logs to Loki
+      [ timeout: <duration> | default = 1ms ]
+      overrides:      
+        [ loki_tag: <string> | default = "tempo" ]
+        [ service_key: <string> | default = "svc" ]
+        [ span_name_key: <string> | default = "span" ]
+        [ status_key: <string> | default = "status" ]
+        [ duration_key: <string> | default = "dur" ]
+        [ trace_id_key: <string> | default = "tid" ]
+
 # Receiver configurations are mapped directly into the OpenTelemetry receivers block.
 #   At least one receiver is required. Supported receivers: otlp, jaeger, kafka, opencensus and zipkin.
 #   Documentation for each receiver can be found at https://github.com/open-telemetry/opentelemetry-collector/blob/7d7ae2eb34b5d387627875c498d7f43619f37ee3/receiver/README.md

diff --git a/example/docker-compose/agent/config/agent.yaml b/example/docker-compose/agent/config/agent.yaml
@@ -73,8 +73,13 @@ tempo:
         key: env
         value: prod
     remote_write:
-      - endpoint: otel-collector:55680
+      - endpoint: tempo:55680
         insecure: true
     batch:
       timeout: 5s
       send_batch_size: 100
+    automatic_logging:
+      loki_name: default
+      spans: true
+      processes: true
+      roots: true
diff --git a/example/docker-compose/docker-compose.yaml b/example/docker-compose/docker-compose.yaml
@@ -13,10 +13,10 @@ services:
     ports:
       - "12345:12345"
     depends_on:
-      - otel-collector
+      - tempo
 
   loki:
-    image: grafana/loki:1.5.0
+    image: grafana/loki:2.2.1
     ports:
       - "3100:3100"
     command: -config.file=/etc/loki/local-config.yaml
@@ -44,15 +44,19 @@ services:
       - agent
 
   # tracing backend
-  otel-collector:
-    image: otel/opentelemetry-collector:0.9.0
-    volumes:
-      - ./otel-collector:/etc/otel-collector
-    command:
-      - --config=/etc/otel-collector/config.yaml
+  tempo:
+    image: grafana/tempo:df7225ae
+    command: 
+      - "-storage.trace.backend=local"                  # tell tempo where to permanently put traces
+      - "-storage.trace.local.path=/tmp/tempo/traces"   
+      - "-storage.trace.wal.path=/tmp/tempo/wal"        # tell tempo where to store the wal
+      - "-auth.enabled=false"                           # disables the requirement for the X-Scope-OrgID header
+      - "-server.http-listen-port=3200"
+    ports:
+      - "3200:3200"
 
   grafana:
-    image: grafana/grafana:6.6.1
+    image: grafana/grafana:7.5.4
     entrypoint:
       - /usr/share/grafana/bin/grafana-server
       - --homepath=/usr/share/grafana

diff --git a/example/docker-compose/grafana/datasources/datasource.yml b/example/docker-compose/grafana/datasources/datasource.yml
@@ -13,6 +13,17 @@ datasources:
   isDefault: false
   version: 1
   editable: false
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://tempo:3200
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false
+  apiVersion: 1
+  uid: tempo
 - name: Loki
   type: loki
   access: proxy 
@@ -22,4 +33,10 @@ datasources:
   isDefault: false
   version: 1
   editable: false
+  jsonData:
+    derivedFields:
+      - datasourceUid: tempo
+        matcherRegex: tid=(\w+)
+        name: TraceID
+        url: $${__value.raw}
 
diff --git a/go.mod b/go.mod
@@ -9,7 +9,6 @@ require (
 	github.com/go-kit/kit v0.10.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/protobuf v1.4.3
-	github.com/golang/snappy v0.0.3 // indirect
 	github.com/google/dnsmasq_exporter v0.0.0-00010101000000-000000000000
 	github.com/gorilla/mux v1.8.0
 	github.com/grafana/loki v1.6.2-0.20210205130758-59a34f9867ce