[Tempo] Automatic Logging

cmd/agent/entrypoint.go

@@ -79,7 +79,7 @@ func NewEntrypoint(logger *util.Logger, cfg *config.Config, reloader Reloader) (
 		return nil, err
 	}
 
-	ep.tempoTraces, err = tempo.New(prometheus.DefaultRegisterer, cfg.Tempo, cfg.Server.LogLevel.Logrus)
+	ep.tempoTraces, err = tempo.New(ep.lokiLogs, prometheus.DefaultRegisterer, cfg.Tempo, cfg.Server.LogLevel.Logrus)
 	if err != nil {
 		return nil, err
 	}
@@ -125,7 +125,7 @@ func (ep *Entrypoint) ApplyConfig(cfg config.Config) error {
 		failed = true
 	}
 
-	if err := ep.tempoTraces.ApplyConfig(cfg.Tempo, cfg.Server.LogLevel.Logrus); err != nil {
+	if err := ep.tempoTraces.ApplyConfig(ep.lokiLogs, cfg.Tempo, cfg.Server.LogLevel.Logrus); err != nil {
 		level.Error(ep.log).Log("msg", "failed to update tempo", "err", err)
 		failed = true
 	}

docs/configuration-reference.md

@@ -2045,6 +2045,21 @@ remote_write:
     [ sending_queue: <otlpexporter.sending_queue> ]
     [ retry_on_failure: <otlpexporter.retry_on_failure> ]
 
+    # automatically log lines to Loki for discovery/metrics
+    automatic_logging:
+      [ spans: <boolean> | default = false ]               # log one line per span. Warning! possibly very high volume
+      [ roots: <boolean> | default = false ]               # log one line for every root span of a trace.
+      [ processes: <boolean> | default = false ]           # log one line for every process
+      [ span_attributes: <string array> | default = []]    # additional span attributes to log
+      [ process_attributes: <string array> | default = []] # additional process attributes to log
+      overrides:      
+        [ loki_tag: <string> | default = "tempologging" ]
+        [ service_key: <string> | default = "svc" ]
+        [ span_name_key: <string> | default = "span" ]
+        [ status_key: <string> | default = "status" ]
+        [ duration_key: <string> | default = "dur" ]
+        [ trace_id_key: <string> | default = "tid" ]
+
 # Receiver configurations are mapped directly into the OpenTelemetry receivers block.
 #   At least one receiver is required. Supported receivers: otlp, jaeger, kafka, opencensus and zipkin.
 #   Documentation for each receiver can be found at https://github.com/open-telemetry/opentelemetry-collector/blob/7d7ae2eb34b5d387627875c498d7f43619f37ee3/receiver/README.md

example/docker-compose/agent/config/agent.yaml

@@ -73,8 +73,13 @@ tempo:
         key: env
         value: prod
     remote_write:
-      - endpoint: otel-collector:55680
+      - endpoint: tempo:55680
         insecure: true
     batch:
       timeout: 5s
       send_batch_size: 100
+    automatic_logging:
+      loki_name: default
+      spans: true
+      processes: true
+      roots: true

example/docker-compose/docker-compose.yaml

@@ -13,10 +13,10 @@ services:
     ports:
       - "12345:12345"
     depends_on:
-      - otel-collector
+      - tempo
 
   loki:
-    image: grafana/loki:1.5.0
+    image: grafana/loki:2.2.1
     ports:
       - "3100:3100"
     command: -config.file=/etc/loki/local-config.yaml
@@ -44,15 +44,19 @@ services:
       - agent
 
   # tracing backend
-  otel-collector:
-    image: otel/opentelemetry-collector:0.9.0
-    volumes:
-      - ./otel-collector:/etc/otel-collector
-    command:
-      - --config=/etc/otel-collector/config.yaml
+  tempo:
+    image: grafana/tempo:df7225ae
+    command: 
+      - "-storage.trace.backend=local"                  # tell tempo where to permanently put traces
+      - "-storage.trace.local.path=/tmp/tempo/traces"   
+      - "-storage.trace.wal.path=/tmp/tempo/wal"        # tell tempo where to store the wal
+      - "-auth.enabled=false"                           # disables the requirement for the X-Scope-OrgID header
+      - "-server.http-listen-port=3200"
+    ports:
+      - "3200:3200"
 
   grafana:
-    image: grafana/grafana:6.6.1
+    image: grafana/grafana:7.5.4
     entrypoint:
       - /usr/share/grafana/bin/grafana-server
       - --homepath=/usr/share/grafana

example/docker-compose/grafana/datasources/datasource.yml

@@ -13,6 +13,17 @@ datasources:
   isDefault: false
   version: 1
   editable: false
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://tempo:3200
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false
+  apiVersion: 1
+  uid: tempo
 - name: Loki
   type: loki
   access: proxy 
@@ -22,4 +33,10 @@ datasources:
   isDefault: false
   version: 1
   editable: false
+  jsonData:
+    derivedFields:
+      - datasourceUid: tempo
+        matcherRegex: tid=(\w+)
+        name: TraceID
+        url: $${__value.raw}
 

go.mod

@@ -9,7 +9,6 @@ require (
 	github.com/go-kit/kit v0.10.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/protobuf v1.4.3
-	github.com/golang/snappy v0.0.3 // indirect
 	github.com/google/dnsmasq_exporter v0.0.0-00010101000000-000000000000
 	github.com/gorilla/mux v1.8.0
 	github.com/grafana/loki v1.6.2-0.20210205130758-59a34f9867ce