From 1033d1c90a8158f847ba557493485b074b68ac37 Mon Sep 17 00:00:00 2001 From: Maksym Ochenashko Date: Fri, 30 Aug 2024 15:10:29 +0300 Subject: [PATCH] docs: add cross-service propagation example --- build.sbt | 6 + docs/instrumentation/directory.conf | 1 + .../tracing-cross-service-propagation.md | 296 ++++++++++++++++++ docs/instrumentation/tracing-java-interop.md | 2 +- .../otel4s/sdk/OpenTelemetrySdk.scala | 6 +- .../SpanExportersAutoConfigure.scala | 2 +- 6 files changed, 308 insertions(+), 5 deletions(-) create mode 100644 docs/instrumentation/tracing-cross-service-propagation.md diff --git a/build.sbt b/build.sbt index 2994757e7..20fdbeb26 100644 --- a/build.sbt +++ b/build.sbt @@ -652,6 +652,7 @@ lazy val docs = project .settings( libraryDependencies ++= Seq( "org.apache.pekko" %% "pekko-http" % PekkoHttpVersion, + "org.http4s" %% "http4s-client" % Http4sVersion, "io.opentelemetry" % "opentelemetry-sdk-extension-autoconfigure" % OpenTelemetryVersion, "io.opentelemetry.instrumentation" % "opentelemetry-instrumentation-annotations" % OpenTelemetryInstrumentationVersion, "io.opentelemetry.instrumentation" % "opentelemetry-runtime-telemetry-java8" % OpenTelemetryInstrumentationAlphaVersion, @@ -681,6 +682,11 @@ lazy val docs = project "scala-version", ChoiceConfig("scala-2", "Scala 2"), ChoiceConfig("scala-3", "Scala 3") + ).withSeparateEbooks, + SelectionConfig( + "otel-backend", + ChoiceConfig("oteljava", "OpenTelemetry Java"), + ChoiceConfig("sdk", "SDK") ).withSeparateEbooks ) ) diff --git a/docs/instrumentation/directory.conf b/docs/instrumentation/directory.conf index 9254b902a..1073b2661 100644 --- a/docs/instrumentation/directory.conf +++ b/docs/instrumentation/directory.conf @@ -4,5 +4,6 @@ laika.navigationOrder = [ metrics.md metrics-jvm-runtime.md tracing.md + tracing-cross-service-propagation.md tracing-java-interop.md ] diff --git a/docs/instrumentation/tracing-cross-service-propagation.md b/docs/instrumentation/tracing-cross-service-propagation.md new file mode 100644 index 000000000..81dfd2dbf --- /dev/null +++ b/docs/instrumentation/tracing-cross-service-propagation.md @@ -0,0 +1,296 @@ +# Tracing | Cross-service propagation + +Cross-service span propagation is a key concept in distributed tracing that enables the tracking +of requests as they move through various services. + +There are two general scenarios: + +#### 1. Join an external span + +This occurs when a service receives a request (response) or message that includes tracing context from an external source. +The service then joins this existing trace, continuing the span from where it was left off. + +#### 2. Propagate current span downstream + +This scenario involves a service injecting the trace context into outgoing requests or messages. +This allows subsequent services to continue the trace, linking their spans to the parent span. + +## Configuration + +There are multiple propagators available out of the box: +- `tracecontext` - [W3C Trace Context](https://www.w3.org/TR/trace-context/) +- `baggage` - [W3C Baggage](https://www.w3.org/TR/baggage/) +- `b3` - [B3 Single](https://github.com/openzipkin/b3-propagation#single-header) +- `b3multi` - [B3 Multi](https://github.com/openzipkin/b3-propagation#multiple-headers) +- `jaeger` - [Jaeger](https://www.jaegertracing.io/docs/1.21/client-libraries/#propagation-format) + +The `tracecontext` is the default propagator. The propagator can be configured via environment variables or system properties: +- `OTEL_PROPAGATORS=b3multi` +- `-Dotel.propagators=b3multi` + +Multiple propagators can be enabled too, for example: `OTEL_PROPAGATORS=b3multi,tracecontext`. + +`Otel4s#propagators` shows the configured propagators: + +@:select(otel-backend) + +@:choice(oteljava) + +```scala mdoc:silent +import cats.effect.IO +import org.typelevel.otel4s.oteljava.OtelJava + +OtelJava.autoConfigured[IO]().use { otel4s => + IO.println("Propagators: " + otel4s.propagators) +} +// Propagators: ContextPropagators.Default{ +// textMapPropagator=[W3CTraceContextPropagator, W3CBaggagePropagator] +// } +``` + +@:choice(sdk) + +```scala mdoc:silent +import cats.effect.IO +import org.typelevel.otel4s.sdk.OpenTelemetrySdk +import org.typelevel.otel4s.sdk.exporter.otlp.autoconfigure.OtlpExportersAutoConfigure + +OpenTelemetrySdk + .autoConfigured[IO](_.addExportersConfigurer(OtlpExportersAutoConfigure[IO])) + .use { auto => + IO.println("Propagators: " + auto.sdk.propagators) + } +// Propagators: ContextPropagators.Default{ +// textMapPropagator=[W3CTraceContextPropagator, W3CBaggagePropagator] +// } +``` + +@:@ + +## Propagation scenarios + +Let's take a look at a common cross-service propagation models. + +#### HTTP + +```mermaid +sequenceDiagram + participant Client + participant ServiceA + participant ServiceB + + Client->>ServiceA: HTTP Request (Start Trace) + ServiceA->>ServiceA: Start Span A + ServiceA->>ServiceA: Inject Trace Context into HTTP Headers + ServiceA->>ServiceB: HTTP Request with Trace Context + ServiceB->>ServiceB: Extract Trace Context from HTTP Headers + ServiceB->>ServiceB: Start Span B (Child of Span A) + ServiceB->>ServiceB: Perform Operation + ServiceB->>ServiceA: HTTP Response + ServiceB->>ServiceB: End Span B + ServiceA->>ServiceA: End Span A + ServiceA->>Client: HTTP Response with Result +``` + +#### MQTT + +```mermaid +sequenceDiagram + participant ServiceA + participant MQTTBroker + participant ServiceB + + ServiceA->>ServiceA: Start Span A + ServiceA->>ServiceA: Inject Trace Context into MQTT Message + ServiceA->>MQTTBroker: Publish MQTT Message with Trace Context + MQTTBroker->>ServiceB: Deliver MQTT Message + ServiceB->>ServiceB: Extract Trace Context from MQTT Message + ServiceB->>ServiceB: Start Span B (Child of Span A) + ServiceB->>ServiceB: Perform Operation + ServiceB->>ServiceB: End Span B + ServiceB->>ServiceB: Inject Trace Context into MQTT Message + ServiceB->>MQTTBroker: Publish response + MQTTBroker->>ServiceA: Deliver MQTT Message + ServiceA->>ServiceA: Extract Trace Context from MQTT Message + ServiceA->>ServiceA: End Span A +``` + +## Propagate current span downstream + +The `Tracer[F].propagate` injects current span details into the given carrier: +```scala +trait Tracer[F[_]] { + def propagate[C: TextMapUpdater](carrier: C): F[C] +} +``` +Any carrier would work as long as `TextMapUpdater` is available for this type. +For example, `Map[String, String]` and `Seq[(String, String)]` work out of the box. + +We can also implement a `TextMapUpdater` for arbitrary types, for example `org.http4s.Headers`: +```scala mdoc:silent +import cats.effect.IO +import org.http4s._ +import org.http4s.client.Client +import org.http4s.syntax.literals._ +import org.typelevel.ci.CIString +import org.typelevel.otel4s.context.propagation._ +import org.typelevel.otel4s.trace.Tracer + +implicit val headersTextMapUpdater: TextMapUpdater[Headers] = + new TextMapUpdater[Headers] { + def updated(headers: Headers, key: String, value: String): Headers = + headers.put(Header.Raw(CIString(key), value)) + } + +def sendRequest(client: Client[IO])(implicit T: Tracer[IO]): IO[String] = + Tracer[IO].span("send-request").surround { + val req = Request[IO](Method.GET, uri"http://localhost:8080") + + for { + traceHeaders <- Tracer[IO].propagate(Headers.empty) + // Headers(traceparent: 00-82383569b2b84276342a70581dc625ad-083b7f94913d787a-01) + + // add trace headers to the request and execute it + result <- client.expect[String](req.withHeaders(req.headers ++ traceHeaders)) + } yield result + } +``` + +## Join an external span + +The `Tracer[F].joinOrRoot` extracts span details from the carrier: +```scala +trait Tracer[F[_]] { + def joinOrRoot[A, C: TextMapGetter](carrier: C)(fa: F[A]): F[A] +} +``` + +Similarly to the `TextMapUpdater`, we can implement a `TextMapGetter` for arbitrary types: +```scala mdoc:silent +import cats.effect.IO +import org.http4s._ +import org.http4s.client.Client +import org.http4s.syntax.literals._ +import org.typelevel.ci.CIString +import org.typelevel.otel4s.context.propagation._ +import org.typelevel.otel4s.trace.Tracer + +implicit val headersTextMapGetter: TextMapGetter[Headers] = + new TextMapGetter[Headers] { + def get(headers: Headers, key: String): Option[String] = + headers.get(CIString(key)).map(_.head.value) + def keys(headers: Headers): Iterable[String] = + headers.headers.map(_.name.toString) + } + +def executeRequest(client: Client[IO])(implicit T: Tracer[IO]): IO[Unit] = { + val req = Request[IO](Method.GET, uri"http://localhost:8080") + + client.run(req).use { response => + // use response's headers to extract tracing details + Tracer[IO].joinOrRoot(response.headers) { + Tracer[IO].span("child-span").surround { + for { + body <- response.as[String] + _ <- IO.println("body: " + body) + // process response there + } yield () + } + } + } +} +``` + +## Implementing a custom propagator + +`TextMapPropagator` injects and extracts values in the form of text into carriers that travel in-band. + +Let's say we use `platform-id` in the HTTP headers. +We can implement a custom `TextMapPropagator` that will use `platform-id` header to carry the identifier. + +@:select(otel-backend) + +@:choice(oteljava) + +```scala mdoc:reset:silent +import cats.effect._ +import org.typelevel.otel4s.context.propagation._ +import org.typelevel.otel4s.oteljava.context._ + +object PlatformIdPropagator extends TextMapPropagator[Context] { + // the value will be stored in the Context under this key + val PlatformIdKey: Context.Key[String] = + Context.Key.unique[SyncIO, String]("platform-id").unsafeRunSync() + + val fields: Iterable[String] = List("platform-id") + + def extract[A: TextMapGetter](ctx: Context, carrier: A): Context = + TextMapGetter[A].get(carrier, "platform-id") match { + case Some(value) => ctx.updated(PlatformIdKey, value) + case None => ctx + } + + def inject[A: TextMapUpdater](ctx: Context, carrier: A): A = + ctx.get(PlatformIdKey) match { + case Some(value) => TextMapUpdater[A].updated(carrier, "platform-id", value) + case None => carrier + } +} +``` + +And wire it up: +```scala mdoc:silent +import org.typelevel.otel4s.oteljava.context.propagation.PropagatorConverters._ +import io.opentelemetry.context.propagation.{TextMapPropagator => JTextMapPropagator} +import org.typelevel.otel4s.oteljava.OtelJava + +OtelJava.autoConfigured[IO] { builder => + builder.addPropagatorCustomizer { (tmp, _) => + JTextMapPropagator.composite(tmp, PlatformIdPropagator.asJava) + } +} +``` + +@:choice(sdk) + +```scala mdoc:reset:silent +import cats.effect._ +import org.typelevel.otel4s.context.propagation._ +import org.typelevel.otel4s.sdk.context._ + +object PlatformIdPropagator extends TextMapPropagator[Context] { + // the value will be stored in the Context under this key + val PlatformIdKey: Context.Key[String] = + Context.Key.unique[SyncIO, String]("platform-id").unsafeRunSync() + + val fields: Iterable[String] = List("platform-id") + + def extract[A: TextMapGetter](ctx: Context, carrier: A): Context = + TextMapGetter[A].get(carrier, "platform-id") match { + case Some(value) => ctx.updated(PlatformIdKey, value) + case None => ctx + } + + def inject[A: TextMapUpdater](ctx: Context, carrier: A): A = + ctx.get(PlatformIdKey) match { + case Some(value) => TextMapUpdater[A].updated(carrier, "platform-id", value) + case None => carrier + } +} +``` + +And wire it up: +```scala mdoc:silent +import org.typelevel.otel4s.sdk.OpenTelemetrySdk +import org.typelevel.otel4s.sdk.autoconfigure.AutoConfigure + +OpenTelemetrySdk.autoConfigured[IO] { builder => + builder.addTextMapPropagatorConfigurer( + AutoConfigure.Named.const("platform-id", PlatformIdPropagator) + ) +} +``` + +And enable propagator via environment variable: `OTEL_PROPAGATORS=platform-id`. + +@:@ diff --git a/docs/instrumentation/tracing-java-interop.md b/docs/instrumentation/tracing-java-interop.md index 2be92c1f8..2dda88478 100644 --- a/docs/instrumentation/tracing-java-interop.md +++ b/docs/instrumentation/tracing-java-interop.md @@ -1,4 +1,4 @@ -# Tracing - interop with Java-instrumented libraries +# Tracing | Interop with Java-instrumented libraries ## Glossary diff --git a/sdk/all/src/main/scala/org/typelevel/otel4s/sdk/OpenTelemetrySdk.scala b/sdk/all/src/main/scala/org/typelevel/otel4s/sdk/OpenTelemetrySdk.scala index 2ad049c44..dc77d46d1 100644 --- a/sdk/all/src/main/scala/org/typelevel/otel4s/sdk/OpenTelemetrySdk.scala +++ b/sdk/all/src/main/scala/org/typelevel/otel4s/sdk/OpenTelemetrySdk.scala @@ -80,7 +80,7 @@ object OpenTelemetrySdk { * import org.typelevel.otel4s.sdk.OpenTelemetrySdk * import org.typelevel.otel4s.sdk.exporter.otlp.autoconfigure.OtlpExportersAutoConfigure * - * OpenTelemetrySdk.autoConfigured[IO](_.addExportersConfigurer(OtlpExporterAutoConfigure[IO])) + * OpenTelemetrySdk.autoConfigured[IO](_.addExportersConfigurer(OtlpExportersAutoConfigure[IO])) * }}} * * @param customize @@ -219,9 +219,9 @@ object OpenTelemetrySdk { * and register the configurer manually: * {{{ * import org.typelevel.otel4s.sdk.OpenTelemetrySdk - * import org.typelevel.otel4s.sdk.exporter.otlp.autoconfigure.OtlpExporterAutoConfigure + * import org.typelevel.otel4s.sdk.exporter.otlp.autoconfigure.OtlpExportersAutoConfigure * - * OpenTelemetrySdk.autoConfigured[IO](_.addExporterConfigurer(OtlpExporterAutoConfigure[IO])) + * OpenTelemetrySdk.autoConfigured[IO](_.addExportersConfigurer(OtlpExportersAutoConfigure[IO])) * }}} * * @param configurer diff --git a/sdk/trace/src/main/scala/org/typelevel/otel4s/sdk/trace/autoconfigure/SpanExportersAutoConfigure.scala b/sdk/trace/src/main/scala/org/typelevel/otel4s/sdk/trace/autoconfigure/SpanExportersAutoConfigure.scala index 17f25d5ab..28cad4080 100644 --- a/sdk/trace/src/main/scala/org/typelevel/otel4s/sdk/trace/autoconfigure/SpanExportersAutoConfigure.scala +++ b/sdk/trace/src/main/scala/org/typelevel/otel4s/sdk/trace/autoconfigure/SpanExportersAutoConfigure.scala @@ -111,7 +111,7 @@ private final class SpanExportersAutoConfigure[F[_]: MonadThrow: Console]( |import org.typelevel.otel4s.sdk.exporter.otlp.autoconfigure.OtlpExportersAutoConfigure | |OpenTelemetrySdk.autoConfigured[IO]( - | _.addExportersConfigurer(OtlpExporterAutoConfigure[IO]) + | _.addExportersConfigurer(OtlpExportersAutoConfigure[IO]) |) | |or via SdkTraces: