From 57b07411b81b73ceb0cf99834cef1773d44c3276 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Wed, 18 Oct 2023 09:03:43 -0400 Subject: [PATCH] Add a jaeger remote sampling recipe. (#48) * Add a jaeger remote sampling recipe. * Add license * updated year * Apply suggestions from code review Co-authored-by: Aaron Abbott * Fixes from review. * Fix recipes list. * Bump reload interval. --------- Co-authored-by: Aaron Abbott --- README.md | 3 + recipes/README.md | 3 + recipes/trace-remote-sampling/README.md | 99 +++++++++++++++++++ .../collector-config.yaml | 60 +++++++++++ .../instrumentation.yaml | 43 ++++++++ .../remote-sampling-config.yaml | 26 +++++ 6 files changed, 234 insertions(+) create mode 100644 recipes/trace-remote-sampling/README.md create mode 100644 recipes/trace-remote-sampling/collector-config.yaml create mode 100644 recipes/trace-remote-sampling/instrumentation.yaml create mode 100644 recipes/trace-remote-sampling/remote-sampling-config.yaml diff --git a/README.md b/README.md index f3854fb..8c16d21 100644 --- a/README.md +++ b/README.md @@ -113,10 +113,13 @@ The [`recipes`](recipes/) directory holds different sample use cases for working operator and auto-instrumentation along with setup guides for each recipe. Currently there are: * [Trace sampling configuration](recipes/trace-sampling) +* [Trace remote sampling config](recipes/trace-remote-sampling) * [Trace filtering](recipes/trace-filtering) * [Trace enhancements](recipes/trace-enhancements) * [Cloud Trace integration](recipes/cloud-trace) * [Resource detection](recipes/resource-detection) +* [Daemonset and Deployment](recipes/daemonset-and-deployment) +* [eBPF HTTP Observability with Beyla](recipes/beyla) ## Contributing diff --git a/recipes/README.md b/recipes/README.md index 4fce5be..1b3f9a6 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -4,7 +4,10 @@ This directory holds different "recipe" configurations for the operator and auto-instrumentation. See below to get started: * [Trace sampling config](trace-sampling) +* [Trace remote sampling config](trace-remote-sampling) * [Trace filtering](trace-filtering) * [Trace enhancements](trace-enhancements) * [Cloud trace integration](cloud-trace) * [Resource detection](resource-detection) +* [Daemonset and Deployment](daemonset-and-deployment) +* [eBPF HTTP Observability with Beyla](beyla) diff --git a/recipes/trace-remote-sampling/README.md b/recipes/trace-remote-sampling/README.md new file mode 100644 index 0000000..c1fc6d8 --- /dev/null +++ b/recipes/trace-remote-sampling/README.md @@ -0,0 +1,99 @@ +# Trace Remote Sampling Configuration Recipe + +This recipe shows how to use the Jaeger Remote Sampling +protocol to provide fine-grained control of trace sampling. + + +This recipe provides three main pieces of configuration: + +- A ConfigMap that provides fine-grained trace sampling controls for the entire cluster. +- An OpenTelemetryCollector deployment that will serve the control protocol to instrumentation. +- An Instrumentation configuration that will leverage the OpenTelemetryCollector deployment to look up trace sampling information. + +## Prerequisites + +* OpenTelemetry Operator installed in your cluster +* Running un-instrumented application (such as one of the [sample apps](../../sample-apps)). + +## Running + +Apply the `ConfigMap` object from [`remote-sampling-config.yaml`](remote-sampling-config.yaml) + +``` +kubectl apply -f remote-sampling-config.yaml +``` + +This creates the configuration for trace sampling. At any point, we can modify the `remote-sampling-config.yaml` file and reperform this step to adjust sampling on the cluster in the future (see [Tuning](#tuning)) + + +Apply the `OpenTelemetryCollector` object from [`collector-config.yaml`](collector-config.yaml) + +``` +kubectl apply -f collector-config.yaml +``` + +Next, create the `Instrumentation` object from [`instrumentation.yaml`](instrumentation.yaml) that will use the remote sampling service: + +``` +kubectl apply -f instrumentation.yaml +``` + +This creates a new object named `instrumentation/trace-remote-sampling` in the current namespace. + +> Note that `jaeger_remote` sampler configuration is +> only available in Java and Go as of 2023-10-18. + +Annotate your application pods to use these settings by editing the `instrumentation.opentelemetry.io` +annotation using one of the following commands: + +> Note that if the app is a standalone Pod you can +>`kubectl annotate` directly on the Pod, but if it is owned by a Deployment or other replica controller +> you must patch the metadata of the Pod template. + +* **Java:** + + Pod: + ``` + kubectl annotate pod/ instrumentation.opentelemetry.io/inject-java="trace-remote-sampling" + ``` + Deployment: + ``` + kubectl patch deployment.apps/ -p '{"spec":{"template":{"metadata":{"annotations":{"instrumentation.opentelemetry.io/inject-java": "trace-remote-sampling"}}}}}' + ``` + +# Tuning + +To tune sampling in the cluster, simply update the `remote-sampling-config.yaml` and reapply: + +``` +kubectl apply -f remote-sampling-config.yaml +``` + +The OpenTelemetryCollector deployment should pick up changes within a few minutes of the ConfigMap rollout, and clients will further pull in those changes within a few minutes. + +This can help, e.g. when needing to increase the sampling rate of a service for better observability "on the fly" and turn it back down after collecting enough data. + +The format of the remote sampling configuration is [documented here](https://www.jaegertracing.io/docs/1.28/sampling/#collector-sampling-configuration). + +An example configuration which disables tracing prometheus metrics and health checks would look as follows: + +```json +{ + "default_strategy": { + "type": "probabilistic", + "param": 0.5, + "operation_strategies": [ + { + "operation": "GET /health", + "type": "probabilistic", + "param": 0.0 + }, + { + "operation": "GET /metrics", + "type": "probabilistic", + "param": 0.0 + } + ] + } +} +``` \ No newline at end of file diff --git a/recipes/trace-remote-sampling/collector-config.yaml b/recipes/trace-remote-sampling/collector-config.yaml new file mode 100644 index 0000000..99cbfce --- /dev/null +++ b/recipes/trace-remote-sampling/collector-config.yaml @@ -0,0 +1,60 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: otel +spec: + image: otel/opentelemetry-collector-contrib:latest + # We need to specify ports so remote sampler is exposed. + ports: + - port: 4317 + name: otlp + - port: 4318 + name: otlp-grpc + - port: 5778 + name: jaeger + - port: 14250 + name: jaeger-grpc + # Connect the config-map w/ our jaeger sampler config so we can update it quickly. + volumes: + - name: sampling-config-volume + configMap: + name: remote-sampling-config + volumeMounts: + - name: sampling-config-volume + mountPath: /etc/otel/sampling + readOnly: true + # Ensure the jaeger remote sampler config is enabled as an extension. + config: | + receivers: + otlp: + protocols: + grpc: + http: + exporters: + debug: + extensions: + jaegerremotesampling: + source: + reload_interval: 60s + file: /etc/otel/sampling/sampling.json + service: + extensions: [jaegerremotesampling] + pipelines: + traces: + receivers: [otlp] + processors: [] + exporters: [debug] diff --git a/recipes/trace-remote-sampling/instrumentation.yaml b/recipes/trace-remote-sampling/instrumentation.yaml new file mode 100644 index 0000000..b3b9688 --- /dev/null +++ b/recipes/trace-remote-sampling/instrumentation.yaml @@ -0,0 +1,43 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: trace-remote-sampling +spec: + +# Default exporting OTLP to the gRPC interface of a collector. + exporter: + endpoint: http://otel-collector:4317 + +# Use w3c `traceparent` and `baggage` for propgation of distributed values. + propagators: + - tracecontext + - baggage + +# Use the Jaeger remote sampling config from the gateway +# +# pollingInterval determines how often clients will refresh sampling configuration. +# initialSamplingRate determines sampling in the event a client cannot connect to the +# the sampling service. + sampler: + type: jaeger_remote + argument: "endpoint=http://otel-collector:14250,pollingIntervalMs=5000,initialSamplingRate=0.25" + +# Note: Python currently supports HTTP not GRPC endpoint + python: + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://otel-collector:4318 diff --git a/recipes/trace-remote-sampling/remote-sampling-config.yaml b/recipes/trace-remote-sampling/remote-sampling-config.yaml new file mode 100644 index 0000000..ab4c2fb --- /dev/null +++ b/recipes/trace-remote-sampling/remote-sampling-config.yaml @@ -0,0 +1,26 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: remote-sampling-config +data: + sampling.json: | + { + "default_strategy": { + "type":"probabilistic", + "param": 0.5 + } + }