From d1bf280ac9a8ad328acf229b082b924d2ee28d14 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 19:10:10 +0000 Subject: [PATCH 01/12] update beyla golden signals sample --- .../gke/beyla-daemonset.yaml | 8 +++++-- .../gke/collector-config.yaml | 24 ++++--------------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/recipes/beyla-golden-signals/gke/beyla-daemonset.yaml b/recipes/beyla-golden-signals/gke/beyla-daemonset.yaml index 50364a7..74f9642 100644 --- a/recipes/beyla-golden-signals/gke/beyla-daemonset.yaml +++ b/recipes/beyla-golden-signals/gke/beyla-daemonset.yaml @@ -35,7 +35,7 @@ spec: hostPID: true initContainers: - name: mount-bpf-fs - image: grafana/beyla:1.6.2 + image: grafana/beyla:1.8.6 args: # Create the directory using the Pod UID, and mount the BPF filesystem. - 'mkdir -p /sys/fs/bpf/$BEYLA_BPF_FS_PATH && mount -t bpf bpf /sys/fs/bpf/$BEYLA_BPF_FS_PATH' @@ -64,7 +64,7 @@ spec: requests: cpu: 10m memory: 100Mi - image: grafana/beyla:1.5.2 + image: grafana/beyla:1.8.6 securityContext: seccompProfile: type: RuntimeDefault @@ -122,5 +122,9 @@ data: attributes: kubernetes: enable: true + drop_external: true + disable_informers: [replicaset, service, node] + routes: + unmatched: wildcard ebpf: bpf_fs_base_dir: /sys/fs/bpf diff --git a/recipes/beyla-golden-signals/gke/collector-config.yaml b/recipes/beyla-golden-signals/gke/collector-config.yaml index cfcbdb0..2177b6a 100644 --- a/recipes/beyla-golden-signals/gke/collector-config.yaml +++ b/recipes/beyla-golden-signals/gke/collector-config.yaml @@ -17,41 +17,27 @@ kind: OpenTelemetryCollector metadata: name: otel spec: - image: otel/opentelemetry-collector-contrib:0.90.0 + image: otel/opentelemetry-collector-contrib:0.112.0 config: | receivers: # receive OTLP metrics from Beyla otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 processors: # detect gke resource attributes resourcedetection: detectors: [env, gcp] timeout: 2s override: false - # Filter out metrics that aren't relevant about pods - filter: - metrics: - metric: - # not about pods - - resource.attributes["k8s.pod.name"] == nil - # not relevant to the GKE applications dashboard. You can comment the line below to - # pass through all metrics generated by Beyla - - name != "http.server.request.duration" and name != "rpc.server.duration" - datapoint: - # Some gRPC metrics appear not to have an RPC method. - - metric.name == "rpc.server.duration" and attributes["rpc.method"] == "" - # Some HTTP metrics appear not to have an http method. - - metric.name == "http.server.request.duration" and attributes["http.request.method"] == "" exporters: googlemanagedprometheus: - logging: - loglevel: debug service: pipelines: metrics: receivers: [otlp] - processors: [filter, resourcedetection] - exporters: [logging, googlemanagedprometheus] + processors: [resourcedetection] + exporters: [googlemanagedprometheus] From 16d1d135903eece2e7ebfd9e2ad4772c116455ea Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 19:35:05 +0000 Subject: [PATCH 02/12] update gce beyla sample. Tested manually on a new instance --- recipes/beyla-golden-signals/gce/install-beyla.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/beyla-golden-signals/gce/install-beyla.sh b/recipes/beyla-golden-signals/gce/install-beyla.sh index ec118f7..9ab3b22 100755 --- a/recipes/beyla-golden-signals/gce/install-beyla.sh +++ b/recipes/beyla-golden-signals/gce/install-beyla.sh @@ -26,7 +26,7 @@ sudo apt install -y openjdk-17-jdk # Download Beyla echo "Downloading Beyla..." -BEYLA_V1_7_RELEASE=https://github.com/grafana/beyla/releases/download/v1.7.0/beyla-linux-amd64-v1.7.0.tar.gz +BEYLA_V1_7_RELEASE=https://github.com/grafana/beyla/releases/download/v1.8.6/beyla-linux-amd64-v1.8.6.tar.gz curl -Lo beyla.tar.gz $BEYLA_V1_7_RELEASE mkdir -p beyla-installation/ tar -xzf beyla.tar.gz -C beyla-installation/ From 0e7cd65a22fc661b539716a77a04e31feb799cf2 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 19:47:23 +0000 Subject: [PATCH 03/12] update operator version and base collector version --- README.md | 2 +- collector-config.yaml | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 30f7027..57ae8dc 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ listed above. Install the the Operator with: ``` -kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.89.0/opentelemetry-operator.yaml +kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.112.0/opentelemetry-operator.yaml ``` ### Starting the Collector diff --git a/collector-config.yaml b/collector-config.yaml index b5582c8..f70bc21 100644 --- a/collector-config.yaml +++ b/collector-config.yaml @@ -22,16 +22,18 @@ spec: otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 processors: exporters: - logging: - loglevel: debug + debug: + verbosity: detailed service: pipelines: traces: receivers: [otlp] processors: [] - exporters: [logging] + exporters: [debug] From 62cb107a0fea23905284192aeacb0f0a30287ade Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 19:49:40 +0000 Subject: [PATCH 04/12] update trace sample; tested manually in a gke cluster --- recipes/cloud-trace/collector-config.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/recipes/cloud-trace/collector-config.yaml b/recipes/cloud-trace/collector-config.yaml index b5b1fff..ecddbb4 100644 --- a/recipes/cloud-trace/collector-config.yaml +++ b/recipes/cloud-trace/collector-config.yaml @@ -17,23 +17,25 @@ kind: OpenTelemetryCollector metadata: name: otel spec: - image: otel/opentelemetry-collector-contrib:0.90.0 + image: otel/opentelemetry-collector-contrib:0.112.0 config: | receivers: otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 processors: exporters: googlecloud: - logging: - loglevel: debug + debug: + verbosity: detailed service: pipelines: traces: receivers: [otlp] processors: [] - exporters: [logging, googlecloud] + exporters: [debug, googlecloud] From 5523cf2713a82b4a68733ff891a16180897377c1 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 19:51:54 +0000 Subject: [PATCH 05/12] update daemonset + deployment sample. tested manually in a GKE cluster --- .../daemonset-and-deployment/daemonset-collector-config.yaml | 4 +++- .../daemonset-and-deployment/deployment-collector-config.yaml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/recipes/daemonset-and-deployment/daemonset-collector-config.yaml b/recipes/daemonset-and-deployment/daemonset-collector-config.yaml index 0e552e2..ed6ddb0 100644 --- a/recipes/daemonset-and-deployment/daemonset-collector-config.yaml +++ b/recipes/daemonset-and-deployment/daemonset-collector-config.yaml @@ -17,7 +17,7 @@ kind: OpenTelemetryCollector metadata: name: otel spec: - image: otel/opentelemetry-collector-contrib:0.90.0 + image: otel/opentelemetry-collector-contrib:0.112.0 mode: daemonset resources: requests: @@ -30,7 +30,9 @@ spec: otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 processors: memory_limiter: check_interval: 1s diff --git a/recipes/daemonset-and-deployment/deployment-collector-config.yaml b/recipes/daemonset-and-deployment/deployment-collector-config.yaml index 4978410..e5b0f10 100644 --- a/recipes/daemonset-and-deployment/deployment-collector-config.yaml +++ b/recipes/daemonset-and-deployment/deployment-collector-config.yaml @@ -17,7 +17,7 @@ kind: OpenTelemetryCollector metadata: name: otel-deployment spec: - image: otel/opentelemetry-collector-contrib:0.90.0 + image: otel/opentelemetry-collector-contrib:0.112.0 resources: requests: cpu: 50m @@ -36,7 +36,9 @@ spec: otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 extensions: file_storage: directory: /var/lib/buffered-telemetry From 8b1fb207f7805e915a700e80db361b670248052b Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 19:55:31 +0000 Subject: [PATCH 06/12] update host-and-kubelet sample and fix gmp exporter bug. tested on gke cluster --- recipes/host-and-kubelet-metrics/collector-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/host-and-kubelet-metrics/collector-config.yaml b/recipes/host-and-kubelet-metrics/collector-config.yaml index 2e2a773..3f3d970 100644 --- a/recipes/host-and-kubelet-metrics/collector-config.yaml +++ b/recipes/host-and-kubelet-metrics/collector-config.yaml @@ -17,7 +17,7 @@ kind: OpenTelemetryCollector metadata: name: otel spec: - image: otel/opentelemetry-collector-contrib:0.98.0 + image: otel/opentelemetry-collector-contrib:0.112.0 mode: daemonset env: - name: K8S_NODE_NAME @@ -105,7 +105,7 @@ spec: extra_metrics_config: enable_target_info: false resource_filters: - regex: "k8s.*" + - regex: "k8s.*" service: pipelines: From 082168cd3f79625b68d5e5d70e497dbe40502166 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 19:56:25 +0000 Subject: [PATCH 07/12] update resource detection sample. tested on a gke cluster --- recipes/resource-detection/collector-config.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/recipes/resource-detection/collector-config.yaml b/recipes/resource-detection/collector-config.yaml index 6319cde..6fe4a6c 100644 --- a/recipes/resource-detection/collector-config.yaml +++ b/recipes/resource-detection/collector-config.yaml @@ -17,13 +17,15 @@ kind: OpenTelemetryCollector metadata: name: otel spec: - image: otel/opentelemetry-collector-contrib:0.90.0 + image: otel/opentelemetry-collector-contrib:0.112.0 config: | receivers: otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 processors: resourcedetection: @@ -32,12 +34,12 @@ spec: override: false exporters: - logging: - loglevel: debug + debug: + verbosity: detailed service: pipelines: traces: receivers: [otlp] processors: [resourcedetection] - exporters: [logging] + exporters: [debug] From 3983a944752390dec9065ea26a614c49c8b1c773 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 19:58:31 +0000 Subject: [PATCH 08/12] update trace enhancements sample and fix bug. tested on a gke cluster --- .../collector-config-resource-detection.yaml | 12 +++++++----- recipes/trace-enhancements/collector-config.yaml | 10 ++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/recipes/trace-enhancements/collector-config-resource-detection.yaml b/recipes/trace-enhancements/collector-config-resource-detection.yaml index bdbc8e2..57e49da 100644 --- a/recipes/trace-enhancements/collector-config-resource-detection.yaml +++ b/recipes/trace-enhancements/collector-config-resource-detection.yaml @@ -17,13 +17,15 @@ kind: OpenTelemetryCollector metadata: name: otel spec: - image: otel/opentelemetry-collector-contrib:0.90.0 + image: otel/opentelemetry-collector-contrib:0.112.0 config: | receivers: otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 processors: resourcedetection: @@ -40,12 +42,12 @@ spec: action: extract exporters: - logging: - loglevel: debug + debug: + verbosity: detailed service: pipelines: traces: receivers: [otlp] - processors: [resourcedetection,attributes] - exporters: [logging] + processors: [resourcedetection,resource] + exporters: [debug] diff --git a/recipes/trace-enhancements/collector-config.yaml b/recipes/trace-enhancements/collector-config.yaml index 34f0bf0..b7d3f1d 100644 --- a/recipes/trace-enhancements/collector-config.yaml +++ b/recipes/trace-enhancements/collector-config.yaml @@ -17,13 +17,15 @@ kind: OpenTelemetryCollector metadata: name: otel spec: - image: otel/opentelemetry-collector-contrib:0.90.0 + image: otel/opentelemetry-collector-contrib:0.112.0 config: | receivers: otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 processors: attributes: @@ -33,12 +35,12 @@ spec: action: insert exporters: - logging: - loglevel: debug + debug: + verbosity: detailed service: pipelines: traces: receivers: [otlp] processors: [attributes] - exporters: [logging] + exporters: [debug] From 8f00501a9f8d16faad6119e2601f5abe45ea67a5 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 19:59:21 +0000 Subject: [PATCH 09/12] update trace filtering sample. tested on a gke cluster --- recipes/trace-filtering/collector-config.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/recipes/trace-filtering/collector-config.yaml b/recipes/trace-filtering/collector-config.yaml index 3ac27ab..e82bb35 100644 --- a/recipes/trace-filtering/collector-config.yaml +++ b/recipes/trace-filtering/collector-config.yaml @@ -17,13 +17,15 @@ kind: OpenTelemetryCollector metadata: name: otel spec: - image: otel/opentelemetry-collector-contrib:0.90.0 + image: otel/opentelemetry-collector-contrib:0.112.0 config: | receivers: otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 processors: filter: @@ -40,12 +42,12 @@ spec: - .*-service exporters: - logging: - loglevel: debug + debug: + verbosity: detailed service: pipelines: traces: receivers: [otlp] processors: [filter] - exporters: [logging] + exporters: [debug] From 2db5608b871e84c191842d84aaf0ca2bdd58fcc6 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 20:01:59 +0000 Subject: [PATCH 10/12] update remote sampling sample. tested on a gke cluster. --- recipes/trace-remote-sampling/collector-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recipes/trace-remote-sampling/collector-config.yaml b/recipes/trace-remote-sampling/collector-config.yaml index 4114e18..ca8a371 100644 --- a/recipes/trace-remote-sampling/collector-config.yaml +++ b/recipes/trace-remote-sampling/collector-config.yaml @@ -17,7 +17,7 @@ kind: OpenTelemetryCollector metadata: name: otel spec: - image: otel/opentelemetry-collector-contrib:0.90.0 + image: otel/opentelemetry-collector-contrib:0.112.0 # We need to specify ports so remote sampler is exposed. ports: - port: 4317 @@ -43,7 +43,9 @@ spec: otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 exporters: debug: extensions: From a76f74bd6bd2f0e2ef01cb5f87ec1ce275877775 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 1 Nov 2024 20:32:24 +0000 Subject: [PATCH 11/12] update beyla service graph sample. tested manually on GKE --- recipes/beyla-service-graph/beyla-daemonset.yaml | 12 ++++++++---- recipes/beyla-service-graph/collector-config.yaml | 8 ++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/recipes/beyla-service-graph/beyla-daemonset.yaml b/recipes/beyla-service-graph/beyla-daemonset.yaml index 4b13c35..404977c 100644 --- a/recipes/beyla-service-graph/beyla-daemonset.yaml +++ b/recipes/beyla-service-graph/beyla-daemonset.yaml @@ -35,7 +35,7 @@ spec: hostPID: true initContainers: - name: mount-bpf-fs - image: grafana/beyla:1.6.2 + image: grafana/beyla:1.8.6 args: # Create the directory using the Pod UID, and mount the BPF filesystem. - 'mkdir -p /sys/fs/bpf/$BEYLA_BPF_FS_PATH && mount -t bpf bpf /sys/fs/bpf/$BEYLA_BPF_FS_PATH' @@ -64,7 +64,7 @@ spec: requests: cpu: 10m memory: 100Mi - image: grafana/beyla:1.5.2 + image: grafana/beyla:1.8.6 securityContext: seccompProfile: type: RuntimeDefault @@ -121,10 +121,14 @@ data: otel_traces_export: endpoint: http://otel-collector:4317 interval: 30s - # Kubernetes decoration must be enabled to use k8s_pod_name service discovery - # https://github.com/grafana/beyla/issues/673 attributes: + instance_id: + dns: false kubernetes: enable: true + drop_external: true + disable_informers: [replicaset, service, node] + routes: + unmatched: wildcard ebpf: bpf_fs_base_dir: /sys/fs/bpf diff --git a/recipes/beyla-service-graph/collector-config.yaml b/recipes/beyla-service-graph/collector-config.yaml index 3fb2ac1..687be52 100644 --- a/recipes/beyla-service-graph/collector-config.yaml +++ b/recipes/beyla-service-graph/collector-config.yaml @@ -17,14 +17,16 @@ kind: OpenTelemetryCollector metadata: name: otel spec: - image: otel/opentelemetry-collector-contrib:0.90.0 + image: otel/opentelemetry-collector-contrib:0.112.0 config: | receivers: # receive OTLP spans from Beyla otlp: protocols: grpc: + endpoint: 0.0.0.0:4317 http: + endpoint: 0.0.0.0:4318 connectors: # convert spans into a calls metric spanmetrics/servicegraph: @@ -98,7 +100,7 @@ spec: # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/connector/servicegraphconnector#how-it-works - context: metric statements: - - set(name, "traces_service_graph_request") where name == "calls" + - set(name, "traces_service_graph_request") where name == "traces.span.metrics.calls" resource/podinstance: attributes: - key: pod @@ -114,8 +116,6 @@ spec: - regex: 'pod' - regex: client - regex: server - logging: - loglevel: debug service: pipelines: traces: From 1dbc2ba94cfb29ef598f0fd5d91a387bcfdd71da Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Thu, 14 Nov 2024 10:29:38 -0700 Subject: [PATCH 12/12] Apply suggestions from code review --- recipes/beyla-golden-signals/gke/beyla-daemonset.yaml | 2 ++ recipes/beyla-service-graph/beyla-daemonset.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/recipes/beyla-golden-signals/gke/beyla-daemonset.yaml b/recipes/beyla-golden-signals/gke/beyla-daemonset.yaml index 74f9642..0c5ec36 100644 --- a/recipes/beyla-golden-signals/gke/beyla-daemonset.yaml +++ b/recipes/beyla-golden-signals/gke/beyla-daemonset.yaml @@ -122,7 +122,9 @@ data: attributes: kubernetes: enable: true + # drop_external only collects golden signal metrics for kubernetes entities (e.g. pods), which reduces resource usage. drop_external: true + # disable_informers prevents Beyla from watching k8s resources, and reduces the load on the kubernetes API Server. disable_informers: [replicaset, service, node] routes: unmatched: wildcard diff --git a/recipes/beyla-service-graph/beyla-daemonset.yaml b/recipes/beyla-service-graph/beyla-daemonset.yaml index 404977c..aa223e4 100644 --- a/recipes/beyla-service-graph/beyla-daemonset.yaml +++ b/recipes/beyla-service-graph/beyla-daemonset.yaml @@ -126,7 +126,9 @@ data: dns: false kubernetes: enable: true + # drop_external only collects spans for kubernetes entities (e.g. pods), which reduces resource usage. drop_external: true + # disable_informers prevents Beyla from watching k8s resources, and reduces the load on the kubernetes API Server. disable_informers: [replicaset, service, node] routes: unmatched: wildcard