From fc52d7660b9cc243f94d432d399c7c0d89ed05f0 Mon Sep 17 00:00:00 2001
From: Povilas Versockas <p.versockas@gmail.com>
Date: Thu, 29 Aug 2024 09:48:22 +0300
Subject: [PATCH] feat: add central cluster collector for tail sampling

---
 otel-integration/CHANGELOG.md                 |   3 +
 otel-integration/k8s-helm/Chart.yaml          |   7 +-
 otel-integration/k8s-helm/README.md           |  53 +++++
 .../k8s-helm/central-agent-values.yaml        |  38 ++++
 .../central-tail-sampling-values.yaml         | 100 ++++++++++
 otel-integration/k8s-helm/values.yaml         | 186 +++++++++++++++++-
 6 files changed, 385 insertions(+), 2 deletions(-)
 create mode 100644 otel-integration/k8s-helm/central-agent-values.yaml
 create mode 100644 otel-integration/k8s-helm/central-tail-sampling-values.yaml

diff --git a/otel-integration/CHANGELOG.md b/otel-integration/CHANGELOG.md
index 0a8491a0..e770410d 100644
--- a/otel-integration/CHANGELOG.md
+++ b/otel-integration/CHANGELOG.md
@@ -2,6 +2,9 @@
 
 ## OpenTelemtry-Integration
 
+### v0.0.98 / 2024-08-29
+- [Feat] Add a way to deploy central collector cluster for tail sampling
+
 ### v0.0.97 / 2024-08-19
 - [Fix] ignore process name not found errors for hostmetrics process preset
 
diff --git a/otel-integration/k8s-helm/Chart.yaml b/otel-integration/k8s-helm/Chart.yaml
index 8be891d0..916d7eac 100644
--- a/otel-integration/k8s-helm/Chart.yaml
+++ b/otel-integration/k8s-helm/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: otel-integration
 description: OpenTelemetry Integration
-version: 0.0.97
+version: 0.0.98
 keywords:
   - OpenTelemetry Collector
   - OpenTelemetry Agent
@@ -24,6 +24,11 @@ dependencies:
     version: "0.90.0"
     repository: https://cgx.jfrog.io/artifactory/coralogix-charts-virtual
     condition: opentelemetry-cluster-collector.enabled
+  - name: opentelemetry-collector
+    alias: opentelemetry-receiver
+    version: "0.90.0"
+    repository: https://cgx.jfrog.io/artifactory/coralogix-charts-virtual
+    condition: opentelemetry-receiver.enabled
   - name: opentelemetry-collector
     alias: opentelemetry-gateway
     version: "0.90.0"
diff --git a/otel-integration/k8s-helm/README.md b/otel-integration/k8s-helm/README.md
index 8bb53bb7..71ac98c8 100644
--- a/otel-integration/k8s-helm/README.md
+++ b/otel-integration/k8s-helm/README.md
@@ -245,6 +245,59 @@ This change will configure otel-agent pods to send span data to coralogix-opente
 
 When running in Openshift make sure to set `distribution: "openshift"` in your `values.yaml`. When running in Windows environments, please use `values-windows-tailsampling.yaml` values file.
 
+### Deploying Central Collector Cluster for Tail Sampling
+
+If you want to deploy OpenTelemetry Collector in a seperate "central" Kubernetes Cluster, that receives telemetry data via OTLP receivers and does [Tail Sampling](https://opentelemetry.io/docs/concepts/sampling/#tail-sampling) you can install `otel-integration` using `central-tail-sampling-values.yaml` values file. Check the values file for configuration.
+
+This will deploy two deployments:
+- opentelemetry-receiver - responsible for receiving otlp data, pushing metrics and logs to Coralogix and loadbalancing spans to opentelemetry-gateway deployment.
+- opentelemetry-gateway - a service that receives span data and does Tail Sampling decisions.
+
+The opentelemetry-receiver will need to be exposed to other Kubernetes Clusters for sending data. You can do that by using service of type LoadBalancer, configuring Ingress object, or manually configuring your load balancer. Also, make sure to configure enough replicas and resource requests and limits to handle the load. Next, you will need to configure [tail sampling processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor) policies with your custom tail sampling policies.
+
+```bash
+helm repo add coralogix-charts-virtual https://cgx.jfrog.io/artifactory/coralogix-charts-virtual
+
+helm upgrade --install otel-coralogix-central-collector coralogix-charts-virtual/otel-integration \
+  --render-subchart-notes -f central-tail-sampling-values.yaml
+```
+
+Once you deploy it, you can validate by sending some otlp data to opentelemetry-receiver Service and checking Coralogix for spans. This can be done via telemetrygen:
+```
+kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: telemetrygen-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: telemetrygen
+  template:
+    metadata:
+      labels:
+        app: telemetrygen
+    spec:
+      containers:
+      - name: telemetrygen
+        image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest
+        args:
+          - "traces"
+          - "--otlp-endpoint=coralogix-opentelemetry-receiver:4317"
+          - "--otlp-insecure"
+          - "--rate=10"
+          - "--duration=120s"
+EOF
+```
+
+Next, you will need to configure regular `otel-integration` deployment to send data to Central Collector Cluster:
+
+```bash
+helm upgrade --install otel-coralogix-integration coralogix-charts-virtual/otel-integration \
+  --render-subchart-notes -f central-agent-values.yaml
+```
+
 #### Why am I getting ResourceExhausted errors when using Tail Sampling?
 
 Typically, the errors look like this:
diff --git a/otel-integration/k8s-helm/central-agent-values.yaml b/otel-integration/k8s-helm/central-agent-values.yaml
new file mode 100644
index 00000000..798f53eb
--- /dev/null
+++ b/otel-integration/k8s-helm/central-agent-values.yaml
@@ -0,0 +1,38 @@
+global:
+  domain: ""
+  clusterName: ""
+  defaultApplicationName: "otel"
+  defaultSubsystemName: "integration"
+  logLevel: "debug"
+  collectionInterval: "30s"
+  version: "0.0.97"
+
+  extensions:
+    kubernetesDashboard:
+      enabled: true
+
+# set distribution to openshift for openshift clusters
+distribution: ""
+opentelemetry-agent:
+  enabled: true
+  config:
+    exporters:
+      otlp:
+        # configure the public endpoint here
+        endpoint: coralogix-opentelemetry-receiver:4317
+        # this is not needed if you have valid tls certificate fronting receivers
+        tls:
+          insecure: true
+    service:
+      pipelines:
+        traces:
+          exporters:
+            - otlp
+
+opentelemetry-cluster-collector:
+  enabled: true
+opentelemetry-agent-windows:
+  enabled: false
+opentelemetry-gateway:
+  enabled: false
+
diff --git a/otel-integration/k8s-helm/central-tail-sampling-values.yaml b/otel-integration/k8s-helm/central-tail-sampling-values.yaml
new file mode 100644
index 00000000..f0bfb0b0
--- /dev/null
+++ b/otel-integration/k8s-helm/central-tail-sampling-values.yaml
@@ -0,0 +1,100 @@
+global:
+  domain: ""
+  clusterName: ""
+  defaultApplicationName: "otel"
+  defaultSubsystemName: "integration"
+  logLevel: "warn"
+  collectionInterval: "30s"
+
+opentelemetry-receiver:
+  enabled: true
+  # Receiver needs to be exposed either via Service of type LoadBalancer or Ingress
+  service:
+    enabled: true
+    type: ClusterIP
+    # type: LoadBalancer
+    # loadBalancerIP: 1.2.3.4
+    # loadBalancerSourceRanges: []
+  
+    # By default, Service of type 'LoadBalancer' will be created setting 'externalTrafficPolicy: Cluster'
+    # unless other value is explicitly set.
+    # Possible values are Cluster or Local (https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip)
+  # externalTrafficPolicy: Cluster
+  ingress:
+    enabled: false
+    # annotations: {}
+    # ingressClassName: nginx
+    # hosts:
+    #   - host: collector.example.com
+    #     paths:
+    #       - path: /
+    #         pathType: Prefix
+    #         port: 4318
+    # tls:
+    #   - secretName: collector-tls
+    #     hosts:
+    #       - collector.example.com
+  # For production use-cases please increase replicas
+  # and resource requests and limits
+  replicaCount: 3
+  # resources:
+  #   requests:
+  #     cpu: 0.5
+  #     memory: 256Mi
+  #   limits:
+  #     cpu: 2
+  #     memory: 2G
+
+  presets:
+    loadBalancing:
+      enabled: true
+      routingKey: "traceID"
+      hostname: coralogix-opentelemetry-gateway
+      # dnsResolverInterval: 20s
+      # dnsResolverTimeout: 5s
+
+  config:
+    service:
+      pipelines:
+        traces:
+          exporters:
+            - loadbalancing
+
+opentelemetry-gateway:
+  enabled: true
+  # For production use-cases please increase replicas
+  # and resource requests and limits
+  replicaCount: 3
+  # resources:
+  #   requests:
+  #     cpu: 0.5
+  #     memory: 256Mi
+  #   limits:
+  #     cpu: 2
+  #     memory: 2G
+
+  config:
+    processors:
+      tail_sampling:
+        # Update configuration here, with your settings and tail sampling policies
+        # Docs: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor
+        policies:
+          [
+            #{
+            #  name: errors-policy,
+            #  type: status_code,
+            #  status_code: {status_codes: [ERROR]}
+            #},
+            {
+              name: randomized-policy,
+              type: probabilistic,
+              probabilistic: {sampling_percentage: 10}
+            },
+          ]
+
+opentelemetry-agent:
+  enabled: false
+opentelemetry-cluster-collector:
+  enabled: false
+opentelemetry-agent-windows:
+  enabled: false
diff --git a/otel-integration/k8s-helm/values.yaml b/otel-integration/k8s-helm/values.yaml
index b8c65cc2..c75a7156 100644
--- a/otel-integration/k8s-helm/values.yaml
+++ b/otel-integration/k8s-helm/values.yaml
@@ -5,7 +5,7 @@ global:
   defaultSubsystemName: "integration"
   logLevel: "warn"
   collectionInterval: "30s"
-  version: "0.0.97"
+  version: "0.0.98"
 
   extensions:
     kubernetesDashboard:
@@ -845,3 +845,187 @@ opentelemetry-gateway:
       enabled: false
     zipkin:
       enabled: false
+
+opentelemetry-receiver:
+  enabled: false
+  mode: deployment
+  fullnameOverride: coralogix-opentelemetry-receiver
+  service:
+    enabled: true
+  extraEnvs:
+    - name: CORALOGIX_PRIVATE_KEY
+      valueFrom:
+        secretKeyRef:
+          name: coralogix-keys
+          key: PRIVATE_KEY
+    - name: KUBE_NODE_NAME
+      valueFrom:
+        fieldRef:
+          apiVersion: v1
+          fieldPath: spec.nodeName
+  presets:
+    fleetManagement:
+      enabled: false
+      agentType: "gateway"
+      clusterName: "{{.Values.global.clusterName}}"
+    kubernetesAttributes:
+      enabled: true
+    metadata:
+      enabled: true
+      clusterName: "{{.Values.global.clusterName}}"
+      integrationName: "coralogix-integration-helm"
+  # Temporary feature gates to prevent breaking changes. Please see changelog for version 0.0.85 for more information.
+  command:
+    name: otelcol-contrib
+    extraArgs: ["--feature-gates=component.UseLocalHostAsDefaultHost,-confmap.strictlyTypedInput"]
+
+  config:
+    extensions:
+      zpages:
+        endpoint: localhost:55679
+      pprof:
+        endpoint: localhost:1777
+    exporters:
+      coralogix:
+        timeout: "30s"
+        private_key: "${env:CORALOGIX_PRIVATE_KEY}"
+        domain: "{{ .Values.global.domain }}"
+        logs:
+          headers:
+            X-Coralogix-Distribution: "helm-otel-integration/{{ .Values.global.version }}"
+        metrics:
+          headers:
+            X-Coralogix-Distribution: "helm-otel-integration/{{ .Values.global.version }}"
+        traces:
+          headers:
+            X-Coralogix-Distribution: "helm-otel-integration/{{ .Values.global.version }}"
+        application_name: "{{ .Values.global.defaultApplicationName }}"
+        subsystem_name: "{{ .Values.global.defaultSubsystemName }}"
+        application_name_attributes:
+          - "k8s.namespace.name"
+          - "service.namespace"
+        subsystem_name_attributes:
+          - "k8s.deployment.name"
+          - "k8s.statefulset.name"
+          - "k8s.daemonset.name"
+          - "k8s.cronjob.name"
+          - "service.name"
+    processors:
+      # needed for self-monitored otel colector metrics
+      k8sattributes:
+        filter:
+          node_from_env_var: KUBE_NODE_NAME
+        extract:
+          metadata:
+            - "k8s.namespace.name"
+            # replace the below by `k8s.deployment.name` after https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/23067
+            - "k8s.replicaset.name"
+            - "k8s.statefulset.name"
+            - "k8s.daemonset.name"
+            - "k8s.cronjob.name"
+            - "k8s.job.name"
+            - "k8s.pod.name"
+            - "k8s.node.name"
+      resourcedetection/env:
+        detectors: ["system", "env"]
+        timeout: 2s
+        override: false
+        system:
+          resource_attributes:
+            host.id:
+              enabled: true
+      resourcedetection/region:
+        detectors: ["gcp", "ec2", "azure"]
+        timeout: 2s
+        override: true
+      batch:
+        send_batch_size: 1024
+        send_batch_max_size: 2048
+        timeout: "1s"
+      transform/prometheus:
+        error_mode: ignore
+        metric_statements:
+          - context: resource
+            statements:
+              - set(attributes["k8s.pod.ip"], attributes["net.host.name"]) where attributes["service.name"] == "opentelemetry-collector"
+    receivers:
+      prometheus:
+        config:
+          scrape_configs:
+            - job_name: opentelemetry-collector
+              scrape_interval: 30s
+              static_configs:
+                - targets:
+                    - ${env:MY_POD_IP}:8888
+      otlp:
+        protocols:
+          grpc:
+            endpoint: ${env:MY_POD_IP}:4317
+            # Default otlp grpc server message size limit is 4mib, which might be too low.
+            max_recv_msg_size_mib: 20
+    service:
+      telemetry:
+        resource:
+          # Supress this attribute, as we don't want the UUID of the collector to be sent,
+          # instead we rely on instance label generated by Prometheus receiver.
+          - service.instance.id:
+          - service.name:
+        logs:
+          level: "{{ .Values.global.logLevel }}"
+          encoding: json
+        metrics:
+          address: ${env:MY_POD_IP}:8888
+      pipelines:
+        metrics/self_monitoring:
+          exporters:
+            - coralogix
+          processors:
+            - transform/prometheus
+            - k8sattributes
+            - resourcedetection/env
+            - resourcedetection/region
+            - memory_limiter
+            - batch
+          receivers:
+            - prometheus
+
+        metrics:
+          exporters:
+            - coralogix
+          processors:
+            - memory_limiter
+            - batch
+          receivers:
+            - otlp
+        traces:
+          exporters:
+            - loadbalancing
+          processors:
+            - memory_limiter
+            - batch
+          receivers:
+            - otlp
+        logs:
+          exporters:
+            - coralogix
+          processors:
+            - memory_limiter
+            - batch
+          receivers:
+            - otlp
+
+  tolerations:
+    - operator: Exists
+  ports:
+    otlp:
+      enabled: true
+    otlp-http:
+      enabled: false
+    jaeger-compact:
+      enabled: false
+    jaeger-thrift:
+      enabled: false
+    jaeger-grpc:
+      enabled: false
+    zipkin:
+      enabled: false