Add simple demo that sets up the OTEL collector in a pull mode (scrap…

…ing another metrics endpoints) and sending it to scaler

Signed-off-by: Jirka Kremser <jiri.kremser@gmail.com>

.github/workflows/release-chart.yaml

@@ -77,7 +77,7 @@ jobs:
             _sec=$(echo "1.5^$i" | bc)
             echo "Waiting ${_sec} seconds.."
             sleep ${_sec}
-            helm repo add kedify-otel https://kedify.github.io/otel-add-on || continue
+            helm repo add kedify-otel https://kedify.github.io/otel-add-on/ || continue
             helm repo update
             set -x
             helm upgrade -i keda-otel-add-on kedify-otel/otel-add-on \

README.md

@@ -81,9 +81,9 @@ helm upgrade -i keda kedacore/keda --namespace keda --create-namespace
 
 ### Then install this add-on
 ```bash
-helm repo add kedify-otel https://kedify.github.io/otel-add-on
+helm repo add kedify-otel https://kedify.github.io/otel-add-on/
 helm repo update
-helm upgrade -i keda-otel kedify-otel/otel-add-on
+helm upgrade -i keda-otel kedify-otel/otel-add-on --version=v0.0.0-1
 ```
 
 ### Create an example scaled object

examples/metric-pull/README.md

@@ -0,0 +1,77 @@
+# Use-case: pull metrics
+
+This use-case demonstrates how OTEL collector can be used as a scraper of another metric endpoints and
+then forwarding the filtered metrics into OTLP receiver in our scaler.
+
+Prepare helm chart repos:
+
+```bash
+helm repo add kedacore https://kedacore.github.io/charts
+helm repo add podinfo https://stefanprodan.github.io/podinfo
+helm repo add kedify-otel https://kedify.github.io/otel-add-on/
+helm repo update
+```
+
+Install demo webapp:
+
+```bash
+helm upgrade -i podinfo podinfo/podinfo -f podinfo-values.yaml
+kubectl -n default port-forward deploy/podinfo 8080:9898
+```
+
+Install this addon:
+```bash
+helm upgrade -i kedify-otel kedify-otel/otel-add-on --version=v0.0.0-1 -f collector-pull-values.yaml
+```
+
+Note the following section in the helm chart values that configures the OTEL collector to scrape targets:
+
+```yaml
+...
+  config:
+    receivers:
+      # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/prometheusreceiver/README.md
+      prometheus:
+        config:
+          scrape_configs:
+            - job_name: 'otelcol'
+              scrape_interval: 5s
+              static_configs:
+                - targets: ['0.0.0.0:8888']
+            - job_name: k8s
+              kubernetes_sd_configs:
+                - role: service
+              relabel_configs:
+                - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+                  regex: "true"
+                  action: keep
+                - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+                  action: replace
+                  target_label: __metrics_path__
+                  regex: (.+)
+...
+```
+We are adding one static target - the metrics from the OTEL collector itself, just for demo purposes, these
+won't be used for scaling decision. And also any service annotated with `prometheus.io/scrape=true`. One can
+also modify the path where the metrics are exported using `prometheus.io/path=/metrics`.
+
+We set these two annotation in our service for podinfo [here](./podinfo-values.yaml).
+
+Install KEDA:
+```bash
+helm upgrade -i keda kedacore/keda --namespace keda --create-namespace
+```
+
+Create `ScaledObject`:
+```bash
+kubectl apply -f podinfo-so.yaml
+```
+
+```bash
+watch kubectl get pods -A
+```
+
+Create some traffic:
+```bash
+hey -n 5000 -c 4 -q 20 -z 70s http://localhost:8080/delay/2
+```

examples/metric-pull/collector-pull-values.yaml

@@ -0,0 +1,71 @@
+settings:
+  metricStoreRetentionSeconds: 60
+  logs:
+    logLvl: debug
+
+opentelemetry-collector:
+  ports:
+    metrics:
+      enabled: true
+  clusterRole:
+    create: true
+    rules:
+     - apiGroups:
+       - ''
+       resources:
+       - 'services'
+       verbs:
+       - 'get'
+       - 'list'
+       - 'watch'
+  config:
+    receivers:
+      opencensus: null
+      # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/prometheusreceiver/README.md
+      prometheus:
+        config:
+          scrape_configs:
+            - job_name: 'otelcol'
+              scrape_interval: 5s
+              static_configs:
+                - targets: ['0.0.0.0:8888']
+            - job_name: k8s
+              kubernetes_sd_configs:
+                - role: service
+              relabel_configs:
+                - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+                  regex: "true"
+                  action: keep
+                - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+                  action: replace
+                  target_label: __metrics_path__
+                  regex: (.+)
+      zipkin: null
+      jaeger: null
+      otlp: null
+    processors:
+      filter/ottl:
+        error_mode: ignore
+        metrics:
+          # runtime/service_invocation/req_sent_total
+          metric:
+            - | # drop all other metrics that are not whitelisted here
+              name != "http_request_duration_seconds_count"
+            - resource.attributes["method"] == "GET"
+            - resource.attributes["path"] == "delay"
+
+    service:
+      telemetry:
+        metrics:
+          address: 0.0.0.0:8888
+      pipelines:
+        traces: null
+        logs: null
+        metrics:
+          receivers:
+            - prometheus
+          processors:
+            - filter/ottl
+          exporters:
+            - debug
+            - otlp

examples/metric-pull/podinfo-so.yaml

@@ -0,0 +1,22 @@
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: podinfo-pull-example
+spec:
+  scaleTargetRef:
+    name: podinfo
+  triggers:
+    - type: external
+      metadata:
+        scalerAddress: "keda-otel-scaler.default.svc:4318"
+        metricQuery: "avg(http_request_duration_seconds_count{path=delay, method=GET, status=200})"
+        operationOverTime: "rate"
+        targetValue: "20"
+        clampMax: "600"
+  advanced:
+    horizontalPodAutoscalerConfig:
+      behavior:
+        scaleDown:
+          stabilizationWindowSeconds: 10
+        scaleUp:
+          stabilizationWindowSeconds: 10

examples/metric-pull/podinfo-values.yaml

@@ -0,0 +1,10 @@
+ui:
+  message: "Hello OTEL+KEDA"
+  color: "#222222"
+  logo: "https://kedify.io/assets/images/logo.svg"
+
+service:
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/path: "/metrics"
+

helmchart/otel-add-on/values.yaml

@@ -84,6 +84,7 @@ tolerations: []
 affinity: {}
 
 #otel collector helm chart:
+# https://github.com/open-telemetry/opentelemetry-helm-charts/blob/main/charts/opentelemetry-collector/values.yaml
 #--------------------------
 opentelemetry-collector:
   enabled: true