From 8df7307bfc35fe2871deaa0a49fabf598fc155bd Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Tue, 22 Oct 2024 16:26:25 +0200 Subject: [PATCH] Add simple demo that sets up the OTEL collector in a pull mode (scraping another metrics endpoints) and sending it to scaler Signed-off-by: Jirka Kremser --- .github/workflows/release-chart.yaml | 2 +- README.md | 4 +- examples/metric-pull/README.md | 77 +++++++++++++++++++ .../metric-pull/collector-pull-values.yaml | 71 +++++++++++++++++ examples/metric-pull/podinfo-so.yaml | 22 ++++++ examples/metric-pull/podinfo-values.yaml | 10 +++ helmchart/otel-add-on/values.yaml | 1 + 7 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 examples/metric-pull/README.md create mode 100644 examples/metric-pull/collector-pull-values.yaml create mode 100644 examples/metric-pull/podinfo-so.yaml create mode 100644 examples/metric-pull/podinfo-values.yaml diff --git a/.github/workflows/release-chart.yaml b/.github/workflows/release-chart.yaml index 25c6c6c..ecfdd71 100644 --- a/.github/workflows/release-chart.yaml +++ b/.github/workflows/release-chart.yaml @@ -77,7 +77,7 @@ jobs: _sec=$(echo "1.5^$i" | bc) echo "Waiting ${_sec} seconds.." sleep ${_sec} - helm repo add kedify-otel https://kedify.github.io/otel-add-on || continue + helm repo add kedify-otel https://kedify.github.io/otel-add-on/ || continue helm repo update set -x helm upgrade -i keda-otel-add-on kedify-otel/otel-add-on \ diff --git a/README.md b/README.md index 13376a8..13e23e8 100644 --- a/README.md +++ b/README.md @@ -81,9 +81,9 @@ helm upgrade -i keda kedacore/keda --namespace keda --create-namespace ### Then install this add-on ```bash -helm repo add kedify-otel https://kedify.github.io/otel-add-on +helm repo add kedify-otel https://kedify.github.io/otel-add-on/ helm repo update -helm upgrade -i keda-otel kedify-otel/otel-add-on +helm upgrade -i keda-otel kedify-otel/otel-add-on --version=v0.0.0-1 ``` ### Create an example scaled object diff --git a/examples/metric-pull/README.md b/examples/metric-pull/README.md new file mode 100644 index 0000000..b4ba376 --- /dev/null +++ b/examples/metric-pull/README.md @@ -0,0 +1,77 @@ +# Use-case: pull metrics + +This use-case demonstrates how OTEL collector can be used as a scraper of another metric endpoints and +then forwarding the filtered metrics into OTLP receiver in our scaler. + +Prepare helm chart repos: + +```bash +helm repo add kedacore https://kedacore.github.io/charts +helm repo add podinfo https://stefanprodan.github.io/podinfo +helm repo add kedify-otel https://kedify.github.io/otel-add-on/ +helm repo update +``` + +Install demo webapp: + +```bash +helm upgrade -i podinfo podinfo/podinfo -f podinfo-values.yaml +kubectl -n default port-forward deploy/podinfo 8080:9898 +``` + +Install this addon: +```bash +helm upgrade -i kedify-otel kedify-otel/otel-add-on --version=v0.0.0-1 -f collector-pull-values.yaml +``` + +Note the following section in the helm chart values that configures the OTEL collector to scrape targets: + +```yaml +... + config: + receivers: + # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/prometheusreceiver/README.md + prometheus: + config: + scrape_configs: + - job_name: 'otelcol' + scrape_interval: 5s + static_configs: + - targets: ['0.0.0.0:8888'] + - job_name: k8s + kubernetes_sd_configs: + - role: service + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + regex: "true" + action: keep + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) +... +``` +We are adding one static target - the metrics from the OTEL collector itself, just for demo purposes, these +won't be used for scaling decision. And also any service annotated with `prometheus.io/scrape=true`. One can +also modify the path where the metrics are exported using `prometheus.io/path=/metrics`. + +We set these two annotation in our service for podinfo [here](./podinfo-values.yaml). + +Install KEDA: +```bash +helm upgrade -i keda kedacore/keda --namespace keda --create-namespace +``` + +Create `ScaledObject`: +```bash +kubectl apply -f podinfo-so.yaml +``` + +```bash +watch kubectl get pods -A +``` + +Create some traffic: +```bash +hey -n 5000 -c 4 -q 20 -z 70s http://localhost:8080/delay/2 +``` diff --git a/examples/metric-pull/collector-pull-values.yaml b/examples/metric-pull/collector-pull-values.yaml new file mode 100644 index 0000000..75d5fdc --- /dev/null +++ b/examples/metric-pull/collector-pull-values.yaml @@ -0,0 +1,71 @@ +settings: + metricStoreRetentionSeconds: 60 + logs: + logLvl: debug + +opentelemetry-collector: + ports: + metrics: + enabled: true + clusterRole: + create: true + rules: + - apiGroups: + - '' + resources: + - 'services' + verbs: + - 'get' + - 'list' + - 'watch' + config: + receivers: + opencensus: null + # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/prometheusreceiver/README.md + prometheus: + config: + scrape_configs: + - job_name: 'otelcol' + scrape_interval: 5s + static_configs: + - targets: ['0.0.0.0:8888'] + - job_name: k8s + kubernetes_sd_configs: + - role: service + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + regex: "true" + action: keep + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + zipkin: null + jaeger: null + otlp: null + processors: + filter/ottl: + error_mode: ignore + metrics: + # runtime/service_invocation/req_sent_total + metric: + - | # drop all other metrics that are not whitelisted here + name != "http_request_duration_seconds_count" + - resource.attributes["method"] == "GET" + - resource.attributes["path"] == "delay" + + service: + telemetry: + metrics: + address: 0.0.0.0:8888 + pipelines: + traces: null + logs: null + metrics: + receivers: + - prometheus + processors: + - filter/ottl + exporters: + - debug + - otlp diff --git a/examples/metric-pull/podinfo-so.yaml b/examples/metric-pull/podinfo-so.yaml new file mode 100644 index 0000000..934fe9e --- /dev/null +++ b/examples/metric-pull/podinfo-so.yaml @@ -0,0 +1,22 @@ +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: podinfo-pull-example +spec: + scaleTargetRef: + name: podinfo + triggers: + - type: external + metadata: + scalerAddress: "keda-otel-scaler.default.svc:4318" + metricQuery: "avg(http_request_duration_seconds_count{path=delay, method=GET, status=200})" + operationOverTime: "rate" + targetValue: "20" + clampMax: "600" + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + stabilizationWindowSeconds: 10 + scaleUp: + stabilizationWindowSeconds: 10 diff --git a/examples/metric-pull/podinfo-values.yaml b/examples/metric-pull/podinfo-values.yaml new file mode 100644 index 0000000..a207999 --- /dev/null +++ b/examples/metric-pull/podinfo-values.yaml @@ -0,0 +1,10 @@ +ui: + message: "Hello OTEL+KEDA" + color: "#222222" + logo: "https://kedify.io/assets/images/logo.svg" + +service: + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + diff --git a/helmchart/otel-add-on/values.yaml b/helmchart/otel-add-on/values.yaml index 34529a3..bf0195b 100644 --- a/helmchart/otel-add-on/values.yaml +++ b/helmchart/otel-add-on/values.yaml @@ -84,6 +84,7 @@ tolerations: [] affinity: {} #otel collector helm chart: +# https://github.com/open-telemetry/opentelemetry-helm-charts/blob/main/charts/opentelemetry-collector/values.yaml #-------------------------- opentelemetry-collector: enabled: true