From 426e83a853a443b746b21ef45cf5e470a9d8ba5a Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Thu, 24 Oct 2024 17:56:35 +0200 Subject: [PATCH] Add another use-case - otel demo that pushes metrics + support also . as a metric chunks separator Signed-off-by: Jirka Kremser --- examples/metric-pull/README.md | 2 +- ...=> scaler-with-collector-pull-values.yaml} | 12 +-- examples/metric-push/README.md | 89 +++++++++++++++++++ .../opentelemetry-demo-values.yaml | 24 +++++ .../metric-push/scaler-only-push-values.yaml | 8 ++ examples/metric-push/sos.yaml | 47 ++++++++++ helmchart/otel-add-on/values.yaml | 1 - metric/mem_store.go | 2 +- metric/mem_store_test.go | 30 +++++++ 9 files changed, 206 insertions(+), 9 deletions(-) rename examples/metric-pull/{collector-pull-values.yaml => scaler-with-collector-pull-values.yaml} (95%) create mode 100644 examples/metric-push/README.md create mode 100644 examples/metric-push/opentelemetry-demo-values.yaml create mode 100644 examples/metric-push/scaler-only-push-values.yaml create mode 100644 examples/metric-push/sos.yaml diff --git a/examples/metric-pull/README.md b/examples/metric-pull/README.md index 3f9072e..66cf730 100644 --- a/examples/metric-pull/README.md +++ b/examples/metric-pull/README.md @@ -28,7 +28,7 @@ open http://localhost:8181/metrics Install this addon: ```bash -helm upgrade -i kedify-otel kedify-otel/otel-add-on --version=v0.0.1-0 -f collector-pull-values.yaml +helm upgrade -i kedify-otel kedify-otel/otel-add-on --version=v0.0.1-0 -f scaler-with-collector-pull-values.yaml ``` Note the following section in the helm chart values that configures the OTEL collector to scrape targets: diff --git a/examples/metric-pull/collector-pull-values.yaml b/examples/metric-pull/scaler-with-collector-pull-values.yaml similarity index 95% rename from examples/metric-pull/collector-pull-values.yaml rename to examples/metric-pull/scaler-with-collector-pull-values.yaml index 1c45563..41a3268 100644 --- a/examples/metric-pull/collector-pull-values.yaml +++ b/examples/metric-pull/scaler-with-collector-pull-values.yaml @@ -22,7 +22,7 @@ opentelemetry-collector: - 'watch' config: receivers: - opencensus: null + opencensus: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/prometheusreceiver/README.md prometheus: config: @@ -48,9 +48,9 @@ opentelemetry-collector: target_label: __address__ regex: (.+)(?::\d+);(\d+) replacement: $1:$2 - zipkin: null - jaeger: null - otlp: null + zipkin: + jaeger: + otlp: exporters: otlp: endpoint: keda-otel-scaler:4317 @@ -72,8 +72,8 @@ opentelemetry-collector: service: pipelines: - traces: null - logs: null + traces: + logs: metrics: receivers: - prometheus diff --git a/examples/metric-push/README.md b/examples/metric-push/README.md new file mode 100644 index 0000000..35e28c2 --- /dev/null +++ b/examples/metric-push/README.md @@ -0,0 +1,89 @@ +# Use-case: pull metrics + +This use-case demonstrates how OTEL collector can be used as a scraper of another metric endpoints and +then forwarding the filtered metrics into OTLP receiver in our scaler. + +Prepare helm chart repos: + +```bash +helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts +helm repo add kedify https://kedify.github.io/charts +helm repo add kedify-otel https://kedify.github.io/otel-add-on +helm repo update +``` + +Any Kubernetes cluster will do: +```bash +k3d cluster create metric-push -p "8080:31198@server:0" +``` + +Install demo app +- architecture: https://opentelemetry.io/docs/demo/architecture/ +- helm chart: https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-demo + +```bash +helm upgrade -i my-otel-demo open-telemetry/opentelemetry-demo -f opentelemetry-demo-values.yaml +# check if the app is running +open http://localhost:8080 +``` + +Install this addon: +```bash +helm upgrade -i kedify-otel kedify-otel/otel-add-on --version=v0.0.1-0 -f scaler-only-push-values.yaml +``` + +In this scenario, we don't install OTEL collector using the `kedify-otel/otel-add-on` helm chart, because +the `opentelemetry-demo` already creates one and it was configured to forward all metrics to our scaler. +If we wanted to filter the metrics, we would need to deploy another OTEL collector and configure the processor +there so that it would look like this: + +```bash + ┌────────────┐ ┌────────────┐ ┌─────────────┐ + │ │ │ │ │ │ + │ OTEL col 1 ├────►│ OTEL col 2 ├────►│ this scaler │ + │ │ │ (filtering)│ │ │ + └────────────┘ └────────────┘ └─────────────┘ + +instead we go w/ simple (w/o filtering): + ┌────────────┐ ┌─────────────┐ + │ │ │ │ + │ OTEL col 1 ├────►│ this scaler │ + │ │ │ │ + └────────────┘ └─────────────┘ +``` + +Install KEDA by Kedify.io: +```bash +helm upgrade -i keda kedify/keda --namespace keda --create-namespace +``` + +We will be scaling two microservices for this application, first let's check what metrics are there in shipped +[grafana](http://localhost:8080/grafana/explore?schemaVersion=1&panes=%7B%222n3%22:%7B%22datasource%22:%22webstore-metrics%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22app_frontend_requests_total%22,%22range%22:true,%22instant%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22webstore-metrics%22%7D,%22editorMode%22:%22code%22,%22legendFormat%22:%22__auto%22,%22useBackend%22:false,%22disableTextWrap%22:false,%22fullMetaSearch%22:false,%22includeNullMetadata%22:true%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1). + +We will use following two metrics for scaling microservices `recommendationservice` and `productcatalogservice`. +```bash +... +app_frontend_requests_total{instance="0b38958c-f169-4a83-9adb-cf2c2830d61e", job="opentelemetry-demo/frontend", method="GET", status="200", target="/api/recommendations"} +1824 +app_frontend_requests_total{instance="0b38958c-f169-4a83-9adb-cf2c2830d61e", job="opentelemetry-demo/frontend", method="GET", status="200", target="/api/products"} +1027 +... +``` + +Create `ScaledObject`s: +```bash +kubectl apply -f sos.yaml +``` + +The demo application contains a load generator that can be further tweaked on http://localhost:8080/loadgen endpoint and by +default, creates a lot of traffic in the eshop. So there is no need to create further load from our side and we can just +observe the effects of autoscaling: + +```bash +watch kubectl get deploy my-otel-demo-recommendationservice my-otel-demo-productcatalogservice +``` + +Once finished, clean the cluster: +```bash +k3d cluster delete metric-push +``` \ No newline at end of file diff --git a/examples/metric-push/opentelemetry-demo-values.yaml b/examples/metric-push/opentelemetry-demo-values.yaml new file mode 100644 index 0000000..ab7a52c --- /dev/null +++ b/examples/metric-push/opentelemetry-demo-values.yaml @@ -0,0 +1,24 @@ +# https://github.com/open-telemetry/opentelemetry-helm-charts/blob/main/charts/opentelemetry-demo/values.yaml +components: + frontendProxy: + service: + type: NodePort + nodePort: 31198 +opentelemetry-collector: + config: + exporters: + # this is the original exporter, we just rename it from otlp to otlp/jaeger to preserve it + otlp/jaeger: + endpoint: '{{ include "otel-demo.name" . }}-jaeger-collector:4317' + tls: + insecure: true + otlp/keda: + endpoint: keda-otel-scaler:4317 + tls: + insecure: true + service: + pipelines: + traces: + exporters: [otlp/jaeger, debug, spanmetrics] + metrics: + exporters: [ otlp/keda, otlphttp/prometheus, debug ] diff --git a/examples/metric-push/scaler-only-push-values.yaml b/examples/metric-push/scaler-only-push-values.yaml new file mode 100644 index 0000000..a927217 --- /dev/null +++ b/examples/metric-push/scaler-only-push-values.yaml @@ -0,0 +1,8 @@ +settings: + metricStoreRetentionSeconds: 60 + logs: + logLvl: debug + +# otel collector will be installed from another helm chart +opentelemetry-collector: + enabled: false diff --git a/examples/metric-push/sos.yaml b/examples/metric-push/sos.yaml new file mode 100644 index 0000000..a4ece65 --- /dev/null +++ b/examples/metric-push/sos.yaml @@ -0,0 +1,47 @@ +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: recommendationservice +spec: + scaleTargetRef: + name: my-otel-demo-recommendationservice + triggers: + - type: external + metadata: + scalerAddress: "keda-otel-scaler.default.svc:4318" + metricQuery: "avg(app_frontend_requests{target=/api/recommendations, method=GET, status=200})" + operationOverTime: "rate" + targetValue: "10" + clampMax: "600" + minReplicaCount: 1 + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + stabilizationWindowSeconds: 10 + scaleUp: + stabilizationWindowSeconds: 10 +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: my-otel-demo-productcatalogservice +spec: + scaleTargetRef: + name: podinfo + triggers: + - type: external + metadata: + scalerAddress: "keda-otel-scaler.default.svc:4318" + metricQuery: "avg(app_frontend_requests{target=/api/products, method=GET, status=200})" + operationOverTime: "rate" + targetValue: "10" + clampMax: "600" + minReplicaCount: 1 + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + stabilizationWindowSeconds: 10 + scaleUp: + stabilizationWindowSeconds: 10 diff --git a/helmchart/otel-add-on/values.yaml b/helmchart/otel-add-on/values.yaml index bf0195b..3c6652b 100644 --- a/helmchart/otel-add-on/values.yaml +++ b/helmchart/otel-add-on/values.yaml @@ -125,7 +125,6 @@ opentelemetry-collector: compression: "none" tls: insecure: true - # endpoint: otel-add-on:4317 # tls: # cert_file: file.cert # key_file: file.key diff --git a/metric/mem_store.go b/metric/mem_store.go index 6c6d53b..764eccc 100644 --- a/metric/mem_store.go +++ b/metric/mem_store.go @@ -124,7 +124,7 @@ func (m ms) Put(entry types.NewMetricEntry) { } func escapeName(name types.MetricName) types.MetricName { - return types.MetricName(strings.ReplaceAll(string(name), "/", "_")) + return types.MetricName(strings.ReplaceAll(strings.ReplaceAll(string(name), "/", "_"), ".", "_")) } func timestampToSeconds(timestamp pcommon.Timestamp) uint32 { diff --git a/metric/mem_store_test.go b/metric/mem_store_test.go index 17fe208..942b330 100644 --- a/metric/mem_store_test.go +++ b/metric/mem_store_test.go @@ -33,6 +33,36 @@ func TestMemStorePutOneAndGetOne(t *testing.T) { assertMetricFound(t, val, found, err, 42.) } +func TestMemStoreEscapeMetrics(t *testing.T) { + // setup + ms := NewMetricStore(5) + ms.Put(types.NewMetricEntry{ + Name: "metric/one", + MeasurementTime: pcommon.Timestamp(time.Now().Unix()), + MeasurementValue: 42., + Labels: map[string]any{ + "a": "1", + "b": "2", + }, + }) + ms.Put(types.NewMetricEntry{ + Name: "metric.two", + MeasurementTime: pcommon.Timestamp(time.Now().Unix()), + MeasurementValue: 43., + Labels: map[string]any{ + "a": "2", + }, + }) + + // checks + val1, found1, err1 := ms.Get("metric_one", map[string]any{"b": "2", "a": "1"}, types.OpLastOne, types.VecSum) + assertMetricFound(t, val1, found1, err1, 42.) + val2, found2, err2 := ms.Get("metric.one", map[string]any{"b": "2", "a": "1"}, types.OpLastOne, types.VecSum) + assertMetricFound(t, val2, found2, err2, 42.) + val3, found3, err3 := ms.Get("metric_two", map[string]any{"a": "2"}, types.OpLastOne, types.VecSum) + assertMetricFound(t, val3, found3, err3, 43.) +} + func TestMemStoreErr(t *testing.T) { // setup ms := NewMetricStore(5)