feat(operator): expose KeptnMetrics as OTel metrics (#684)

Signed-off-by: odubajDT <ondrej.dubaj@dynatrace.com> Signed-off-by: odubajDT <93584209+odubajDT@users.noreply.github.com> Co-authored-by: RealAnna <89971034+RealAnna@users.noreply.github.com>
keptn · Jan 26, 2023 · eab9397 · eab9397
1 parent 4971a8b
commit eab9397
Show file tree

Hide file tree

Showing 18 changed files with 596 additions and 142 deletions.
diff --git a/README.md b/README.md
@@ -354,6 +354,29 @@ spec:
   secretName: prometheusLoginCredentials
 ```
 
+### Keptn Metric
+A `KeptnMetric` is a CRD used to define SLI provider with a query and to store metric data fetched from the provider.
+Providing the metrics as CRD into a K8s cluster will facilitate the reusability of this data across multiple components.
+Furthermore, this allows using multiple observability platforms for different metrics.
+
+A `KeptnMetric` looks like the following:
+
+```yaml
+apiVersion: metrics.keptn.sh/v1alpha1
+kind: KeptnMetric
+metadata:
+  name: keptnmetric-sample
+  namespace: keptn-lifecycle-toolkit-system
+spec:
+  provider:
+    name: "prometheus"
+  query: "sum(kube_pod_container_resource_limits{resource='cpu'})"
+  fetchIntervalSeconds: 5
+```
+
+To be able to use `KeptnMetric` as part of your evaluation, you need to add `keptn-metric` as your value for `.spec.source` in `KeptnEvaluationDefiniton`. Further you need specify
+the `.spec.objectives[i].name` of `KeptnEvaluationDefiniton` to the same value as it is stored in `.metadata.name` of `KeptnMetric` resource. The `.spec.objectives[i].query` parameter
+of `KeptnEvaluationDefiniton` will be ignored and `.spec.query` of `KeptnMetric` will be use instead as a query to fetch the data.
 
 ## Install a dev build
 

diff --git a/docs/content/en/docs/concepts/metrics/_index.md b/docs/content/en/docs/concepts/metrics/_index.md
@@ -0,0 +1,38 @@
+---
+title: Metrics
+description: Learn what Keptn Metrics are and how to use them
+icon: concepts
+layout: quickstart
+weight: 10
+hidechildren: true # this flag hides all sub-pages in the sidebar-multicard.html
+---
+
+### Keptn Metric
+A `KeptnMetric` is a CRD representing a metric. The metric will be collected from the provider specified in the specs.provider.name field. The query is a string in the provider-specific query language, used to obtain a metric. Providing the metrics as CRD into a K8s cluster will facilitate the reusability of this data across multiple components. Furthermore, this allows using multiple observability platforms for different metrics. Please note, there is a limitation that `KeptnMetric` resource needs to be created only in `keptn-lifecycle-toolkit-system` namespace.
+
+A `KeptnMetric` looks like the following:
+
+```yaml
+apiVersion: metrics.keptn.sh/v1alpha1
+kind: KeptnMetric
+metadata:
+  name: keptnmetric-sample
+  namespace: keptn-lifecycle-toolkit-system
+spec:
+  provider:
+    name: "<your-keptn-evaluation-provider-crd-name>"
+  query: "sum(kube_pod_container_resource_limits{resource='cpu'})"
+  fetchIntervalSeconds: 5
+```
+
+Keptn metrics can be exposed as OTel metrics via port `9999` of the KLT operator. To expose them, the env variable `EXPOSE_KEPTN_METRICS` in the operator manifest needs to be set to `true`. The default value of this variable is `true`. To access the metrics, use the following command:
+
+```
+kubectl port-forward deployment/klc-controller-manager 9999 -n keptn-lifecycle-toolkit-system
+```
+
+and access the metrics via your browser with:
+
+```
+http://localhost:9999/metrics
+```
diff --git a/functions-runtime/Dockerfile b/functions-runtime/Dockerfile
@@ -1,4 +1,4 @@
-FROM denoland/deno:alpine-1.29.1 as production
+FROM denoland/deno:alpine-1.29.1 AS production
 
 LABEL org.opencontainers.image.source="https://github.com/keptn/lifecycle-toolkit" \
     org.opencontainers.image.url="https://keptn.sh" \

diff --git a/klt-cert-manager/Dockerfile b/klt-cert-manager/Dockerfile
@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM --platform=$BUILDPLATFORM golang:1.19.5-alpine3.16 as builder
+FROM --platform=$BUILDPLATFORM golang:1.19.5-alpine3.16 AS builder
 
 ENV CGO_ENABLED=0
 
@@ -32,7 +32,7 @@ RUN controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..." &&
     -X main.buildVersion=${RELEASE_VERSION}" \
     -a -o manager main.go
 
-FROM gcr.io/distroless/base-debian11:debug-nonroot as debug
+FROM gcr.io/distroless/base-debian11:debug-nonroot AS debug
 
 LABEL org.opencontainers.image.source="https://github.com/keptn/lifecycle-toolkit" \
     org.opencontainers.image.url="https://keptn.sh" \
@@ -45,7 +45,7 @@ COPY --from=builder /workspace/manager .
 
 ENTRYPOINT ["/manager"]
 
-FROM gcr.io/distroless/base-debian11:nonroot as production
+FROM gcr.io/distroless/base-debian11:nonroot AS production
 
 LABEL org.opencontainers.image.source="https://github.com/keptn/lifecycle-toolkit/klt-cert-manager" \
     org.opencontainers.image.url="https://keptn.sh" \

diff --git a/operator/Dockerfile b/operator/Dockerfile
@@ -1,4 +1,4 @@
-FROM --platform=$BUILDPLATFORM golang:1.19.5-alpine3.16 as builder
+FROM --platform=$BUILDPLATFORM golang:1.19.5-alpine3.16 AS builder
 
 ENV CGO_ENABLED=0
 
@@ -30,7 +30,7 @@ RUN controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..." &&
     -X main.buildVersion=${RELEASE_VERSION}" \
     -o bin/manager main.go
 
-FROM gcr.io/distroless/base-debian11:debug-nonroot as debug
+FROM gcr.io/distroless/base-debian11:debug-nonroot AS debug
 
 LABEL org.opencontainers.image.source="https://github.com/keptn/lifecycle-toolkit" \
     org.opencontainers.image.url="https://keptn.sh" \
@@ -43,7 +43,7 @@ COPY --from=builder /workspace/bin/manager .
 
 ENTRYPOINT ["/manager"]
 
-FROM gcr.io/distroless/base-debian11:nonroot as production
+FROM gcr.io/distroless/base-debian11:nonroot AS production
 
 LABEL org.opencontainers.image.source="https://github.com/keptn/lifecycle-toolkit" \
     org.opencontainers.image.url="https://keptn.sh" \

diff --git a/operator/config/manager/manager.yaml b/operator/config/manager/manager.yaml
@@ -23,6 +23,7 @@ spec:
         kubectl.kubernetes.io/default-container: manager
         metrics.dynatrace.com/scrape: 'true'
         metrics.dynatrace.com/port: '2222'
+        openfeature.dev/enabled: "true"
       labels:
         control-plane: controller-manager
     spec:
@@ -52,6 +53,9 @@ spec:
           - containerPort: 9443
             name: webhook-server
             protocol: TCP
+          - name: keptn-metrics
+            containerPort: 9999
+            protocol: TCP
         imagePullPolicy: Always
         env:
           - name: POD_NAMESPACE
@@ -64,6 +68,8 @@ spec:
                 fieldPath: metadata.name
           - name: OTEL_COLLECTOR_URL
             value: otel-collector:4317
+          - name: EXPOSE_KEPTN_METRICS
+            value: "true"
           - name: FUNCTION_RUNNER_IMAGE
             value: ghcr.keptn.sh/keptn/functions-runtime:v0.5.0 #x-release-please-version
         securityContext:
@@ -103,3 +109,4 @@ spec:
             mountPath: /tmp/k8s-webhook-server/serving-certs/
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10
+
diff --git a/operator/config/rbac/auth_proxy_service.yaml b/operator/config/rbac/auth_proxy_service.yaml
@@ -15,5 +15,9 @@ spec:
     port: 2222
     protocol: TCP
     targetPort: 2222
+  - name: keptn-metrics
+    protocol: TCP
+    port: 9999
+    targetPort: 9999
   selector:
     control-plane: controller-manager
diff --git a/operator/go.mod b/operator/go.mod
@@ -3,13 +3,16 @@ module github.com/keptn/lifecycle-toolkit/operator
 go 1.19
 
 require (
+	github.com/benbjohnson/clock v1.1.0
 	github.com/go-logr/logr v1.2.3
+	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/imdario/mergo v0.3.13
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/magiconair/properties v1.8.7
 	github.com/onsi/ginkgo/v2 v2.7.0
 	github.com/onsi/gomega v1.24.2
+	github.com/open-feature/go-sdk v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/common v0.39.0
@@ -24,42 +27,41 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v0.34.0
 	go.opentelemetry.io/otel/trace v1.11.2
 	golang.org/x/exp v0.0.0-20230118134722-a68e582fa157
-	google.golang.org/grpc v1.51.0
-	k8s.io/api v0.25.5
-	k8s.io/apiextensions-apiserver v0.25.5
-	k8s.io/apimachinery v0.25.6
-	k8s.io/apiserver v0.25.5
-	k8s.io/client-go v0.25.5
-	sigs.k8s.io/controller-runtime v0.13.1
+	google.golang.org/grpc v1.52.0
+	k8s.io/api v0.26.0
+	k8s.io/apiextensions-apiserver v0.26.0
+	k8s.io/apimachinery v0.26.0
+	k8s.io/apiserver v0.26.0
+	k8s.io/client-go v0.26.0
+	k8s.io/klog/v2 v2.80.1
+	sigs.k8s.io/controller-runtime v0.14.1
 )
 
 require (
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.5 // indirect
-	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-openapi/jsonreference v0.20.0 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -71,27 +73,28 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.21.0 // indirect
-	golang.org/x/net v0.4.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/net v0.5.0 // indirect
 	golang.org/x/oauth2 v0.3.0 // indirect
-	golang.org/x/sys v0.3.0 // indirect
-	golang.org/x/term v0.3.0 // indirect
-	golang.org/x/text v0.5.0 // indirect
-	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
+	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/term v0.4.0 // indirect
+	golang.org/x/text v0.6.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect
+	google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/component-base v0.25.5 // indirect
-	k8s.io/klog/v2 v2.70.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
-	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	k8s.io/component-base v0.26.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a // indirect
+	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
+
+replace github.com/open-feature/go-sdk-contrib/providers/flagd => github.com/open-feature/flagd v0.3.1