From 96c44f2e8775fc0d52d865ef6104e20c52b1595b Mon Sep 17 00:00:00 2001 From: Jean-Francois Roy Date: Fri, 26 Apr 2024 21:28:51 -0700 Subject: [PATCH] feat(kube-prometheus-stack): add kube-state-metrics configmap The configmap cleans up the main helmrelease file, and includes the changes in https://github.com/fluxcd/flux2-monitoring-example/pull/33. --- .../app/helmrelease.yaml | 308 +----------------- .../app/kustomization.yaml | 7 +- .../app/kustomizeconfig.yaml | 6 + .../resources/kube-state-metrics-config.yaml | 270 +++++++++++++++ 4 files changed, 290 insertions(+), 301 deletions(-) create mode 100644 kubernetes/apps/observability/kube-prometheus-stack/app/kustomizeconfig.yaml create mode 100644 kubernetes/apps/observability/kube-prometheus-stack/app/resources/kube-state-metrics-config.yaml diff --git a/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml b/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml index 85a31b0cd..363b13f9c 100644 --- a/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml +++ b/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml @@ -29,6 +29,15 @@ spec: namespace: openebs-system driftDetection: mode: enabled + ignore: + # Ignore "validated" annotation which is not inserted during install + - paths: [ "/metadata/annotations/prometheus-operator-validated" ] + target: + kind: PrometheusRule + valuesFrom: + - kind: ConfigMap + name: kube-state-metrics-config + valuesKey: kube-state-metrics-config.yaml values: cleanPrometheusOperatorObjectNames: true @@ -127,307 +136,8 @@ spec: kubeProxy: enabled: false - kube-state-metrics: - fullnameOverride: kube-state-metrics - rbac: - extraRules: - - apiGroups: - - source.toolkit.fluxcd.io - - kustomize.toolkit.fluxcd.io - - helm.toolkit.fluxcd.io - - notification.toolkit.fluxcd.io - - image.toolkit.fluxcd.io - resources: - - gitrepositories - - buckets - - helmrepositories - - helmcharts - - ocirepositories - - kustomizations - - helmreleases - - alerts - - providers - - receivers - - imagerepositories - - imagepolicies - - imageupdateautomations - verbs: [ "list", "watch" ] - # prometheus: - # monitor: - # relabelings: - # - action: replace - # sourceLabels: [__meta_kubernetes_pod_node_name] - # targetLabel: nodename - # - action: replace - # regex: (.*) - # replacement: $1 - # sourceLabels: ["__meta_kubernetes_pod_node_name"] - # targetLabel: kubernetes_node - # metricLabelsAllowlist: - # - pods=[*] - # - deployments=[*] - # - persistentvolumeclaims=[*] - customResourceState: - enabled: true - config: - spec: - resources: - - groupVersionKind: - group: kustomize.toolkit.fluxcd.io - version: v1 - kind: Kustomization - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - revision: [ status, lastAppliedRevision ] - source_name: [ spec, sourceRef, name ] - - groupVersionKind: - group: helm.toolkit.fluxcd.io - version: v2beta2 - kind: HelmRelease - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - revision: [ status, lastAppliedRevision ] - chart_name: [ spec, chart, spec, chart ] - chart_source_name: [ spec, chart, spec, sourceRef, name ] - - groupVersionKind: - group: source.toolkit.fluxcd.io - version: v1 - kind: GitRepository - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - revision: [ status, artifact, revision ] - url: [ spec, url ] - - groupVersionKind: - group: source.toolkit.fluxcd.io - version: v1beta2 - kind: Bucket - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - revision: [ status, artifact, revision ] - endpoint: [ spec, endpoint ] - bucket_name: [ spec, bucketName ] - - groupVersionKind: - group: source.toolkit.fluxcd.io - version: v1beta2 - kind: HelmRepository - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - revision: [ status, artifact, revision ] - url: [ spec, url ] - - groupVersionKind: - group: source.toolkit.fluxcd.io - version: v1beta2 - kind: HelmChart - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - revision: [ status, artifact, revision ] - chart_name: [ spec, chart ] - chart_version: [ spec, version ] - - groupVersionKind: - group: source.toolkit.fluxcd.io - version: v1beta2 - kind: OCIRepository - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - revision: [ status, artifact, revision ] - url: [ spec, url ] - - groupVersionKind: - group: notification.toolkit.fluxcd.io - version: v1beta3 - kind: Alert - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - suspended: [ spec, suspend ] - - groupVersionKind: - group: notification.toolkit.fluxcd.io - version: v1beta3 - kind: Provider - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - suspended: [ spec, suspend ] - - groupVersionKind: - group: notification.toolkit.fluxcd.io - version: v1 - kind: Receiver - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - webhook_path: [ status, webhookPath ] - - groupVersionKind: - group: image.toolkit.fluxcd.io - version: v1beta2 - kind: ImageRepository - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - image: [ spec, image ] - - groupVersionKind: - group: image.toolkit.fluxcd.io - version: v1beta2 - kind: ImagePolicy - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - source_name: [ spec, imageRepositoryRef, name ] - - groupVersionKind: - group: image.toolkit.fluxcd.io - version: v1beta1 - kind: ImageUpdateAutomation - metricNamePrefix: gotk - metrics: - - name: "resource_info" - help: "The current state of a GitOps Toolkit resource." - each: - type: Info - info: - labelsFromPath: - name: [ metadata, name ] - labelsFromPath: - exported_namespace: [ metadata, namespace ] - ready: [ status, conditions, "[type=Ready]", status ] - suspended: [ spec, suspend ] - source_name: [ spec, sourceRef, name ] - prometheus-node-exporter: fullnameOverride: node-exporter - # prometheus: - # monitor: - # relabelings: - # - action: replace - # sourceLabels: [__meta_kubernetes_pod_node_name] - # targetLabel: nodename - # - action: labeldrop - # regex: (pod) - # - action: replace - # regex: (.*) - # replacement: $1 - # sourceLabels: ["__meta_kubernetes_pod_node_name"] - # targetLabel: kubernetes_node prometheusOperator: fullnameOverride: prometheus-operator diff --git a/kubernetes/apps/observability/kube-prometheus-stack/app/kustomization.yaml b/kubernetes/apps/observability/kube-prometheus-stack/app/kustomization.yaml index 2084239de..ed68d55f5 100644 --- a/kubernetes/apps/observability/kube-prometheus-stack/app/kustomization.yaml +++ b/kubernetes/apps/observability/kube-prometheus-stack/app/kustomization.yaml @@ -9,7 +9,10 @@ resources: - ./prometheusrule.yaml configMapGenerator: - name: alertmanager-config-tpl - files: - - alertmanager.yaml=./resources/alertmanager.yaml + files: [./resources/alertmanager.yaml] + - name: kube-state-metrics-config + files: [./resources/kube-state-metrics-config.yaml] generatorOptions: disableNameSuffixHash: true +configurations: + - kustomizeconfig.yaml diff --git a/kubernetes/apps/observability/kube-prometheus-stack/app/kustomizeconfig.yaml b/kubernetes/apps/observability/kube-prometheus-stack/app/kustomizeconfig.yaml new file mode 100644 index 000000000..a80be15a1 --- /dev/null +++ b/kubernetes/apps/observability/kube-prometheus-stack/app/kustomizeconfig.yaml @@ -0,0 +1,6 @@ +nameReference: + - kind: ConfigMap + version: v1 + fieldSpecs: + - path: spec/valuesFrom/name + kind: HelmRelease diff --git a/kubernetes/apps/observability/kube-prometheus-stack/app/resources/kube-state-metrics-config.yaml b/kubernetes/apps/observability/kube-prometheus-stack/app/resources/kube-state-metrics-config.yaml new file mode 100644 index 000000000..c647e1f7a --- /dev/null +++ b/kubernetes/apps/observability/kube-prometheus-stack/app/resources/kube-state-metrics-config.yaml @@ -0,0 +1,270 @@ +kube-state-metrics: + fullnameOverride: kube-state-metrics + rbac: + extraRules: + - apiGroups: + - source.toolkit.fluxcd.io + - kustomize.toolkit.fluxcd.io + - helm.toolkit.fluxcd.io + - notification.toolkit.fluxcd.io + - image.toolkit.fluxcd.io + resources: + - gitrepositories + - buckets + - helmrepositories + - helmcharts + - ocirepositories + - kustomizations + - helmreleases + - alerts + - providers + - receivers + - imagerepositories + - imagepolicies + - imageupdateautomations + verbs: [ "list", "watch" ] + customResourceState: + enabled: true + config: + spec: + resources: + - groupVersionKind: + group: kustomize.toolkit.fluxcd.io + version: v1 + kind: Kustomization + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux Kustomization resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, lastAppliedRevision ] + source_name: [ spec, sourceRef, name ] + - groupVersionKind: + group: helm.toolkit.fluxcd.io + version: v2beta2 + kind: HelmRelease + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux HelmRelease resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, lastAppliedRevision ] + chart_name: [ spec, chart, spec, chart ] + chart_source_name: [ spec, chart, spec, sourceRef, name ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1 + kind: GitRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux GitRepository resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + url: [ spec, url ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: Bucket + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux Bucket resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + endpoint: [ spec, endpoint ] + bucket_name: [ spec, bucketName ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: HelmRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux HelmRepository resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + url: [ spec, url ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: HelmChart + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux HelmChart resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + chart_name: [ spec, chart ] + chart_version: [ spec, version ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: OCIRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux OCIRepository resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + url: [ spec, url ] + - groupVersionKind: + group: notification.toolkit.fluxcd.io + version: v1beta3 + kind: Alert + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux Alert resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + suspended: [ spec, suspend ] + - groupVersionKind: + group: notification.toolkit.fluxcd.io + version: v1beta3 + kind: Provider + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux Provider resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + suspended: [ spec, suspend ] + - groupVersionKind: + group: notification.toolkit.fluxcd.io + version: v1 + kind: Receiver + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux Receiver resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + webhook_path: [ status, webhookPath ] + - groupVersionKind: + group: image.toolkit.fluxcd.io + version: v1beta2 + kind: ImageRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux ImageRepository resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + image: [ spec, image ] + - groupVersionKind: + group: image.toolkit.fluxcd.io + version: v1beta2 + kind: ImagePolicy + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux ImagePolicy resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + source_name: [ spec, imageRepositoryRef, name ] + - groupVersionKind: + group: image.toolkit.fluxcd.io + version: v1beta1 + kind: ImageUpdateAutomation + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a Flux ImageUpdateAutomation resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + source_name: [ spec, sourceRef, name ]