🌱 observability: move metrics to config and use sidecar in kube-state-metrics

Makefile

@@ -549,16 +549,16 @@ generate-e2e-templates-main: $(KUSTOMIZE)
 	$(KUSTOMIZE) build $(INMEMORY_TEMPLATES)/main/cluster-template --load-restrictor LoadRestrictionsNone > $(INMEMORY_TEMPLATES)/main/cluster-template.yaml
 
 .PHONY: generate-metrics-config
-generate-metrics-config: $(ENVSUBST_BIN) ## Generate ./hack/observability/kube-state-metrics/crd-config.yaml
-	OUTPUT_FILE="${OBSERVABILITY_DIR}/kube-state-metrics/crd-config.yaml"; \
-	METRICS_DIR="${OBSERVABILITY_DIR}/kube-state-metrics/metrics"; \
+generate-metrics-config: $(ENVSUBST_BIN) ## Generate ./config/metrics/crd-metrics-config.yaml
+	OUTPUT_FILE="./config/metrics/crd-metrics-config.yaml"; \
+	METRIC_TEMPLATES_DIR="./config/metrics/templates"; \
 	echo "# This file was auto-generated via: make generate-metrics-config" > "$${OUTPUT_FILE}"; \
-	cat "$${METRICS_DIR}/header.yaml" >> "$${OUTPUT_FILE}"; \
+	cat "$${METRIC_TEMPLATES_DIR}/header.yaml" >> "$${OUTPUT_FILE}"; \
 	for resource in clusterclass cluster kubeadmcontrolplane kubeadmconfig machine machinedeployment machinehealthcheck machineset machinepool; do \
-		cat "$${METRICS_DIR}/$${resource}.yaml"; \
-		RESOURCE="$${resource}" ${ENVSUBST_BIN} < "$${METRICS_DIR}/common_metrics.yaml"; \
+		cat "$${METRIC_TEMPLATES_DIR}/$${resource}.yaml"; \
+		RESOURCE="$${resource}" ${ENVSUBST_BIN} < "$${METRIC_TEMPLATES_DIR}/common_metrics.yaml"; \
 		if [[ "$${resource}" != "cluster" ]]; then \
-			cat "$${METRICS_DIR}/owner_metric.yaml"; \
+			cat "$${METRIC_TEMPLATES_DIR}/owner_metric.yaml"; \
 		fi \
 	done >> "$${OUTPUT_FILE}"; \
 

Tiltfile

@@ -482,6 +482,19 @@ def deploy_observability():
             objects = ["capi-visualizer:serviceaccount"],
         )
 
+def deploy_kustomizations():
+    for name in settings.get("deploy_kustomizations", []):
+        yaml = read_file("./.tiltbuild/yaml/{}.kustomization.yaml".format(name))
+        k8s_yaml(yaml)
+        objs = decode_yaml_stream(yaml)
+        print("objects")
+        print(find_all_objects_names(objs))
+        k8s_resource(
+            new_name = name,
+            objects = find_all_objects_names(objs),
+            labels = ["kustomization"],
+        )
+
 def prepare_all():
     tools_arg = "--tools kustomize,envsubst,clusterctl "
     tilt_settings_file_arg = "--tilt-settings-file " + tilt_file
@@ -640,6 +653,8 @@ deploy_provider_crds()
 
 deploy_observability()
 
+deploy_kustomizations()
+
 enable_providers()
 
 cluster_templates()

config/metrics/crd-clusterrole.yaml

@@ -0,0 +1,38 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics-custom-resource-capi
+  labels:
+    kube-state-metrics/aggregate-to-manager: "true"
+rules:
+- apiGroups:
+  - cluster.x-k8s.io
+  resources:
+  - clusterclasses
+  - clusters
+  - machinedeployments
+  - machinepools
+  - machinesets
+  - machines
+  - machinehealthchecks
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - controlplane.cluster.x-k8s.io
+  resources:
+  - kubeadmcontrolplanes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - bootstrap.cluster.x-k8s.io
+  resources:
+  - kubeadmconfigs
+  verbs:
+  - get
+  - list
+  - watch

config/metrics/kustomization.yaml

@@ -0,0 +1,13 @@
+resources:
+  - ./crd-clusterrole.yaml
+
+namespace: observability
+
+configMapGenerator:
+- name: kube-state-metrics-crd-config-capi
+  files:
+  - capi.yaml=crd-metrics-config.yaml
+  options:
+    disableNameSuffixHash: true
+    labels:
+      kube-state-metrics/custom-resource: "true"

hack/observability/kube-state-metrics/metrics/README.md → config/metrics/templates/README.md

@@ -4,4 +4,4 @@
 
 The make target `generate-metrics-config` is used to generate a single file which contains the Cluster API specific custom resource configuration for kube-state-metrics.
 
-To regenerate the file `../crd-config.yaml`, execute the `make generate-metrics-config` command.
+To regenerate the file `../crd-metrics-config.yaml`, execute the `make generate-metrics-config` command.

docs/book/src/developer/tilt.md

@@ -214,6 +214,14 @@ Supported values are:
 
 \*: Note: the UI will be accessible via a link in the tilt console
 
+**deploy_kustomizations** (map[string]string, default={}): If set, installs the additional kustomizations to the cluster.
+
+Example:
+```yaml
+deploy_kustomizations:
+  capv-metrics: ../cluster-api-provider-vsphere/config/metrics
+```
+
 **debug** (Map{string: Map} default{}): A map of named configurations for the provider. The key is the name of the provider.
 
 Supported settings:

hack/observability/kube-state-metrics/crd-sidecar-cm.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kube-state-metrics-crd-sidecar-script
+data:
+  compile.sh: |
+    #!/bin/sh
+    set -x
+
+    SOURCE_DIR="/tmp"
+    TARGET_FILE="/etc/config/crd-metrics-config.yaml"
+    # This script will use a temporary file to only overwrite the target file once.
+    TARGET_FILE_TMP="${TARGET_FILE}.tmp"
+    # Create header
+    cat << EOF > "${TARGET_FILE_TMP}"
+    kind: CustomResourceStateMetrics
+    spec:
+      resources:
+    EOF
+    # Append custom resource config of all files but remove headers
+    for f in $(ls -1 ${SOURCE_DIR}/*.yaml); do
+      cat $f | grep -v -E -e '^(-|kind: CustomResourceStateMetrics|spec:| +resources:)' \
+        >> "${TARGET_FILE_TMP}"
+    done
+    # Overwrite target file
+    mv "${TARGET_FILE_TMP}" "${TARGET_FILE}"

hack/observability/kube-state-metrics/crd-sidecar-patch.yaml

@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+spec:
+  template:
+    spec:
+      containers:
+      - env:
+        - name: LABEL
+          value: kube-state-metrics/custom-resource
+        - name: FOLDER
+          value: /tmp
+        - name: RESOURCE
+          value: configmap
+        - name: NAMESPACE
+          value: observability
+        - name: SCRIPT
+          value: /script/compile.sh
+        # This image continuously collects config maps with the specified label and
+        # updates the configuration for kube-state-metrics using a script.
+        image: kiwigrid/k8s-sidecar:latest
+        name: crd-sidecar
+        volumeMounts:
+        - mountPath: /etc/config
+          name: config-volume
+        - mountPath: /script
+          name: compile-script
+      initContainers:
+      - command:
+        - /bin/sh
+        - -c
+        - |
+          cat << EOF > "/etc/config/crd-metrics-config.yaml"
+          kind: CustomResourceStateMetrics
+          spec:
+            resources: []
+          EOF
+        # This container initializes an empty configuration for kube-state-metrics
+        # to have a smooth start.
+        image: kiwigrid/k8s-sidecar:latest
+        name: init-crd-config
+        volumeMounts:
+        - mountPath: /etc/config
+          name: config-volume
+      volumes:
+      - configMap:
+          defaultMode: 511
+          name: kube-state-metrics-crd-sidecar-script
+        name: compile-script

hack/observability/kube-state-metrics/kustomization.yaml

@@ -1,15 +1,30 @@
 resources:
   - ../namespace.yaml
-  # The kube-state-metrics helm chart will reference a configmap with name `kube-state-metrics-crd-config`.
-  # The configMapGenerator below will create the configmap and append a hash suffix calculated from its
-  # content to the name. Kustomize will append the suffix hash to all references in the helm chart, but
-  # only when the helm chart content is referenced in "resources".
-  # This would not work if the helm chart is configured in this file via the "helmCharts" option.
-  - ./chart
+  # Aggregated ClusterRole and ClusterRoleBinding for kube-state-metrics crd resources.
+  - rbac-crd-aggregation.yaml
+  # Adds the configmap which contains the script to build the final crd configuration.
+  - crd-sidecar-cm.yaml
+  # Adds the core CAPI crd metrics configmap and CusterRole.
+  - ../../../config/metrics
 
 namespace: observability
 
-configMapGenerator:
-- name: kube-state-metrics-crd-config
-  files:
-  - crd-config.yaml
+helmCharts:
+  - name: kube-state-metrics
+    repo: https://prometheus-community.github.io/helm-charts
+    namespace: observability
+    releaseName: kube-state-metrics
+    valuesFile: values.yaml
+    version: 5.12.1
+
+helmGlobals:
+    # Store chart in ".charts" folder instead of "charts".
+    # Otherwise "go mod tidy" picks up dependencies of go files contained in the Helm Chart.
+    # "go mod tidy" ignores folders that begin with ".": https://pkg.go.dev/cmd/go#hdr-Package_lists_and_patterns.
+    chartHome: .charts
+
+patches:
+# Adds the sidecar container which assembles the final configuration for kube-state-metrics
+- path: crd-sidecar-patch.yaml
+  target:
+    kind: Deployment

hack/observability/kube-state-metrics/rbac-crd-aggregation.yaml

@@ -0,0 +1,24 @@
+---
+# ClusterRole to aggregate other ClusterRoles for different Custom Resource Configurations
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics-aggregated-role
+aggregationRule:
+  clusterRoleSelectors:
+  - matchLabels:
+      kube-state-metrics/aggregate-to-manager: "true"
+---
+# ClusterRoleBinding for the aggregation role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics-custom-resource-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics-aggregated-role
+subjects:
+- kind: ServiceAccount
+  name: kube-state-metrics
+  namespace: observability

hack/observability/kube-state-metrics/values.yaml

@@ -0,0 +1,22 @@
+# Add the CR configuration from the config map.
+volumeMounts:
+- mountPath: /etc/config
+  name: config-volume
+
+volumes:
+- emptyDir: {}
+  name: config-volume
+
+extraArgs:
+- "--custom-resource-state-config-file=/etc/config/crd-metrics-config.yaml"
+
+rbac:
+  extraRules:
+  - apiGroups:
+    - apiextensions.k8s.io
+    resources:
+    - customresourcedefinitions
+    verbs:
+    - get
+    - list
+    - watch

hack/tools/internal/tilt-prepare/main.go

@@ -104,13 +104,14 @@ var (
 // Types used to de-serialize the tilt-settings.yaml/json file from the Cluster API repository.
 
 type tiltSettings struct {
-	Debug               map[string]tiltSettingsDebugConfig `json:"debug,omitempty"`
-	ExtraArgs           map[string]tiltSettingsExtraArgs   `json:"extra_args,omitempty"`
-	DeployCertManager   *bool                              `json:"deploy_cert_manager,omitempty"`
-	DeployObservability []string                           `json:"deploy_observability,omitempty"`
-	EnableProviders     []string                           `json:"enable_providers,omitempty"`
-	AllowedContexts     []string                           `json:"allowed_contexts,omitempty"`
-	ProviderRepos       []string                           `json:"provider_repos,omitempty"`
+	Debug                map[string]tiltSettingsDebugConfig `json:"debug,omitempty"`
+	ExtraArgs            map[string]tiltSettingsExtraArgs   `json:"extra_args,omitempty"`
+	DeployCertManager    *bool                              `json:"deploy_cert_manager,omitempty"`
+	DeployObservability  []string                           `json:"deploy_observability,omitempty"`
+	DeployKustomizations map[string]string                  `json:"deploy_kustomizations,omitempty"`
+	EnableProviders      []string                           `json:"enable_providers,omitempty"`
+	AllowedContexts      []string                           `json:"allowed_contexts,omitempty"`
+	ProviderRepos        []string                           `json:"provider_repos,omitempty"`
 }
 
 type tiltSettingsDebugConfig struct {
@@ -308,6 +309,13 @@ func tiltResources(ctx context.Context, ts *tiltSettings) error {
 		)
 	}
 
+	for name, path := range ts.DeployKustomizations {
+		name := fmt.Sprintf("%s.kustomization", name)
+		tasks[name] = sequential(
+			kustomizeTask(path, fmt.Sprintf("%s.yaml", name)),
+		)
+	}
+
 	// Add read configurations from provider repos
 	for _, p := range ts.ProviderRepos {
 		tiltProviderConfigs, err := loadTiltProvider(p)