add sidecar for kube-state-metrics and read crd config from cm + add …

…option to add kustomizations via tilt

Tiltfile

@@ -1,4 +1,4 @@
-# -*- mode: Python -*-
+Tiltfile# -*- mode: Python -*-
 
 envsubst_cmd = "./hack/tools/bin/envsubst"
 clusterctl_cmd = "./bin/clusterctl"
@@ -482,6 +482,19 @@ def deploy_observability():
             objects = ["capi-visualizer:serviceaccount"],
         )
 
+def deploy_kustomizations():
+    for name in settings.get("deploy_kustomizations", []):
+        yaml = read_file("./.tiltbuild/yaml/{}.kustomization.yaml".format(name))
+        k8s_yaml(yaml)
+        objs = decode_yaml_stream(yaml)
+        print("objects") 
+        print(find_all_objects_names(objs))
+        k8s_resource(
+            new_name = name,
+            objects = find_all_objects_names(objs),
+            labels = ["kustomization"],
+        )
+
 def prepare_all():
     tools_arg = "--tools kustomize,envsubst,clusterctl "
     tilt_settings_file_arg = "--tilt-settings-file " + tilt_file
@@ -640,6 +653,8 @@ deploy_provider_crds()
 
 deploy_observability()
 
+deploy_kustomizations()
+
 enable_providers()
 
 cluster_templates()

hack/observability/kube-state-metrics/chart/cm-crd-sidecar.yaml

@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kube-state-metrics-crd-sidecar-script
+data:
+  compile.sh: |
+    #!/bin/sh
+
+    set -x
+    
+    SOURCE_DIR="/tmp"
+    TARGET_FILE="/etc/config/crd-config.yaml"
+    # This script will use a temporary file to only overwrite the target file once.
+    TARGET_FILE_TMP="${TARGET_FILE}.tmp"
+
+    # Create header
+    cat << EOF > "${TARGET_FILE_TMP}"
+    kind: CustomResourceStateMetrics
+    spec:
+      resources:
+    EOF
+
+    # Append custom resource config of all files but remove headers
+    for f in $(ls -1 ${SOURCE_DIR}/*.yaml); do
+      cat $f | grep -v -E -e '^(-|kind: CustomResourceStateMetrics|spec:| +resources:)' \
+        >> "${TARGET_FILE_TMP}"
+    done
+
+    # Overwrite target file
+    mv "${TARGET_FILE_TMP}" "${TARGET_FILE}"

hack/observability/kube-state-metrics/chart/kustomization.yaml

@@ -11,3 +11,12 @@ helmGlobals:
     # Otherwise "go mod tidy" picks up dependencies of go files contained in the Helm Chart.
     # "go mod tidy" ignores folders that begin with ".": https://pkg.go.dev/cmd/go#hdr-Package_lists_and_patterns.
     chartHome: .charts
+
+resources:
+- rbac-crd-aggregation.yaml
+- cm-crd-sidecar.yaml
+
+patches:
+- path: patch-crd-sidecar.yaml
+  target:
+    kind: Deployment

hack/observability/kube-state-metrics/chart/patch-crd-sidecar.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+spec:
+  template:
+    spec:
+      containers:
+      - env:
+        - name: LABEL
+          value: kube-state-metrics/custom-resource
+        - name: FOLDER
+          value: /tmp
+        - name: RESOURCE
+          value: configmap
+        - name: NAMESPACE
+          value: observability
+        - name: SCRIPT
+          value: /script/compile.sh
+        image: kiwigrid/k8s-sidecar:latest
+        name: crd-sidecar
+        volumeMounts:
+        - mountPath: /etc/config
+          name: config-volume
+        - mountPath: /script
+          name: compile-script
+      initContainers:
+      - command:
+        - /bin/sh
+        - -c
+        - |
+          cat << EOF > "/etc/config/crd-config.yaml"
+          kind: CustomResourceStateMetrics
+          spec:
+            resources: []
+          EOF
+        image: kiwigrid/k8s-sidecar:latest
+        name: init-crd-config
+        volumeMounts:
+        - mountPath: /etc/config
+          name: config-volume
+      volumes:
+      - configMap:
+          defaultMode: 511
+          name: kube-state-metrics-crd-sidecar-script
+        name: compile-script

hack/observability/kube-state-metrics/chart/rbac-crd-aggregation.yaml

@@ -0,0 +1,24 @@
+---
+# ClusterRole to aggregate other ClusterRoles for different Custom Resource Configurations
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics-aggregated-role
+aggregationRule:
+  clusterRoleSelectors:
+  - matchLabels:
+      kube-state-metrics/aggregate-to-manager: "true"
+---
+# ClusterRoleBinding for the aggregation role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics-custom-resource-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics-aggregated-role
+subjects:
+- kind: ServiceAccount
+  name: kube-state-metrics
+  namespace: observability

hack/observability/kube-state-metrics/chart/values.yaml

@@ -4,9 +4,8 @@ volumeMounts:
    name: config-volume
 
 volumes:
- - configMap:
-     name: kube-state-metrics-crd-config
-   name: config-volume
+  - emptyDir: {}
+    name: config-volume
 
 extraArgs:
 - "--custom-resource-state-config-file=/etc/config/crd-config.yaml"
@@ -21,33 +20,3 @@ rbac:
     - get
     - list
     - watch
-  - apiGroups:
-    - cluster.x-k8s.io
-    resources:
-    - clusterclasses
-    - clusters
-    - machinedeployments
-    - machinepools
-    - machinesets
-    - machines
-    - machinehealthchecks
-    verbs:
-    - get
-    - list
-    - watch
-  - apiGroups:
-    - controlplane.cluster.x-k8s.io
-    resources:
-    - kubeadmcontrolplanes
-    verbs:
-    - get
-    - list
-    - watch
-  - apiGroups:
-    - bootstrap.cluster.x-k8s.io
-    resources:
-    - kubeadmconfigs
-    verbs:
-    - get
-    - list
-    - watch

hack/observability/kube-state-metrics/crd-clusterrole.yaml

@@ -0,0 +1,38 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics-custom-resource-capi
+  labels:
+    kube-state-metrics/aggregate-to-manager: "true"
+rules:
+- apiGroups:
+  - cluster.x-k8s.io
+  resources:
+  - clusterclasses
+  - clusters
+  - machinedeployments
+  - machinepools
+  - machinesets
+  - machines
+  - machinehealthchecks
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - controlplane.cluster.x-k8s.io
+  resources:
+  - kubeadmcontrolplanes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - bootstrap.cluster.x-k8s.io
+  resources:
+  - kubeadmconfigs
+  verbs:
+  - get
+  - list
+  - watch

hack/observability/kube-state-metrics/kustomization.yaml

@@ -1,15 +1,15 @@
 resources:
   - ../namespace.yaml
-  # The kube-state-metrics helm chart will reference a configmap with name `kube-state-metrics-crd-config`.
-  # The configMapGenerator below will create the configmap and append a hash suffix calculated from its
-  # content to the name. Kustomize will append the suffix hash to all references in the helm chart, but
-  # only when the helm chart content is referenced in "resources".
-  # This would not work if the helm chart is configured in this file via the "helmCharts" option.
   - ./chart
+  - ./crd-clusterrole.yaml
 
 namespace: observability
 
 configMapGenerator:
-- name: kube-state-metrics-crd-config
+- name: kube-state-metrics-crd-config-capi
   files:
-  - crd-config.yaml
+  - capi.yaml=crd-config.yaml
+  options:
+    disableNameSuffixHash: true
+    labels:
+      kube-state-metrics/custom-resource: "true"

hack/tools/internal/tilt-prepare/main.go

@@ -104,13 +104,14 @@ var (
 // Types used to de-serialize the tilt-settings.yaml/json file from the Cluster API repository.
 
 type tiltSettings struct {
-	Debug               map[string]tiltSettingsDebugConfig `json:"debug,omitempty"`
-	ExtraArgs           map[string]tiltSettingsExtraArgs   `json:"extra_args,omitempty"`
-	DeployCertManager   *bool                              `json:"deploy_cert_manager,omitempty"`
-	DeployObservability []string                           `json:"deploy_observability,omitempty"`
-	EnableProviders     []string                           `json:"enable_providers,omitempty"`
-	AllowedContexts     []string                           `json:"allowed_contexts,omitempty"`
-	ProviderRepos       []string                           `json:"provider_repos,omitempty"`
+	Debug                map[string]tiltSettingsDebugConfig `json:"debug,omitempty"`
+	ExtraArgs            map[string]tiltSettingsExtraArgs   `json:"extra_args,omitempty"`
+	DeployCertManager    *bool                              `json:"deploy_cert_manager,omitempty"`
+	DeployObservability  []string                           `json:"deploy_observability,omitempty"`
+	DeployKustomizations map[string]string                  `json:"deploy_kustomizations,omitempty"`
+	EnableProviders      []string                           `json:"enable_providers,omitempty"`
+	AllowedContexts      []string                           `json:"allowed_contexts,omitempty"`
+	ProviderRepos        []string                           `json:"provider_repos,omitempty"`
 }
 
 type tiltSettingsDebugConfig struct {
@@ -308,6 +309,13 @@ func tiltResources(ctx context.Context, ts *tiltSettings) error {
 		)
 	}
 
+	for name, path := range ts.DeployKustomizations {
+		name := fmt.Sprintf("%s.kustomization", name)
+		tasks[name] = sequential(
+			kustomizeTask(path, fmt.Sprintf("%s.yaml", name)),
+		)
+	}
+
 	// Add read configurations from provider repos
 	for _, p := range ts.ProviderRepos {
 		tiltProviderConfigs, err := loadTiltProvider(p)