From 4b4df56135a4215b36a8ab44f49228cc1a021dba Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Mon, 11 Sep 2023 09:23:53 +0200 Subject: [PATCH] observability: move metrics to config and use sidecar in kube-state-metrics --- Makefile | 14 ++--- Tiltfile | 15 ++++++ config/metrics/crd-clusterrole.yaml | 38 +++++++++++++ .../metrics/crd-metrics-config.yaml | 0 config/metrics/kustomization.yaml | 13 +++++ .../metrics/templates}/README.md | 2 +- .../metrics/templates}/cluster.yaml | 0 .../metrics/templates}/clusterclass.yaml | 0 .../metrics/templates}/common_metrics.yaml | 0 .../metrics/templates}/header.yaml | 0 .../metrics/templates}/kubeadmconfig.yaml | 0 .../templates}/kubeadmcontrolplane.yaml | 0 .../metrics/templates}/machine.yaml | 0 .../metrics/templates}/machinedeployment.yaml | 0 .../templates}/machinehealthcheck.yaml | 0 .../metrics/templates}/machinepool.yaml | 0 .../metrics/templates}/machineset.yaml | 0 .../metrics/templates}/owner_metric.yaml | 0 docs/book/src/developer/tilt.md | 8 +++ .../chart/kustomization.yaml | 13 ----- .../kube-state-metrics/chart/values.yaml | 53 ------------------- .../kube-state-metrics/crd-sidecar-cm.yaml | 26 +++++++++ .../kube-state-metrics/crd-sidecar-patch.yaml | 50 +++++++++++++++++ .../kube-state-metrics/kustomization.yaml | 35 ++++++++---- .../rbac-crd-aggregation.yaml | 24 +++++++++ .../kube-state-metrics/values.yaml | 22 ++++++++ hack/tools/internal/tilt-prepare/main.go | 22 +++++--- 27 files changed, 244 insertions(+), 91 deletions(-) create mode 100644 config/metrics/crd-clusterrole.yaml rename hack/observability/kube-state-metrics/crd-config.yaml => config/metrics/crd-metrics-config.yaml (100%) create mode 100644 config/metrics/kustomization.yaml rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/README.md (74%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/cluster.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/clusterclass.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/common_metrics.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/header.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/kubeadmconfig.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/kubeadmcontrolplane.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/machine.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/machinedeployment.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/machinehealthcheck.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/machinepool.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/machineset.yaml (100%) rename {hack/observability/kube-state-metrics/metrics => config/metrics/templates}/owner_metric.yaml (100%) delete mode 100644 hack/observability/kube-state-metrics/chart/kustomization.yaml delete mode 100644 hack/observability/kube-state-metrics/chart/values.yaml create mode 100644 hack/observability/kube-state-metrics/crd-sidecar-cm.yaml create mode 100644 hack/observability/kube-state-metrics/crd-sidecar-patch.yaml create mode 100644 hack/observability/kube-state-metrics/rbac-crd-aggregation.yaml create mode 100644 hack/observability/kube-state-metrics/values.yaml diff --git a/Makefile b/Makefile index 2bd6cefb9b55..8b5d4882fc00 100644 --- a/Makefile +++ b/Makefile @@ -549,16 +549,16 @@ generate-e2e-templates-main: $(KUSTOMIZE) $(KUSTOMIZE) build $(INMEMORY_TEMPLATES)/main/cluster-template --load-restrictor LoadRestrictionsNone > $(INMEMORY_TEMPLATES)/main/cluster-template.yaml .PHONY: generate-metrics-config -generate-metrics-config: $(ENVSUBST_BIN) ## Generate ./hack/observability/kube-state-metrics/crd-config.yaml - OUTPUT_FILE="${OBSERVABILITY_DIR}/kube-state-metrics/crd-config.yaml"; \ - METRICS_DIR="${OBSERVABILITY_DIR}/kube-state-metrics/metrics"; \ +generate-metrics-config: $(ENVSUBST_BIN) ## Generate ./config/metrics/crd-metrics-config.yaml + OUTPUT_FILE="./config/metrics/crd-metrics-config.yaml"; \ + METRIC_TEMPLATES_DIR="./config/metrics/templates"; \ echo "# This file was auto-generated via: make generate-metrics-config" > "$${OUTPUT_FILE}"; \ - cat "$${METRICS_DIR}/header.yaml" >> "$${OUTPUT_FILE}"; \ + cat "$${METRIC_TEMPLATES_DIR}/header.yaml" >> "$${OUTPUT_FILE}"; \ for resource in clusterclass cluster kubeadmcontrolplane kubeadmconfig machine machinedeployment machinehealthcheck machineset machinepool; do \ - cat "$${METRICS_DIR}/$${resource}.yaml"; \ - RESOURCE="$${resource}" ${ENVSUBST_BIN} < "$${METRICS_DIR}/common_metrics.yaml"; \ + cat "$${METRIC_TEMPLATES_DIR}/$${resource}.yaml"; \ + RESOURCE="$${resource}" ${ENVSUBST_BIN} < "$${METRIC_TEMPLATES_DIR}/common_metrics.yaml"; \ if [[ "$${resource}" != "cluster" ]]; then \ - cat "$${METRICS_DIR}/owner_metric.yaml"; \ + cat "$${METRIC_TEMPLATES_DIR}/owner_metric.yaml"; \ fi \ done >> "$${OUTPUT_FILE}"; \ diff --git a/Tiltfile b/Tiltfile index 65e60870ac6a..d2090f09d988 100644 --- a/Tiltfile +++ b/Tiltfile @@ -482,6 +482,19 @@ def deploy_observability(): objects = ["capi-visualizer:serviceaccount"], ) +def deploy_kustomizations(): + for name in settings.get("deploy_kustomizations", []): + yaml = read_file("./.tiltbuild/yaml/{}.kustomization.yaml".format(name)) + k8s_yaml(yaml) + objs = decode_yaml_stream(yaml) + print("objects") + print(find_all_objects_names(objs)) + k8s_resource( + new_name = name, + objects = find_all_objects_names(objs), + labels = ["kustomization"], + ) + def prepare_all(): tools_arg = "--tools kustomize,envsubst,clusterctl " tilt_settings_file_arg = "--tilt-settings-file " + tilt_file @@ -640,6 +653,8 @@ deploy_provider_crds() deploy_observability() +deploy_kustomizations() + enable_providers() cluster_templates() diff --git a/config/metrics/crd-clusterrole.yaml b/config/metrics/crd-clusterrole.yaml new file mode 100644 index 000000000000..bcd5bde16ba3 --- /dev/null +++ b/config/metrics/crd-clusterrole.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-custom-resource-capi + labels: + kube-state-metrics/aggregate-to-manager: "true" +rules: +- apiGroups: + - cluster.x-k8s.io + resources: + - clusterclasses + - clusters + - machinedeployments + - machinepools + - machinesets + - machines + - machinehealthchecks + verbs: + - get + - list + - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - kubeadmcontrolplanes + verbs: + - get + - list + - watch +- apiGroups: + - bootstrap.cluster.x-k8s.io + resources: + - kubeadmconfigs + verbs: + - get + - list + - watch diff --git a/hack/observability/kube-state-metrics/crd-config.yaml b/config/metrics/crd-metrics-config.yaml similarity index 100% rename from hack/observability/kube-state-metrics/crd-config.yaml rename to config/metrics/crd-metrics-config.yaml diff --git a/config/metrics/kustomization.yaml b/config/metrics/kustomization.yaml new file mode 100644 index 000000000000..3955c8d0d945 --- /dev/null +++ b/config/metrics/kustomization.yaml @@ -0,0 +1,13 @@ +resources: + - ./crd-clusterrole.yaml + +namespace: observability + +configMapGenerator: +- name: kube-state-metrics-crd-config-capi + files: + - capi.yaml=crd-metrics-config.yaml + options: + disableNameSuffixHash: true + labels: + kube-state-metrics/custom-resource: "true" diff --git a/hack/observability/kube-state-metrics/metrics/README.md b/config/metrics/templates/README.md similarity index 74% rename from hack/observability/kube-state-metrics/metrics/README.md rename to config/metrics/templates/README.md index 753ba822f79a..c52a76817f1f 100644 --- a/hack/observability/kube-state-metrics/metrics/README.md +++ b/config/metrics/templates/README.md @@ -4,4 +4,4 @@ The make target `generate-metrics-config` is used to generate a single file which contains the Cluster API specific custom resource configuration for kube-state-metrics. -To regenerate the file `../crd-config.yaml`, execute the `make generate-metrics-config` command. +To regenerate the file `../crd-metrics-config.yaml`, execute the `make generate-metrics-config` command. diff --git a/hack/observability/kube-state-metrics/metrics/cluster.yaml b/config/metrics/templates/cluster.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/cluster.yaml rename to config/metrics/templates/cluster.yaml diff --git a/hack/observability/kube-state-metrics/metrics/clusterclass.yaml b/config/metrics/templates/clusterclass.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/clusterclass.yaml rename to config/metrics/templates/clusterclass.yaml diff --git a/hack/observability/kube-state-metrics/metrics/common_metrics.yaml b/config/metrics/templates/common_metrics.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/common_metrics.yaml rename to config/metrics/templates/common_metrics.yaml diff --git a/hack/observability/kube-state-metrics/metrics/header.yaml b/config/metrics/templates/header.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/header.yaml rename to config/metrics/templates/header.yaml diff --git a/hack/observability/kube-state-metrics/metrics/kubeadmconfig.yaml b/config/metrics/templates/kubeadmconfig.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/kubeadmconfig.yaml rename to config/metrics/templates/kubeadmconfig.yaml diff --git a/hack/observability/kube-state-metrics/metrics/kubeadmcontrolplane.yaml b/config/metrics/templates/kubeadmcontrolplane.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/kubeadmcontrolplane.yaml rename to config/metrics/templates/kubeadmcontrolplane.yaml diff --git a/hack/observability/kube-state-metrics/metrics/machine.yaml b/config/metrics/templates/machine.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/machine.yaml rename to config/metrics/templates/machine.yaml diff --git a/hack/observability/kube-state-metrics/metrics/machinedeployment.yaml b/config/metrics/templates/machinedeployment.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/machinedeployment.yaml rename to config/metrics/templates/machinedeployment.yaml diff --git a/hack/observability/kube-state-metrics/metrics/machinehealthcheck.yaml b/config/metrics/templates/machinehealthcheck.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/machinehealthcheck.yaml rename to config/metrics/templates/machinehealthcheck.yaml diff --git a/hack/observability/kube-state-metrics/metrics/machinepool.yaml b/config/metrics/templates/machinepool.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/machinepool.yaml rename to config/metrics/templates/machinepool.yaml diff --git a/hack/observability/kube-state-metrics/metrics/machineset.yaml b/config/metrics/templates/machineset.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/machineset.yaml rename to config/metrics/templates/machineset.yaml diff --git a/hack/observability/kube-state-metrics/metrics/owner_metric.yaml b/config/metrics/templates/owner_metric.yaml similarity index 100% rename from hack/observability/kube-state-metrics/metrics/owner_metric.yaml rename to config/metrics/templates/owner_metric.yaml diff --git a/docs/book/src/developer/tilt.md b/docs/book/src/developer/tilt.md index 958fcce2c25b..40d1262705cc 100644 --- a/docs/book/src/developer/tilt.md +++ b/docs/book/src/developer/tilt.md @@ -214,6 +214,14 @@ Supported values are: \*: Note: the UI will be accessible via a link in the tilt console +**deploy_kustomizations** (map[string]string, default={}): If set, installs the additional kustomizations to the cluster. + +Example: +```yaml +deploy_kustomizations: + capv-metrics: ../cluster-api-provider-vsphere/config/metrics +``` + **debug** (Map{string: Map} default{}): A map of named configurations for the provider. The key is the name of the provider. Supported settings: diff --git a/hack/observability/kube-state-metrics/chart/kustomization.yaml b/hack/observability/kube-state-metrics/chart/kustomization.yaml deleted file mode 100644 index 3555c80e7e16..000000000000 --- a/hack/observability/kube-state-metrics/chart/kustomization.yaml +++ /dev/null @@ -1,13 +0,0 @@ -helmCharts: - - name: kube-state-metrics - repo: https://prometheus-community.github.io/helm-charts - namespace: observability - releaseName: kube-state-metrics - valuesFile: values.yaml - version: 5.12.1 - -helmGlobals: - # Store chart in ".charts" folder instead of "charts". - # Otherwise "go mod tidy" picks up dependencies of go files contained in the Helm Chart. - # "go mod tidy" ignores folders that begin with ".": https://pkg.go.dev/cmd/go#hdr-Package_lists_and_patterns. - chartHome: .charts diff --git a/hack/observability/kube-state-metrics/chart/values.yaml b/hack/observability/kube-state-metrics/chart/values.yaml deleted file mode 100644 index 7413a56439bd..000000000000 --- a/hack/observability/kube-state-metrics/chart/values.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Add the CR configuration from the config map. -volumeMounts: - - mountPath: /etc/config - name: config-volume - -volumes: - - configMap: - name: kube-state-metrics-crd-config - name: config-volume - -extraArgs: -- "--custom-resource-state-config-file=/etc/config/crd-config.yaml" - -rbac: - extraRules: - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch - - apiGroups: - - cluster.x-k8s.io - resources: - - clusterclasses - - clusters - - machinedeployments - - machinepools - - machinesets - - machines - - machinehealthchecks - verbs: - - get - - list - - watch - - apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - kubeadmcontrolplanes - verbs: - - get - - list - - watch - - apiGroups: - - bootstrap.cluster.x-k8s.io - resources: - - kubeadmconfigs - verbs: - - get - - list - - watch diff --git a/hack/observability/kube-state-metrics/crd-sidecar-cm.yaml b/hack/observability/kube-state-metrics/crd-sidecar-cm.yaml new file mode 100644 index 000000000000..588ddabe090c --- /dev/null +++ b/hack/observability/kube-state-metrics/crd-sidecar-cm.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-state-metrics-crd-sidecar-script +data: + compile.sh: | + #!/bin/sh + set -x + + SOURCE_DIR="/tmp" + TARGET_FILE="/etc/config/crd-metrics-config.yaml" + # This script will use a temporary file to only overwrite the target file once. + TARGET_FILE_TMP="${TARGET_FILE}.tmp" + # Create header + cat << EOF > "${TARGET_FILE_TMP}" + kind: CustomResourceStateMetrics + spec: + resources: + EOF + # Append custom resource config of all files but remove headers + for f in $(ls -1 ${SOURCE_DIR}/*.yaml); do + cat $f | grep -v -E -e '^(-|kind: CustomResourceStateMetrics|spec:| +resources:)' \ + >> "${TARGET_FILE_TMP}" + done + # Overwrite target file + mv "${TARGET_FILE_TMP}" "${TARGET_FILE}" diff --git a/hack/observability/kube-state-metrics/crd-sidecar-patch.yaml b/hack/observability/kube-state-metrics/crd-sidecar-patch.yaml new file mode 100644 index 000000000000..3690e92f42a3 --- /dev/null +++ b/hack/observability/kube-state-metrics/crd-sidecar-patch.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-state-metrics +spec: + template: + spec: + containers: + - env: + - name: LABEL + value: kube-state-metrics/custom-resource + - name: FOLDER + value: /tmp + - name: RESOURCE + value: configmap + - name: NAMESPACE + value: observability + - name: SCRIPT + value: /script/compile.sh + # This image continuously collects config maps with the specified label and + # updates the configuration for kube-state-metrics using a script. + image: kiwigrid/k8s-sidecar:latest + name: crd-sidecar + volumeMounts: + - mountPath: /etc/config + name: config-volume + - mountPath: /script + name: compile-script + initContainers: + - command: + - /bin/sh + - -c + - | + cat << EOF > "/etc/config/crd-metrics-config.yaml" + kind: CustomResourceStateMetrics + spec: + resources: [] + EOF + # This container initializes an empty configuration for kube-state-metrics + # to have a smooth start. + image: kiwigrid/k8s-sidecar:latest + name: init-crd-config + volumeMounts: + - mountPath: /etc/config + name: config-volume + volumes: + - configMap: + defaultMode: 511 + name: kube-state-metrics-crd-sidecar-script + name: compile-script \ No newline at end of file diff --git a/hack/observability/kube-state-metrics/kustomization.yaml b/hack/observability/kube-state-metrics/kustomization.yaml index cdcfcee5dc5d..e44d34102986 100644 --- a/hack/observability/kube-state-metrics/kustomization.yaml +++ b/hack/observability/kube-state-metrics/kustomization.yaml @@ -1,15 +1,30 @@ resources: - ../namespace.yaml - # The kube-state-metrics helm chart will reference a configmap with name `kube-state-metrics-crd-config`. - # The configMapGenerator below will create the configmap and append a hash suffix calculated from its - # content to the name. Kustomize will append the suffix hash to all references in the helm chart, but - # only when the helm chart content is referenced in "resources". - # This would not work if the helm chart is configured in this file via the "helmCharts" option. - - ./chart + # Aggregated ClusterRole and ClusterRoleBinding for kube-state-metrics crd resources. + - rbac-crd-aggregation.yaml + # Adds the configmap which contains the script to build the final crd configuration. + - crd-sidecar-cm.yaml + # Adds the core CAPI crd metrics configmap and CusterRole. + - ../../../config/metrics namespace: observability -configMapGenerator: -- name: kube-state-metrics-crd-config - files: - - crd-config.yaml +helmCharts: + - name: kube-state-metrics + repo: https://prometheus-community.github.io/helm-charts + namespace: observability + releaseName: kube-state-metrics + valuesFile: values.yaml + version: 5.12.1 + +helmGlobals: + # Store chart in ".charts" folder instead of "charts". + # Otherwise "go mod tidy" picks up dependencies of go files contained in the Helm Chart. + # "go mod tidy" ignores folders that begin with ".": https://pkg.go.dev/cmd/go#hdr-Package_lists_and_patterns. + chartHome: .charts + +patches: +# Adds the sidecar container which assembles the final configuration for kube-state-metrics +- path: crd-sidecar-patch.yaml + target: + kind: Deployment diff --git a/hack/observability/kube-state-metrics/rbac-crd-aggregation.yaml b/hack/observability/kube-state-metrics/rbac-crd-aggregation.yaml new file mode 100644 index 000000000000..a7c08eb2480b --- /dev/null +++ b/hack/observability/kube-state-metrics/rbac-crd-aggregation.yaml @@ -0,0 +1,24 @@ +--- +# ClusterRole to aggregate other ClusterRoles for different Custom Resource Configurations +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-aggregated-role +aggregationRule: + clusterRoleSelectors: + - matchLabels: + kube-state-metrics/aggregate-to-manager: "true" +--- +# ClusterRoleBinding for the aggregation role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-state-metrics-custom-resource-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics-aggregated-role +subjects: +- kind: ServiceAccount + name: kube-state-metrics + namespace: observability diff --git a/hack/observability/kube-state-metrics/values.yaml b/hack/observability/kube-state-metrics/values.yaml new file mode 100644 index 000000000000..b9dbfa5085fd --- /dev/null +++ b/hack/observability/kube-state-metrics/values.yaml @@ -0,0 +1,22 @@ +# Add the CR configuration from the config map. +volumeMounts: +- mountPath: /etc/config + name: config-volume + +volumes: +- emptyDir: {} + name: config-volume + +extraArgs: +- "--custom-resource-state-config-file=/etc/config/crd-metrics-config.yaml" + +rbac: + extraRules: + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch diff --git a/hack/tools/internal/tilt-prepare/main.go b/hack/tools/internal/tilt-prepare/main.go index 5e7f4f7e6ba9..ddd41ee9531a 100644 --- a/hack/tools/internal/tilt-prepare/main.go +++ b/hack/tools/internal/tilt-prepare/main.go @@ -104,13 +104,14 @@ var ( // Types used to de-serialize the tilt-settings.yaml/json file from the Cluster API repository. type tiltSettings struct { - Debug map[string]tiltSettingsDebugConfig `json:"debug,omitempty"` - ExtraArgs map[string]tiltSettingsExtraArgs `json:"extra_args,omitempty"` - DeployCertManager *bool `json:"deploy_cert_manager,omitempty"` - DeployObservability []string `json:"deploy_observability,omitempty"` - EnableProviders []string `json:"enable_providers,omitempty"` - AllowedContexts []string `json:"allowed_contexts,omitempty"` - ProviderRepos []string `json:"provider_repos,omitempty"` + Debug map[string]tiltSettingsDebugConfig `json:"debug,omitempty"` + ExtraArgs map[string]tiltSettingsExtraArgs `json:"extra_args,omitempty"` + DeployCertManager *bool `json:"deploy_cert_manager,omitempty"` + DeployObservability []string `json:"deploy_observability,omitempty"` + DeployKustomizations map[string]string `json:"deploy_kustomizations,omitempty"` + EnableProviders []string `json:"enable_providers,omitempty"` + AllowedContexts []string `json:"allowed_contexts,omitempty"` + ProviderRepos []string `json:"provider_repos,omitempty"` } type tiltSettingsDebugConfig struct { @@ -308,6 +309,13 @@ func tiltResources(ctx context.Context, ts *tiltSettings) error { ) } + for name, path := range ts.DeployKustomizations { + name := fmt.Sprintf("%s.kustomization", name) + tasks[name] = sequential( + kustomizeTask(path, fmt.Sprintf("%s.yaml", name)), + ) + } + // Add read configurations from provider repos for _, p := range ts.ProviderRepos { tiltProviderConfigs, err := loadTiltProvider(p)