From a43f40b6376922b3d7fabe45edfa604e4fd23e85 Mon Sep 17 00:00:00 2001 From: killianmuldoon Date: Mon, 25 Sep 2023 16:53:58 +0100 Subject: [PATCH] Add debug logging patch to ClusterClass Signed-off-by: killianmuldoon --- .../cluster.yaml | 4 + .../cluster.yaml | 4 + .../main/clusterclass-quick-start.yaml | 115 +++++++++++++++--- .../v1.5/clusterclass-quick-start.yaml | 98 +++++++++++++-- 4 files changed, 192 insertions(+), 29 deletions(-) diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary/cluster.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary/cluster.yaml index 8193123cde52..4becb1d1d900 100644 --- a/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary/cluster.yaml +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary/cluster.yaml @@ -11,6 +11,10 @@ spec: value: false - name: externalCloudProvider value: true + - name: kubeControlPlaneLogLevel + value: "2" + - name: kubeletLogLevel + value: "4" clusterNetwork: services: cidrBlocks: diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary/cluster.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary/cluster.yaml index 4ff80d16ccbc..d43dc35b4c8a 100644 --- a/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary/cluster.yaml +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary/cluster.yaml @@ -11,6 +11,10 @@ spec: value: true - name: externalCloudProvider value: true + - name: kubeControlPlaneLogLevel + value: "2" + - name: kubeletLogLevel + value: "4" clusterNetwork: services: cidrBlocks: diff --git a/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml index 770fad20c4c3..6c9d2326f997 100644 --- a/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml +++ b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml @@ -130,6 +130,18 @@ spec: openAPIV3Schema: type: boolean default: false + - name: kubeControlPlaneLogLevel + schema: + openAPIV3Schema: + type: string + description: "Log level for kube-apiserver, kube-scheduler and kube-controller-manager" + example: "2" + - name: kubeletLogLevel + schema: + openAPIV3Schema: + type: string + description: "Log level for kubelets on control plane and worker nodes" + example: "2" patches: - name: lbImageRepository definitions: @@ -290,13 +302,11 @@ spec: controlPlane: true jsonPatches: - op: add - path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs" - value: - cloud-provider: "external" + path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/cloud-provider" + value: "external" - op: add - path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs" - value: - cloud-provider: "external" + path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/cloud-provider" + value: "external" - name: machineDeploymentExternalCloudProvider enabledIf: "{{ .externalCloudProvider }}" description: "Configures kubelet to run with an external cloud provider for machineDeployment nodes." @@ -310,9 +320,8 @@ spec: - '*-worker' jsonPatches: - op: add - path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs" - value: - cloud-provider: "external" + path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/cloud-provider" + value: "external" - selector: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate @@ -322,9 +331,8 @@ spec: - '*-worker' jsonPatches: - op: add - path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs" - value: - cloud-provider: "external" + path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/cloud-provider" + value: "external" - name: localEndpointIPv6 enabledIf: "{{ .ipv6Primary }}" description: "Configures KCP to use IPv6 for its localAPIEndpoint." @@ -349,9 +357,8 @@ spec: controlPlane: true jsonPatches: - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs" - value: - admission-control-config-file: "/etc/kubernetes/kube-apiserver-admission-pss.yaml" + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/admission-control-config-file" + value: "/etc/kubernetes/kube-apiserver-admission-pss.yaml" - op: add path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraVolumes" value: @@ -385,6 +392,74 @@ spec: namespaces: [kube-system] path: /etc/kubernetes/kube-apiserver-admission-pss.yaml enabledIf: '{{ semverCompare ">= v1.24" .builtin.controlPlane.version }}' + - name: controlPlaneLogLevel + enabledIf: "{{ if .kubeControlPlaneLogLevel }}true{{end}}" + description: "Configures control plane components and kubelet to log at v=4." + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/v" + valueFrom: + variable: kubeControlPlaneLogLevel + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/controllerManager/extraArgs/v" + valueFrom: + variable: kubeControlPlaneLogLevel + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/scheduler/extraArgs/v" + valueFrom: + variable: kubeControlPlaneLogLevel + - name: controlPlaneKubeletLogLevel + enabledIf: "{{ if .kubeletLogLevel }}true{{end}}" + description: "Configures worker kubelets to log at v=4." + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/v" + valueFrom: + variable: kubeletLogLevel + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/v" + valueFrom: + variable: kubeletLogLevel + - name: workerKubeletLogLevel + enabledIf: "{{ if .kubeletLogLevel }}true{{end}}" + description: "Configures worker kubelets to log at v=4." + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - '*-worker' + jsonPatches: + - op: add + path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/v" + valueFrom: + variable: kubeletLogLevel + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machinePoolClass: + names: + - '*-worker' + jsonPatches: + - op: add + path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/v" + valueFrom: + variable: kubeletLogLevel --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerClusterTemplate @@ -447,9 +522,17 @@ spec: nodeDrainTimeout: 1s kubeadmConfigSpec: clusterConfiguration: + # extraArgs must be non-empty for control plane components to enable patches from ClusterClass to work. controllerManager: - extraArgs: { enable-hostpath-provisioner: 'true' } + extraArgs: + enable-hostpath-provisioner: 'true' + v: "0" + scheduler: + extraArgs: + v: "0" apiServer: + extraArgs: + v: "0" # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied. certSANs: [localhost, host.docker.internal, "::", "::1", "127.0.0.1", "0.0.0.0"] initConfiguration: diff --git a/test/e2e/data/infrastructure-docker/v1.5/clusterclass-quick-start.yaml b/test/e2e/data/infrastructure-docker/v1.5/clusterclass-quick-start.yaml index 3d747dd49625..605d96d83f13 100644 --- a/test/e2e/data/infrastructure-docker/v1.5/clusterclass-quick-start.yaml +++ b/test/e2e/data/infrastructure-docker/v1.5/clusterclass-quick-start.yaml @@ -112,6 +112,18 @@ spec: openAPIV3Schema: type: boolean default: false + - name: kubeControlPlaneLogLevel + schema: + openAPIV3Schema: + type: string + description: "Log level for kube-apiserver, kube-scheduler and kube-controller-manager" + example: "2" + - name: kubeletLogLevel + schema: + openAPIV3Schema: + type: string + description: "Log level for kubelets on control plane and worker nodes" + example: "2" patches: - name: lbImageRepository definitions: @@ -241,13 +253,11 @@ spec: controlPlane: true jsonPatches: - op: add - path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs" - value: - cloud-provider: "external" + path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/cloud-provider" + value: "external" - op: add - path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs" - value: - cloud-provider: "external" + path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/cloud-provider" + value: "external" - name: machineDeploymentExternalCloudProvider enabledIf: "{{ .externalCloudProvider }}" description: "Configures kubelet to run with an external cloud provider for machineDeployment nodes." @@ -261,9 +271,8 @@ spec: - '*-worker' jsonPatches: - op: add - path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs" - value: - cloud-provider: "external" + path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/cloud-provider" + value: "external" - name: localEndpointIPv6 enabledIf: "{{ .ipv6Primary }}" description: "Configures KCP to use IPv6 for its localAPIEndpoint." @@ -288,9 +297,8 @@ spec: controlPlane: true jsonPatches: - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs" - value: - admission-control-config-file: "/etc/kubernetes/kube-apiserver-admission-pss.yaml" + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/admission-control-config-file" + value: "/etc/kubernetes/kube-apiserver-admission-pss.yaml" - op: add path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraVolumes" value: @@ -324,6 +332,62 @@ spec: namespaces: [kube-system] path: /etc/kubernetes/kube-apiserver-admission-pss.yaml enabledIf: '{{ semverCompare ">= v1.24" .builtin.controlPlane.version }}' + - name: controlPlaneLogLevel + enabledIf: "{{ if .kubeControlPlaneLogLevel }}true{{end}}" + description: "Configures control plane components and kubelet to log at v=4." + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/v" + valueFrom: + variable: kubeControlPlaneLogLevel + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/controllerManager/extraArgs/v" + valueFrom: + variable: kubeControlPlaneLogLevel + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/scheduler/extraArgs/v" + valueFrom: + variable: kubeControlPlaneLogLevel + - name: controlPlaneKubeletLogLevel + enabledIf: "{{ if .kubeletLogLevel }}true{{end}}" + description: "Configures worker kubelets to log at v=4." + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/v" + valueFrom: + variable: kubeletLogLevel + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/v" + valueFrom: + variable: kubeletLogLevel + - name: workerKubeletLogLevel + enabledIf: "{{ if .kubeletLogLevel }}true{{end}}" + description: "Configures worker kubelets to log at v=4." + definitions: + - selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - '*-worker' + jsonPatches: + - op: add + path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/v" + valueFrom: + variable: kubeletLogLevel --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerClusterTemplate @@ -386,9 +450,17 @@ spec: nodeDrainTimeout: 1s kubeadmConfigSpec: clusterConfiguration: + # extraArgs must be non-empty for control plane components to enable patches from ClusterClass to work. controllerManager: - extraArgs: { enable-hostpath-provisioner: 'true' } + extraArgs: + enable-hostpath-provisioner: 'true' + v: "0" + scheduler: + extraArgs: + v: "0" apiServer: + extraArgs: + v: "0" # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied. certSANs: [localhost, host.docker.internal, "::", "::1", "127.0.0.1", "0.0.0.0"] initConfiguration: