Support for Kubernetes v1.23 (gardener#5255)

* Extend docs for 1.23 and allow Kubernetes client creation

* Maintain Kubernetes feature gates

$ ./hack/compare-k8s-feature-gates.sh 1.22 1.23
Feature gates added in 1.23 compared to 1.22:
CPUManagerPolicyAlphaOptions
CPUManagerPolicyBetaOptions
CSIMigrationPortworx
CustomResourceValidationExpressions
DisableKubeletCloudCredentialProviders
GRPCContainerProbe
GracefulNodeShutdownBasedOnPodPriority
HonorPVReclaimPolicy
IdentifyPodOS
InTreePluginPortworxUnregister
InTreePluginRBDUnregister
JobMutableNodeSchedulingDirectives
JobReadyPods
MigrationRBD
OpenAPIEnums
OpenAPIV3
PodAndContainerStatsFromCRI
RecoverVolumeExpansionFailure
ServerSideFieldValidation
StatefulSetAutoDeletePVC

Feature gates removed in 1.23 compared to 1.22:
AllowInsecureBackendProxy
BoundServiceAccountTokenVolume
CronJobControllerV2
NodeLease
ServiceAccountIssuerDiscovery
StartupProbe
SupportNodePidsLimit
SupportPodPidsLimit
Sysctls

* Adapt TLS cipher suites for Kubernetes v1.23

according to gardener#4823

* Use `systemd` as cgroup driver for shoots >= 1.23

Ref gardener#5102 (comment)

* Adapt to deprecated seccomp annotations

* Use v1beta3 version for kube-scheduler config

* Do no longer use ineffective flags for shoots >= 1.23

* Switch to autoscaling/v2 or add TODOs

* Bump `kindest/node` image to `v1.23.1`

* Use 1.23 for local shoot and local e2e test

* Revert "Bump `kindest/node` image to `v1.23.1`"

This reverts commit 7c31fa8.

* Address PR review feedback

* Address PR review feedback

charts/istio/istio-ingress/templates/autoscale.yaml

@@ -16,4 +16,10 @@ spec:
   - type: Resource
     resource:
       name: cpu
+      {{- if semverCompare "<= 1.22-0" .Capabilities.KubeVersion.GitVersion }}
       targetAverageUtilization: 80
+      {{- else }}
+      target:
+        type: Utilization
+        averageUtilization: 80
+      {{- end }}

charts/shoot-addons/charts/kubernetes-dashboard/templates/deployment-metrics-scraper.yaml

@@ -22,7 +22,9 @@ spec:
         k8s-app: dashboard-metrics-scraper
         origin: gardener
       annotations:
+        {{- if semverCompare "<= 1.22.x" .Capabilities.KubeVersion.GitVersion }}
         seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
+        {{- end }}
         # TODO(rfranzke): Remove in a future release.
         security.gardener.cloud/trigger: rollout
     spec:
@@ -47,6 +49,11 @@ spec:
           readOnlyRootFilesystem: true
           runAsUser: 1001
           runAsGroup: 2001
+      {{- if semverCompare "> 1.22.x" .Capabilities.KubeVersion.GitVersion }}
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      {{- end }}
       serviceAccountName: kubernetes-dashboard
       nodeSelector:
         worker.gardener.cloud/system-components: "true"

charts/utils-templates/templates/_versions.tpl

@@ -43,11 +43,15 @@ batch/v1beta1
 {{- end -}}
 
 {{- define "hpaversion" -}}
+{{- if semverCompare "<= 1.22-0" .Capabilities.KubeVersion.GitVersion -}}
 autoscaling/v2beta1
+{{- else -}}
+autoscaling/v2
+{{- end -}}
 {{- end -}}
 
 {{- define "webhookadmissionregistration" -}}
-{{- if semverCompare "<= 1.15.x" .Capabilities.KubeVersion.GitVersion -}}
+{{- if semverCompare "<= 1.15-0" .Capabilities.KubeVersion.GitVersion -}}
 admissionregistration.k8s.io/v1beta1
 {{- else -}}
 admissionregistration.k8s.io/v1

docs/usage/supported_k8s_versions.md

@@ -16,6 +16,6 @@ It also enables the Gardener code base to leverage more advanced Kubernetes feat
 
 ## Shoot cluster versions
 
-Gardener itself is capable of spinning up clusters with Kubernetes versions **`1.15`** up to **`1.22`**.
+Gardener itself is capable of spinning up clusters with Kubernetes versions **`1.15`** up to **`1.23`**.
 However, the concrete versions that can be used for shoot clusters depend on the installed provider extension.
 Consequently, please consult the documentation of your provider extension to see which Kubernetes versions are supported for shoot clusters.

example/provider-local/base/cloudprofile.yaml

@@ -8,6 +8,7 @@ spec:
   - name: local
   kubernetes:
     versions:
+    - version: 1.23.1
     - version: 1.22.0
     - version: 1.21.0
     - version: 1.20.0

example/provider-local/shoot.yaml

@@ -25,4 +25,4 @@ spec:
       maxSurge: 1
       maxUnavailable: 0
   kubernetes:
-    version: 1.21.0
+    version: 1.23.1

hack/compare-k8s-feature-gates.sh

@@ -19,6 +19,8 @@ set -e
 usage() {
   echo "Usage:"
   echo "> compare-k8s-feature-gates.sh [ -h | <old version> <new version> ]"
+  echo
+  echo ">> For example: compare-k8s-feature-gates.sh 1.22 1.23"
 
   exit 0
 }
@@ -58,4 +60,4 @@ echo "Feature gates added in $2 compared to $1:"
 diff "${out_dir}/featuregates-$1.txt" "${out_dir}/featuregates-$2.txt" | grep '>' | awk '{print $2}'
 echo
 echo "Feature gates removed in $2 compared to $1:"
-diff "${out_dir}/featuregates-$1.txt" "${out_dir}/featuregates-$2.txt" | grep '<' | awk '{print $2}'
+diff "${out_dir}/featuregates-$1.txt" "${out_dir}/featuregates-$2.txt" | grep '<' | awk '{print $2}'

hack/test-e2e-local.sh

@@ -28,7 +28,7 @@ go test -mod=vendor -timeout=15m ./test/system/shoot_creation \
   -project-namespace=garden-local \
   -shoot-name=e2e-local \
   -annotations=shoot.gardener.cloud/infrastructure-cleanup-wait-period-seconds=0 \
-  -k8s-version=1.21.0 \
+  -k8s-version=1.23.1 \
   -cloud-profile=local \
   -seed=local \
   -region=local \

pkg/client/kubernetes/client.go

@@ -247,6 +247,7 @@ var supportedKubernetesVersions = []string{
 	"1.20",
 	"1.21",
 	"1.22",
+	"1.23",
 }
 
 func checkIfSupportedKubernetesVersion(gitVersion string) error {

pkg/operation/botanist/component/coredns/coredns.go

@@ -504,6 +504,8 @@ import custom/*.server
 			},
 		}
 
+		// TODO: Switch to autoscaling/v2 for v1.23+ clusters as soon as we have revendored the k8s.io/* libraries with
+		//  v0.23.0 or above.
 		horizontalPodAutoscaler = &autoscalingv2beta1.HorizontalPodAutoscaler{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "coredns",

pkg/operation/botanist/component/extensions/operatingsystemconfig/original/components/kubelet/config.go

@@ -51,7 +51,7 @@ func Config(kubernetesVersion *semver.Version, clusterDNSAddress, clusterDomain
 				CacheUnauthorizedTTL: metav1.Duration{Duration: 30 * time.Second},
 			},
 		},
-		CgroupDriver:                     "cgroupfs",
+		CgroupDriver:                     "systemd",
 		CgroupRoot:                       "/",
 		CgroupsPerQOS:                    pointer.Bool(true),
 		ClusterDNS:                       []string{clusterDNSAddress},
@@ -103,6 +103,10 @@ func Config(kubernetesVersion *semver.Version, clusterDNSAddress, clusterDomain
 		config.VolumePluginDir = pathVolumePluginDirectory
 	}
 
+	if version.ConstraintK8sLessEqual122.Check(kubernetesVersion) {
+		config.CgroupDriver = "cgroupfs"
+	}
+
 	return config
 }
 

pkg/operation/botanist/component/extensions/operatingsystemconfig/original/components/kubelet/config_test.go

@@ -369,5 +369,82 @@ var _ = Describe("Config", func() {
 				cfg.VolumePluginDir = "/var/lib/kubelet/volumeplugins"
 			},
 		),
+
+		Entry(
+			"kubernetes 1.21 w/o defaults",
+			"1.21.1",
+			clusterDNSAddress,
+			clusterDomain,
+			components.ConfigurableKubeletConfigParameters{},
+			kubeletConfigWithDefaults,
+			func(cfg *kubeletconfigv1beta1.KubeletConfiguration) {
+				cfg.RotateCertificates = true
+				cfg.VolumePluginDir = "/var/lib/kubelet/volumeplugins"
+			},
+		),
+		Entry(
+			"kubernetes 1.21 w/ defaults",
+			"1.21.1",
+			clusterDNSAddress,
+			clusterDomain,
+			params,
+			kubeletConfigWithParams,
+			func(cfg *kubeletconfigv1beta1.KubeletConfiguration) {
+				cfg.RotateCertificates = true
+				cfg.VolumePluginDir = "/var/lib/kubelet/volumeplugins"
+			},
+		),
+
+		Entry(
+			"kubernetes 1.22 w/o defaults",
+			"1.22.1",
+			clusterDNSAddress,
+			clusterDomain,
+			components.ConfigurableKubeletConfigParameters{},
+			kubeletConfigWithDefaults,
+			func(cfg *kubeletconfigv1beta1.KubeletConfiguration) {
+				cfg.RotateCertificates = true
+				cfg.VolumePluginDir = "/var/lib/kubelet/volumeplugins"
+			},
+		),
+		Entry(
+			"kubernetes 1.22 w/ defaults",
+			"1.22.1",
+			clusterDNSAddress,
+			clusterDomain,
+			params,
+			kubeletConfigWithParams,
+			func(cfg *kubeletconfigv1beta1.KubeletConfiguration) {
+				cfg.RotateCertificates = true
+				cfg.VolumePluginDir = "/var/lib/kubelet/volumeplugins"
+			},
+		),
+
+		Entry(
+			"kubernetes 1.23 w/o defaults",
+			"1.23.1",
+			clusterDNSAddress,
+			clusterDomain,
+			components.ConfigurableKubeletConfigParameters{},
+			kubeletConfigWithDefaults,
+			func(cfg *kubeletconfigv1beta1.KubeletConfiguration) {
+				cfg.CgroupDriver = "systemd"
+				cfg.RotateCertificates = true
+				cfg.VolumePluginDir = "/var/lib/kubelet/volumeplugins"
+			},
+		),
+		Entry(
+			"kubernetes 1.23 w/ defaults",
+			"1.23.1",
+			clusterDNSAddress,
+			clusterDomain,
+			params,
+			kubeletConfigWithParams,
+			func(cfg *kubeletconfigv1beta1.KubeletConfiguration) {
+				cfg.CgroupDriver = "systemd"
+				cfg.RotateCertificates = true
+				cfg.VolumePluginDir = "/var/lib/kubelet/volumeplugins"
+			},
+		),
 	)
 })

pkg/operation/botanist/component/gardenerkubescheduler/gardener_kube_scheduler.go

@@ -20,7 +20,19 @@ import (
 	"fmt"
 	"time"
 
+	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	"github.com/gardener/gardener/pkg/client/kubernetes"
+	"github.com/gardener/gardener/pkg/controllerutils"
+	"github.com/gardener/gardener/pkg/operation/botanist/component"
+	"github.com/gardener/gardener/pkg/operation/botanist/component/gardenerkubescheduler/configurator"
 	"github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references"
+	"github.com/gardener/gardener/pkg/utils"
+	"github.com/gardener/gardener/pkg/utils/imagevector"
+	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
+	"github.com/gardener/gardener/pkg/utils/managedresources"
+	"github.com/gardener/gardener/pkg/utils/version"
+
+	"github.com/Masterminds/semver"
 	admissionv1 "k8s.io/api/admission/v1"
 	admissionv1beta1 "k8s.io/api/admission/v1beta1"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
@@ -37,16 +49,6 @@ import (
 	autoscalingv1beta2 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
-	"github.com/gardener/gardener/pkg/client/kubernetes"
-	"github.com/gardener/gardener/pkg/controllerutils"
-	"github.com/gardener/gardener/pkg/operation/botanist/component"
-	"github.com/gardener/gardener/pkg/operation/botanist/component/gardenerkubescheduler/configurator"
-	"github.com/gardener/gardener/pkg/utils"
-	"github.com/gardener/gardener/pkg/utils/imagevector"
-	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
-	"github.com/gardener/gardener/pkg/utils/managedresources"
 )
 
 const (
@@ -69,6 +71,7 @@ func New(
 	client client.Client,
 	namespace string,
 	image *imagevector.Image,
+	version *semver.Version,
 	config configurator.Configurator,
 	webhookClientConfig *admissionregistrationv1.WebhookClientConfig,
 ) (
@@ -88,10 +91,10 @@ func New(
 	}
 
 	s := &kubeScheduler{
-		client:    client,
-		namespace: namespace,
-		image:     image,
-
+		client:              client,
+		namespace:           namespace,
+		image:               image,
+		version:             version,
 		config:              config,
 		webhookClientConfig: webhookClientConfig,
 	}
@@ -103,6 +106,7 @@ type kubeScheduler struct {
 	client              client.Client
 	namespace           string
 	image               *imagevector.Image
+	version             *semver.Version
 	config              configurator.Configurator
 	webhookClientConfig *admissionregistrationv1.WebhookClientConfig
 }
@@ -468,11 +472,16 @@ func (k *kubeScheduler) WaitCleanup(ctx context.Context) error {
 }
 
 func (k *kubeScheduler) command(port int32) []string {
-	return []string{
+	command := []string{
 		"/usr/local/bin/kube-scheduler",
 		fmt.Sprintf("--config=%s/%s", volumeMountPathConfig, dataKeyComponentConfig),
 		fmt.Sprintf("--secure-port=%d", port),
-		"--port=0",
 		"--v=2",
 	}
+
+	if version.ConstraintK8sLessEqual122.Check(k.version) {
+		command = append(command, "--port=0")
+	}
+
+	return command
 }

pkg/operation/botanist/component/gardenerkubescheduler/gardener_kube_scheduler_bootstrap.go

@@ -130,6 +130,7 @@ func Bootstrap(
 		c,
 		Name,
 		image,
+		seedVersion,
 		config,
 		&admissionregistrationv1.WebhookClientConfig{
 			Service: &admissionregistrationv1.ServiceReference{