From 0a05a0221b3453fbc8ac0089ee6ef0d3e189d205 Mon Sep 17 00:00:00 2001 From: Ismail Alidzhikov Date: Thu, 30 Jun 2022 11:36:48 +0300 Subject: [PATCH] Add CSI migration feature gates to the cluster-autoscaler (#463) --- pkg/webhook/controlplane/ensurer.go | 44 ++++++++++++ pkg/webhook/controlplane/ensurer_test.go | 85 +++++++++++++++++++++++- 2 files changed, 126 insertions(+), 3 deletions(-) diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go index 35f429d7a..39cbd7165 100644 --- a/pkg/webhook/controlplane/ensurer.go +++ b/pkg/webhook/controlplane/ensurer.go @@ -156,6 +156,38 @@ func (e *ensurer) EnsureKubeSchedulerDeployment(ctx context.Context, gctx gconte return nil } +// EnsureClusterAutoscalerDeployment ensures that the cluster-autoscaler deployment conforms to the provider requirements. +func (e *ensurer) EnsureClusterAutoscalerDeployment(ctx context.Context, gctx gcontext.GardenContext, new, _ *appsv1.Deployment) error { + template := &new.Spec.Template + ps := &template.Spec + + cluster, err := gctx.GetCluster(ctx) + if err != nil { + return err + } + + // cluster-autoscaler supports the "--feature-gates" flag starting 1.20. + // Exit early and do not add the "--feature-gates" flag for K8s < 1.20 Shoots. + k8sLessThan120, err := versionutils.CompareVersions(cluster.Shoot.Spec.Kubernetes.Version, "<", "1.20") + if err != nil { + return err + } + if k8sLessThan120 { + return nil + } + + // At this point K8s >= 1.20. As CSIMigrationKubernetesVersion is 1.19, we can assume that CSI is enabled and CSI migration is complete. + csiMigrationCompleteFeatureGate, err := computeCSIMigrationCompleteFeatureGate(cluster.Shoot.Spec.Kubernetes.Version) + if err != nil { + return err + } + + if c := extensionswebhook.ContainerWithName(ps.Containers, "cluster-autoscaler"); c != nil { + ensureClusterAutoscalerCommandLineArgs(c, csiMigrationCompleteFeatureGate) + } + return nil +} + func ensureKubeAPIServerCommandLineArgs(c *corev1.Container, csiEnabled, csiMigrationComplete bool, csiMigrationCompleteFeatureGate string) { if csiEnabled { c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", @@ -223,6 +255,18 @@ func ensureKubeSchedulerCommandLineArgs(c *corev1.Container, csiEnabled, csiMigr } } +// ensureClusterAutoscalerCommandLineArgs ensures the cluster-autoscaler command line args. +// cluster-autoscaler supports the "--feature-gates" flag starting 1.20. This func assumes that +// the K8s version is >= 1.20 which means that CSI is enabled and CSI migration is complete. +func ensureClusterAutoscalerCommandLineArgs(c *corev1.Container, csiMigrationCompleteFeatureGate string) { + c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + "CSIMigration=true", ",") + c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + "CSIMigrationOpenStack=true", ",") + c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + csiMigrationCompleteFeatureGate+"=true", ",") +} + func ensureKubeControllerManagerLabels(t *corev1.PodTemplateSpec, csiEnabled, csiMigrationComplete bool) { // TODO: This can be removed in a future version. delete(t.Labels, v1beta1constants.LabelNetworkPolicyToBlockedCIDRs) diff --git a/pkg/webhook/controlplane/ensurer_test.go b/pkg/webhook/controlplane/ensurer_test.go index 7535c8816..99df282e4 100644 --- a/pkg/webhook/controlplane/ensurer_test.go +++ b/pkg/webhook/controlplane/ensurer_test.go @@ -139,7 +139,22 @@ var _ = Describe("Ensurer", func() { }, }, ) - + eContextK8s120WithCSIAnnotation = gcontext.NewInternalGardenContext( + &extensionscontroller.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + csimigration.AnnotationKeyNeedsComplete: "true", + }, + }, + Shoot: &gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Kubernetes: gardencorev1beta1.Kubernetes{ + Version: "1.20.0", + }, + }, + }, + }, + ) eContextK8s121 = gcontext.NewInternalGardenContext( &extensionscontroller.Cluster{ Shoot: &gardencorev1beta1.Shoot{ @@ -407,7 +422,7 @@ var _ = Describe("Ensurer", func() { BeforeEach(func() { dep = &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameKubeControllerManager}, + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameKubeScheduler}, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ @@ -424,7 +439,7 @@ var _ = Describe("Ensurer", func() { ensurer = NewEnsurer(logger) }) - It("should add missing elements to kube-scheduler deployment (k8s < 1.19)", func() { + It("should not add anything to kube-scheduler deployment (k8s < 1.19)", func() { err := ensurer.EnsureKubeSchedulerDeployment(ctx, eContextK8s116, dep, nil) Expect(err).To(Not(HaveOccurred())) @@ -453,6 +468,53 @@ var _ = Describe("Ensurer", func() { }) }) + Describe("#EnsureClusterAutoscalerDeployment", func() { + var ( + dep *appsv1.Deployment + ensurer genericmutator.Ensurer + ) + + BeforeEach(func() { + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameClusterAutoscaler}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "cluster-autoscaler", + }, + }, + }, + }, + }, + } + + ensurer = NewEnsurer(logger) + }) + + It("should not add anything to cluster-autoscaler deployment (k8s < 1.20)", func() { + err := ensurer.EnsureClusterAutoscalerDeployment(ctx, eContextK8s119, dep, nil) + Expect(err).To(Not(HaveOccurred())) + + checkClusterAutoscalerDeployment(dep, "1.19.0") + }) + + It("should add missing elements to cluster-autoscaler deployment (k8s 1.20)", func() { + err := ensurer.EnsureClusterAutoscalerDeployment(ctx, eContextK8s120WithCSIAnnotation, dep, nil) + Expect(err).To(Not(HaveOccurred())) + + checkClusterAutoscalerDeployment(dep, "1.20.0") + }) + + It("should add missing elements to cluster-autoscaler deployment (k8s >= 1.21)", func() { + err := ensurer.EnsureClusterAutoscalerDeployment(ctx, eContextK8s121WithCSIAnnotation, dep, nil) + Expect(err).To(Not(HaveOccurred())) + + checkClusterAutoscalerDeployment(dep, "1.21.0") + }) + }) + Describe("#EnsureKubeletServiceUnitOptions", func() { var ( oldUnitOptions []*unit.UnitOption @@ -815,6 +877,23 @@ func checkKubeSchedulerDeployment(dep *appsv1.Deployment, k8sVersion string, nee } } +func checkClusterAutoscalerDeployment(dep *appsv1.Deployment, k8sVersion string) { + if k8sVersionAtLeast120, _ := version.CompareVersions(k8sVersion, ">=", "1.20"); !k8sVersionAtLeast120 { + return + } + k8sVersionAtLeast121, _ := version.CompareVersions(k8sVersion, ">=", "1.21") + + // Check that the cluster-autoscaler container still exists and contains all needed command line args. + c := extensionswebhook.ContainerWithName(dep.Spec.Template.Spec.Containers, "cluster-autoscaler") + Expect(c).To(Not(BeNil())) + + if k8sVersionAtLeast121 { + Expect(c.Command).To(ContainElement("--feature-gates=CSIMigration=true,CSIMigrationOpenStack=true,InTreePluginOpenStackUnregister=true")) + } else { + Expect(c.Command).To(ContainElement("--feature-gates=CSIMigration=true,CSIMigrationOpenStack=true,CSIMigrationOpenStackComplete=true")) + } +} + func clientGet(result runtime.Object) interface{} { return func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { switch obj.(type) {