Add CSI migration feature gates to the cluster-autoscaler (#463)

gardener · Jun 30, 2022 · 0a05a02 · 0a05a02
1 parent 184b640
commit 0a05a02
Show file tree

Hide file tree

Showing 2 changed files with 126 additions and 3 deletions.
diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go
@@ -156,6 +156,38 @@ func (e *ensurer) EnsureKubeSchedulerDeployment(ctx context.Context, gctx gconte
 	return nil
 }
 
+// EnsureClusterAutoscalerDeployment ensures that the cluster-autoscaler deployment conforms to the provider requirements.
+func (e *ensurer) EnsureClusterAutoscalerDeployment(ctx context.Context, gctx gcontext.GardenContext, new, _ *appsv1.Deployment) error {
+	template := &new.Spec.Template
+	ps := &template.Spec
+
+	cluster, err := gctx.GetCluster(ctx)
+	if err != nil {
+		return err
+	}
+
+	// cluster-autoscaler supports the "--feature-gates" flag starting 1.20.
+	// Exit early and do not add the "--feature-gates" flag for K8s < 1.20 Shoots.
+	k8sLessThan120, err := versionutils.CompareVersions(cluster.Shoot.Spec.Kubernetes.Version, "<", "1.20")
+	if err != nil {
+		return err
+	}
+	if k8sLessThan120 {
+		return nil
+	}
+
+	// At this point K8s >= 1.20. As CSIMigrationKubernetesVersion is 1.19, we can assume that CSI is enabled and CSI migration is complete.
+	csiMigrationCompleteFeatureGate, err := computeCSIMigrationCompleteFeatureGate(cluster.Shoot.Spec.Kubernetes.Version)
+	if err != nil {
+		return err
+	}
+
+	if c := extensionswebhook.ContainerWithName(ps.Containers, "cluster-autoscaler"); c != nil {
+		ensureClusterAutoscalerCommandLineArgs(c, csiMigrationCompleteFeatureGate)
+	}
+	return nil
+}
+
 func ensureKubeAPIServerCommandLineArgs(c *corev1.Container, csiEnabled, csiMigrationComplete bool, csiMigrationCompleteFeatureGate string) {
 	if csiEnabled {
 		c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
@@ -223,6 +255,18 @@ func ensureKubeSchedulerCommandLineArgs(c *corev1.Container, csiEnabled, csiMigr
 	}
 }
 
+// ensureClusterAutoscalerCommandLineArgs ensures the cluster-autoscaler command line args.
+// cluster-autoscaler supports the "--feature-gates" flag starting 1.20. This func assumes that
+// the K8s version is >= 1.20 which means that CSI is enabled and CSI migration is complete.
+func ensureClusterAutoscalerCommandLineArgs(c *corev1.Container, csiMigrationCompleteFeatureGate string) {
+	c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
+		"CSIMigration=true", ",")
+	c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
+		"CSIMigrationOpenStack=true", ",")
+	c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
+		csiMigrationCompleteFeatureGate+"=true", ",")
+}
+
 func ensureKubeControllerManagerLabels(t *corev1.PodTemplateSpec, csiEnabled, csiMigrationComplete bool) {
 	// TODO: This can be removed in a future version.
 	delete(t.Labels, v1beta1constants.LabelNetworkPolicyToBlockedCIDRs)

diff --git a/pkg/webhook/controlplane/ensurer_test.go b/pkg/webhook/controlplane/ensurer_test.go
@@ -139,7 +139,22 @@ var _ = Describe("Ensurer", func() {
 				},
 			},
 		)
-
+		eContextK8s120WithCSIAnnotation = gcontext.NewInternalGardenContext(
+			&extensionscontroller.Cluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						csimigration.AnnotationKeyNeedsComplete: "true",
+					},
+				},
+				Shoot: &gardencorev1beta1.Shoot{
+					Spec: gardencorev1beta1.ShootSpec{
+						Kubernetes: gardencorev1beta1.Kubernetes{
+							Version: "1.20.0",
+						},
+					},
+				},
+			},
+		)
 		eContextK8s121 = gcontext.NewInternalGardenContext(
 			&extensionscontroller.Cluster{
 				Shoot: &gardencorev1beta1.Shoot{
@@ -407,7 +422,7 @@ var _ = Describe("Ensurer", func() {
 
 		BeforeEach(func() {
 			dep = &appsv1.Deployment{
-				ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameKubeControllerManager},
+				ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameKubeScheduler},
 				Spec: appsv1.DeploymentSpec{
 					Template: corev1.PodTemplateSpec{
 						Spec: corev1.PodSpec{
@@ -424,7 +439,7 @@ var _ = Describe("Ensurer", func() {
 			ensurer = NewEnsurer(logger)
 		})
 
-		It("should add missing elements to kube-scheduler deployment (k8s < 1.19)", func() {
+		It("should not add anything to kube-scheduler deployment (k8s < 1.19)", func() {
 			err := ensurer.EnsureKubeSchedulerDeployment(ctx, eContextK8s116, dep, nil)
 			Expect(err).To(Not(HaveOccurred()))
 
@@ -453,6 +468,53 @@ var _ = Describe("Ensurer", func() {
 		})
 	})
 
+	Describe("#EnsureClusterAutoscalerDeployment", func() {
+		var (
+			dep     *appsv1.Deployment
+			ensurer genericmutator.Ensurer
+		)
+
+		BeforeEach(func() {
+			dep = &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameClusterAutoscaler},
+				Spec: appsv1.DeploymentSpec{
+					Template: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name: "cluster-autoscaler",
+								},
+							},
+						},
+					},
+				},
+			}
+
+			ensurer = NewEnsurer(logger)
+		})
+
+		It("should not add anything to cluster-autoscaler deployment (k8s < 1.20)", func() {
+			err := ensurer.EnsureClusterAutoscalerDeployment(ctx, eContextK8s119, dep, nil)
+			Expect(err).To(Not(HaveOccurred()))
+
+			checkClusterAutoscalerDeployment(dep, "1.19.0")
+		})
+
+		It("should add missing elements to cluster-autoscaler deployment (k8s 1.20)", func() {
+			err := ensurer.EnsureClusterAutoscalerDeployment(ctx, eContextK8s120WithCSIAnnotation, dep, nil)
+			Expect(err).To(Not(HaveOccurred()))
+
+			checkClusterAutoscalerDeployment(dep, "1.20.0")
+		})
+
+		It("should add missing elements to cluster-autoscaler deployment (k8s >= 1.21)", func() {
+			err := ensurer.EnsureClusterAutoscalerDeployment(ctx, eContextK8s121WithCSIAnnotation, dep, nil)
+			Expect(err).To(Not(HaveOccurred()))
+
+			checkClusterAutoscalerDeployment(dep, "1.21.0")
+		})
+	})
+
 	Describe("#EnsureKubeletServiceUnitOptions", func() {
 		var (
 			oldUnitOptions        []*unit.UnitOption
@@ -815,6 +877,23 @@ func checkKubeSchedulerDeployment(dep *appsv1.Deployment, k8sVersion string, nee
 	}
 }
 
+func checkClusterAutoscalerDeployment(dep *appsv1.Deployment, k8sVersion string) {
+	if k8sVersionAtLeast120, _ := version.CompareVersions(k8sVersion, ">=", "1.20"); !k8sVersionAtLeast120 {
+		return
+	}
+	k8sVersionAtLeast121, _ := version.CompareVersions(k8sVersion, ">=", "1.21")
+
+	// Check that the cluster-autoscaler container still exists and contains all needed command line args.
+	c := extensionswebhook.ContainerWithName(dep.Spec.Template.Spec.Containers, "cluster-autoscaler")
+	Expect(c).To(Not(BeNil()))
+
+	if k8sVersionAtLeast121 {
+		Expect(c.Command).To(ContainElement("--feature-gates=CSIMigration=true,CSIMigrationOpenStack=true,InTreePluginOpenStackUnregister=true"))
+	} else {
+		Expect(c.Command).To(ContainElement("--feature-gates=CSIMigration=true,CSIMigrationOpenStack=true,CSIMigrationOpenStackComplete=true"))
+	}
+}
+
 func clientGet(result runtime.Object) interface{} {
 	return func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
 		switch obj.(type) {