diff --git a/controlplane/kubeadm/internal/controllers/controller_test.go b/controlplane/kubeadm/internal/controllers/controller_test.go index aaf7c09a180b..6459174a918a 100644 --- a/controlplane/kubeadm/internal/controllers/controller_test.go +++ b/controlplane/kubeadm/internal/controllers/controller_test.go @@ -1279,7 +1279,8 @@ dns: type: CoreDNS imageRepository: registry.k8s.io kind: ClusterConfiguration -kubernetesVersion: metav1.16.1`, +kubernetesVersion: metav1.16.1 +`, }, } g.Expect(env.Create(ctx, kubeadmCM)).To(Succeed()) diff --git a/controlplane/kubeadm/internal/controllers/fakes_test.go b/controlplane/kubeadm/internal/controllers/fakes_test.go index 3c7348bc4831..aa583a1f9639 100644 --- a/controlplane/kubeadm/internal/controllers/fakes_test.go +++ b/controlplane/kubeadm/internal/controllers/fakes_test.go @@ -48,7 +48,7 @@ func (f *fakeManagementCluster) List(ctx context.Context, list client.ObjectList return f.Reader.List(ctx, list, opts...) } -func (f *fakeManagementCluster) GetWorkloadCluster(_ context.Context, _ client.ObjectKey) (internal.WorkloadCluster, error) { +func (f *fakeManagementCluster) GetWorkloadCluster(context.Context, client.ObjectKey) (internal.WorkloadCluster, error) { return f.Workload, nil } @@ -80,65 +80,69 @@ func (f fakeWorkloadCluster) ForwardEtcdLeadership(_ context.Context, _ *cluster return nil } -func (f fakeWorkloadCluster) ReconcileEtcdMembers(_ context.Context, _ []string, _ semver.Version) ([]string, error) { +func (f fakeWorkloadCluster) ReconcileEtcdMembers(context.Context, []string, semver.Version) ([]string, error) { return nil, nil } -func (f fakeWorkloadCluster) ClusterStatus(_ context.Context) (internal.ClusterStatus, error) { +func (f fakeWorkloadCluster) ClusterStatus(context.Context) (internal.ClusterStatus, error) { return f.Status, nil } -func (f fakeWorkloadCluster) GetAPIServerCertificateExpiry(_ context.Context, _ *bootstrapv1.KubeadmConfig, _ string) (*time.Time, error) { +func (f fakeWorkloadCluster) GetAPIServerCertificateExpiry(context.Context, *bootstrapv1.KubeadmConfig, string) (*time.Time, error) { return f.APIServerCertificateExpiry, nil } -func (f fakeWorkloadCluster) AllowBootstrapTokensToGetNodes(_ context.Context) error { +func (f fakeWorkloadCluster) AllowBootstrapTokensToGetNodes(context.Context) error { return nil } -func (f fakeWorkloadCluster) AllowClusterAdminPermissions(_ context.Context, _ semver.Version) error { +func (f fakeWorkloadCluster) AllowClusterAdminPermissions(context.Context, semver.Version) error { return nil } -func (f fakeWorkloadCluster) ReconcileKubeletRBACRole(_ context.Context, _ semver.Version) error { +func (f fakeWorkloadCluster) ReconcileKubeletRBACRole(context.Context, semver.Version) error { return nil } -func (f fakeWorkloadCluster) ReconcileKubeletRBACBinding(_ context.Context, _ semver.Version) error { +func (f fakeWorkloadCluster) ReconcileKubeletRBACBinding(context.Context, semver.Version) error { return nil } -func (f fakeWorkloadCluster) UpdateKubernetesVersionInKubeadmConfigMap(_ context.Context, _ semver.Version) error { +func (f fakeWorkloadCluster) UpdateKubernetesVersionInKubeadmConfigMap(semver.Version) func(*bootstrapv1.ClusterConfiguration, *[]string) { return nil } -func (f fakeWorkloadCluster) UpdateEtcdVersionInKubeadmConfigMap(_ context.Context, _, _ string, _ semver.Version) error { +func (f fakeWorkloadCluster) UpdateEtcdVersionInKubeadmConfigMap(string, string) func(*bootstrapv1.ClusterConfiguration, *[]string) { return nil } -func (f fakeWorkloadCluster) UpdateKubeletConfigMap(_ context.Context, _ semver.Version) error { +func (f fakeWorkloadCluster) UpdateKubeletConfigMap(context.Context, semver.Version) error { return nil } -func (f fakeWorkloadCluster) RemoveEtcdMemberForMachine(_ context.Context, _ *clusterv1.Machine) error { +func (f fakeWorkloadCluster) RemoveEtcdMemberForMachine(context.Context, *clusterv1.Machine) error { return nil } -func (f fakeWorkloadCluster) RemoveMachineFromKubeadmConfigMap(_ context.Context, _ *clusterv1.Machine, _ semver.Version) error { +func (f fakeWorkloadCluster) RemoveMachineFromKubeadmConfigMap(context.Context, *clusterv1.Machine, semver.Version) error { return nil } -func (f fakeWorkloadCluster) EtcdMembers(_ context.Context) ([]string, error) { +func (f fakeWorkloadCluster) EtcdMembers(context.Context) ([]string, error) { return f.EtcdMembersResult, nil } +func (f fakeWorkloadCluster) UpdateClusterConfiguration(context.Context, semver.Version, ...func(*bootstrapv1.ClusterConfiguration, *[]string)) error { + return nil +} + type fakeMigrator struct { migrateCalled bool migrateErr error migratedCorefile string } -func (m *fakeMigrator) Migrate(_, _, _ string, _ bool) (string, error) { +func (m *fakeMigrator) Migrate(string, string, string, bool) (string, error) { m.migrateCalled = true if m.migrateErr != nil { return "", m.migrateErr diff --git a/controlplane/kubeadm/internal/controllers/upgrade.go b/controlplane/kubeadm/internal/controllers/upgrade.go index 6abf136947ab..d9f5f72c6466 100644 --- a/controlplane/kubeadm/internal/controllers/upgrade.go +++ b/controlplane/kubeadm/internal/controllers/upgrade.go @@ -23,6 +23,7 @@ import ( "github.com/pkg/errors" ctrl "sigs.k8s.io/controller-runtime" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/controlplane/kubeadm/internal" "sigs.k8s.io/cluster-api/util" @@ -73,9 +74,8 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane( return ctrl.Result{}, errors.Wrap(err, "failed to set cluster-admin ClusterRoleBinding for kubeadm") } - if err := workloadCluster.UpdateKubernetesVersionInKubeadmConfigMap(ctx, parsedVersion); err != nil { - return ctrl.Result{}, errors.Wrap(err, "failed to update the kubernetes version in the kubeadm config map") - } + kubeadmCMMutators := make([]func(*bootstrapv1.ClusterConfiguration, *[]string), 0) + kubeadmCMMutators = append(kubeadmCMMutators, workloadCluster.UpdateKubernetesVersionInKubeadmConfigMap(parsedVersion)) if controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration != nil { // We intentionally only parse major/minor/patch so that the subsequent code @@ -84,38 +84,29 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane( if err != nil { return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version) } + // Get the imageRepository or the correct value if nothing is set and a migration is necessary. imageRepository := internal.ImageRepositoryFromClusterConfig(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration, parsedVersionTolerant) - if err := workloadCluster.UpdateImageRepositoryInKubeadmConfigMap(ctx, imageRepository, parsedVersion); err != nil { - return ctrl.Result{}, errors.Wrap(err, "failed to update the image repository in the kubeadm config map") - } - } - - if controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration != nil && controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local != nil { - meta := controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local.ImageMeta - if err := workloadCluster.UpdateEtcdVersionInKubeadmConfigMap(ctx, meta.ImageRepository, meta.ImageTag, parsedVersion); err != nil { - return ctrl.Result{}, errors.Wrap(err, "failed to update the etcd version in the kubeadm config map") - } - - extraArgs := controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local.ExtraArgs - if err := workloadCluster.UpdateEtcdExtraArgsInKubeadmConfigMap(ctx, extraArgs, parsedVersion); err != nil { - return ctrl.Result{}, errors.Wrap(err, "failed to update the etcd extra args in the kubeadm config map") + kubeadmCMMutators = append(kubeadmCMMutators, + workloadCluster.UpdateImageRepositoryInKubeadmConfigMap(imageRepository), + workloadCluster.UpdateFeatureGatesInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates), + workloadCluster.UpdateAPIServerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer), + workloadCluster.UpdateControllerManagerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager), + workloadCluster.UpdateSchedulerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Scheduler)) + + if controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local != nil { + meta := controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local.ImageMeta + extraArgs := controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local.ExtraArgs + kubeadmCMMutators = append(kubeadmCMMutators, + workloadCluster.UpdateEtcdVersionInKubeadmConfigMap(meta.ImageRepository, meta.ImageTag), + workloadCluster.UpdateEtcdExtraArgsInKubeadmConfigMap(extraArgs)) } } - if controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration != nil { - if err := workloadCluster.UpdateAPIServerInKubeadmConfigMap(ctx, controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer, parsedVersion); err != nil { - return ctrl.Result{}, errors.Wrap(err, "failed to update api server in the kubeadm config map") - } - - if err := workloadCluster.UpdateControllerManagerInKubeadmConfigMap(ctx, controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager, parsedVersion); err != nil { - return ctrl.Result{}, errors.Wrap(err, "failed to update controller manager in the kubeadm config map") - } - - if err := workloadCluster.UpdateSchedulerInKubeadmConfigMap(ctx, controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Scheduler, parsedVersion); err != nil { - return ctrl.Result{}, errors.Wrap(err, "failed to update scheduler in the kubeadm config map") - } + // collectively update Kubeadm config map + if err = workloadCluster.UpdateClusterConfiguration(ctx, parsedVersion, kubeadmCMMutators...); err != nil { + return ctrl.Result{}, err } if err := workloadCluster.UpdateKubeletConfigMap(ctx, parsedVersion); err != nil { diff --git a/controlplane/kubeadm/internal/webhooks/kubeadm_control_plane.go b/controlplane/kubeadm/internal/webhooks/kubeadm_control_plane.go index 4338b8b11b15..d848d4616bba 100644 --- a/controlplane/kubeadm/internal/webhooks/kubeadm_control_plane.go +++ b/controlplane/kubeadm/internal/webhooks/kubeadm_control_plane.go @@ -150,6 +150,7 @@ const ( ntp = "ntp" ignition = "ignition" diskSetup = "diskSetup" + featureGates = "featureGates" ) const minimumCertificatesExpiryDays = 7 @@ -176,6 +177,8 @@ func (webhook *KubeadmControlPlane) ValidateUpdate(_ context.Context, oldObj, ne {spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageRepository"}, {spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageTag"}, {spec, kubeadmConfigSpec, clusterConfiguration, "imageRepository"}, + {spec, kubeadmConfigSpec, clusterConfiguration, featureGates}, + {spec, kubeadmConfigSpec, clusterConfiguration, featureGates, "*"}, {spec, kubeadmConfigSpec, clusterConfiguration, apiServer}, {spec, kubeadmConfigSpec, clusterConfiguration, apiServer, "*"}, {spec, kubeadmConfigSpec, clusterConfiguration, controllerManager}, diff --git a/controlplane/kubeadm/internal/webhooks/kubeadm_control_plane_test.go b/controlplane/kubeadm/internal/webhooks/kubeadm_control_plane_test.go index 699d3776d0e1..9c62b22ad4e6 100644 --- a/controlplane/kubeadm/internal/webhooks/kubeadm_control_plane_test.go +++ b/controlplane/kubeadm/internal/webhooks/kubeadm_control_plane_test.go @@ -886,8 +886,8 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) { kcp: imageRepository, }, { - name: "should fail when making a change to the cluster config's featureGates", - expectErr: true, + name: "should succeed when making a change to the cluster config's featureGates", + expectErr: false, before: before, kcp: featureGates, }, diff --git a/controlplane/kubeadm/internal/workload_cluster.go b/controlplane/kubeadm/internal/workload_cluster.go index d4c41eb89f4c..a9c2958a4bcd 100644 --- a/controlplane/kubeadm/internal/workload_cluster.go +++ b/controlplane/kubeadm/internal/workload_cluster.go @@ -27,6 +27,7 @@ import ( "fmt" "math/big" "reflect" + "strings" "time" "github.com/blang/semver/v4" @@ -105,13 +106,14 @@ type WorkloadCluster interface { // Upgrade related tasks. ReconcileKubeletRBACBinding(ctx context.Context, version semver.Version) error ReconcileKubeletRBACRole(ctx context.Context, version semver.Version) error - UpdateKubernetesVersionInKubeadmConfigMap(ctx context.Context, version semver.Version) error - UpdateImageRepositoryInKubeadmConfigMap(ctx context.Context, imageRepository string, version semver.Version) error - UpdateEtcdVersionInKubeadmConfigMap(ctx context.Context, imageRepository, imageTag string, version semver.Version) error - UpdateEtcdExtraArgsInKubeadmConfigMap(ctx context.Context, extraArgs map[string]string, version semver.Version) error - UpdateAPIServerInKubeadmConfigMap(ctx context.Context, apiServer bootstrapv1.APIServer, version semver.Version) error - UpdateControllerManagerInKubeadmConfigMap(ctx context.Context, controllerManager bootstrapv1.ControlPlaneComponent, version semver.Version) error - UpdateSchedulerInKubeadmConfigMap(ctx context.Context, scheduler bootstrapv1.ControlPlaneComponent, version semver.Version) error + UpdateKubernetesVersionInKubeadmConfigMap(version semver.Version) func(*bootstrapv1.ClusterConfiguration, *[]string) + UpdateImageRepositoryInKubeadmConfigMap(imageRepository string) func(*bootstrapv1.ClusterConfiguration, *[]string) + UpdateFeatureGatesInKubeadmConfigMap(featureGates map[string]bool) func(*bootstrapv1.ClusterConfiguration, *[]string) + UpdateEtcdVersionInKubeadmConfigMap(imageRepository, imageTag string) func(*bootstrapv1.ClusterConfiguration, *[]string) + UpdateEtcdExtraArgsInKubeadmConfigMap(extraArgs map[string]string) func(*bootstrapv1.ClusterConfiguration, *[]string) + UpdateAPIServerInKubeadmConfigMap(apiServer bootstrapv1.APIServer) func(*bootstrapv1.ClusterConfiguration, *[]string) + UpdateControllerManagerInKubeadmConfigMap(controllerManager bootstrapv1.ControlPlaneComponent) func(*bootstrapv1.ClusterConfiguration, *[]string) + UpdateSchedulerInKubeadmConfigMap(scheduler bootstrapv1.ControlPlaneComponent) func(*bootstrapv1.ClusterConfiguration, *[]string) UpdateKubeletConfigMap(ctx context.Context, version semver.Version) error UpdateKubeProxyImageInfo(ctx context.Context, kcp *controlplanev1.KubeadmControlPlane, version semver.Version) error UpdateCoreDNS(ctx context.Context, kcp *controlplanev1.KubeadmControlPlane, version semver.Version) error @@ -121,6 +123,7 @@ type WorkloadCluster interface { ForwardEtcdLeadership(ctx context.Context, machine *clusterv1.Machine, leaderCandidate *clusterv1.Machine) error AllowBootstrapTokensToGetNodes(ctx context.Context) error AllowClusterAdminPermissions(ctx context.Context, version semver.Version) error + UpdateClusterConfiguration(ctx context.Context, version semver.Version, mutators ...func(*bootstrapv1.ClusterConfiguration, *[]string)) error // State recovery tasks. ReconcileEtcdMembers(ctx context.Context, nodeNames []string, version semver.Version) ([]string, error) @@ -173,20 +176,35 @@ func (w *Workload) getConfigMap(ctx context.Context, configMap ctrlclient.Object } // UpdateImageRepositoryInKubeadmConfigMap updates the image repository in the kubeadm config map. -func (w *Workload) UpdateImageRepositoryInKubeadmConfigMap(ctx context.Context, imageRepository string, version semver.Version) error { - return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) { +func (w *Workload) UpdateImageRepositoryInKubeadmConfigMap(imageRepository string) func(*bootstrapv1.ClusterConfiguration, *[]string) { + return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) { if imageRepository == "" { return } + c.ImageRepository = imageRepository - }, version) + *updatedKeys = append(*updatedKeys, "imageRepository") + } +} + +// UpdateFeatureGatesInKubeadmConfigMap updates the feature gates in the kubeadm config map. +func (w *Workload) UpdateFeatureGatesInKubeadmConfigMap(featureGates map[string]bool) func(*bootstrapv1.ClusterConfiguration, *[]string) { + return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) { + if featureGates == nil { + return + } + + c.FeatureGates = featureGates + *updatedKeys = append(*updatedKeys, "featureGates") + } } // UpdateKubernetesVersionInKubeadmConfigMap updates the kubernetes version in the kubeadm config map. -func (w *Workload) UpdateKubernetesVersionInKubeadmConfigMap(ctx context.Context, version semver.Version) error { - return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) { +func (w *Workload) UpdateKubernetesVersionInKubeadmConfigMap(version semver.Version) func(*bootstrapv1.ClusterConfiguration, *[]string) { + return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) { c.KubernetesVersion = fmt.Sprintf("v%s", version.String()) - }, version) + *updatedKeys = append(*updatedKeys, "kubernetesVersion") + } } // UpdateKubeletConfigMap will create a new kubelet-config-1.x config map for a new version of the kubelet. @@ -270,24 +288,27 @@ func (w *Workload) UpdateKubeletConfigMap(ctx context.Context, version semver.Ve } // UpdateAPIServerInKubeadmConfigMap updates api server configuration in kubeadm config map. -func (w *Workload) UpdateAPIServerInKubeadmConfigMap(ctx context.Context, apiServer bootstrapv1.APIServer, version semver.Version) error { - return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) { +func (w *Workload) UpdateAPIServerInKubeadmConfigMap(apiServer bootstrapv1.APIServer) func(*bootstrapv1.ClusterConfiguration, *[]string) { + return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) { c.APIServer = apiServer - }, version) + *updatedKeys = append(*updatedKeys, "apiServer") + } } // UpdateControllerManagerInKubeadmConfigMap updates controller manager configuration in kubeadm config map. -func (w *Workload) UpdateControllerManagerInKubeadmConfigMap(ctx context.Context, controllerManager bootstrapv1.ControlPlaneComponent, version semver.Version) error { - return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) { +func (w *Workload) UpdateControllerManagerInKubeadmConfigMap(controllerManager bootstrapv1.ControlPlaneComponent) func(*bootstrapv1.ClusterConfiguration, *[]string) { + return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) { c.ControllerManager = controllerManager - }, version) + *updatedKeys = append(*updatedKeys, "controllerManager") + } } // UpdateSchedulerInKubeadmConfigMap updates scheduler configuration in kubeadm config map. -func (w *Workload) UpdateSchedulerInKubeadmConfigMap(ctx context.Context, scheduler bootstrapv1.ControlPlaneComponent, version semver.Version) error { - return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) { +func (w *Workload) UpdateSchedulerInKubeadmConfigMap(scheduler bootstrapv1.ControlPlaneComponent) func(*bootstrapv1.ClusterConfiguration, *[]string) { + return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) { c.Scheduler = scheduler - }, version) + *updatedKeys = append(*updatedKeys, "scheduler") + } } // RemoveMachineFromKubeadmConfigMap removes the entry for the machine from the kubeadm configmap. @@ -350,11 +371,11 @@ func (w *Workload) updateClusterStatus(ctx context.Context, mutator func(status }) } -// updateClusterConfiguration gets the ClusterConfiguration kubeadm-config ConfigMap, converts it to the +// UpdateClusterConfiguration gets the ClusterConfiguration kubeadm-config ConfigMap, converts it to the // Cluster API representation, and then applies a mutation func; if changes are detected, the // data are converted back into the Kubeadm API version in use for the target Kubernetes version and the // kubeadm-config ConfigMap updated. -func (w *Workload) updateClusterConfiguration(ctx context.Context, mutator func(*bootstrapv1.ClusterConfiguration), version semver.Version) error { +func (w *Workload) UpdateClusterConfiguration(ctx context.Context, version semver.Version, mutators ...func(*bootstrapv1.ClusterConfiguration, *[]string)) error { return retry.RetryOnConflict(retry.DefaultBackoff, func() error { key := ctrlclient.ObjectKey{Name: kubeadmConfigKey, Namespace: metav1.NamespaceSystem} configMap, err := w.getConfigMap(ctx, key) @@ -373,7 +394,10 @@ func (w *Workload) updateClusterConfiguration(ctx context.Context, mutator func( } updatedObj := currentObj.DeepCopy() - mutator(updatedObj) + updatedKeys := make([]string, 0) + for i := range mutators { + mutators[i](updatedObj, &updatedKeys) + } if !reflect.DeepEqual(currentObj, updatedObj) { updatedData, err := kubeadmtypes.MarshalClusterConfigurationForVersion(updatedObj, version) @@ -382,7 +406,7 @@ func (w *Workload) updateClusterConfiguration(ctx context.Context, mutator func( } configMap.Data[clusterConfigurationKey] = updatedData if err := w.Client.Update(ctx, configMap); err != nil { - return errors.Wrap(err, "failed to upgrade the kubeadmConfigMap") + return errors.Wrapf(err, "failed to upgrade the kubeadmConfigMap with the following properties %s", strings.Join(updatedKeys, ",")) } } return nil diff --git a/controlplane/kubeadm/internal/workload_cluster_coredns.go b/controlplane/kubeadm/internal/workload_cluster_coredns.go index 5699c9c06656..5663b64117ea 100644 --- a/controlplane/kubeadm/internal/workload_cluster_coredns.go +++ b/controlplane/kubeadm/internal/workload_cluster_coredns.go @@ -145,7 +145,7 @@ func (w *Workload) UpdateCoreDNS(ctx context.Context, kcp *controlplanev1.Kubead } // Perform the upgrade. - if err := w.updateCoreDNSImageInfoInKubeadmConfigMap(ctx, &clusterConfig.DNS, version); err != nil { + if err := w.UpdateClusterConfiguration(ctx, version, w.updateCoreDNSImageInfoInKubeadmConfigMap(&clusterConfig.DNS)); err != nil { return err } if err := w.updateCoreDNSCorefile(ctx, info); err != nil { @@ -270,11 +270,12 @@ func (w *Workload) updateCoreDNSDeployment(ctx context.Context, info *coreDNSInf } // updateCoreDNSImageInfoInKubeadmConfigMap updates the kubernetes version in the kubeadm config map. -func (w *Workload) updateCoreDNSImageInfoInKubeadmConfigMap(ctx context.Context, dns *bootstrapv1.DNS, version semver.Version) error { - return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) { +func (w *Workload) updateCoreDNSImageInfoInKubeadmConfigMap(dns *bootstrapv1.DNS) func(*bootstrapv1.ClusterConfiguration, *[]string) { + return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) { c.DNS.ImageRepository = dns.ImageRepository c.DNS.ImageTag = dns.ImageTag - }, version) + *updatedKeys = append(*updatedKeys, "dns.imageRepository", "dns.imageTag") + } } // updateCoreDNSClusterRole updates the CoreDNS ClusterRole when necessary. diff --git a/controlplane/kubeadm/internal/workload_cluster_coredns_test.go b/controlplane/kubeadm/internal/workload_cluster_coredns_test.go index 12bf01c42863..a9f1f2a8c369 100644 --- a/controlplane/kubeadm/internal/workload_cluster_coredns_test.go +++ b/controlplane/kubeadm/internal/workload_cluster_coredns_test.go @@ -1451,7 +1451,7 @@ func TestUpdateCoreDNSImageInfoInKubeadmConfigMap(t *testing.T) { w := &Workload{ Client: fakeClient, } - err := w.updateCoreDNSImageInfoInKubeadmConfigMap(ctx, &tt.newDNS, semver.MustParse("1.19.1")) + err := w.UpdateClusterConfiguration(ctx, semver.MustParse("1.19.1"), w.updateCoreDNSImageInfoInKubeadmConfigMap(&tt.newDNS)) g.Expect(err).ToNot(HaveOccurred()) var actualConfig corev1.ConfigMap diff --git a/controlplane/kubeadm/internal/workload_cluster_etcd.go b/controlplane/kubeadm/internal/workload_cluster_etcd.go index bb4c4d417960..a4ad0f0d2d32 100644 --- a/controlplane/kubeadm/internal/workload_cluster_etcd.go +++ b/controlplane/kubeadm/internal/workload_cluster_etcd.go @@ -93,22 +93,24 @@ loopmembers: } // UpdateEtcdVersionInKubeadmConfigMap sets the imageRepository or the imageTag or both in the kubeadm config map. -func (w *Workload) UpdateEtcdVersionInKubeadmConfigMap(ctx context.Context, imageRepository, imageTag string, version semver.Version) error { - return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) { +func (w *Workload) UpdateEtcdVersionInKubeadmConfigMap(imageRepository, imageTag string) func(*bootstrapv1.ClusterConfiguration, *[]string) { + return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) { if c.Etcd.Local != nil { c.Etcd.Local.ImageRepository = imageRepository c.Etcd.Local.ImageTag = imageTag + *updatedKeys = append(*updatedKeys, "etcd.local.imageRepository, etcd.local.imageTag") } - }, version) + } } // UpdateEtcdExtraArgsInKubeadmConfigMap sets extraArgs in the kubeadm config map. -func (w *Workload) UpdateEtcdExtraArgsInKubeadmConfigMap(ctx context.Context, extraArgs map[string]string, version semver.Version) error { - return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) { +func (w *Workload) UpdateEtcdExtraArgsInKubeadmConfigMap(extraArgs map[string]string) func(*bootstrapv1.ClusterConfiguration, *[]string) { + return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) { if c.Etcd.Local != nil { c.Etcd.Local.ExtraArgs = extraArgs + *updatedKeys = append(*updatedKeys, "etcd.local.extraArgs") } - }, version) + } } // RemoveEtcdMemberForMachine removes the etcd member from the target cluster's etcd cluster. diff --git a/controlplane/kubeadm/internal/workload_cluster_etcd_test.go b/controlplane/kubeadm/internal/workload_cluster_etcd_test.go index e80e869a51ba..db61ac8f2125 100644 --- a/controlplane/kubeadm/internal/workload_cluster_etcd_test.go +++ b/controlplane/kubeadm/internal/workload_cluster_etcd_test.go @@ -104,7 +104,7 @@ func TestUpdateEtcdVersionInKubeadmConfigMap(t *testing.T) { w := &Workload{ Client: fakeClient, } - err := w.UpdateEtcdVersionInKubeadmConfigMap(ctx, tt.newImageRepository, tt.newImageTag, semver.MustParse("1.19.1")) + err := w.UpdateClusterConfiguration(ctx, semver.MustParse("1.19.1"), w.UpdateEtcdVersionInKubeadmConfigMap(tt.newImageRepository, tt.newImageTag)) g.Expect(err).ToNot(HaveOccurred()) var actualConfig corev1.ConfigMap @@ -186,7 +186,7 @@ func TestUpdateEtcdExtraArgsInKubeadmConfigMap(t *testing.T) { w := &Workload{ Client: fakeClient, } - err := w.UpdateEtcdExtraArgsInKubeadmConfigMap(ctx, tt.newExtraArgs, semver.MustParse("1.19.1")) + err := w.UpdateClusterConfiguration(ctx, semver.MustParse("1.19.1"), w.UpdateEtcdExtraArgsInKubeadmConfigMap(tt.newExtraArgs)) g.Expect(err).ToNot(HaveOccurred()) var actualConfig corev1.ConfigMap diff --git a/controlplane/kubeadm/internal/workload_cluster_test.go b/controlplane/kubeadm/internal/workload_cluster_test.go index bb15b6d99da2..a76e15dfaa9f 100644 --- a/controlplane/kubeadm/internal/workload_cluster_test.go +++ b/controlplane/kubeadm/internal/workload_cluster_test.go @@ -30,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + yaml2 "sigs.k8s.io/yaml" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" @@ -531,7 +532,7 @@ func TestUpdateUpdateClusterConfigurationInKubeadmConfigMap(t *testing.T) { name string version semver.Version objs []client.Object - mutator func(*bootstrapv1.ClusterConfiguration) + mutator func(*bootstrapv1.ClusterConfiguration, *[]string) wantConfigMap *corev1.ConfigMap wantErr bool }{ @@ -583,7 +584,7 @@ func TestUpdateUpdateClusterConfigurationInKubeadmConfigMap(t *testing.T) { `), }, }}, - mutator: func(*bootstrapv1.ClusterConfiguration) {}, + mutator: func(*bootstrapv1.ClusterConfiguration, *[]string) {}, wantConfigMap: &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: kubeadmConfigKey, @@ -614,7 +615,7 @@ func TestUpdateUpdateClusterConfigurationInKubeadmConfigMap(t *testing.T) { `), }, }}, - mutator: func(c *bootstrapv1.ClusterConfiguration) { + mutator: func(c *bootstrapv1.ClusterConfiguration, _ *[]string) { c.KubernetesVersion = "v1.17.2" }, wantConfigMap: &corev1.ConfigMap{ @@ -653,7 +654,7 @@ func TestUpdateUpdateClusterConfigurationInKubeadmConfigMap(t *testing.T) { `), }, }}, - mutator: func(c *bootstrapv1.ClusterConfiguration) { + mutator: func(c *bootstrapv1.ClusterConfiguration, _ *[]string) { c.KubernetesVersion = "v1.28.0" }, wantConfigMap: &corev1.ConfigMap{ @@ -686,7 +687,7 @@ func TestUpdateUpdateClusterConfigurationInKubeadmConfigMap(t *testing.T) { w := &Workload{ Client: fakeClient, } - err := w.updateClusterConfiguration(ctx, tt.mutator, tt.version) + err := w.UpdateClusterConfiguration(ctx, tt.version, tt.mutator) if tt.wantErr { g.Expect(err).To(HaveOccurred()) return @@ -882,7 +883,8 @@ func TestUpdateKubernetesVersionInKubeadmConfigMap(t *testing.T) { w := &Workload{ Client: fakeClient, } - err := w.UpdateKubernetesVersionInKubeadmConfigMap(ctx, tt.version) + + err := w.UpdateClusterConfiguration(ctx, tt.version, w.UpdateKubernetesVersionInKubeadmConfigMap(tt.version)) g.Expect(err).ToNot(HaveOccurred()) var actualConfig corev1.ConfigMap @@ -938,7 +940,7 @@ func TestUpdateImageRepositoryInKubeadmConfigMap(t *testing.T) { w := &Workload{ Client: fakeClient, } - err := w.UpdateImageRepositoryInKubeadmConfigMap(ctx, tt.newImageRepository, semver.MustParse("1.19.1")) + err := w.UpdateClusterConfiguration(ctx, semver.MustParse("1.19.1"), w.UpdateImageRepositoryInKubeadmConfigMap(tt.newImageRepository)) g.Expect(err).ToNot(HaveOccurred()) var actualConfig corev1.ConfigMap @@ -1016,7 +1018,7 @@ func TestUpdateApiServerInKubeadmConfigMap(t *testing.T) { w := &Workload{ Client: fakeClient, } - err := w.UpdateAPIServerInKubeadmConfigMap(ctx, tt.newAPIServer, semver.MustParse("1.19.1")) + err := w.UpdateClusterConfiguration(ctx, semver.MustParse("1.19.1"), w.UpdateAPIServerInKubeadmConfigMap(tt.newAPIServer)) g.Expect(err).ToNot(HaveOccurred()) var actualConfig corev1.ConfigMap @@ -1092,7 +1094,7 @@ func TestUpdateControllerManagerInKubeadmConfigMap(t *testing.T) { w := &Workload{ Client: fakeClient, } - err := w.UpdateControllerManagerInKubeadmConfigMap(ctx, tt.newControllerManager, semver.MustParse("1.19.1")) + err := w.UpdateClusterConfiguration(ctx, semver.MustParse("1.19.1"), w.UpdateControllerManagerInKubeadmConfigMap(tt.newControllerManager)) g.Expect(err).ToNot(HaveOccurred()) var actualConfig corev1.ConfigMap @@ -1167,7 +1169,7 @@ func TestUpdateSchedulerInKubeadmConfigMap(t *testing.T) { w := &Workload{ Client: fakeClient, } - err := w.UpdateSchedulerInKubeadmConfigMap(ctx, tt.newScheduler, semver.MustParse("1.19.1")) + err := w.UpdateClusterConfiguration(ctx, semver.MustParse("1.19.1"), w.UpdateSchedulerInKubeadmConfigMap(tt.newScheduler)) g.Expect(err).ToNot(HaveOccurred()) var actualConfig corev1.ConfigMap @@ -1260,6 +1262,69 @@ func TestClusterStatus(t *testing.T) { } } +func TestUpdateFeatureGatesInKubeadmConfigMap(t *testing.T) { + tests := []struct { + name string + clusterConfigurationData string + newFeatureGates map[string]bool + wantFeatureGates map[string]bool + }{ + { + name: "it updates feature gates", + clusterConfigurationData: yaml.Raw(` + apiVersion: kubeadm.k8s.io/v1beta2 + kind: ClusterConfiguration`), + newFeatureGates: map[string]bool{"EtcdLearnerMode": true}, + wantFeatureGates: map[string]bool{"EtcdLearnerMode": true}, + }, + { + name: "it should preserve existing feature gates if new value is nil", + clusterConfigurationData: yaml.Raw(` + apiVersion: kubeadm.k8s.io/v1beta2 + kind: ClusterConfiguration + featureGates: + EtcdLearnerMode: true + `), + newFeatureGates: nil, + wantFeatureGates: map[string]bool{"EtcdLearnerMode": true}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithObjects(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmConfigKey, + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{ + clusterConfigurationKey: tt.clusterConfigurationData, + }, + }).Build() + + w := &Workload{ + Client: fakeClient, + } + err := w.UpdateClusterConfiguration(ctx, semver.MustParse("1.19.1"), w.UpdateFeatureGatesInKubeadmConfigMap(tt.newFeatureGates)) + g.Expect(err).ToNot(HaveOccurred()) + + var actualConfig corev1.ConfigMap + g.Expect(w.Client.Get( + ctx, + client.ObjectKey{Name: kubeadmConfigKey, Namespace: metav1.NamespaceSystem}, + &actualConfig, + )).To(Succeed()) + + marshal, err := yaml2.Marshal(tt.wantFeatureGates) + if err != nil { + return + } + g.Expect(actualConfig.Data[clusterConfigurationKey]).To(ContainSubstring(string(marshal))) + }) + } +} + func getProxyImageInfo(ctx context.Context, c client.Client) (string, error) { ds := &appsv1.DaemonSet{}