From 95097652f0432c200b7b313ac97665139226719b Mon Sep 17 00:00:00 2001 From: Nicole Morales Date: Fri, 17 May 2024 13:28:18 +0100 Subject: [PATCH] add feature flag, add e2e tests --- controllers/cluster_controller.go | 1 + .../replace_misconfigured_process_groups.go | 2 +- controllers/update_pods.go | 2 +- e2e/fixtures/fdb_cluster.go | 30 ++++++- e2e/fixtures/fdb_operator_client.go | 1 + e2e/test_operator/operator_test.go | 44 +++++++++ internal/replacements/replacements.go | 22 +++-- internal/replacements/replacements_test.go | 90 ++++++++++--------- setup/setup.go | 5 ++ 9 files changed, 145 insertions(+), 52 deletions(-) diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go index 6cefb27c9..7715c3dd9 100644 --- a/controllers/cluster_controller.go +++ b/controllers/cluster_controller.go @@ -62,6 +62,7 @@ type FoundationDBClusterReconciler struct { ServerSideApply bool EnableRecoveryState bool CacheDatabaseStatusForReconciliationDefault bool + ReplaceOnSecurityContextChange bool PodLifecycleManager podmanager.PodLifecycleManager PodClientProvider func(*fdbv1beta2.FoundationDBCluster, *corev1.Pod) (podclient.FdbPodClient, error) DatabaseClientProvider fdbadminclient.DatabaseClientProvider diff --git a/controllers/replace_misconfigured_process_groups.go b/controllers/replace_misconfigured_process_groups.go index 81e665e57..c97bfa2a9 100644 --- a/controllers/replace_misconfigured_process_groups.go +++ b/controllers/replace_misconfigured_process_groups.go @@ -46,7 +46,7 @@ func (c replaceMisconfiguredProcessGroups) reconcile(ctx context.Context, r *Fou return &requeue{curError: err} } - hasReplacements, err := replacements.ReplaceMisconfiguredProcessGroups(ctx, r.PodLifecycleManager, r, logger, cluster, internal.CreatePVCMap(cluster, pvcs)) + hasReplacements, err := replacements.ReplaceMisconfiguredProcessGroups(ctx, r.PodLifecycleManager, r, logger, cluster, internal.CreatePVCMap(cluster, pvcs), r.ReplaceOnSecurityContextChange) if err != nil { return &requeue{curError: err} } diff --git a/controllers/update_pods.go b/controllers/update_pods.go index 8b11e3523..309fa8f4c 100644 --- a/controllers/update_pods.go +++ b/controllers/update_pods.go @@ -207,7 +207,7 @@ func getPodsToUpdate(ctx context.Context, logger logr.Logger, reconciler *Founda continue } - needsRemoval, err := replacements.ProcessGroupNeedsRemoval(ctx, reconciler.PodLifecycleManager, reconciler, logger, cluster, processGroup, pvcMap) + needsRemoval, err := replacements.ProcessGroupNeedsRemoval(ctx, reconciler.PodLifecycleManager, reconciler, logger, cluster, processGroup, pvcMap, reconciler.ReplaceOnSecurityContextChange) // Do not update the Pod if unable to determine if it needs to be removed. if err != nil { logger.V(1).Info("Skip process group, error checking if it requires a removal", diff --git a/e2e/fixtures/fdb_cluster.go b/e2e/fixtures/fdb_cluster.go index 9c4a2664d..194a77da5 100644 --- a/e2e/fixtures/fdb_cluster.go +++ b/e2e/fixtures/fdb_cluster.go @@ -399,7 +399,7 @@ func (fdbCluster *FdbCluster) UpdateClusterSpec() { fdbCluster.UpdateClusterSpecWithSpec(fdbCluster.cluster.Spec.DeepCopy()) } -// UpdateClusterSpecWithSpec ensures that the FoundationDBCluster will be updated in Kubernetes. This method as a retry mechanism +// UpdateClusterSpecWithSpec ensures that the FoundationDBCluster will be updated in Kubernetes. This method has a retry mechanism // implemented and ensures that the provided (local) Spec matches the Spec in Kubernetes. You must make sure that you call // fdbCluster.GetCluster() before updating the spec, to make sure you are not overwriting the current state with an outdated state. // An example on how to update a field with this method: @@ -1117,6 +1117,34 @@ func (fdbCluster *FdbCluster) GetCustomParameters( return fdbCluster.cluster.Spec.Processes[processClass].CustomParameters } +// SetPodTemplateSpec allows to set the pod template spec of the provided process class. +func (fdbCluster *FdbCluster) SetPodTemplateSpec( + processClass fdbv1beta2.ProcessClass, + podTemplateSpec *corev1.PodSpec, + waitForReconcile bool, +) error { + setting, ok := fdbCluster.cluster.Spec.Processes[processClass] + if !ok { + return fmt.Errorf("could not find process settings for process class %s", processClass) + } + setting.PodTemplate.Spec = *podTemplateSpec + + fdbCluster.cluster.Spec.Processes[processClass] = setting + fdbCluster.UpdateClusterSpec() + if !waitForReconcile { + return nil + } + + return fdbCluster.WaitForReconciliation() +} + +// GetPodTemplateSpec returns the current pod template spec for the specified process class. +func (fdbCluster *FdbCluster) GetPodTemplateSpec( + processClass fdbv1beta2.ProcessClass, +) *corev1.PodSpec { + return &fdbCluster.cluster.Spec.Processes[processClass].PodTemplate.Spec +} + // CheckPodIsDeleted return true if Pod no longer exists at the executed time point func (fdbCluster *FdbCluster) CheckPodIsDeleted(podName string) bool { pod := &corev1.Pod{} diff --git a/e2e/fixtures/fdb_operator_client.go b/e2e/fixtures/fdb_operator_client.go index 7f42d5c74..f72ba5362 100644 --- a/e2e/fixtures/fdb_operator_client.go +++ b/e2e/fixtures/fdb_operator_client.go @@ -476,6 +476,7 @@ spec: - --minimum-recovery-time-for-exclusion=1.0 - --cluster-label-key-for-node-trigger=foundationdb.org/fdb-cluster-name - --enable-node-index + - --replace-on-security-context-change ` ) diff --git a/e2e/test_operator/operator_test.go b/e2e/test_operator/operator_test.go index 3b8d8e054..041a3b33e 100644 --- a/e2e/test_operator/operator_test.go +++ b/e2e/test_operator/operator_test.go @@ -428,6 +428,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() { fdbCluster.EnsurePodIsDeleted(replacedPod.Name) }) }) + }) When("setting storageServersPerPod", func() { @@ -1428,6 +1429,49 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() { }) }) + // TODO also test container level security context change + When("replacing Pods due to securityContext changes", func() { + var initialPods *corev1.PodList + var originalPodSpec, modifiedPodSpec *corev1.PodSpec + + BeforeEach(func() { + originalPodSpec = fdbCluster.GetPodTemplateSpec(fdbv1beta2.ProcessClassLog) + modifiedPodSpec = originalPodSpec.DeepCopy() + modifiedPodSpec.SecurityContext.FSGroupChangePolicy = &[]corev1.PodFSGroupChangePolicy{corev1.FSGroupChangeOnRootMismatch}[0] + initialPods = fdbCluster.GetLogPods() + Expect(fdbCluster.SetPodTemplateSpec(fdbv1beta2.ProcessClassLog, modifiedPodSpec, false)).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + Expect(fdbCluster.SetPodTemplateSpec(fdbv1beta2.ProcessClassLog, originalPodSpec, false)).NotTo(HaveOccurred()) + // TODO maybe test that it does the same for the revert? + }) + + It("should replace all the log pods (which had the security context change)", func() { + lastForcedReconciliationTime := time.Now() + forceReconcileDuration := 4 * time.Minute + Eventually(func() bool { + // Force a reconcile if needed to make sure we speed up the reconciliation if needed. + if time.Since(lastForcedReconciliationTime) >= forceReconcileDuration { + fdbCluster.ForceReconcile() + lastForcedReconciliationTime = time.Now() + } + + // check that all original pods are gone + pods := fdbCluster.GetLogPods() + Expect(pods.Items).NotTo(ContainElements(initialPods.Items)) + // TODO maybe instead check if all logs are marked for removal? unclear how long *all* would take? + //for _, processGroup := range fdbCluster.GetCluster().Status.ProcessGroups { + // if processGroup.IsMarkedForRemoval() && processGroup.IsExcluded() { + // continue + // } + // + //} + return true + }).WithTimeout(20 * time.Minute).WithPolling(5 * time.Second).Should(BeTrue()) + }) + }) + When("migrating a cluster to make use of DNS in the cluster file", func() { BeforeEach(func() { cluster := fdbCluster.GetCluster() diff --git a/internal/replacements/replacements.go b/internal/replacements/replacements.go index b10fdce31..c027076a4 100644 --- a/internal/replacements/replacements.go +++ b/internal/replacements/replacements.go @@ -40,7 +40,7 @@ import ( ) // ReplaceMisconfiguredProcessGroups checks if the cluster has any misconfigured process groups that must be replaced. -func ReplaceMisconfiguredProcessGroups(ctx context.Context, podManager podmanager.PodLifecycleManager, client client.Client, log logr.Logger, cluster *fdbv1beta2.FoundationDBCluster, pvcMap map[fdbv1beta2.ProcessGroupID]corev1.PersistentVolumeClaim) (bool, error) { +func ReplaceMisconfiguredProcessGroups(ctx context.Context, podManager podmanager.PodLifecycleManager, client client.Client, log logr.Logger, cluster *fdbv1beta2.FoundationDBCluster, pvcMap map[fdbv1beta2.ProcessGroupID]corev1.PersistentVolumeClaim, replaceOnSecurityContextChange bool) (bool, error) { hasReplacements := false maxReplacements, _ := getReplacementInformation(cluster, cluster.GetMaxConcurrentReplacements()) @@ -54,7 +54,7 @@ func ReplaceMisconfiguredProcessGroups(ctx context.Context, podManager podmanage continue } - needsRemoval, err := ProcessGroupNeedsRemoval(ctx, podManager, client, log, cluster, processGroup, pvcMap) + needsRemoval, err := ProcessGroupNeedsRemoval(ctx, podManager, client, log, cluster, processGroup, pvcMap, replaceOnSecurityContextChange) // Do not mark for removal if there is an error if err != nil { @@ -72,7 +72,7 @@ func ReplaceMisconfiguredProcessGroups(ctx context.Context, podManager podmanage } // ProcessGroupNeedsRemoval checks if a process group needs to be removed. -func ProcessGroupNeedsRemoval(ctx context.Context, podManager podmanager.PodLifecycleManager, client client.Client, log logr.Logger, cluster *fdbv1beta2.FoundationDBCluster, processGroup *fdbv1beta2.ProcessGroupStatus, pvcMap map[fdbv1beta2.ProcessGroupID]corev1.PersistentVolumeClaim) (bool, error) { +func ProcessGroupNeedsRemoval(ctx context.Context, podManager podmanager.PodLifecycleManager, client client.Client, log logr.Logger, cluster *fdbv1beta2.FoundationDBCluster, processGroup *fdbv1beta2.ProcessGroupStatus, pvcMap map[fdbv1beta2.ProcessGroupID]corev1.PersistentVolumeClaim, replaceOnSecurityContextChange bool) (bool, error) { // TODO(johscheuer): Fix how we fetch the pvc to make better use of the controller runtime cache. pvc, hasPVC := pvcMap[processGroup.ProcessGroupID] pod, podErr := podManager.GetPod(ctx, client, cluster, processGroup.GetPodName(cluster)) @@ -96,7 +96,7 @@ func ProcessGroupNeedsRemoval(ctx context.Context, podManager podmanager.PodLife return false, podErr } - return processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + return processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, replaceOnSecurityContextChange) } func processGroupNeedsRemovalForPVC(cluster *fdbv1beta2.FoundationDBCluster, pvc corev1.PersistentVolumeClaim, log logr.Logger, processGroup *fdbv1beta2.ProcessGroupStatus) (bool, error) { @@ -140,7 +140,7 @@ func processGroupNeedsRemovalForPVC(cluster *fdbv1beta2.FoundationDBCluster, pvc return false, nil } -func processGroupNeedsRemovalForPod(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, processGroupStatus *fdbv1beta2.ProcessGroupStatus, log logr.Logger) (bool, error) { +func processGroupNeedsRemovalForPod(cluster *fdbv1beta2.FoundationDBCluster, pod *corev1.Pod, processGroupStatus *fdbv1beta2.ProcessGroupStatus, log logr.Logger, replaceOnSecurityContextChange bool) (bool, error) { if pod == nil { return false, nil } @@ -251,10 +251,14 @@ func processGroupNeedsRemovalForPod(cluster *fdbv1beta2.FoundationDBCluster, pod if err != nil { return false, err } - // TODO deprecated builtin k8s features edited securityContext automatically, and it doesn't seem outlandish that someone's cluster - // could use it or a similar feature, and it would result in constant replacements with no solution unless we feature - // guard this... (https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/) - return fileSecurityContextChanged(desiredPod, pod, log), nil + // Some k8s instances have security context vetting which may edit the spec automatically. + // This would cause changes to security context on a pod or container + // to constantly be seen as having a security context change, hence we want to feature guard this. + // https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/ + if replaceOnSecurityContextChange { + return fileSecurityContextChanged(desiredPod, pod, log), nil + } + return false, nil } func resourcesNeedsReplacement(desired []corev1.Container, current []corev1.Container) bool { diff --git a/internal/replacements/replacements_test.go b/internal/replacements/replacements_test.go index 5488860e2..559b38ecc 100644 --- a/internal/replacements/replacements_test.go +++ b/internal/replacements/replacements_test.go @@ -98,7 +98,7 @@ var _ = Describe("replace_misconfigured_pods", func() { Describe("Check process group", func() { When("process group has no Pod", func() { It("should not need removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, nil, nil, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, nil, nil, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -111,7 +111,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -125,13 +125,13 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) // Change the process group ID should trigger a removal cluster.Spec.ProcessGroupIDPrefix = "test" - needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -144,13 +144,13 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) // Change the process group ID should trigger a removal cluster.Spec.ProcessGroupIDPrefix = "test" - needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -164,13 +164,13 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) ipSource := fdbv1beta2.PublicIPSourceService cluster.Spec.Routing.PublicIPSource = &ipSource - needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -191,12 +191,12 @@ var _ = Describe("replace_misconfigured_pods", func() { ipSource := fdbv1beta2.PublicIPSourceService cluster.Spec.Routing.PublicIPSource = &ipSource - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) cluster.Spec.Routing.PublicIPSource = nil - needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -209,13 +209,13 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) ipSource := fdbv1beta2.PublicIPSourcePod cluster.Spec.Routing.PublicIPSource = &ipSource - needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -228,12 +228,12 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) cluster.Spec.StorageServersPerPod = 2 - needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -246,12 +246,12 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) cluster.Spec.StorageServersPerPod = 2 - needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -264,14 +264,14 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) cluster.Spec.Processes[fdbv1beta2.ProcessClassGeneral].PodTemplate.Spec.NodeSelector = map[string]string{ "dummy": "test", } - needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err = processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -290,7 +290,7 @@ var _ = Describe("replace_misconfigured_pods", func() { pod.Spec.NodeSelector = map[string]string{ "dummy": "test", } - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -308,7 +308,7 @@ var _ = Describe("replace_misconfigured_pods", func() { pod.Spec.SecurityContext = &corev1.PodSecurityContext{RunAsUser: new(int64)} cluster.Spec.AutomationOptions.PodUpdateStrategy = fdbv1beta2.PodUpdateStrategyReplacement - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -318,17 +318,27 @@ var _ = Describe("replace_misconfigured_pods", func() { pod.Spec.SecurityContext = &corev1.PodSecurityContext{FSGroup: new(int64)} cluster.Spec.AutomationOptions.PodUpdateStrategy = fdbv1beta2.PodUpdateStrategyReplacement - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) It("should need a removal with ReplaceInstancesWhenResourcesChange (even with no explicit spec change)", func() { pod.Spec.SecurityContext = &corev1.PodSecurityContext{RunAsUser: new(int64)} cluster.Spec.AutomationOptions.PodUpdateStrategy = fdbv1beta2.PodUpdateStrategyReplacement - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) + It("with replaceOnSecurityContextChange false, it should not need a removal for FSGroup change", func() { + // if ReplaceInstancesWhenResourcesChange is true, any spec change should result in replacement + cluster.Spec.ReplaceInstancesWhenResourcesChange = new(bool) + + pod.Spec.SecurityContext = &corev1.PodSecurityContext{FSGroup: new(int64)} + cluster.Spec.AutomationOptions.PodUpdateStrategy = fdbv1beta2.PodUpdateStrategyReplacement + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, false) + Expect(needsRemoval).To(BeFalse()) + Expect(err).NotTo(HaveOccurred()) + }) }) @@ -342,7 +352,7 @@ var _ = Describe("replace_misconfigured_pods", func() { pod.Spec = corev1.PodSpec{ Containers: []corev1.Container{{}}, } - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -357,7 +367,7 @@ var _ = Describe("replace_misconfigured_pods", func() { It("should need a removal", func() { pod.ObjectMeta.Annotations[fdbv1beta2.LastSpecKey] = "-1" cluster.Spec.AutomationOptions.PodUpdateStrategy = fdbv1beta2.PodUpdateStrategyReplacement - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -372,7 +382,7 @@ var _ = Describe("replace_misconfigured_pods", func() { It("should not need a removal", func() { pod.ObjectMeta.Annotations[fdbv1beta2.LastSpecKey] = "-1" cluster.Spec.AutomationOptions.PodUpdateStrategy = fdbv1beta2.PodUpdateStrategyTransactionReplacement - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -387,7 +397,7 @@ var _ = Describe("replace_misconfigured_pods", func() { It("should need a removal", func() { pod.ObjectMeta.Annotations[fdbv1beta2.LastSpecKey] = "-1" cluster.Spec.AutomationOptions.PodUpdateStrategy = fdbv1beta2.PodUpdateStrategyTransactionReplacement - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, processGroup, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -439,7 +449,7 @@ var _ = Describe("replace_misconfigured_pods", func() { ProcessClass: fdbv1beta2.ProcessClassStorage, } - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -461,7 +471,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -479,7 +489,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -497,7 +507,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -515,7 +525,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -536,7 +546,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log, true) Expect(needsRemoval).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) }) @@ -560,7 +570,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -578,7 +588,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -596,7 +606,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -614,7 +624,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not need a removal", func() { - needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log) + needsRemoval, err := processGroupNeedsRemovalForPod(cluster, pod, status, log, true) Expect(needsRemoval).To(BeFalse()) Expect(err).NotTo(HaveOccurred()) }) @@ -674,7 +684,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not have a replacements", func() { - hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap) + hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap, true) Expect(err).NotTo(HaveOccurred()) Expect(hasReplacement).To(BeFalse()) @@ -697,7 +707,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should have two replacements", func() { - hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap) + hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap, true) Expect(err).NotTo(HaveOccurred()) Expect(hasReplacement).To(BeTrue()) @@ -716,7 +726,7 @@ var _ = Describe("replace_misconfigured_pods", func() { When("Setting is unset", func() { It("should replace all process groups", func() { - hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap) + hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap, true) Expect(err).NotTo(HaveOccurred()) Expect(hasReplacement).To(BeTrue()) @@ -761,7 +771,7 @@ var _ = Describe("replace_misconfigured_pods", func() { }) It("should not have any replacements", func() { - hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap) + hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap, true) Expect(err).NotTo(HaveOccurred()) Expect(hasReplacement).To(BeFalse()) diff --git a/setup/setup.go b/setup/setup.go index 29707f857..5489d363c 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -65,6 +65,7 @@ type Options struct { EnableRecoveryState bool CacheDatabaseStatus bool EnableNodeIndex bool + ReplaceOnSecurityContextChange bool MetricsAddr string LeaderElectionID string LogFile string @@ -139,6 +140,9 @@ func (o *Options) BindFlags(fs *flag.FlagSet) { fs.BoolVar(&o.EnableRecoveryState, "enable-recovery-state", true, "This flag enables the use of the recovery state for the minimum uptime between bounced if the FDB version supports it.") fs.BoolVar(&o.CacheDatabaseStatus, "cache-database-status", true, "Defines the default value for caching the database status.") fs.BoolVar(&o.EnableNodeIndex, "enable-node-index", false, "Deprecated, not used anymore. Defines if the operator should add an index for accessing node objects. This requires a ClusterRoleBinding with node access. If the taint feature should be used, this setting should be set to true.") + fs.BoolVar(&o.ReplaceOnSecurityContextChange, "replace-on-security-context-change", false, "This flag enables the operator"+ + " to automatically replace pods whose effective security context has one of the following fields change: "+ + "FSGroup, FSGroupPolicy, RunAsGroup, RunAsUser") fs.Float64Var(&o.MinimumRecoveryTimeForInclusion, "minimum-recovery-time-for-inclusion", 600.0, "Defines the minimum uptime of the cluster before inclusions are allowed. For clusters after 7.1 this will use the recovery state. This should reduce the risk of frequent recoveries because of inclusions.") fs.Float64Var(&o.MinimumRecoveryTimeForExclusion, "minimum-recovery-time-for-exclusion", 120.0, "Defines the minimum uptime of the cluster before exclusions are allowed. For clusters after 7.1 this will use the recovery state. This should reduce the risk of frequent recoveries because of exclusions.") } @@ -258,6 +262,7 @@ func StartManager( clusterReconciler.ServerSideApply = operatorOpts.ServerSideApply clusterReconciler.EnableRecoveryState = operatorOpts.EnableRecoveryState clusterReconciler.CacheDatabaseStatusForReconciliationDefault = operatorOpts.CacheDatabaseStatus + clusterReconciler.ReplaceOnSecurityContextChange = operatorOpts.ReplaceOnSecurityContextChange clusterReconciler.MinimumRequiredUptimeCCBounce = operatorOpts.MinimumRequiredUptimeCCBounce clusterReconciler.MaintenanceListStaleDuration = operatorOpts.MaintenanceListStaleDuration clusterReconciler.MaintenanceListWaitDuration = operatorOpts.MaintenanceListWaitDuration