From 688289a8d2ff7862971ebfeede0479c1b3b3e5f1 Mon Sep 17 00:00:00 2001 From: Roy Paulin Date: Fri, 19 Apr 2024 12:09:40 +0200 Subject: [PATCH] Update upgrade reconciler to handle subclusters in sandbox (#772) This change will modify the upgrade reconcilers, both offline and online, to function in either the main cluster or a sandbox. Our immediate plan is to use the offline upgrade reconciler within the sandbox controller, although this will be done in a follow-on task. --- pkg/builder/builder.go | 2 +- pkg/builder/labels_annotations.go | 11 ++ pkg/controllers/vdb/crashloop_reconciler.go | 2 +- .../vdb/dbremovesubcluster_reconciler.go | 2 +- pkg/controllers/vdb/obj_reconciler.go | 5 +- .../vdb/offlineupgrade_reconciler.go | 38 ++++-- .../vdb/onlineupgrade_reconciler.go | 39 +++--- .../vdb/onlineupgrade_reconciler_test.go | 2 +- pkg/controllers/vdb/podfacts.go | 11 ++ .../vdb/replicatedupgrade_reconciler.go | 14 +- pkg/controllers/vdb/revivedb_reconciler.go | 2 +- pkg/controllers/vdb/upgrade.go | 121 +++++++++++++----- pkg/controllers/vdb/upgrade_test.go | 42 +++++- .../vdb/upgradeoperator120_reconciler.go | 2 +- pkg/iter/sc_finder.go | 71 +++++----- pkg/iter/sc_finder_test.go | 44 ++++--- pkg/meta/labels.go | 6 +- 17 files changed, 282 insertions(+), 132 deletions(-) diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 3470d4af8..c049f5d99 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -1312,7 +1312,7 @@ func BuildPod(vdb *vapi.VerticaDB, sc *vapi.Subcluster, podIndex int32) *corev1. ObjectMeta: metav1.ObjectMeta{ Name: nm.Name, Namespace: nm.Namespace, - Labels: MakeLabelsForPodObject(vdb, sc), + Labels: MakeLabelsForSandboxPodObject(vdb, sc), Annotations: MakeAnnotationsForObject(vdb), }, Spec: buildPodSpec(vdb, sc), diff --git a/pkg/builder/labels_annotations.go b/pkg/builder/labels_annotations.go index 77a4e0365..45f7e7e2c 100644 --- a/pkg/builder/labels_annotations.go +++ b/pkg/builder/labels_annotations.go @@ -91,6 +91,17 @@ func MakeLabelsForPodObject(vdb *vapi.VerticaDB, sc *vapi.Subcluster) map[string return makeLabelsForObject(vdb, sc, true) } +// MakeLabelsForSandboxPodObject constructs the labels that are common for all pods plus +// the sandbox name label. It is for testing purposes. +func MakeLabelsForSandboxPodObject(vdb *vapi.VerticaDB, sc *vapi.Subcluster) map[string]string { + labels := makeLabelsForObject(vdb, sc, true) + sandbox := vdb.GetSubclusterSandboxName(sc.Name) + if sandbox != vapi.MainCluster { + labels[vmeta.SandboxNameLabel] = sandbox + } + return labels +} + // MakeLabelsForStsObject constructs the labels that are common for all statefulsets. func MakeLabelsForStsObject(vdb *vapi.VerticaDB, sc *vapi.Subcluster) map[string]string { labels := makeLabelsForObject(vdb, sc, false) diff --git a/pkg/controllers/vdb/crashloop_reconciler.go b/pkg/controllers/vdb/crashloop_reconciler.go index 6827c1789..3d94515d7 100644 --- a/pkg/controllers/vdb/crashloop_reconciler.go +++ b/pkg/controllers/vdb/crashloop_reconciler.go @@ -67,7 +67,7 @@ func (c *CrashLoopReconciler) Reconcile(ctx context.Context, _ *ctrl.Request) (c func (c *CrashLoopReconciler) reconcileStatefulSets(ctx context.Context) { finder := iter.MakeSubclusterFinder(c.VRec.Client, c.VDB) - stss, err := finder.FindStatefulSets(ctx, iter.FindExisting|iter.FindSorted) + stss, err := finder.FindStatefulSets(ctx, iter.FindExisting|iter.FindSorted, vapi.MainCluster) if err != nil { // This reconciler is a best effort. It only tries to surface meaningful // error messages based on the events it see. For this reason, no errors diff --git a/pkg/controllers/vdb/dbremovesubcluster_reconciler.go b/pkg/controllers/vdb/dbremovesubcluster_reconciler.go index 9296cbc14..fd81823e3 100644 --- a/pkg/controllers/vdb/dbremovesubcluster_reconciler.go +++ b/pkg/controllers/vdb/dbremovesubcluster_reconciler.go @@ -159,7 +159,7 @@ func (d *DBRemoveSubclusterReconciler) resetDefaultSubcluster(ctx context.Contex scFinder := iter.MakeSubclusterFinder(d.VRec.Client, d.Vdb) // We use the FindServices() API to get subclusters that already exist. // We can only change the default subcluster to one of those. - svcs, err := scFinder.FindServices(ctx, iter.FindInVdb) + svcs, err := scFinder.FindServices(ctx, iter.FindInVdb, vapi.MainCluster) if err != nil { return err } diff --git a/pkg/controllers/vdb/obj_reconciler.go b/pkg/controllers/vdb/obj_reconciler.go index b3fac4ea4..d34eb809f 100644 --- a/pkg/controllers/vdb/obj_reconciler.go +++ b/pkg/controllers/vdb/obj_reconciler.go @@ -237,8 +237,9 @@ func (o *ObjReconciler) checkForDeletedSubcluster(ctx context.Context) (ctrl.Res finder := iter.MakeSubclusterFinder(o.VRec.Client, o.Vdb) + sandbox := o.PFacts.GetSandboxName() // Find any statefulsets that need to be deleted - stss, err := finder.FindStatefulSets(ctx, iter.FindNotInVdb) + stss, err := finder.FindStatefulSets(ctx, iter.FindNotInVdb, sandbox) if err != nil { return ctrl.Result{}, err } @@ -258,7 +259,7 @@ func (o *ObjReconciler) checkForDeletedSubcluster(ctx context.Context) (ctrl.Res } // Find any service objects that need to be deleted - svcs, err := finder.FindServices(ctx, iter.FindNotInVdb) + svcs, err := finder.FindServices(ctx, iter.FindNotInVdb, vapi.MainCluster) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/controllers/vdb/offlineupgrade_reconciler.go b/pkg/controllers/vdb/offlineupgrade_reconciler.go index 1c7b98328..75545f493 100644 --- a/pkg/controllers/vdb/offlineupgrade_reconciler.go +++ b/pkg/controllers/vdb/offlineupgrade_reconciler.go @@ -63,8 +63,8 @@ var OfflineUpgradeStatusMsgs = []string{ } // MakeOfflineUpgradeReconciler will build an OfflineUpgradeReconciler object -func MakeOfflineUpgradeReconciler(vdbrecon *VerticaDBReconciler, log logr.Logger, - vdb *vapi.VerticaDB, prunner cmds.PodRunner, pfacts *PodFacts, dispatcher vadmin.Dispatcher) controllers.ReconcileActor { +func MakeOfflineUpgradeReconciler(vdbrecon *VerticaDBReconciler, log logr.Logger, vdb *vapi.VerticaDB, + prunner cmds.PodRunner, pfacts *PodFacts, dispatcher vadmin.Dispatcher) controllers.ReconcileActor { return &OfflineUpgradeReconciler{ VRec: vdbrecon, Log: log.WithName("OfflineUpgradeReconciler"), @@ -80,7 +80,12 @@ func MakeOfflineUpgradeReconciler(vdbrecon *VerticaDBReconciler, log logr.Logger // Reconcile will handle the process of the vertica image changing. For // example, this can automate the process for an upgrade. func (o *OfflineUpgradeReconciler) Reconcile(ctx context.Context, _ *ctrl.Request) (ctrl.Result, error) { - if ok, err := o.Manager.IsUpgradeNeeded(ctx); !ok || err != nil { + sandbox := o.PFacts.GetSandboxName() + if ok, err := o.Manager.IsUpgradeNeeded(ctx, sandbox); !ok || err != nil { + return ctrl.Result{}, err + } + + if err := o.Manager.logUpgradeStarted(sandbox); err != nil { return ctrl.Result{}, err } @@ -131,7 +136,7 @@ func (o *OfflineUpgradeReconciler) Reconcile(ctx context.Context, _ *ctrl.Reques } } - return ctrl.Result{}, nil + return ctrl.Result{}, o.Manager.logUpgradeSucceeded(sandbox) } // logEventIfOnlineUpgradeRequested will log an event if the vdb has @@ -203,7 +208,8 @@ func (o *OfflineUpgradeReconciler) postReschedulePodsMsg(ctx context.Context) (c // Since there will be processing after to delete the pods so that they come up // with the new image. func (o *OfflineUpgradeReconciler) updateImageInStatefulSets(ctx context.Context) (ctrl.Result, error) { - numStsChanged, err := o.Manager.updateImageInStatefulSets(ctx) + sandbox := o.PFacts.GetSandboxName() + numStsChanged, err := o.Manager.updateImageInStatefulSets(ctx, sandbox) if numStsChanged > 0 { o.PFacts.Invalidate() } @@ -215,7 +221,8 @@ func (o *OfflineUpgradeReconciler) updateImageInStatefulSets(ctx context.Context // the sts is OnDelete. Deleting the pods ensures they get rescheduled with the // new image. func (o *OfflineUpgradeReconciler) deletePods(ctx context.Context) (ctrl.Result, error) { - numPodsDeleted, err := o.Manager.deletePodsRunningOldImage(ctx, "") + sandbox := o.PFacts.GetSandboxName() + numPodsDeleted, err := o.Manager.deletePodsRunningOldImage(ctx, "", sandbox) if numPodsDeleted > 0 { o.PFacts.Invalidate() } @@ -228,7 +235,8 @@ func (o *OfflineUpgradeReconciler) checkNMADeploymentChange(ctx context.Context) return ctrl.Result{}, nil } - stss, err := o.Finder.FindStatefulSets(ctx, iter.FindExisting) + sandbox := o.PFacts.GetSandboxName() + stss, err := o.Finder.FindStatefulSets(ctx, iter.FindExisting, sandbox) if err != nil { return ctrl.Result{}, err } @@ -251,7 +259,12 @@ func (o *OfflineUpgradeReconciler) checkNMADeploymentChange(ctx context.Context) // only occur if there is at least one pod that exists. func (o *OfflineUpgradeReconciler) checkForNewPods(ctx context.Context) (ctrl.Result, error) { foundPodWithNewImage := false - pods, err := o.Finder.FindPods(ctx, iter.FindExisting, vapi.MainCluster) + sandbox := o.PFacts.GetSandboxName() + pods, err := o.Finder.FindPods(ctx, iter.FindExisting, sandbox) + if err != nil { + return ctrl.Result{}, err + } + targetImage, err := o.Manager.getTargetImage(sandbox) if err != nil { return ctrl.Result{}, err } @@ -261,7 +274,7 @@ func (o *OfflineUpgradeReconciler) checkForNewPods(ctx context.Context) (ctrl.Re if err != nil { return ctrl.Result{}, err } - if cntImage == o.Vdb.Spec.Image { + if cntImage == targetImage { foundPodWithNewImage = true break } @@ -342,6 +355,11 @@ func (o *OfflineUpgradeReconciler) addClientRoutingLabel(ctx context.Context) (c // anyPodsRunningWithOldImage will check if any upNode pods are running with the old image. func (o *OfflineUpgradeReconciler) anyPodsRunningWithOldImage(ctx context.Context) (bool, error) { + sandbox := o.PFacts.GetSandboxName() + targetImage, err := o.Manager.getTargetImage(sandbox) + if err != nil { + return false, err + } for pn, pf := range o.PFacts.Detail { if !pf.upNode { continue @@ -359,7 +377,7 @@ func (o *OfflineUpgradeReconciler) anyPodsRunningWithOldImage(ctx context.Contex if err != nil { return false, err } - if cntImage != o.Vdb.Spec.Image { + if cntImage != targetImage { return true, nil } } diff --git a/pkg/controllers/vdb/onlineupgrade_reconciler.go b/pkg/controllers/vdb/onlineupgrade_reconciler.go index 3d0ba0e39..d7a9d003a 100644 --- a/pkg/controllers/vdb/onlineupgrade_reconciler.go +++ b/pkg/controllers/vdb/onlineupgrade_reconciler.go @@ -58,13 +58,14 @@ type OnlineUpgradeReconciler struct { func MakeOnlineUpgradeReconciler(vdbrecon *VerticaDBReconciler, log logr.Logger, vdb *vapi.VerticaDB, prunner cmds.PodRunner, pfacts *PodFacts, dispatcher vadmin.Dispatcher) controllers.ReconcileActor { return &OnlineUpgradeReconciler{ - VRec: vdbrecon, - Log: log.WithName("OnlineUpgradeReconciler"), - Vdb: vdb, - PRunner: prunner, - PFacts: pfacts, - Finder: iter.MakeSubclusterFinder(vdbrecon.Client, vdb), - Manager: *MakeUpgradeManager(vdbrecon, log, vdb, vapi.OnlineUpgradeInProgress, onlineUpgradeAllowed), + VRec: vdbrecon, + Log: log.WithName("OnlineUpgradeReconciler"), + Vdb: vdb, + PRunner: prunner, + PFacts: pfacts, + Finder: iter.MakeSubclusterFinder(vdbrecon.Client, vdb), + Manager: *MakeUpgradeManager(vdbrecon, log, vdb, vapi.OnlineUpgradeInProgress, + onlineUpgradeAllowed), Dispatcher: dispatcher, } } @@ -72,10 +73,14 @@ func MakeOnlineUpgradeReconciler(vdbrecon *VerticaDBReconciler, log logr.Logger, // Reconcile will handle the process of the vertica image changing. For // example, this can automate the process for an upgrade. func (o *OnlineUpgradeReconciler) Reconcile(ctx context.Context, _ *ctrl.Request) (ctrl.Result, error) { - if ok, err := o.Manager.IsUpgradeNeeded(ctx); !ok || err != nil { + sandbox := o.PFacts.GetSandboxName() + if ok, err := o.Manager.IsUpgradeNeeded(ctx, sandbox); !ok || err != nil { return ctrl.Result{}, err } + if err := o.Manager.logUpgradeStarted(sandbox); err != nil { + return ctrl.Result{}, err + } // Functions to perform when the image changes. Order matters. funcs := []func(context.Context) (ctrl.Result, error){ // Initiate an upgrade by setting condition and event recording @@ -122,7 +127,7 @@ func (o *OnlineUpgradeReconciler) Reconcile(ctx context.Context, _ *ctrl.Request } } - return ctrl.Result{}, nil + return ctrl.Result{}, o.Manager.logUpgradeSucceeded(sandbox) } // loadSubclusterState will load state into the OnlineUpgradeReconciler that @@ -135,8 +140,8 @@ func (o *OnlineUpgradeReconciler) loadSubclusterState(ctx context.Context) (ctrl } o.TransientSc = o.Vdb.FindTransientSubcluster() - - err = o.Manager.cachePrimaryImages(ctx) + sandbox := o.PFacts.GetSandboxName() + err = o.Manager.cachePrimaryImages(ctx, sandbox) return ctrl.Result{}, err } @@ -202,7 +207,8 @@ func (o *OnlineUpgradeReconciler) addTransientToVdb(ctx context.Context) (ctrl.R return ctrl.Result{}, nil } - oldImage, ok := o.Manager.fetchOldImage() + sandbox := o.PFacts.GetSandboxName() + oldImage, ok := o.Manager.fetchOldImage(sandbox) if !ok { return ctrl.Result{}, fmt.Errorf("could not determine the old image name. "+ "Only available image is %s", o.Vdb.Spec.Image) @@ -327,7 +333,8 @@ func (o *OnlineUpgradeReconciler) addClientRoutingLabelToTransientNodes(ctx cont // processFunc for each one that matches the given type. func (o *OnlineUpgradeReconciler) iterateSubclusterType(ctx context.Context, scType string, processFunc func(context.Context, *appsv1.StatefulSet) (ctrl.Result, error)) (ctrl.Result, error) { - stss, err := o.Finder.FindStatefulSets(ctx, iter.FindExisting|iter.FindSorted) + sandbox := o.PFacts.GetSandboxName() + stss, err := o.Finder.FindStatefulSets(ctx, iter.FindExisting|iter.FindSorted, sandbox) if err != nil { return ctrl.Result{}, err } @@ -454,7 +461,8 @@ func (o *OnlineUpgradeReconciler) recreateSubclusterWithNewImage(ctx context.Con } scName := sts.Labels[vmeta.SubclusterNameLabel] - podsDeleted, err := o.Manager.deletePodsRunningOldImage(ctx, scName) + sandbox := o.PFacts.GetSandboxName() + podsDeleted, err := o.Manager.deletePodsRunningOldImage(ctx, scName, sandbox) if err != nil { return ctrl.Result{}, err } @@ -498,6 +506,7 @@ func (o *OnlineUpgradeReconciler) checkVersion(ctx context.Context, sts *appsv1. } func (o *OnlineUpgradeReconciler) handleDeploymentChange(ctx context.Context, _ *appsv1.StatefulSet) (ctrl.Result, error) { + sandbox := o.PFacts.GetSandboxName() // We need to check if we are changing deployment types. This isn't allowed // for online upgrade because the vclusterops library won't know how to talk // to the pods that are still running the old admintools deployment since it @@ -508,7 +517,7 @@ func (o *OnlineUpgradeReconciler) handleDeploymentChange(ctx context.Context, _ if primaryRunningVClusterOps > 0 && secondaryRunningAdmintools > 0 { o.Log.Info("online upgrade isn't supported when changing deployment types from admintools to vclusterops", "primaryRunningVClusterOps", primaryRunningVClusterOps, "secondaryRunningAdmintools", secondaryRunningAdmintools) - if err := o.Manager.deleteStsRunningOldImage(ctx); err != nil { + if err := o.Manager.deleteStsRunningOldImage(ctx, sandbox); err != nil { return ctrl.Result{}, err } } diff --git a/pkg/controllers/vdb/onlineupgrade_reconciler_test.go b/pkg/controllers/vdb/onlineupgrade_reconciler_test.go index aa574b62c..76a17cf0f 100644 --- a/pkg/controllers/vdb/onlineupgrade_reconciler_test.go +++ b/pkg/controllers/vdb/onlineupgrade_reconciler_test.go @@ -113,7 +113,7 @@ var _ = Describe("onlineupgrade_reconcile", func() { r := createOnlineUpgradeReconciler(ctx, vdb) Expect(r.loadSubclusterState(ctx)).Should(Equal(ctrl.Result{})) - oldImage, ok := r.Manager.fetchOldImage() + oldImage, ok := r.Manager.fetchOldImage(vapi.MainCluster) Expect(ok).Should(BeTrue()) Expect(oldImage).Should(Equal(OldImage)) }) diff --git a/pkg/controllers/vdb/podfacts.go b/pkg/controllers/vdb/podfacts.go index bcc9d47dc..32e2f699a 100644 --- a/pkg/controllers/vdb/podfacts.go +++ b/pkg/controllers/vdb/podfacts.go @@ -1141,6 +1141,17 @@ func (p *PodFacts) findExpectedNodeNames() []string { return expectedNodeNames } +// GetSandboxName returns the name of the sandbox, or empty string +// for main cluster, the pods belong to +func (p *PodFacts) GetSandboxName() string { + for _, v := range p.Detail { + // all pods in the podfacts belong to either + // the same sandbox or the main cluster + return v.sandbox + } + return "" +} + // setSandboxNodeType sets the isPrimary state for a sandboxed // subcluster's node func setSandboxNodeType(pf *PodFact) { diff --git a/pkg/controllers/vdb/replicatedupgrade_reconciler.go b/pkg/controllers/vdb/replicatedupgrade_reconciler.go index 09028feac..9b6da4ea3 100644 --- a/pkg/controllers/vdb/replicatedupgrade_reconciler.go +++ b/pkg/controllers/vdb/replicatedupgrade_reconciler.go @@ -68,7 +68,7 @@ func MakeReplicatedUpgradeReconciler(vdbrecon *VerticaDBReconciler, log logr.Log // Reconcile will automate the process of a replicated upgrade. func (r *ReplicatedUpgradeReconciler) Reconcile(ctx context.Context, _ *ctrl.Request) (ctrl.Result, error) { - if ok, err := r.Manager.IsUpgradeNeeded(ctx); !ok || err != nil { + if ok, err := r.Manager.IsUpgradeNeeded(ctx, vapi.MainCluster); !ok || err != nil { return ctrl.Result{}, err } @@ -76,6 +76,10 @@ func (r *ReplicatedUpgradeReconciler) Reconcile(ctx context.Context, _ *ctrl.Req return ctrl.Result{}, err } + if err := r.Manager.logUpgradeStarted(vapi.MainCluster); err != nil { + return ctrl.Result{}, err + } + // Functions to perform when the image changes. Order matters. funcs := []func(context.Context) (ctrl.Result, error){ // Initiate an upgrade by setting condition and event recording @@ -127,13 +131,13 @@ func (r *ReplicatedUpgradeReconciler) Reconcile(ctx context.Context, _ *ctrl.Req } } - return ctrl.Result{}, nil + return ctrl.Result{}, r.Manager.logUpgradeSucceeded(vapi.MainCluster) } // loadUpgradeState will load state into the reconciler that // is used in subsequent steps. func (r *ReplicatedUpgradeReconciler) loadUpgradeState(ctx context.Context) (ctrl.Result, error) { - err := r.Manager.cachePrimaryImages(ctx) + err := r.Manager.cachePrimaryImages(ctx, vapi.MainCluster) if err != nil { return ctrl.Result{}, err } @@ -493,7 +497,7 @@ func (r *ReplicatedUpgradeReconciler) scaleOutSecondariesInReplicaGroupB(ctx con // be added directly to r.VDB. This is a callback function for // updateVDBWithRetry to prepare the vdb for update. func (r *ReplicatedUpgradeReconciler) addNewSubclustersForPrimaries() (bool, error) { - oldImage, found := r.Manager.fetchOldImage() + oldImage, found := r.Manager.fetchOldImage(vapi.MainCluster) if !found { return false, errors.New("Could not find old image needed for new subclusters") } @@ -547,7 +551,7 @@ func (r *ReplicatedUpgradeReconciler) assignSubclustersToReplicaGroupACallback() // replica group B into the sandbox. This is a callback function for // updateVDBWithRetry to prepare the vdb for an update. func (r *ReplicatedUpgradeReconciler) moveReplicaGroupBSubclusterToSandbox() (bool, error) { - oldImage, found := r.Manager.fetchOldImage() + oldImage, found := r.Manager.fetchOldImage(vapi.MainCluster) if !found { return false, errors.New("Could not find old image") } diff --git a/pkg/controllers/vdb/revivedb_reconciler.go b/pkg/controllers/vdb/revivedb_reconciler.go index c3f11d8a7..cdc76e950 100644 --- a/pkg/controllers/vdb/revivedb_reconciler.go +++ b/pkg/controllers/vdb/revivedb_reconciler.go @@ -286,7 +286,7 @@ func (r *ReviveDBReconciler) genDescribeOpts(initiatorPod types.NamespacedName, func (r *ReviveDBReconciler) deleteRevisionPendingSts(ctx context.Context) (ctrl.Result, error) { numStsDeleted := 0 finder := iter.MakeSubclusterFinder(r.VRec.Client, r.Vdb) - stss, err := finder.FindStatefulSets(ctx, iter.FindInVdb) + stss, err := finder.FindStatefulSets(ctx, iter.FindInVdb, vapi.MainCluster) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/controllers/vdb/upgrade.go b/pkg/controllers/vdb/upgrade.go index 9caf9fdb9..d8fbd869f 100644 --- a/pkg/controllers/vdb/upgrade.go +++ b/pkg/controllers/vdb/upgrade.go @@ -67,7 +67,7 @@ func MakeUpgradeManager(vdbrecon *VerticaDBReconciler, log logr.Logger, vdb *vap // IsUpgradeNeeded checks whether an upgrade is needed and/or in // progress. It will return true for the first parm if an upgrade should // proceed. -func (i *UpgradeManager) IsUpgradeNeeded(ctx context.Context) (bool, error) { +func (i *UpgradeManager) IsUpgradeNeeded(ctx context.Context, sandbox string) (bool, error) { // no-op for ScheduleOnly init policy if i.Vdb.Spec.InitPolicy == vapi.CommunalInitPolicyScheduleOnly { return false, nil @@ -81,7 +81,7 @@ func (i *UpgradeManager) IsUpgradeNeeded(ctx context.Context) (bool, error) { return ok, nil } - return i.isVDBImageDifferent(ctx) + return i.isVDBImageDifferent(ctx, sandbox) } // isUpgradeInProgress returns true if state indicates that an upgrade @@ -96,9 +96,14 @@ func (i *UpgradeManager) isUpgradeInProgress() bool { } // isVDBImageDifferent will check if an upgrade is needed based on the -// image being different between the Vdb and any of the statefulset's. -func (i *UpgradeManager) isVDBImageDifferent(ctx context.Context) (bool, error) { - stss, err := i.Finder.FindStatefulSets(ctx, iter.FindInVdb) +// image being different between the Vdb and any of the statefulset's or +// between a sandbox and any of its statefulsets if the sandbox name is non-empty. +func (i *UpgradeManager) isVDBImageDifferent(ctx context.Context, sandbox string) (bool, error) { + stss, err := i.Finder.FindStatefulSets(ctx, iter.FindInVdb, sandbox) + if err != nil { + return false, err + } + targetImage, err := i.getTargetImage(sandbox) if err != nil { return false, err } @@ -108,7 +113,7 @@ func (i *UpgradeManager) isVDBImageDifferent(ctx context.Context) (bool, error) if err != nil { return false, err } - if cntImage != i.Vdb.Spec.Image { + if cntImage != targetImage { return true, nil } } @@ -116,10 +121,19 @@ func (i *UpgradeManager) isVDBImageDifferent(ctx context.Context) (bool, error) return false, nil } +// logUpgradeStarted logs an event msg when upgrade is sstarting +func (i *UpgradeManager) logUpgradeStarted(sandbox string) error { + targetImage, err := i.getTargetImage(sandbox) + if err != nil { + return err + } + i.Log.Info("Starting upgrade for reconciliation iteration", "ContinuingUpgrade", i.ContinuingUpgrade, + "New Image", targetImage, "Sandbox", sandbox) + return nil +} + // startUpgrade handles condition status and event recording for start of an upgrade func (i *UpgradeManager) startUpgrade(ctx context.Context) (ctrl.Result, error) { - i.Log.Info("Starting upgrade for reconciliation iteration", "ContinuingUpgrade", i.ContinuingUpgrade, - "New Image", i.Vdb.Spec.Image) if err := i.toggleUpgradeInProgress(ctx, metav1.ConditionTrue); err != nil { return ctrl.Result{}, err } @@ -147,13 +161,21 @@ func (i *UpgradeManager) finishUpgrade(ctx context.Context) (ctrl.Result, error) return ctrl.Result{}, err } - i.Log.Info("The upgrade has completed successfully") - i.VRec.Eventf(i.Vdb, corev1.EventTypeNormal, events.UpgradeSucceeded, - "Vertica server upgrade has completed successfully. New image is '%s'", i.Vdb.Spec.Image) - return ctrl.Result{}, nil } +// logUpgradeSucceeded logs an event msg when upgrade is successful +func (i *UpgradeManager) logUpgradeSucceeded(sandbox string) error { + targetImage, err := i.getTargetImage(sandbox) + if err != nil { + return err + } + i.Log.Info("The upgrade has completed successfully", "Sandbox", sandbox) + i.VRec.Eventf(i.Vdb, corev1.EventTypeNormal, events.UpgradeSucceeded, + "Vertica server upgrade has completed successfully. New image is '%s'", targetImage) + return nil +} + // toggleUpgradeInProgress is a helper for updating the // UpgradeInProgress condition's. We set the UpgradeInProgress plus the // one defined in i.StatusCondition. @@ -210,14 +232,14 @@ func (i *UpgradeManager) clearReplicatedUpgradeAnnotationCallback() (updated boo // updateImageInStatefulSets will change the image in each of the statefulsets. // This changes the images in all subclusters except any transient ones. -func (i *UpgradeManager) updateImageInStatefulSets(ctx context.Context) (int, error) { +func (i *UpgradeManager) updateImageInStatefulSets(ctx context.Context, sandbox string) (int, error) { numStsChanged := 0 // Count to keep track of the nubmer of statefulsets updated // We use FindExisting for the finder because we only want to work with sts // that already exist. This is necessary incase the upgrade was paired // with a scaling operation. The pod change due to the scaling operation // doesn't take affect until after the upgrade. - stss, err := i.Finder.FindStatefulSets(ctx, iter.FindExisting) + stss, err := i.Finder.FindStatefulSets(ctx, iter.FindExisting, sandbox) if err != nil { return numStsChanged, err } @@ -250,11 +272,15 @@ func (i *UpgradeManager) updateImageInStatefulSet(ctx context.Context, sts *apps if svrCnt == nil { return false, fmt.Errorf("could not find the server container in the sts %s", sts.Name) } - if svrCnt.Image != i.Vdb.Spec.Image { + targetImage, err := i.getTargetImage(sts.Labels[vmeta.SandboxNameLabel]) + if err != nil { + return false, err + } + if svrCnt.Image != targetImage { i.Log.Info("Updating image in old statefulset", "name", sts.ObjectMeta.Name) - svrCnt.Image = i.Vdb.Spec.Image + svrCnt.Image = targetImage if nmaCnt := vk8s.GetNMAContainer(sts.Spec.Template.Spec.Containers); nmaCnt != nil { - nmaCnt.Image = i.Vdb.Spec.Image + nmaCnt.Image = targetImage } // We change the update strategy to OnDelete. We don't want the k8s // sts controller to interphere and do a rolling update after the @@ -272,14 +298,18 @@ func (i *UpgradeManager) updateImageInStatefulSet(ctx context.Context, sts *apps // deletePodsRunningOldImage will delete pods that have the old image. It will return the // number of pods that were deleted. Callers can control whether to delete pods // for a specific subcluster or all -- passing an empty string for scName will delete all. -func (i *UpgradeManager) deletePodsRunningOldImage(ctx context.Context, scName string) (int, error) { +func (i *UpgradeManager) deletePodsRunningOldImage(ctx context.Context, scName, sandbox string) (int, error) { numPodsDeleted := 0 // Tracks the number of pods that were deleted // We use FindExisting for the finder because we only want to work with pods // that already exist. This is necessary in case the upgrade was paired // with a scaling operation. The pod change due to the scaling operation // doesn't take affect until after the upgrade. - pods, err := i.Finder.FindPods(ctx, iter.FindExisting, vapi.MainCluster) + pods, err := i.Finder.FindPods(ctx, iter.FindExisting, sandbox) + if err != nil { + return numPodsDeleted, err + } + targetImage, err := i.getTargetImage(sandbox) if err != nil { return numPodsDeleted, err } @@ -299,7 +329,7 @@ func (i *UpgradeManager) deletePodsRunningOldImage(ctx context.Context, scName s if err != nil { return numPodsDeleted, err } - if cntImage != i.Vdb.Spec.Image { + if cntImage != targetImage { i.Log.Info("Deleting pod that had old image", "name", pod.ObjectMeta.Name) err = i.VRec.Client.Delete(ctx, pod) if err != nil { @@ -312,8 +342,12 @@ func (i *UpgradeManager) deletePodsRunningOldImage(ctx context.Context, scName s } // deleteStsRunningOldImage will delete statefulsets that have the old image. -func (i *UpgradeManager) deleteStsRunningOldImage(ctx context.Context) error { - stss, err := i.Finder.FindStatefulSets(ctx, iter.FindExisting) +func (i *UpgradeManager) deleteStsRunningOldImage(ctx context.Context, sandbox string) error { + stss, err := i.Finder.FindStatefulSets(ctx, iter.FindExisting, sandbox) + if err != nil { + return err + } + targetImage, err := i.getTargetImage(sandbox) if err != nil { return err } @@ -324,7 +358,7 @@ func (i *UpgradeManager) deleteStsRunningOldImage(ctx context.Context) error { if err != nil { return err } - if cntImage != i.Vdb.Spec.Image { + if cntImage != targetImage { i.Log.Info("Deleting sts that had old image", "name", sts.ObjectMeta.Name) err = i.VRec.Client.Delete(ctx, sts) if err != nil { @@ -367,7 +401,7 @@ func (i *UpgradeManager) changeNMASidecarDeploymentIfNeeded(ctx context.Context, // the version in the VerticaDB. Since this is a CR wide value, it applies to // all subclusters. i.Log.Info("Detected that we need to switch to NMA sidecar. Deleting all sts") - err = i.deleteStsRunningOldImage(ctx) + err = i.deleteStsRunningOldImage(ctx, sts.Labels[vmeta.SandboxNameLabel]) if err != nil { return ctrl.Result{}, err } @@ -464,14 +498,14 @@ func replicatedUpgradeAllowed(vdb *vapi.VerticaDB) bool { } // cachePrimaryImages will update o.PrimaryImages with the names of all of the primary images -func (i *UpgradeManager) cachePrimaryImages(ctx context.Context) error { - stss, err := i.Finder.FindStatefulSets(ctx, iter.FindExisting) +func (i *UpgradeManager) cachePrimaryImages(ctx context.Context, sandbox string) error { + stss, err := i.Finder.FindStatefulSets(ctx, iter.FindExisting, sandbox) if err != nil { return err } for inx := range stss.Items { sts := &stss.Items[inx] - if sts.Labels[vmeta.SubclusterTypeLabel] == vapi.PrimarySubcluster { + if i.isPrimary(sts.Labels, sandbox) { img, err := vk8s.GetServerImage(sts.Spec.Template.Spec.Containers) if err != nil { return err @@ -494,15 +528,44 @@ func (i *UpgradeManager) cachePrimaryImages(ctx context.Context) error { // fetchOldImage will return the old image that existed prior to the image // change process. If we cannot determine the old image, then the bool return // value returns false. -func (i *UpgradeManager) fetchOldImage() (string, bool) { +func (i *UpgradeManager) fetchOldImage(sandbox string) (string, bool) { + targetImage, err := i.getTargetImage(sandbox) + if err != nil { + return "", false + } for inx := range i.PrimaryImages { - if i.PrimaryImages[inx] != i.Vdb.Spec.Image { + if i.PrimaryImages[inx] != targetImage { return i.PrimaryImages[inx], true } } return "", false } +// getTargetImage returns the image that must be running +// in the main cluster or in a specific sandbox +func (i *UpgradeManager) getTargetImage(sandbox string) (string, error) { + if sandbox == vapi.MainCluster { + return i.Vdb.Spec.Image, nil + } + sb := i.Vdb.GetSandbox(sandbox) + if sb == nil { + return "", fmt.Errorf("could not find sandbox %q", sandbox) + } + // if the target cluster is a sandbox, the target image + // is the one set for that specific sandbox + return sb.Image, nil +} + +// isPrimary returns true if the subcluster is primary +func (i *UpgradeManager) isPrimary(l map[string]string, sandbox string) bool { + if sandbox != vapi.MainCluster { + // For now, there can be only one subcluster + // in a sandbox and so it is primary + return true + } + return l[vmeta.SubclusterTypeLabel] == vapi.PrimarySubcluster +} + func (i *UpgradeManager) traceActorReconcile(actor controllers.ReconcileActor) { i.Log.Info("starting actor for upgrade", "name", fmt.Sprintf("%T", actor)) } diff --git a/pkg/controllers/vdb/upgrade_test.go b/pkg/controllers/vdb/upgrade_test.go index afdaa30d3..32ec41fab 100644 --- a/pkg/controllers/vdb/upgrade_test.go +++ b/pkg/controllers/vdb/upgrade_test.go @@ -65,7 +65,37 @@ var _ = Describe("upgrade", func() { mgr := MakeUpgradeManager(vdbRec, logger, vdb, vapi.OnlineUpgradeInProgress, func(vdb *vapi.VerticaDB) bool { return true }) - Expect(mgr.IsUpgradeNeeded(ctx)).Should(Equal(false)) + Expect(mgr.IsUpgradeNeeded(ctx, vapi.MainCluster)).Should(Equal(false)) + }) + + It("should need an upgrade if images don't match in sts and sandbox", func() { + vdb := vapi.MakeVDB() + vdb.Spec.Image = OldImage + vdb.Spec.Subclusters = []vapi.Subcluster{ + {Name: "sc1", Size: 2, Type: vapi.PrimarySubcluster}, + {Name: "sc2", Size: 1, Type: vapi.SecondarySubcluster}, + } + const sbName = "sand" + vdb.Spec.Sandboxes = []vapi.Sandbox{ + {Name: sbName, Image: "new-img", Subclusters: []vapi.SubclusterName{{Name: "sc2"}}}, + } + test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning) + defer test.DeletePods(ctx, k8sClient, vdb) + test.CreateVDB(ctx, k8sClient, vdb) + defer test.DeleteVDB(ctx, k8sClient, vdb) + + vdb.Status.Sandboxes = []vapi.SandboxStatus{ + {Name: sbName, Subclusters: []string{"sc2"}}, + } + + // upgrade not needed on main cluster + mgr := MakeUpgradeManager(vdbRec, logger, vdb, vapi.OnlineUpgradeInProgress, + func(vdb *vapi.VerticaDB) bool { return true }) + Expect(mgr.IsUpgradeNeeded(ctx, vapi.MainCluster)).Should(Equal(false)) + // upgrade needed on sandbox + mgr = MakeUpgradeManager(vdbRec, logger, vdb, vapi.OnlineUpgradeInProgress, + func(vdb *vapi.VerticaDB) bool { return true }) + Expect(mgr.IsUpgradeNeeded(ctx, sbName)).Should(Equal(true)) }) It("should change the image of both primaries and secondaries", func() { @@ -83,8 +113,8 @@ var _ = Describe("upgrade", func() { mgr := MakeUpgradeManager(vdbRec, logger, vdb, vapi.OfflineUpgradeInProgress, func(vdb *vapi.VerticaDB) bool { return true }) - Expect(mgr.IsUpgradeNeeded(ctx)).Should(Equal(true)) - stsChange, err := mgr.updateImageInStatefulSets(ctx) + Expect(mgr.IsUpgradeNeeded(ctx, vapi.MainCluster)).Should(Equal(true)) + stsChange, err := mgr.updateImageInStatefulSets(ctx, vapi.MainCluster) Expect(err).Should(Succeed()) Expect(stsChange).Should(Equal(2)) @@ -111,7 +141,7 @@ var _ = Describe("upgrade", func() { mgr := MakeUpgradeManager(vdbRec, logger, vdb, vapi.OfflineUpgradeInProgress, func(vdb *vapi.VerticaDB) bool { return true }) - numPodsDeleted, err := mgr.deletePodsRunningOldImage(ctx, "") // pods from primaries only + numPodsDeleted, err := mgr.deletePodsRunningOldImage(ctx, "", vapi.MainCluster) // pods from primaries only Expect(err).Should(Succeed()) Expect(numPodsDeleted).Should(Equal(2)) @@ -132,7 +162,7 @@ var _ = Describe("upgrade", func() { mgr := MakeUpgradeManager(vdbRec, logger, vdb, vapi.OfflineUpgradeInProgress, func(vdb *vapi.VerticaDB) bool { return true }) - numPodsDeleted, err := mgr.deletePodsRunningOldImage(ctx, vdb.Spec.Subclusters[1].Name) + numPodsDeleted, err := mgr.deletePodsRunningOldImage(ctx, vdb.Spec.Subclusters[1].Name, vapi.MainCluster) Expect(err).Should(Succeed()) Expect(numPodsDeleted).Should(Equal(1)) @@ -140,7 +170,7 @@ var _ = Describe("upgrade", func() { Expect(k8sClient.Get(ctx, names.GenPodName(vdb, &vdb.Spec.Subclusters[0], 0), pod)).Should(Succeed()) Expect(k8sClient.Get(ctx, names.GenPodName(vdb, &vdb.Spec.Subclusters[1], 0), pod)).ShouldNot(Succeed()) - numPodsDeleted, err = mgr.deletePodsRunningOldImage(ctx, vdb.Spec.Subclusters[0].Name) + numPodsDeleted, err = mgr.deletePodsRunningOldImage(ctx, vdb.Spec.Subclusters[0].Name, vapi.MainCluster) Expect(err).Should(Succeed()) Expect(numPodsDeleted).Should(Equal(1)) diff --git a/pkg/controllers/vdb/upgradeoperator120_reconciler.go b/pkg/controllers/vdb/upgradeoperator120_reconciler.go index fd2e2446b..0d9120521 100644 --- a/pkg/controllers/vdb/upgradeoperator120_reconciler.go +++ b/pkg/controllers/vdb/upgradeoperator120_reconciler.go @@ -46,7 +46,7 @@ func MakeUpgradeOperator120Reconciler(vdbrecon *VerticaDBReconciler, log logr.Lo // Reconcile will handle any upgrade actions for k8s objects created in 1.2.0 or prior. func (u *UpgradeOperator120Reconciler) Reconcile(ctx context.Context, _ *ctrl.Request) (ctrl.Result, error) { finder := iter.MakeSubclusterFinder(u.VRec.Client, u.Vdb) - stss, err := finder.FindStatefulSets(ctx, iter.FindExisting) + stss, err := finder.FindStatefulSets(ctx, iter.FindExisting, vapi.MainCluster) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/iter/sc_finder.go b/pkg/iter/sc_finder.go index 90e75a236..6b6d514f4 100644 --- a/pkg/iter/sc_finder.go +++ b/pkg/iter/sc_finder.go @@ -64,9 +64,9 @@ func MakeSubclusterFinder(cli client.Client, vdb *vapi.VerticaDB) SubclusterFind // FindStatefulSets returns the statefulsets that were created by the operator. // You can limit it so that it only returns statefulsets that match subclusters // in Vdb, ones that don't match or all. -func (m *SubclusterFinder) FindStatefulSets(ctx context.Context, flags FindFlags) (*appsv1.StatefulSetList, error) { +func (m *SubclusterFinder) FindStatefulSets(ctx context.Context, flags FindFlags, sandbox string) (*appsv1.StatefulSetList, error) { sts := &appsv1.StatefulSetList{} - if err := m.buildObjList(ctx, sts, flags); err != nil { + if err := m.buildObjList(ctx, sts, flags, sandbox); err != nil { return nil, err } if flags&FindSorted != 0 { @@ -78,9 +78,9 @@ func (m *SubclusterFinder) FindStatefulSets(ctx context.Context, flags FindFlags } // FindServices returns service objects that are in use for subclusters -func (m *SubclusterFinder) FindServices(ctx context.Context, flags FindFlags) (*corev1.ServiceList, error) { +func (m *SubclusterFinder) FindServices(ctx context.Context, flags FindFlags, sandbox string) (*corev1.ServiceList, error) { svcs := &corev1.ServiceList{} - if err := m.buildObjList(ctx, svcs, flags); err != nil { + if err := m.buildObjList(ctx, svcs, flags, sandbox); err != nil { return nil, err } if flags&FindSorted != 0 { @@ -95,14 +95,9 @@ func (m *SubclusterFinder) FindServices(ctx context.Context, flags FindFlags) (* // pods that were created by the VerticaDB object. func (m *SubclusterFinder) FindPods(ctx context.Context, flags FindFlags, sandbox string) (*corev1.PodList, error) { pods := &corev1.PodList{} - if err := m.buildObjList(ctx, pods, flags); err != nil { + if err := m.buildObjList(ctx, pods, flags, sandbox); err != nil { return nil, err } - // we can skip the filtering if all of the pods we are - // looking for are not part of a subcluster in vdb - if flags&FindInVdb != 0 || flags&FindExisting != 0 { - pods = m.filterPodsBySandbox(pods, sandbox) - } if flags&FindSorted != 0 { sort.Slice(pods.Items, func(i, j int) bool { return pods.Items[i].Name < pods.Items[j].Name @@ -116,16 +111,13 @@ func (m *SubclusterFinder) FindPods(ctx context.Context, flags FindFlags, sandbo // not in the vdb or both. func (m *SubclusterFinder) FindSubclusters(ctx context.Context, flags FindFlags, sandbox string) ([]*vapi.Subcluster, error) { subclusters := []*vapi.Subcluster{} - isMainCluster := sandbox == vapi.MainCluster if flags&FindInVdb != 0 { subclusters = append(subclusters, m.getVdbSubclusters(sandbox)...) } - // a sandboxed subcluster can only be one of those in the vdb so we skip this part if - // we are looking for sandboxed subclusters - if isMainCluster && (flags&FindNotInVdb != 0 || flags&FindExisting != 0) { - missingSts, err := m.FindStatefulSets(ctx, flags & ^FindInVdb) + if flags&FindNotInVdb != 0 || flags&FindExisting != 0 { + missingSts, err := m.FindStatefulSets(ctx, flags & ^FindInVdb, sandbox) if err != nil { return nil, err } @@ -157,24 +149,36 @@ func (m *SubclusterFinder) hasSubclusterLabelFromVdb(objLabels map[string]string // buildObjList will populate list with an object type owned by the operator. // Caller can use flags to return a list of all objects, only those in the vdb, // or only those not in the vdb. -func (m *SubclusterFinder) buildObjList(ctx context.Context, list client.ObjectList, flags FindFlags) error { +func (m *SubclusterFinder) buildObjList(ctx context.Context, list client.ObjectList, flags FindFlags, sandbox string) error { if err := listObjectsOwnedByOperator(ctx, m.Client, m.Vdb, list); err != nil { return err } - if flags&FindAll == FindAll { - return nil - } rawObjs := []runtime.Object{} if err := meta.EachListItem(list, func(obj runtime.Object) error { l, ok := getLabelsFromObject(obj) if !ok { return fmt.Errorf("could not find labels from k8s object %s", obj) } + if flags&FindAll == FindAll { + // When FindAll is passed, we want the entire list to be returned, + // but still want to filter out objects that do not belong to the given + // sandbox or main cluster. + if !shouldSkipBasedOnSandboxState(l, sandbox) { + rawObjs = append(rawObjs, obj) + } + return nil + } // Skip if object is not subcluster specific. This is necessary for objects like // the headless service object that is cluster wide. if !hasSubclusterNameLabel(l) { return nil } + + // Skip if the object does not belong to the given sandbox + if shouldSkipBasedOnSandboxState(l, sandbox) { + return nil + } + if flags&FindExisting != 0 { rawObjs = append(rawObjs, obj) return nil @@ -194,6 +198,12 @@ func (m *SubclusterFinder) buildObjList(ctx context.Context, list client.ObjectL return meta.SetList(list, rawObjs) } +// shouldSkipBasedOnSandboxState returns true if the object whose labels +// is passed does not belong to the given sandbox or main cluster +func shouldSkipBasedOnSandboxState(l map[string]string, sandbox string) bool { + return l[vmeta.SandboxNameLabel] != sandbox +} + // hasSubclusterNameLabel returns true if there exists a label that indicates // the object is for a subcluster func hasSubclusterNameLabel(l map[string]string) bool { @@ -273,26 +283,3 @@ func (m *SubclusterFinder) getSandboxedSubclusters(sandbox string) []*vapi.Subcl } return subclusters } - -// filterPodsBySandbox returns a pod list without main cluster pods if a non-empty sandbox named is -// passed in, or pod list with only main cluster pods if no sandbox name is passed -func (m *SubclusterFinder) filterPodsBySandbox(oldPods *corev1.PodList, sandbox string) *corev1.PodList { - newPods := []corev1.Pod{} - scMap := m.getSubclusterSandboxStatusMap() - for i := range oldPods.Items { - pod := oldPods.Items[i] - sc := pod.Labels[vmeta.SubclusterNameLabel] - sbName, isSandbox := scMap[sc] - if sandbox == vapi.MainCluster { - if !isSandbox { - newPods = append(newPods, pod) - } - } else { - if sbName == sandbox { - newPods = append(newPods, pod) - } - } - } - oldPods.Items = newPods - return oldPods -} diff --git a/pkg/iter/sc_finder_test.go b/pkg/iter/sc_finder_test.go index 1d2b17426..6e539fcd5 100644 --- a/pkg/iter/sc_finder_test.go +++ b/pkg/iter/sc_finder_test.go @@ -98,7 +98,7 @@ var _ = Describe("sc_finder", func() { lookupVdb.Spec.Subclusters[0] = vapi.Subcluster{Name: scNames[1], Size: scSizes[1]} finder := MakeSubclusterFinder(k8sClient, lookupVdb) - sts, err := finder.FindStatefulSets(ctx, FindNotInVdb) + sts, err := finder.FindStatefulSets(ctx, FindNotInVdb, vapi.MainCluster) Expect(err).Should(Succeed()) Expect(len(sts.Items)).Should(Equal(1)) Expect(sts.Items[0].Name).Should(Equal(names.GenStsName(vdb, &vdb.Spec.Subclusters[0]).Name)) @@ -114,6 +114,11 @@ var _ = Describe("sc_finder", func() { {Name: scNames[0], Size: scSizes[0], Type: vapi.PrimarySubcluster}, {Name: scNames[1], Size: scSizes[1], Type: vapi.SecondarySubcluster}, } + const sbName = "sand" + // sandbox scNames[0] + vdb.Spec.Sandboxes = []vapi.Sandbox{ + {Name: sbName, Subclusters: []vapi.SubclusterName{{Name: scNames[0]}}}, + } test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning) vdbCopy := *vdb // Make a copy for cleanup since we will mutate vdb defer test.DeletePods(ctx, k8sClient, &vdbCopy) @@ -128,17 +133,26 @@ var _ = Describe("sc_finder", func() { vdb.Spec.Image = "should-not-report-this-image" finder := MakeSubclusterFinder(k8sClient, vdb) - sts, err := finder.FindStatefulSets(ctx, FindExisting) + sts, err := finder.FindStatefulSets(ctx, FindExisting, vapi.MainCluster) Expect(err).Should(Succeed()) - Expect(len(sts.Items)).Should(Equal(2)) - Expect(sts.Items[0].Name).Should(Equal(names.GenStsName(vdb, &vdb.Spec.Subclusters[0]).Name)) - Expect(sts.Items[1].Name).Should(Equal(names.GenStsName(vdb, &vdb.Spec.Subclusters[1]).Name)) + Expect(len(sts.Items)).Should(Equal(1)) + Expect(sts.Items[0].Name).Should(Equal(names.GenStsName(vdb, &vdb.Spec.Subclusters[1]).Name)) + + vdb.Status.Sandboxes = []vapi.SandboxStatus{ + {Name: sbName, Subclusters: scNames[:1]}, + } scs, err := finder.FindSubclusters(ctx, FindExisting, vapi.MainCluster) Expect(err).Should(Succeed()) - Expect(len(scs)).Should(Equal(2)) - Expect(scs[0].Name).Should(Equal(vdb.Spec.Subclusters[0].Name)) - Expect(scs[1].Name).Should(Equal(vdb.Spec.Subclusters[1].Name)) + Expect(len(scs)).Should(Equal(1)) + Expect(scs[0].Name).Should(Equal(vdb.Spec.Subclusters[1].Name)) + + // only the sandboxed sts should be returned + finder = MakeSubclusterFinder(k8sClient, vdb) + sts, err = finder.FindStatefulSets(ctx, FindExisting, sbName) + Expect(err).Should(Succeed()) + Expect(len(sts.Items)).Should(Equal(1)) + Expect(sts.Items[0].Name).Should(Equal(names.GenStsName(vdb, &vdb.Spec.Subclusters[0]).Name)) }) It("should find all pods that exist in k8s for the VerticaDB", func() { @@ -150,8 +164,8 @@ var _ = Describe("sc_finder", func() { {Name: scNames[1], Size: scSizes[1]}, } const sbName = "sand" - vdb.Status.Sandboxes = []vapi.SandboxStatus{ - {Name: sbName, Subclusters: scNames[:1]}, + vdb.Spec.Sandboxes = []vapi.Sandbox{ + {Name: sbName, Subclusters: []vapi.SubclusterName{{Name: scNames[0]}}}, } test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning) defer test.DeletePods(ctx, k8sClient, vdb) @@ -159,7 +173,7 @@ var _ = Describe("sc_finder", func() { // When use the finder, pass in a Vdb that is entirely different then // the one we used above. It will be ignored anyway when using // FindExisting. - findPods(ctx, vapi.MakeVDB(), int(scSizes[0]+scSizes[1]), vapi.MainCluster) + findPods(ctx, vapi.MakeVDB(), int(scSizes[1]), vapi.MainCluster) // Only the pods belonging to the sandboxed subcluster // will be collected findPods(ctx, vdb, int(scSizes[0]), sbName) @@ -172,7 +186,7 @@ var _ = Describe("sc_finder", func() { defer test.DeleteSvcs(ctx, k8sClient, vdb) finder := MakeSubclusterFinder(k8sClient, vdb) - svcs, err := finder.FindServices(ctx, FindInVdb) + svcs, err := finder.FindServices(ctx, FindInVdb, vapi.MainCluster) Expect(err).Should(Succeed()) const SvcsPerSubcluster = 1 Expect(len(svcs.Items)).Should(Equal(SvcsPerSubcluster)) @@ -201,7 +215,7 @@ var _ = Describe("sc_finder", func() { lookupVdb.Spec.Subclusters[0] = vapi.Subcluster{Name: scNames[0]} finder := MakeSubclusterFinder(k8sClient, lookupVdb) - svcs, err := finder.FindServices(ctx, FindNotInVdb) + svcs, err := finder.FindServices(ctx, FindNotInVdb, vapi.MainCluster) Expect(err).Should(Succeed()) const SvcsPerSubcluster = 1 Expect(len(svcs.Items)).Should(Equal(SvcsPerSubcluster)) @@ -224,7 +238,7 @@ var _ = Describe("sc_finder", func() { defer test.DeleteSvcs(ctx, k8sClient, vdb) finder := MakeSubclusterFinder(k8sClient, vdb) - svcs, err := finder.FindServices(ctx, FindExisting|FindSorted) + svcs, err := finder.FindServices(ctx, FindExisting|FindSorted, vapi.MainCluster) Expect(err).Should(Succeed()) Expect(svcs.Items[0].Name).Should(ContainSubstring(scNames[1])) Expect(svcs.Items[1].Name).Should(ContainSubstring(scNames[0])) @@ -241,7 +255,7 @@ var _ = Describe("sc_finder", func() { defer test.DeletePods(ctx, k8sClient, vdb) finder := MakeSubclusterFinder(k8sClient, vdb) - stss, err := finder.FindStatefulSets(ctx, FindExisting|FindSorted) + stss, err := finder.FindStatefulSets(ctx, FindExisting|FindSorted, vapi.MainCluster) Expect(err).Should(Succeed()) Expect(stss.Items[0].Name).Should(ContainSubstring(scNames[1])) Expect(stss.Items[1].Name).Should(ContainSubstring(scNames[0])) diff --git a/pkg/meta/labels.go b/pkg/meta/labels.go index 0fef76296..e6aaf2752 100644 --- a/pkg/meta/labels.go +++ b/pkg/meta/labels.go @@ -59,8 +59,10 @@ const ( WatchedBySandboxLabel = "vertica.com/watched-by-sandbox-controller" WatchedBySandboxTrue = "true" - // This label is added is added to a statefulset to indicate the sandbox - // it belongs to. The sandbox controller will be watching statefulsets + // This label is added to a statefulset or a pod to indicate the sandbox + // it belongs to. This will allow the operator to filter these objects based on + // the cluster(main cluster or sandbox) they belong to. + // Moreover, the sandbox controller will be watching statefulsets // with this label and will trigger a reconcile loop if it finds a configmap // with a sandbox name equal to this label's value SandboxNameLabel = "vertica.com/sandbox"