diff --git a/pkg/placement/controllers/scheduling/schedule.go b/pkg/placement/controllers/scheduling/schedule.go index 7c530d9a2..169179b1b 100644 --- a/pkg/placement/controllers/scheduling/schedule.go +++ b/pkg/placement/controllers/scheduling/schedule.go @@ -56,8 +56,8 @@ type ScheduleResult interface { // PrioritizerScores returns total score for each cluster PrioritizerScores() PrioritizerScore - // Decisions returns the decisions of the schedule - Decisions() []clusterapiv1beta1.ClusterDecision + // Decision returns the decision groups of the schedule + Decisions() []*clusterapiv1.ManagedCluster // NumOfUnscheduled returns the number of unscheduled. NumOfUnscheduled() int @@ -82,7 +82,7 @@ type PrioritizerResult struct { // ScheduleResult is the result for a certain schedule. type scheduleResult struct { feasibleClusters []*clusterapiv1.ManagedCluster - scheduledDecisions []clusterapiv1beta1.ClusterDecision + scheduledDecisions []*clusterapiv1.ManagedCluster unscheduledDecisions int filteredRecords map[string][]*clusterapiv1.ManagedCluster @@ -290,9 +290,8 @@ func (s *pluginScheduler) Schedule( return results, finalStatus } -// makeClusterDecisions selects clusters based on given cluster slice and then creates -// cluster decisions. -func selectClusters(placement *clusterapiv1beta1.Placement, clusters []*clusterapiv1.ManagedCluster) []clusterapiv1beta1.ClusterDecision { +// selects clusters based on given cluster slice and number of clusters +func selectClusters(placement *clusterapiv1beta1.Placement, clusters []*clusterapiv1.ManagedCluster) []*clusterapiv1.ManagedCluster { numOfDecisions := len(clusters) if placement.Spec.NumberOfClusters != nil { numOfDecisions = int(*placement.Spec.NumberOfClusters) @@ -304,13 +303,7 @@ func selectClusters(placement *clusterapiv1beta1.Placement, clusters []*clustera clusters = clusters[:numOfDecisions] } - decisions := []clusterapiv1beta1.ClusterDecision{} - for _, cluster := range clusters { - decisions = append(decisions, clusterapiv1beta1.ClusterDecision{ - ClusterName: cluster.Name, - }) - } - return decisions + return clusters } // setRequeueAfter selects minimal time.Duration as requeue time @@ -427,7 +420,7 @@ func (r *scheduleResult) PrioritizerScores() PrioritizerScore { return r.scoreSum } -func (r *scheduleResult) Decisions() []clusterapiv1beta1.ClusterDecision { +func (r *scheduleResult) Decisions() []*clusterapiv1.ManagedCluster { return r.scheduledDecisions } diff --git a/pkg/placement/controllers/scheduling/schedule_test.go b/pkg/placement/controllers/scheduling/schedule_test.go index ae26b52a4..423490e43 100644 --- a/pkg/placement/controllers/scheduling/schedule_test.go +++ b/pkg/placement/controllers/scheduling/schedule_test.go @@ -31,10 +31,9 @@ func TestSchedule(t *testing.T) { placement *clusterlisterv1beta1.Placement initObjs []runtime.Object clusters []*clusterapiv1.ManagedCluster - decisions []runtime.Object expectedFilterResult []FilterResult expectedScoreResult []PrioritizerResult - expectedDecisions []clusterapiv1beta1.ClusterDecision + expectedDecisions []*clusterapiv1.ManagedCluster expectedUnScheduled int expectedStatus framework.Status }{ @@ -45,9 +44,13 @@ func TestSchedule(t *testing.T) { testinghelpers.NewClusterSet(clusterSetName).Build(), testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), }, - decisions: []runtime.Object{}, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel( + clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), + }, + expectedDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel( + clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, expectedFilterResult: []FilterResult{ { @@ -71,10 +74,6 @@ func TestSchedule(t *testing.T) { Scores: PrioritizerScore{"cluster1": 0}, }, }, - clusters: []*clusterapiv1.ManagedCluster{ - testinghelpers.NewManagedCluster("cluster1").WithLabel( - clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), - }, expectedUnScheduled: 0, expectedStatus: *framework.NewStatus("", framework.Success, ""), }, @@ -85,12 +84,11 @@ func TestSchedule(t *testing.T) { testinghelpers.NewClusterSet(clusterSetName).Build(), testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), }, - decisions: []runtime.Object{}, clusters: []*clusterapiv1.ManagedCluster{ testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, + expectedDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, expectedFilterResult: []FilterResult{ { @@ -128,11 +126,10 @@ func TestSchedule(t *testing.T) { testinghelpers.NewClusterSet(clusterSetName).Build(), testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), }, - decisions: []runtime.Object{}, clusters: []*clusterapiv1.ManagedCluster{ testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{}, + expectedDecisions: []*clusterapiv1.ManagedCluster{}, expectedFilterResult: []FilterResult{ { Name: "Predicate", @@ -157,19 +154,14 @@ func TestSchedule(t *testing.T) { WithLabel(clusterapiv1beta1.PlacementLabel, placementName). WithDecisions("cluster1", "cluster2").Build(), }, - decisions: []runtime.Object{ - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). - WithLabel(clusterapiv1beta1.PlacementLabel, placementName). - WithDecisions("cluster1", "cluster2").Build(), - }, clusters: []*clusterapiv1.ManagedCluster{ testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, - {ClusterName: "cluster2"}, + expectedDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, expectedFilterResult: []FilterResult{ { @@ -203,9 +195,8 @@ func TestSchedule(t *testing.T) { testinghelpers.NewClusterSet(clusterSetName).Build(), testinghelpers.NewClusterSetBinding(placementNamespace, clusterSetName), }, - decisions: []runtime.Object{}, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, + expectedDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, expectedFilterResult: []FilterResult{ { @@ -267,10 +258,15 @@ func TestSchedule(t *testing.T) { }).Build(), testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, - decisions: []runtime.Object{}, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, - {ClusterName: "cluster3"}, + expectedDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).WithTaint( + &clusterapiv1.Taint{ + Key: "key1", + Value: "value1", + Effect: clusterapiv1.TaintEffectNoSelect, + TimeAdded: metav1.Time{}, + }).Build(), + testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, expectedFilterResult: []FilterResult{ { @@ -312,10 +308,9 @@ func TestSchedule(t *testing.T) { testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).WithResource(clusterapiv1.ResourceMemory, "50", "100").Build(), testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).WithResource(clusterapiv1.ResourceMemory, "0", "100").Build(), }, - decisions: []runtime.Object{}, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, - {ClusterName: "cluster2"}, + expectedDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).WithResource(clusterapiv1.ResourceMemory, "100", "100").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).WithResource(clusterapiv1.ResourceMemory, "50", "100").Build(), }, expectedFilterResult: []FilterResult{ { @@ -364,10 +359,9 @@ func TestSchedule(t *testing.T) { testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).WithResource(clusterapiv1.ResourceMemory, "50", "100").Build(), testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).WithResource(clusterapiv1.ResourceMemory, "0", "100").Build(), }, - decisions: []runtime.Object{}, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, - {ClusterName: "cluster2"}, + expectedDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).WithResource(clusterapiv1.ResourceMemory, "100", "100").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).WithResource(clusterapiv1.ResourceMemory, "50", "100").Build(), }, expectedFilterResult: []FilterResult{ { @@ -408,14 +402,9 @@ func TestSchedule(t *testing.T) { testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, - decisions: []runtime.Object{ - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). - WithLabel(clusterapiv1beta1.PlacementLabel, placementName). - WithDecisions("cluster1").Build(), - }, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, - {ClusterName: "cluster2"}, + expectedDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, expectedFilterResult: []FilterResult{ { @@ -451,17 +440,13 @@ func TestSchedule(t *testing.T) { testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 1)). WithDecisions("cluster1", "cluster2").Build(), }, - decisions: []runtime.Object{ - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 1)). - WithDecisions("cluster1", "cluster2").Build(), - }, clusters: []*clusterapiv1.ManagedCluster{ testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster3"}, + expectedDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, expectedFilterResult: []FilterResult{ { @@ -502,22 +487,13 @@ func TestSchedule(t *testing.T) { WithLabel(clusterapiv1beta1.PlacementLabel, placementName). WithDecisions("cluster3").Build(), }, - decisions: []runtime.Object{ - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 1)). - WithDecisions("cluster2", "cluster3").Build(), - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName("others", 2)). - WithDecisions("cluster1", "cluster2").Build(), - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). - WithLabel(clusterapiv1beta1.PlacementLabel, placementName). - WithDecisions("cluster3").Build(), - }, clusters: []*clusterapiv1.ManagedCluster{ testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), testinghelpers.NewManagedCluster("cluster2").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, - expectedDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster3"}, + expectedDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster3").WithLabel(clusterapiv1beta2.ClusterSetLabel, clusterSetName).Build(), }, expectedFilterResult: []FilterResult{ { diff --git a/pkg/placement/controllers/scheduling/scheduling_controller.go b/pkg/placement/controllers/scheduling/scheduling_controller.go index 86a013da7..d4ca67bcc 100644 --- a/pkg/placement/controllers/scheduling/scheduling_controller.go +++ b/pkg/placement/controllers/scheduling/scheduling_controller.go @@ -3,8 +3,10 @@ package scheduling import ( "context" "fmt" + "math" "reflect" "sort" + "strconv" "strings" "time" @@ -12,13 +14,13 @@ import ( "github.com/openshift/library-go/pkg/operator/events" errorhelpers "github.com/openshift/library-go/pkg/operator/v1helpers" corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/intstr" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" cache "k8s.io/client-go/tools/cache" @@ -37,8 +39,10 @@ import ( clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1" clusterapiv1beta2 "open-cluster-management.io/api/cluster/v1beta2" + "open-cluster-management.io/ocm/pkg/common/patcher" "open-cluster-management.io/ocm/pkg/common/queue" "open-cluster-management.io/ocm/pkg/placement/controllers/framework" + "open-cluster-management.io/ocm/pkg/placement/helpers" ) const ( @@ -47,6 +51,14 @@ const ( maxEventMessageLength = 1000 //the event message can have at most 1024 characters, use 1000 as limitation here to keep some buffer ) +// decisionGroups groups the cluster decisions by group strategy +type clusterDecisionGroups []clusterDecisionGroup + +type clusterDecisionGroup struct { + decisionGroupName string + clusterDecisions []clusterapiv1beta1.ClusterDecision +} + var ResyncInterval = time.Minute * 5 // schedulingController schedules cluster decisions for Placements @@ -226,6 +238,11 @@ func (c *schedulingController) syncPlacement(ctx context.Context, syncCtx factor // schedule placement with scheduler scheduleResult, status := c.scheduler.Schedule(ctx, placement, clusters) + // generate placement decision and status + decisions, groupStatus, s := c.generatePlacementDecisionsAndStatus(placement, scheduleResult.Decisions()) + if s.IsError() { + status = s + } misconfiguredCondition := newMisconfiguredCondition(status) satisfiedCondition := newSatisfiedCondition( placement.Spec.ClusterSets, @@ -245,12 +262,14 @@ func (c *schedulingController) syncPlacement(ctx context.Context, syncCtx factor syncCtx.Queue().AddAfter(key, *t) } - if err := c.bind(ctx, placement, scheduleResult.Decisions(), scheduleResult.PrioritizerScores(), status); err != nil { + // create/update placement decisions + err = c.bind(ctx, placement, decisions, scheduleResult.PrioritizerScores(), status) + if err != nil { return err } // update placement status if necessary to signal no bindings - if err := c.updateStatus(ctx, placement, int32(len(scheduleResult.Decisions())), misconfiguredCondition, satisfiedCondition); err != nil { + if err := c.updateStatus(ctx, placement, groupStatus, int32(len(scheduleResult.Decisions())), misconfiguredCondition, satisfiedCondition); err != nil { return err } @@ -345,11 +364,17 @@ func (c *schedulingController) getAvailableClusters(clusterSetNames []string) ([ func (c *schedulingController) updateStatus( ctx context.Context, placement *clusterapiv1beta1.Placement, + decisionGroupStatus []*clusterapiv1beta1.DecisionGroupStatus, numberOfSelectedClusters int32, conditions ...metav1.Condition, ) error { newPlacement := placement.DeepCopy() newPlacement.Status.NumberOfSelectedClusters = numberOfSelectedClusters + newPlacement.Status.DecisionGroups = []clusterapiv1beta1.DecisionGroupStatus{} + + for _, status := range decisionGroupStatus { + newPlacement.Status.DecisionGroups = append(newPlacement.Status.DecisionGroups, *status) + } for _, c := range conditions { meta.SetStatusCondition(&newPlacement.Status.Conditions, c) @@ -358,9 +383,11 @@ func (c *schedulingController) updateStatus( return nil } - _, err := c.clusterClient.ClusterV1beta1(). - Placements(newPlacement.Namespace). - UpdateStatus(ctx, newPlacement, metav1.UpdateOptions{}) + placementPatcher := patcher.NewPatcher[ + *clusterapiv1beta1.Placement, clusterapiv1beta1.PlacementSpec, clusterapiv1beta1.PlacementStatus]( + c.clusterClient.ClusterV1beta1().Placements(newPlacement.Namespace)) + + _, err := placementPatcher.PatchStatus(ctx, newPlacement, newPlacement.Status, placement.Status) return err } @@ -428,24 +455,132 @@ func newMisconfiguredCondition(status *framework.Status) metav1.Condition { } } -// bind updates the cluster decisions in the status of the placementdecisions with the given -// cluster decision slice. New placementdecisions will be created if no one exists. -func (c *schedulingController) bind( - ctx context.Context, +// generate placement decision and decision group status of placement +func (c *schedulingController) generatePlacementDecisionsAndStatus( placement *clusterapiv1beta1.Placement, - clusterDecisions []clusterapiv1beta1.ClusterDecision, - clusterScores PrioritizerScore, - status *framework.Status, -) error { - // sort clusterdecisions by cluster name - sort.SliceStable(clusterDecisions, func(i, j int) bool { - return clusterDecisions[i].ClusterName < clusterDecisions[j].ClusterName - }) + clusters []*clusterapiv1.ManagedCluster, +) ([]*clusterapiv1beta1.PlacementDecision, []*clusterapiv1beta1.DecisionGroupStatus, *framework.Status) { + placementDecisionIndex := 0 + placementDecisions := []*clusterapiv1beta1.PlacementDecision{} + decisionGroupStatus := []*clusterapiv1beta1.DecisionGroupStatus{} + + // generate decision group + decisionGroups, status := c.generateDecisionGroups(placement, clusters) + + // generate placement decision for each decision group + for decisionGroupIndex, decisionGroup := range decisionGroups { + // sort clusterdecisions by cluster name + clusterDecisions := decisionGroup.clusterDecisions + sort.SliceStable(clusterDecisions, func(i, j int) bool { + return clusterDecisions[i].ClusterName < clusterDecisions[j].ClusterName + }) + + // generate placement decisions and status, group and placement name index starts from 0. + pds, groupStatus := c.generateDecision(placement, decisionGroup, decisionGroupIndex, placementDecisionIndex) + + placementDecisions = append(placementDecisions, pds...) + decisionGroupStatus = append(decisionGroupStatus, groupStatus) + placementDecisionIndex += len(pds) + } + + return placementDecisions, decisionGroupStatus, status +} + +// generateDecisionGroups group clusters based on the placement decision strategy. +func (c *schedulingController) generateDecisionGroups( + placement *clusterapiv1beta1.Placement, + clusters []*clusterapiv1.ManagedCluster, +) (clusterDecisionGroups, *framework.Status) { + groups := []clusterDecisionGroup{} + // Calculate the group length + groupLength, status := calculateLength(&placement.Spec.DecisionStrategy.GroupStrategy.ClustersPerDecisionGroup, len(clusters)) + if status.IsError() { + return groups, status + } + + // Record the cluster names + clusterNames := sets.NewString() + for _, cluster := range clusters { + clusterNames.Insert(cluster.Name) + } + + // First groups the clusters by ClusterSelector defined in spec.DecisionStrategy.GroupStrategy.DecisionGroups. + for _, d := range placement.Spec.DecisionStrategy.GroupStrategy.DecisionGroups { + // create cluster label selector + clusterSelector, err := helpers.NewClusterSelector(d.ClusterSelector) + if err != nil { + status = framework.NewStatus("", framework.Misconfigured, err.Error()) + return groups, status + } + + // filter clusters by label selector + groupName := d.GroupName + matched := []clusterapiv1beta1.ClusterDecision{} + for _, cluster := range clusters { + if ok := clusterSelector.Matches(cluster.Labels, helpers.GetClusterClaims(cluster)); !ok { + continue + } + if !clusterNames.Has(cluster.Name) { + continue + } + + matched = append(matched, clusterapiv1beta1.ClusterDecision{ + ClusterName: cluster.Name, + }) + clusterNames.Delete(cluster.Name) + } + + if len(matched) > 0 { + decisionGroup := clusterDecisionGroup{ + decisionGroupName: groupName, + clusterDecisions: matched, + } + groups = append(groups, decisionGroup) + } + } + + // The rest of the clusters will also be put into decision groups. + // The number of items in each group is determined by the specific number or + // percentage defined in spec.DecisionStrategy.GroupStrategy.ClustersPerDecisionGroup. + for len(clusterNames) > 0 { + clusterList := clusterNames.List() + resultNumber := len(clusterList) + if groupLength < resultNumber { + resultNumber = groupLength + } + + matched := []clusterapiv1beta1.ClusterDecision{} + for i := 0; i < resultNumber; i++ { + matched = append(matched, clusterapiv1beta1.ClusterDecision{ + ClusterName: clusterList[i], + }) + delete(clusterNames, clusterList[i]) + } + decisionGroup := clusterDecisionGroup{ + decisionGroupName: "", + clusterDecisions: matched, + } + groups = append(groups, decisionGroup) + } + + // generate at least on empty decisionGroup, this is to ensure there's an empty placement decision if no cluster selected. + if len(groups) == 0 { + groups = append(groups, clusterDecisionGroup{}) + } + + return groups, framework.NewStatus("", framework.Success, "") +} + +func (c *schedulingController) generateDecision( + placement *clusterapiv1beta1.Placement, + clusterDecisionGroup clusterDecisionGroup, + decisionGroupIndex, placementDecisionIndex int, +) ([]*clusterapiv1beta1.PlacementDecision, *clusterapiv1beta1.DecisionGroupStatus) { // split the cluster decisions into slices, the size of each slice cannot exceed // maxNumOfClusterDecisions. decisionSlices := [][]clusterapiv1beta1.ClusterDecision{} - remainingDecisions := clusterDecisions + remainingDecisions := clusterDecisionGroup.clusterDecisions for index := 0; len(remainingDecisions) > 0; index++ { var decisionSlice []clusterapiv1beta1.ClusterDecision switch { @@ -458,28 +593,68 @@ func (c *schedulingController) bind( } decisionSlices = append(decisionSlices, decisionSlice) } + // if decisionSlices is empty, append one empty slice. // so that can create a PlacementDecision with empty decisions in status. if len(decisionSlices) == 0 { decisionSlices = append(decisionSlices, []clusterapiv1beta1.ClusterDecision{}) } - // bind cluster decision slices to placementdecisions. - errs := []error{} + placementDecisionNames := []string{} + placementDecisions := []*clusterapiv1beta1.PlacementDecision{} + for index, decisionSlice := range decisionSlices { + placementDecisionName := fmt.Sprintf("%s-decision-%d", placement.Name, placementDecisionIndex+index) + owner := metav1.NewControllerRef(placement, clusterapiv1beta1.GroupVersion.WithKind("Placement")) + placementDecision := &clusterapiv1beta1.PlacementDecision{ + ObjectMeta: metav1.ObjectMeta{ + Name: placementDecisionName, + Namespace: placement.Namespace, + Labels: map[string]string{ + clusterapiv1beta1.PlacementLabel: placement.Name, + clusterapiv1beta1.DecisionGroupNameLabel: clusterDecisionGroup.decisionGroupName, + clusterapiv1beta1.DecisionGroupIndexLabel: fmt.Sprint(decisionGroupIndex), + }, + OwnerReferences: []metav1.OwnerReference{*owner}, + }, + Status: clusterapiv1beta1.PlacementDecisionStatus{ + Decisions: decisionSlice, + }, + } + placementDecisions = append(placementDecisions, placementDecision) + placementDecisionNames = append(placementDecisionNames, placementDecisionName) + } + decisionGroupStatus := &clusterapiv1beta1.DecisionGroupStatus{ + DecisionGroupIndex: int32(decisionGroupIndex), + DecisionGroupName: clusterDecisionGroup.decisionGroupName, + Decisions: placementDecisionNames, + ClustersCount: int32(len(clusterDecisionGroup.clusterDecisions)), + } + + return placementDecisions, decisionGroupStatus +} + +// bind updates the cluster decisions in the status of the placementdecisions with the given +// cluster decision slice. New placementdecisions will be created if no one exists. +// bind will also return the decision groups for placement status. +func (c *schedulingController) bind( + ctx context.Context, + placement *clusterapiv1beta1.Placement, + placementdecisions []*clusterapiv1beta1.PlacementDecision, + clusterScores PrioritizerScore, + status *framework.Status, +) error { + errs := []error{} placementDecisionNames := sets.NewString() - for index, decisionSlice := range decisionSlices { - placementDecisionName := fmt.Sprintf("%s-decision-%d", placement.Name, index+1) - placementDecisionNames.Insert(placementDecisionName) - err := c.createOrUpdatePlacementDecision( - ctx, placement, placementDecisionName, decisionSlice, clusterScores, status) + + // create/update placement decisions + for _, pd := range placementdecisions { + placementDecisionNames.Insert(pd.Name) + err := c.createOrUpdatePlacementDecision(ctx, placement, pd, clusterScores, status) if err != nil { errs = append(errs, err) } } - if len(errs) != 0 { - return errorhelpers.NewMultiLineAggregate(errs) - } // query all placementdecisions of the placement requirement, err := labels.NewRequirement(clusterapiv1beta1.PlacementLabel, selection.Equals, []string{placement.Name}) @@ -487,19 +662,19 @@ func (c *schedulingController) bind( return err } labelSelector := labels.NewSelector().Add(*requirement) - placementDecisions, err := c.placementDecisionLister.PlacementDecisions(placement.Namespace).List(labelSelector) + pds, err := c.placementDecisionLister.PlacementDecisions(placement.Namespace).List(labelSelector) if err != nil { return err } // delete redundant placementdecisions errs = []error{} - for _, placementDecision := range placementDecisions { - if placementDecisionNames.Has(placementDecision.Name) { + for _, pd := range pds { + if placementDecisionNames.Has(pd.Name) { continue } err := c.clusterClient.ClusterV1beta1().PlacementDecisions( - placementDecision.Namespace).Delete(ctx, placementDecision.Name, metav1.DeleteOptions{}) + pd.Namespace).Delete(ctx, pd.Name, metav1.DeleteOptions{}) if errors.IsNotFound(err) { continue } @@ -507,9 +682,9 @@ func (c *schedulingController) bind( errs = append(errs, err) } c.recorder.Eventf( - placement, placementDecision, corev1.EventTypeNormal, + placement, pd, corev1.EventTypeNormal, "DecisionDelete", "DecisionDeleted", - "Decision %s is deleted with placement %s in namespace %s", placementDecision.Name, placement.Name, placement.Namespace) + "Decision %s is deleted with placement %s in namespace %s", pd.Name, placement.Name, placement.Namespace) } return errorhelpers.NewMultiLineAggregate(errs) } @@ -519,54 +694,49 @@ func (c *schedulingController) bind( func (c *schedulingController) createOrUpdatePlacementDecision( ctx context.Context, placement *clusterapiv1beta1.Placement, - placementDecisionName string, - clusterDecisions []clusterapiv1beta1.ClusterDecision, + placementDecision *clusterapiv1beta1.PlacementDecision, clusterScores PrioritizerScore, status *framework.Status, ) error { + placementDecisionName := placementDecision.Name + clusterDecisions := placementDecision.Status.Decisions + if len(clusterDecisions) > maxNumOfClusterDecisions { return fmt.Errorf("the number of clusterdecisions %q exceeds the max limitation %q", len(clusterDecisions), maxNumOfClusterDecisions) } - placementDecision, err := c.placementDecisionLister.PlacementDecisions(placement.Namespace).Get(placementDecisionName) + existPlacementDecision, err := c.placementDecisionLister.PlacementDecisions(placementDecision.Namespace).Get(placementDecisionName) switch { case errors.IsNotFound(err): - // create the placementdecision if not exists - owner := metav1.NewControllerRef(placement, clusterapiv1beta1.GroupVersion.WithKind("Placement")) - placementDecision = &clusterapiv1beta1.PlacementDecision{ - ObjectMeta: metav1.ObjectMeta{ - Name: placementDecisionName, - Namespace: placement.Namespace, - Labels: map[string]string{ - clusterapiv1beta1.PlacementLabel: placement.Name, - }, - OwnerReferences: []metav1.OwnerReference{*owner}, - }, - } var err error - placementDecision, err = c.clusterClient.ClusterV1beta1().PlacementDecisions( + existPlacementDecision, err = c.clusterClient.ClusterV1beta1().PlacementDecisions( placement.Namespace).Create(ctx, placementDecision, metav1.CreateOptions{}) if err != nil { return err } c.recorder.Eventf( - placement, placementDecision, corev1.EventTypeNormal, + placement, existPlacementDecision, corev1.EventTypeNormal, "DecisionCreate", "DecisionCreated", - "Decision %s is created with placement %s in namespace %s", placementDecision.Name, placement.Name, placement.Namespace) + "Decision %s is created with placement %s in namespace %s", existPlacementDecision.Name, placement.Name, placement.Namespace) case err != nil: return err } - // update the status of the placementdecision if decisions change - if apiequality.Semantic.DeepEqual(placementDecision.Status.Decisions, clusterDecisions) { - return nil - } + // update the status and labels of the placementdecision if decisions change + placementDecisionPatcher := patcher.NewPatcher[ + *clusterapiv1beta1.PlacementDecision, interface{}, clusterapiv1beta1.PlacementDecisionStatus]( + c.clusterClient.ClusterV1beta1().PlacementDecisions(placementDecision.Namespace)) - newPlacementDecision := placementDecision.DeepCopy() + newPlacementDecision := existPlacementDecision.DeepCopy() + newPlacementDecision.Labels = placementDecision.Labels newPlacementDecision.Status.Decisions = clusterDecisions - newPlacementDecision, err = c.clusterClient.ClusterV1beta1().PlacementDecisions(newPlacementDecision.Namespace). - UpdateStatus(ctx, newPlacementDecision, metav1.UpdateOptions{}) - + updated, err := placementDecisionPatcher.PatchStatus(ctx, newPlacementDecision, newPlacementDecision.Status, existPlacementDecision.Status) + // If status has been updated, just return, this is to avoid conflict when updating the label later. + // Labels and annotations will still be updated in next reconcile. + if updated { + return err + } + _, err = placementDecisionPatcher.PatchLabelAnnotations(ctx, newPlacementDecision, newPlacementDecision.ObjectMeta, existPlacementDecision.ObjectMeta) if err != nil { return err } @@ -574,16 +744,16 @@ func (c *schedulingController) createOrUpdatePlacementDecision( // update the event with warning if status.Code() == framework.Warning { c.recorder.Eventf( - placement, placementDecision, corev1.EventTypeWarning, + placement, existPlacementDecision, corev1.EventTypeWarning, "DecisionUpdate", "DecisionUpdated", - "Decision %s is updated with placement %s in namespace %s: %s in plugin %s", placementDecision.Name, placement.Name, placement.Namespace, + "Decision %s is updated with placement %s in namespace %s: %s in plugin %s", existPlacementDecision.Name, placement.Name, placement.Namespace, status.Message(), status.Plugin()) } else { c.recorder.Eventf( - placement, placementDecision, corev1.EventTypeNormal, + placement, existPlacementDecision, corev1.EventTypeNormal, "DecisionUpdate", "DecisionUpdated", - "Decision %s is updated with placement %s in namespace %s", placementDecision.Name, placement.Name, placement.Namespace) + "Decision %s is updated with placement %s in namespace %s", existPlacementDecision.Name, placement.Name, placement.Namespace) } // update the event with prioritizer score. @@ -599,9 +769,36 @@ func (c *schedulingController) createOrUpdatePlacementDecision( } c.recorder.Eventf( - placement, placementDecision, corev1.EventTypeNormal, + placement, existPlacementDecision, corev1.EventTypeNormal, "ScoreUpdate", "ScoreUpdated", scoreStr) return nil } + +func calculateLength(intOrStr *intstr.IntOrString, total int) (int, *framework.Status) { + length := total + + switch intOrStr.Type { + case intstr.Int: + length = intOrStr.IntValue() + case intstr.String: + str := intOrStr.StrVal + if strings.HasSuffix(str, "%") { + f, err := strconv.ParseFloat(str[:len(str)-1], 64) + if err != nil { + msg := fmt.Sprintf("%v invalid type: string is not a percentage", intOrStr) + return length, framework.NewStatus("", framework.Misconfigured, msg) + } + length = int(math.Ceil(f / 100 * float64(total))) + } else { + msg := fmt.Sprintf("%v invalid type: string is not a percentage", intOrStr) + return length, framework.NewStatus("", framework.Misconfigured, msg) + } + } + + if length <= 0 || length > total { + length = total + } + return length, framework.NewStatus("", framework.Success, "") +} diff --git a/pkg/placement/controllers/scheduling/scheduling_controller_test.go b/pkg/placement/controllers/scheduling/scheduling_controller_test.go index 904eee269..e30d9d657 100644 --- a/pkg/placement/controllers/scheduling/scheduling_controller_test.go +++ b/pkg/placement/controllers/scheduling/scheduling_controller_test.go @@ -2,13 +2,16 @@ package scheduling import ( "context" + "encoding/json" "fmt" + "reflect" "sort" "strings" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" clienttesting "k8s.io/client-go/testing" @@ -56,26 +59,39 @@ func TestSchedulingController_sync(t *testing.T) { testinghelpers.NewManagedCluster("cluster2").Build(), testinghelpers.NewManagedCluster("cluster3").Build(), }, - scheduledDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, - {ClusterName: "cluster2"}, - {ClusterName: "cluster3"}, + scheduledDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").Build(), + testinghelpers.NewManagedCluster("cluster2").Build(), + testinghelpers.NewManagedCluster("cluster3").Build(), }, unscheduledDecisions: 0, }, validateActions: func(t *testing.T, actions []clienttesting.Action) { // check if PlacementDecision has been updated - testingcommon.AssertActions(t, actions, "create", "update", "update") + testingcommon.AssertActions(t, actions, "create", "patch") // check if Placement has been updated - actual := actions[2].(clienttesting.UpdateActionImpl).Object - placement, ok := actual.(*clusterapiv1beta1.Placement) - if !ok { - t.Errorf("expected Placement was updated") + placement := &clusterapiv1beta1.Placement{} + patchData := actions[1].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placement) + if err != nil { + t.Fatal(err) } if placement.Status.NumberOfSelectedClusters != int32(3) { t.Errorf("expecte %d cluster selected, but got %d", 3, placement.Status.NumberOfSelectedClusters) } + expectDecisionGroups := []clusterapiv1beta1.DecisionGroupStatus{ + { + DecisionGroupIndex: 0, + DecisionGroupName: "", + Decisions: []string{"placement1-decision-0"}, + ClustersCount: 3, + }, + } + if !reflect.DeepEqual(placement.Status.DecisionGroups, expectDecisionGroups) { + t.Errorf("expect %v cluster decision gorups, but got %v", expectDecisionGroups, placement.Status.DecisionGroups) + } + util.HasCondition( placement.Status.Conditions, clusterapiv1beta1.PlacementConditionSatisfied, @@ -99,26 +115,30 @@ func TestSchedulingController_sync(t *testing.T) { testinghelpers.NewManagedCluster("cluster2").Build(), testinghelpers.NewManagedCluster("cluster3").Build(), }, - scheduledDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, - {ClusterName: "cluster2"}, - {ClusterName: "cluster3"}, + scheduledDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").Build(), + testinghelpers.NewManagedCluster("cluster2").Build(), + testinghelpers.NewManagedCluster("cluster3").Build(), }, unscheduledDecisions: 1, }, validateActions: func(t *testing.T, actions []clienttesting.Action) { // check if PlacementDecision has been updated - testingcommon.AssertActions(t, actions, "create", "update", "update") + testingcommon.AssertActions(t, actions, "create", "patch") // check if Placement has been updated - actual := actions[2].(clienttesting.UpdateActionImpl).Object - placement, ok := actual.(*clusterapiv1beta1.Placement) - if !ok { - t.Errorf("expected Placement was updated") + placement := &clusterapiv1beta1.Placement{} + patchData := actions[1].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placement) + if err != nil { + t.Fatal(err) } if placement.Status.NumberOfSelectedClusters != int32(3) { t.Errorf("expecte %d cluster selected, but got %d", 3, placement.Status.NumberOfSelectedClusters) } + if len(placement.Status.DecisionGroups) != 1 || placement.Status.DecisionGroups[0].ClustersCount != 3 { + t.Errorf("expecte %d cluster decision gorups, but got %d", 1, len(placement.Status.DecisionGroups)) + } util.HasCondition( placement.Status.Conditions, clusterapiv1beta1.PlacementConditionSatisfied, @@ -127,17 +147,224 @@ func TestSchedulingController_sync(t *testing.T) { ) }, }, + { + name: "placement with canary group strategy", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithGroupStrategy(clusterapiv1beta1.GroupStrategy{ + ClustersPerDecisionGroup: intstr.FromInt(1), + DecisionGroups: []clusterapiv1beta1.DecisionGroup{ + { + GroupName: "canary", + ClusterSelector: clusterapiv1beta1.ClusterSelector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"cloud": "Azure"}}, + }, + }, + }}).Build(), + scheduleResult: &scheduleResult{ + feasibleClusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").Build(), + testinghelpers.NewManagedCluster("cluster2").Build(), + testinghelpers.NewManagedCluster("cluster3").Build(), + testinghelpers.NewManagedCluster("cluster4").Build(), + }, + scheduledDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster3").WithLabel("cloud", "Azure").Build(), + testinghelpers.NewManagedCluster("cluster4").WithLabel("cloud", "Azure").Build(), + }, + unscheduledDecisions: 0, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertActions(t, actions, "create", "create", "create", "patch") + // check if Placement has been updated + placement := &clusterapiv1beta1.Placement{} + patchData := actions[3].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placement) + if err != nil { + t.Fatal(err) + } + + if placement.Status.NumberOfSelectedClusters != int32(4) { + t.Errorf("expecte %d cluster selected, but got %d", 4, placement.Status.NumberOfSelectedClusters) + } + + expectDecisionGroups := []clusterapiv1beta1.DecisionGroupStatus{ + { + DecisionGroupIndex: 0, + DecisionGroupName: "canary", + Decisions: []string{"placement1-decision-0"}, + ClustersCount: 2, + }, + { + DecisionGroupIndex: 1, + DecisionGroupName: "", + Decisions: []string{"placement1-decision-1"}, + ClustersCount: 1, + }, + { + DecisionGroupIndex: 2, + DecisionGroupName: "", + Decisions: []string{"placement1-decision-2"}, + ClustersCount: 1, + }, + } + if !reflect.DeepEqual(placement.Status.DecisionGroups, expectDecisionGroups) { + t.Errorf("expect %v cluster decision gorups, but got %v", expectDecisionGroups, placement.Status.DecisionGroups) + } + + util.HasCondition( + placement.Status.Conditions, + clusterapiv1beta1.PlacementConditionSatisfied, + "AllDecisionsScheduled", + metav1.ConditionTrue, + ) + }, + }, + { + name: "placement with multiple group strategy", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithGroupStrategy(clusterapiv1beta1.GroupStrategy{ + DecisionGroups: []clusterapiv1beta1.DecisionGroup{ + { + GroupName: "group1", + ClusterSelector: clusterapiv1beta1.ClusterSelector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"cloud": "Amazon"}}, + }, + }, + { + GroupName: "group2", + ClusterSelector: clusterapiv1beta1.ClusterSelector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"cloud": "Azure"}}, + }, + }, + }}).Build(), + scheduleResult: &scheduleResult{ + feasibleClusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").Build(), + testinghelpers.NewManagedCluster("cluster2").Build(), + testinghelpers.NewManagedCluster("cluster3").Build(), + }, + scheduledDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster3").WithLabel("cloud", "Azure").Build(), + }, + unscheduledDecisions: 0, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertActions(t, actions, "create", "create", "patch") + // check if Placement has been updated + placement := &clusterapiv1beta1.Placement{} + patchData := actions[2].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placement) + if err != nil { + t.Fatal(err) + } + + if placement.Status.NumberOfSelectedClusters != int32(3) { + t.Errorf("expecte %d cluster selected, but got %d", 3, placement.Status.NumberOfSelectedClusters) + } + + expectDecisionGroups := []clusterapiv1beta1.DecisionGroupStatus{ + { + DecisionGroupIndex: 0, + DecisionGroupName: "group1", + Decisions: []string{"placement1-decision-0"}, + ClustersCount: 2, + }, + { + DecisionGroupIndex: 1, + DecisionGroupName: "group2", + Decisions: []string{"placement1-decision-1"}, + ClustersCount: 1, + }, + } + if !reflect.DeepEqual(placement.Status.DecisionGroups, expectDecisionGroups) { + t.Errorf("expect %v cluster decision gorups, but got %v", expectDecisionGroups, placement.Status.DecisionGroups) + } + + util.HasCondition( + placement.Status.Conditions, + clusterapiv1beta1.PlacementConditionSatisfied, + "AllDecisionsScheduled", + metav1.ConditionTrue, + ) + }, + }, + { + name: "placement with only cluster per decision group", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithGroupStrategy(clusterapiv1beta1.GroupStrategy{ + ClustersPerDecisionGroup: intstr.FromString("25%"), + }).Build(), + scheduleResult: &scheduleResult{ + feasibleClusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").Build(), + testinghelpers.NewManagedCluster("cluster2").Build(), + testinghelpers.NewManagedCluster("cluster3").Build(), + }, + scheduledDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster3").WithLabel("cloud", "Azure").Build(), + }, + unscheduledDecisions: 0, + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertActions(t, actions, "create", "create", "create", "patch") + // check if Placement has been updated + placement := &clusterapiv1beta1.Placement{} + patchData := actions[3].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placement) + if err != nil { + t.Fatal(err) + } + + if placement.Status.NumberOfSelectedClusters != int32(3) { + t.Errorf("expecte %d cluster selected, but got %d", 3, placement.Status.NumberOfSelectedClusters) + } + + expectDecisionGroups := []clusterapiv1beta1.DecisionGroupStatus{ + { + DecisionGroupIndex: 0, + DecisionGroupName: "", + Decisions: []string{"placement1-decision-0"}, + ClustersCount: 1, + }, + { + DecisionGroupIndex: 1, + DecisionGroupName: "", + Decisions: []string{"placement1-decision-1"}, + ClustersCount: 1, + }, + { + DecisionGroupIndex: 2, + DecisionGroupName: "", + Decisions: []string{"placement1-decision-2"}, + ClustersCount: 1, + }, + } + if !reflect.DeepEqual(placement.Status.DecisionGroups, expectDecisionGroups) { + t.Errorf("expect %v cluster decision gorups, but got %v", expectDecisionGroups, placement.Status.DecisionGroups) + } + + util.HasCondition( + placement.Status.Conditions, + clusterapiv1beta1.PlacementConditionSatisfied, + "AllDecisionsScheduled", + metav1.ConditionTrue, + ) + }, + }, { name: "placement missing managedclustersetbindings", placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), scheduleResult: &scheduleResult{ feasibleClusters: []*clusterapiv1.ManagedCluster{}, - scheduledDecisions: []clusterapiv1beta1.ClusterDecision{}, + scheduledDecisions: []*clusterapiv1.ManagedCluster{}, unscheduledDecisions: 0, }, validateActions: func(t *testing.T, actions []clienttesting.Action) { // check if PlacementDecision has been updated - testingcommon.AssertActions(t, actions, "create", "update") + testingcommon.AssertActions(t, actions, "create", "patch") // check if emtpy PlacementDecision has been created actual := actions[0].(clienttesting.CreateActionImpl).Object placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) @@ -149,15 +376,19 @@ func TestSchedulingController_sync(t *testing.T) { t.Errorf("expecte %d cluster selected, but got %d", 0, len(placementDecision.Status.Decisions)) } // check if Placement has been updated - actual = actions[1].(clienttesting.UpdateActionImpl).Object - placement, ok := actual.(*clusterapiv1beta1.Placement) - if !ok { - t.Errorf("expected Placement was updated") + placement := &clusterapiv1beta1.Placement{} + patchData := actions[1].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placement) + if err != nil { + t.Fatal(err) } if placement.Status.NumberOfSelectedClusters != int32(0) { t.Errorf("expecte %d cluster selected, but got %d", 0, placement.Status.NumberOfSelectedClusters) } + if len(placement.Status.DecisionGroups) != 1 || placement.Status.DecisionGroups[0].ClustersCount != 0 { + t.Errorf("expecte %d cluster decision gorups, but got %d", 1, len(placement.Status.DecisionGroups)) + } util.HasCondition( placement.Status.Conditions, clusterapiv1beta1.PlacementConditionSatisfied, @@ -175,12 +406,12 @@ func TestSchedulingController_sync(t *testing.T) { }, scheduleResult: &scheduleResult{ feasibleClusters: []*clusterapiv1.ManagedCluster{}, - scheduledDecisions: []clusterapiv1beta1.ClusterDecision{}, + scheduledDecisions: []*clusterapiv1.ManagedCluster{}, unscheduledDecisions: 0, }, validateActions: func(t *testing.T, actions []clienttesting.Action) { // check if PlacementDecision has been updated - testingcommon.AssertActions(t, actions, "create", "update") + testingcommon.AssertActions(t, actions, "create", "patch") // check if emtpy PlacementDecision has been created actual := actions[0].(clienttesting.CreateActionImpl).Object placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) @@ -192,15 +423,19 @@ func TestSchedulingController_sync(t *testing.T) { t.Errorf("expecte %d cluster selected, but got %d", 0, len(placementDecision.Status.Decisions)) } // check if Placement has been updated - actual = actions[1].(clienttesting.UpdateActionImpl).Object - placement, ok := actual.(*clusterapiv1beta1.Placement) - if !ok { - t.Errorf("expected Placement was updated") + placement := &clusterapiv1beta1.Placement{} + patchData := actions[1].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placement) + if err != nil { + t.Fatal(err) } if placement.Status.NumberOfSelectedClusters != int32(0) { t.Errorf("expecte %d cluster selected, but got %d", 0, placement.Status.NumberOfSelectedClusters) } + if len(placement.Status.DecisionGroups) != 1 || placement.Status.DecisionGroups[0].ClustersCount != 0 { + t.Errorf("expecte %d cluster decision gorups, but got %d", 1, len(placement.Status.DecisionGroups)) + } util.HasCondition( placement.Status.Conditions, clusterapiv1beta1.PlacementConditionSatisfied, @@ -221,12 +456,12 @@ func TestSchedulingController_sync(t *testing.T) { feasibleClusters: []*clusterapiv1.ManagedCluster{ testinghelpers.NewManagedCluster("cluster1").Build(), }, - scheduledDecisions: []clusterapiv1beta1.ClusterDecision{}, + scheduledDecisions: []*clusterapiv1.ManagedCluster{}, unscheduledDecisions: 0, }, validateActions: func(t *testing.T, actions []clienttesting.Action) { // check if PlacementDecision has been updated - testingcommon.AssertActions(t, actions, "create", "update") + testingcommon.AssertActions(t, actions, "create", "patch") // check if emtpy PlacementDecision has been created actual := actions[0].(clienttesting.CreateActionImpl).Object placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) @@ -238,15 +473,19 @@ func TestSchedulingController_sync(t *testing.T) { t.Errorf("expecte %d cluster selected, but got %d", 0, len(placementDecision.Status.Decisions)) } // check if Placement has been updated - actual = actions[1].(clienttesting.UpdateActionImpl).Object - placement, ok := actual.(*clusterapiv1beta1.Placement) - if !ok { - t.Errorf("expected Placement was updated") + placement := &clusterapiv1beta1.Placement{} + patchData := actions[1].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placement) + if err != nil { + t.Fatal(err) } if placement.Status.NumberOfSelectedClusters != int32(0) { t.Errorf("expecte %d cluster selected, but got %d", 0, placement.Status.NumberOfSelectedClusters) } + if len(placement.Status.DecisionGroups) != 1 || placement.Status.DecisionGroups[0].ClustersCount != 0 { + t.Errorf("expecte %d cluster decision gorups, but got %d", 1, len(placement.Status.DecisionGroups)) + } util.HasCondition( placement.Status.Conditions, clusterapiv1beta1.PlacementConditionSatisfied, @@ -258,13 +497,15 @@ func TestSchedulingController_sync(t *testing.T) { { name: "placement status not changed", placement: testinghelpers.NewPlacement(placementNamespace, placementName). - WithNumOfSelectedClusters(3).WithSatisfiedCondition(3, 0).WithMisconfiguredCondition(metav1.ConditionFalse).Build(), + WithNumOfSelectedClusters(3, placementName).WithSatisfiedCondition(3, 0).WithMisconfiguredCondition(metav1.ConditionFalse).Build(), initObjs: []runtime.Object{ testinghelpers.NewClusterSet("clusterset1").Build(), testinghelpers.NewClusterSetBinding(placementNamespace, "clusterset1"), testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterapiv1beta2.ClusterSetLabel, "clusterset1").Build(), - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 0)). WithLabel(clusterapiv1beta1.PlacementLabel, placementName). + WithLabel(clusterapiv1beta1.DecisionGroupNameLabel, ""). + WithLabel(clusterapiv1beta1.DecisionGroupIndexLabel, "0"). WithDecisions("cluster1", "cluster2", "cluster3").Build(), }, scheduleResult: &scheduleResult{ @@ -273,10 +514,10 @@ func TestSchedulingController_sync(t *testing.T) { testinghelpers.NewManagedCluster("cluster2").Build(), testinghelpers.NewManagedCluster("cluster3").Build(), }, - scheduledDecisions: []clusterapiv1beta1.ClusterDecision{ - {ClusterName: "cluster1"}, - {ClusterName: "cluster2"}, - {ClusterName: "cluster3"}, + scheduledDecisions: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").Build(), + testinghelpers.NewManagedCluster("cluster2").Build(), + testinghelpers.NewManagedCluster("cluster3").Build(), }, unscheduledDecisions: 0, }, @@ -294,7 +535,7 @@ func TestSchedulingController_sync(t *testing.T) { testinghelpers.NewManagedCluster("cluster2").Build(), testinghelpers.NewManagedCluster("cluster3").Build(), }, - scheduledDecisions: []clusterapiv1beta1.ClusterDecision{}, + scheduledDecisions: []*clusterapiv1.ManagedCluster{}, }, validateActions: testingcommon.AssertNoActions, }, @@ -732,48 +973,64 @@ func TestBind(t *testing.T) { placementName := "placement1" cases := []struct { - name string - initObjs []runtime.Object - clusterDecisions []clusterapiv1beta1.ClusterDecision - validateActions func(t *testing.T, actions []clienttesting.Action) + name string + initObjs []runtime.Object + placement *clusterapiv1beta1.Placement + clusters []*clusterapiv1.ManagedCluster + validateActions func(t *testing.T, actions []clienttesting.Action) }{ { - name: "create single placementdecision", - clusterDecisions: newClusterDecisions(10), + name: "create single placementdecision", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + clusters: newClusters(10), validateActions: func(t *testing.T, actions []clienttesting.Action) { - testingcommon.AssertActions(t, actions, "create", "update") - actual := actions[1].(clienttesting.UpdateActionImpl).Object + testingcommon.AssertActions(t, actions, "create") + actual := actions[0].(clienttesting.CreateActionImpl).Object placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) if !ok { t.Errorf("expected PlacementDecision was updated") } assertClustersSelected(t, placementDecision.Status.Decisions, newSelectedClusters(10)...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "0" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } }, }, { - name: "create multiple placementdecisions", - clusterDecisions: newClusterDecisions(101), + name: "create multiple placementdecisions", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + clusters: newClusters(101), validateActions: func(t *testing.T, actions []clienttesting.Action) { - testingcommon.AssertActions(t, actions, "create", "update", "create", "update") + testingcommon.AssertActions(t, actions, "create", "create") selectedClusters := newSelectedClusters(101) - actual := actions[1].(clienttesting.UpdateActionImpl).Object + actual := actions[0].(clienttesting.CreateActionImpl).Object placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) if !ok { t.Errorf("expected PlacementDecision was updated") } assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters[0:100]...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "0" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } - actual = actions[3].(clienttesting.UpdateActionImpl).Object + actual = actions[1].(clienttesting.CreateActionImpl).Object placementDecision, ok = actual.(*clusterapiv1beta1.PlacementDecision) if !ok { t.Errorf("expected PlacementDecision was updated") } assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters[100:]...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "0" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } }, }, { - name: "create empty placementdecision", - clusterDecisions: newClusterDecisions(0), + name: "create empty placementdecision", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + clusters: newClusters(0), validateActions: func(t *testing.T, actions []clienttesting.Action) { testingcommon.AssertActions(t, actions, "create") actual := actions[0].(clienttesting.CreateActionImpl).Object @@ -787,30 +1044,249 @@ func TestBind(t *testing.T) { }, }, { - name: "no change", - clusterDecisions: newClusterDecisions(128), + name: "create placementdecision with canary group strategy", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithGroupStrategy(clusterapiv1beta1.GroupStrategy{ + ClustersPerDecisionGroup: intstr.FromString("25%"), + DecisionGroups: []clusterapiv1beta1.DecisionGroup{ + { + GroupName: "canary", + ClusterSelector: clusterapiv1beta1.ClusterSelector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"cloud": "Azure"}}, + }, + }, + }}).Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster3").WithLabel("cloud", "Azure").Build(), + testinghelpers.NewManagedCluster("cluster4").WithLabel("cloud", "Azure").Build(), + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertActions(t, actions, "create", "create", "create") + selectedClusters := newSelectedClusters(4) + actual := actions[0].(clienttesting.CreateActionImpl).Object + placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was created") + } + assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters[2:]...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "0" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "canary" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + + actual = actions[1].(clienttesting.CreateActionImpl).Object + placementDecision, ok = actual.(*clusterapiv1beta1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was created") + } + assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters[0:1]...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "1" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + + actual = actions[2].(clienttesting.CreateActionImpl).Object + placementDecision, ok = actual.(*clusterapiv1beta1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was created") + } + assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters[1:2]...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "2" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + }, + }, + { + name: "create placementdecision when no cluster selected by canary group strategy", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithGroupStrategy(clusterapiv1beta1.GroupStrategy{ + DecisionGroups: []clusterapiv1beta1.DecisionGroup{ + { + GroupName: "canary", + ClusterSelector: clusterapiv1beta1.ClusterSelector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"cloud": "Azure"}}, + }, + }, + }}).Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster3").WithLabel("cloud", "Amazon").Build(), + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertActions(t, actions, "create") + selectedClusters := newSelectedClusters(3) + actual := actions[0].(clienttesting.CreateActionImpl).Object + placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was created") + } + assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "0" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + }, + }, + { + name: "create placementdecision with multiple group strategy", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithGroupStrategy(clusterapiv1beta1.GroupStrategy{ + DecisionGroups: []clusterapiv1beta1.DecisionGroup{ + { + GroupName: "group1", + ClusterSelector: clusterapiv1beta1.ClusterSelector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"cloud": "Amazon"}}, + }, + }, + { + GroupName: "group2", + ClusterSelector: clusterapiv1beta1.ClusterSelector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"cloud": "Azure"}}, + }, + }, + }}).Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster3").WithLabel("cloud", "Azure").Build(), + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertActions(t, actions, "create", "create") + selectedClusters := newSelectedClusters(4) + actual := actions[0].(clienttesting.CreateActionImpl).Object + placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was created") + } + assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters[0:2]...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "0" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "group1" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + + actual = actions[1].(clienttesting.CreateActionImpl).Object + placementDecision, ok = actual.(*clusterapiv1beta1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was created") + } + assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters[2:3]...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "1" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "group2" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + }, + }, + { + name: "create placementdecision with only cluster per decision group", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).WithGroupStrategy(clusterapiv1beta1.GroupStrategy{ + ClustersPerDecisionGroup: intstr.FromString("25%"), + }).Build(), + clusters: []*clusterapiv1.ManagedCluster{ + testinghelpers.NewManagedCluster("cluster1").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster2").WithLabel("cloud", "Amazon").Build(), + testinghelpers.NewManagedCluster("cluster3").WithLabel("cloud", "Azure").Build(), + }, + validateActions: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertActions(t, actions, "create", "create", "create") + selectedClusters := newSelectedClusters(4) + actual := actions[0].(clienttesting.CreateActionImpl).Object + placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was created") + } + assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters[0:1]...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "0" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + + actual = actions[1].(clienttesting.CreateActionImpl).Object + placementDecision, ok = actual.(*clusterapiv1beta1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was created") + } + assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters[1:2]...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "1" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + + actual = actions[2].(clienttesting.CreateActionImpl).Object + placementDecision, ok = actual.(*clusterapiv1beta1.PlacementDecision) + if !ok { + t.Errorf("expected PlacementDecision was created") + } + assertClustersSelected(t, placementDecision.Status.Decisions, selectedClusters[2:3]...) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupIndexLabel] != "2" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + }, + }, + { + name: "no change", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + clusters: newClusters(128), initObjs: []runtime.Object{ - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 0)). WithLabel(clusterapiv1beta1.PlacementLabel, placementName). + WithLabel(clusterapiv1beta1.DecisionGroupNameLabel, ""). + WithLabel(clusterapiv1beta1.DecisionGroupIndexLabel, "0"). WithDecisions(newSelectedClusters(128)[:100]...).Build(), - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 2)). + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). WithLabel(clusterapiv1beta1.PlacementLabel, placementName). + WithLabel(clusterapiv1beta1.DecisionGroupNameLabel, ""). + WithLabel(clusterapiv1beta1.DecisionGroupIndexLabel, "0"). WithDecisions(newSelectedClusters(128)[100:]...).Build(), }, validateActions: testingcommon.AssertNoActions, }, { - name: "update one of placementdecisions", - clusterDecisions: newClusterDecisions(128), + name: "update one of placementdecisions", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + clusters: newClusters(128), initObjs: []runtime.Object{ - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 0)). WithLabel(clusterapiv1beta1.PlacementLabel, placementName). + WithLabel(clusterapiv1beta1.DecisionGroupNameLabel, "fakegroup"). + WithLabel(clusterapiv1beta1.DecisionGroupIndexLabel, "0"). WithDecisions(newSelectedClusters(128)[:100]...).Build(), }, validateActions: func(t *testing.T, actions []clienttesting.Action) { - testingcommon.AssertActions(t, actions, "create", "update") + testingcommon.AssertActions(t, actions, "patch", "create") selectedClusters := newSelectedClusters(128) - actual := actions[1].(clienttesting.UpdateActionImpl).Object + placementDecision := &clusterapiv1beta1.PlacementDecision{} + patchData := actions[0].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placementDecision) + if err != nil { + t.Fatal(err) + } + // only update labels + assertClustersSelected(t, placementDecision.Status.Decisions) + if placementDecision.Labels[clusterapiv1beta1.DecisionGroupNameLabel] != "" { + t.Errorf("unexpected PlacementDecision labels %v", placementDecision.Labels) + } + + actual := actions[1].(clienttesting.CreateActionImpl).Object placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) if !ok { t.Errorf("expected PlacementDecision was updated") @@ -819,43 +1295,55 @@ func TestBind(t *testing.T) { }, }, { - name: "delete redundant placementdecisions", - clusterDecisions: newClusterDecisions(10), + name: "delete redundant placementdecisions", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + clusters: newClusters(10), initObjs: []runtime.Object{ - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 0)). WithLabel(clusterapiv1beta1.PlacementLabel, placementName). + WithLabel(clusterapiv1beta1.DecisionGroupNameLabel, ""). + WithLabel(clusterapiv1beta1.DecisionGroupIndexLabel, "0"). WithDecisions(newSelectedClusters(128)[:100]...).Build(), - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 2)). + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). WithLabel(clusterapiv1beta1.PlacementLabel, placementName). + WithLabel(clusterapiv1beta1.DecisionGroupNameLabel, ""). + WithLabel(clusterapiv1beta1.DecisionGroupIndexLabel, "0"). WithDecisions(newSelectedClusters(128)[100:]...).Build(), }, validateActions: func(t *testing.T, actions []clienttesting.Action) { - testingcommon.AssertActions(t, actions, "update", "delete") - actual := actions[0].(clienttesting.UpdateActionImpl).Object - placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) - if !ok { - t.Errorf("expected PlacementDecision was updated") + testingcommon.AssertActions(t, actions, "patch", "delete") + placementDecision := &clusterapiv1beta1.PlacementDecision{} + patchData := actions[0].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placementDecision) + if err != nil { + t.Fatal(err) } assertClustersSelected(t, placementDecision.Status.Decisions, newSelectedClusters(10)...) }, }, { - name: "delete all placementdecisions and leave one empty placementdecision", - clusterDecisions: newClusterDecisions(0), + name: "delete all placementdecisions and leave one empty placementdecision", + placement: testinghelpers.NewPlacement(placementNamespace, placementName).Build(), + clusters: newClusters(0), initObjs: []runtime.Object{ - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 0)). WithLabel(clusterapiv1beta1.PlacementLabel, placementName). + WithLabel(clusterapiv1beta1.DecisionGroupNameLabel, ""). + WithLabel(clusterapiv1beta1.DecisionGroupIndexLabel, "0"). WithDecisions(newSelectedClusters(128)[:100]...).Build(), - testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 2)). + testinghelpers.NewPlacementDecision(placementNamespace, placementDecisionName(placementName, 1)). WithLabel(clusterapiv1beta1.PlacementLabel, placementName). + WithLabel(clusterapiv1beta1.DecisionGroupNameLabel, ""). + WithLabel(clusterapiv1beta1.DecisionGroupIndexLabel, "0"). WithDecisions(newSelectedClusters(128)[100:]...).Build(), }, validateActions: func(t *testing.T, actions []clienttesting.Action) { - testingcommon.AssertActions(t, actions, "update", "delete") - actual := actions[0].(clienttesting.UpdateActionImpl).Object - placementDecision, ok := actual.(*clusterapiv1beta1.PlacementDecision) - if !ok { - t.Errorf("expected PlacementDecision was updated") + testingcommon.AssertActions(t, actions, "patch", "delete") + placementDecision := &clusterapiv1beta1.PlacementDecision{} + patchData := actions[0].(clienttesting.PatchActionImpl).Patch + err := json.Unmarshal(patchData, placementDecision) + if err != nil { + t.Fatal(err) } if len(placementDecision.Status.Decisions) != 0 { t.Errorf("expecte %d cluster selected, but got %d", 0, len(placementDecision.Status.Decisions)) @@ -895,10 +1383,11 @@ func TestBind(t *testing.T) { recorder: kevents.NewFakeRecorder(100), } + decisions, _, _ := ctrl.generatePlacementDecisionsAndStatus(c.placement, c.clusters) err := ctrl.bind( context.TODO(), - testinghelpers.NewPlacement(placementNamespace, placementName).Build(), - c.clusterDecisions, + c.placement, + decisions, nil, nil, ) @@ -923,13 +1412,13 @@ func assertClustersSelected(t *testing.T, decisons []clusterapiv1beta1.ClusterDe } } -func newClusterDecisions(num int) (decisions []clusterapiv1beta1.ClusterDecision) { +func newClusters(num int) (clusters []*clusterapiv1.ManagedCluster) { for i := 0; i < num; i++ { - decisions = append(decisions, clusterapiv1beta1.ClusterDecision{ - ClusterName: fmt.Sprintf("cluster%d", i+1), - }) + ClusterName := fmt.Sprintf("cluster%d", i+1) + c := testinghelpers.NewManagedCluster(ClusterName).Build() + clusters = append(clusters, c) } - return decisions + return clusters } func newSelectedClusters(num int) (clusters []string) { diff --git a/pkg/placement/debugger/debugger_test.go b/pkg/placement/debugger/debugger_test.go index e498295b9..9fd70cabe 100644 --- a/pkg/placement/debugger/debugger_test.go +++ b/pkg/placement/debugger/debugger_test.go @@ -44,8 +44,8 @@ func (r *testResult) PrioritizerScores() scheduling.PrioritizerScore { return r.scoreSum } -func (r *testResult) Decisions() []clusterapiv1beta1.ClusterDecision { - return []clusterapiv1beta1.ClusterDecision{} +func (r *testResult) Decisions() []*clusterapiv1.ManagedCluster { + return []*clusterapiv1.ManagedCluster{} } func (r *testResult) NumOfUnscheduled() int { diff --git a/pkg/placement/helpers/testing/builders.go b/pkg/placement/helpers/testing/builders.go index 46aad4e5b..43a7eb6cb 100644 --- a/pkg/placement/helpers/testing/builders.go +++ b/pkg/placement/helpers/testing/builders.go @@ -52,6 +52,11 @@ func (b *placementBuilder) WithNOC(noc int32) *placementBuilder { return b } +func (b *placementBuilder) WithGroupStrategy(groupStrategy clusterapiv1beta1.GroupStrategy) *placementBuilder { + b.placement.Spec.DecisionStrategy.GroupStrategy = groupStrategy + return b +} + func (b *placementBuilder) WithPrioritizerPolicy(mode clusterapiv1beta1.PrioritizerPolicyModeType) *placementBuilder { b.placement.Spec.PrioritizerPolicy = clusterapiv1beta1.PrioritizerPolicy{ Mode: mode, @@ -118,8 +123,16 @@ func (b *placementBuilder) AddToleration(toleration *clusterapiv1beta1.Toleratio return b } -func (b *placementBuilder) WithNumOfSelectedClusters(nosc int) *placementBuilder { +func (b *placementBuilder) WithNumOfSelectedClusters(nosc int, placementName string) *placementBuilder { b.placement.Status.NumberOfSelectedClusters = int32(nosc) + b.placement.Status.DecisionGroups = []clusterapiv1beta1.DecisionGroupStatus{ + { + DecisionGroupIndex: 0, + DecisionGroupName: "", + ClustersCount: int32(nosc), + Decisions: []string{fmt.Sprintf("%s-decision-%d", placementName, 0)}, + }, + } return b } diff --git a/test/integration/placement/assertion_test.go b/test/integration/placement/assertion_test.go index 35ba05b1b..dca087078 100644 --- a/test/integration/placement/assertion_test.go +++ b/test/integration/placement/assertion_test.go @@ -3,6 +3,7 @@ package placement import ( "context" "fmt" + "reflect" "sort" "time" @@ -79,10 +80,8 @@ func assertPlacementDeleted(placementName, namespace string) { }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) } -func assertNumberOfDecisions(placementName, namespace string, desiredNOD int) { +func assertNumberOfDecisions(placementName, namespace string, desiredNOD, desiredNOPD int) { ginkgo.By("Check the number of decisions in placementdecisions") - // at least one decision for each placement - desiredNOPD := desiredNOD/maxNumOfClusterDecisions + 1 gomega.Eventually(func() bool { pdl, err := clusterClient.ClusterV1beta1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{ LabelSelector: placementLabel + "=" + placementName, @@ -126,6 +125,18 @@ func assertClusterNamesOfDecisions(placementName, namespace string, desiredClust }, eventuallyTimeout*2, eventuallyInterval).Should(gomega.BeTrue()) } +func assertPlacementDecisionGroupStatus(placementName, namespace string, decisionGroupStatus []clusterapiv1beta1.DecisionGroupStatus) { + ginkgo.By("Check the group status of placement") + gomega.Eventually(func() bool { + placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) + if err != nil { + return false + } + ginkgo.By(fmt.Sprintf("actual decision groups %v", placement.Status.DecisionGroups)) + return reflect.DeepEqual(placement.Status.DecisionGroups, decisionGroupStatus) + }, eventuallyTimeout*2, eventuallyInterval).Should(gomega.BeTrue()) +} + func assertPlacementConditionSatisfied(placementName, namespace string, numOfSelectedClusters int, satisfied bool) { ginkgo.By("Check the condition PlacementSatisfied of placement") gomega.Eventually(func() bool { @@ -380,11 +391,11 @@ func assertDeletingClusters(clusterNames ...string) { return false } return errors.IsNotFound(err) - }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + }, eventuallyTimeout*2, eventuallyInterval).Should(gomega.BeTrue()) } } -func assertCreatingPlacement(name, namespace string, noc *int32, prioritizerPolicy clusterapiv1beta1.PrioritizerPolicy, tolerations []clusterapiv1beta1.Toleration) *clusterapiv1beta1.Placement { +func assertCreatingPlacement(name, namespace string, noc *int32, prioritizerPolicy clusterapiv1beta1.PrioritizerPolicy, tolerations []clusterapiv1beta1.Toleration, groupStrategy clusterapiv1beta1.GroupStrategy) *clusterapiv1beta1.Placement { ginkgo.By("Create placement") placement := &clusterapiv1beta1.Placement{ ObjectMeta: metav1.ObjectMeta{ @@ -395,6 +406,7 @@ func assertCreatingPlacement(name, namespace string, noc *int32, prioritizerPoli NumberOfClusters: noc, PrioritizerPolicy: prioritizerPolicy, Tolerations: tolerations, + DecisionStrategy: clusterapiv1beta1.DecisionStrategy{GroupStrategy: groupStrategy}, }, } placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{}) @@ -403,12 +415,12 @@ func assertCreatingPlacement(name, namespace string, noc *int32, prioritizerPoli return placement } -func assertCreatingPlacementWithDecision(name, namespace string, noc *int32, nod int, prioritizerPolicy clusterapiv1beta1.PrioritizerPolicy, tolerations []clusterapiv1beta1.Toleration) { - placement := assertCreatingPlacement(name, namespace, noc, prioritizerPolicy, tolerations) +func assertCreatingPlacementWithDecision(name, namespace string, numberOfClusters *int32, numberOfDecisionClusters, numberOfPlacementDecisions int, prioritizerPolicy clusterapiv1beta1.PrioritizerPolicy, tolerations []clusterapiv1beta1.Toleration, groupStrategy clusterapiv1beta1.GroupStrategy) { + placement := assertCreatingPlacement(name, namespace, numberOfClusters, prioritizerPolicy, tolerations, groupStrategy) assertPlacementDecisionCreated(placement) - assertNumberOfDecisions(name, namespace, nod) - if noc != nil { - assertPlacementConditionSatisfied(name, namespace, nod, nod == int(*noc)) + assertNumberOfDecisions(name, namespace, numberOfDecisionClusters, numberOfPlacementDecisions) + if numberOfClusters != nil { + assertPlacementConditionSatisfied(name, namespace, numberOfDecisionClusters, numberOfDecisionClusters == int(*numberOfClusters)) } } diff --git a/test/integration/placement/placement_test.go b/test/integration/placement/placement_test.go index 0db1592fa..de67d2b99 100644 --- a/test/integration/placement/placement_test.go +++ b/test/integration/placement/placement_test.go @@ -10,6 +10,7 @@ import ( "github.com/openshift/library-go/pkg/controller/controllercmd" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1" @@ -83,7 +84,7 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should re-create placementdecisions successfully once placementdecisions are deleted", func() { assertBindingClusterSet(clusterSet1Name, namespace) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) ginkgo.By("Delete placementdecisions") placementDecisions, err := clusterClient.ClusterV1beta1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{ @@ -99,22 +100,25 @@ var _ = ginkgo.Describe("Placement", func() { placement, err := clusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) assertPlacementDecisionCreated(placement) - assertNumberOfDecisions(placementName, namespace, 5) + assertNumberOfDecisions(placementName, namespace, 5, 1) + assertPlacementDecisionGroupStatus(placementName, namespace, []clusterapiv1beta1.DecisionGroupStatus{{Decisions: []string{placementName + "-decision-0"}, ClustersCount: 5}}) }) ginkgo.It("Should create empty placementdecision when no cluster selected", func() { - placement := assertCreatingPlacement(placementName, namespace, nil, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + placement := assertCreatingPlacement(placementName, namespace, nil, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) assertPlacementDecisionCreated(placement) - assertNumberOfDecisions(placementName, namespace, 0) + assertNumberOfDecisions(placementName, namespace, 0, 1) + assertPlacementDecisionGroupStatus(placementName, namespace, []clusterapiv1beta1.DecisionGroupStatus{{Decisions: []string{placementName + "-decision-0"}, ClustersCount: 0}}) }) ginkgo.It("Should create multiple placementdecisions once scheduled", func() { assertBindingClusterSet(clusterSet1Name, namespace) assertCreatingClusters(clusterSet1Name, 101) - assertCreatingPlacementWithDecision(placementName, namespace, nil, 101, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, nil, 101, 2, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) nod := 101 - assertNumberOfDecisions(placementName, namespace, nod) + assertNumberOfDecisions(placementName, namespace, nod, 2) + assertPlacementDecisionGroupStatus(placementName, namespace, []clusterapiv1beta1.DecisionGroupStatus{{Decisions: []string{placementName + "-decision-0", placementName + "-decision-1"}, ClustersCount: 101}}) assertPlacementConditionSatisfied(placementName, namespace, nod, true) }) @@ -123,19 +127,19 @@ var _ = ginkgo.Describe("Placement", func() { assertBindingClusterSet(clusterSet2Name, namespace) assertCreatingClusters(clusterSet1Name, 2) assertCreatingClusters(clusterSet2Name, 3) - assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) // update ClusterSets assertUpdatingPlacement(placementName, namespace, noc(10), []string{clusterSet1Name}, []clusterapiv1beta1.ClusterPredicate{}, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) - assertNumberOfDecisions(placementName, namespace, 2) + assertNumberOfDecisions(placementName, namespace, 2, 1) }) ginkgo.It("Should schedule placement successfully once spec.Predicates LabelSelector changes", func() { assertBindingClusterSet(clusterSet1Name, namespace) assertCreatingClusters(clusterSet1Name, 2, "cloud", "Azure") assertCreatingClusters(clusterSet1Name, 3, "cloud", "Amazon") - assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) ginkgo.By("add the predicates") // add a predicates @@ -151,7 +155,8 @@ var _ = ginkgo.Describe("Placement", func() { }, } assertUpdatingPlacement(placementName, namespace, noc(10), []string{}, predicates, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) - assertNumberOfDecisions(placementName, namespace, 3) + assertNumberOfDecisions(placementName, namespace, 3, 1) + assertPlacementDecisionGroupStatus(placementName, namespace, []clusterapiv1beta1.DecisionGroupStatus{{Decisions: []string{placementName + "-decision-0"}, ClustersCount: 3}}) ginkgo.By("change the predicates") // change the predicates @@ -167,15 +172,14 @@ var _ = ginkgo.Describe("Placement", func() { }, } assertUpdatingPlacement(placementName, namespace, noc(10), []string{}, predicates, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) - assertNumberOfDecisions(placementName, namespace, 2) + assertNumberOfDecisions(placementName, namespace, 2, 1) }) ginkgo.It("Should schedule placement successfully once spec.Predicates ClaimSelector changes", func() { assertBindingClusterSet(clusterSet1Name, namespace) assertCreatingClusters(clusterSet1Name, 2, "cloud", "Azure") assertCreatingClusters(clusterSet1Name, 3, "cloud", "Amazon") - assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) - + assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) ginkgo.By("add the predicates") // add a predicates predicates := []clusterapiv1beta1.ClusterPredicate{ @@ -194,7 +198,7 @@ var _ = ginkgo.Describe("Placement", func() { }, } assertUpdatingPlacement(placementName, namespace, noc(10), []string{}, predicates, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) - assertNumberOfDecisions(placementName, namespace, 3) + assertNumberOfDecisions(placementName, namespace, 3, 1) ginkgo.By("change the predicates") // change the predicates @@ -214,34 +218,33 @@ var _ = ginkgo.Describe("Placement", func() { }, } assertUpdatingPlacement(placementName, namespace, noc(10), []string{}, predicates, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) - assertNumberOfDecisions(placementName, namespace, 2) + assertNumberOfDecisions(placementName, namespace, 2, 1) }) ginkgo.It("Should schedule successfully once spec.NumberOfClusters is reduced", func() { assertBindingClusterSet(clusterSet1Name, namespace) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) - + assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) ginkgo.By("Reduce NOC of the placement") noc := int32(4) assertUpdatingPlacement(placementName, namespace, &noc, []string{}, []clusterapiv1beta1.ClusterPredicate{}, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) nod := int(noc) - assertNumberOfDecisions(placementName, namespace, nod) + assertNumberOfDecisions(placementName, namespace, nod, 1) assertPlacementConditionSatisfied(placementName, namespace, nod, true) }) ginkgo.It("Should schedule successfully once spec.NumberOfClusters is increased", func() { assertBindingClusterSet(clusterSet1Name, namespace) assertCreatingClusters(clusterSet1Name, 10) - assertCreatingPlacementWithDecision(placementName, namespace, noc(5), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(5), 5, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) ginkgo.By("Increase NOC of the placement") noc := int32(8) assertUpdatingPlacement(placementName, namespace, &noc, []string{}, []clusterapiv1beta1.ClusterPredicate{}, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) nod := int(noc) - assertNumberOfDecisions(placementName, namespace, nod) + assertNumberOfDecisions(placementName, namespace, nod, 1) assertPlacementConditionSatisfied(placementName, namespace, nod, true) }) @@ -255,12 +258,12 @@ var _ = ginkgo.Describe("Placement", func() { Operator: clusterapiv1beta1.TolerationOpExists, Value: "value1", }, - }) - assertNumberOfDecisions(placementName, namespace, 0) + }, clusterapiv1beta1.GroupStrategy{}) + assertNumberOfDecisions(placementName, namespace, 0, 1) assertPlacementConditionMisconfigured(placementName, namespace, true) assertUpdatingPlacement(placementName, namespace, noc(1), []string{}, []clusterapiv1beta1.ClusterPredicate{}, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) - assertNumberOfDecisions(placementName, namespace, 1) + assertNumberOfDecisions(placementName, namespace, 1, 1) assertPlacementConditionMisconfigured(placementName, namespace, false) assertPlacementConditionSatisfied(placementName, namespace, 1, true) }) @@ -268,14 +271,14 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should be satisfied once new clusters are added", func() { assertBindingClusterSet(clusterSet1Name, namespace) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) // add more clusters ginkgo.By("Add the cluster") assertCreatingClusters(clusterSet1Name, 5) nod := 10 - assertNumberOfDecisions(placementName, namespace, nod) + assertNumberOfDecisions(placementName, namespace, nod, 1) assertPlacementConditionSatisfied(placementName, namespace, nod, true) }) @@ -284,21 +287,129 @@ var _ = ginkgo.Describe("Placement", func() { assertCreatingClusterSet(clusterSet1Name, "vendor", "openShift") assertCreatingClusterSetBinding(clusterSet1Name, namespace) clusters1 := assertCreatingClusters(clusterName+"1", 1, "vendor", "openShift") - assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 1, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) // add more clusters ginkgo.By("Add the cluster") clusters2 := assertCreatingClusters(clusterName+"2", 1, "vendor", "openShift") - assertNumberOfDecisions(placementName, namespace, 2) + assertNumberOfDecisions(placementName, namespace, 2, 1) assertPlacementConditionSatisfied(placementName, namespace, 2, true) ginkgo.By("Delete the cluster") assertDeletingClusters(clusters1...) - assertNumberOfDecisions(placementName, namespace, 1) + assertNumberOfDecisions(placementName, namespace, 1, 1) assertDeletingClusters(clusters2...) - assertNumberOfDecisions(placementName, namespace, 0) + assertNumberOfDecisions(placementName, namespace, 0, 1) + }) + + ginkgo.It("Should update the decision group once clusters added/deleted", func() { + ginkgo.By("Bind clusterset to the placement namespace") + assertCreatingClusterSet("global") + assertCreatingClusterSetBinding("global", namespace) + + canary := assertCreatingClusters(clusterSet1Name, 3, "vendor", "openShift") + noncanary := assertCreatingClusters(clusterSet1Name, 3) + assertCreatingPlacementWithDecision(placementName, namespace, nil, 6, 3, + clusterapiv1beta1.PrioritizerPolicy{}, + []clusterapiv1beta1.Toleration{}, + clusterapiv1beta1.GroupStrategy{ + ClustersPerDecisionGroup: intstr.FromInt(2), + DecisionGroups: []clusterapiv1beta1.DecisionGroup{ + { + GroupName: "canary", + ClusterSelector: clusterapiv1beta1.ClusterSelector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"vendor": "openShift"}}, + }, + }, + }, + }) + + assertNumberOfDecisions(placementName, namespace, 6, 3) + assertPlacementConditionSatisfied(placementName, namespace, 6, true) + assertPlacementDecisionGroupStatus(placementName, namespace, []clusterapiv1beta1.DecisionGroupStatus{ + { + DecisionGroupIndex: 0, + DecisionGroupName: "canary", + Decisions: []string{placementName + "-decision-0"}, + ClustersCount: 3, + }, + { + DecisionGroupIndex: 1, + Decisions: []string{placementName + "-decision-1"}, + ClustersCount: 2, + }, + { + DecisionGroupIndex: 2, + Decisions: []string{placementName + "-decision-2"}, + ClustersCount: 1, + }, + }) + + ginkgo.By("Delete the cluster") + assertDeletingClusters(canary[0], noncanary[0]) + assertPlacementDecisionGroupStatus(placementName, namespace, []clusterapiv1beta1.DecisionGroupStatus{ + { + DecisionGroupIndex: 0, + DecisionGroupName: "canary", + Decisions: []string{placementName + "-decision-0"}, + ClustersCount: 2, + }, + { + DecisionGroupIndex: 1, + Decisions: []string{placementName + "-decision-1"}, + ClustersCount: 2, + }, + }) + + ginkgo.By("Add the canary cluster") + c := assertCreatingClusters(clusterSet1Name, 1, "vendor", "openShift") + canary = append(canary, c...) + assertNumberOfDecisions(placementName, namespace, 5, 2) + assertPlacementDecisionGroupStatus(placementName, namespace, []clusterapiv1beta1.DecisionGroupStatus{ + { + DecisionGroupIndex: 0, + DecisionGroupName: "canary", + Decisions: []string{placementName + "-decision-0"}, + ClustersCount: 3, + }, + { + DecisionGroupIndex: 1, + Decisions: []string{placementName + "-decision-1"}, + ClustersCount: 2, + }, + }) + ginkgo.By("Add the non canary cluster") + c = assertCreatingClusters(clusterSet1Name, 1) + noncanary = append(noncanary, c...) + assertNumberOfDecisions(placementName, namespace, 6, 3) + assertPlacementDecisionGroupStatus(placementName, namespace, []clusterapiv1beta1.DecisionGroupStatus{ + { + DecisionGroupIndex: 0, + DecisionGroupName: "canary", + Decisions: []string{placementName + "-decision-0"}, + ClustersCount: 3, + }, + { + DecisionGroupIndex: 1, + Decisions: []string{placementName + "-decision-1"}, + ClustersCount: 2, + }, + { + DecisionGroupIndex: 2, + Decisions: []string{placementName + "-decision-2"}, + ClustersCount: 1, + }, + }) + + ginkgo.By("Delete all the cluster") + assertDeletingClusters(canary[1:]...) + assertDeletingClusters(noncanary[1:]...) + assertDeletingClusterSet("global") + assertNumberOfDecisions(placementName, namespace, 0, 1) + assertPlacementDecisionGroupStatus(placementName, namespace, []clusterapiv1beta1.DecisionGroupStatus{{Decisions: []string{placementName + "-decision-0"}, ClustersCount: 0}}) + }) ginkgo.It("Should schedule successfully once clusters belong to global(empty labelselector) clusterset are added/deleted)", func() { @@ -306,33 +417,33 @@ var _ = ginkgo.Describe("Placement", func() { assertCreatingClusterSet("global") assertCreatingClusterSetBinding("global", namespace) clusters1 := assertCreatingClusters(clusterName+"1", 1) - assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 1, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) ginkgo.By("Add the cluster") clusters2 := assertCreatingClusters(clusterName+"2", 1) - assertNumberOfDecisions(placementName, namespace, 2) + assertNumberOfDecisions(placementName, namespace, 2, 1) assertPlacementConditionSatisfied(placementName, namespace, 2, true) ginkgo.By("Delete the cluster") assertDeletingClusters(clusters1...) - assertNumberOfDecisions(placementName, namespace, 1) + assertNumberOfDecisions(placementName, namespace, 1, 1) assertDeletingClusters(clusters2...) - assertNumberOfDecisions(placementName, namespace, 0) + assertNumberOfDecisions(placementName, namespace, 0, 1) }) ginkgo.It("Should schedule successfully once new clusterset is bound", func() { assertBindingClusterSet(clusterSet1Name, namespace) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) ginkgo.By("Bind one more clusterset to the placement namespace") assertBindingClusterSet(clusterSet2Name, namespace) assertCreatingClusters(clusterSet2Name, 3) nod := 8 - assertNumberOfDecisions(placementName, namespace, nod) + assertNumberOfDecisions(placementName, namespace, nod, 1) assertPlacementConditionSatisfied(placementName, namespace, nod, false) }) @@ -341,7 +452,7 @@ var _ = ginkgo.Describe("Placement", func() { assertCreatingClusterSet(clusterSet1Name, "vendor", "openShift") assertCreatingClusterSetBinding(clusterSet1Name, namespace) clusters1 := assertCreatingClusters(clusterName+"1", 1, "vendor", "openShift") - assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 1, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) ginkgo.By("Bind one more labelselector clusterset to the placement namespace") assertCreatingClusterSet(clusterSet2Name, "vendor", "IKS") @@ -349,7 +460,7 @@ var _ = ginkgo.Describe("Placement", func() { clusters2 := assertCreatingClusters(clusterName+"2", 1, "vendor", "IKS") nod := 2 - assertNumberOfDecisions(placementName, namespace, nod) + assertNumberOfDecisions(placementName, namespace, nod, 1) assertPlacementConditionSatisfied(placementName, namespace, nod, true) assertDeletingClusters(clusters1[0], clusters2[0]) @@ -358,19 +469,19 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should schedule successfully once a clusterset deleted/added", func() { assertBindingClusterSet(clusterSet1Name, namespace) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) - assertNumberOfDecisions(placementName, namespace, 5) + assertNumberOfDecisions(placementName, namespace, 5, 1) assertPlacementConditionSatisfied(placementName, namespace, 5, false) ginkgo.By("Delete the clusterset") assertDeletingClusterSet(clusterSet1Name) - assertNumberOfDecisions(placementName, namespace, 0) + assertNumberOfDecisions(placementName, namespace, 0, 1) ginkgo.By("Add the clusterset back") assertCreatingClusterSet(clusterSet1Name) - assertNumberOfDecisions(placementName, namespace, 5) + assertNumberOfDecisions(placementName, namespace, 5, 1) assertPlacementConditionSatisfied(placementName, namespace, 5, false) }) @@ -379,24 +490,24 @@ var _ = ginkgo.Describe("Placement", func() { assertCreatingClusterSetBinding(clusterSet1Name, namespace) clusters1 := assertCreatingClusters(clusterName+"1", 1, "vendor", "openShift") clusters2 := assertCreatingClusters(clusterName+"2", 1, "vendor", "IKS") - assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 1, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) - assertNumberOfDecisions(placementName, namespace, 1) + assertNumberOfDecisions(placementName, namespace, 1, 1) assertPlacementConditionSatisfied(placementName, namespace, 1, false) ginkgo.By("Delete the clusterset") assertDeletingClusterSet(clusterSet1Name) - assertNumberOfDecisions(placementName, namespace, 0) + assertNumberOfDecisions(placementName, namespace, 0, 1) ginkgo.By("Add the clusterset back") assertCreatingClusterSet(clusterSet1Name, "vendor", "openShift") - assertNumberOfDecisions(placementName, namespace, 1) + assertNumberOfDecisions(placementName, namespace, 1, 1) assertPlacementConditionSatisfied(placementName, namespace, 1, false) ginkgo.By("Delete the cluster") assertDeletingClusters(clusters1...) - assertNumberOfDecisions(placementName, namespace, 0) + assertNumberOfDecisions(placementName, namespace, 0, 1) assertDeletingClusters(clusters2...) }) @@ -404,20 +515,19 @@ var _ = ginkgo.Describe("Placement", func() { ginkgo.It("Should schedule successfully once a clustersetbinding deleted/added", func() { assertBindingClusterSet(clusterSet1Name, namespace) assertCreatingClusters(clusterSet1Name, 5) - assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}) - - assertNumberOfDecisions(placementName, namespace, 5) + assertCreatingPlacementWithDecision(placementName, namespace, noc(10), 5, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) + assertNumberOfDecisions(placementName, namespace, 5, 1) assertPlacementConditionSatisfied(placementName, namespace, 5, false) ginkgo.By("Delete the clustersetbinding") assertDeletingClusterSetBinding(clusterSet1Name, namespace) - assertNumberOfDecisions(placementName, namespace, 0) + assertNumberOfDecisions(placementName, namespace, 0, 1) ginkgo.By("Add the clustersetbinding back") assertCreatingClusterSetBinding(clusterSet1Name, namespace) - assertNumberOfDecisions(placementName, namespace, 5) + assertNumberOfDecisions(placementName, namespace, 5, 1) assertPlacementConditionSatisfied(placementName, namespace, 5, false) }) diff --git a/test/integration/placement/prioritizer_test.go b/test/integration/placement/prioritizer_test.go index 0bd48a9c8..67ff1b588 100644 --- a/test/integration/placement/prioritizer_test.go +++ b/test/integration/placement/prioritizer_test.go @@ -79,7 +79,7 @@ var _ = ginkgo.Describe("Prioritizers", func() { } //Checking the result of the placement - assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, 1, prioritizerPolicy, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) assertClusterNamesOfDecisions(placementName, namespace, []string{clusterNames[0], clusterNames[1]}) }) @@ -119,7 +119,7 @@ var _ = ginkgo.Describe("Prioritizers", func() { } //Checking the result of the placement - assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, 1, prioritizerPolicy, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) assertClusterNamesOfDecisions(placementName, namespace, []string{clusterNames[0], clusterNames[2]}) }) @@ -166,7 +166,7 @@ var _ = ginkgo.Describe("Prioritizers", func() { } //Checking the result of the placement - assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, 1, prioritizerPolicy, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) assertClusterNamesOfDecisions(placementName, namespace, []string{clusterNames[0], clusterNames[2]}) ginkgo.By("Adding fake placement decisions") @@ -219,7 +219,7 @@ var _ = ginkgo.Describe("Prioritizers", func() { assertCreatingPlacementDecision("fake-1", namespace, []string{clusterNames[0]}) //Checking the result of the placement - assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, 1, prioritizerPolicy, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) assertClusterNamesOfDecisions(placementName, namespace, []string{clusterNames[1], clusterNames[2]}) }) @@ -259,7 +259,7 @@ var _ = ginkgo.Describe("Prioritizers", func() { } //Checking the result of the placement - assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, 1, prioritizerPolicy, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) assertClusterNamesOfDecisions(placementName, namespace, []string{clusterNames[0], clusterNames[2]}) ginkgo.By("Adding a new cluster with resources") @@ -308,7 +308,7 @@ var _ = ginkgo.Describe("Prioritizers", func() { assertCreatingAddOnPlacementScores(clusterNames[2], "demo", "demo", 100) //Checking the result of the placement - assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, 1, prioritizerPolicy, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) assertClusterNamesOfDecisions(placementName, namespace, []string{clusterNames[1], clusterNames[2]}) }) @@ -335,7 +335,7 @@ var _ = ginkgo.Describe("Prioritizers", func() { clusterNames := assertCreatingClusters(clusterSet1Name, 3) //Creating the placement - assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, prioritizerPolicy, []clusterapiv1beta1.Toleration{}) + assertCreatingPlacementWithDecision(placementName, namespace, noc(2), 2, 1, prioritizerPolicy, []clusterapiv1beta1.Toleration{}, clusterapiv1beta1.GroupStrategy{}) //Checking the result of the placement when no AddOnPlacementScores assertClusterNamesOfDecisions(placementName, namespace, []string{clusterNames[0], clusterNames[1]}) diff --git a/test/integration/placement/toleration_test.go b/test/integration/placement/toleration_test.go index 1055df78a..106e76725 100644 --- a/test/integration/placement/toleration_test.go +++ b/test/integration/placement/toleration_test.go @@ -83,7 +83,7 @@ var _ = ginkgo.Describe("TaintToleration", func() { }) //Checking the result of the placement - assertCreatingPlacementWithDecision(placementName, namespace, noc(3), 2, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{ + assertCreatingPlacementWithDecision(placementName, namespace, noc(3), 2, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{ { Key: "key1", Operator: clusterapiv1beta1.TolerationOpExists, @@ -93,7 +93,7 @@ var _ = ginkgo.Describe("TaintToleration", func() { Operator: clusterapiv1beta1.TolerationOpEqual, Value: "value2", }, - }) + }, clusterapiv1beta1.GroupStrategy{}) assertClusterNamesOfDecisions(placementName, namespace, []string{clusterNames[0], clusterNames[1]}) }) @@ -119,7 +119,7 @@ var _ = ginkgo.Describe("TaintToleration", func() { }) //Taint/toleration matches, effect not match - assertCreatingPlacementWithDecision(placementName, namespace, noc(4), 2, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{ + assertCreatingPlacementWithDecision(placementName, namespace, noc(4), 2, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{ { Key: "key1", Operator: clusterapiv1beta1.TolerationOpExists, @@ -140,7 +140,7 @@ var _ = ginkgo.Describe("TaintToleration", func() { Operator: clusterapiv1beta1.TolerationOpExists, Effect: clusterapiv1.TaintEffectNoSelect, }, - }) + }, clusterapiv1beta1.GroupStrategy{}) //Checking the result of the placement assertClusterNamesOfDecisions(placementName, namespace, []string{clusterNames[2], clusterNames[3]}) @@ -183,7 +183,7 @@ var _ = ginkgo.Describe("TaintToleration", func() { }) //Checking the result of the placement - assertCreatingPlacementWithDecision(placementName, namespace, noc(4), 3, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{ + assertCreatingPlacementWithDecision(placementName, namespace, noc(4), 3, 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{ { Key: "key1", Operator: clusterapiv1beta1.TolerationOpExists, @@ -194,7 +194,7 @@ var _ = ginkgo.Describe("TaintToleration", func() { Operator: clusterapiv1beta1.TolerationOpExists, TolerationSeconds: &tolerationSeconds_20, }, - }) + }, clusterapiv1beta1.GroupStrategy{}) assertClusterNamesOfDecisions(placementName, namespace, []string{clusterNames[0], clusterNames[1], clusterNames[3]}) //Check placement requeue, clusterNames[0] should be removed when TolerationSeconds expired.