diff --git a/apis/extension/reservation.go b/apis/extension/reservation.go index b43d4335a..0cff991ec 100644 --- a/apis/extension/reservation.go +++ b/apis/extension/reservation.go @@ -161,3 +161,57 @@ func SetReservationRestrictedOptions(obj metav1.Object, options *ReservationRest obj.SetAnnotations(annotations) return nil } + +const ( + AnnotationExactMatchReservationSpec = SchedulingDomainPrefix + "/exact-match-reservation" +) + +type ExactMatchReservationSpec struct { + ResourceNames []corev1.ResourceName `json:"resourceNames,omitempty"` +} + +func SetExactMatchReservationSpec(obj metav1.Object, spec *ExactMatchReservationSpec) error { + data, err := json.Marshal(spec) + if err != nil { + return err + } + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[AnnotationExactMatchReservationSpec] = string(data) + obj.SetAnnotations(annotations) + return nil +} + +func GetExactMatchReservationSpec(annotations map[string]string) (*ExactMatchReservationSpec, error) { + if s := annotations[AnnotationExactMatchReservationSpec]; s != "" { + var exactMatchReservationSpec ExactMatchReservationSpec + if err := json.Unmarshal([]byte(s), &exactMatchReservationSpec); err != nil { + return nil, err + } + return &exactMatchReservationSpec, nil + } + return nil, nil +} + +func ExactMatchReservation(podRequests, reservationAllocatable corev1.ResourceList, spec *ExactMatchReservationSpec) bool { + if spec == nil || len(spec.ResourceNames) == 0 { + return true + } + for _, resourceName := range spec.ResourceNames { + allocatable, existsInReservation := reservationAllocatable[resourceName] + request, existsInPod := podRequests[resourceName] + if !existsInReservation || !existsInPod { + if !existsInReservation && !existsInPod { + return true + } + return false + } + + if allocatable.Cmp(request) != 0 { + return false + } + } + return true +} diff --git a/apis/extension/reservation_test.go b/apis/extension/reservation_test.go index 9aa09c497..549f20dc3 100644 --- a/apis/extension/reservation_test.go +++ b/apis/extension/reservation_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -44,3 +45,85 @@ func TestSetReservationAllocated(t *testing.T) { } assert.Equal(t, expectReservationAllocated, reservationAllocated) } + +func TestExactMatchReservation(t *testing.T) { + tests := []struct { + name string + podRequests corev1.ResourceList + reservationAllocatable corev1.ResourceList + spec *ExactMatchReservationSpec + want bool + }{ + { + name: "exact matched cpu", + spec: &ExactMatchReservationSpec{ + ResourceNames: []corev1.ResourceName{ + corev1.ResourceCPU, + }, + }, + podRequests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + reservationAllocatable: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + want: true, + }, + { + name: "exact matched cpu", + spec: &ExactMatchReservationSpec{ + ResourceNames: []corev1.ResourceName{ + corev1.ResourceCPU, + }, + }, + podRequests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + reservationAllocatable: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + want: true, + }, + { + name: "exact matched cpu, memory not exact matched", + spec: &ExactMatchReservationSpec{ + ResourceNames: []corev1.ResourceName{ + corev1.ResourceCPU, + corev1.ResourceMemory, + }, + }, + podRequests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + reservationAllocatable: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + want: false, + }, + { + name: "exact matched cpu, memory exact match spec doesn't matter", + spec: &ExactMatchReservationSpec{ + ResourceNames: []corev1.ResourceName{ + corev1.ResourceCPU, + corev1.ResourceMemory, + }, + }, + podRequests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + reservationAllocatable: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, ExactMatchReservation(tt.podRequests, tt.reservationAllocatable, tt.spec)) + }) + } +} diff --git a/pkg/scheduler/plugins/reservation/plugin.go b/pkg/scheduler/plugins/reservation/plugin.go index 00365a042..b36ddf065 100644 --- a/pkg/scheduler/plugins/reservation/plugin.go +++ b/pkg/scheduler/plugins/reservation/plugin.go @@ -178,6 +178,7 @@ type nodeDiagnosisState struct { ownerMatched int // owner matched isUnschedulableUnmatched int // owner matched but BeforePreFilter unmatched due to unschedulable affinityUnmatched int // owner matched but BeforePreFilter unmatched due to affinity + notExactMatched int // owner matched but BeforePreFilter unmatched due to not exact match } func (s *stateData) Clone() framework.StateData { @@ -540,15 +541,17 @@ func (pl *Plugin) PostFilter(_ context.Context, cycleState *framework.CycleState } func (pl *Plugin) makePostFilterReasons(state *stateData, filteredNodeStatusMap framework.NodeToStatusMap) []string { - ownerMatched, affinityUnmatched, isUnSchedulableUnmatched := 0, 0, 0 + ownerMatched, affinityUnmatched, isUnSchedulableUnmatched, notExactMatched := 0, 0, 0, 0 // failure reasons and counts for the nodes which have not been handled by the Reservation's Filter reasonsByNode := map[string]int{} for nodeName, diagnosisState := range state.nodeReservationDiagnosis { isUnSchedulableUnmatched += diagnosisState.isUnschedulableUnmatched affinityUnmatched += diagnosisState.affinityUnmatched ownerMatched += diagnosisState.ownerMatched + notExactMatched += diagnosisState.notExactMatched + // calculate the remaining unmatched which is owner-matched and Reservation BeforePreFilter matched - remainUnmatched := diagnosisState.ownerMatched - diagnosisState.affinityUnmatched - diagnosisState.isUnschedulableUnmatched + remainUnmatched := diagnosisState.ownerMatched - diagnosisState.affinityUnmatched - diagnosisState.isUnschedulableUnmatched - diagnosisState.notExactMatched if remainUnmatched <= 0 { // no need to check other reasons continue } @@ -591,6 +594,12 @@ func (pl *Plugin) makePostFilterReasons(state *stateData, filteredNodeStatusMap reasons = append(reasons, b.String()) b.Reset() } + if notExactMatched > 0 { + b.WriteString(strconv.Itoa(notExactMatched)) + b.WriteString(" Reservation(s) is not exact matched") + reasons = append(reasons, b.String()) + b.Reset() + } for nodeReason, count := range reasonsByNode { // node reason Filter failed b.WriteString(strconv.Itoa(count)) b.WriteString(" Reservation(s) for node reason that ") diff --git a/pkg/scheduler/plugins/reservation/plugin_test.go b/pkg/scheduler/plugins/reservation/plugin_test.go index 66c4a7d40..f6a0503e4 100644 --- a/pkg/scheduler/plugins/reservation/plugin_test.go +++ b/pkg/scheduler/plugins/reservation/plugin_test.go @@ -1899,6 +1899,60 @@ func TestPostFilter(t *testing.T) { "4 Reservation(s) is unschedulable", "4 Reservation(s) matched owner total"), }, + { + name: "show reservation matched owner, unschedulable and exact matched", + args: args{ + hasStateData: true, + nodeReservationDiagnosis: map[string]*nodeDiagnosisState{ + "test-node-0": { + ownerMatched: 3, + isUnschedulableUnmatched: 0, + notExactMatched: 3, + }, + "test-node-1": { + ownerMatched: 2, + isUnschedulableUnmatched: 1, + notExactMatched: 1, + }, + }, + filteredNodeStatusMap: framework.NodeToStatusMap{ + "test-node-0": {}, + "test-node-1": {}, + }, + }, + want: nil, + want1: framework.NewStatus(framework.Unschedulable, + "1 Reservation(s) is unschedulable", + "4 Reservation(s) is not exact matched", + "5 Reservation(s) matched owner total"), + }, + { + name: "show reservation matched owner, unschedulable and exact matched", + args: args{ + hasStateData: true, + nodeReservationDiagnosis: map[string]*nodeDiagnosisState{ + "test-node-0": { + ownerMatched: 3, + isUnschedulableUnmatched: 0, + notExactMatched: 3, + }, + "test-node-1": { + ownerMatched: 2, + isUnschedulableUnmatched: 1, + notExactMatched: 1, + }, + }, + filteredNodeStatusMap: framework.NodeToStatusMap{ + "test-node-0": {}, + "test-node-1": {}, + }, + }, + want: nil, + want1: framework.NewStatus(framework.Unschedulable, + "1 Reservation(s) is unschedulable", + "4 Reservation(s) is not exact matched", + "5 Reservation(s) matched owner total"), + }, { name: "show reservation matched owner, unschedulable and affinity unmatched", args: args{ diff --git a/pkg/scheduler/plugins/reservation/transformer.go b/pkg/scheduler/plugins/reservation/transformer.go index 090ca29c7..22862ea04 100644 --- a/pkg/scheduler/plugins/reservation/transformer.go +++ b/pkg/scheduler/plugins/reservation/transformer.go @@ -33,6 +33,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework/parallelize" schedutil "k8s.io/kubernetes/pkg/scheduler/util" + "github.com/koordinator-sh/koordinator/apis/extension" "github.com/koordinator-sh/koordinator/pkg/scheduler/frameworkext" "github.com/koordinator-sh/koordinator/pkg/util" reservationutil "github.com/koordinator-sh/koordinator/pkg/util/reservation" @@ -61,6 +62,13 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState * } requiredNodeAffinity := nodeaffinity.GetRequiredNodeAffinity(pod) + podRequests := resourceapi.PodRequests(pod, resourceapi.PodResourcesOptions{}) + exactMatchReservationSpec, err := extension.GetExactMatchReservationSpec(pod.Annotations) + if err != nil { + klog.ErrorS(err, "Failed to parse exact match reservation spec", "pod", klog.KObj(pod)) + return nil, false, framework.AsStatus(err) + } + var stateIndex, diagnosisIndex int32 allNodes := pl.reservationCache.listAllNodes() allNodeReservationStates := make([]*nodeReservationState, len(allNodes)) @@ -106,6 +114,7 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState * ownerMatched: 0, isUnschedulableUnmatched: 0, affinityUnmatched: 0, + notExactMatched: 0, } status := pl.reservationCache.forEachAvailableReservationOnNode(node.Name, func(rInfo *frameworkext.ReservationInfo) (bool, *framework.Status) { if !rInfo.IsAvailable() || rInfo.ParseError != nil { @@ -121,7 +130,8 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState * isOwnerMatched := rInfo.Match(pod) isUnschedulable := rInfo.IsUnschedulable() isMatchReservationAffinity := matchReservationAffinity(node, rInfo, reservationAffinity) - if !isReservedPod && !isUnschedulable && isOwnerMatched && isMatchReservationAffinity { + isExactMatched := extension.ExactMatchReservation(podRequests, rInfo.Allocatable, exactMatchReservationSpec) + if !isReservedPod && !isUnschedulable && isOwnerMatched && isMatchReservationAffinity && isExactMatched { matched = append(matched, rInfo.Clone()) } else if len(rInfo.AssignedPods) > 0 { @@ -133,6 +143,8 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState * diagnosisState.isUnschedulableUnmatched++ } else if !isMatchReservationAffinity { diagnosisState.affinityUnmatched++ + } else if !isExactMatched { + diagnosisState.notExactMatched++ } } @@ -224,8 +236,6 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState * allNodeReservationStates = allNodeReservationStates[:stateIndex] allPluginToRestoreState = allPluginToRestoreState[:stateIndex] allNodeDiagnosisStates = allNodeDiagnosisStates[:diagnosisIndex] - - podRequests := resourceapi.PodRequests(pod, resourceapi.PodResourcesOptions{}) podRequestResources := framework.NewResource(podRequests) state := &stateData{ hasAffinity: reservationAffinity != nil,