scheduler: support pod request exact match reservation (#2121)

Signed-off-by: wangjianyu.wjy <wangjianyu.wjy@alibaba-inc.com>
Co-authored-by: wangjianyu.wjy <wangjianyu.wjy@alibaba-inc.com>

apis/extension/reservation.go

@@ -161,3 +161,57 @@ func SetReservationRestrictedOptions(obj metav1.Object, options *ReservationRest
 	obj.SetAnnotations(annotations)
 	return nil
 }
+
+const (
+	AnnotationExactMatchReservationSpec = SchedulingDomainPrefix + "/exact-match-reservation"
+)
+
+type ExactMatchReservationSpec struct {
+	ResourceNames []corev1.ResourceName `json:"resourceNames,omitempty"`
+}
+
+func SetExactMatchReservationSpec(obj metav1.Object, spec *ExactMatchReservationSpec) error {
+	data, err := json.Marshal(spec)
+	if err != nil {
+		return err
+	}
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	annotations[AnnotationExactMatchReservationSpec] = string(data)
+	obj.SetAnnotations(annotations)
+	return nil
+}
+
+func GetExactMatchReservationSpec(annotations map[string]string) (*ExactMatchReservationSpec, error) {
+	if s := annotations[AnnotationExactMatchReservationSpec]; s != "" {
+		var exactMatchReservationSpec ExactMatchReservationSpec
+		if err := json.Unmarshal([]byte(s), &exactMatchReservationSpec); err != nil {
+			return nil, err
+		}
+		return &exactMatchReservationSpec, nil
+	}
+	return nil, nil
+}
+
+func ExactMatchReservation(podRequests, reservationAllocatable corev1.ResourceList, spec *ExactMatchReservationSpec) bool {
+	if spec == nil || len(spec.ResourceNames) == 0 {
+		return true
+	}
+	for _, resourceName := range spec.ResourceNames {
+		allocatable, existsInReservation := reservationAllocatable[resourceName]
+		request, existsInPod := podRequests[resourceName]
+		if !existsInReservation || !existsInPod {
+			if !existsInReservation && !existsInPod {
+				return true
+			}
+			return false
+		}
+
+		if allocatable.Cmp(request) != 0 {
+			return false
+		}
+	}
+	return true
+}

apis/extension/reservation_test.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
 
@@ -44,3 +45,85 @@ func TestSetReservationAllocated(t *testing.T) {
 	}
 	assert.Equal(t, expectReservationAllocated, reservationAllocated)
 }
+
+func TestExactMatchReservation(t *testing.T) {
+	tests := []struct {
+		name                   string
+		podRequests            corev1.ResourceList
+		reservationAllocatable corev1.ResourceList
+		spec                   *ExactMatchReservationSpec
+		want                   bool
+	}{
+		{
+			name: "exact matched cpu",
+			spec: &ExactMatchReservationSpec{
+				ResourceNames: []corev1.ResourceName{
+					corev1.ResourceCPU,
+				},
+			},
+			podRequests: corev1.ResourceList{
+				corev1.ResourceCPU: resource.MustParse("1"),
+			},
+			reservationAllocatable: corev1.ResourceList{
+				corev1.ResourceCPU: resource.MustParse("1"),
+			},
+			want: true,
+		},
+		{
+			name: "exact matched cpu",
+			spec: &ExactMatchReservationSpec{
+				ResourceNames: []corev1.ResourceName{
+					corev1.ResourceCPU,
+				},
+			},
+			podRequests: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("1"),
+				corev1.ResourceMemory: resource.MustParse("1Gi"),
+			},
+			reservationAllocatable: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("1"),
+				corev1.ResourceMemory: resource.MustParse("2Gi"),
+			},
+			want: true,
+		},
+		{
+			name: "exact matched cpu, memory not exact matched",
+			spec: &ExactMatchReservationSpec{
+				ResourceNames: []corev1.ResourceName{
+					corev1.ResourceCPU,
+					corev1.ResourceMemory,
+				},
+			},
+			podRequests: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("1"),
+				corev1.ResourceMemory: resource.MustParse("1Gi"),
+			},
+			reservationAllocatable: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("1"),
+				corev1.ResourceMemory: resource.MustParse("2Gi"),
+			},
+			want: false,
+		},
+		{
+			name: "exact matched cpu, memory exact match spec doesn't matter",
+			spec: &ExactMatchReservationSpec{
+				ResourceNames: []corev1.ResourceName{
+					corev1.ResourceCPU,
+					corev1.ResourceMemory,
+				},
+			},
+			podRequests: corev1.ResourceList{
+				corev1.ResourceCPU: resource.MustParse("1"),
+			},
+			reservationAllocatable: corev1.ResourceList{
+				corev1.ResourceCPU: resource.MustParse("1"),
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, ExactMatchReservation(tt.podRequests, tt.reservationAllocatable, tt.spec))
+		})
+	}
+}

pkg/scheduler/plugins/reservation/plugin.go

@@ -178,6 +178,7 @@ type nodeDiagnosisState struct {
 	ownerMatched             int // owner matched
 	isUnschedulableUnmatched int // owner matched but BeforePreFilter unmatched due to unschedulable
 	affinityUnmatched        int // owner matched but BeforePreFilter unmatched due to affinity
+	notExactMatched          int // owner matched but BeforePreFilter unmatched due to not exact match
 }
 
 func (s *stateData) Clone() framework.StateData {
@@ -540,15 +541,17 @@ func (pl *Plugin) PostFilter(_ context.Context, cycleState *framework.CycleState
 }
 
 func (pl *Plugin) makePostFilterReasons(state *stateData, filteredNodeStatusMap framework.NodeToStatusMap) []string {
-	ownerMatched, affinityUnmatched, isUnSchedulableUnmatched := 0, 0, 0
+	ownerMatched, affinityUnmatched, isUnSchedulableUnmatched, notExactMatched := 0, 0, 0, 0
 	// failure reasons and counts for the nodes which have not been handled by the Reservation's Filter
 	reasonsByNode := map[string]int{}
 	for nodeName, diagnosisState := range state.nodeReservationDiagnosis {
 		isUnSchedulableUnmatched += diagnosisState.isUnschedulableUnmatched
 		affinityUnmatched += diagnosisState.affinityUnmatched
 		ownerMatched += diagnosisState.ownerMatched
+		notExactMatched += diagnosisState.notExactMatched
+
 		// calculate the remaining unmatched which is owner-matched and Reservation BeforePreFilter matched
-		remainUnmatched := diagnosisState.ownerMatched - diagnosisState.affinityUnmatched - diagnosisState.isUnschedulableUnmatched
+		remainUnmatched := diagnosisState.ownerMatched - diagnosisState.affinityUnmatched - diagnosisState.isUnschedulableUnmatched - diagnosisState.notExactMatched
 		if remainUnmatched <= 0 { // no need to check other reasons
 			continue
 		}
@@ -591,6 +594,12 @@ func (pl *Plugin) makePostFilterReasons(state *stateData, filteredNodeStatusMap
 		reasons = append(reasons, b.String())
 		b.Reset()
 	}
+	if notExactMatched > 0 {
+		b.WriteString(strconv.Itoa(notExactMatched))
+		b.WriteString(" Reservation(s) is not exact matched")
+		reasons = append(reasons, b.String())
+		b.Reset()
+	}
 	for nodeReason, count := range reasonsByNode { // node reason Filter failed
 		b.WriteString(strconv.Itoa(count))
 		b.WriteString(" Reservation(s) for node reason that ")

pkg/scheduler/plugins/reservation/plugin_test.go

@@ -1899,6 +1899,60 @@ func TestPostFilter(t *testing.T) {
 				"4 Reservation(s) is unschedulable",
 				"4 Reservation(s) matched owner total"),
 		},
+		{
+			name: "show reservation matched owner, unschedulable and exact matched",
+			args: args{
+				hasStateData: true,
+				nodeReservationDiagnosis: map[string]*nodeDiagnosisState{
+					"test-node-0": {
+						ownerMatched:             3,
+						isUnschedulableUnmatched: 0,
+						notExactMatched:          3,
+					},
+					"test-node-1": {
+						ownerMatched:             2,
+						isUnschedulableUnmatched: 1,
+						notExactMatched:          1,
+					},
+				},
+				filteredNodeStatusMap: framework.NodeToStatusMap{
+					"test-node-0": {},
+					"test-node-1": {},
+				},
+			},
+			want: nil,
+			want1: framework.NewStatus(framework.Unschedulable,
+				"1 Reservation(s) is unschedulable",
+				"4 Reservation(s) is not exact matched",
+				"5 Reservation(s) matched owner total"),
+		},
+		{
+			name: "show reservation matched owner, unschedulable and exact matched",
+			args: args{
+				hasStateData: true,
+				nodeReservationDiagnosis: map[string]*nodeDiagnosisState{
+					"test-node-0": {
+						ownerMatched:             3,
+						isUnschedulableUnmatched: 0,
+						notExactMatched:          3,
+					},
+					"test-node-1": {
+						ownerMatched:             2,
+						isUnschedulableUnmatched: 1,
+						notExactMatched:          1,
+					},
+				},
+				filteredNodeStatusMap: framework.NodeToStatusMap{
+					"test-node-0": {},
+					"test-node-1": {},
+				},
+			},
+			want: nil,
+			want1: framework.NewStatus(framework.Unschedulable,
+				"1 Reservation(s) is unschedulable",
+				"4 Reservation(s) is not exact matched",
+				"5 Reservation(s) matched owner total"),
+		},
 		{
 			name: "show reservation matched owner, unschedulable and affinity unmatched",
 			args: args{

pkg/scheduler/plugins/reservation/transformer.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/framework/parallelize"
 	schedutil "k8s.io/kubernetes/pkg/scheduler/util"
 
+	"github.com/koordinator-sh/koordinator/apis/extension"
 	"github.com/koordinator-sh/koordinator/pkg/scheduler/frameworkext"
 	"github.com/koordinator-sh/koordinator/pkg/util"
 	reservationutil "github.com/koordinator-sh/koordinator/pkg/util/reservation"
@@ -61,6 +62,13 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
 	}
 	requiredNodeAffinity := nodeaffinity.GetRequiredNodeAffinity(pod)
 
+	podRequests := resourceapi.PodRequests(pod, resourceapi.PodResourcesOptions{})
+	exactMatchReservationSpec, err := extension.GetExactMatchReservationSpec(pod.Annotations)
+	if err != nil {
+		klog.ErrorS(err, "Failed to parse exact match reservation spec", "pod", klog.KObj(pod))
+		return nil, false, framework.AsStatus(err)
+	}
+
 	var stateIndex, diagnosisIndex int32
 	allNodes := pl.reservationCache.listAllNodes()
 	allNodeReservationStates := make([]*nodeReservationState, len(allNodes))
@@ -106,6 +114,7 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
 			ownerMatched:             0,
 			isUnschedulableUnmatched: 0,
 			affinityUnmatched:        0,
+			notExactMatched:          0,
 		}
 		status := pl.reservationCache.forEachAvailableReservationOnNode(node.Name, func(rInfo *frameworkext.ReservationInfo) (bool, *framework.Status) {
 			if !rInfo.IsAvailable() || rInfo.ParseError != nil {
@@ -121,7 +130,8 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
 			isOwnerMatched := rInfo.Match(pod)
 			isUnschedulable := rInfo.IsUnschedulable()
 			isMatchReservationAffinity := matchReservationAffinity(node, rInfo, reservationAffinity)
-			if !isReservedPod && !isUnschedulable && isOwnerMatched && isMatchReservationAffinity {
+			isExactMatched := extension.ExactMatchReservation(podRequests, rInfo.Allocatable, exactMatchReservationSpec)
+			if !isReservedPod && !isUnschedulable && isOwnerMatched && isMatchReservationAffinity && isExactMatched {
 				matched = append(matched, rInfo.Clone())
 
 			} else if len(rInfo.AssignedPods) > 0 {
@@ -133,6 +143,8 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
 					diagnosisState.isUnschedulableUnmatched++
 				} else if !isMatchReservationAffinity {
 					diagnosisState.affinityUnmatched++
+				} else if !isExactMatched {
+					diagnosisState.notExactMatched++
 				}
 			}
 
@@ -224,8 +236,6 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
 	allNodeReservationStates = allNodeReservationStates[:stateIndex]
 	allPluginToRestoreState = allPluginToRestoreState[:stateIndex]
 	allNodeDiagnosisStates = allNodeDiagnosisStates[:diagnosisIndex]
-
-	podRequests := resourceapi.PodRequests(pod, resourceapi.PodResourcesOptions{})
 	podRequestResources := framework.NewResource(podRequests)
 	state := &stateData{
 		hasAffinity:              reservationAffinity != nil,