diff --git a/apis/extension/reservation.go b/apis/extension/reservation.go index 19dc58a6c..cced21860 100644 --- a/apis/extension/reservation.go +++ b/apis/extension/reservation.go @@ -38,6 +38,9 @@ const ( // AnnotationReservationAffinity represents the constraints of Pod selection Reservation AnnotationReservationAffinity = SchedulingDomainPrefix + "/reservation-affinity" + + // AnnotationReservationRestrictedOptions represent the Reservation Restricted options + AnnotationReservationRestrictedOptions = SchedulingDomainPrefix + "/reservation-restricted-options" ) type ReservationAllocated struct { @@ -67,6 +70,14 @@ type ReservationAffinitySelector struct { ReservationSelectorTerms []corev1.NodeSelectorTerm `json:"reservationSelectorTerms,omitempty"` } +type ReservationRestrictedOptions struct { + // Resources means that when the Pod intersects with these resources, + // it can only allocate the reserved amount at most. + // If the Reservation's AllocatePolicy is Restricted, and no resources configured, + // by default the resources equal all reserved resources by the Reservation. + Resources []corev1.ResourceName `json:"resources,omitempty"` +} + func GetReservationAllocated(pod *corev1.Pod) (*ReservationAllocated, error) { if pod.Annotations == nil { return nil, nil @@ -122,3 +133,27 @@ func SetReservationAffinity(obj metav1.Object, affinity *ReservationAffinity) er obj.SetAnnotations(annotations) return nil } + +func GetReservationRestrictedOptions(annotations map[string]string) (*ReservationRestrictedOptions, error) { + var options ReservationRestrictedOptions + if s, ok := annotations[AnnotationReservationRestrictedOptions]; ok && s != "" { + if err := json.Unmarshal([]byte(s), &options); err != nil { + return nil, err + } + } + return &options, nil +} + +func SetReservationRestrictedOptions(obj metav1.Object, options *ReservationRestrictedOptions) error { + data, err := json.Marshal(options) + if err != nil { + return err + } + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[AnnotationReservationRestrictedOptions] = string(data) + obj.SetAnnotations(annotations) + return nil +} diff --git a/pkg/scheduler/frameworkext/reservation_info.go b/pkg/scheduler/frameworkext/reservation_info.go index 6f30ac2a9..068fafad3 100644 --- a/pkg/scheduler/frameworkext/reservation_info.go +++ b/pkg/scheduler/frameworkext/reservation_info.go @@ -20,6 +20,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" quotav1 "k8s.io/apiserver/pkg/quota/v1" corev1helpers "k8s.io/component-helpers/scheduling/corev1" "k8s.io/klog/v2" @@ -77,15 +78,30 @@ func (p *PodRequirement) Clone() *PodRequirement { } func NewReservationInfo(r *schedulingv1alpha1.Reservation) *ReservationInfo { + var parseErrors []error allocatable := reservationutil.ReservationRequests(r) resourceNames := quotav1.ResourceNames(allocatable) + if r.Spec.AllocatePolicy == schedulingv1alpha1.ReservationAllocatePolicyRestricted { + options, err := apiext.GetReservationRestrictedOptions(r.Annotations) + if err == nil { + resourceNames = reservationutil.GetReservationRestrictedResources(resourceNames, options) + } else { + parseErrors = append(parseErrors, err) + } + } reservedPod := reservationutil.NewReservePod(r) ownerMatchers, err := reservationutil.ParseReservationOwnerMatchers(r.Spec.Owners) if err != nil { + parseErrors = append(parseErrors, err) klog.ErrorS(err, "Failed to parse reservation owner matchers", "reservation", klog.KObj(r)) } + var parseError error + if len(parseErrors) > 0 { + parseError = utilerrors.NewAggregate(parseErrors) + } + return &ReservationInfo{ Reservation: r, Pod: reservedPod, @@ -94,13 +110,21 @@ func NewReservationInfo(r *schedulingv1alpha1.Reservation) *ReservationInfo { AllocatablePorts: util.RequestedHostPorts(reservedPod), AssignedPods: map[types.UID]*PodRequirement{}, OwnerMatchers: ownerMatchers, - ParseError: err, + ParseError: parseError, } } func NewReservationInfoFromPod(pod *corev1.Pod) *ReservationInfo { + var parseErrors []error + allocatable, _ := resource.PodRequestsAndLimits(pod) resourceNames := quotav1.ResourceNames(allocatable) + options, err := apiext.GetReservationRestrictedOptions(pod.Annotations) + if err == nil { + resourceNames = reservationutil.GetReservationRestrictedResources(resourceNames, options) + } else { + parseErrors = append(parseErrors, err) + } owners, err := apiext.GetReservationOwners(pod.Annotations) if err != nil { @@ -110,10 +134,16 @@ func NewReservationInfoFromPod(pod *corev1.Pod) *ReservationInfo { if owners != nil { ownerMatchers, err = reservationutil.ParseReservationOwnerMatchers(owners) if err != nil { + parseErrors = append(parseErrors, err) klog.ErrorS(err, "Failed to parse reservation owner matchers of pod", "pod", klog.KObj(pod)) } } + var parseError error + if len(parseErrors) > 0 { + parseError = utilerrors.NewAggregate(parseErrors) + } + return &ReservationInfo{ Pod: pod, ResourceNames: resourceNames, @@ -121,7 +151,7 @@ func NewReservationInfoFromPod(pod *corev1.Pod) *ReservationInfo { AllocatablePorts: util.RequestedHostPorts(pod), AssignedPods: map[types.UID]*PodRequirement{}, OwnerMatchers: ownerMatchers, - ParseError: err, + ParseError: parseError, } } diff --git a/pkg/util/reservation/reservation.go b/pkg/util/reservation/reservation.go index 6a6acc47c..2cad2ff31 100644 --- a/pkg/util/reservation/reservation.go +++ b/pkg/util/reservation/reservation.go @@ -528,3 +528,22 @@ func UpdateReservationResizeAllocatable(obj metav1.Object, resources corev1.Reso } return SetReservationResizeAllocatable(obj, resizeAllocatable) } + +func GetReservationRestrictedResources(allocatableResources []corev1.ResourceName, options *extension.ReservationRestrictedOptions) []corev1.ResourceName { + if options == nil { + return allocatableResources + } + result := make([]corev1.ResourceName, 0, len(allocatableResources)) + for _, resourceName := range allocatableResources { + for _, v := range options.Resources { + if resourceName == v { + result = append(result, resourceName) + break + } + } + } + if len(result) == 0 { + result = allocatableResources + } + return result +} diff --git a/pkg/util/reservation/reservation_test.go b/pkg/util/reservation/reservation_test.go index 6bb28e87d..2cfd6048d 100644 --- a/pkg/util/reservation/reservation_test.go +++ b/pkg/util/reservation/reservation_test.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/utils/pointer" + apiext "github.com/koordinator-sh/koordinator/apis/extension" schedulingv1alpha1 "github.com/koordinator-sh/koordinator/apis/scheduling/v1alpha1" ) @@ -747,3 +748,49 @@ func TestReservationRequests(t *testing.T) { }) } } + +func TestGetReservationRestrictedResources(t *testing.T) { + tests := []struct { + name string + resourceNames []corev1.ResourceName + options *apiext.ReservationRestrictedOptions + want []corev1.ResourceName + }{ + { + name: "no options, got all allocatable resources", + resourceNames: []corev1.ResourceName{"cpu", "memory"}, + options: nil, + want: []corev1.ResourceName{"cpu", "memory"}, + }, + { + name: "has options and same as resourceNames", + resourceNames: []corev1.ResourceName{"cpu", "memory"}, + options: &apiext.ReservationRestrictedOptions{ + Resources: []corev1.ResourceName{"cpu", "memory"}, + }, + want: []corev1.ResourceName{"cpu", "memory"}, + }, + { + name: "has options but different resourceNames", + resourceNames: []corev1.ResourceName{"cpu", "memory"}, + options: &apiext.ReservationRestrictedOptions{ + Resources: []corev1.ResourceName{"cpu"}, + }, + want: []corev1.ResourceName{"cpu"}, + }, + { + name: "has options but no resourceNames", + resourceNames: []corev1.ResourceName{"cpu", "memory"}, + options: &apiext.ReservationRestrictedOptions{ + Resources: []corev1.ResourceName{}, + }, + want: []corev1.ResourceName{"cpu", "memory"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetReservationRestrictedResources(tt.resourceNames, tt.options) + assert.Equal(t, tt.want, got) + }) + } +}