From 663da5b9b473cc853cc51072ea4b599e3eb0d218 Mon Sep 17 00:00:00 2001 From: "wangjianyu.wjy" Date: Tue, 23 Apr 2024 11:55:41 +0800 Subject: [PATCH] scheduler: try best to distribute cpu and memory evenly across numa Signed-off-by: wangjianyu.wjy --- .../nodenumaresource/resource_manager.go | 109 ++- .../nodenumaresource/resource_manager_test.go | 695 ++++++++++++++++++ .../nodenumaresource/topology_hint_test.go | 8 + 3 files changed, 778 insertions(+), 34 deletions(-) diff --git a/pkg/scheduler/plugins/nodenumaresource/resource_manager.go b/pkg/scheduler/plugins/nodenumaresource/resource_manager.go index be3a438c5..c1374bba5 100644 --- a/pkg/scheduler/plugins/nodenumaresource/resource_manager.go +++ b/pkg/scheduler/plugins/nodenumaresource/resource_manager.go @@ -19,6 +19,7 @@ package nodenumaresource import ( "errors" "fmt" + "sort" "sync" corev1 "k8s.io/api/core/v1" @@ -151,15 +152,12 @@ func (c *resourceManager) trimNUMANodeResources(nodeName string, totalAvailable if cpuQuantity.IsZero() { continue } - availableCPUs := cpuDetails.CPUsInNUMANodes(numaNode) - if int64(availableCPUs.Size()*1000) >= cpuQuantity.MilliValue() { - availableCPUs = filterCPUsByRequiredCPUBindPolicy( - options.cpuBindPolicy, - availableCPUs, - cpuDetails, - options.topologyOptions.CPUTopology.CPUsPerCore(), - ) - } + availableCPUs := filterCPUsByRequiredCPUBindPolicy( + options.cpuBindPolicy, + cpuDetails.CPUsInNUMANodes(numaNode), + cpuDetails, + options.topologyOptions.CPUTopology.CPUsPerCore(), + ) if int64(availableCPUs.Size())*1000 < cpuQuantity.MilliValue() { cpuQuantity.SetMilli(int64(availableCPUs.Size() * 1000)) available[corev1.ResourceCPU] = cpuQuantity @@ -202,6 +200,10 @@ func (c *resourceManager) allocateResourcesByHint(node *corev1.Node, pod *corev1 return nil, err } + if err := c.trimNUMANodeResources(node.Name, totalAvailable, options); err != nil { + return nil, err + } + var requests corev1.ResourceList if options.requestCPUBind { requests = options.originalRequests.DeepCopy() @@ -209,44 +211,83 @@ func (c *resourceManager) allocateResourcesByHint(node *corev1.Node, pod *corev1 requests = options.requests.DeepCopy() } - intersectionResources := sets.NewString() - var result []NUMANodeResource - for _, numaNodeID := range options.hint.NUMANodeAffinity.GetBits() { - allocatable := totalAvailable[numaNodeID] - r := NUMANodeResource{ - Node: numaNodeID, - Resources: corev1.ResourceList{}, + result, reasons := tryBestToDistributeEvenly(requests, totalAvailable, options) + if len(reasons) > 0 { + return nil, framework.NewStatus(framework.Unschedulable, reasons...).AsError() + } + return result, nil +} + +func tryBestToDistributeEvenly(requests corev1.ResourceList, totalAvailable map[int]corev1.ResourceList, options *ResourceOptions) ([]NUMANodeResource, []string) { + resourceNamesByNUMA := sets.NewString() + for _, available := range totalAvailable { + for resourceName := range available { + resourceNamesByNUMA.Insert(string(resourceName)) } - for resourceName, quantity := range requests { - if allocatableQuantity, ok := allocatable[resourceName]; ok { - intersectionResources.Insert(string(resourceName)) - var allocated resource.Quantity - allocatable[resourceName], requests[resourceName], allocated = allocateRes(allocatableQuantity, quantity) - if !allocated.IsZero() { - r.Resources[resourceName] = allocated + } + sortedNUMANodeByResource := map[corev1.ResourceName][]int{} + numaNodes := options.hint.NUMANodeAffinity.GetBits() + for resourceName := range resourceNamesByNUMA { + sortedNUMANodes := make([]int, len(numaNodes)) + copy(sortedNUMANodes, numaNodes) + sort.Slice(sortedNUMANodes, func(i, j int) bool { + iAvailableOfResource := totalAvailable[i][corev1.ResourceName(resourceName)] + return (&iAvailableOfResource).Cmp(totalAvailable[j][corev1.ResourceName(resourceName)]) < 0 + }) + sortedNUMANodeByResource[corev1.ResourceName(resourceName)] = sortedNUMANodes + } + allocatedNUMANodeResources := map[int]*NUMANodeResource{} + for resourceName, quantity := range requests { + for i, numaNodeID := range sortedNUMANodeByResource[resourceName] { + splittedQuantity := splitQuantity(resourceName, quantity, len(numaNodes)-i, options) + _, _, allocated := allocateRes(totalAvailable[numaNodeID][resourceName], splittedQuantity) + if !allocated.IsZero() { + allocatedNUMANodeResource := allocatedNUMANodeResources[numaNodeID] + if allocatedNUMANodeResource == nil { + allocatedNUMANodeResource = &NUMANodeResource{ + Node: numaNodeID, + Resources: corev1.ResourceList{}, + } + allocatedNUMANodeResources[numaNodeID] = allocatedNUMANodeResource } + allocatedNUMANodeResource.Resources[resourceName] = allocated + quantity.Sub(allocated) } } - if !quotav1.IsZero(r.Resources) { - result = append(result, r) - } - if quotav1.IsZero(requests) { - break - } + requests[resourceName] = quantity } - var reasons []string for resourceName, quantity := range requests { - if intersectionResources.Has(string(resourceName)) { + if resourceNamesByNUMA.Has(string(resourceName)) { if !quantity.IsZero() { reasons = append(reasons, fmt.Sprintf("Insufficient NUMA %s", resourceName)) } } } - if len(reasons) > 0 { - return nil, framework.NewStatus(framework.Unschedulable, reasons...).AsError() + result := make([]NUMANodeResource, 0, len(allocatedNUMANodeResources)) + for _, numaNodeResource := range allocatedNUMANodeResources { + result = append(result, *numaNodeResource) } - return result, nil + sort.Slice(result, func(i, j int) bool { + return result[i].Node < result[j].Node + }) + return result, reasons +} + +func splitQuantity(resourceName corev1.ResourceName, quantity resource.Quantity, numaNodeCount int, options *ResourceOptions) resource.Quantity { + if resourceName != corev1.ResourceCPU { + return *resource.NewQuantity(quantity.Value()/int64(numaNodeCount), quantity.Format) + } + if !options.requestCPUBind { + return *resource.NewMilliQuantity(quantity.MilliValue()/int64(numaNodeCount), quantity.Format) + } + if options.requiredCPUBindPolicy && options.cpuBindPolicy == schedulingconfig.CPUBindPolicyFullPCPUs { + cpusPerCore := int64(options.topologyOptions.CPUTopology.CPUsPerCore()) + numOfPCPUs := quantity.Value() / cpusPerCore + numOfPCPUsPerNUMA := numOfPCPUs / int64(numaNodeCount) + return *resource.NewQuantity(numOfPCPUsPerNUMA*cpusPerCore, quantity.Format) + } + return *resource.NewQuantity(quantity.Value()/int64(numaNodeCount), quantity.Format) } func allocateRes(available, request resource.Quantity) (resource.Quantity, resource.Quantity, resource.Quantity) { diff --git a/pkg/scheduler/plugins/nodenumaresource/resource_manager_test.go b/pkg/scheduler/plugins/nodenumaresource/resource_manager_test.go index 9ec9215e5..77d08f25a 100644 --- a/pkg/scheduler/plugins/nodenumaresource/resource_manager_test.go +++ b/pkg/scheduler/plugins/nodenumaresource/resource_manager_test.go @@ -23,6 +23,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + quotav1 "k8s.io/apiserver/pkg/quota/v1" + "k8s.io/klog/v2" apiext "github.com/koordinator-sh/koordinator/apis/extension" schedulingconfig "github.com/koordinator-sh/koordinator/pkg/scheduler/apis/config" @@ -583,6 +585,693 @@ func TestResourceManagerAllocate(t *testing.T) { if tt.wantErr != (err != nil) { t.Errorf("wantErr %v but got %v", tt.wantErr, err != nil) } + if tt.want != nil { + for i := range tt.want.NUMANodeResources { + assert.Equal(t, tt.want.NUMANodeResources[i].Node, got.NUMANodeResources[i].Node) + assert.True(t, quotav1.Equals(tt.want.NUMANodeResources[i].Resources, got.NUMANodeResources[i].Resources)) + } + tt.want.NUMANodeResources = nil + got.NUMANodeResources = nil + } + assert.Equal(t, tt.want, got) + }) + } +} + +func TestAllocateDistributeEvenly(t *testing.T) { + tests := []struct { + name string + pod *corev1.Pod + options *ResourceOptions + amplificationRatios map[corev1.ResourceName]apiext.Ratio + allocated *PodAllocation + want *PodAllocation + wantErr bool + }{ + { + name: "allocate with non-existing resources in NUMA", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: false, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + apiext.ResourceGPUMemory: resource.MustParse("10Gi"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + want: &PodAllocation{ + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "allocate with insufficient resources", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: false, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("108"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + want: nil, + wantErr: true, + }, + { + name: "allocate with required CPUBindPolicyFullPCPUs", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: true, + requiredCPUBindPolicy: true, + cpuBindPolicy: schedulingconfig.CPUBindPolicyFullPCPUs, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + want: &PodAllocation{ + CPUSet: cpuset.MustParse("0-1,52-53"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "allocate with required CPUBindPolicyFullPCPUs and allocated", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: true, + requiredCPUBindPolicy: true, + cpuBindPolicy: schedulingconfig.CPUBindPolicyFullPCPUs, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + allocated: &PodAllocation{ + UID: "123456", + Name: "test-xxx", + Namespace: "default", + CPUSet: cpuset.MustParse("4-101"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("48"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50"), + }, + }, + }, + }, + want: &PodAllocation{ + CPUSet: cpuset.MustParse("0-1,102-103"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(2, resource.DecimalSI), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(2, resource.DecimalSI), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "failed to allocate with required CPUBindPolicyFullPCPUs and allocated", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: true, + requiredCPUBindPolicy: true, + cpuBindPolicy: schedulingconfig.CPUBindPolicyFullPCPUs, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0) + return mask + }(), + }, + }, + allocated: &PodAllocation{ + UID: "123456", + Name: "test-xxx", + Namespace: "default", + CPUSet: cpuset.MustParse("1,3,5,7-104"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("48"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("52"), + }, + }, + }, + }, + want: nil, + wantErr: true, + }, + { + name: "allocate with required CPUBindPolicySpreadByPCPUs", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: true, + requiredCPUBindPolicy: true, + cpuBindPolicy: schedulingconfig.CPUBindPolicySpreadByPCPUs, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + want: &PodAllocation{ + CPUSet: cpuset.MustParse("0,2,52,54"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "allocate with required CPUBindPolicySpreadByPCPUs and allocated, plural numCPUsNeeded, not balanced", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: true, + requiredCPUBindPolicy: true, + cpuBindPolicy: schedulingconfig.CPUBindPolicySpreadByPCPUs, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + allocated: &PodAllocation{ + UID: "123456", + Name: "test-xxx", + Namespace: "default", + CPUSet: cpuset.MustParse("1,3,5,7-101,103"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("48"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50"), + }, + }, + }, + }, + want: &PodAllocation{ + CPUSet: cpuset.MustParse("0,2,4,102"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(3, resource.DecimalSI), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(1, resource.DecimalSI), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "allocate with required CPUBindPolicySpreadByPCPUs and allocated, plural numCPUsNeeded, balanced", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: true, + requiredCPUBindPolicy: true, + cpuBindPolicy: schedulingconfig.CPUBindPolicySpreadByPCPUs, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + allocated: &PodAllocation{ + UID: "123456", + Name: "test-xxx", + Namespace: "default", + CPUSet: cpuset.MustParse("4-100"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("48"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("49"), + }, + }, + }, + }, + want: &PodAllocation{ + CPUSet: cpuset.MustParse("0,2,101-102"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(2, resource.DecimalSI), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(2, resource.DecimalSI), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "allocate with required CPUBindPolicySpreadByPCPUs and allocated, singular num cpus needed", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 5, + requestCPUBind: true, + requiredCPUBindPolicy: true, + cpuBindPolicy: schedulingconfig.CPUBindPolicySpreadByPCPUs, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + allocated: &PodAllocation{ + UID: "123456", + Name: "test-xxx", + Namespace: "default", + CPUSet: cpuset.MustParse("1,3,5,7-97,103"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("48"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("47"), + }, + }, + }, + }, + want: &PodAllocation{ + CPUSet: cpuset.MustParse("0,2,4,98,100"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(3, resource.DecimalSI), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(2, resource.DecimalSI), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "failed to allocate with required CPUBindPolicySpreadByPCPUs and allocated", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: true, + requiredCPUBindPolicy: true, + cpuBindPolicy: schedulingconfig.CPUBindPolicySpreadByPCPUs, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + allocated: &PodAllocation{ + UID: "123456", + Name: "test-xxx", + Namespace: "default", + CPUSet: cpuset.MustParse("4-104"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("48"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("52"), + }, + }, + }, + }, + want: nil, + wantErr: true, + }, + { + name: "allocate with required CPUBindPolicySpreadByPCPUs and amplified requests", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: true, + requiredCPUBindPolicy: true, + cpuBindPolicy: schedulingconfig.CPUBindPolicySpreadByPCPUs, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("6"), + }, + originalRequests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + amplificationRatios: map[corev1.ResourceName]apiext.Ratio{ + corev1.ResourceCPU: 1.5, + }, + want: &PodAllocation{ + CPUSet: cpuset.MustParse("0,2,52,54"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "allocate with required CPUBindPolicySpreadByPCPUs and allocated and amplified requests", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 4, + requestCPUBind: true, + requiredCPUBindPolicy: true, + cpuBindPolicy: schedulingconfig.CPUBindPolicySpreadByPCPUs, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("6"), + }, + originalRequests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + amplificationRatios: map[corev1.ResourceName]apiext.Ratio{ + corev1.ResourceCPU: 1.5, + }, + allocated: &PodAllocation{ + UID: "123456", + Name: "test-xxx", + Namespace: "default", + CPUSet: cpuset.MustParse("1,3,5,7-101"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("48"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50"), + }, + }, + }, + }, + want: &PodAllocation{ + CPUSet: cpuset.MustParse("0,2,4,102"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "allocate with CPU Share and allocated and amplified ratios", + pod: &corev1.Pod{}, + options: &ResourceOptions{ + numCPUsNeeded: 0, + requestCPUBind: false, + requiredCPUBindPolicy: false, + requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3.5"), + }, + originalRequests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3.5"), + }, + hint: topologymanager.NUMATopologyHint{ + NUMANodeAffinity: func() bitmask.BitMask { + mask, _ := bitmask.NewBitMask(0, 1) + return mask + }(), + }, + }, + amplificationRatios: map[corev1.ResourceName]apiext.Ratio{ + corev1.ResourceCPU: 1.5, + }, + allocated: &PodAllocation{ + UID: "123456", + Name: "test-xxx", + Namespace: "default", + CPUSet: cpuset.MustParse("0-49,52-101"), + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50"), + }, + }, + }, + }, + want: &PodAllocation{ + NUMANodeResources: []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1.75"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1.75"), + }, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + suit := newPluginTestSuit(t, nil, nil) + tom := NewTopologyOptionsManager() + tom.UpdateTopologyOptions("test-node", func(options *TopologyOptions) { + options.CPUTopology = buildCPUTopologyForTest(2, 1, 26, 2) + options.NUMANodeResources = []NUMANodeResource{ + { + Node: 0, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("52"), + corev1.ResourceMemory: resource.MustParse("128Gi"), + }, + }, + { + Node: 1, + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("52"), + corev1.ResourceMemory: resource.MustParse("128Gi"), + }, + }, + } + }) + tt.options.topologyOptions = tom.GetTopologyOptions("test-node") + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: corev1.NodeStatus{ + Allocatable: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("104"), + corev1.ResourceMemory: resource.MustParse("256Gi"), + }, + }, + } + apiext.SetNodeResourceAmplificationRatios(node, tt.amplificationRatios) + resourceManager := NewResourceManager(suit.Handle, schedulingconfig.NUMALeastAllocated, tom) + if tt.allocated != nil { + resourceManager.Update(node.Name, tt.allocated) + } + if tt.options.originalRequests == nil { + tt.options.originalRequests = tt.options.requests.DeepCopy() + } + assert.NoError(t, amplifyNUMANodeResources(node, &tt.options.topologyOptions)) + + got, err := resourceManager.Allocate(node, tt.pod, tt.options) + if tt.wantErr != (err != nil) { + t.Errorf("wantErr %v but got %v", tt.wantErr, err != nil) + } + if tt.want != nil { + for i := range tt.want.NUMANodeResources { + assert.Equal(t, tt.want.NUMANodeResources[i].Node, got.NUMANodeResources[i].Node) + assert.True(t, quotav1.Equals(tt.want.NUMANodeResources[i].Resources, got.NUMANodeResources[i].Resources)) + } + tt.want.NUMANodeResources = nil + got.NUMANodeResources = nil + } assert.Equal(t, tt.want, got) }) } @@ -1025,3 +1714,9 @@ func TestResourceManagerGetTopologyHint(t *testing.T) { }) } } + +func TestA(t *testing.T) { + a := map[string]corev1.ResourceList{} + b := a["a"] + klog.Info(b["cpu"]) +} diff --git a/pkg/scheduler/plugins/nodenumaresource/topology_hint_test.go b/pkg/scheduler/plugins/nodenumaresource/topology_hint_test.go index 5f7ed9a4c..4cbe35826 100644 --- a/pkg/scheduler/plugins/nodenumaresource/topology_hint_test.go +++ b/pkg/scheduler/plugins/nodenumaresource/topology_hint_test.go @@ -24,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + quotav1 "k8s.io/apiserver/pkg/quota/v1" "k8s.io/kubernetes/pkg/scheduler/framework" apiext "github.com/koordinator-sh/koordinator/apis/extension" @@ -116,5 +117,12 @@ func TestReserveByNUMANode(t *testing.T) { }, }, } + for i := range expectPodAllocation.NUMANodeResources { + assert.Equal(t, expectPodAllocation.NUMANodeResources[i].Node, state.allocation.NUMANodeResources[i].Node) + assert.True(t, quotav1.Equals(expectPodAllocation.NUMANodeResources[i].Resources, state.allocation.NUMANodeResources[i].Resources)) + } + expectPodAllocation.NUMANodeResources = nil + state.allocation.NUMANodeResources = nil assert.Equal(t, expectPodAllocation, state.allocation) + }