koordlet: fix be cpuset for numaResource (#1818)

Signed-off-by: saintube <saintube@foxmail.com>

pkg/koordlet/runtimehooks/hooks/cpuset/rule.go

@@ -30,7 +30,8 @@ import (
 	"github.com/koordinator-sh/koordinator/pkg/features"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/statesinformer"
-	"github.com/koordinator-sh/koordinator/pkg/koordlet/util"
+	koordletutil "github.com/koordinator-sh/koordinator/pkg/koordlet/util"
+	"github.com/koordinator-sh/koordinator/pkg/util"
 	"github.com/koordinator-sh/koordinator/pkg/util/cpuset"
 )
 
@@ -64,8 +65,12 @@ func (r *cpusetRule) getContainerCPUSet(containerReq *protocol.ContainerRequest)
 	// check if numa-aware
 	isNUMAAware := false
 	for _, numaNode := range podAlloc.NUMANodeResources {
+		if numaNode.Resources == nil {
+			continue
+		}
 		// check if cpu resource is allocated in numa-level since there can be numa allocation without cpu
-		if numaNode.Resources != nil && !numaNode.Resources.Cpu().IsZero() {
+		if !numaNode.Resources.Cpu().IsZero() ||
+			util.GetBatchMilliCPUFromResourceList(numaNode.Resources) > 0 {
 			isNUMAAware = true
 			break
 		}
@@ -117,7 +122,7 @@ func (r *cpusetRule) getContainerCPUSet(containerReq *protocol.ContainerRequest)
 		return pointer.String(strings.Join(allSharePoolCPUs, ",")), nil
 	}
 
-	kubeQOS := util.GetKubeQoSByCgroupParent(containerReq.CgroupParent)
+	kubeQOS := koordletutil.GetKubeQoSByCgroupParent(containerReq.CgroupParent)
 	if kubeQOS == corev1.PodQOSBestEffort {
 		// besteffort pods including QoS=BE, clear cpuset of BE container to avoid conflict with kubelet static policy,
 		// which will pass cpuset in StartContainerRequest of CRI

pkg/koordlet/runtimehooks/hooks/cpuset/rule_test.go

@@ -382,6 +382,59 @@ func Test_cpusetRule_getContainerCPUSet(t *testing.T) {
 			want:    pointer.String("9-15"),
 			wantErr: false,
 		},
+		{
+			name: "get cpuset from annotation be share pool",
+			fields: fields{
+				sharePools: []ext.CPUSharedPool{
+					{
+						Socket: 0,
+						Node:   0,
+						CPUSet: "1-7",
+					},
+					{
+						Socket: 1,
+						Node:   1,
+						CPUSet: "9-15",
+					},
+				},
+				beSharePools: []ext.CPUSharedPool{
+					{
+						Socket: 0,
+						Node:   0,
+						CPUSet: "0-7",
+					},
+					{
+						Socket: 1,
+						Node:   1,
+						CPUSet: "8-15",
+					},
+				},
+			},
+			args: args{
+				beCPUManagerEnabled: true,
+				containerReq: &protocol.ContainerRequest{
+					PodMeta:       protocol.PodMeta{},
+					ContainerMeta: protocol.ContainerMeta{},
+					PodLabels: map[string]string{
+						ext.LabelPodQoS: string(ext.QoSBE),
+					},
+					PodAnnotations: map[string]string{},
+					CgroupParent:   "besteffort/test-pod/test-container",
+				},
+				podAlloc: &ext.ResourceStatus{
+					NUMANodeResources: []ext.NUMANodeResource{
+						{
+							Node: 1,
+							Resources: map[corev1.ResourceName]resource.Quantity{
+								ext.BatchCPU: *resource.NewQuantity(2000, resource.DecimalSI),
+							},
+						},
+					},
+				},
+			},
+			want:    pointer.String("8-15"),
+			wantErr: false,
+		},
 		{
 			name: "get cpuset from annotation system qos resource",
 			fields: fields{
@@ -453,11 +506,8 @@ func Test_cpusetRule_getContainerCPUSet(t *testing.T) {
 			features.DefaultMutableKoordletFeatureGate.SetFromMap(
 				map[string]bool{string(features.BECPUManager): tt.args.beCPUManagerEnabled})
 			got, err := r.getContainerCPUSet(tt.args.containerReq)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("getCPUSet() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			assert.Equal(t, tt.want, got, "cpuset of container should be equal")
+			assert.Equal(t, tt.wantErr, err != nil, err)
+			assert.Equal(t, tt.want, got, "cpuset of container should be equal, want %+v, got %+v", util.DumpJSON(tt.want), util.DumpJSON(got))
 		})
 	}
 	// node.koordinator.sh/cpu-shared-pools: '[{"cpuset":"2-7"}]'