koordlet: add resctrl updater

pkg/koordlet/resourceexecutor/config.go

@@ -22,6 +22,7 @@ const (
 	ReasonUpdateCgroups      = "UpdateCgroups"
 	ReasonUpdateSystemConfig = "UpdateSystemConfig"
 	ReasonUpdateResctrl      = "UpdateResctrl" // update resctrl tasks, schemata
+	CreateCATGroup           = "CreateCATGroup"
 
 	EvictPodByNodeMemoryUsage   = "EvictPodByNodeMemoryUsage"
 	EvictPodByBECPUSatisfaction = "EvictPodByBECPUSatisfaction"

pkg/koordlet/resourceexecutor/resctrl_updater.go

@@ -55,6 +55,64 @@ func (r *ResctrlSchemataResourceUpdater) Clone() ResourceUpdater {
 	}
 }
 
+func NewResctrlSchemataResource(group, schemata string, e *audit.EventHelper) (ResourceUpdater, error) {
+	if schemata == "" {
+		return nil, fmt.Errorf("schemata is nil")
+	}
+	schemataFile := sysutil.ResctrlSchemata.Path(group)
+	schemataKey := sysutil.ResctrlSchemataName + ":" + schemataFile
+	// The current assumption is that the cache ids obtained through
+	// resctrl schemata will not go wrong. TODO: Use the ability of node info
+	// to obtain cache ids to replace the current method.
+	ids, _ := sysutil.CacheIdsCacheFunc()
+	schemataRaw := sysutil.NewResctrlSchemataRaw(ids).WithL3Num(len(ids))
+	err := schemataRaw.ParseResctrlSchemata(schemata, -1)
+	if err != nil {
+		klog.Errorf("failed to parse %v", err)
+	}
+	items := []string{}
+	for _, item := range []struct {
+		validFunc func() (bool, string)
+		value     func() string
+	}{
+		{validFunc: schemataRaw.ValidateL3, value: schemataRaw.L3String},
+		{validFunc: schemataRaw.ValidateMB, value: schemataRaw.MBString},
+	} {
+		if valid, _ := item.validFunc(); valid {
+			items = append(items, item.value())
+		}
+	}
+	schemataStr := strings.Join(items, "")
+	klog.V(6).Infof("generate new resctrl schemata resource, file %s, key %s, value %s, schemata %s",
+		schemataFile, schemataKey, schemataStr, schemata)
+	return &ResctrlSchemataResourceUpdater{
+		DefaultResourceUpdater: DefaultResourceUpdater{
+			key:         schemataKey,
+			file:        schemataFile,
+			value:       schemataStr,
+			updateFunc:  UpdateResctrlSchemataFunc,
+			eventHelper: e,
+		},
+		schemataRaw: schemataRaw,
+	}, err
+}
+
+func NewCatGroupResource(group string, e *audit.EventHelper) (ResourceUpdater, error) {
+	if group == "" {
+		return nil, fmt.Errorf("group is nil")
+	}
+	schemataFile := sysutil.ResctrlSchemata.Path(group)
+
+	klog.V(6).Infof("generate new cat group resource, file %s", schemataFile)
+	return &DefaultResourceUpdater{
+		key:         group,
+		file:        schemataFile,
+		value:       "",
+		updateFunc:  InitCatGroupFunc,
+		eventHelper: e,
+	}, nil
+}
+
 func NewResctrlL3SchemataResource(group, schemataDelta string, l3Num int) ResourceUpdater {
 	schemataFile := sysutil.ResctrlSchemata.Path(group)
 	l3SchemataKey := sysutil.L3SchemataPrefix + ":" + schemataFile
@@ -116,6 +174,25 @@ func CalculateResctrlL3TasksResource(group string, taskIds []int32) (ResourceUpd
 	return NewCommonDefaultUpdaterWithUpdateFunc(tasksPath, tasksPath, builder.String(), UpdateResctrlTasksFunc, eventHelper)
 }
 
+func InitCatGroupFunc(u ResourceUpdater) error {
+	r, ok := u.(*DefaultResourceUpdater)
+	if !ok {
+		return fmt.Errorf("not a DefaultResourceUpdater")
+	}
+
+	if updated, err := sysutil.InitCatGroupIfNotExist(r.key); err != nil {
+		klog.Errorf("init cat group dir %v failed, error %v", r.key, err)
+		return err
+	} else if updated {
+		klog.V(4).Infof("create cat dir for group %v successfully", r.key)
+	} else {
+		klog.V(6).Infof("cat dir for group %v is already created", r.key)
+	}
+
+	_ = audit.V(3).Reason(CreateCATGroup).Message("Create %v to %v", u.Key(), u.Value()).Do()
+	return nil
+}
+
 func UpdateResctrlSchemataFunc(u ResourceUpdater) error {
 	r, ok := u.(*ResctrlSchemataResourceUpdater)
 	if !ok {

pkg/koordlet/resourceexecutor/resctrl_updater_test.go

@@ -112,3 +112,101 @@ func TestNewResctrlMbSchemataResource(t *testing.T) {
 		assert.NoError(t, err)
 	})
 }
+
+func TestNewResctrlSchemataResource(t *testing.T) {
+	t.Run("test_all_schemata", func(t *testing.T) {
+		helper := system.NewFileTestUtil(t)
+		defer helper.Cleanup()
+
+		sysFSRootDirName := "NewResctrlSchemataResource"
+		helper.MkDirAll(sysFSRootDirName)
+		system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName)
+		testingPrepareResctrlL3CatGroups(t, "7ff", "    L3:0=ff;1=ff\n    MB:0=100;1=100")
+		updater, _ := NewResctrlSchemataResource("BE", "L3:0=f;1=f\nMB:0=60;1=60", nil)
+		assert.Equal(t, "L3:0=f;1=f;\nMB:0=60;1=60;\n", updater.Value())
+		err := updater.update()
+		assert.NoError(t, err)
+	})
+
+	t.Run("test_LLC_resource", func(t *testing.T) {
+		helper := system.NewFileTestUtil(t)
+		defer helper.Cleanup()
+
+		sysFSRootDirName := "NewResctrlSchemataResourceSingleLLC"
+		helper.MkDirAll(sysFSRootDirName)
+		system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName)
+		testingPrepareResctrlL3CatGroups(t, "7ff", "    L3:0=ff;1=ff")
+		updater, _ := NewResctrlSchemataResource("BE", "L3:0=f;1=f", nil)
+		assert.Equal(t, "L3:0=f;1=f;\n", updater.Value())
+		err := updater.update()
+		assert.NoError(t, err)
+	})
+
+	t.Run("test_MB_resource", func(t *testing.T) {
+		helper := system.NewFileTestUtil(t)
+		defer helper.Cleanup()
+
+		sysFSRootDirName := "NewResctrlSchemataResourceSingleMB"
+		helper.MkDirAll(sysFSRootDirName)
+		system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName)
+		testingPrepareResctrlL3CatGroups(t, "", "    MB:0=10;1=10")
+		updater, _ := NewResctrlSchemataResource("BE", "MB:0=20;1=20", nil)
+		assert.Equal(t, "MB:0=20;1=20;\n", updater.Value())
+		err := updater.update()
+		assert.NoError(t, err)
+	})
+}
+
+func TestNewCatGroupResource(t *testing.T) {
+	t.Run("test_create_cat_success", func(t *testing.T) {
+		helper := system.NewFileTestUtil(t)
+		defer helper.Cleanup()
+
+		sysFSRootDirName := "NewCatGroupResource"
+		helper.MkDirAll(sysFSRootDirName)
+		system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName)
+		resctrlDir := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir)
+		l3CatDir := filepath.Join(resctrlDir, system.RdtInfoDir, system.L3CatDir)
+		err := os.MkdirAll(l3CatDir, 0700)
+		assert.NoError(t, err)
+		updater, _ := NewCatGroupResource("ga", nil)
+		err = updater.update()
+		assert.NoError(t, err)
+	})
+
+	t.Run("test_create_cat_exist", func(t *testing.T) {
+		helper := system.NewFileTestUtil(t)
+		defer helper.Cleanup()
+
+		sysFSRootDirName := "NewCatGroupResource"
+		helper.MkDirAll(sysFSRootDirName)
+		system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName)
+		resctrlDir := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir)
+		l3CatDir := filepath.Join(resctrlDir, system.RdtInfoDir, system.L3CatDir)
+		err := os.MkdirAll(l3CatDir, 0700)
+		assert.NoError(t, err)
+		err = os.MkdirAll("ga", 0700)
+		assert.NoError(t, err)
+		updater, _ := NewCatGroupResource("ga", nil)
+		err = updater.update()
+		assert.NoError(t, err)
+	})
+
+	t.Run("test_create_cat_fail", func(t *testing.T) {
+		helper := system.NewFileTestUtil(t)
+		defer helper.Cleanup()
+
+		sysFSRootDirName := "NewCatGroupResource"
+		helper.MkDirAll(sysFSRootDirName)
+		system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName)
+		resctrlDir := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir)
+		l3CatDir := filepath.Join(resctrlDir, system.RdtInfoDir, system.L3CatDir)
+		err := os.MkdirAll(l3CatDir, 0700)
+		assert.NoError(t, err)
+		err = os.MkdirAll("ga", 0700)
+		assert.NoError(t, err)
+		updater, _ := NewCatGroupResource("ga/ga/", nil)
+		err = updater.update()
+		assert.Error(t, err)
+	})
+}

pkg/koordlet/util/system/resctrl.go

@@ -256,11 +256,18 @@ func (r *ResctrlSchemataRaw) L3Number() int {
 }
 
 func (r *ResctrlSchemataRaw) CacheIds() []int {
-	ids := []int{}
+	ids1 := []int{}
 	for id := range r.L3 {
-		ids = append(ids, id)
+		ids1 = append(ids1, id)
 	}
-	return ids
+	ids2 := []int{}
+	for id := range r.MB {
+		ids2 = append(ids2, id)
+	}
+	if len(ids1) >= len(ids2) {
+		return ids1
+	}
+	return ids2
 }
 
 func (r *ResctrlSchemataRaw) L3String() string {
@@ -342,6 +349,11 @@ func (r *ResctrlSchemataRaw) ValidateL3() (bool, string) {
 	if r.L3Num != len(r.L3) {
 		return false, "unmatched L3 number and CAT infos"
 	}
+	for _, value := range r.L3 {
+		if value <= 0 {
+			return false, "wrong value of L3 mask"
+		}
+	}
 	return true, ""
 }
 
@@ -352,6 +364,11 @@ func (r *ResctrlSchemataRaw) ValidateMB() (bool, string) {
 	if len(r.MB) <= 0 {
 		return false, "no MBA info"
 	}
+	for _, value := range r.MB {
+		if value <= 0 {
+			return false, "wrong value of MB mask"
+		}
+	}
 	return true, ""
 }
 
@@ -444,7 +461,13 @@ func ReadResctrlSchemataRaw(schemataFile string, l3Num int) (*ResctrlSchemataRaw
 		return nil, fmt.Errorf("failed to parse l3 schemata, content %s, err: %v", string(content), err)
 	}
 	if l3Num == -1 {
-		schemataRaw.WithL3Num(len(schemataRaw.L3))
+		len1 := len(schemataRaw.L3)
+		len2 := len(schemataRaw.MB)
+		if len1 >= len2 {
+			schemataRaw.WithL3Num(len1)
+		} else {
+			schemataRaw.WithL3Num(len2)
+		}
 	}
 
 	return schemataRaw, nil
@@ -560,6 +583,21 @@ func CheckAndTryEnableResctrlCat() error {
 	return nil
 }
 
+func InitCatGroupIfNotExist(group string) (bool, error) {
+	path := GetResctrlGroupRootDirPath(group)
+	_, err := os.Stat(path)
+	if err == nil {
+		return false, nil
+	} else if !os.IsNotExist(err) {
+		return false, fmt.Errorf("check dir %v for group %s but got unexpected err: %v", path, group, err)
+	}
+	err = os.Mkdir(path, 0755)
+	if err != nil {
+		return false, fmt.Errorf("create dir %v failed for group %s, err: %v", path, group, err)
+	}
+	return true, nil
+}
+
 func CheckResctrlSchemataValid() error {
 	schemataPath := GetResctrlSchemataFilePath("")
 	schemataRaw, err := ReadResctrlSchemataRaw(schemataPath, -1)