From 32c354dfad755fa5911cdefc653d3abe3386a283 Mon Sep 17 00:00:00 2001 From: Zhang Kang Date: Wed, 22 May 2024 13:55:30 +0800 Subject: [PATCH 1/2] add resctrl updater Signed-off-by: Zhang Kang --- pkg/koordlet/resourceexecutor/config.go | 1 + .../resourceexecutor/resctrl_updater.go | 77 +++++++++++++++ .../resourceexecutor/resctrl_updater_test.go | 98 +++++++++++++++++++ pkg/koordlet/util/system/resctrl.go | 46 ++++++++- 4 files changed, 218 insertions(+), 4 deletions(-) diff --git a/pkg/koordlet/resourceexecutor/config.go b/pkg/koordlet/resourceexecutor/config.go index c66ca6e54..a1b8e22bf 100644 --- a/pkg/koordlet/resourceexecutor/config.go +++ b/pkg/koordlet/resourceexecutor/config.go @@ -22,6 +22,7 @@ const ( ReasonUpdateCgroups = "UpdateCgroups" ReasonUpdateSystemConfig = "UpdateSystemConfig" ReasonUpdateResctrl = "UpdateResctrl" // update resctrl tasks, schemata + CreateCATGroup = "CreateCATGroup" EvictPodByNodeMemoryUsage = "EvictPodByNodeMemoryUsage" EvictPodByBECPUSatisfaction = "EvictPodByBECPUSatisfaction" diff --git a/pkg/koordlet/resourceexecutor/resctrl_updater.go b/pkg/koordlet/resourceexecutor/resctrl_updater.go index ba8a19b63..1d899be94 100644 --- a/pkg/koordlet/resourceexecutor/resctrl_updater.go +++ b/pkg/koordlet/resourceexecutor/resctrl_updater.go @@ -55,6 +55,64 @@ func (r *ResctrlSchemataResourceUpdater) Clone() ResourceUpdater { } } +func NewResctrlSchemataResource(group, schemata string, e *audit.EventHelper) (ResourceUpdater, error) { + if schemata == "" { + return nil, fmt.Errorf("schemata is nil") + } + schemataFile := sysutil.ResctrlSchemata.Path(group) + schemataKey := sysutil.ResctrlSchemataName + ":" + schemataFile + // The current assumption is that the cache ids obtained through + // resctrl schemata will not go wrong. TODO: Use the ability of node info + // to obtain cache ids to replace the current method. + ids, _ := sysutil.CacheIdsCacheFunc() + schemataRaw := sysutil.NewResctrlSchemataRaw(ids).WithL3Num(len(ids)) + err := schemataRaw.ParseResctrlSchemata(schemata, -1) + if err != nil { + klog.Errorf("failed to parse %v", err) + } + items := []string{} + for _, item := range []struct { + validFunc func() (bool, string) + value func() string + }{ + {validFunc: schemataRaw.ValidateL3, value: schemataRaw.L3String}, + {validFunc: schemataRaw.ValidateMB, value: schemataRaw.MBString}, + } { + if valid, _ := item.validFunc(); valid { + items = append(items, item.value()) + } + } + schemataStr := strings.Join(items, "") + klog.V(6).Infof("generate new resctrl schemata resource, file %s, key %s, value %s, schemata %s", + schemataFile, schemataKey, schemataStr, schemata) + return &ResctrlSchemataResourceUpdater{ + DefaultResourceUpdater: DefaultResourceUpdater{ + key: schemataKey, + file: schemataFile, + value: schemataStr, + updateFunc: UpdateResctrlSchemataFunc, + eventHelper: e, + }, + schemataRaw: schemataRaw, + }, err +} + +func NewCatGroupResource(group string, e *audit.EventHelper) (ResourceUpdater, error) { + if group == "" { + return nil, fmt.Errorf("group is nil") + } + schemataFile := sysutil.ResctrlSchemata.Path(group) + + klog.V(6).Infof("generate new cat group resource, file %s", schemataFile) + return &DefaultResourceUpdater{ + key: group, + file: schemataFile, + value: "", + updateFunc: InitCatGroupFunc, + eventHelper: e, + }, nil +} + func NewResctrlL3SchemataResource(group, schemataDelta string, l3Num int) ResourceUpdater { schemataFile := sysutil.ResctrlSchemata.Path(group) l3SchemataKey := sysutil.L3SchemataPrefix + ":" + schemataFile @@ -116,6 +174,25 @@ func CalculateResctrlL3TasksResource(group string, taskIds []int32) (ResourceUpd return NewCommonDefaultUpdaterWithUpdateFunc(tasksPath, tasksPath, builder.String(), UpdateResctrlTasksFunc, eventHelper) } +func InitCatGroupFunc(u ResourceUpdater) error { + r, ok := u.(*DefaultResourceUpdater) + if !ok { + return fmt.Errorf("not a DefaultResourceUpdater") + } + + if updated, err := sysutil.InitCatGroupIfNotExist(r.key); err != nil { + klog.Errorf("init cat group dir %v failed, error %v", r.key, err) + return err + } else if updated { + klog.V(4).Infof("create cat dir for group %v successfully", r.key) + } else { + klog.V(6).Infof("cat dir for group %v is already created", r.key) + } + + _ = audit.V(3).Reason(CreateCATGroup).Message("Create %v to %v", u.Key(), u.Value()).Do() + return nil +} + func UpdateResctrlSchemataFunc(u ResourceUpdater) error { r, ok := u.(*ResctrlSchemataResourceUpdater) if !ok { diff --git a/pkg/koordlet/resourceexecutor/resctrl_updater_test.go b/pkg/koordlet/resourceexecutor/resctrl_updater_test.go index 201927cf1..797c204fd 100644 --- a/pkg/koordlet/resourceexecutor/resctrl_updater_test.go +++ b/pkg/koordlet/resourceexecutor/resctrl_updater_test.go @@ -112,3 +112,101 @@ func TestNewResctrlMbSchemataResource(t *testing.T) { assert.NoError(t, err) }) } + +func TestNewResctrlSchemataResource(t *testing.T) { + t.Run("test_all_schemata", func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "NewResctrlSchemataResource" + helper.MkDirAll(sysFSRootDirName) + system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName) + testingPrepareResctrlL3CatGroups(t, "7ff", " L3:0=ff;1=ff\n MB:0=100;1=100") + updater, _ := NewResctrlSchemataResource("BE", "L3:0=f;1=f\nMB:0=60;1=60", nil) + assert.Equal(t, "L3:0=f;1=f;\nMB:0=60;1=60;\n", updater.Value()) + err := updater.update() + assert.NoError(t, err) + }) + + t.Run("test_LLC_resource", func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "NewResctrlSchemataResourceSingleLLC" + helper.MkDirAll(sysFSRootDirName) + system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName) + testingPrepareResctrlL3CatGroups(t, "7ff", " L3:0=ff;1=ff") + updater, _ := NewResctrlSchemataResource("BE", "L3:0=f;1=f", nil) + assert.Equal(t, "L3:0=f;1=f;\n", updater.Value()) + err := updater.update() + assert.NoError(t, err) + }) + + t.Run("test_MB_resource", func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "NewResctrlSchemataResourceSingleMB" + helper.MkDirAll(sysFSRootDirName) + system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName) + testingPrepareResctrlL3CatGroups(t, "", " MB:0=10;1=10") + updater, _ := NewResctrlSchemataResource("BE", "MB:0=20;1=20", nil) + assert.Equal(t, "MB:0=20;1=20;\n", updater.Value()) + err := updater.update() + assert.NoError(t, err) + }) +} + +func TestNewCatGroupResource(t *testing.T) { + t.Run("test_create_cat_success", func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "NewCatGroupResource" + helper.MkDirAll(sysFSRootDirName) + system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName) + resctrlDir := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir) + l3CatDir := filepath.Join(resctrlDir, system.RdtInfoDir, system.L3CatDir) + err := os.MkdirAll(l3CatDir, 0700) + assert.NoError(t, err) + updater, _ := NewCatGroupResource("ga", nil) + err = updater.update() + assert.NoError(t, err) + }) + + t.Run("test_create_cat_exist", func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "NewCatGroupResource" + helper.MkDirAll(sysFSRootDirName) + system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName) + resctrlDir := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir) + l3CatDir := filepath.Join(resctrlDir, system.RdtInfoDir, system.L3CatDir) + err := os.MkdirAll(l3CatDir, 0700) + assert.NoError(t, err) + err = os.MkdirAll("ga", 0700) + assert.NoError(t, err) + updater, _ := NewCatGroupResource("ga", nil) + err = updater.update() + assert.NoError(t, err) + }) + + t.Run("test_create_cat_fail", func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "NewCatGroupResource" + helper.MkDirAll(sysFSRootDirName) + system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName) + resctrlDir := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir) + l3CatDir := filepath.Join(resctrlDir, system.RdtInfoDir, system.L3CatDir) + err := os.MkdirAll(l3CatDir, 0700) + assert.NoError(t, err) + err = os.MkdirAll("ga", 0700) + assert.NoError(t, err) + updater, _ := NewCatGroupResource("ga/ga/", nil) + err = updater.update() + assert.Error(t, err) + }) +} diff --git a/pkg/koordlet/util/system/resctrl.go b/pkg/koordlet/util/system/resctrl.go index 9580bdd15..c2d67d3ce 100644 --- a/pkg/koordlet/util/system/resctrl.go +++ b/pkg/koordlet/util/system/resctrl.go @@ -256,11 +256,18 @@ func (r *ResctrlSchemataRaw) L3Number() int { } func (r *ResctrlSchemataRaw) CacheIds() []int { - ids := []int{} + ids1 := []int{} for id := range r.L3 { - ids = append(ids, id) + ids1 = append(ids1, id) } - return ids + ids2 := []int{} + for id := range r.MB { + ids2 = append(ids2, id) + } + if len(ids1) >= len(ids2) { + return ids1 + } + return ids2 } func (r *ResctrlSchemataRaw) L3String() string { @@ -342,6 +349,11 @@ func (r *ResctrlSchemataRaw) ValidateL3() (bool, string) { if r.L3Num != len(r.L3) { return false, "unmatched L3 number and CAT infos" } + for _, value := range r.L3 { + if value <= 0 { + return false, "wrong value of L3 mask" + } + } return true, "" } @@ -352,6 +364,11 @@ func (r *ResctrlSchemataRaw) ValidateMB() (bool, string) { if len(r.MB) <= 0 { return false, "no MBA info" } + for _, value := range r.MB { + if value <= 0 { + return false, "wrong value of MB mask" + } + } return true, "" } @@ -444,7 +461,13 @@ func ReadResctrlSchemataRaw(schemataFile string, l3Num int) (*ResctrlSchemataRaw return nil, fmt.Errorf("failed to parse l3 schemata, content %s, err: %v", string(content), err) } if l3Num == -1 { - schemataRaw.WithL3Num(len(schemataRaw.L3)) + len1 := len(schemataRaw.L3) + len2 := len(schemataRaw.MB) + if len1 >= len2 { + schemataRaw.WithL3Num(len1) + } else { + schemataRaw.WithL3Num(len2) + } } return schemataRaw, nil @@ -560,6 +583,21 @@ func CheckAndTryEnableResctrlCat() error { return nil } +func InitCatGroupIfNotExist(group string) (bool, error) { + path := GetResctrlGroupRootDirPath(group) + _, err := os.Stat(path) + if err == nil { + return false, nil + } else if !os.IsNotExist(err) { + return false, fmt.Errorf("check dir %v for group %s but got unexpected err: %v", path, group, err) + } + err = os.Mkdir(path, 0755) + if err != nil { + return false, fmt.Errorf("create dir %v failed for group %s, err: %v", path, group, err) + } + return true, nil +} + func CheckResctrlSchemataValid() error { schemataPath := GetResctrlSchemataFilePath("") schemataRaw, err := ReadResctrlSchemataRaw(schemataPath, -1) From a1a952998c133918de83e3949ac97fa60e2e94b2 Mon Sep 17 00:00:00 2001 From: Zhang Kang Date: Thu, 30 May 2024 17:08:54 +0800 Subject: [PATCH 2/2] add todo Signed-off-by: Zhang Kang --- pkg/koordlet/util/system/resctrl.go | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/pkg/koordlet/util/system/resctrl.go b/pkg/koordlet/util/system/resctrl.go index c2d67d3ce..26c866f0c 100644 --- a/pkg/koordlet/util/system/resctrl.go +++ b/pkg/koordlet/util/system/resctrl.go @@ -256,18 +256,12 @@ func (r *ResctrlSchemataRaw) L3Number() int { } func (r *ResctrlSchemataRaw) CacheIds() []int { - ids1 := []int{} + // TODO: consider situation that L3 number and the MB number are the same. + ids := []int{} for id := range r.L3 { - ids1 = append(ids1, id) + ids = append(ids, id) } - ids2 := []int{} - for id := range r.MB { - ids2 = append(ids2, id) - } - if len(ids1) >= len(ids2) { - return ids1 - } - return ids2 + return ids } func (r *ResctrlSchemataRaw) L3String() string { @@ -461,13 +455,7 @@ func ReadResctrlSchemataRaw(schemataFile string, l3Num int) (*ResctrlSchemataRaw return nil, fmt.Errorf("failed to parse l3 schemata, content %s, err: %v", string(content), err) } if l3Num == -1 { - len1 := len(schemataRaw.L3) - len2 := len(schemataRaw.MB) - if len1 >= len2 { - schemataRaw.WithL3Num(len1) - } else { - schemataRaw.WithL3Num(len2) - } + schemataRaw.WithL3Num(len(schemataRaw.L3)) } return schemataRaw, nil