diff --git a/apis/slo/v1alpha1/nodeslo_types.go b/apis/slo/v1alpha1/nodeslo_types.go index 0c3c0bc7a..faa96ab88 100644 --- a/apis/slo/v1alpha1/nodeslo_types.go +++ b/apis/slo/v1alpha1/nodeslo_types.go @@ -104,7 +104,8 @@ type MemoryQoSCfg struct { } type ResourceQoS struct { - MemoryQoS *MemoryQoSCfg `json:"memoryQoS,omitempty"` + MemoryQoS *MemoryQoSCfg `json:"memoryQoS,omitempty"` + ResctrlQoS *ResctrlQoSCfg `json:"resctrlQoS,omitempty"` } type ResourceQoSStrategy struct { @@ -151,17 +152,42 @@ type ResourceThresholdStrategy struct { MemoryEvictLowerPercent *int64 `json:"memoryEvictLowerPercent,omitempty"` } +// ResctrlQoSCfg stores node-level config of resctrl qos +type ResctrlQoSCfg struct { + // Enable indicates whether the resctrl qos is enabled. + Enable *bool `json:"enable,omitempty"` + ResctrlQoS `json:",inline"` +} + +type ResctrlQoS struct { + // LLC available range start for pods by percentage + // +kubebuilder:default=0 + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=100 + CATRangeStartPercent *int64 `json:"catRangeStartPercent,omitempty"` + // LLC available range end for pods by percentage + // +kubebuilder:default=100 + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=100 + CATRangeEndPercent *int64 `json:"catRangeEndPercent,omitempty"` + // MBA percent + // +kubebuilder:default=100 + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=100 + MBAPercent *int64 `json:"mbaPercent,omitempty"` +} + type CPUBurstPolicy string const ( // disable cpu burst policy - CPUBurstNone = "none" + CPUBurstNone CPUBurstPolicy = "none" // only enable cpu burst policy by setting cpu.cfs_burst_us - CPUBurstOnly = "cpuBurstOnly" + CPUBurstOnly CPUBurstPolicy = "cpuBurstOnly" // only enable cfs quota burst policy by scale up cpu.cfs_quota_us if pod throttled - CFSQuotaBurstOnly = "cfsQuotaBurstOnly" + CFSQuotaBurstOnly CPUBurstPolicy = "cfsQuotaBurstOnly" // enable both - CPUBurstAuto = "auto" + CPUBurstAuto CPUBurstPolicy = "auto" ) type CPUBurstConfig struct { diff --git a/apis/slo/v1alpha1/zz_generated.deepcopy.go b/apis/slo/v1alpha1/zz_generated.deepcopy.go index f8b228816..98e7772db 100644 --- a/apis/slo/v1alpha1/zz_generated.deepcopy.go +++ b/apis/slo/v1alpha1/zz_generated.deepcopy.go @@ -419,6 +419,57 @@ func (in *PodMetricInfo) DeepCopy() *PodMetricInfo { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResctrlQoS) DeepCopyInto(out *ResctrlQoS) { + *out = *in + if in.CATRangeStartPercent != nil { + in, out := &in.CATRangeStartPercent, &out.CATRangeStartPercent + *out = new(int64) + **out = **in + } + if in.CATRangeEndPercent != nil { + in, out := &in.CATRangeEndPercent, &out.CATRangeEndPercent + *out = new(int64) + **out = **in + } + if in.MBAPercent != nil { + in, out := &in.MBAPercent, &out.MBAPercent + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResctrlQoS. +func (in *ResctrlQoS) DeepCopy() *ResctrlQoS { + if in == nil { + return nil + } + out := new(ResctrlQoS) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResctrlQoSCfg) DeepCopyInto(out *ResctrlQoSCfg) { + *out = *in + if in.Enable != nil { + in, out := &in.Enable, &out.Enable + *out = new(bool) + **out = **in + } + in.ResctrlQoS.DeepCopyInto(&out.ResctrlQoS) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResctrlQoSCfg. +func (in *ResctrlQoSCfg) DeepCopy() *ResctrlQoSCfg { + if in == nil { + return nil + } + out := new(ResctrlQoSCfg) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceMap) DeepCopyInto(out *ResourceMap) { *out = *in @@ -449,6 +500,11 @@ func (in *ResourceQoS) DeepCopyInto(out *ResourceQoS) { *out = new(MemoryQoSCfg) (*in).DeepCopyInto(*out) } + if in.ResctrlQoS != nil { + in, out := &in.ResctrlQoS, &out.ResctrlQoS + *out = new(ResctrlQoSCfg) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceQoS. diff --git a/config/crd/bases/slo.koordinator.sh_nodeslos.yaml b/config/crd/bases/slo.koordinator.sh_nodeslos.yaml index 379bdd73c..4b9eeade0 100644 --- a/config/crd/bases/slo.koordinator.sh_nodeslos.yaml +++ b/config/crd/bases/slo.koordinator.sh_nodeslos.yaml @@ -163,6 +163,36 @@ spec: format: int64 type: integer type: object + resctrlQoS: + description: ResctrlQoSCfg stores node-level config of resctrl + qos + properties: + catRangeEndPercent: + default: 100 + description: LLC available range end for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + catRangeStartPercent: + default: 0 + description: LLC available range start for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: Enable indicates whether the resctrl qos + is enabled. + type: boolean + mbaPercent: + default: 100 + description: MBA percent + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object type: object cgroupRoot: description: ResourceQoS for root cgroup. @@ -259,6 +289,36 @@ spec: format: int64 type: integer type: object + resctrlQoS: + description: ResctrlQoSCfg stores node-level config of resctrl + qos + properties: + catRangeEndPercent: + default: 100 + description: LLC available range end for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + catRangeStartPercent: + default: 0 + description: LLC available range start for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: Enable indicates whether the resctrl qos + is enabled. + type: boolean + mbaPercent: + default: 100 + description: MBA percent + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object type: object ls: description: ResourceQoS for LS pods. @@ -355,6 +415,36 @@ spec: format: int64 type: integer type: object + resctrlQoS: + description: ResctrlQoSCfg stores node-level config of resctrl + qos + properties: + catRangeEndPercent: + default: 100 + description: LLC available range end for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + catRangeStartPercent: + default: 0 + description: LLC available range start for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: Enable indicates whether the resctrl qos + is enabled. + type: boolean + mbaPercent: + default: 100 + description: MBA percent + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object type: object lsr: description: ResourceQoS for LSR pods. @@ -451,6 +541,36 @@ spec: format: int64 type: integer type: object + resctrlQoS: + description: ResctrlQoSCfg stores node-level config of resctrl + qos + properties: + catRangeEndPercent: + default: 100 + description: LLC available range end for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + catRangeStartPercent: + default: 0 + description: LLC available range start for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: Enable indicates whether the resctrl qos + is enabled. + type: boolean + mbaPercent: + default: 100 + description: MBA percent + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object type: object system: description: ResourceQoS for system pods @@ -547,6 +667,36 @@ spec: format: int64 type: integer type: object + resctrlQoS: + description: ResctrlQoSCfg stores node-level config of resctrl + qos + properties: + catRangeEndPercent: + default: 100 + description: LLC available range end for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + catRangeStartPercent: + default: 0 + description: LLC available range start for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: Enable indicates whether the resctrl qos + is enabled. + type: boolean + mbaPercent: + default: 100 + description: MBA percent + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object type: object type: object resourceUsedThresholdWithBE: diff --git a/pkg/cache/expiration_cache.go b/pkg/cache/expiration_cache.go deleted file mode 100644 index b8ec86153..000000000 --- a/pkg/cache/expiration_cache.go +++ /dev/null @@ -1,128 +0,0 @@ -/* - Copyright 2022 The Koordinator Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cache - -import ( - "fmt" - "sync" - "time" - - "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog/v2" -) - -const ( - defaultExpiration = 2 * time.Minute - defaultGCInterval = time.Minute -) - -type item struct { - object interface{} - expirationTime time.Time -} - -type Cache struct { - items map[string]item - defaultExpiration time.Duration - gcInterval time.Duration - gcStarted bool - mu sync.Mutex -} - -func NewCacheDefault() *Cache { - return &Cache{ - items: map[string]item{}, - defaultExpiration: defaultExpiration, - gcInterval: defaultGCInterval, - } -} - -func NewCache(expiration time.Duration, gcInterval time.Duration) *Cache { - cache := Cache{ - items: map[string]item{}, - defaultExpiration: expiration, - gcInterval: gcInterval, - } - if cache.defaultExpiration <= 0 { - cache.defaultExpiration = defaultExpiration - } - if cache.gcInterval <= time.Second { - cache.gcInterval = defaultGCInterval - } - return &cache -} - -func (c *Cache) Run(stopCh <-chan struct{}) error { - defer runtime.HandleCrash() - c.gcStarted = true - go wait.Until(func() { - c.gcExpiredCache() - }, c.gcInterval, stopCh) - return nil -} - -func (c *Cache) gcExpiredCache() { - c.mu.Lock() - defer c.mu.Unlock() - gcTime := time.Now() - var gcKeys []string - for key, item := range c.items { - if gcTime.After(item.expirationTime) { - gcKeys = append(gcKeys, key) - } - } - for _, key := range gcKeys { - delete(c.items, key) - } - klog.V(5).Infof("gc resource update executor, current size %v", len(c.items)) -} - -func (c *Cache) Set(key string, value interface{}, expiration time.Duration) error { - return c.set(key, value, expiration) -} - -func (c *Cache) SetDefault(key string, value interface{}) error { - return c.set(key, value, c.defaultExpiration) -} - -func (c *Cache) set(key string, value interface{}, expiration time.Duration) error { - if !c.gcStarted { - return fmt.Errorf("cache GC is not started yet") - } - item := item{ - object: value, - expirationTime: time.Now().Add(expiration), - } - c.mu.Lock() - defer c.mu.Unlock() - c.items[key] = item - return nil -} - -func (c *Cache) Get(key string) (interface{}, bool) { - c.mu.Lock() - defer c.mu.Unlock() - item, ok := c.items[key] - if !ok { - return nil, false - } - if item.expirationTime.Before(time.Now()) { - return nil, false - } - return item.object, true -} diff --git a/pkg/cache/expiration_cache_test.go b/pkg/cache/expiration_cache_test.go deleted file mode 100644 index 984943682..000000000 --- a/pkg/cache/expiration_cache_test.go +++ /dev/null @@ -1,114 +0,0 @@ -/* - Copyright 2022 The Koordinator Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cache - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func Test_Cache_Get(t *testing.T) { - cache := NewCacheDefault() - cache.gcStarted = true - cache.items = map[string]item{ - "keyExpire": {object: "value1", expirationTime: time.Now().Add(-1 * time.Minute)}, - "keyNotExpire": {object: "value2", expirationTime: time.Now().Add(1 * time.Minute)}, - } - value, found := cache.Get("keyExpire") - assert.True(t, !found, "value not found", "keyExpire") - assert.Nil(t, value, "value must be nil", "keyExpire") - - value, found = cache.Get("keyNotExpire") - assert.True(t, found, "value found", "keyNotExpire") - assert.Equal(t, "value2", value, "keyNotExpire") -} - -func Test_Cache_Set(t *testing.T) { - cache := NewCacheDefault() - cache.gcStarted = true - value, found := cache.Get("key") - assert.True(t, !found, "value not found") - assert.Nil(t, value, "value must be nil") - - _ = cache.SetDefault("key", "value") - value, found = cache.Get("key") - assert.True(t, found, "value found", "checkSetDefault") - assert.Equal(t, "value", value, "checkSetDefault") - - _ = cache.Set("key", "value", -1*time.Minute) - value, found = cache.Get("key") - assert.True(t, !found, "value not found", "checkSet") - assert.Nil(t, value, "value must be nil", "checkSet") - -} - -func Test_gcExpiredCache(t *testing.T) { - tests := []struct { - name string - initItems map[string]item - cache *Cache - expectItemsAfterGC map[string]item - }{ - { - name: "test_gcExpiredCache_NewCacheDefault", - initItems: map[string]item{ - "keyNeedExpire": {object: "value1", expirationTime: time.Now().Add(-1 * time.Minute)}, - "keyNotExpire": {object: "value2", expirationTime: time.Now().Add(time.Minute)}, - }, - cache: NewCacheDefault(), - expectItemsAfterGC: map[string]item{ - "keyNotExpire": {object: "value2", expirationTime: time.Now().Add(time.Minute)}, - }, - }, - { - name: "test_gcExpiredCache_NewCache", - initItems: map[string]item{ - "keyNeedExpire": {object: "value1", expirationTime: time.Now().Add(-1 * time.Minute)}, - "keyNotExpire": {object: "value2", expirationTime: time.Now().Add(time.Minute)}, - }, - cache: NewCache(time.Minute, time.Minute), - expectItemsAfterGC: map[string]item{ - "keyNotExpire": {object: "value2", expirationTime: time.Now().Add(time.Minute)}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.cache.items = tt.initItems - tt.cache.gcStarted = true - tt.cache.gcExpiredCache() - got := tt.cache.items - assert.Equal(t, len(tt.expectItemsAfterGC), len(got), "checkLen") - checkValueEqual(t, tt.expectItemsAfterGC, got) - }) - } -} - -func checkValueEqual(t *testing.T, expect, got map[string]item) { - assert.Equal(t, len(expect), len(got), "checkLen") - for key, item := range expect { - gotItem, ok := got[key] - if !ok { - assert.True(t, ok, "checkFound", key) - return - } - assert.Equal(t, item.object, gotItem.object, "checkValue", key) - } -} diff --git a/pkg/features/koordlet_features.go b/pkg/features/koordlet_features.go index 0509b0c3c..89f908b99 100644 --- a/pkg/features/koordlet_features.go +++ b/pkg/features/koordlet_features.go @@ -34,11 +34,14 @@ const ( // BECPUSuppress suppresses for best-effort pod BECPUSuppress featuregate.Feature = "BECPUSuppress" + // BEMemoryEvict evict best-effort pod based on Memory + BEMemoryEvict featuregate.Feature = "BEMemoryEvict" + // CPUBurst set cpu.cfs_burst_us; scale up cpu.cfs_quota_us if pod cpu throttled CPUBurst featuregate.Feature = "CPUBurst" - // BEMemoryEvict evict best-effort pod based on Memory - BEMemoryEvict featuregate.Feature = "BEMemoryEvict" + // RdtResctrl sets intel rdt resctrl for processes belonging to ls or be pods + RdtResctrl featuregate.Feature = "RdtResctrl" // CgroupReconcile reconciles qos config for resources like cpu, memory, disk, etc. CgroupReconcile featuregate.Feature = "CgroupReconcile" @@ -59,6 +62,7 @@ var ( BECPUSuppress: {Default: false, PreRelease: featuregate.Alpha}, BEMemoryEvict: {Default: false, PreRelease: featuregate.Alpha}, CPUBurst: {Default: false, PreRelease: featuregate.Alpha}, + RdtResctrl: {Default: false, PreRelease: featuregate.Alpha}, CgroupReconcile: {Default: false, PreRelease: featuregate.Alpha}, } ) diff --git a/pkg/koordlet/resmanager/cgroup_reconcile_test.go b/pkg/koordlet/resmanager/cgroup_reconcile_test.go index cf4a45796..56b1528f9 100644 --- a/pkg/koordlet/resmanager/cgroup_reconcile_test.go +++ b/pkg/koordlet/resmanager/cgroup_reconcile_test.go @@ -824,7 +824,8 @@ func TestCgroupResourcesReconcile_getMergedPodResourceQoS(t *testing.T) { testingMemoryQoSNoneResourceQoS.MemoryQoS = util.NoneResourceQoSStrategy().BE.MemoryQoS testingMemoryQoSNoneResourceQoS1 := util.DefaultResourceQoSStrategy().BE // qos partially disable testingMemoryQoSNoneResourceQoS1.MemoryQoS = util.NoneResourceQoSStrategy().BE.MemoryQoS - testingMemoryQoSAutoResourceQoS := util.DefaultResourceQoSStrategy().BE + testingMemoryQoSAutoResourceQoS := util.NoneResourceQoSStrategy().BE + testingMemoryQoSAutoResourceQoS.MemoryQoS.MemoryQoS = *util.DefaultMemoryQoS(apiext.QoSBE) testingMemoryQoSAutoResourceQoS1 := util.DefaultResourceQoSStrategy().BE testingMemoryQoSAutoResourceQoS1.MemoryQoS.ThrottlingPercent = pointer.Int64Ptr(90) testingMemoryQoSAutoResourceQoS2 := &slov1alpha1.ResourceQoS{ diff --git a/pkg/koordlet/resmanager/reason.go b/pkg/koordlet/resmanager/reason.go index 3186905b6..3952b3881 100644 --- a/pkg/koordlet/resmanager/reason.go +++ b/pkg/koordlet/resmanager/reason.go @@ -17,10 +17,11 @@ limitations under the License. package resmanager const ( - updateCPU = "UpdateCPU" - updateMemory = "UpdateMemory" - updateSystemConfig = "UpdateSystemConfig" - updateCgroups = "UpdateCgroups" // update cgroups excluding the options already stated above + updateCPU = "UpdateCPU" + updateMemory = "UpdateMemory" + updateCgroups = "UpdateCgroups" // update cgroups excluding the options already stated above + updateResctrlSchemata = "UpdateResctrlSchemata" // update resctrl l3 cat schemata + updateResctrlTasks = "UpdateResctrlTasks" // update resctrl tasks evictPodByNodeMemoryUsage = "EvictPodByNodeMemoryUsage" diff --git a/pkg/koordlet/resmanager/resctrl_reconcile.go b/pkg/koordlet/resmanager/resctrl_reconcile.go new file mode 100644 index 000000000..1dca2128e --- /dev/null +++ b/pkg/koordlet/resmanager/resctrl_reconcile.go @@ -0,0 +1,482 @@ +/* + Copyright 2022 The Koordinator Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package resmanager + +import ( + "fmt" + "math" + "math/bits" + "os" + "sort" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + + "github.com/koordinator-sh/koordinator/apis/extension" + slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1" + "github.com/koordinator-sh/koordinator/pkg/koordlet/metriccache" + "github.com/koordinator-sh/koordinator/pkg/koordlet/statesinformer" + "github.com/koordinator-sh/koordinator/pkg/util" + "github.com/koordinator-sh/koordinator/pkg/util/system" +) + +const ( + // RootResctrlGroup is the name of the root resctrl group + RootResctrlGroup = "" + // LSRResctrlGroup is the name of LSR resctrl group + LSRResctrlGroup = "LSR" + // LSResctrlGroup is the name of LS resctrl group + LSResctrlGroup = "LS" + // BEResctrlGroup is the name of BE resctrl group + BEResctrlGroup = "BE" + // UnknownResctrlGroup is the resctrl group which is unknown to reconcile + UnknownResctrlGroup = "Unknown" + // L3SchemataPrefix is the prefix of l3 cat schemata + L3SchemataPrefix = "L3:" + // MbSchemataPrefix is the prefix of l3 cat schemata + MbSchemataPrefix = "MB:" +) + +var ( + // resctrlGroupList is the list of resctrl groups to be reconcile + resctrlGroupList = []string{LSRResctrlGroup, LSResctrlGroup, BEResctrlGroup} +) + +type ResctrlReconcile struct { + resManager *resmanager + executor *ResourceUpdateExecutor +} + +func NewResctrlReconcile(resManager *resmanager) *ResctrlReconcile { + executor := NewResourceUpdateExecutor("ResctrlExecutor", resManager.config.ReconcileIntervalSeconds*60) + return &ResctrlReconcile{ + resManager: resManager, + executor: executor, + } +} + +func (r *ResctrlReconcile) RunInit(stopCh <-chan struct{}) error { + r.executor.Run(stopCh) + return nil +} + +func getPodResctrlGroup(pod *corev1.Pod) string { + podQoS := extension.GetPodQoSClass(pod) + switch podQoS { + case extension.QoSLSR: + return LSRResctrlGroup + case extension.QoSLS: + return LSResctrlGroup + case extension.QoSBE: + return BEResctrlGroup + } + return UnknownResctrlGroup +} + +func getResourceQoSForResctrlGroup(strategy *slov1alpha1.ResourceQoSStrategy, group string) *slov1alpha1.ResourceQoS { + if strategy == nil { + return nil + } + switch group { + case LSRResctrlGroup: + return strategy.LSR + case LSResctrlGroup: + return strategy.LS + case BEResctrlGroup: + return strategy.BE + } + return nil +} + +func initCatResctrl() error { + // check if the resctrl root and l3_cat feature are enabled correctly + if err := system.CheckAndTryEnableResctrlCat(); err != nil { + klog.Errorf("check resctrl cat failed, err: %s", err) + return err + } + for _, group := range resctrlGroupList { + if err := initCatGroupIfNotExist(group); err != nil { + klog.Errorf("init cat group dir %v failed, error %v", group, err) + } else { + klog.V(5).Infof("create cat dir for group %v successfully", group) + } + } + return nil +} + +func initCatGroupIfNotExist(group string) error { + path := system.GetResctrlGroupRootDirPath(group) + _, err := os.Stat(path) + if err == nil { + return nil + } else if !os.IsNotExist(err) { + return fmt.Errorf("check dir %v for group %s but got unexpected err: %v", path, group, err) + } + err = os.Mkdir(path, 0755) + if err != nil { + return fmt.Errorf("create dir %v failed for group %s, err: %v", path, group, err) + } + return nil +} + +func calculateCatL3MaskValue(cbm uint, startPercent, endPercent int64) (string, error) { + // check if the parsed cbm value is valid, eg. 0xff, 0x1, 0x7ff, ... + // NOTE: (Cache Bit Masks) X86 hardware requires that these masks have all the '1' bits in a contiguous block. + // ref: https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt + // since the input cbm here is the cbm value of the resctrl root, every lower bit is required to be `1` additionally + if bits.OnesCount(cbm+1) != 1 { + return "", fmt.Errorf("illegal cbm %v", cbm) + } + + // check if the startPercent and endPercent are valid + if startPercent < 0 || endPercent > 100 || endPercent <= startPercent { + return "", fmt.Errorf("illegal l3 cat percent: start %v, end %v", startPercent, endPercent) + } + + // calculate a bit mask belonging to interval [startPercent% * ways, endPercent% * ways) + // eg. + // cbm 0x3ff ('b1111111111), start 10%, end 80% + // ways 10, l3Mask 0xfe ('b11111110) + // cbm 0x7ff ('b11111111111), start 10%, end 50% + // ways 11, l3Mask 0x3c ('b111100) + // cbm 0x7ff ('b11111111111), start 0%, end 30% + // ways 11, l3Mask 0xf ('b1111) + ways := float64(bits.Len(cbm)) + startWay := uint64(math.Ceil(ways * float64(startPercent) / 100)) + endWay := uint64(math.Ceil(ways * float64(endPercent) / 100)) + + var l3Mask uint64 = (1 << endWay) - (1 << startWay) + return strconv.FormatUint(l3Mask, 16), nil +} + +func calculateL3SchemataResource(group, schemataDelta string, l3Num int) ResourceUpdater { + schemata := L3SchemataPrefix + // the last ';' will be auto ignored + for i := 0; i < l3Num; i++ { + schemata = schemata + strconv.Itoa(i) + "=" + schemataDelta + ";" + } + // the trailing '\n' is necessary to append + schemata += "\n" + + schemataFile := system.GetResctrlSchemataFilePath(group) + + // write to $schemataFile with valued $schemata + updaterKey := schemataFile + ":" + L3SchemataPrefix + return NewDetailCommonResourceUpdater(updaterKey, schemataFile, schemata, GroupOwnerRef(group), updateResctrlSchemataFunc) +} + +func calculateMbSchemataResource(group, schemataDelta string, l3Num int) ResourceUpdater { + schemata := MbSchemataPrefix + // the last ';' will be auto ignored + for i := 0; i < l3Num; i++ { + schemata = schemata + strconv.Itoa(i) + "=" + schemataDelta + ";" + } + // the trailing '\n' is necessary to append + schemata += "\n" + + schemataFile := system.GetResctrlSchemataFilePath(group) + + // write to $schemataFile with valued $schemata + updaterKey := schemataFile + ":" + MbSchemataPrefix + return NewDetailCommonResourceUpdater(updaterKey, schemataFile, schemata, GroupOwnerRef(group), updateResctrlSchemataFunc) +} + +func calculateMbaPercentForGroup(group string, mbaPercentConfig *int64) string { + if mbaPercentConfig == nil { + klog.Warningf("cat MBA will not change, since MBAPercent is nil for group %v, "+ + "mbaPercentConfig %v", mbaPercentConfig, group) + return "" + } + + if *mbaPercentConfig <= 0 || *mbaPercentConfig > 100 { + klog.Warningf("cat MBA will not change, since MBAPercent is not in [1,100] for group %v, "+ + "MBAPercent %d", group, *mbaPercentConfig) + return "" + } + + if *mbaPercentConfig%10 != 0 { + actualPercent := *mbaPercentConfig/10*10 + 10 + klog.Warningf("cat MBA must multiple of 10, group: %v, mbaPercentConfig is %d, actualMBAPercent will be %d", + group, *mbaPercentConfig, actualPercent) + return strconv.FormatInt(actualPercent, 10) + } + + return strconv.FormatInt(*mbaPercentConfig, 10) +} + +func getPodCgroupNewTaskIds(podMeta *statesinformer.PodMeta, tasksMap map[int]struct{}) []int { + var taskIds []int + + pod := podMeta.Pod + containerMap := make(map[string]*corev1.Container, len(pod.Spec.Containers)) + for i := range pod.Spec.Containers { + container := &pod.Spec.Containers[i] + containerMap[container.Name] = container + } + for _, containerStat := range pod.Status.ContainerStatuses { + // reconcile containers + container, exist := containerMap[containerStat.Name] + if !exist { + klog.Warningf("container %s/%s/%s lost during reconcile resctrl group", pod.Namespace, + pod.Name, containerStat.Name) + continue + } + + ids, err := util.GetContainerCurTasks(podMeta.CgroupDir, &containerStat) + if err != nil { + klog.Warningf("failed to get pod container cgroup task ids for container %s/%s/%s, err: %s", + pod.Namespace, pod.Name, container.Name, err) + continue + } + + // only append the non-mapped ids + if tasksMap == nil { + taskIds = append(taskIds, ids...) + continue + } + for _, id := range ids { + if _, ok := tasksMap[id]; !ok { + taskIds = append(taskIds, id) + } + } + } + + return taskIds +} + +func calculateL3TasksResource(group string, taskIds []int) ResourceUpdater { + // join ids into updater value and make the id updates one by one + tasksPath := system.GetResctrlTasksFilePath(group) + + // use ordered slice + sort.Ints(taskIds) + var builder strings.Builder + for _, id := range taskIds { + builder.WriteString(strconv.Itoa(id)) + builder.WriteByte('\n') + } + + return NewDetailCommonResourceUpdater(tasksPath, tasksPath, builder.String(), GroupOwnerRef(group), updateResctrlTasksFunc) +} + +func (r *ResctrlReconcile) calculateAndApplyCatL3PolicyForGroup(group string, cbm uint, l3Num int, + resourceQoS *slov1alpha1.ResourceQoS) error { + if resourceQoS == nil || resourceQoS.ResctrlQoS == nil || resourceQoS.ResctrlQoS.CATRangeStartPercent == nil || + resourceQoS.ResctrlQoS.CATRangeEndPercent == nil { + klog.Warningf("skipped, since resourceQoS or startPercent or endPercent is nil for group %v, "+ + "resourceQoS %v", resourceQoS, group) + return nil + } + + startPercent, endPercent := *resourceQoS.ResctrlQoS.CATRangeStartPercent, *resourceQoS.ResctrlQoS.CATRangeEndPercent + // calculate policy + l3MaskValue, err := calculateCatL3MaskValue(cbm, startPercent, endPercent) + if err != nil { + klog.Warningf("failed to calculate l3 cat schemata for group %v, err: %v", group, err) + return err + } + + // calculate updating resource + resource := calculateL3SchemataResource(group, l3MaskValue, l3Num) + + // write policy into resctrl files if need update + isUpdated, err := r.executor.UpdateByCache(resource) + if err != nil { + klog.Warningf("failed to write l3 cat policy on schemata for group %s, err: %s", group, err) + return err + } + klog.V(5).Infof("apply l3 cat policy for group %s finished, schemata %v, l3 number %v, isUpdated %v", + group, l3MaskValue, l3Num, isUpdated) + + return nil +} + +func (r *ResctrlReconcile) calculateAndApplyCatMbPolicyForGroup(group string, l3Num int, resourceQoS *slov1alpha1.ResourceQoS) error { + if resourceQoS == nil || resourceQoS.ResctrlQoS == nil { + klog.Warningf("skipped, since resourceQoS or ResctrlQoS is nil for group %v, "+ + "resourceQoS %v", resourceQoS, group) + return nil + } + + memBwPercent := calculateMbaPercentForGroup(group, resourceQoS.ResctrlQoS.MBAPercent) + if memBwPercent == "" { + return nil + } + // calculate updating resource + resource := calculateMbSchemataResource(group, memBwPercent, l3Num) + + // write policy into resctrl files if need update + isUpdated, err := r.executor.UpdateByCache(resource) + if err != nil { + klog.Warningf("failed to write mb cat policy on schemata for group %s, err: %s", group, err) + return err + } + klog.V(5).Infof("apply mb cat policy for group %s finished, schemata %v, l3 number %v, isUpdated %v", + group, memBwPercent, l3Num, isUpdated) + return nil +} + +func (r *ResctrlReconcile) calculateAndApplyCatL3GroupTasks(group string, taskIds []int) error { + resource := calculateL3TasksResource(group, taskIds) + + // write policy into resctrl files + // NOTE: the operation should not be cacheable, since old tid has chance to be reused by a new task and here the + // tasks ids are the realtime diff between cgroup and resctrl + err := r.executor.Update(resource) + if err != nil { + klog.Warningf("failed to write l3 cat policy on tasks for group %s, err: %s", group, err) + return err + } + klog.V(5).Infof("apply l3 cat tasks for group %s finished, len(taskIds) %v", group, len(taskIds)) + + return nil +} + +func (r *ResctrlReconcile) reconcileCatResctrlPolicy(qosStrategy *slov1alpha1.ResourceQoSStrategy) { + // 1. retrieve rdt configs from nodeSLOSpec + // 2.1 get cbm and l3 numbers, which are general for all resctrl groups + // 2.2 calculate applying resctrl policies, like cat policy and so on, with each rdt config + // 3. apply the policies onto resctrl groups + + // read cat l3 cbm + nodeCPUInfo, err := r.resManager.metricCache.GetNodeCPUInfo(&metriccache.QueryParam{}) + if err != nil { + klog.Warningf("failed to get nodeCPUInfo, err: %v", err) + return + } + if nodeCPUInfo == nil { + klog.Warning("failed to get nodeCPUInfo, the value is nil") + return + } + cbmStr := nodeCPUInfo.BasicInfo.CatL3CbmMask + if len(cbmStr) <= 0 { + klog.Warning("failed to get cat l3 cbm, cbm is empty") + return + } + cbmValue, err := strconv.ParseUint(cbmStr, 16, 32) + if err != nil { + klog.Warningf("failed to parse cat l3 cbm %s, err: %v", cbmStr, err) + return + } + cbm := uint(cbmValue) + + // get the number of l3 caches; it is larger than 0 + l3Num := int(nodeCPUInfo.TotalInfo.NumberL3s) + if l3Num <= 0 { + klog.Warningf("failed to get the number of l3 caches, invalid value %v", l3Num) + return + } + + // calculate and apply l3 cat policy for each group + for _, group := range resctrlGroupList { + resQoSStrategy := getResourceQoSForResctrlGroup(qosStrategy, group) + err = r.calculateAndApplyCatL3PolicyForGroup(group, cbm, l3Num, resQoSStrategy) + if err != nil { + klog.Warningf("failed to apply l3 cat policy for group %v, err: %v", group, err) + } + err = r.calculateAndApplyCatMbPolicyForGroup(group, l3Num, resQoSStrategy) + if err != nil { + klog.Warningf("failed to apply cat MB policy for group %v, err: %v", group, err) + } + } +} + +func (r *ResctrlReconcile) reconcileResctrlGroups(qosStrategy *slov1alpha1.ResourceQoSStrategy) { + // 1. retrieve task ids for each slo by reading cgroup task file of every pod container + // 2. add the related task ids in resctrl groups + + // NOTE: pid_max can be found in `/proc/sys/kernel/pid_max` on linux. + // the maximum pid on 32-bit/64-bit platforms is always less than 4194304, so the int type is bigger enough. + // here we only append the task ids which only appear in cgroup but not in resctrl to reduce resctrl writes + var err error + + curTaskMaps := map[string]map[int]struct{}{} + for _, group := range resctrlGroupList { + curTaskMaps[group], err = system.ReadResctrlTasksMap(group) + if err != nil { + klog.Warningf("failed to read Cat L3 tasks for resctrl group %s, err: %s", group, err) + } + } + + taskIds := map[string][]int{} + podsMeta := r.resManager.statesInformer.GetAllPods() + for _, podMeta := range podsMeta { + pod := podMeta.Pod + // only Running and Pending pods are considered + if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodPending { + continue + } + + // only extension-QoS-specified pod are considered + podQoSCfg := getPodResourceQoSByQoSClass(pod, qosStrategy, r.resManager.config) + if podQoSCfg.ResctrlQoS.Enable == nil || !(*podQoSCfg.ResctrlQoS.Enable) { + klog.V(5).Infof("pod %v with qos %v disabled resctrl", util.GetPodKey(pod), extension.GetPodQoSClass(pod)) + continue + } + + // TODO https://github.com/koordinator-sh/koordinator/pull/94#discussion_r858779795 + if group := getPodResctrlGroup(pod); group != UnknownResctrlGroup { + ids := getPodCgroupNewTaskIds(podMeta, curTaskMaps[group]) + taskIds[group] = append(taskIds[group], ids...) + } + } + + // write Cat L3 tasks for each resctrl group + for _, group := range resctrlGroupList { + err = r.calculateAndApplyCatL3GroupTasks(group, taskIds[group]) + if err != nil { + klog.Warningf("failed to apply l3 cat tasks for group %s, err %s", group, err) + } + } +} + +func (r *ResctrlReconcile) reconcile() { + // Step 0. create and init them if resctrl groups do not exist + // Step 1. reconcile rdt policies against `schemata` file + // Step 2. reconcile resctrl groups against `tasks` file + + // Step 0. + if r.resManager == nil || r.executor == nil { + klog.Warning("ResctrlReconcile failed, uninitialized") + return + } + nodeSLO := r.resManager.getNodeSLOCopy() + if nodeSLO == nil || nodeSLO.Spec.ResourceQoSStrategy == nil { + // do nothing if nodeSLO == nil || nodeSLO.spec.ResourceStrategy == nil + klog.Warningf("nodeSLO is nil %v, or nodeSLO.Spec.ResourceQoSStrategy is nil %v", + nodeSLO == nil, nodeSLO.Spec.ResourceQoSStrategy == nil) + return + } + + // skip if host not support resctrl + if support, err := system.IsSupportResctrl(); err != nil { + klog.Warningf("check support resctrl failed, err: %s", err) + return + } else if !support { + klog.V(5).Infof("ResctrlReconcile skipped, cpu not support CAT/MBA") + return + } + + if err := initCatResctrl(); err != nil { + klog.Warningf("ResctrlReconcile failed, cannot initialize cat resctrl group, err: %s", err) + return + } + r.reconcileCatResctrlPolicy(nodeSLO.Spec.ResourceQoSStrategy) + r.reconcileResctrlGroups(nodeSLO.Spec.ResourceQoSStrategy) +} diff --git a/pkg/koordlet/resmanager/resctrl_reconcile_test.go b/pkg/koordlet/resmanager/resctrl_reconcile_test.go new file mode 100644 index 000000000..248d712cc --- /dev/null +++ b/pkg/koordlet/resmanager/resctrl_reconcile_test.go @@ -0,0 +1,1313 @@ +/* + Copyright 2022 The Koordinator Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package resmanager + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + "github.com/koordinator-sh/koordinator/apis/extension" + slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1" + "github.com/koordinator-sh/koordinator/pkg/koordlet/metriccache" + mock_metriccache "github.com/koordinator-sh/koordinator/pkg/koordlet/metriccache/mockmetriccache" + "github.com/koordinator-sh/koordinator/pkg/koordlet/statesinformer" + mock_statesinformer "github.com/koordinator-sh/koordinator/pkg/koordlet/statesinformer/mockstatesinformer" + "github.com/koordinator-sh/koordinator/pkg/util" + "github.com/koordinator-sh/koordinator/pkg/util/system" +) + +func testingPrepareResctrlL3CatPath(t *testing.T, cbmStr, rootSchemataStr string) { + resctrlDir := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir) + l3CatDir := filepath.Join(resctrlDir, system.RdtInfoDir, system.L3CatDir) + err := os.MkdirAll(l3CatDir, 0700) + assert.NoError(t, err) + + cbmPath := filepath.Join(l3CatDir, system.CbmMaskFileName) + err = ioutil.WriteFile(cbmPath, []byte(cbmStr), 0666) + assert.NoError(t, err) + + schemataPath := filepath.Join(resctrlDir, system.SchemataFileName) + err = ioutil.WriteFile(schemataPath, []byte(rootSchemataStr), 0666) + assert.NoError(t, err) +} + +func testingPrepareResctrlL3CatGroups(t *testing.T, cbmStr, rootSchemataStr string) { + testingPrepareResctrlL3CatPath(t, cbmStr, rootSchemataStr) + resctrlDir := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir) + + beSchemataData := []byte(" L3:0=f;1=f\n MB:0=100;1=100") + beSchemataDir := filepath.Join(resctrlDir, BEResctrlGroup) + err := os.MkdirAll(beSchemataDir, 0700) + assert.NoError(t, err) + beSchemataPath := filepath.Join(beSchemataDir, system.SchemataFileName) + err = ioutil.WriteFile(beSchemataPath, beSchemataData, 0666) + assert.NoError(t, err) + beTasksPath := filepath.Join(beSchemataDir, system.ResctrlTaskFileName) + err = ioutil.WriteFile(beTasksPath, []byte{}, 0666) + assert.NoError(t, err) + + lsSchemataData := []byte(" L3:0=ff;1=ff\n MB:0=100;1=100") + lsSchemataDir := filepath.Join(resctrlDir, LSResctrlGroup) + err = os.MkdirAll(lsSchemataDir, 0700) + assert.NoError(t, err) + lsSchemataPath := filepath.Join(lsSchemataDir, system.SchemataFileName) + err = ioutil.WriteFile(lsSchemataPath, lsSchemataData, 0666) + assert.NoError(t, err) + lsTasksPath := filepath.Join(lsSchemataDir, system.ResctrlTaskFileName) + err = ioutil.WriteFile(lsTasksPath, []byte{}, 0666) + assert.NoError(t, err) + + lsrSchemataData := []byte(" L3:0=ff;1=ff\n MB:0=100;1=100") + lsrSchemataDir := filepath.Join(resctrlDir, LSRResctrlGroup) + err = os.MkdirAll(lsrSchemataDir, 0700) + assert.NoError(t, err) + lsrSchemataPath := filepath.Join(lsrSchemataDir, system.SchemataFileName) + err = ioutil.WriteFile(lsrSchemataPath, lsrSchemataData, 0666) + assert.NoError(t, err) + lsrTasksPath := filepath.Join(lsrSchemataDir, system.ResctrlTaskFileName) + err = ioutil.WriteFile(lsrTasksPath, []byte{}, 0666) + assert.NoError(t, err) +} + +func testingPrepareContainerCgroupCPUTasks(t *testing.T, containerParentPath, tasksStr string) { + containerCgroupDir := filepath.Join(system.Conf.CgroupRootDir, system.CgroupCPUDir, containerParentPath) + err := os.MkdirAll(containerCgroupDir, 0700) + assert.NoError(t, err) + + containerTasksPath := filepath.Join(containerCgroupDir, system.CPUTaskFileName) + err = ioutil.WriteFile(containerTasksPath, []byte(tasksStr), 0666) + assert.NoError(t, err) +} + +func Test_calculateCatL3Schemata(t *testing.T) { + type args struct { + cbm uint + startPercent int64 + endPercent int64 + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "do not panic but throw an error for empty input", + want: "", + wantErr: true, + }, + { + name: "cbm value is invalid", + args: args{ + cbm: 0x101, + startPercent: 0, + endPercent: 100, + }, + want: "", + wantErr: true, + }, + { + name: "cbm value is invalid 1", + args: args{ + cbm: 4, + startPercent: 0, + endPercent: 100, + }, + want: "", + wantErr: true, + }, + { + name: "percent value is invalid", + args: args{ + cbm: 0xff, + startPercent: -10, + endPercent: 100, + }, + want: "", + wantErr: true, + }, + { + name: "percent value is invalid 1", + args: args{ + cbm: 0xff, + startPercent: 30, + endPercent: 30, + }, + want: "", + wantErr: true, + }, + { + name: "calculate l3 schemata correctly", + args: args{ + cbm: 0xff, + startPercent: 0, + endPercent: 100, + }, + want: "ff", + wantErr: false, + }, + { + name: "calculate l3 schemata correctly 1", + args: args{ + cbm: 0x3ff, + startPercent: 10, + endPercent: 80, + }, + want: "fe", + wantErr: false, + }, + { + name: "calculate l3 schemata correctly 2", + args: args{ + cbm: 0x7ff, + startPercent: 10, + endPercent: 50, + }, + want: "3c", + wantErr: false, + }, + { + name: "calculate l3 schemata correctly 3", + args: args{ + cbm: 0x3ff, + startPercent: 0, + endPercent: 30, + }, + want: "7", + wantErr: false, + }, + { + name: "calculate l3 schemata correctly 4", + args: args{ + cbm: 0x3ff, + startPercent: 10, + endPercent: 85, + }, + want: "1fe", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := calculateCatL3MaskValue(tt.args.cbm, tt.args.startPercent, tt.args.endPercent) + assert.Equal(t, tt.wantErr, err != nil) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_initCatResctrl(t *testing.T) { + t.Run("test", func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "initCatResctrl" + helper.MkDirAll(sysFSRootDirName) + + system.Conf.SysFSRootDir = path.Join(helper.TempDir, sysFSRootDirName) + + testingPrepareResctrlL3CatGroups(t, "ff", "L3:0=ff") + + resctrlDirPath := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir) + _, err := os.Stat(resctrlDirPath) + assert.NoError(t, err) + + err = initCatResctrl() + // skip init if resctrl group path exists + assert.NoError(t, err) + + err = os.RemoveAll(system.Conf.SysFSRootDir) + assert.NoError(t, err) + helper.MkDirAll(sysFSRootDirName) + + testingPrepareResctrlL3CatPath(t, "ff", "L3:0=ff") + + // do not panic but create resctrl group if the path does not exist + err = initCatResctrl() + assert.NoError(t, err) + + resctrlDirPath = filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir) + beResctrlGroupPath := filepath.Join(resctrlDirPath, BEResctrlGroup) + lsResctrlGroupPath := filepath.Join(resctrlDirPath, LSResctrlGroup) + _, err = os.Stat(beResctrlGroupPath) + assert.NoError(t, err) + _, err = os.Stat(lsResctrlGroupPath) + assert.NoError(t, err) + + // path is invalid, do not panic but log the error + system.Conf.SysFSRootDir = "invalidPath" + err = initCatResctrl() + assert.Error(t, err) + }) +} + +func Test_getPodCgroupNewTaskIds(t *testing.T) { + type args struct { + podMeta *statesinformer.PodMeta + tasksMap map[int]struct{} + } + type fields struct { + containerParentDir string + containerTasksStr string + invalidPath bool + } + tests := []struct { + name string + args args + fields fields + want []int + }{ + { + name: "do nothing for empty pod", + args: args{ + podMeta: &statesinformer.PodMeta{Pod: &corev1.Pod{}}, + }, + want: nil, + }, + { + name: "successfully get task ids for the pod", + fields: fields{ + containerParentDir: "kubepods.slice/p0/cri-containerd-c0.scope", + containerTasksStr: "122450\n122454\n123111\n128912", + }, + args: args{ + podMeta: &statesinformer.PodMeta{ + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod0", + UID: "p0", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container0", + }, + }, + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "container0", + ContainerID: "containerd://c0", + }, + }, + }, + }, + CgroupDir: "p0", + }, + tasksMap: map[int]struct{}{ + 122450: {}, + }, + }, + want: []int{122454, 123111, 128912}, + }, + { + name: "return empty for invalid path", + fields: fields{ + containerParentDir: "p0/cri-containerd-c0.scope", + containerTasksStr: "122454\n123111\n128912", + invalidPath: true, + }, + args: args{ + podMeta: &statesinformer.PodMeta{ + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod0", + UID: "p0", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container0", + }, + }, + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "container0", + ContainerID: "containerd://c0", + }, + }, + }, + }, + CgroupDir: "p0", + }, + tasksMap: map[int]struct{}{ + 122450: {}, + }, + }, + want: nil, + }, + { + name: "missing container's status", + fields: fields{ + containerParentDir: "p0/cri-containerd-c0.scope", + containerTasksStr: "122454\n123111\n128912", + invalidPath: true, + }, + args: args{ + podMeta: &statesinformer.PodMeta{ + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod0", + UID: "p0", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container0", + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{}, + }, + }, + CgroupDir: "p0", + }, + tasksMap: map[int]struct{}{ + 122450: {}, + }, + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + testingPrepareContainerCgroupCPUTasks(t, + tt.fields.containerParentDir, tt.fields.containerTasksStr) + + system.CommonRootDir = "" + if tt.fields.invalidPath { + system.Conf.CgroupRootDir = "invalidPath" + } + + got := getPodCgroupNewTaskIds(tt.args.podMeta, tt.args.tasksMap) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestResctrlReconcile_calculateAndApplyCatL3PolicyForGroup(t *testing.T) { + type args struct { + group string + cbm uint + l3Num int + qosStrategy *slov1alpha1.ResourceQoSStrategy + } + type field struct { + invalidPath bool + noUpdate bool + } + tests := []struct { + name string + field field + args args + want string + wantErr bool + }{ + { + name: "warning for empty input", + want: "", + wantErr: false, + }, + { + name: "throw an error for write on invalid path", + args: args{ + group: LSResctrlGroup, + cbm: 0xf, + l3Num: 2, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + }, + }, + }, + }, + }, + field: field{invalidPath: true}, + want: " L3:0=ff;1=ff\n MB:0=100;1=100", + wantErr: true, + }, + { + name: "warning to empty policy", + args: args{ + group: LSResctrlGroup, + cbm: 0xf, + l3Num: 2, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + BE: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + }, + }, + }, + }, + }, + want: " L3:0=ff;1=ff\n MB:0=100;1=100", + wantErr: false, + }, + { + name: "throw an error for calculating schemata failed", + args: args{ + group: LSResctrlGroup, + cbm: 0x4, + l3Num: 2, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + }, + }, + }, + }, + }, + want: " L3:0=ff;1=ff\n MB:0=100;1=100", + wantErr: true, + }, + { + name: "apply policy correctly", + args: args{ + group: LSResctrlGroup, + cbm: 0xf, + l3Num: 2, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + }, + }, + }, + }, + }, + want: "L3:0=f;1=f;\n", + wantErr: false, + }, + { + name: "apply policy correctly 1", + args: args{ + group: LSResctrlGroup, + cbm: 0x7ff, + l3Num: 1, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(10), + CATRangeEndPercent: pointer.Int64Ptr(50), + }, + }, + }, + BE: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + }, + }, + }, + }, + }, + want: "L3:0=3c;\n", + wantErr: false, + }, + { + name: "apply policy correctly 2", + args: args{ + group: LSRResctrlGroup, + cbm: 0x7ff, + l3Num: 1, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + LSR: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(10), + CATRangeEndPercent: pointer.Int64Ptr(50), + }, + }, + }, + BE: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + }, + }, + }, + }, + }, + want: "L3:0=3c;\n", + wantErr: false, + }, + { + name: "calculate the policy but no need to update", + args: args{ + group: BEResctrlGroup, + cbm: 0x7ff, + l3Num: 1, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + }, + }, + }, + BE: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(10), + CATRangeEndPercent: pointer.Int64Ptr(50), + }, + }, + }, + }, + }, + field: field{noUpdate: true}, + want: "L3:0=3c;\n", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "calculateAndApplyCatL3PolicyForGroup" + helper.MkDirAll(sysFSRootDirName) + + system.Conf.SysFSRootDir = path.Join(helper.TempDir, sysFSRootDirName) + validSysFSRootDir := system.Conf.SysFSRootDir + system.CommonRootDir = "" + + testingPrepareResctrlL3CatGroups(t, "ff", "") + + r := ResctrlReconcile{ + executor: NewResourceUpdateExecutor("ResctrlExecutor", 60), + } + stop := make(chan struct{}) + r.RunInit(stop) + defer func() { stop <- struct{}{} }() + + if tt.field.invalidPath { + system.Conf.SysFSRootDir = "invalidPath" + } + if tt.field.noUpdate { + // prepare fake record + schemataFilePath := system.GetResctrlSchemataFilePath(tt.args.group) + updaterKey := schemataFilePath + ":" + L3SchemataPrefix + fakeResource := NewDetailCommonResourceUpdater(updaterKey, schemataFilePath, + tt.want, GroupOwnerRef(tt.args.group), updateResctrlSchemataFunc) + isUpdate := r.executor.UpdateBatchByCache(fakeResource) + assert.True(t, isUpdate) + } + + // execute function + err := r.calculateAndApplyCatL3PolicyForGroup(tt.args.group, tt.args.cbm, tt.args.l3Num, + getResourceQoSForResctrlGroup(tt.args.qosStrategy, tt.args.group)) + assert.Equal(t, tt.wantErr, err != nil) + + schemataPath := filepath.Join(validSysFSRootDir, system.ResctrlDir, tt.args.group, system.SchemataFileName) + got, _ := ioutil.ReadFile(schemataPath) + assert.Equal(t, tt.want, string(got)) + }) + } +} + +func TestResctrlReconcile_calculateAndApplyCatMbPolicyForGroup(t *testing.T) { + type args struct { + group string + l3Num int + qosStrategy *slov1alpha1.ResourceQoSStrategy + } + type field struct { + invalidPath bool + noUpdate bool + } + tests := []struct { + name string + field field + args args + want string + wantErr bool + }{ + { + name: "warning for empty input", + want: "", + wantErr: false, + }, + { + name: "throw an error for write on invalid path", + args: args{ + group: LSResctrlGroup, + l3Num: 2, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + MBAPercent: pointer.Int64Ptr(90), + }, + }, + }, + }, + }, + field: field{invalidPath: true}, + want: " L3:0=ff;1=ff\n MB:0=100;1=100", + wantErr: true, + }, + { + name: "warning to empty policy", + args: args{ + group: LSResctrlGroup, + l3Num: 2, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + BE: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + MBAPercent: pointer.Int64Ptr(90), + }, + }, + }, + }, + }, + want: " L3:0=ff;1=ff\n MB:0=100;1=100", + wantErr: false, + }, + { + name: "apply policy correctly", + args: args{ + group: LSResctrlGroup, + l3Num: 2, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + MBAPercent: pointer.Int64Ptr(90), + }, + }, + }, + }, + }, + want: "MB:0=90;1=90;\n", + wantErr: false, + }, + { + name: "calculate the policy but no need to update", + args: args{ + group: BEResctrlGroup, + l3Num: 2, + qosStrategy: &slov1alpha1.ResourceQoSStrategy{ + LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + MBAPercent: pointer.Int64Ptr(100), + }, + }, + }, + BE: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + MBAPercent: pointer.Int64Ptr(90), + }, + }, + }, + }, + }, + field: field{noUpdate: true}, + want: "MB:0=90;1=90;\n", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "calculateAndApplyCatMbPolicyForGroup" + helper.MkDirAll(sysFSRootDirName) + + system.Conf.SysFSRootDir = path.Join(helper.TempDir, sysFSRootDirName) + validSysFSRootDir := system.Conf.SysFSRootDir + system.CommonRootDir = "" + + testingPrepareResctrlL3CatGroups(t, "ff", "") + r := ResctrlReconcile{ + executor: NewResourceUpdateExecutor("ResctrlExecutor", 60), + } + stop := make(chan struct{}) + r.RunInit(stop) + defer func() { stop <- struct{}{} }() + + if tt.field.invalidPath { + system.Conf.SysFSRootDir = "invalidPath" + } + if tt.field.noUpdate { + // prepare fake record + schemataFilePath := system.GetResctrlSchemataFilePath(tt.args.group) + updaterKey := schemataFilePath + ":" + MbSchemataPrefix + fakeResource := NewDetailCommonResourceUpdater(updaterKey, schemataFilePath, + tt.want, GroupOwnerRef(tt.args.group), updateResctrlSchemataFunc) + isUpdate := r.executor.UpdateBatchByCache(fakeResource) + assert.True(t, isUpdate) + } + + // execute function + err := r.calculateAndApplyCatMbPolicyForGroup(tt.args.group, tt.args.l3Num, + getResourceQoSForResctrlGroup(tt.args.qosStrategy, tt.args.group)) + assert.Equal(t, tt.wantErr, err != nil) + + schemataPath := filepath.Join(validSysFSRootDir, system.ResctrlDir, tt.args.group, system.SchemataFileName) + got, _ := ioutil.ReadFile(schemataPath) + assert.Equal(t, tt.want, string(got)) + }) + } +} + +func TestResctrlReconcile_calculateAndApplyCatL3GroupTasks(t *testing.T) { + type args struct { + group string + taskIds []int + } + type fields struct { + invalidPath bool + } + tests := []struct { + name string + args args + fields fields + want string + wantErr bool + }{ + { + name: "write nothing", + args: args{group: LSResctrlGroup}, + want: "", + wantErr: false, + }, + { + name: "abort writing for invalid path", + args: args{group: LSResctrlGroup}, + fields: fields{ + invalidPath: true, + }, + want: "", + wantErr: true, + }, + { + name: "write successfully", + args: args{ + group: BEResctrlGroup, + taskIds: []int{0, 1, 2, 5, 7, 9}, + }, + want: "012579", // the real content of resctrl tasks file would be "0\n\1\n2\n..." + wantErr: false, + }, + { + name: "write successfully 1", + args: args{ + group: BEResctrlGroup, + taskIds: []int{0, 1, 2, 4, 5, 6}, + }, + want: "012456", // the real content of resctrl tasks file would be "0\n\1\n2\n..." + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "writeCatL3GroupTasks" + helper.MkDirAll(sysFSRootDirName) + + system.Conf.SysFSRootDir = path.Join(helper.TempDir, sysFSRootDirName) + validSysFSRootDir := system.Conf.SysFSRootDir + + testingPrepareResctrlL3CatGroups(t, "", "") + + if tt.fields.invalidPath { + system.Conf.SysFSRootDir = "invalidPath" + } + r := ResctrlReconcile{ + executor: NewResourceUpdateExecutor("ResctrlExecutor", 60), + } + stop := make(chan struct{}) + r.RunInit(stop) + defer func() { stop <- struct{}{} }() + + err := r.calculateAndApplyCatL3GroupTasks(tt.args.group, tt.args.taskIds) + assert.Equal(t, tt.wantErr, err != nil) + + out, err := ioutil.ReadFile(filepath.Join(validSysFSRootDir, system.ResctrlDir, tt.args.group, + system.CPUTaskFileName)) + assert.NoError(t, err) + assert.Equal(t, tt.want, string(out)) + }) + } +} + +func TestResctrlReconcile_reconcileCatResctrlPolicy(t *testing.T) { + t.Run("test", func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "reconcileCatResctrlPolicy" + helper.MkDirAll(sysFSRootDirName) + + system.Conf.SysFSRootDir = path.Join(helper.TempDir, sysFSRootDirName) + validSysFSRootDir := system.Conf.SysFSRootDir + system.CommonRootDir = "" + + testingPrepareResctrlL3CatGroups(t, "7ff", "L3:0=7ff;1=7ff\n") + + resctrlDirPath := filepath.Join(validSysFSRootDir, system.ResctrlDir) + _, err := os.Stat(resctrlDirPath) + assert.NoError(t, err) + + nodeSLO := &slov1alpha1.NodeSLO{ + Spec: slov1alpha1.NodeSLOSpec{ + ResourceQoSStrategy: &slov1alpha1.ResourceQoSStrategy{ + LSR: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + }, + }, + }, + LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + MBAPercent: pointer.Int64Ptr(90), + }, + }, + }, + BE: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(30), + }, + }, + }, + }, + }, + } + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + metricCache := mock_metriccache.NewMockMetricCache(ctrl) + metricCache.EXPECT().GetNodeCPUInfo(&metriccache.QueryParam{}).Return(&metriccache.NodeCPUInfo{ + BasicInfo: util.CPUBasicInfo{CatL3CbmMask: "7ff"}, + TotalInfo: util.CPUTotalInfo{NumberL3s: 2}, + }, nil).Times(3) + rm := &resmanager{metricCache: metricCache} + r := ResctrlReconcile{ + resManager: rm, + executor: NewResourceUpdateExecutor("ResctrlReconcile", 60), + } + stop := make(chan struct{}) + r.RunInit(stop) + defer func() { stop <- struct{}{} }() + + // reconcile and check if the result is correct + r.reconcileCatResctrlPolicy(nodeSLO.Spec.ResourceQoSStrategy) + + beSchemataPath := filepath.Join(resctrlDirPath, BEResctrlGroup, system.SchemataFileName) + expectBESchemataStr := "L3:0=f;1=f;\n" + got, _ := ioutil.ReadFile(beSchemataPath) + assert.Equal(t, expectBESchemataStr, string(got)) + + lsSchemataPath := filepath.Join(resctrlDirPath, LSResctrlGroup, system.SchemataFileName) + expectLSSchemataStr := "MB:0=90;1=90;\n" + got, _ = ioutil.ReadFile(lsSchemataPath) + assert.Equal(t, expectLSSchemataStr, string(got)) + + // log error for invalid be resctrl path + err = os.RemoveAll(filepath.Join(resctrlDirPath, BEResctrlGroup)) + assert.NoError(t, err) + r.reconcileCatResctrlPolicy(nodeSLO.Spec.ResourceQoSStrategy) + + // log error for invalid root resctrl path + system.Conf.SysFSRootDir = "invalidPath" + r.reconcileCatResctrlPolicy(nodeSLO.Spec.ResourceQoSStrategy) + system.Conf.SysFSRootDir = validSysFSRootDir + + // log error for invalid l3 number + metricCache.EXPECT().GetNodeCPUInfo(&metriccache.QueryParam{}).Return(&metriccache.NodeCPUInfo{ + BasicInfo: util.CPUBasicInfo{CatL3CbmMask: "7ff"}, + TotalInfo: util.CPUTotalInfo{NumberL3s: -1}, + }, nil).Times(1) + r.reconcileCatResctrlPolicy(nodeSLO.Spec.ResourceQoSStrategy) + + // log error for invalid l3 cbm + metricCache.EXPECT().GetNodeCPUInfo(&metriccache.QueryParam{}).Return(&metriccache.NodeCPUInfo{ + BasicInfo: util.CPUBasicInfo{CatL3CbmMask: "invalid"}, + TotalInfo: util.CPUTotalInfo{NumberL3s: 2}, + }, nil).Times(1) + r.reconcileCatResctrlPolicy(nodeSLO.Spec.ResourceQoSStrategy) + metricCache.EXPECT().GetNodeCPUInfo(&metriccache.QueryParam{}).Return(&metriccache.NodeCPUInfo{ + BasicInfo: util.CPUBasicInfo{CatL3CbmMask: ""}, + TotalInfo: util.CPUTotalInfo{NumberL3s: 2}, + }, nil).Times(1) + r.reconcileCatResctrlPolicy(nodeSLO.Spec.ResourceQoSStrategy) + + // log error for invalid nodeCPUInfo + metricCache.EXPECT().GetNodeCPUInfo(&metriccache.QueryParam{}).Return(nil, nil) + r.reconcileCatResctrlPolicy(nodeSLO.Spec.ResourceQoSStrategy) + + // log error for get nodeCPUInfo failed + metricCache.EXPECT().GetNodeCPUInfo(&metriccache.QueryParam{}).Return(nil, fmt.Errorf("error")) + r.reconcileCatResctrlPolicy(nodeSLO.Spec.ResourceQoSStrategy) + }) +} + +func TestResctrlReconcile_reconcileResctrlGroups(t *testing.T) { + // preparing + wantResctrlTaskStr := "122450122454123111128912" + testingContainerParentDir := "kubepods.slice/p0/cri-containerd-c0.scope" + testingContainerTasksStr := "122450\n122454\n123111\n128912" + testingBEResctrlTasksStr := "122450" + testingPodMeta := &statesinformer.PodMeta{ + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod0", + UID: "p0", + Labels: map[string]string{ + extension.LabelPodQoS: string(extension.QoSBE), + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container0", + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "container0", + ContainerID: "containerd://c0", + }, + }, + }, + }, + CgroupDir: "p0", + } + testQOSStrategy := util.DefaultResourceQoSStrategy() + testQOSStrategy.BE.ResctrlQoS.Enable = pointer.BoolPtr(true) + + t.Run("test", func(t *testing.T) { + // initialization + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + statesInformer := mock_statesinformer.NewMockStatesInformer(ctrl) + rm := &resmanager{statesInformer: statesInformer} + r := ResctrlReconcile{ + resManager: rm, + executor: NewResourceUpdateExecutor("ResctrlReconcile", 30), + } + stop := make(chan struct{}) + r.RunInit(stop) + defer func() { stop <- struct{}{} }() + + statesInformer.EXPECT().GetAllPods().Return([]*statesinformer.PodMeta{testingPodMeta}).MaxTimes(2) + + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "reconcileResctrlGroups" + helper.MkDirAll(sysFSRootDirName) + + system.Conf.SysFSRootDir = path.Join(helper.TempDir, sysFSRootDirName) + + testingPrepareResctrlL3CatGroups(t, "", "") + testingPrepareContainerCgroupCPUTasks(t, testingContainerParentDir, testingContainerTasksStr) + + // run reconcileResctrlGroups for BE tasks not exist + r.reconcileResctrlGroups(testQOSStrategy) + + // check if the reconciliation is a success + out, err := ioutil.ReadFile(filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir, BEResctrlGroup, + system.CPUTaskFileName)) + assert.NoError(t, err) + assert.Equal(t, wantResctrlTaskStr, string(out)) + + beTasksPath := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir, BEResctrlGroup, system.ResctrlTaskFileName) + err = ioutil.WriteFile(beTasksPath, []byte(testingBEResctrlTasksStr), 0666) + assert.NoError(t, err) + + // run reconcileResctrlGroups + r.reconcileResctrlGroups(testQOSStrategy) + + // check if the reconciliation is a success + out, err = ioutil.ReadFile(filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir, BEResctrlGroup, + system.CPUTaskFileName)) + assert.NoError(t, err) + assert.Equal(t, wantResctrlTaskStr, string(out)) + }) +} + +func TestResctrlReconcile_reconcile(t *testing.T) { + // preparing + testingContainerParentDir := "kubepods.slice/p0/cri-containerd-c0.scope" + testingContainerTasksStr := "122450\n122454\n123111\n128912" + + testingNodeSLO := &slov1alpha1.NodeSLO{ + Spec: slov1alpha1.NodeSLOSpec{ + ResourceQoSStrategy: &slov1alpha1.ResourceQoSStrategy{ + LSR: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + }, + }, + }, + LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + }, + }, + }, + BE: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + ResctrlQoS: slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(30), + }, + }, + }, + }, + }, + } + testingPodMeta := &statesinformer.PodMeta{ + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod0", + UID: "p0", + Labels: map[string]string{ + extension.LabelPodQoS: string(extension.QoSBE), + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container0", + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "container0", + ContainerID: "containerd://c0", + }, + }, + }, + }, + CgroupDir: "p0", + } + testingNodeCPUInfo := &metriccache.NodeCPUInfo{ + BasicInfo: util.CPUBasicInfo{CatL3CbmMask: "7ff"}, + TotalInfo: util.CPUTotalInfo{NumberL3s: 2}, + } + + t.Run("test not panic", func(t *testing.T) { + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + statesInformer := mock_statesinformer.NewMockStatesInformer(ctrl) + metricCache := mock_metriccache.NewMockMetricCache(ctrl) + statesInformer.EXPECT().GetAllPods().Return([]*statesinformer.PodMeta{testingPodMeta}).AnyTimes() + metricCache.EXPECT().GetNodeCPUInfo(&metriccache.QueryParam{}).Return(testingNodeCPUInfo, nil).AnyTimes() + rm := &resmanager{ + statesInformer: statesInformer, + metricCache: metricCache, + config: NewDefaultConfig(), + nodeSLO: testingNodeSLO, + } + + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "ResctrlReconcile" + helper.MkDirAll(sysFSRootDirName) + system.Conf.SysFSRootDir = path.Join(helper.TempDir, sysFSRootDirName) + validSysFSRootDir := system.Conf.SysFSRootDir + system.CommonRootDir = "" + + testingPrepareContainerCgroupCPUTasks(t, testingContainerParentDir, testingContainerTasksStr) + testingPrepareResctrlL3CatGroups(t, "", "") + + r := NewResctrlReconcile(rm) + stop := make(chan struct{}) + r.RunInit(stop) + defer func() { stop <- struct{}{} }() + + cpuInfoContents := "flags : fpu vme de pse cat_l3 mba" + helper.WriteProcSubFileContents("cpuinfo", cpuInfoContents) + + r.reconcile() + + // test nil resmgr + r.resManager = nil + r.reconcile() + r.resManager = rm + + // test init cat resctrl failed + system.Conf.SysFSRootDir = "invalidPath" + r.reconcile() + system.Conf.SysFSRootDir = validSysFSRootDir + + r.reconcile() + + // test strategy parse error + r.resManager.nodeSLO.Spec.ResourceQoSStrategy = nil + r.reconcile() + + }) +} + +func Test_calculateMbaPercentForGroup(t *testing.T) { + + type args struct { + group string + mbPercent *int64 + } + tests := []struct { + name string + args args + want string + }{ + { + name: "mbPercent not config", + args: args{ + group: "BE", + }, + want: "", + }, + { + name: "mbPercent value is invalid,not between (0,100]", + args: args{ + group: "BE", + mbPercent: pointer.Int64Ptr(0), + }, + want: "", + }, + { + name: "mbPercent value is invalid,not between (0,100]", + args: args{ + group: "BE", + mbPercent: pointer.Int64Ptr(101), + }, + want: "", + }, + { + name: "mbPercent value is invalid, not multiple of 10", + args: args{ + group: "BE", + mbPercent: pointer.Int64Ptr(85), + }, + want: "90", + }, + { + name: "mbPercent value is valid", + args: args{ + group: "BE", + mbPercent: pointer.Int64Ptr(80), + }, + want: "80", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := calculateMbaPercentForGroup(tt.args.group, tt.args.mbPercent) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_calculateL3SchemataResource(t *testing.T) { + t.Run("test", func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "reconcileCatResctrlPolicy" + helper.MkDirAll(sysFSRootDirName) + system.Conf.SysFSRootDir = path.Join(helper.TempDir, sysFSRootDirName) + + testingPrepareResctrlL3CatGroups(t, "7ff", " L3:0=ff;1=ff\n MB:0=100;1=100") + updater := calculateL3SchemataResource(BEResctrlGroup, "3c", 2) + assert.Equal(t, updater.Value(), "L3:0=3c;1=3c;\n") + + }) +} + +func Test_calculateMbSchemataResource(t *testing.T) { + t.Run("test", func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "reconcileCatResctrlPolicy" + helper.MkDirAll(sysFSRootDirName) + system.Conf.SysFSRootDir = path.Join(helper.TempDir, sysFSRootDirName) + + testingPrepareResctrlL3CatGroups(t, "7ff", " L3:0=ff;1=ff\n MB:0=100;1=100") + updater := calculateMbSchemataResource(BEResctrlGroup, "90", 2) + assert.Equal(t, updater.Value(), "MB:0=90;1=90;\n") + + }) +} diff --git a/pkg/koordlet/resmanager/resmanager.go b/pkg/koordlet/resmanager/resmanager.go index abff7b186..8e35e7853 100644 --- a/pkg/koordlet/resmanager/resmanager.go +++ b/pkg/koordlet/resmanager/resmanager.go @@ -38,7 +38,6 @@ import ( "k8s.io/klog/v2" slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1" - expireCache "github.com/koordinator-sh/koordinator/pkg/cache" koordclientset "github.com/koordinator-sh/koordinator/pkg/client/clientset/versioned" slolisterv1alpha1 "github.com/koordinator-sh/koordinator/pkg/client/listers/slo/v1alpha1" "github.com/koordinator-sh/koordinator/pkg/features" @@ -47,6 +46,7 @@ import ( "github.com/koordinator-sh/koordinator/pkg/koordlet/metrics" "github.com/koordinator-sh/koordinator/pkg/koordlet/statesinformer" "github.com/koordinator-sh/koordinator/pkg/runtime" + expireCache "github.com/koordinator-sh/koordinator/pkg/tools/cache" "github.com/koordinator-sh/koordinator/pkg/util" ) @@ -98,8 +98,8 @@ func newNodeSLOInformer(client koordclientset.Interface, nodeName string) cache. ) } -// mergeDefaultNodeSLO merges nodeSLO with default config; ensure use the function with a RWMutex -func (r *resmanager) mergeDefaultNodeSLO(nodeSLO *slov1alpha1.NodeSLO) { +// mergeNodeSLOSpec merges nodeSLO with default config; ensure use the function with a RWMutex +func (r *resmanager) mergeNodeSLOSpec(nodeSLO *slov1alpha1.NodeSLO) { if r.nodeSLO == nil || nodeSLO == nil { klog.Errorf("failed to merge with nil nodeSLO, old: %v, new: %v", r.nodeSLO, nodeSLO) return @@ -131,7 +131,7 @@ func (r *resmanager) createNodeSLO(nodeSLO *slov1alpha1.NodeSLO) { r.nodeSLO.Spec = nodeSLO.Spec // merge nodeSLO spec with the default config - r.mergeDefaultNodeSLO(nodeSLO) + r.mergeNodeSLOSpec(nodeSLO) newNodeSLOStr := util.DumpJSON(r.nodeSLO) klog.Infof("update nodeSLO content: old %s, new %s", oldNodeSLOStr, newNodeSLOStr) @@ -157,7 +157,7 @@ func (r *resmanager) updateNodeSLOSpec(nodeSLO *slov1alpha1.NodeSLO) { r.nodeSLO.Spec = nodeSLO.Spec // merge nodeSLO spec with the default config - r.mergeDefaultNodeSLO(nodeSLO) + r.mergeNodeSLOSpec(nodeSLO) newNodeSLOStr := util.DumpJSON(r.nodeSLO) klog.Infof("update nodeSLO content: old %s, new %s", oldNodeSLOStr, newNodeSLOStr) diff --git a/pkg/koordlet/resmanager/strategy_util.go b/pkg/koordlet/resmanager/strategy_util.go index ddd4cc302..e49bb4757 100644 --- a/pkg/koordlet/resmanager/strategy_util.go +++ b/pkg/koordlet/resmanager/strategy_util.go @@ -55,10 +55,27 @@ func mergeSLOSpecResourceQoSStrategy(defaultSpec, // mergeNoneResourceQoSIfDisabled complete ResourceQoSStrategy according to enable statuses of qos features func mergeNoneResourceQoSIfDisabled(resourceQoS *slov1alpha1.ResourceQoSStrategy) { + mergeNoneResctrlQoSIfDisabled(resourceQoS) mergeNoneMemoryQoSIfDisabled(resourceQoS) klog.V(5).Infof("get merged node ResourceQoS %v", util.DumpJSON(resourceQoS)) } +// mergeNoneResctrlQoSIfDisabled completes node's resctrl qos config according to Enable options in ResctrlQoS +func mergeNoneResctrlQoSIfDisabled(resourceQoS *slov1alpha1.ResourceQoSStrategy) { + if resourceQoS.LSR != nil && resourceQoS.LSR.ResctrlQoS != nil && + resourceQoS.LSR.ResctrlQoS.Enable != nil && !(*resourceQoS.LSR.ResctrlQoS.Enable) { + resourceQoS.LSR.ResctrlQoS.ResctrlQoS = *util.NoneResctrlQoS() + } + if resourceQoS.LS != nil && resourceQoS.LS.ResctrlQoS != nil && + resourceQoS.LS.ResctrlQoS.Enable != nil && !(*resourceQoS.LS.ResctrlQoS.Enable) { + resourceQoS.LS.ResctrlQoS.ResctrlQoS = *util.NoneResctrlQoS() + } + if resourceQoS.BE != nil && resourceQoS.BE.ResctrlQoS != nil && + resourceQoS.BE.ResctrlQoS.Enable != nil && !(*resourceQoS.BE.ResctrlQoS.Enable) { + resourceQoS.BE.ResctrlQoS.ResctrlQoS = *util.NoneResctrlQoS() + } +} + // mergeNoneMemoryQoSIfDisabled completes node's memory qos config according to Enable options in MemoryQoS func mergeNoneMemoryQoSIfDisabled(resourceQoS *slov1alpha1.ResourceQoSStrategy) { // if MemoryQoS.Enable=false, merge with NoneMemoryQoS diff --git a/pkg/koordlet/resmanager/types.go b/pkg/koordlet/resmanager/types.go index 6002d7d71..76d0e1790 100644 --- a/pkg/koordlet/resmanager/types.go +++ b/pkg/koordlet/resmanager/types.go @@ -17,7 +17,9 @@ limitations under the License. package resmanager import ( + "os" "strconv" + "strings" "time" "k8s.io/klog/v2" @@ -113,14 +115,14 @@ func (c *CommonResourceUpdater) Update() error { return c.updateFunc(c) } -func CommonUpdateFunc(resource ResourceUpdater) error { - info := resource.(*CommonResourceUpdater) - audit.V(5).Node().Reason(updateSystemConfig).Message("update %v to %v", info.file, info.value).Do() - return system.CommonFileWriteIfDifferent(info.file, info.Value()) -} - -func NewCommonResourceUpdater(file string, value string) *CommonResourceUpdater { - return &CommonResourceUpdater{key: file, file: file, value: value, updateFunc: CommonUpdateFunc} +func NewDetailCommonResourceUpdater(key, file, value string, owner *OwnerRef, updateFunc UpdateFunc) *CommonResourceUpdater { + return &CommonResourceUpdater{ + owner: owner, + key: key, + file: file, + value: value, + updateFunc: updateFunc, + } } type CgroupResourceUpdater struct { @@ -222,6 +224,67 @@ func ContainerOwnerRef(ns string, name string, container string) *OwnerRef { return &OwnerRef{Type: podType, Namespace: ns, Name: name, Container: container} } +func updateResctrlSchemataFunc(resource ResourceUpdater) error { + // NOTE: currently, only l3 schemata is to update, so do not read or compare before the write + // eg. + // $ cat /sys/fs/resctrl/schemata/BE/schemata + // L3:0=7ff;1=7ff + // MB:0=100;1=100 + // $ echo "L3:0=3f;1=3f" > /sys/fs/resctrl/BE/schemata + // $ cat /sys/fs/resctrl/BE/schemata + // L3:0=03f;1=03f + // MB:0=100;1=100 + info := resource.(*CommonResourceUpdater) + audit.V(5).Group(info.owner.Name).Reason(updateResctrlSchemata).Message("update %v with value %v", + resource.Key(), resource.Value()).Do() + return system.CommonFileWrite(info.file, info.value) +} + +func updateResctrlTasksFunc(resource ResourceUpdater) error { + // NOTE: resctrl/{...}/tasks file is required to appending write a task id once a time, and any duplicate would be + // dropped automatically without an exception + // eg. + // $ echo 123 > /sys/fs/resctrl/BE/tasks + // $ echo 124 > /sys/fs/resctrl/BE/tasks + // $ echo 123 > /sys/fs/resctrl/BE/tasks + // $ echo 122 > /sys/fs/resctrl/BE/tasks + // $ tail -n 3 /sys/fs/resctrl/BE/tasks + // 122 + // 123 + // 124 + info := resource.(*CommonResourceUpdater) + audit.V(5).Group(info.owner.Name).Reason(updateResctrlTasks).Message("update %v with value %v", + resource.Key(), resource.Value()).Do() + + f, err := os.OpenFile(info.file, os.O_RDWR|os.O_APPEND, 0644) + if err != nil { + return err + } + + success, total := 0, 0 + + ids := strings.Split(strings.Trim(info.Value(), "\n"), "\n") + for _, id := range ids { + if strings.TrimSpace(id) == "" { + continue + } + total++ + _, err = f.WriteString(id) + // any thread can exit before the writing + if err == nil { + success++ + continue + } + klog.V(6).Infof("failed to write resctrl task id %v for group %s, err: %s", id, + info.Owner().Name, err) + } + + klog.V(5).Infof("write Cat L3 task ids for group %s finished: %v succeed, %v total", + info.Owner().Name, success, total) + + return f.Close() +} + func mergeFuncUpdateCgroupIfLarger(resource MergeableResourceUpdater) (MergeableResourceUpdater, error) { info := resource.(*CgroupResourceUpdater) diff --git a/pkg/slo-controller/nodeslo/nodeslo_controller.go b/pkg/slo-controller/nodeslo/nodeslo_controller.go index f14d232b9..2211d725b 100644 --- a/pkg/slo-controller/nodeslo/nodeslo_controller.go +++ b/pkg/slo-controller/nodeslo/nodeslo_controller.go @@ -76,6 +76,7 @@ func (r *NodeSLOReconciler) getNodeSLOSpec(node *corev1.Node, oldSpec *slov1alph nodeSLOSpec := &slov1alpha1.NodeSLOSpec{ ResourceUsedThresholdWithBE: util.DefaultResourceThresholdStrategy(), + ResourceQoSStrategy: &slov1alpha1.ResourceQoSStrategy{}, } // TODO: record an event about the failure reason on configmap/crd when failed to load the config diff --git a/pkg/slo-controller/nodeslo/nodeslo_controller_test.go b/pkg/slo-controller/nodeslo/nodeslo_controller_test.go index b3578243a..ac3510e49 100644 --- a/pkg/slo-controller/nodeslo/nodeslo_controller_test.go +++ b/pkg/slo-controller/nodeslo/nodeslo_controller_test.go @@ -83,6 +83,7 @@ func TestNodeSLOReconciler_initNodeSLO(t *testing.T) { fields: fields{client: fake.NewClientBuilder().Build()}, want: &slov1alpha1.NodeSLOSpec{ ResourceUsedThresholdWithBE: util.DefaultResourceThresholdStrategy(), + ResourceQoSStrategy: &slov1alpha1.ResourceQoSStrategy{}, }, wantErr: false, }, diff --git a/pkg/slo-controller/nodeslo/resource_strategy_test.go b/pkg/slo-controller/nodeslo/resource_strategy_test.go index 813cbd6a7..6f091aa58 100644 --- a/pkg/slo-controller/nodeslo/resource_strategy_test.go +++ b/pkg/slo-controller/nodeslo/resource_strategy_test.go @@ -447,3 +447,15 @@ func Test_getResourceQoSSpec(t *testing.T) { }) } } + +func Test_generateResourceQoSStrategeCfg(t *testing.T) { + cfg := config.ResourceQoSCfg{} + cfg.ClusterStrategy = util.DefaultResourceQoSStrategy() + labelSelector := &metav1.LabelSelector{MatchLabels: map[string]string{}} + labelSelector.MatchLabels["machineType"] = "F53" + selectCfg := config.NodeResourceQoSStrategy{NodeSelector: labelSelector, ResourceQoSStrategy: util.DefaultResourceQoSStrategy()} + cfg.NodeStrategies = []config.NodeResourceQoSStrategy{selectCfg} + + cfgJson, _ := json.MarshalIndent(cfg, "", " ") + fmt.Print(string(cfgJson)) +} diff --git a/pkg/tools/cache/expiration_cache.go b/pkg/tools/cache/expiration_cache.go index 67752f71d..b8ec86153 100644 --- a/pkg/tools/cache/expiration_cache.go +++ b/pkg/tools/cache/expiration_cache.go @@ -1,17 +1,17 @@ /* -Copyright 2022 The Koordinator Authors. + Copyright 2022 The Koordinator Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ package cache @@ -21,14 +21,14 @@ import ( "sync" "time" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ) const ( - DefaultExpiration = 2 * time.Minute - DefaultGCInterval = time.Minute + defaultExpiration = 2 * time.Minute + defaultGCInterval = time.Minute ) type item struct { @@ -45,30 +45,53 @@ type Cache struct { } func NewCacheDefault() *Cache { - return &Cache{items: map[string]item{}, defaultExpiration: DefaultExpiration, gcInterval: DefaultGCInterval} + return &Cache{ + items: map[string]item{}, + defaultExpiration: defaultExpiration, + gcInterval: defaultGCInterval, + } } -func NewCache(defaultExpiration time.Duration, gcInterval time.Duration) *Cache { - cache := Cache{items: map[string]item{}, defaultExpiration: defaultExpiration, gcInterval: gcInterval} +func NewCache(expiration time.Duration, gcInterval time.Duration) *Cache { + cache := Cache{ + items: map[string]item{}, + defaultExpiration: expiration, + gcInterval: gcInterval, + } if cache.defaultExpiration <= 0 { - cache.defaultExpiration = DefaultExpiration + cache.defaultExpiration = defaultExpiration } if cache.gcInterval <= time.Second { - cache.gcInterval = DefaultGCInterval + cache.gcInterval = defaultGCInterval } return &cache } func (c *Cache) Run(stopCh <-chan struct{}) error { - defer utilruntime.HandleCrash() + defer runtime.HandleCrash() c.gcStarted = true go wait.Until(func() { c.gcExpiredCache() }, c.gcInterval, stopCh) - return nil } +func (c *Cache) gcExpiredCache() { + c.mu.Lock() + defer c.mu.Unlock() + gcTime := time.Now() + var gcKeys []string + for key, item := range c.items { + if gcTime.After(item.expirationTime) { + gcKeys = append(gcKeys, key) + } + } + for _, key := range gcKeys { + delete(c.items, key) + } + klog.V(5).Infof("gc resource update executor, current size %v", len(c.items)) +} + func (c *Cache) Set(key string, value interface{}, expiration time.Duration) error { return c.set(key, value, expiration) } @@ -79,9 +102,12 @@ func (c *Cache) SetDefault(key string, value interface{}) error { func (c *Cache) set(key string, value interface{}, expiration time.Duration) error { if !c.gcStarted { - return fmt.Errorf("Cache GC need start! ") + return fmt.Errorf("cache GC is not started yet") + } + item := item{ + object: value, + expirationTime: time.Now().Add(expiration), } - item := item{object: value, expirationTime: time.Now().Add(expiration)} c.mu.Lock() defer c.mu.Unlock() c.items[key] = item @@ -100,19 +126,3 @@ func (c *Cache) Get(key string) (interface{}, bool) { } return item.object, true } - -func (c *Cache) gcExpiredCache() { - c.mu.Lock() - defer c.mu.Unlock() - gcTime := time.Now() - var gcKeys []string - for key, item := range c.items { - if gcTime.After(item.expirationTime) { - gcKeys = append(gcKeys, key) - } - } - for _, key := range gcKeys { - delete(c.items, key) - } - klog.V(5).Infof("gc resource update executor, current size %v", len(c.items)) -} diff --git a/pkg/tools/cache/expiration_cache_test.go b/pkg/tools/cache/expiration_cache_test.go index b226c86cf..984943682 100644 --- a/pkg/tools/cache/expiration_cache_test.go +++ b/pkg/tools/cache/expiration_cache_test.go @@ -1,17 +1,17 @@ /* -Copyright 2022 The Koordinator Authors. + Copyright 2022 The Koordinator Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ package cache @@ -46,12 +46,12 @@ func Test_Cache_Set(t *testing.T) { assert.True(t, !found, "value not found") assert.Nil(t, value, "value must be nil") - cache.SetDefault("key", "value") + _ = cache.SetDefault("key", "value") value, found = cache.Get("key") assert.True(t, found, "value found", "checkSetDefault") assert.Equal(t, "value", value, "checkSetDefault") - cache.Set("key", "value", -1*time.Minute) + _ = cache.Set("key", "value", -1*time.Minute) value, found = cache.Get("key") assert.True(t, !found, "value not found", "checkSet") assert.Nil(t, value, "value must be nil", "checkSet") diff --git a/pkg/util/config.go b/pkg/util/config.go index 00af85f48..57a90de8f 100644 --- a/pkg/util/config.go +++ b/pkg/util/config.go @@ -41,6 +41,34 @@ func DefaultResourceThresholdStrategy() *slov1alpha1.ResourceThresholdStrategy { } } +// TODO https://github.com/koordinator-sh/koordinator/pull/94#discussion_r858786733 +func DefaultResctrlQoS(qos apiext.QoSClass) *slov1alpha1.ResctrlQoS { + var resctrlQoS *slov1alpha1.ResctrlQoS + switch qos { + case apiext.QoSLSR: + resctrlQoS = &slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + MBAPercent: pointer.Int64Ptr(100), + } + case apiext.QoSLS: + resctrlQoS = &slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + MBAPercent: pointer.Int64Ptr(100), + } + case apiext.QoSBE: + resctrlQoS = &slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(30), + MBAPercent: pointer.Int64Ptr(100), + } + default: + klog.Infof("resctrl qos has no auto config for qos %s", qos) + } + return resctrlQoS +} + // DefaultMemoryQoS returns the recommended configuration for memory qos strategy. // Please refer to `apis/slo/v1alpha1` for the definition of each field. // In the recommended configuration, all abilities of memcg qos are disable, including `MinLimitPercent`, @@ -99,18 +127,30 @@ func DefaultMemoryQoS(qos apiext.QoSClass) *slov1alpha1.MemoryQoS { func DefaultResourceQoSStrategy() *slov1alpha1.ResourceQoSStrategy { return &slov1alpha1.ResourceQoSStrategy{ LSR: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + Enable: pointer.BoolPtr(false), + ResctrlQoS: *DefaultResctrlQoS(apiext.QoSLSR), + }, MemoryQoS: &slov1alpha1.MemoryQoSCfg{ Enable: pointer.BoolPtr(false), MemoryQoS: *DefaultMemoryQoS(apiext.QoSLSR), }, }, LS: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + Enable: pointer.BoolPtr(false), + ResctrlQoS: *DefaultResctrlQoS(apiext.QoSLS), + }, MemoryQoS: &slov1alpha1.MemoryQoSCfg{ Enable: pointer.BoolPtr(false), MemoryQoS: *DefaultMemoryQoS(apiext.QoSLS), }, }, BE: &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + Enable: pointer.BoolPtr(false), + ResctrlQoS: *DefaultResctrlQoS(apiext.QoSBE), + }, MemoryQoS: &slov1alpha1.MemoryQoSCfg{ Enable: pointer.BoolPtr(false), MemoryQoS: *DefaultMemoryQoS(apiext.QoSBE), @@ -121,6 +161,10 @@ func DefaultResourceQoSStrategy() *slov1alpha1.ResourceQoSStrategy { func NoneResourceQoS(qos apiext.QoSClass) *slov1alpha1.ResourceQoS { return &slov1alpha1.ResourceQoS{ + ResctrlQoS: &slov1alpha1.ResctrlQoSCfg{ + Enable: pointer.BoolPtr(false), + ResctrlQoS: *NoneResctrlQoS(), + }, MemoryQoS: &slov1alpha1.MemoryQoSCfg{ Enable: pointer.BoolPtr(false), MemoryQoS: *NoneMemoryQoS(), @@ -128,6 +172,14 @@ func NoneResourceQoS(qos apiext.QoSClass) *slov1alpha1.ResourceQoS { } } +func NoneResctrlQoS() *slov1alpha1.ResctrlQoS { + return &slov1alpha1.ResctrlQoS{ + CATRangeStartPercent: pointer.Int64Ptr(0), + CATRangeEndPercent: pointer.Int64Ptr(100), + MBAPercent: pointer.Int64Ptr(100), + } +} + // NoneMemoryQoS returns the all-disabled configuration for memory qos strategy. func NoneMemoryQoS() *slov1alpha1.MemoryQoS { return &slov1alpha1.MemoryQoS{ diff --git a/pkg/util/container.go b/pkg/util/container.go index 68217ca6b..e11028d62 100644 --- a/pkg/util/container.go +++ b/pkg/util/container.go @@ -149,6 +149,14 @@ func GetContainerCgroupCFSQuotaPath(podParentDir string, c *corev1.ContainerStat return system.GetCgroupFilePath(containerPath, system.CPUCFSQuota), nil } +func GetContainerCurTasksPath(podParentDir string, c *corev1.ContainerStatus) (string, error) { + containerPath, err := GetContainerCgroupPathWithKube(podParentDir, c) + if err != nil { + return "", err + } + return system.GetCgroupFilePath(containerPath, system.CPUTask), nil +} + func GetContainerCurCPUShare(podParentDir string, c *corev1.ContainerStatus) (int64, error) { cgroupPath, err := GetContainerCgroupCPUSharePath(podParentDir, c) if err != nil { @@ -197,6 +205,14 @@ func GetContainerCurMemLimitBytes(podParentDir string, c *corev1.ContainerStatus return strconv.ParseInt(strings.TrimSpace(string(rawContent)), 10, 64) } +func GetContainerCurTasks(podParentDir string, c *corev1.ContainerStatus) ([]int, error) { + cgroupPath, err := GetContainerCurTasksPath(podParentDir, c) + if err != nil { + return nil, err + } + return system.GetCgroupCurTasks(cgroupPath) +} + func FindContainerIdAndStatusByName(status *corev1.PodStatus, name string) (string, *corev1.ContainerStatus, error) { allStatuses := status.InitContainerStatuses allStatuses = append(allStatuses, status.ContainerStatuses...) diff --git a/pkg/util/container_test.go b/pkg/util/container_test.go index 6fac22e74..5378f5f45 100644 --- a/pkg/util/container_test.go +++ b/pkg/util/container_test.go @@ -18,6 +18,9 @@ package util import ( "fmt" + "io/ioutil" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -172,6 +175,125 @@ func Test_getContainerCgroupPathWithCgroupfsDriver(t *testing.T) { } } +func Test_GetContainerCurTasks(t *testing.T) { + type args struct { + podParentDir string + c *corev1.ContainerStatus + } + type field struct { + containerParentDir string + tasksFileStr string + invalidPath bool + } + tests := []struct { + name string + args args + field field + want []int + wantErr bool + }{ + { + name: "throw an error for empty input", + args: args{ + c: &corev1.ContainerStatus{}, + }, + want: nil, + wantErr: true, + }, + { + name: "parse tasks correctly", + field: field{ + containerParentDir: "pod0/cri-containerd-1.scope", + tasksFileStr: "22264\n22265\n22266\n22267\n29925\n29926\n37587\n41340\n45169\n", + }, + args: args{ + podParentDir: "pod0", + c: &corev1.ContainerStatus{ + ContainerID: "containerd://1", + }, + }, + want: []int{22264, 22265, 22266, 22267, 29925, 29926, 37587, 41340, 45169}, + wantErr: false, + }, + { + name: "throw an error for invalid path", + field: field{ + containerParentDir: "pod0/cri-containerd-1.scope", + tasksFileStr: "22264\n22265\n22266\n22267\n29925\n29926\n37587\n41340\n45169\n", + invalidPath: true, + }, + args: args{ + podParentDir: "pod0", + c: &corev1.ContainerStatus{ + ContainerID: "containerd://1", + }, + }, + want: nil, + wantErr: true, + }, + { + name: "parse error", + field: field{ + containerParentDir: "pod0/cri-containerd-1.scope", + tasksFileStr: "22264\n22265\n22266\n22587\nabs", + }, + args: args{ + podParentDir: "pod0", + c: &corev1.ContainerStatus{ + ContainerID: "containerd://1", + }, + }, + want: nil, + wantErr: true, + }, + { + name: "parse empty", + field: field{ + containerParentDir: "pod0/cri-containerd-1.scope", + tasksFileStr: "", + }, + args: args{ + podParentDir: "pod0", + c: &corev1.ContainerStatus{ + ContainerID: "containerd://1", + }, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var cgroupRootDir string + cgroupRootDir, _ = ioutil.TempDir("", "GetContainerCurTasks") + defer os.RemoveAll(cgroupRootDir) + + dname := filepath.Join(cgroupRootDir, system.CgroupCPUDir, tt.field.containerParentDir) + err := os.MkdirAll(dname, 0700) + assert.NoError(t, err) + fname := filepath.Join(dname, system.CPUTaskFileName) + _ = ioutil.WriteFile(fname, []byte(tt.field.tasksFileStr), 0666) + + system.Conf = &system.Config{ + CgroupRootDir: cgroupRootDir, + } + // reset Formatter after testing + rawParentDir := system.CgroupPathFormatter.ParentDir + system.CgroupPathFormatter.ParentDir = "" + defer func() { + system.CgroupPathFormatter.ParentDir = rawParentDir + }() + if tt.field.invalidPath { + system.Conf.CgroupRootDir = "invalidPath" + } + + got, err := GetContainerCurTasks(tt.args.podParentDir, tt.args.c) + assert.Equal(t, tt.wantErr, err != nil) + assert.Equal(t, tt.want, got) + }) + } +} + func Test_FindContainerIdAndStatusByName(t *testing.T) { type args struct { diff --git a/pkg/util/cpuinfo.go b/pkg/util/cpuinfo.go index b91f62c36..18bae816a 100644 --- a/pkg/util/cpuinfo.go +++ b/pkg/util/cpuinfo.go @@ -35,7 +35,8 @@ const cpuCmdTimeout = 3 * time.Second // CPUBasicInfo describes the cpu basic features and status type CPUBasicInfo struct { - HyperThreadEnabled bool `json:"hyperThreadEnabled,omitempty"` + HyperThreadEnabled bool `json:"hyperThreadEnabled,omitempty"` + CatL3CbmMask string `json:"catL3CbmMask,omitempty"` } // ProcessorInfo describes the processor topology information of a single logic cpu, including the core, socket and numa @@ -111,6 +112,9 @@ func getCPUBasicInfo() (*CPUBasicInfo, error) { if cpuBasicInfo.HyperThreadEnabled, err = getHyperThreadEnabled(); err != nil { klog.V(5).Infof("get hyperthreadEnabled info error: %v", err) } + if cpuBasicInfo.CatL3CbmMask, err = system.ReadCatL3CbmString(); err != nil { + klog.V(5).Infof("get l3 cache bit mask error: %v", err) + } return cpuBasicInfo, nil } diff --git a/pkg/util/system/cgroup.go b/pkg/util/system/cgroup.go index 5bb99d4c0..392389a41 100644 --- a/pkg/util/system/cgroup.go +++ b/pkg/util/system/cgroup.go @@ -115,6 +115,27 @@ func GetCgroupFilePath(cgroupTaskDir string, file CgroupFile) string { return path.Join(Conf.CgroupRootDir, file.Subfs, cgroupTaskDir, file.ResourceFileName) } +func GetCgroupCurTasks(cgroupPath string) ([]int, error) { + var tasks []int + rawContent, err := ioutil.ReadFile(cgroupPath) + if err != nil { + return nil, err + } + lines := strings.Split(string(rawContent), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if len(line) <= 0 { + continue + } + task, err := strconv.Atoi(line) + if err != nil { + return nil, err + } + tasks = append(tasks, task) + } + return tasks, nil +} + func GetCPUStatRaw(cgroupPath string) (*CPUStatRaw, error) { content, err := ioutil.ReadFile(cgroupPath) if err != nil { diff --git a/pkg/util/system/cgroup_resource.go b/pkg/util/system/cgroup_resource.go index f37bb3bab..218b830be 100644 --- a/pkg/util/system/cgroup_resource.go +++ b/pkg/util/system/cgroup_resource.go @@ -40,6 +40,7 @@ const ( CPUBVTWarpNsName = "cpu.bvt_warp_ns" CPUBurstName = "cpu.cfs_burst_us" CPUSFileName = "cpuset.cpus" + CPUTaskFileName = "tasks" CpuacctStatFileName = "cpuacct.stat" @@ -75,6 +76,7 @@ var ( CPUShares = CgroupFile{ResourceFileName: CPUSharesFileName, Subfs: CgroupCPUDir, IsAnolisOS: false} CPUCFSQuota = CgroupFile{ResourceFileName: CPUCFSQuotaName, Subfs: CgroupCPUDir, IsAnolisOS: false} CPUCFSPeriod = CgroupFile{ResourceFileName: CPUCFSPeriodName, Subfs: CgroupCPUDir, IsAnolisOS: false} + CPUTask = CgroupFile{ResourceFileName: CPUTaskFileName, Subfs: CgroupCPUDir, IsAnolisOS: false} CPUBurst = CgroupFile{ResourceFileName: CPUBurstName, Subfs: CgroupCPUDir, IsAnolisOS: true, Validator: CPUBurstValidator} CPUSet = CgroupFile{ResourceFileName: CPUSFileName, Subfs: CgroupCPUSetDir, IsAnolisOS: false} diff --git a/pkg/util/system/resctrl.go b/pkg/util/system/resctrl.go new file mode 100644 index 000000000..13de70c29 --- /dev/null +++ b/pkg/util/system/resctrl.go @@ -0,0 +1,176 @@ +/* + Copyright 2022 The Koordinator Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package system + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + + "k8s.io/klog/v2" +) + +const ( + ResctrlDir string = "resctrl/" + RdtInfoDir string = "info" + L3CatDir string = "L3" + + SchemataFileName string = "schemata" + CbmMaskFileName string = "cbm_mask" + ResctrlTaskFileName string = "tasks" + + ResctrlName string = "resctrl" +) + +var ( + initLock sync.Mutex + isInit bool + isSupportResctrl bool +) + +func isCPUSupportResctrl() (bool, error) { + isCatFlagSet, isMbaFlagSet, err := isResctrlAvailableByCpuInfo(filepath.Join(Conf.ProcRootDir, "cpuinfo")) + if err != nil { + klog.Errorf("isResctrlAvailableByCpuInfo error: %v", err) + return false, err + } + klog.Infof("isResctrlAvailableByCpuInfo result,isCatFlagSet: %v,isMbaFlagSet: %v", isCatFlagSet, isMbaFlagSet) + isInit = true + return isCatFlagSet && isMbaFlagSet, nil +} + +func isKernelSupportResctrl() (bool, error) { + isCatFlagSet, isMbaFlagSet, err := isResctrlAvailableByKernelCmd(filepath.Join(Conf.ProcRootDir, "cmdline")) + if err != nil { + klog.Errorf("isResctrlAvailableByKernelCmd error: %v", err) + return false, err + } + klog.Infof("isResctrlAvailableByKernelCmd result,isCatFlagSet: %v,isMbaFlagSet: %v", isCatFlagSet, isMbaFlagSet) + isInit = true + return isCatFlagSet && isMbaFlagSet, nil +} + +func IsSupportResctrl() (bool, error) { + initLock.Lock() + defer initLock.Unlock() + if !isInit { + cpuSupport, err := isCPUSupportResctrl() + if err != nil { + return false, err + } + kernelSupport, err := isKernelSupportResctrl() + if err != nil { + return false, err + } + isInit = true + isSupportResctrl = kernelSupport && cpuSupport + } + return isSupportResctrl, nil +} + +// @return /sys/fs/resctrl +func GetResctrlSubsystemDirPath() string { + return filepath.Join(Conf.SysFSRootDir, ResctrlDir) +} + +// @groupPath BE +// @return /sys/fs/resctrl/BE +func GetResctrlGroupRootDirPath(groupPath string) string { + return filepath.Join(Conf.SysFSRootDir, ResctrlDir, groupPath) +} + +// @return /sys/fs/resctrl/info/L3/cbm_mask +func GetResctrlL3CbmFilePath() string { + return filepath.Join(Conf.SysFSRootDir, ResctrlDir, RdtInfoDir, L3CatDir, CbmMaskFileName) +} + +// @groupPath BE +// @return /sys/fs/resctrl/BE/schemata +func GetResctrlSchemataFilePath(groupPath string) string { + return filepath.Join(Conf.SysFSRootDir, ResctrlDir, groupPath, SchemataFileName) +} + +// @groupPath BE +// @return /sys/fs/resctrl/BE/tasks +func GetResctrlTasksFilePath(groupPath string) string { + return filepath.Join(Conf.SysFSRootDir, ResctrlDir, groupPath, ResctrlTaskFileName) +} + +// ReadCatL3Cbm reads and returns the value of cat l3 cbm_mask +func ReadCatL3CbmString() (string, error) { + cbmFile := GetResctrlL3CbmFilePath() + out, err := ioutil.ReadFile(cbmFile) + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} + +// ReadResctrlTasksMap reads and returns the map of given resctrl group's task ids +func ReadResctrlTasksMap(groupPath string) (map[int]struct{}, error) { + tasksPath := GetResctrlTasksFilePath(groupPath) + rawContent, err := ioutil.ReadFile(tasksPath) + if err != nil { + return nil, err + } + + tasksMap := map[int]struct{}{} + + lines := strings.Split(string(rawContent), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if len(line) <= 0 { + continue + } + task, err := strconv.Atoi(line) + if err != nil { + return nil, err + } + tasksMap[task] = struct{}{} + } + return tasksMap, nil +} + +// CheckAndTryEnableResctrlCat checks if resctrl and l3_cat are enabled; if not, try to enable the features by mount +// resctrl subsystem; See MountResctrlSubsystem() for the detail. +// It returns whether the resctrl cat is enabled, and the error if failed to enable or to check resctrl interfaces +func CheckAndTryEnableResctrlCat() error { + // resctrl cat is correctly enabled: l3_cbm path exists + l3CbmFilePath := GetResctrlL3CbmFilePath() + _, err := os.Stat(l3CbmFilePath) + if err == nil { + return nil + } + newMount, err := MountResctrlSubsystem() + if err != nil { + return err + } + if newMount { + klog.Infof("mount resctrl successfully, resctrl enabled") + } + // double check l3_cbm path to ensure both resctrl and cat are correctly enabled + l3CbmFilePath = GetResctrlL3CbmFilePath() + _, err = os.Stat(l3CbmFilePath) + if err != nil { + return fmt.Errorf("resctrl cat is not enabled, err: %s", err) + } + return nil +} diff --git a/pkg/util/system/resctrl_linux.go b/pkg/util/system/resctrl_linux.go new file mode 100644 index 000000000..bc40a41dc --- /dev/null +++ b/pkg/util/system/resctrl_linux.go @@ -0,0 +1,121 @@ +//go:build linux +// +build linux + +/* + Copyright 2022 The Koordinator Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package system + +import ( + "bufio" + "fmt" + "os" + "regexp" + "strings" + "syscall" +) + +// MountResctrlSubsystem mounts resctrl fs under the sysFSRoot to enable the kernel feature on supported environment +// NOTE: Linux kernel (>= 4.10), Intel cpu and bare-mental host are required; Also, Intel RDT +// features should be enabled in kernel configurations and kernel commandline. +// For more info, please see https://github.com/intel/intel-cmt-cat/wiki/resctrl +func MountResctrlSubsystem() (bool, error) { + schemataPath := GetResctrlSchemataFilePath("") + // use schemata path to check since the subsystem root dir could keep exist when unmounted + _, err := os.Stat(schemataPath) + if err == nil { + return false, nil + } + subsystemPath := GetResctrlSubsystemDirPath() + err = syscall.Mount(ResctrlName, subsystemPath, ResctrlName, syscall.MS_RELATIME, "") + if err != nil { + return false, err + } + _, err = os.Stat(schemataPath) + if err != nil { + return false, fmt.Errorf("resctrl subsystem is mounted, but path %s does not exist, err: %s", + subsystemPath, err) + } + return true, nil +} + +func isResctrlAvailableByCpuInfo(path string) (bool, bool, error) { + isCatFlagSet := false + isMbaFlagSet := false + + f, err := os.Open(path) + if err != nil { + return false, false, err + } + defer f.Close() + + s := bufio.NewScanner(f) + for s.Scan() { + if err := s.Err(); err != nil { + return false, false, err + } + + line := s.Text() + + // Search "cat_l3" and "mba" flags in first "flags" line + if strings.Contains(line, "flags") { + flags := strings.Split(line, " ") + // "cat_l3" flag for CAT and "mba" flag for MBA + for _, flag := range flags { + switch flag { + case "cat_l3": + isCatFlagSet = true + case "mba": + isMbaFlagSet = true + } + } + return isCatFlagSet, isMbaFlagSet, nil + } + } + return isCatFlagSet, isMbaFlagSet, nil +} + +// file content example: +// BOOT_IMAGE=/boot/vmlinuz-4.19.91-24.1.al7.x86_64 root=UUID=231efa3b-302b-4e82-9445-0f7d5d353dda \ +// crashkernel=0M-2G:0M,2G-8G:192M,8G-:256M cryptomgr.notests cgroup.memory=nokmem rcupdate.rcu_cpu_stall_timeout=300 \ +// vring_force_dma_api biosdevname=0 net.ifnames=0 console=tty0 console=ttyS0,115200n8 noibrs \ +// nvme_core.io_timeout=4294967295 nomodeset intel_idle.max_cstate=1 rdt=cmt,l3cat,l3cdp,mba +func isResctrlAvailableByKernelCmd(path string) (bool, bool, error) { + isCatFlagSet := false + isMbaFlagSet := false + f, err := os.Open(path) + if err != nil { + return false, false, err + } + defer f.Close() + s := bufio.NewScanner(f) + for s.Scan() { + if err := s.Err(); err != nil { + return false, false, err + } + line := s.Text() + l3Reg, regErr := regexp.Compile(".* rdt=.*l3cat.*") + if regErr == nil && l3Reg.Match([]byte(line)) { + isCatFlagSet = true + } + + mbaReg, regErr := regexp.Compile(".* rdt=.*mba.*") + if regErr == nil && mbaReg.Match([]byte(line)) { + isMbaFlagSet = true + } + } + return isCatFlagSet, isMbaFlagSet, nil +} diff --git a/pkg/util/system/resctrl_linux_test.go b/pkg/util/system/resctrl_linux_test.go new file mode 100644 index 000000000..6b8d9fbac --- /dev/null +++ b/pkg/util/system/resctrl_linux_test.go @@ -0,0 +1,129 @@ +//go:build linux +// +build linux + +/* + Copyright 2022 The Koordinator Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package system + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_isResctrlAvailableByCpuInfo(t *testing.T) { + type args struct { + name string + cpuInfoContents string + expectIsCatFlagSet bool + expectIsMbaFlagSet bool + } + + tests := []args{ + { + name: "testResctrlEnable", + cpuInfoContents: "flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts pku ospke avx512_vnni md_clear flush_l1d arch_capabilities", + expectIsCatFlagSet: true, + expectIsMbaFlagSet: true, + }, + { + name: "testResctrlUnable", + cpuInfoContents: "flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault invpcid_single ibrs_enhanced tsc_adjust bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves wbnoinvd arat avx512vbmi pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm arch_capabilities", + expectIsCatFlagSet: false, + expectIsMbaFlagSet: false, + }, + { + name: "testContentsInvalid", + cpuInfoContents: "invalid contents", + expectIsCatFlagSet: false, + expectIsMbaFlagSet: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + helper := NewFileTestUtil(t) + defer helper.Cleanup() + + helper.WriteProcSubFileContents("cpuinfo", tt.cpuInfoContents) + gotIsCatFlagSet, gotIsMbaFlagSet, err := isResctrlAvailableByCpuInfo(filepath.Join(Conf.ProcRootDir, "cpuinfo")) + assert.NoError(t, err, "testError") + assert.Equal(t, tt.expectIsCatFlagSet, gotIsCatFlagSet, "checkIsCatFlagSet") + assert.Equal(t, tt.expectIsMbaFlagSet, gotIsMbaFlagSet, "checkIsMbaFlagSet") + }) + } +} + +func Test_isResctrlAvailableByKernelCmd(t *testing.T) { + type args struct { + content string + } + tests := []struct { + name string + args args + wantCat bool + wantMba bool + }{ + { + name: "testResctrlEnable", + args: args{ + content: "BOOT_IMAGE=/boot/vmlinuz-4.19.91-24.1.al7.x86_64 root=UUID=231efa3b-302b-4e82-9445-0f7d5d353dda rdt=cmt,l3cat,l3cdp,mba", + }, + wantCat: true, + wantMba: true, + }, + { + name: "testResctrlCatDisable", + args: args{ + content: "BOOT_IMAGE=/boot/vmlinuz-4.19.91-24.1.al7.x86_64 root=UUID=231efa3b-302b-4e82-9445-0f7d5d353dda rdt=cmt,mba,l3cdp", + }, + wantCat: false, + wantMba: true, + }, + { + name: "testResctrlMBADisable", + args: args{ + content: "BOOT_IMAGE=/boot/vmlinuz-4.19.91-24.1.al7.x86_64 root=UUID=231efa3b-302b-4e82-9445-0f7d5d353dda rdt=cmt,l3cat,l3cdp", + }, + wantCat: true, + wantMba: false, + }, + { + name: "testContentsInvalid", + args: args{ + content: "invalid contents", + }, + wantCat: false, + wantMba: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + helper := NewFileTestUtil(t) + defer helper.Cleanup() + helper.WriteProcSubFileContents("cmdline", tt.args.content) + isCatFlagSet, isMbaFlagSet, _ := isResctrlAvailableByKernelCmd(filepath.Join(Conf.ProcRootDir, "cmdline")) + if isCatFlagSet != tt.wantCat { + t.Errorf("isResctrlAvailableByKernelCmd() isCatFlagSet = %v, want %v", isCatFlagSet, tt.wantCat) + } + if isMbaFlagSet != tt.wantMba { + t.Errorf("isResctrlAvailableByKernelCmd() got1 = %v, want %v", isMbaFlagSet, tt.wantMba) + } + }) + } +} diff --git a/pkg/util/system/resctrl_test.go b/pkg/util/system/resctrl_test.go new file mode 100644 index 000000000..b80a8689d --- /dev/null +++ b/pkg/util/system/resctrl_test.go @@ -0,0 +1,183 @@ +/* + Copyright 2022 The Koordinator Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package system + +import ( + "io/ioutil" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_ReadResctrlTasksMap(t *testing.T) { + type args struct { + groupPath string + } + type fields struct { + tasksStr string + invalidPath bool + } + tests := []struct { + name string + args args + fields fields + want map[int]struct{} + wantErr bool + }{ + { + name: "do not panic but throw an error for empty input", + want: map[int]struct{}{}, + wantErr: false, + }, + { + name: "invalid path", + fields: fields{invalidPath: true}, + want: nil, + wantErr: true, + }, + { + name: "parse correctly", + fields: fields{tasksStr: "101\n111\n"}, + want: map[int]struct{}{101: {}, 111: {}}, + wantErr: false, + }, + { + name: "parse correctly 1", + args: args{groupPath: "BE"}, + fields: fields{tasksStr: "101\n111\n"}, + want: map[int]struct{}{101: {}, 111: {}}, + wantErr: false, + }, + { + name: "parse error for invalid task str", + fields: fields{tasksStr: "101\n1aa\n"}, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var sysFSRootDir string + sysFSRootDir, _ = ioutil.TempDir("", "ReadResctrlTasksMap") + resctrlDir := filepath.Join(sysFSRootDir, ResctrlDir, tt.args.groupPath) + err := os.MkdirAll(resctrlDir, 0700) + assert.NoError(t, err) + + tasksPath := filepath.Join(resctrlDir, ResctrlTaskFileName) + err = ioutil.WriteFile(tasksPath, []byte(tt.fields.tasksStr), 0666) + assert.NoError(t, err) + + defer os.RemoveAll(sysFSRootDir) + + Conf = &Config{ + SysFSRootDir: sysFSRootDir, + } + if tt.fields.invalidPath { + Conf.SysFSRootDir = "invalidPath" + } + + got, err := ReadResctrlTasksMap(tt.args.groupPath) + assert.Equal(t, tt.wantErr, err != nil) + assert.Equal(t, tt.want, got) + }) + } + +} + +func Test_CheckAndTryEnableResctrlCat(t *testing.T) { + type fields struct { + cbmStr string + invalidPath bool + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "return disabled for a invalid path", + fields: fields{invalidPath: true}, + wantErr: true, + }, + { + name: "return enabled for a valid l3_cbm", + fields: fields{cbmStr: "3f"}, + wantErr: false, + }, + // TODO: add mount case + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var sysFSRootDir string + sysFSRootDir, _ = ioutil.TempDir("", "CheckAndTryEnableResctrlCat") + defer os.RemoveAll(sysFSRootDir) + resctrlDir := filepath.Join(sysFSRootDir, ResctrlDir) + l3CatDir := filepath.Join(resctrlDir, RdtInfoDir, L3CatDir) + err := os.MkdirAll(l3CatDir, 0700) + assert.NoError(t, err) + + cbmPath := filepath.Join(l3CatDir, CbmMaskFileName) + err = ioutil.WriteFile(cbmPath, []byte(tt.fields.cbmStr), 0666) + assert.NoError(t, err) + + Conf = &Config{ + SysFSRootDir: sysFSRootDir, + } + if tt.fields.invalidPath { + Conf.SysFSRootDir = "invalidPath" + } + + gotErr := CheckAndTryEnableResctrlCat() + + assert.Equal(t, tt.wantErr, gotErr != nil) + }) + } +} + +func Test_MountResctrlSubsystem(t *testing.T) { + t.Run("test not panic", func(t *testing.T) { + var sysFSRootDir string + sysFSRootDir, _ = ioutil.TempDir("", "MountResctrlSubsystem") + defer os.RemoveAll(sysFSRootDir) + resctrlDir := filepath.Join(sysFSRootDir, ResctrlDir) + err := os.MkdirAll(resctrlDir, 0700) + assert.NoError(t, err) + + schemataPath := filepath.Join(resctrlDir, SchemataFileName) + err = ioutil.WriteFile(schemataPath, []byte(" L3:0=ff;1=ff\n MB:0=100;1=100\n"), 0666) + assert.NoError(t, err) + + Conf = &Config{ + SysFSRootDir: sysFSRootDir, + } + + got, err := MountResctrlSubsystem() + + // resctrl is only supported by linux + if runtime.GOOS != "linux" { + assert.Equal(t, false, got) + assert.EqualError(t, err, "only support linux") + return + } + + assert.Equal(t, false, got) + assert.NoError(t, err) + }) +} diff --git a/pkg/util/system/resctrl_unsupported.go b/pkg/util/system/resctrl_unsupported.go new file mode 100644 index 000000000..8a06caf07 --- /dev/null +++ b/pkg/util/system/resctrl_unsupported.go @@ -0,0 +1,35 @@ +//go:build !linux +// +build !linux + +/* + Copyright 2022 The Koordinator Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package system + +import "fmt" + +// MountResctrlSubsystem is not supported for non-linux os +func MountResctrlSubsystem() (bool, error) { + return false, fmt.Errorf("only support linux") +} + +func isResctrlAvailableByCpuInfo(path string) (bool, bool, error) { + return false, false, nil +} + +func isResctrlAvailableByKernelCmd(path string) (bool, bool, error) { + return false, false, nil +}