From 59607d91211f66fc9f5a5a60cc144b6ea546e3f8 Mon Sep 17 00:00:00 2001 From: Zhang Kang Date: Thu, 4 Jul 2024 10:38:38 +0800 Subject: [PATCH 01/10] add resctrl runtime hook Signed-off-by: Zhang Kang --- .../plugins/resctrl/resctrl_reconcile.go | 5 + pkg/koordlet/runtimehooks/config.go | 9 + pkg/koordlet/runtimehooks/hooks/hooks.go | 6 +- .../runtimehooks/hooks/resctrl/resctrl.go | 209 ++++++++++ .../runtimehooks/hooks/resctrl/rule.go | 72 ++++ .../runtimehooks/hooks/resctrl/updater.go | 138 +++++++ .../runtimehooks/protocol/pod_context.go | 2 + .../runtimehooks/protocol/protocol.go | 23 ++ .../runtimehooks/reconciler/reconciler.go | 50 +++ pkg/koordlet/util/resctrl/ctrl_mgr.go | 194 ++++++++++ pkg/koordlet/util/resctrl/ctrl_mgr_test.go | 357 ++++++++++++++++++ pkg/koordlet/util/resctrl/resctrl.go | 318 ++++++++++++++++ pkg/koordlet/util/system/resctrl.go | 4 +- 13 files changed, 1384 insertions(+), 3 deletions(-) create mode 100644 pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go create mode 100644 pkg/koordlet/runtimehooks/hooks/resctrl/rule.go create mode 100755 pkg/koordlet/runtimehooks/hooks/resctrl/updater.go create mode 100644 pkg/koordlet/util/resctrl/ctrl_mgr.go create mode 100644 pkg/koordlet/util/resctrl/ctrl_mgr_test.go create mode 100644 pkg/koordlet/util/resctrl/resctrl.go diff --git a/pkg/koordlet/qosmanager/plugins/resctrl/resctrl_reconcile.go b/pkg/koordlet/qosmanager/plugins/resctrl/resctrl_reconcile.go index 973d12d75..7b41cbb2b 100644 --- a/pkg/koordlet/qosmanager/plugins/resctrl/resctrl_reconcile.go +++ b/pkg/koordlet/qosmanager/plugins/resctrl/resctrl_reconcile.go @@ -459,6 +459,11 @@ func (r *resctrlReconcile) reconcileResctrlGroups(qosStrategy *slov1alpha1.Resou podsMeta := r.statesInformer.GetAllPods() for _, podMeta := range podsMeta { pod := podMeta.Pod + // only QoS class level pod are considered + if _, ok := pod.Annotations[extension.AnnotationResctrl]; ok { + continue + } + // only Running and Pending pods are considered if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodPending { continue diff --git a/pkg/koordlet/runtimehooks/config.go b/pkg/koordlet/runtimehooks/config.go index fdc070d68..ad517ad5d 100644 --- a/pkg/koordlet/runtimehooks/config.go +++ b/pkg/koordlet/runtimehooks/config.go @@ -32,6 +32,7 @@ import ( "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/cpuset" "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/gpu" "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/groupidentity" + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/resctrl" "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/tc" "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/terwayqos" "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" @@ -87,6 +88,12 @@ const ( // owner: @lucming // alpha: v1.5 TCNetworkQoS featuregate.Feature = "TCNetworkQoS" + + // Resctrl adjusts LLC/MB value for pod. + // + // owner: @kangclzjc @saintube @zwzhang0107 + // alpha: v1.5 + Resctrl featuregate.Feature = "Resctrl" ) var ( @@ -99,6 +106,7 @@ var ( CoreSched: {Default: false, PreRelease: featuregate.Alpha}, TerwayQoS: {Default: false, PreRelease: featuregate.Alpha}, TCNetworkQoS: {Default: false, PreRelease: featuregate.Alpha}, + Resctrl: {Default: false, PreRelease: featuregate.Alpha}, } runtimeHookPlugins = map[featuregate.Feature]HookPlugin{ @@ -110,6 +118,7 @@ var ( CoreSched: coresched.Object(), TerwayQoS: terwayqos.Object(), TCNetworkQoS: tc.Object(), + Resctrl: resctrl.Object(), } ) diff --git a/pkg/koordlet/runtimehooks/hooks/hooks.go b/pkg/koordlet/runtimehooks/hooks/hooks.go index 7c6c38fca..d8437066a 100644 --- a/pkg/koordlet/runtimehooks/hooks/hooks.go +++ b/pkg/koordlet/runtimehooks/hooks/hooks.go @@ -25,6 +25,7 @@ import ( "github.com/koordinator-sh/koordinator/pkg/koordlet/metrics" "github.com/koordinator-sh/koordinator/pkg/koordlet/resourceexecutor" "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol" + "github.com/koordinator-sh/koordinator/pkg/koordlet/statesinformer" rmconfig "github.com/koordinator-sh/koordinator/pkg/runtimeproxy/config" ) @@ -36,8 +37,9 @@ type Hook struct { } type Options struct { - Reader resourceexecutor.CgroupReader - Executor resourceexecutor.ResourceUpdateExecutor + Reader resourceexecutor.CgroupReader + Executor resourceexecutor.ResourceUpdateExecutor + StatesInformer statesinformer.StatesInformer } type HookFn func(protocol.HooksProtocol) error diff --git a/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go b/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go new file mode 100644 index 000000000..379b72629 --- /dev/null +++ b/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go @@ -0,0 +1,209 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resctrl + +import ( + "fmt" + "strings" + + "k8s.io/klog/v2" + + apiext "github.com/koordinator-sh/koordinator/apis/extension" + "github.com/koordinator-sh/koordinator/pkg/koordlet/resourceexecutor" + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks" + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol" + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/reconciler" + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/rule" + "github.com/koordinator-sh/koordinator/pkg/koordlet/statesinformer" + util "github.com/koordinator-sh/koordinator/pkg/koordlet/util/resctrl" + "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" + sysutil "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" + rmconfig "github.com/koordinator-sh/koordinator/pkg/runtimeproxy/config" +) + +const ( + name = "Resctrl" + description = "set resctrl for pod" + ruleNameForAllPods = name + " (AllPods)" +) + +type plugin struct { + rule *Rule + engine util.ResctrlEngine + executor resourceexecutor.ResourceUpdateExecutor + statesInformer statesinformer.StatesInformer +} + +var singleton *plugin + +func Object() *plugin { + if singleton == nil { + singleton = newPlugin() + } + return singleton +} + +func newPlugin() *plugin { + return &plugin{ + rule: newRule(), + } +} + +func (p *plugin) Register(op hooks.Options) { + // skip if host not support resctrl + if support, err := system.IsSupportResctrl(); err != nil { + klog.Warningf("check support resctrl failed, err: %s", err) + return + } else if !support { + klog.V(5).Infof("resctrl runtime hook skipped, cpu not support CAT/MBA") + return + } + + if vendorID, err := sysutil.GetVendorIDByCPUInfo(sysutil.GetCPUInfoPath()); err == nil { + p.engine, err = util.NewRDTEngine(vendorID) + if err != nil { + klog.Errorf("New RDT Engine failed, error is %v", err) + return + } + } + p.executor = op.Executor + p.statesInformer = op.StatesInformer + p.engine.Rebuild() + + rule.Register(ruleNameForAllPods, description, + rule.WithParseFunc(statesinformer.RegisterTypeAllPods, p.parseRuleForAllPods), + rule.WithUpdateCallback(p.ruleUpdateCbForAllPods)) + + hooks.Register(rmconfig.PreRunPodSandbox, name, description+" (pod)", p.SetPodResctrlResourcesForHooks) + hooks.Register(rmconfig.PreCreateContainer, name, description+" (pod)", p.SetContainerResctrlResources) + hooks.Register(rmconfig.PreRemoveRunPodSandbox, name, description+" (pod)", p.RemovePodResctrlResources) + + reconciler.RegisterCgroupReconciler(reconciler.PodLevel, system.ResctrlSchemata, description+" (pod resctrl schema)", p.SetPodResctrlResourcesForReconciler, reconciler.NoneFilter()) + reconciler.RegisterCgroupReconciler(reconciler.PodLevel, system.ResctrlTasks, description+" (pod resctrl tasks)", p.UpdatePodTaskIds, reconciler.NoneFilter()) + reconciler.RegisterCgroupReconciler4AllPods(reconciler.AllPodsLevel, system.ResctrlRoot, description+" (pod resctl schema)", p.RemoveUnusedResctrlPath, reconciler.PodAnnotationResctrlFilter(), "resctrl") + +} + +func (p *plugin) SetPodResctrlResourcesForHooks(proto protocol.HooksProtocol) error { + return p.setPodResctrlResources(proto, true) +} + +func (p *plugin) SetPodResctrlResourcesForReconciler(proto protocol.HooksProtocol) error { + return p.setPodResctrlResources(proto, false) +} + +func (p *plugin) setPodResctrlResources(proto protocol.HooksProtocol, fromNRI bool) error { + podCtx, ok := proto.(*protocol.PodContext) + if !ok { + return fmt.Errorf("pod protocol is nil for plugin %v", name) + } + + if v, ok := podCtx.Request.Annotations[apiext.AnnotationResctrl]; ok { + app, ok := p.engine.GetApp(podCtx.Request.PodMeta.UID) + if ok && app.Annotation == v { + return nil + } + updater := NewCreateResctrlProtocolUpdater(proto) + err := p.engine.RegisterApp(podCtx.Request.PodMeta.UID, v, fromNRI, updater) + if err != nil { + return err + } + } + + return nil +} + +func (p *plugin) RemoveUnusedResctrlPath(protos []protocol.HooksProtocol) error { + currentPods := make(map[string]protocol.HooksProtocol) + + for _, proto := range protos { + podCtx, ok := proto.(*protocol.PodContext) + if !ok { + return fmt.Errorf("pod protocol is nil for plugin %v", name) + } + + if _, ok := podCtx.Request.Annotations[apiext.AnnotationResctrl]; ok { + group := podCtx.Request.PodMeta.UID + currentPods[group] = podCtx + } + } + + apps := p.engine.GetApps() + for k, v := range apps { + if _, ok := currentPods[k]; !ok { + updater := NewRemoveResctrlUpdater(v.Closid) + p.engine.UnRegisterApp(strings.TrimPrefix(v.Closid, util.ClosdIdPrefix), false, updater) + } + } + return nil +} + +func (p *plugin) UpdatePodTaskIds(proto protocol.HooksProtocol) error { + podCtx, ok := proto.(*protocol.PodContext) + if !ok { + return fmt.Errorf("pod protocol is nil for plugin %v", name) + } + + if _, ok := podCtx.Request.Annotations[apiext.AnnotationResctrl]; ok { + curTaskMaps := map[string]map[int32]struct{}{} + var err error + group := podCtx.Request.PodMeta.UID + curTaskMaps[group], err = system.ReadResctrlTasksMap(util.ClosdIdPrefix + group) + if err != nil { + klog.Warningf("failed to read Cat L3 tasks for resctrl group %s, err: %s", group, err) + } + + newTaskIds := util.GetPodCgroupNewTaskIdsFromPodCtx(podCtx, curTaskMaps[group]) + resctrlInfo := &protocol.Resctrl{ + Closid: util.ClosdIdPrefix + group, + NewTaskIds: make([]int32, 0), + } + resctrlInfo.NewTaskIds = newTaskIds + podCtx.Response.Resources.Resctrl = resctrlInfo + } + return nil +} + +func (p *plugin) SetContainerResctrlResources(proto protocol.HooksProtocol) error { + containerCtx, ok := proto.(*protocol.ContainerContext) + if !ok { + return fmt.Errorf("container protocol is nil for plugin %v", name) + } + + if _, ok := containerCtx.Request.PodAnnotations[apiext.AnnotationResctrl]; ok { + containerCtx.Response.Resources.Resctrl = &protocol.Resctrl{ + Schemata: "", + Closid: util.ClosdIdPrefix + containerCtx.Request.PodMeta.UID, + NewTaskIds: make([]int32, 0), + } + } + + return nil +} + +func (p *plugin) RemovePodResctrlResources(proto protocol.HooksProtocol) error { + podCtx, ok := proto.(*protocol.PodContext) + if !ok { + return fmt.Errorf("pod protocol is nil for plugin %v", name) + } + + if _, ok := podCtx.Request.Annotations[apiext.AnnotationResctrl]; ok { + updater := NewRemoveResctrlProtocolUpdater(proto) + p.engine.UnRegisterApp(podCtx.Request.PodMeta.UID, true, updater) + } + return nil +} diff --git a/pkg/koordlet/runtimehooks/hooks/resctrl/rule.go b/pkg/koordlet/runtimehooks/hooks/resctrl/rule.go new file mode 100644 index 000000000..afc1aa08b --- /dev/null +++ b/pkg/koordlet/runtimehooks/hooks/resctrl/rule.go @@ -0,0 +1,72 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resctrl + +import ( + "strings" + "sync" + + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + + apiext "github.com/koordinator-sh/koordinator/apis/extension" + "github.com/koordinator-sh/koordinator/pkg/koordlet/statesinformer" + util "github.com/koordinator-sh/koordinator/pkg/koordlet/util/resctrl" +) + +type Rule struct { + lock sync.RWMutex +} + +func newRule() *Rule { + return &Rule{} +} + +func (p *plugin) parseRuleForAllPods(allPods interface{}) (bool, error) { + return true, nil +} + +func (p *plugin) ruleUpdateCbForAllPods(target *statesinformer.CallbackTarget) error { + if target == nil { + klog.Warningf("callback target is nil") + return nil + } + + if p.rule == nil { + klog.V(5).Infof("hook plugin rule is nil, nothing to do for plugin %v", ruleNameForAllPods) + return nil + } + + apps := p.engine.GetApps() + + currentPods := make(map[string]*corev1.Pod) + for _, podMeta := range target.Pods { + pod := podMeta.Pod + if _, ok := podMeta.Pod.Annotations[apiext.AnnotationResctrl]; ok { + group := string(podMeta.Pod.UID) + currentPods[group] = pod + } + } + + for k, v := range apps { + if _, ok := currentPods[k]; !ok { + updater := NewRemoveResctrlUpdater(v.Closid) + p.engine.UnRegisterApp(strings.TrimPrefix(v.Closid, util.ClosdIdPrefix), false, updater) + } + } + return nil +} diff --git a/pkg/koordlet/runtimehooks/hooks/resctrl/updater.go b/pkg/koordlet/runtimehooks/hooks/resctrl/updater.go new file mode 100755 index 000000000..d9b4ffa89 --- /dev/null +++ b/pkg/koordlet/runtimehooks/hooks/resctrl/updater.go @@ -0,0 +1,138 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resctrl + +import ( + "fmt" + "os" + + "k8s.io/klog/v2" + + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol" + util "github.com/koordinator-sh/koordinator/pkg/koordlet/util/resctrl" + "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" +) + +type UpdateFunc func(resource util.ResctrlUpdater) error + +type DefaultResctrlProtocolUpdater struct { + hooksProtocol protocol.HooksProtocol + group string + schemata string + updateFunc UpdateFunc +} + +func (u DefaultResctrlProtocolUpdater) Name() string { + return "default" +} + +func (u DefaultResctrlProtocolUpdater) Key() string { + return u.group +} + +func (u DefaultResctrlProtocolUpdater) Value() string { + return u.schemata +} + +func (r *DefaultResctrlProtocolUpdater) SetKey(key string) { + r.group = key +} + +func (r *DefaultResctrlProtocolUpdater) SetValue(val string) { + r.schemata = val +} + +func (u *DefaultResctrlProtocolUpdater) Update() error { + return u.updateFunc(u) +} + +type Updater func(u DefaultResctrlProtocolUpdater) error + +func NewCreateResctrlProtocolUpdater(hooksProtocol protocol.HooksProtocol) util.ResctrlUpdater { + return &DefaultResctrlProtocolUpdater{ + hooksProtocol: hooksProtocol, + updateFunc: CreateResctrlProtocolUpdaterFunc, + } +} + +func NewRemoveResctrlProtocolUpdater(hooksProtocol protocol.HooksProtocol) util.ResctrlUpdater { + return &DefaultResctrlProtocolUpdater{ + hooksProtocol: hooksProtocol, + updateFunc: RemoveResctrlProtocolUpdaterFunc, + } +} + +func NewRemoveResctrlUpdater(group string) util.ResctrlUpdater { + return &DefaultResctrlProtocolUpdater{ + group: group, + updateFunc: RemoveResctrlUpdaterFunc, + } +} + +func CreateResctrlProtocolUpdaterFunc(u util.ResctrlUpdater) error { + r, ok := u.(*DefaultResctrlProtocolUpdater) + if !ok { + return fmt.Errorf("not a ResctrlSchemataResourceUpdater") + } + + podCtx, ok := r.hooksProtocol.(*protocol.PodContext) + if !ok { + return fmt.Errorf("pod protocol is nil for plugin %v", name) + } + if podCtx.Response.Resources.Resctrl != nil { + podCtx.Response.Resources.Resctrl.Schemata = r.Value() + podCtx.Response.Resources.Resctrl.Closid = r.Key() + } else { + resctrlInfo := &protocol.Resctrl{ + NewTaskIds: make([]int32, 0), + } + resctrlInfo.Schemata = r.Value() + resctrlInfo.Closid = r.Key() + podCtx.Response.Resources.Resctrl = resctrlInfo + } + return nil +} + +func RemoveResctrlProtocolUpdaterFunc(u util.ResctrlUpdater) error { + r, ok := u.(*DefaultResctrlProtocolUpdater) + if !ok { + return fmt.Errorf("not a ResctrlSchemataResourceUpdater") + } + resctrlInfo := &protocol.Resctrl{ + NewTaskIds: make([]int32, 0), + } + podCtx, ok := r.hooksProtocol.(*protocol.PodContext) + if !ok { + return fmt.Errorf("pod protocol is nil for plugin %v", name) + } + resctrlInfo.Closid = util.ClosdIdPrefix + podCtx.Request.PodMeta.UID + podCtx.Response.Resources.Resctrl = resctrlInfo + return nil +} + +func RemoveResctrlUpdaterFunc(u util.ResctrlUpdater) error { + r, ok := u.(*DefaultResctrlProtocolUpdater) + if !ok { + return fmt.Errorf("not a ResctrlSchemataResourceUpdater") + } + if err := os.Remove(system.GetResctrlGroupRootDirPath(r.group)); err != nil { + return err + } else { + klog.V(5).Infof("successfully remove ctrl group %s", r.group) + } + return nil +} diff --git a/pkg/koordlet/runtimehooks/protocol/pod_context.go b/pkg/koordlet/runtimehooks/protocol/pod_context.go index be7576ed6..ed3a3bbe0 100644 --- a/pkg/koordlet/runtimehooks/protocol/pod_context.go +++ b/pkg/koordlet/runtimehooks/protocol/pod_context.go @@ -70,6 +70,7 @@ type PodRequest struct { CgroupParent string Resources *Resources // TODO: support proxy & nri mode ExtendedResources *apiext.ExtendedResourceSpec + ContainerTaskIds map[string][]int32 } func (p *PodRequest) FromNri(pod *api.PodSandbox) { @@ -109,6 +110,7 @@ func (p *PodRequest) FromReconciler(podMeta *statesinformer.PodMeta) { p.Labels = podMeta.Pod.Labels p.Annotations = podMeta.Pod.Annotations p.CgroupParent = podMeta.CgroupDir + p.ContainerTaskIds = podMeta.ContainerTaskIds p.Resources = &Resources{} p.Resources.FromPod(podMeta.Pod) // retrieve ExtendedResources from pod spec and pod annotations (prefer pod spec) diff --git a/pkg/koordlet/runtimehooks/protocol/protocol.go b/pkg/koordlet/runtimehooks/protocol/protocol.go index 6a0a4401c..bf7fc2ea9 100644 --- a/pkg/koordlet/runtimehooks/protocol/protocol.go +++ b/pkg/koordlet/runtimehooks/protocol/protocol.go @@ -73,6 +73,12 @@ var HooksProtocolBuilder = hooksProtocolBuilder{ }, } +type Resctrl struct { + Schemata string + Closid string + NewTaskIds []int32 +} + type Resources struct { // origin resources CPUShares *int64 @@ -84,6 +90,7 @@ type Resources struct { // extended resources CPUBvt *int64 CPUIdle *int64 + Resctrl *Resctrl } func (r *Resources) IsOriginResSet() bool { @@ -189,3 +196,19 @@ func injectNetClsClassId(cgroupParent string, classId uint32, a *audit.EventHelp } return updater, nil } + +func createCatGroup(closid string, a *audit.EventHelper, e resourceexecutor.ResourceUpdateExecutor) (resourceexecutor.ResourceUpdater, error) { + updater, err := resourceexecutor.NewCatGroupResource(closid, a) + if err != nil { + return nil, err + } + return updater, nil +} + +func injectResctrl(closid string, schemata string, e *audit.EventHelper, executor resourceexecutor.ResourceUpdateExecutor) (resourceexecutor.ResourceUpdater, error) { + updater, err := resourceexecutor.NewResctrlSchemataResource(closid, schemata, e) + if err != nil { + return nil, err + } + return updater, nil +} diff --git a/pkg/koordlet/runtimehooks/reconciler/reconciler.go b/pkg/koordlet/runtimehooks/reconciler/reconciler.go index cf10bcc62..501d69f71 100644 --- a/pkg/koordlet/runtimehooks/reconciler/reconciler.go +++ b/pkg/koordlet/runtimehooks/reconciler/reconciler.go @@ -160,6 +160,34 @@ func PodHostNetworkFilter() *podHostNetworkFilter { return singletonPodHostNetworkFilter } +type podAnnotationResctrlFilter struct{} + +const ( + podAnnotationResctrlFilterName = "resctrl" +) + +func (p *podAnnotationResctrlFilter) Name() string { + return podAnnotationResctrlFilterName +} + +func (p *podAnnotationResctrlFilter) Filter(podMeta *statesinformer.PodMeta) string { + if _, ok := podMeta.Pod.Annotations[apiext.AnnotationResctrl]; ok { + return podAnnotationResctrlFilterName + } + + return "" +} + +var singletonPodAnnotationResctrlFilter *podAnnotationResctrlFilter + +// PodQOSFilter returns a Filter which filters pod qos class +func PodAnnotationResctrlFilter() *podAnnotationResctrlFilter { + if singletonPodQOSFilter == nil { + singletonPodQOSFilter = &podQOSFilter{} + } + return singletonPodAnnotationResctrlFilter +} + type reconcileFunc func(protocol.HooksProtocol) error type reconcileFunc4AllPods func([]protocol.HooksProtocol) error @@ -471,6 +499,28 @@ func (c *reconciler) reconcilePodCgroup(stopCh <-chan struct{}) { } } } + + for _, r := range globalCgroupReconcilers.allPodsLevel { + currentPods := make([]protocol.HooksProtocol, 0) + for _, podMeta := range podsMeta { + if _, ok := r.fn4AllPods[r.filter.Filter(podMeta)]; ok { + podCtx := protocol.HooksProtocolBuilder.Pod(podMeta) + currentPods = append(currentPods, podCtx) + } + } + + reconcileFn, ok := r.fn4AllPods[r.filter.Name()] + if !ok { + klog.V(5).Infof("calling reconcile function %v aborted, condition %s not registered", + r.description, r.filter.Name()) + continue + } + + if err := reconcileFn(currentPods); err != nil { + klog.Warningf("calling reconcile function %v for pod %v failed, error %v", + r.description, err) + } + } } for _, r := range globalCgroupReconcilers.allPodsLevel { diff --git a/pkg/koordlet/util/resctrl/ctrl_mgr.go b/pkg/koordlet/util/resctrl/ctrl_mgr.go new file mode 100644 index 000000000..d79dda9bc --- /dev/null +++ b/pkg/koordlet/util/resctrl/ctrl_mgr.go @@ -0,0 +1,194 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resctrl + +import ( + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + gocache "github.com/patrickmn/go-cache" + "k8s.io/klog/v2" + + sysutil "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" +) + +const ( + Remove = "Remove" + Add = "Add" + ExpirationTime int64 = 10 + CleanupInterval = 600 * time.Second +) + +type Updater interface { + Update(string) error +} + +type SchemataUpdater interface { + Update(id, schemata string) error +} + +type ControlGroup struct { + AppId string + GroupId string + Schemata string + Status string + CreatedTime int64 +} + +type ControlGroupManager struct { + rdtcgs *gocache.Cache + reconcileInterval int64 + sync.Mutex +} + +func NewControlGroupManager() ControlGroupManager { + return ControlGroupManager{ + rdtcgs: gocache.New(time.Duration(ExpirationTime), CleanupInterval), + } +} + +func (c *ControlGroupManager) Init() { + c.Lock() + defer c.Unlock() + + // get resctrl filesystem root + root := sysutil.GetResctrlSubsystemDirPath() + files, err := os.ReadDir(root) + if err != nil { + klog.Errorf("read %s failed err is %v", root, err) + return + } + + // rebuild c.rdtcgs when restart + for _, file := range files { + if file.IsDir() && strings.HasPrefix(file.Name(), ClosdIdPrefix) { + path := filepath.Join(root, file.Name(), "schemata") + if _, err := os.Stat(path); err == nil { + reader, err := os.Open(path) + if err != nil { + klog.Errorf("open resctrl file path fail, %v", err) + } + content, err := io.ReadAll(reader) + if err != nil { + klog.Errorf("read resctrl file path fail, %v", err) + return + } + schemata := string(content) + podid := strings.TrimPrefix(file.Name(), ClosdIdPrefix) + c.rdtcgs.Set(podid, &ControlGroup{ + AppId: podid, + GroupId: file.Name(), + Schemata: schemata, + Status: Add, + CreatedTime: time.Now().UnixNano(), + }, -1) + klog.Infof("podid is %s, ctrl group is %v", podid, file.Name()) + } + } + } +} + +func (c *ControlGroupManager) AddPod(podid string, schemata string, fromNRI bool, createUpdater ResctrlUpdater, schemataUpdater ResctrlUpdater) { + c.Lock() + defer c.Unlock() + + p, ok := c.rdtcgs.Get(podid) + var pod *ControlGroup + if !ok { + pod = &ControlGroup{ + AppId: podid, + GroupId: "", + Schemata: "", + Status: Add, + } + } else { + pod = p.(*ControlGroup) + } + + if pod.Status == Add && pod.GroupId == "" { + if createUpdater != nil { + err := createUpdater.Update() + if err != nil { + klog.Errorf("create ctrl group error %v", err) + } else { + pod.GroupId = ClosdIdPrefix + podid + pod.CreatedTime = time.Now().UnixNano() + } + } + + if schemataUpdater != nil { + err := schemataUpdater.Update() + if err != nil { + klog.Errorf("updater ctrl group schemata error %v", err) + } + pod.Schemata = schemata + } + + c.rdtcgs.Set(podid, pod, -1) + } else { + if pod.Status == Add && pod.GroupId != "" && !fromNRI { + // Update Schemata + if schemataUpdater != nil { + err := schemataUpdater.Update() + if err != nil { + klog.Errorf("updater ctrl group schemata error %v", err) + } + pod.Schemata = schemata + } + c.rdtcgs.Set(podid, pod, -1) + } + } +} + +func (c *ControlGroupManager) RemovePod(podid string, fromNRI bool, removeUpdater ResctrlUpdater) bool { + c.Lock() + defer c.Unlock() + + p, ok := c.rdtcgs.Get(podid) + if !ok { + pod := &ControlGroup{podid, "", "", Remove, -1} + if removeUpdater != nil { + err := removeUpdater.Update() + if err != nil { + klog.Errorf("remove updater fail %v", err) + return false + } + } + + c.rdtcgs.Set(podid, pod, gocache.DefaultExpiration) + return true + } + pod := p.(*ControlGroup) + if (fromNRI || time.Now().UnixNano()-pod.CreatedTime >= ExpirationTime*time.Second.Nanoseconds()) && pod.Status == Add { + pod.Status = Remove + if removeUpdater != nil { + err := removeUpdater.Update() + if err != nil { + klog.Errorf("remove updater fail %v", err) + return false + } + } + + c.rdtcgs.Set(podid, pod, gocache.DefaultExpiration) + return true + } + return false +} diff --git a/pkg/koordlet/util/resctrl/ctrl_mgr_test.go b/pkg/koordlet/util/resctrl/ctrl_mgr_test.go new file mode 100644 index 000000000..72f450127 --- /dev/null +++ b/pkg/koordlet/util/resctrl/ctrl_mgr_test.go @@ -0,0 +1,357 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resctrl + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + gocache "github.com/patrickmn/go-cache" + "github.com/stretchr/testify/assert" + + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol" + "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" +) + +type UpdateFunc func(resource ResctrlUpdater) error + +type DefaultTestResctrlProtocolUpdater struct { + name string + hooksProtocol protocol.HooksProtocol + group string + schemata string + updateFunc UpdateFunc +} + +func (u DefaultTestResctrlProtocolUpdater) Name() string { + return u.name +} + +func (u DefaultTestResctrlProtocolUpdater) Key() string { + return u.group +} + +func (u DefaultTestResctrlProtocolUpdater) Value() string { + return u.schemata +} + +func (r *DefaultTestResctrlProtocolUpdater) SetKey(group string) { + r.group = group +} + +func (r *DefaultTestResctrlProtocolUpdater) SetValue(schemata string) { + r.schemata = schemata +} + +func (u *DefaultTestResctrlProtocolUpdater) Update() error { + return u.updateFunc(u) +} + +func NewTestCreateResctrlUpdater(podid string) ResctrlUpdater { + return &DefaultTestResctrlProtocolUpdater{ + name: podid, + updateFunc: createResctrlProtocolUpdaterFunc, + } +} + +func NewTestUpdateResctrlUpdater(podid string, schemata string) ResctrlUpdater { + return &DefaultTestResctrlProtocolUpdater{ + name: podid, + schemata: schemata, + updateFunc: updateResctrlProtocolUpdaterFunc, + } +} + +func NewTestRemoveResctrlUpdater(group string) ResctrlUpdater { + return &DefaultTestResctrlProtocolUpdater{ + group: group, + updateFunc: removeResctrlUpdaterFunc, + } +} + +func createResctrlProtocolUpdaterFunc(u ResctrlUpdater) error { + r, ok := u.(*DefaultTestResctrlProtocolUpdater) + if !ok { + return fmt.Errorf("not a ResctrlSchemataResourceUpdater") + } + r.SetKey(ClosdIdPrefix + r.name) + return nil +} + +func updateResctrlProtocolUpdaterFunc(u ResctrlUpdater) error { + r, ok := u.(*DefaultTestResctrlProtocolUpdater) + if !ok { + return fmt.Errorf("not a ResctrlSchemataResourceUpdater") + } + r.SetValue(r.schemata) + return nil +} + +func removeResctrlUpdaterFunc(u ResctrlUpdater) error { + r, ok := u.(*DefaultTestResctrlProtocolUpdater) + if !ok { + return fmt.Errorf("not a ResctrlSchemataResourceUpdater") + } + r.SetKey("") + r.SetValue("") + return nil +} + +func TestControlGroupManager_AddPod(t *testing.T) { + type fields struct { + rdtcgs *gocache.Cache + reconcileInterval int64 + groupExist bool + } + type args struct { + podid string + schemata string + fromNRI bool + createUpdater ResctrlUpdater + schemataUpdater ResctrlUpdater + } + tests := []struct { + name string + fields fields + args args + }{ + { + name: "AddPod from NRI while ctrl group is not exist", + fields: fields{ + rdtcgs: gocache.New(time.Duration(ExpirationTime), CleanupInterval), + reconcileInterval: 0, + groupExist: false, + }, + args: args{ + podid: "pod1", + schemata: "", + fromNRI: true, + createUpdater: NewTestCreateResctrlUpdater("pod1"), + schemataUpdater: NewTestUpdateResctrlUpdater("pod1", "testschemata"), + }, + }, + { + name: "AddPod from NRI while ctrl group exist", + fields: fields{ + rdtcgs: gocache.New(time.Duration(ExpirationTime), CleanupInterval), + reconcileInterval: 0, + groupExist: true, + }, + args: args{ + podid: "pod2", + schemata: "testschemata", + fromNRI: true, + createUpdater: NewTestCreateResctrlUpdater("pod2"), + schemataUpdater: NewTestUpdateResctrlUpdater("pod2", "testschemata"), + }, + }, + { + name: "AddPod from reconciler while ctrl group is not exist", + fields: fields{ + rdtcgs: gocache.New(time.Duration(ExpirationTime), CleanupInterval), + reconcileInterval: 0, + groupExist: false, + }, + args: args{ + podid: "pod3", + schemata: "", + fromNRI: false, + createUpdater: NewTestCreateResctrlUpdater("pod3"), + schemataUpdater: NewTestUpdateResctrlUpdater("pod3", "testschemata"), + }, + }, + { + name: "AddPod from NRI while ctrl group exist", + fields: fields{ + rdtcgs: gocache.New(time.Duration(ExpirationTime), CleanupInterval), + reconcileInterval: 0, + groupExist: true, + }, + args: args{ + podid: "pod4", + schemata: "testschemata", + fromNRI: false, + createUpdater: NewTestCreateResctrlUpdater("pod4"), + schemataUpdater: NewTestUpdateResctrlUpdater("pod4", "testschemata"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ControlGroupManager{ + rdtcgs: tt.fields.rdtcgs, + reconcileInterval: tt.fields.reconcileInterval, + } + // prepare ctrl group if group exist + if tt.fields.groupExist { + c.rdtcgs.Set(tt.args.podid, &ControlGroup{ + AppId: tt.args.podid, + GroupId: ClosdIdPrefix + tt.args.podid, + Schemata: tt.args.schemata, + Status: Add, + CreatedTime: 0, + }, -1) + } + c.AddPod(tt.args.podid, tt.args.schemata, tt.args.fromNRI, tt.args.createUpdater, tt.args.schemataUpdater) + p, ok := c.rdtcgs.Get(tt.args.podid) + assert.Equal(t, true, ok) + pod := p.(*ControlGroup) + assert.Equal(t, ClosdIdPrefix+tt.args.podid, pod.GroupId) + assert.Equal(t, tt.args.schemata, pod.Schemata) + }) + } +} + +func testingPrepareResctrlL3CatPath(t *testing.T, cbmStr, rootSchemataStr string) { + resctrlDir := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir) + l3CatDir := filepath.Join(resctrlDir, system.RdtInfoDir, system.L3CatDir) + err := os.MkdirAll(l3CatDir, 0700) + assert.NoError(t, err) + + cbmPath := filepath.Join(l3CatDir, system.ResctrlCbmMaskName) + err = os.WriteFile(cbmPath, []byte(cbmStr), 0666) + assert.NoError(t, err) + + schemataPath := filepath.Join(resctrlDir, system.ResctrlSchemataName) + err = os.WriteFile(schemataPath, []byte(rootSchemataStr), 0666) + assert.NoError(t, err) +} + +// @schemataData: schemata for pod1, pod2 +func testingPrepareResctrlL3CatGroups(t *testing.T, cbmStr, rootSchemataStr string, schemataData ...string) { + testingPrepareResctrlL3CatPath(t, cbmStr, rootSchemataStr) + resctrlDir := filepath.Join(system.Conf.SysFSRootDir, system.ResctrlDir) + + pod1SchemataData := []byte(" L3:0=f;1=f\n MB:0=100;1=100") + if len(schemataData) >= 1 { + pod1SchemataData = []byte(schemataData[0]) + } + pod1SchemataDir := filepath.Join(resctrlDir, "koordlet-pod1") + err := os.MkdirAll(pod1SchemataDir, 0700) + assert.NoError(t, err) + pod1SchemataPath := filepath.Join(pod1SchemataDir, system.ResctrlSchemataName) + err = os.WriteFile(pod1SchemataPath, pod1SchemataData, 0666) + assert.NoError(t, err) + pod1TasksPath := filepath.Join(pod1SchemataDir, system.ResctrlTasksName) + err = os.WriteFile(pod1TasksPath, []byte{}, 0666) + assert.NoError(t, err) + + pod2SchemataData := []byte(" L3:0=f;1=f\n MB:0=100;1=100") + if len(schemataData) >= 1 { + pod2SchemataData = []byte(schemataData[1]) + } + pod2SchemataDir := filepath.Join(resctrlDir, "koordlet-pod2") + err = os.MkdirAll(pod2SchemataDir, 0700) + assert.NoError(t, err) + pod2SchemataPath := filepath.Join(pod2SchemataDir, system.ResctrlSchemataName) + err = os.WriteFile(pod2SchemataPath, pod2SchemataData, 0666) + assert.NoError(t, err) + pod2TasksPath := filepath.Join(pod2SchemataDir, system.ResctrlTasksName) + err = os.WriteFile(pod2TasksPath, []byte{}, 0666) + assert.NoError(t, err) +} + +func TestControlGroupManager_Init(t *testing.T) { + type fields struct { + rdtcgs *gocache.Cache + reconcileInterval int64 + schemataData []string + mockSchemata string + } + tests := []struct { + name string + fields fields + }{ + { + name: "rdt cgroup Init", + fields: fields{ + rdtcgs: gocache.New(time.Duration(ExpirationTime), CleanupInterval), + reconcileInterval: 0, + schemataData: []string{"L3:0=f0\nMB:0=100", "L3:0=fc\nMB:0=100"}, + mockSchemata: "L3:0=ff\nMB:0=100\n", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + sysFSRootDirName := "ctrlmgr" + helper.MkDirAll(sysFSRootDirName) + system.Conf.SysFSRootDir = filepath.Join(helper.TempDir, sysFSRootDirName) + testingPrepareResctrlL3CatGroups(t, "ff", tt.fields.mockSchemata, tt.fields.schemataData...) + c := &ControlGroupManager{ + rdtcgs: tt.fields.rdtcgs, + reconcileInterval: tt.fields.reconcileInterval, + } + c.Init() + p, ok := c.rdtcgs.Get("pod1") + assert.Equal(t, true, ok) + pod1 := p.(*ControlGroup) + assert.Equal(t, ClosdIdPrefix+"pod1", pod1.GroupId) + assert.Equal(t, "L3:0=f0\nMB:0=100", pod1.Schemata) + + p, ok = c.rdtcgs.Get("pod2") + assert.Equal(t, true, ok) + pod2 := p.(*ControlGroup) + assert.Equal(t, ClosdIdPrefix+"pod2", pod2.GroupId) + assert.Equal(t, "L3:0=fc\nMB:0=100", pod2.Schemata) + }) + } +} + +func TestControlGroupManager_RemovePod(t *testing.T) { + type fields struct { + rdtcgs *gocache.Cache + reconcileInterval int64 + } + type args struct { + podid string + fromNRI bool + removeUpdater ResctrlUpdater + } + tests := []struct { + name string + fields fields + args args + want bool + }{ + { + name: "RemovePod from NRI while not exist in rdtcgs", + fields: fields{ + rdtcgs: gocache.New(time.Duration(ExpirationTime), CleanupInterval), + reconcileInterval: 0, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ControlGroupManager{ + rdtcgs: tt.fields.rdtcgs, + reconcileInterval: tt.fields.reconcileInterval, + } + if got := c.RemovePod(tt.args.podid, tt.args.fromNRI, tt.args.removeUpdater); got != tt.want { + t.Errorf("RemovePod() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/koordlet/util/resctrl/resctrl.go b/pkg/koordlet/util/resctrl/resctrl.go new file mode 100644 index 000000000..199efe703 --- /dev/null +++ b/pkg/koordlet/util/resctrl/resctrl.go @@ -0,0 +1,318 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resctrl + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "sync" + + "k8s.io/klog/v2" + + apiext "github.com/koordinator-sh/koordinator/apis/extension" + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol" + koordletutil "github.com/koordinator-sh/koordinator/pkg/koordlet/util" + sysutil "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" +) + +const ( + // Max memory bandwidth for AMD CPU, Gb/s, since the extreme limit is hard to reach, we set a discount by 0.8 + // TODO The max memory bandwidth varies across SKU, so koordlet should be aware of the maximum automatically, + // or support an configuration list. + // Currently, the value is measured on "AMD EPYC(TM) MILAN" + + AMDCCDMaxMBGbps = 25 * 8 * 0.8 + + // the AMD CPU use 2048 to express the unlimited memory bandwidth + AMDCCDUnlimitedMB = 2048 +) + +type ResctrlUpdater interface { + Name() string + Key() string + Value() string + Update() error + SetKey(key string) + SetValue(key string) +} + +const ClosdIdPrefix = "koordlet-" + +type App struct { + Resctrl *sysutil.ResctrlSchemataRaw + Closid string + Annotation string +} + +type ResctrlEngine interface { + Rebuild() + RegisterApp(podid, annotation string, fromNRI bool, updater ResctrlUpdater) error + UnRegisterApp(podid string, fromNRI bool, updater ResctrlUpdater) error + GetApp(podid string) (App, bool) + GetApps() map[string]App +} + +func NewRDTEngine(vendor string) (ResctrlEngine, error) { + var CatL3CbmMask string + var err error + if CatL3CbmMask, err = sysutil.ReadCatL3CbmString(); err != nil { + klog.Errorf("get l3 cache bit mask error: %v", err) + return nil, err + } + + if len(CatL3CbmMask) <= 0 { + return nil, fmt.Errorf("failed to get cat l3 cbm, cbm is empty") + } + cbmValue, err := strconv.ParseUint(CatL3CbmMask, 16, 32) + if err != nil { + return nil, fmt.Errorf("failed to parse cat l3 cbm %s, err: %v", CatL3CbmMask, err) + } + cbm := uint(cbmValue) + + return &RDTEngine{ + Apps: make(map[string]App), + CtrlGroups: make(map[string]apiext.Resctrl), + CBM: cbm, + Cgm: NewControlGroupManager(), + Vendor: vendor, + }, nil +} + +type RDTEngine struct { + Apps map[string]App + Cgm ControlGroupManager + CtrlGroups map[string]apiext.Resctrl + l sync.RWMutex + CBM uint + Vendor string +} + +func (R *RDTEngine) GetApps() map[string]App { + R.l.RLock() + defer R.l.RUnlock() + apps := make(map[string]App) + for podid, app := range R.Apps { + apps[podid] = app + } + return apps +} + +func (R *RDTEngine) Rebuild() { + R.l.RLock() + defer R.l.RUnlock() + R.Cgm.Init() + for podid, item := range R.Cgm.rdtcgs.Items() { + v, ok := item.Object.(*ControlGroup) + if !ok { + continue + } + + ids, _ := sysutil.CacheIdsCacheFunc() + schemataRaw := sysutil.NewResctrlSchemataRaw(ids).WithL3Num(len(ids)) + err := schemataRaw.ParseResctrlSchemata(v.Schemata, -1) + if err != nil { + klog.Errorf("failed to parse %v", err) + } + R.Apps[podid] = App{ + Resctrl: schemataRaw, + Closid: v.GroupId, + } + } +} + +func (R *RDTEngine) RegisterApp(podid, annotation string, fromNRI bool, updater ResctrlUpdater) error { + R.l.Lock() + defer R.l.Unlock() + var res apiext.ResctrlConfig + err := json.Unmarshal([]byte(annotation), &res) + if err != nil { + klog.Errorf("error is %v", err) + return err + } + + schemata := R.ParseSchemata(res, R.CBM) + app := App{ + Resctrl: schemata, + Closid: ClosdIdPrefix + podid, + Annotation: annotation, + } + + items := []string{} + for _, item := range []struct { + validFunc func() (bool, string) + value func() string + }{ + {validFunc: app.Resctrl.ValidateL3, value: app.Resctrl.L3String}, + {validFunc: app.Resctrl.ValidateMB, value: app.Resctrl.MBString}, + } { + if valid, _ := item.validFunc(); valid { + items = append(items, item.value()) + } + } + schemataStr := strings.Join(items, "") + if updater != nil { + updater.SetKey(ClosdIdPrefix + podid) + updater.SetValue(schemataStr) + } + R.Cgm.AddPod(podid, schemataStr, fromNRI, updater, nil) + + R.Apps[podid] = app + return nil +} + +func (R *RDTEngine) UnRegisterApp(podid string, fromNRI bool, updater ResctrlUpdater) error { + R.l.Lock() + defer R.l.Unlock() + + if _, ok := R.Apps[podid]; !ok { + return fmt.Errorf("pod %s not registered", podid) + } + removed := R.Cgm.RemovePod(podid, fromNRI, updater) + if removed { + delete(R.Apps, podid) + } + + return nil +} + +func (R *RDTEngine) GetApp(id string) (App, bool) { + R.l.RLock() + defer R.l.RUnlock() + + if v, ok := R.Apps[id]; ok { + return v, true + } else { + return App{}, false + } +} + +func (R *RDTEngine) calculateMba(mbaPercent int64) int64 { + if R.Vendor == sysutil.INTEL_VENDOR_ID { + return calculateIntelMba(mbaPercent) + } else if R.Vendor == sysutil.AMD_VENDOR_ID { + return calculateAMDMba(mbaPercent) + } + return 0 +} + +func calculateIntelMba(mbaPercent int64) int64 { + if mbaPercent%10 != 0 { + actualPercent := mbaPercent/10*10 + 10 + klog.V(4).Infof("cat MBA must multiple of 10, mbaPercentConfig is %d, actualMBAPercent will be %d", + mbaPercent, actualPercent) + return actualPercent + } + + return mbaPercent +} + +func calculateAMDMba(mbaPercent int64) int64 { + if mbaPercent == 100 { + return AMDCCDUnlimitedMB + } + mbaLimitValue := float64(AMDCCDMaxMBGbps*mbaPercent) / 100 + return int64(mbaLimitValue) +} + +func (R *RDTEngine) ParseSchemata(config apiext.ResctrlConfig, cbm uint) *sysutil.ResctrlSchemataRaw { + ids, _ := sysutil.CacheIdsCacheFunc() + schemataRaw := sysutil.NewResctrlSchemataRaw(ids).WithL3Num(len(ids)) + if config.MB.Schemata.Percent != 0 { + percent := R.calculateMba(int64(config.MB.Schemata.Percent)) + for k := range schemataRaw.MB { + schemataRaw.MB[k] = percent + } + } + + if config.MB.SchemataPerCache != nil { + for _, v := range config.MB.SchemataPerCache { + percent := R.calculateMba(int64(v.Percent)) + schemataRaw.MB[v.CacheID] = percent + } + } + + if config.LLC.Schemata.Range != nil && len(config.LLC.Schemata.Range) == 2 { + start := config.LLC.Schemata.Range[0] + end := config.LLC.Schemata.Range[1] + + l3MaskValue, err := sysutil.CalculateCatL3MaskValue(cbm, int64(start), int64(end)) + if err != nil { + klog.Warningf("failed to calculate l3 cat schemata err: %v", err) + return schemataRaw + } + + schemataRaw.WithL3Num(len(ids)).WithL3Mask(l3MaskValue) + } + + if config.LLC.SchemataPerCache != nil { + for _, v := range config.LLC.SchemataPerCache { + if len(v.Range) == 2 { + start := v.Range[0] + end := v.Range[1] + l3MaskValue, err := sysutil.CalculateCatL3MaskValue(cbm, int64(start), int64(end)) + if err != nil { + klog.Warningf("failed to calculate l3 cat schemata err: %v", err) + return schemataRaw + } + // l3 mask MUST be a valid hex + maskValue, err := strconv.ParseInt(strings.TrimSpace(l3MaskValue), 16, 64) + if err != nil { + klog.V(5).Infof("failed to parse l3 mask %s, err: %v", l3MaskValue, err) + } + schemataRaw.L3[v.CacheID] = maskValue + } + } + } + return schemataRaw +} + +func GetPodCgroupNewTaskIdsFromPodCtx(podMeta *protocol.PodContext, tasksMap map[int32]struct{}) []int32 { + var taskIds []int32 + + for containerId, v := range podMeta.Request.ContainerTaskIds { + containerDir, err := koordletutil.GetContainerCgroupParentDirByID(podMeta.Request.CgroupParent, containerId) + if err != nil { + klog.Errorf("container %s lost during reconcile", containerDir) + continue + } + ids, err := GetNewTaskIds(v, tasksMap) + if err != nil { + klog.Warningf("failed to get pod container cgroup task ids for container %s/%s/%s, err: %s", + podMeta.Request.PodMeta.Name, containerId) + continue + } + taskIds = append(taskIds, ids...) + } + return taskIds +} + +func GetNewTaskIds(ids []int32, tasksMap map[int32]struct{}) ([]int32, error) { + if tasksMap == nil { + return ids, nil + } + + // only append the non-mapped ids + var taskIDs []int32 + for _, id := range ids { + if _, ok := tasksMap[id]; !ok { + taskIDs = append(taskIDs, id) + } + } + return taskIDs, nil +} diff --git a/pkg/koordlet/util/system/resctrl.go b/pkg/koordlet/util/system/resctrl.go index 26c866f0c..d4ed33682 100644 --- a/pkg/koordlet/util/system/resctrl.go +++ b/pkg/koordlet/util/system/resctrl.go @@ -51,7 +51,8 @@ const ( MbSchemataPrefix = "MB" // other cpu vendor like "GenuineIntel" - AMD_VENDOR_ID = "AuthenticAMD" + AMD_VENDOR_ID = "AuthenticAMD" + INTEL_VENDOR_ID = "GenuineIntel" ) var ( @@ -112,6 +113,7 @@ func IsSupportResctrl() (bool, error) { } var ( + ResctrlRoot = NewCommonResctrlResource("", "") ResctrlSchemata = NewCommonResctrlResource(ResctrlSchemataName, "") ResctrlTasks = NewCommonResctrlResource(ResctrlTasksName, "") ResctrlL3CbmMask = NewCommonResctrlResource(ResctrlCbmMaskName, filepath.Join(RdtInfoDir, L3CatDir)) From 91547ac56d75777369518e6fc30a3ad501322d00 Mon Sep 17 00:00:00 2001 From: Zhang Kang Date: Wed, 17 Jul 2024 10:59:55 +0800 Subject: [PATCH 02/10] add event recorder Signed-off-by: Zhang Kang --- pkg/koordlet/runtimehooks/nri/server.go | 2 ++ pkg/koordlet/runtimehooks/proxyserver/server.go | 3 +++ pkg/koordlet/runtimehooks/runtimehooks.go | 2 ++ 3 files changed, 7 insertions(+) diff --git a/pkg/koordlet/runtimehooks/nri/server.go b/pkg/koordlet/runtimehooks/nri/server.go index b80aa3b5a..5630a9f87 100644 --- a/pkg/koordlet/runtimehooks/nri/server.go +++ b/pkg/koordlet/runtimehooks/nri/server.go @@ -19,6 +19,7 @@ package nri import ( "context" "fmt" + "k8s.io/client-go/tools/record" "path/filepath" "strings" "time" @@ -52,6 +53,7 @@ type Options struct { DisableStages map[string]struct{} Executor resourceexecutor.ResourceUpdateExecutor BackOff wait.Backoff + EventRecorder record.EventRecorder } func (o Options) Validate() error { diff --git a/pkg/koordlet/runtimehooks/proxyserver/server.go b/pkg/koordlet/runtimehooks/proxyserver/server.go index 7bd3b4bad..034e1480c 100644 --- a/pkg/koordlet/runtimehooks/proxyserver/server.go +++ b/pkg/koordlet/runtimehooks/proxyserver/server.go @@ -19,6 +19,7 @@ package proxyserver import ( "encoding/json" "fmt" + "k8s.io/client-go/tools/record" "net" "path/filepath" "syscall" @@ -44,6 +45,8 @@ type Options struct { ConfigFilePath string DisableStages map[string]struct{} Executor resourceexecutor.ResourceUpdateExecutor + EventRecorder record.EventRecorder + } type Server interface { diff --git a/pkg/koordlet/runtimehooks/runtimehooks.go b/pkg/koordlet/runtimehooks/runtimehooks.go index bdae743bf..dc98fc6ba 100644 --- a/pkg/koordlet/runtimehooks/runtimehooks.go +++ b/pkg/koordlet/runtimehooks/runtimehooks.go @@ -110,6 +110,7 @@ func NewRuntimeHook(si statesinformer.StatesInformer, cfg *Config, schema *apiru ConfigFilePath: cfg.RuntimeHookConfigFilePath, DisableStages: getDisableStagesMap(cfg.RuntimeHookDisableStages), Executor: e, + EventRecorder: recorder, } backOff := wait.Backoff{ @@ -130,6 +131,7 @@ func NewRuntimeHook(si statesinformer.StatesInformer, cfg *Config, schema *apiru DisableStages: getDisableStagesMap(cfg.RuntimeHookDisableStages), Executor: e, BackOff: backOff, + EventRecorder: recorder, } nriServer, err = nri.NewNriServer(nriServerOptions) if err != nil { From 92219c9f61755662a5ea50508a2661706384bb2d Mon Sep 17 00:00:00 2001 From: Zhang Kang Date: Wed, 17 Jul 2024 11:26:31 +0800 Subject: [PATCH 03/10] fmt Signed-off-by: Zhang Kang --- pkg/koordlet/runtimehooks/nri/server.go | 2 +- pkg/koordlet/runtimehooks/proxyserver/server.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/koordlet/runtimehooks/nri/server.go b/pkg/koordlet/runtimehooks/nri/server.go index 5630a9f87..e7b151c77 100644 --- a/pkg/koordlet/runtimehooks/nri/server.go +++ b/pkg/koordlet/runtimehooks/nri/server.go @@ -53,7 +53,7 @@ type Options struct { DisableStages map[string]struct{} Executor resourceexecutor.ResourceUpdateExecutor BackOff wait.Backoff - EventRecorder record.EventRecorder + EventRecorder record.EventRecorder } func (o Options) Validate() error { diff --git a/pkg/koordlet/runtimehooks/proxyserver/server.go b/pkg/koordlet/runtimehooks/proxyserver/server.go index 034e1480c..d177a76aa 100644 --- a/pkg/koordlet/runtimehooks/proxyserver/server.go +++ b/pkg/koordlet/runtimehooks/proxyserver/server.go @@ -46,7 +46,6 @@ type Options struct { DisableStages map[string]struct{} Executor resourceexecutor.ResourceUpdateExecutor EventRecorder record.EventRecorder - } type Server interface { From 388cb54e2ebea7a1e2d7069ebab03dceb1bebd24 Mon Sep 17 00:00:00 2001 From: "Zhang, Kang" Date: Fri, 26 Jul 2024 15:31:20 +0800 Subject: [PATCH 04/10] format --- pkg/koordlet/runtimehooks/hooks/hooks.go | 2 ++ pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go | 4 +++- pkg/koordlet/runtimehooks/nri/server.go | 2 +- pkg/koordlet/runtimehooks/proxyserver/server.go | 2 +- pkg/koordlet/runtimehooks/runtimehooks.go | 5 +++-- pkg/koordlet/runtimehooks/runtimehooks_test.go | 2 ++ pkg/koordlet/util/resctrl/ctrl_mgr.go | 6 +++++- pkg/koordlet/util/resctrl/resctrl.go | 5 ++++- 8 files changed, 21 insertions(+), 7 deletions(-) diff --git a/pkg/koordlet/runtimehooks/hooks/hooks.go b/pkg/koordlet/runtimehooks/hooks/hooks.go index d8437066a..0de707460 100644 --- a/pkg/koordlet/runtimehooks/hooks/hooks.go +++ b/pkg/koordlet/runtimehooks/hooks/hooks.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "github.com/koordinator-sh/koordinator/pkg/koordlet/metrics" @@ -40,6 +41,7 @@ type Options struct { Reader resourceexecutor.CgroupReader Executor resourceexecutor.ResourceUpdateExecutor StatesInformer statesinformer.StatesInformer + EventRecorder record.EventRecorder } type HookFn func(protocol.HooksProtocol) error diff --git a/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go b/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go index 379b72629..eacda071a 100644 --- a/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go +++ b/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go @@ -18,6 +18,7 @@ package resctrl import ( "fmt" + "k8s.io/client-go/tools/record" "strings" "k8s.io/klog/v2" @@ -46,6 +47,7 @@ type plugin struct { engine util.ResctrlEngine executor resourceexecutor.ResourceUpdateExecutor statesInformer statesinformer.StatesInformer + EventRecorder record.EventRecorder } var singleton *plugin @@ -82,6 +84,7 @@ func (p *plugin) Register(op hooks.Options) { } p.executor = op.Executor p.statesInformer = op.StatesInformer + p.EventRecorder = op.EventRecorder p.engine.Rebuild() rule.Register(ruleNameForAllPods, description, @@ -95,7 +98,6 @@ func (p *plugin) Register(op hooks.Options) { reconciler.RegisterCgroupReconciler(reconciler.PodLevel, system.ResctrlSchemata, description+" (pod resctrl schema)", p.SetPodResctrlResourcesForReconciler, reconciler.NoneFilter()) reconciler.RegisterCgroupReconciler(reconciler.PodLevel, system.ResctrlTasks, description+" (pod resctrl tasks)", p.UpdatePodTaskIds, reconciler.NoneFilter()) reconciler.RegisterCgroupReconciler4AllPods(reconciler.AllPodsLevel, system.ResctrlRoot, description+" (pod resctl schema)", p.RemoveUnusedResctrlPath, reconciler.PodAnnotationResctrlFilter(), "resctrl") - } func (p *plugin) SetPodResctrlResourcesForHooks(proto protocol.HooksProtocol) error { diff --git a/pkg/koordlet/runtimehooks/nri/server.go b/pkg/koordlet/runtimehooks/nri/server.go index e7b151c77..50bd5e1ba 100644 --- a/pkg/koordlet/runtimehooks/nri/server.go +++ b/pkg/koordlet/runtimehooks/nri/server.go @@ -19,7 +19,6 @@ package nri import ( "context" "fmt" - "k8s.io/client-go/tools/record" "path/filepath" "strings" "time" @@ -28,6 +27,7 @@ import ( "github.com/containerd/nri/pkg/stub" "go.uber.org/atomic" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "sigs.k8s.io/yaml" diff --git a/pkg/koordlet/runtimehooks/proxyserver/server.go b/pkg/koordlet/runtimehooks/proxyserver/server.go index d177a76aa..4f0b49f45 100644 --- a/pkg/koordlet/runtimehooks/proxyserver/server.go +++ b/pkg/koordlet/runtimehooks/proxyserver/server.go @@ -19,13 +19,13 @@ package proxyserver import ( "encoding/json" "fmt" - "k8s.io/client-go/tools/record" "net" "path/filepath" "syscall" "google.golang.org/grpc" "google.golang.org/grpc/reflection" + "k8s.io/client-go/tools/record" "k8s.io/klog/v2" runtimeapi "github.com/koordinator-sh/koordinator/apis/runtime/v1alpha1" diff --git a/pkg/koordlet/runtimehooks/runtimehooks.go b/pkg/koordlet/runtimehooks/runtimehooks.go index dc98fc6ba..8b7068000 100644 --- a/pkg/koordlet/runtimehooks/runtimehooks.go +++ b/pkg/koordlet/runtimehooks/runtimehooks.go @@ -150,8 +150,9 @@ func NewRuntimeHook(si statesinformer.StatesInformer, cfg *Config, schema *apiru } newPluginOptions := hooks.Options{ - Reader: cr, - Executor: e, + Reader: cr, + Executor: e, + EventRecorder: recorder, } if err != nil { diff --git a/pkg/koordlet/runtimehooks/runtimehooks_test.go b/pkg/koordlet/runtimehooks/runtimehooks_test.go index 545b875ee..4a3d2b584 100644 --- a/pkg/koordlet/runtimehooks/runtimehooks_test.go +++ b/pkg/koordlet/runtimehooks/runtimehooks_test.go @@ -17,6 +17,8 @@ limitations under the License. package runtimehooks import ( + apiruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" "path" "testing" "time" diff --git a/pkg/koordlet/util/resctrl/ctrl_mgr.go b/pkg/koordlet/util/resctrl/ctrl_mgr.go index d79dda9bc..568c3a5c3 100644 --- a/pkg/koordlet/util/resctrl/ctrl_mgr.go +++ b/pkg/koordlet/util/resctrl/ctrl_mgr.go @@ -106,7 +106,7 @@ func (c *ControlGroupManager) Init() { } } -func (c *ControlGroupManager) AddPod(podid string, schemata string, fromNRI bool, createUpdater ResctrlUpdater, schemataUpdater ResctrlUpdater) { +func (c *ControlGroupManager) AddPod(podid string, schemata string, fromNRI bool, createUpdater ResctrlUpdater, schemataUpdater ResctrlUpdater) error { c.Lock() defer c.Unlock() @@ -128,6 +128,7 @@ func (c *ControlGroupManager) AddPod(podid string, schemata string, fromNRI bool err := createUpdater.Update() if err != nil { klog.Errorf("create ctrl group error %v", err) + return err } else { pod.GroupId = ClosdIdPrefix + podid pod.CreatedTime = time.Now().UnixNano() @@ -138,6 +139,7 @@ func (c *ControlGroupManager) AddPod(podid string, schemata string, fromNRI bool err := schemataUpdater.Update() if err != nil { klog.Errorf("updater ctrl group schemata error %v", err) + return err } pod.Schemata = schemata } @@ -150,12 +152,14 @@ func (c *ControlGroupManager) AddPod(podid string, schemata string, fromNRI bool err := schemataUpdater.Update() if err != nil { klog.Errorf("updater ctrl group schemata error %v", err) + return err } pod.Schemata = schemata } c.rdtcgs.Set(podid, pod, -1) } } + return nil } func (c *ControlGroupManager) RemovePod(podid string, fromNRI bool, removeUpdater ResctrlUpdater) bool { diff --git a/pkg/koordlet/util/resctrl/resctrl.go b/pkg/koordlet/util/resctrl/resctrl.go index 199efe703..a8d707910 100644 --- a/pkg/koordlet/util/resctrl/resctrl.go +++ b/pkg/koordlet/util/resctrl/resctrl.go @@ -170,7 +170,10 @@ func (R *RDTEngine) RegisterApp(podid, annotation string, fromNRI bool, updater updater.SetKey(ClosdIdPrefix + podid) updater.SetValue(schemataStr) } - R.Cgm.AddPod(podid, schemataStr, fromNRI, updater, nil) + err = R.Cgm.AddPod(podid, schemataStr, fromNRI, updater, nil) + if err != nil { + return err + } R.Apps[podid] = app return nil From 32c7fe4710bf6feff414d9478c4f8c00ac370142 Mon Sep 17 00:00:00 2001 From: "Zhang, Kang" Date: Fri, 26 Jul 2024 16:18:16 +0800 Subject: [PATCH 05/10] format --- pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go b/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go index eacda071a..eb991ab35 100644 --- a/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go +++ b/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl.go @@ -18,9 +18,9 @@ package resctrl import ( "fmt" - "k8s.io/client-go/tools/record" "strings" + "k8s.io/client-go/tools/record" "k8s.io/klog/v2" apiext "github.com/koordinator-sh/koordinator/apis/extension" From 5c9f1c5b1355b65d0ffa3824d994fc4d00fbc2ba Mon Sep 17 00:00:00 2001 From: "Zhang, Kang" Date: Fri, 26 Jul 2024 16:30:21 +0800 Subject: [PATCH 06/10] fix lint --- pkg/koordlet/util/resctrl/resctrl_test.go | 368 ++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 pkg/koordlet/util/resctrl/resctrl_test.go diff --git a/pkg/koordlet/util/resctrl/resctrl_test.go b/pkg/koordlet/util/resctrl/resctrl_test.go new file mode 100644 index 000000000..0ca04ad00 --- /dev/null +++ b/pkg/koordlet/util/resctrl/resctrl_test.go @@ -0,0 +1,368 @@ +package resctrl + +import ( + "fmt" + "sync" + "testing" + + "github.com/koordinator-sh/koordinator/apis/extension" + apiext "github.com/koordinator-sh/koordinator/apis/extension" + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol" + sysutil "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" + "github.com/stretchr/testify/assert" +) + +func TestGetNewTaskIds(t *testing.T) { + type args struct { + ids []int32 + tasksMap map[int32]struct{} + } + tests := []struct { + name string + args args + want []int32 + wantErr assert.ErrorAssertionFunc + }{ + { + name: "tasksMap is nil", + args: args{ + ids: []int32{1, 2, 3}, + tasksMap: nil, + }, + want: []int32{1, 2, 3}, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetNewTaskIds(tt.args.ids, tt.args.tasksMap) + if !tt.wantErr(t, err, fmt.Sprintf("GetNewTaskIds(%v, %v)", tt.args.ids, tt.args.tasksMap)) { + return + } + assert.Equalf(t, tt.want, got, "GetNewTaskIds(%v, %v)", tt.args.ids, tt.args.tasksMap) + }) + } +} + +func TestGetPodCgroupNewTaskIdsFromPodCtx(t *testing.T) { + type args struct { + podMeta *protocol.PodContext + tasksMap map[int32]struct{} + } + tests := []struct { + name string + args args + want []int32 + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, GetPodCgroupNewTaskIdsFromPodCtx(tt.args.podMeta, tt.args.tasksMap), "GetPodCgroupNewTaskIdsFromPodCtx(%v, %v)", tt.args.podMeta, tt.args.tasksMap) + }) + } +} + +func TestNewRDTEngine(t *testing.T) { + type args struct { + vendor string + } + tests := []struct { + name string + args args + want ResctrlEngine + wantErr assert.ErrorAssertionFunc + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewRDTEngine(tt.args.vendor) + if !tt.wantErr(t, err, fmt.Sprintf("NewRDTEngine(%v)", tt.args.vendor)) { + return + } + assert.Equalf(t, tt.want, got, "NewRDTEngine(%v)", tt.args.vendor) + }) + } +} + +func TestRDTEngine_GetApp(t *testing.T) { + type fields struct { + Apps map[string]App + Cgm ControlGroupManager + CtrlGroups map[string]apiext.Resctrl + l sync.RWMutex + CBM uint + Vendor string + } + type args struct { + id string + } + tests := []struct { + name string + fields fields + args args + want App + want1 bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + R := &RDTEngine{ + Apps: tt.fields.Apps, + Cgm: tt.fields.Cgm, + CtrlGroups: tt.fields.CtrlGroups, + l: tt.fields.l, + CBM: tt.fields.CBM, + Vendor: tt.fields.Vendor, + } + got, got1 := R.GetApp(tt.args.id) + assert.Equalf(t, tt.want, got, "GetApp(%v)", tt.args.id) + assert.Equalf(t, tt.want1, got1, "GetApp(%v)", tt.args.id) + }) + } +} + +func TestRDTEngine_GetApps(t *testing.T) { + type fields struct { + Apps map[string]App + Cgm ControlGroupManager + CtrlGroups map[string]apiext.Resctrl + l sync.RWMutex + CBM uint + Vendor string + } + tests := []struct { + name string + fields fields + want map[string]App + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + R := &RDTEngine{ + Apps: tt.fields.Apps, + Cgm: tt.fields.Cgm, + CtrlGroups: tt.fields.CtrlGroups, + l: tt.fields.l, + CBM: tt.fields.CBM, + Vendor: tt.fields.Vendor, + } + assert.Equalf(t, tt.want, R.GetApps(), "GetApps()") + }) + } +} + +func TestRDTEngine_ParseSchemata(t *testing.T) { + type fields struct { + Apps map[string]App + Cgm ControlGroupManager + CtrlGroups map[string]apiext.Resctrl + l sync.RWMutex + CBM uint + Vendor string + } + type args struct { + config extension.ResctrlConfig + cbm uint + } + tests := []struct { + name string + fields fields + args args + want *sysutil.ResctrlSchemataRaw + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + R := &RDTEngine{ + Apps: tt.fields.Apps, + Cgm: tt.fields.Cgm, + CtrlGroups: tt.fields.CtrlGroups, + l: tt.fields.l, + CBM: tt.fields.CBM, + Vendor: tt.fields.Vendor, + } + assert.Equalf(t, tt.want, R.ParseSchemata(tt.args.config, tt.args.cbm), "ParseSchemata(%v, %v)", tt.args.config, tt.args.cbm) + }) + } +} + +func TestRDTEngine_Rebuild(t *testing.T) { + type fields struct { + Apps map[string]App + Cgm ControlGroupManager + CtrlGroups map[string]apiext.Resctrl + l sync.RWMutex + CBM uint + Vendor string + } + tests := []struct { + name string + fields fields + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + R := &RDTEngine{ + Apps: tt.fields.Apps, + Cgm: tt.fields.Cgm, + CtrlGroups: tt.fields.CtrlGroups, + l: tt.fields.l, + CBM: tt.fields.CBM, + Vendor: tt.fields.Vendor, + } + R.Rebuild() + }) + } +} + +func TestRDTEngine_RegisterApp(t *testing.T) { + type fields struct { + Apps map[string]App + Cgm ControlGroupManager + CtrlGroups map[string]apiext.Resctrl + l sync.RWMutex + CBM uint + Vendor string + } + type args struct { + podid string + annotation string + fromNRI bool + updater ResctrlUpdater + } + tests := []struct { + name string + fields fields + args args + wantErr assert.ErrorAssertionFunc + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + R := &RDTEngine{ + Apps: tt.fields.Apps, + Cgm: tt.fields.Cgm, + CtrlGroups: tt.fields.CtrlGroups, + l: tt.fields.l, + CBM: tt.fields.CBM, + Vendor: tt.fields.Vendor, + } + tt.wantErr(t, R.RegisterApp(tt.args.podid, tt.args.annotation, tt.args.fromNRI, tt.args.updater), fmt.Sprintf("RegisterApp(%v, %v, %v, %v)", tt.args.podid, tt.args.annotation, tt.args.fromNRI, tt.args.updater)) + }) + } +} + +func TestRDTEngine_UnRegisterApp(t *testing.T) { + type fields struct { + Apps map[string]App + Cgm ControlGroupManager + CtrlGroups map[string]apiext.Resctrl + l sync.RWMutex + CBM uint + Vendor string + } + type args struct { + podid string + fromNRI bool + updater ResctrlUpdater + } + tests := []struct { + name string + fields fields + args args + wantErr assert.ErrorAssertionFunc + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + R := &RDTEngine{ + Apps: tt.fields.Apps, + Cgm: tt.fields.Cgm, + CtrlGroups: tt.fields.CtrlGroups, + l: tt.fields.l, + CBM: tt.fields.CBM, + Vendor: tt.fields.Vendor, + } + tt.wantErr(t, R.UnRegisterApp(tt.args.podid, tt.args.fromNRI, tt.args.updater), fmt.Sprintf("UnRegisterApp(%v, %v, %v)", tt.args.podid, tt.args.fromNRI, tt.args.updater)) + }) + } +} + +func TestRDTEngine_calculateMba(t *testing.T) { + type fields struct { + Apps map[string]App + Cgm ControlGroupManager + CtrlGroups map[string]apiext.Resctrl + l sync.RWMutex + CBM uint + Vendor string + } + type args struct { + mbaPercent int64 + } + tests := []struct { + name string + fields fields + args args + want int64 + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + R := &RDTEngine{ + Apps: tt.fields.Apps, + Cgm: tt.fields.Cgm, + CtrlGroups: tt.fields.CtrlGroups, + l: tt.fields.l, + CBM: tt.fields.CBM, + Vendor: tt.fields.Vendor, + } + assert.Equalf(t, tt.want, R.calculateMba(tt.args.mbaPercent), "calculateMba(%v)", tt.args.mbaPercent) + }) + } +} + +func Test_calculateAMDMba(t *testing.T) { + type args struct { + mbaPercent int64 + } + tests := []struct { + name string + args args + want int64 + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, calculateAMDMba(tt.args.mbaPercent), "calculateAMDMba(%v)", tt.args.mbaPercent) + }) + } +} + +func Test_calculateIntelMba(t *testing.T) { + type args struct { + mbaPercent int64 + } + tests := []struct { + name string + args args + want int64 + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, calculateIntelMba(tt.args.mbaPercent), "calculateIntelMba(%v)", tt.args.mbaPercent) + }) + } +} From e8019fe73305b21d4d2ac32bfc391e289485b732 Mon Sep 17 00:00:00 2001 From: "Zhang, Kang" Date: Fri, 26 Jul 2024 16:42:51 +0800 Subject: [PATCH 07/10] fix lint --- .../runtimehooks/runtimehooks_test.go | 2 - pkg/koordlet/util/resctrl/resctrl_test.go | 339 +----------------- 2 files changed, 11 insertions(+), 330 deletions(-) diff --git a/pkg/koordlet/runtimehooks/runtimehooks_test.go b/pkg/koordlet/runtimehooks/runtimehooks_test.go index 4a3d2b584..545b875ee 100644 --- a/pkg/koordlet/runtimehooks/runtimehooks_test.go +++ b/pkg/koordlet/runtimehooks/runtimehooks_test.go @@ -17,8 +17,6 @@ limitations under the License. package runtimehooks import ( - apiruntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes" "path" "testing" "time" diff --git a/pkg/koordlet/util/resctrl/resctrl_test.go b/pkg/koordlet/util/resctrl/resctrl_test.go index 0ca04ad00..7b8fa4578 100644 --- a/pkg/koordlet/util/resctrl/resctrl_test.go +++ b/pkg/koordlet/util/resctrl/resctrl_test.go @@ -2,13 +2,8 @@ package resctrl import ( "fmt" - "sync" "testing" - "github.com/koordinator-sh/koordinator/apis/extension" - apiext "github.com/koordinator-sh/koordinator/apis/extension" - "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol" - sysutil "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" "github.com/stretchr/testify/assert" ) @@ -32,6 +27,17 @@ func TestGetNewTaskIds(t *testing.T) { want: []int32{1, 2, 3}, wantErr: assert.NoError, }, + { + name: "taskMap is not nil", + args: args{ + ids: []int32{1, 2, 3}, + tasksMap: map[int32]struct{}{ + 1: struct{}{}, + }, + }, + want: []int32{2, 3}, + wantErr: assert.NoError, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -43,326 +49,3 @@ func TestGetNewTaskIds(t *testing.T) { }) } } - -func TestGetPodCgroupNewTaskIdsFromPodCtx(t *testing.T) { - type args struct { - podMeta *protocol.PodContext - tasksMap map[int32]struct{} - } - tests := []struct { - name string - args args - want []int32 - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, GetPodCgroupNewTaskIdsFromPodCtx(tt.args.podMeta, tt.args.tasksMap), "GetPodCgroupNewTaskIdsFromPodCtx(%v, %v)", tt.args.podMeta, tt.args.tasksMap) - }) - } -} - -func TestNewRDTEngine(t *testing.T) { - type args struct { - vendor string - } - tests := []struct { - name string - args args - want ResctrlEngine - wantErr assert.ErrorAssertionFunc - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := NewRDTEngine(tt.args.vendor) - if !tt.wantErr(t, err, fmt.Sprintf("NewRDTEngine(%v)", tt.args.vendor)) { - return - } - assert.Equalf(t, tt.want, got, "NewRDTEngine(%v)", tt.args.vendor) - }) - } -} - -func TestRDTEngine_GetApp(t *testing.T) { - type fields struct { - Apps map[string]App - Cgm ControlGroupManager - CtrlGroups map[string]apiext.Resctrl - l sync.RWMutex - CBM uint - Vendor string - } - type args struct { - id string - } - tests := []struct { - name string - fields fields - args args - want App - want1 bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - R := &RDTEngine{ - Apps: tt.fields.Apps, - Cgm: tt.fields.Cgm, - CtrlGroups: tt.fields.CtrlGroups, - l: tt.fields.l, - CBM: tt.fields.CBM, - Vendor: tt.fields.Vendor, - } - got, got1 := R.GetApp(tt.args.id) - assert.Equalf(t, tt.want, got, "GetApp(%v)", tt.args.id) - assert.Equalf(t, tt.want1, got1, "GetApp(%v)", tt.args.id) - }) - } -} - -func TestRDTEngine_GetApps(t *testing.T) { - type fields struct { - Apps map[string]App - Cgm ControlGroupManager - CtrlGroups map[string]apiext.Resctrl - l sync.RWMutex - CBM uint - Vendor string - } - tests := []struct { - name string - fields fields - want map[string]App - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - R := &RDTEngine{ - Apps: tt.fields.Apps, - Cgm: tt.fields.Cgm, - CtrlGroups: tt.fields.CtrlGroups, - l: tt.fields.l, - CBM: tt.fields.CBM, - Vendor: tt.fields.Vendor, - } - assert.Equalf(t, tt.want, R.GetApps(), "GetApps()") - }) - } -} - -func TestRDTEngine_ParseSchemata(t *testing.T) { - type fields struct { - Apps map[string]App - Cgm ControlGroupManager - CtrlGroups map[string]apiext.Resctrl - l sync.RWMutex - CBM uint - Vendor string - } - type args struct { - config extension.ResctrlConfig - cbm uint - } - tests := []struct { - name string - fields fields - args args - want *sysutil.ResctrlSchemataRaw - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - R := &RDTEngine{ - Apps: tt.fields.Apps, - Cgm: tt.fields.Cgm, - CtrlGroups: tt.fields.CtrlGroups, - l: tt.fields.l, - CBM: tt.fields.CBM, - Vendor: tt.fields.Vendor, - } - assert.Equalf(t, tt.want, R.ParseSchemata(tt.args.config, tt.args.cbm), "ParseSchemata(%v, %v)", tt.args.config, tt.args.cbm) - }) - } -} - -func TestRDTEngine_Rebuild(t *testing.T) { - type fields struct { - Apps map[string]App - Cgm ControlGroupManager - CtrlGroups map[string]apiext.Resctrl - l sync.RWMutex - CBM uint - Vendor string - } - tests := []struct { - name string - fields fields - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - R := &RDTEngine{ - Apps: tt.fields.Apps, - Cgm: tt.fields.Cgm, - CtrlGroups: tt.fields.CtrlGroups, - l: tt.fields.l, - CBM: tt.fields.CBM, - Vendor: tt.fields.Vendor, - } - R.Rebuild() - }) - } -} - -func TestRDTEngine_RegisterApp(t *testing.T) { - type fields struct { - Apps map[string]App - Cgm ControlGroupManager - CtrlGroups map[string]apiext.Resctrl - l sync.RWMutex - CBM uint - Vendor string - } - type args struct { - podid string - annotation string - fromNRI bool - updater ResctrlUpdater - } - tests := []struct { - name string - fields fields - args args - wantErr assert.ErrorAssertionFunc - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - R := &RDTEngine{ - Apps: tt.fields.Apps, - Cgm: tt.fields.Cgm, - CtrlGroups: tt.fields.CtrlGroups, - l: tt.fields.l, - CBM: tt.fields.CBM, - Vendor: tt.fields.Vendor, - } - tt.wantErr(t, R.RegisterApp(tt.args.podid, tt.args.annotation, tt.args.fromNRI, tt.args.updater), fmt.Sprintf("RegisterApp(%v, %v, %v, %v)", tt.args.podid, tt.args.annotation, tt.args.fromNRI, tt.args.updater)) - }) - } -} - -func TestRDTEngine_UnRegisterApp(t *testing.T) { - type fields struct { - Apps map[string]App - Cgm ControlGroupManager - CtrlGroups map[string]apiext.Resctrl - l sync.RWMutex - CBM uint - Vendor string - } - type args struct { - podid string - fromNRI bool - updater ResctrlUpdater - } - tests := []struct { - name string - fields fields - args args - wantErr assert.ErrorAssertionFunc - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - R := &RDTEngine{ - Apps: tt.fields.Apps, - Cgm: tt.fields.Cgm, - CtrlGroups: tt.fields.CtrlGroups, - l: tt.fields.l, - CBM: tt.fields.CBM, - Vendor: tt.fields.Vendor, - } - tt.wantErr(t, R.UnRegisterApp(tt.args.podid, tt.args.fromNRI, tt.args.updater), fmt.Sprintf("UnRegisterApp(%v, %v, %v)", tt.args.podid, tt.args.fromNRI, tt.args.updater)) - }) - } -} - -func TestRDTEngine_calculateMba(t *testing.T) { - type fields struct { - Apps map[string]App - Cgm ControlGroupManager - CtrlGroups map[string]apiext.Resctrl - l sync.RWMutex - CBM uint - Vendor string - } - type args struct { - mbaPercent int64 - } - tests := []struct { - name string - fields fields - args args - want int64 - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - R := &RDTEngine{ - Apps: tt.fields.Apps, - Cgm: tt.fields.Cgm, - CtrlGroups: tt.fields.CtrlGroups, - l: tt.fields.l, - CBM: tt.fields.CBM, - Vendor: tt.fields.Vendor, - } - assert.Equalf(t, tt.want, R.calculateMba(tt.args.mbaPercent), "calculateMba(%v)", tt.args.mbaPercent) - }) - } -} - -func Test_calculateAMDMba(t *testing.T) { - type args struct { - mbaPercent int64 - } - tests := []struct { - name string - args args - want int64 - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, calculateAMDMba(tt.args.mbaPercent), "calculateAMDMba(%v)", tt.args.mbaPercent) - }) - } -} - -func Test_calculateIntelMba(t *testing.T) { - type args struct { - mbaPercent int64 - } - tests := []struct { - name string - args args - want int64 - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, calculateIntelMba(tt.args.mbaPercent), "calculateIntelMba(%v)", tt.args.mbaPercent) - }) - } -} From 3d2fc63173ec714f71f7b674631c96758d979923 Mon Sep 17 00:00:00 2001 From: "Zhang, Kang" Date: Mon, 19 Aug 2024 10:37:28 +0800 Subject: [PATCH 08/10] fix lint --- pkg/koordlet/util/resctrl/resctrl_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/koordlet/util/resctrl/resctrl_test.go b/pkg/koordlet/util/resctrl/resctrl_test.go index 7b8fa4578..7070be5ef 100644 --- a/pkg/koordlet/util/resctrl/resctrl_test.go +++ b/pkg/koordlet/util/resctrl/resctrl_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package resctrl import ( From 270f8f556e5ccf4984e7d0eb1b5799dd648fbe89 Mon Sep 17 00:00:00 2001 From: "Zhang, Kang" Date: Mon, 19 Aug 2024 14:25:33 +0800 Subject: [PATCH 09/10] fix lint --- pkg/koordlet/util/resctrl/resctrl_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/koordlet/util/resctrl/resctrl_test.go b/pkg/koordlet/util/resctrl/resctrl_test.go index 7070be5ef..23676e646 100644 --- a/pkg/koordlet/util/resctrl/resctrl_test.go +++ b/pkg/koordlet/util/resctrl/resctrl_test.go @@ -48,7 +48,7 @@ func TestGetNewTaskIds(t *testing.T) { args: args{ ids: []int32{1, 2, 3}, tasksMap: map[int32]struct{}{ - 1: struct{}{}, + 1: {}, }, }, want: []int32{2, 3}, From 3c66337e682342c38415d07e7baab0407259fefd Mon Sep 17 00:00:00 2001 From: "Zhang, Kang" Date: Wed, 21 Aug 2024 15:53:55 +0800 Subject: [PATCH 10/10] add ut --- .../hooks/resctrl/resctrl_test.go | 272 ++++++++++++++++++ .../runtimehooks/hooks/resctrl/updater.go | 8 +- .../hooks/resctrl/updater_test.go | 251 ++++++++++++++++ 3 files changed, 527 insertions(+), 4 deletions(-) create mode 100644 pkg/koordlet/runtimehooks/hooks/resctrl/resctrl_test.go create mode 100644 pkg/koordlet/runtimehooks/hooks/resctrl/updater_test.go diff --git a/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl_test.go b/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl_test.go new file mode 100644 index 000000000..153308c30 --- /dev/null +++ b/pkg/koordlet/runtimehooks/hooks/resctrl/resctrl_test.go @@ -0,0 +1,272 @@ +package resctrl + +import ( + "github.com/koordinator-sh/koordinator/pkg/koordlet/resourceexecutor" + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks" + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol" + "github.com/koordinator-sh/koordinator/pkg/koordlet/statesinformer" + "github.com/koordinator-sh/koordinator/pkg/koordlet/util/resctrl" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/tools/record" + "testing" +) + +func TestObject(t *testing.T) { + t.Run("test not panic", func(t *testing.T) { + p := Object() + assert.Equal(t, &plugin{rule: newRule()}, p) + }) +} + +func Test_plugin_Register(t *testing.T) { + t.Run("test not panic", func(t *testing.T) { + p := newPlugin() + p.Register(hooks.Options{}) + }) +} + +func Test_plugin_RemovePodResctrlResources(t *testing.T) { + type fields struct { + rule *Rule + engine resctrl.ResctrlEngine + executor resourceexecutor.ResourceUpdateExecutor + statesInformer statesinformer.StatesInformer + EventRecorder record.EventRecorder + } + type args struct { + proto protocol.HooksProtocol + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &plugin{ + rule: tt.fields.rule, + engine: tt.fields.engine, + executor: tt.fields.executor, + statesInformer: tt.fields.statesInformer, + EventRecorder: tt.fields.EventRecorder, + } + if err := p.RemovePodResctrlResources(tt.args.proto); (err != nil) != tt.wantErr { + t.Errorf("RemovePodResctrlResources() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_plugin_RemoveUnusedResctrlPath(t *testing.T) { + type fields struct { + rule *Rule + engine resctrl.ResctrlEngine + executor resourceexecutor.ResourceUpdateExecutor + statesInformer statesinformer.StatesInformer + EventRecorder record.EventRecorder + } + type args struct { + protos []protocol.HooksProtocol + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &plugin{ + rule: tt.fields.rule, + engine: tt.fields.engine, + executor: tt.fields.executor, + statesInformer: tt.fields.statesInformer, + EventRecorder: tt.fields.EventRecorder, + } + if err := p.RemoveUnusedResctrlPath(tt.args.protos); (err != nil) != tt.wantErr { + t.Errorf("RemoveUnusedResctrlPath() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_plugin_SetContainerResctrlResources(t *testing.T) { + type fields struct { + rule *Rule + engine resctrl.ResctrlEngine + executor resourceexecutor.ResourceUpdateExecutor + statesInformer statesinformer.StatesInformer + EventRecorder record.EventRecorder + } + type args struct { + proto protocol.HooksProtocol + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &plugin{ + rule: tt.fields.rule, + engine: tt.fields.engine, + executor: tt.fields.executor, + statesInformer: tt.fields.statesInformer, + EventRecorder: tt.fields.EventRecorder, + } + if err := p.SetContainerResctrlResources(tt.args.proto); (err != nil) != tt.wantErr { + t.Errorf("SetContainerResctrlResources() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_plugin_SetPodResctrlResourcesForHooks(t *testing.T) { + type fields struct { + rule *Rule + engine resctrl.ResctrlEngine + executor resourceexecutor.ResourceUpdateExecutor + statesInformer statesinformer.StatesInformer + EventRecorder record.EventRecorder + } + type args struct { + proto protocol.HooksProtocol + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &plugin{ + rule: tt.fields.rule, + engine: tt.fields.engine, + executor: tt.fields.executor, + statesInformer: tt.fields.statesInformer, + EventRecorder: tt.fields.EventRecorder, + } + if err := p.SetPodResctrlResourcesForHooks(tt.args.proto); (err != nil) != tt.wantErr { + t.Errorf("SetPodResctrlResourcesForHooks() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_plugin_SetPodResctrlResourcesForReconciler(t *testing.T) { + type fields struct { + rule *Rule + engine resctrl.ResctrlEngine + executor resourceexecutor.ResourceUpdateExecutor + statesInformer statesinformer.StatesInformer + EventRecorder record.EventRecorder + } + type args struct { + proto protocol.HooksProtocol + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &plugin{ + rule: tt.fields.rule, + engine: tt.fields.engine, + executor: tt.fields.executor, + statesInformer: tt.fields.statesInformer, + EventRecorder: tt.fields.EventRecorder, + } + if err := p.SetPodResctrlResourcesForReconciler(tt.args.proto); (err != nil) != tt.wantErr { + t.Errorf("SetPodResctrlResourcesForReconciler() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_plugin_UpdatePodTaskIds(t *testing.T) { + type fields struct { + rule *Rule + engine resctrl.ResctrlEngine + executor resourceexecutor.ResourceUpdateExecutor + statesInformer statesinformer.StatesInformer + EventRecorder record.EventRecorder + } + type args struct { + proto protocol.HooksProtocol + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &plugin{ + rule: tt.fields.rule, + engine: tt.fields.engine, + executor: tt.fields.executor, + statesInformer: tt.fields.statesInformer, + EventRecorder: tt.fields.EventRecorder, + } + if err := p.UpdatePodTaskIds(tt.args.proto); (err != nil) != tt.wantErr { + t.Errorf("UpdatePodTaskIds() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_plugin_setPodResctrlResources(t *testing.T) { + type fields struct { + rule *Rule + engine resctrl.ResctrlEngine + executor resourceexecutor.ResourceUpdateExecutor + statesInformer statesinformer.StatesInformer + EventRecorder record.EventRecorder + } + type args struct { + proto protocol.HooksProtocol + fromNRI bool + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &plugin{ + rule: tt.fields.rule, + engine: tt.fields.engine, + executor: tt.fields.executor, + statesInformer: tt.fields.statesInformer, + EventRecorder: tt.fields.EventRecorder, + } + if err := p.setPodResctrlResources(tt.args.proto, tt.args.fromNRI); (err != nil) != tt.wantErr { + t.Errorf("setPodResctrlResources() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/koordlet/runtimehooks/hooks/resctrl/updater.go b/pkg/koordlet/runtimehooks/hooks/resctrl/updater.go index d9b4ffa89..c7eaf5e4b 100755 --- a/pkg/koordlet/runtimehooks/hooks/resctrl/updater.go +++ b/pkg/koordlet/runtimehooks/hooks/resctrl/updater.go @@ -48,12 +48,12 @@ func (u DefaultResctrlProtocolUpdater) Value() string { return u.schemata } -func (r *DefaultResctrlProtocolUpdater) SetKey(key string) { - r.group = key +func (r *DefaultResctrlProtocolUpdater) SetKey(group string) { + r.group = group } -func (r *DefaultResctrlProtocolUpdater) SetValue(val string) { - r.schemata = val +func (r *DefaultResctrlProtocolUpdater) SetValue(schemata string) { + r.schemata = schemata } func (u *DefaultResctrlProtocolUpdater) Update() error { diff --git a/pkg/koordlet/runtimehooks/hooks/resctrl/updater_test.go b/pkg/koordlet/runtimehooks/hooks/resctrl/updater_test.go new file mode 100644 index 000000000..443448bb4 --- /dev/null +++ b/pkg/koordlet/runtimehooks/hooks/resctrl/updater_test.go @@ -0,0 +1,251 @@ +package resctrl + +import ( + "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol" + "github.com/koordinator-sh/koordinator/pkg/koordlet/util/resctrl" + "testing" +) + +func TestCreateResctrlProtocolUpdaterFunc(t *testing.T) { + type args struct { + u resctrl.ResctrlUpdater + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Default Resctrl Protocol Updater", + args: args{u: &DefaultResctrlProtocolUpdater{ + hooksProtocol: &protocol.ContainerContext{}, + group: "guaranteed", + schemata: "MB:0=0xff", + updateFunc: nil, + }}, + wantErr: true, + }, + { + name: "Response resctrl is nil", + args: args{u: &DefaultResctrlProtocolUpdater{ + hooksProtocol: &protocol.PodContext{ + Request: protocol.PodRequest{ + PodMeta: protocol.PodMeta{ + Namespace: "default", + Name: "test", + UID: "uid", + }, + Resources: &protocol.Resources{}, + }, + Response: protocol.PodResponse{ + Resources: protocol.Resources{}, + }, + }, + group: "guaranteed", + schemata: "MB:0=0xff", + updateFunc: nil, + }}, + wantErr: false, + }, + { + name: "Response resctrl is not nil", + args: args{u: &DefaultResctrlProtocolUpdater{ + hooksProtocol: &protocol.PodContext{ + Request: protocol.PodRequest{ + PodMeta: protocol.PodMeta{ + Namespace: "default", + Name: "test", + UID: "uid", + }, + Resources: &protocol.Resources{}, + }, + Response: protocol.PodResponse{ + Resources: protocol.Resources{ + Resctrl: &protocol.Resctrl{ + Schemata: "", + Closid: "", + NewTaskIds: nil, + }, + }, + }, + }, + group: "guaranteed", + schemata: "MB:0=0xff", + updateFunc: nil, + }}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := CreateResctrlProtocolUpdaterFunc(tt.args.u); (err != nil) != tt.wantErr { + t.Errorf("CreateResctrlProtocolUpdaterFunc() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestNewCreateResctrlProtocolUpdater(t *testing.T) { + type args struct { + hooksProtocol protocol.HooksProtocol + } + hooksProtocol := &protocol.PodContext{} + tests := []struct { + name string + args args + }{ + { + name: "Default Resctrl Protocol Updater", + args: args{ + hooksProtocol: hooksProtocol, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + NewCreateResctrlProtocolUpdater(tt.args.hooksProtocol) + }) + } +} + +func TestNewRemoveResctrlProtocolUpdater(t *testing.T) { + type args struct { + hooksProtocol protocol.HooksProtocol + } + hooksProtocol := &protocol.PodContext{} + tests := []struct { + name string + args args + want resctrl.ResctrlUpdater + }{ + { + name: "Default Remove Resctrl Protocol Updater", + args: args{ + hooksProtocol: hooksProtocol, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + NewRemoveResctrlProtocolUpdater(tt.args.hooksProtocol) + }) + } +} + +func TestNewRemoveResctrlUpdater(t *testing.T) { + type args struct { + group string + } + tests := []struct { + name string + args args + }{ + { + name: "New Remove Resctrl Updater", + args: args{group: "test"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + NewRemoveResctrlUpdater(tt.args.group) + }) + } +} + +func TestRemoveResctrlProtocolUpdaterFunc(t *testing.T) { + type args struct { + u resctrl.ResctrlUpdater + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "not pod context", + args: args{u: &DefaultResctrlProtocolUpdater{hooksProtocol: &protocol.ContainerContext{}}}, + wantErr: true, + }, + { + name: "Default Remove Resctrl Protocol Updater", + args: args{u: &DefaultResctrlProtocolUpdater{ + hooksProtocol: &protocol.PodContext{ + Request: protocol.PodRequest{ + PodMeta: protocol.PodMeta{ + Namespace: "default", + Name: "test", + UID: "uid", + }, + Resources: &protocol.Resources{}, + }, + Response: protocol.PodResponse{ + Resources: protocol.Resources{ + Resctrl: &protocol.Resctrl{ + Schemata: "", + Closid: "", + NewTaskIds: nil, + }, + }, + }, + }, + group: "guaranteed", + schemata: "MB:0=0xff", + updateFunc: nil, + }}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := RemoveResctrlProtocolUpdaterFunc(tt.args.u); (err != nil) != tt.wantErr { + t.Errorf("RemoveResctrlProtocolUpdaterFunc() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestRemoveResctrlUpdaterFunc(t *testing.T) { + type args struct { + u resctrl.ResctrlUpdater + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "remove err", + args: args{u: &DefaultResctrlProtocolUpdater{ + hooksProtocol: &protocol.PodContext{ + Request: protocol.PodRequest{ + PodMeta: protocol.PodMeta{ + Namespace: "default", + Name: "test", + UID: "uid", + }, + Resources: &protocol.Resources{}, + }, + Response: protocol.PodResponse{ + Resources: protocol.Resources{ + Resctrl: &protocol.Resctrl{ + Schemata: "", + Closid: "", + NewTaskIds: nil, + }, + }, + }, + }, + group: "guaranteed", + schemata: "MB:0=0xff", + updateFunc: nil, + }}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := RemoveResctrlUpdaterFunc(tt.args.u); (err != nil) != tt.wantErr { + t.Errorf("RemoveResctrlUpdaterFunc() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +}