add group identity plugin

cmd/koordlet/main.go

@@ -32,6 +32,7 @@ import (
 	agent "github.com/koordinator-sh/koordinator/pkg/koordlet"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/audit"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/config"
+	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks"
 )
 
 func init() {}
@@ -46,8 +47,10 @@ func main() {
 	defer klog.Flush()
 
 	if err := features.DefaultMutableKoordletFeatureGate.SetFromMap(cfg.FeatureGates); err != nil {
-		klog.Error("Unable to setup feature-gates: ", err)
-		os.Exit(1)
+		klog.Fatalf("Unable to setup feature-gates: %v", err)
+	}
+	if err := runtimehooks.DefaultMutableRuntimeHooksFG.SetFromMap(cfg.RuntimeHookConf.FeatureGates); err != nil {
+		klog.Fatalf("Unable to setup runtime-hooks: %v", err)
 	}
 
 	stopCtx := signals.SetupSignalHandler()

examples/nginx/nginx.yaml

@@ -14,6 +14,7 @@ spec:
       name: nginx
       labels:
         app: nginx
+        koordinator.sh/qosClass: LS
     spec:
       containers:
         - name: nginx
@@ -29,4 +30,4 @@ spec:
               memory: "1Gi"
             limits:
               cpu: 4
-              memory: "1Gi"
+              memory: "1Gi"

pkg/koordlet/runtimehooks/config.go

@@ -23,7 +23,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/runtime"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/featuregate"
-	// "github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/groupidentity"
+
+	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks/groupidentity"
 )
 
 const (
@@ -39,7 +40,7 @@ var (
 	}
 
 	runtimeHookPlugins = map[featuregate.Feature]HookPlugin{
-		// GroupIdentity: groupidentity.Object(),
+		GroupIdentity: groupidentity.Object(),
 	}
 )
 

pkg/koordlet/runtimehooks/hooks/groupidentity/bvt.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package groupidentity
+
+import (
+	"sync"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"k8s.io/utils/pointer"
+
+	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks"
+	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/rule"
+	rmconfig "github.com/koordinator-sh/koordinator/pkg/runtime-manager/config"
+	"github.com/koordinator-sh/koordinator/pkg/util"
+	sysutil "github.com/koordinator-sh/koordinator/pkg/util/system"
+)
+
+const (
+	name        = "GroupIdentity"
+	description = "set bvt value by priority and qos class"
+)
+
+type bvtPlugin struct {
+	rule         *bvtRule
+	ruleRWMutex  sync.RWMutex
+	sysSupported *bool
+}
+
+func (b *bvtPlugin) Register() {
+	klog.V(5).Infof("register hook %v", name)
+	hooks.Register(rmconfig.PreRunPodSandbox, name, description, b.PreRunPodSandbox)
+	rule.Register(name, description,
+		rule.WithParseFunc(b.parseRule),
+		rule.WithUpdateCallback(b.ruleUpdateCb),
+		rule.WithSystemSupported(b.SystemSupported))
+}
+
+func (b *bvtPlugin) SystemSupported() bool {
+	if b.sysSupported == nil {
+		bvtFilePath := sysutil.GetCgroupFilePath(
+			util.GetKubeQosRelativePath(corev1.PodQOSGuaranteed), sysutil.CPUBVTWarpNs)
+		b.sysSupported = pointer.BoolPtr(sysutil.FileExists(bvtFilePath))
+		klog.Infof("update system supported info to %v for plugin %v", *b.sysSupported, name)
+	}
+	return *b.sysSupported
+}
+
+var singleton *bvtPlugin
+
+func Object() *bvtPlugin {
+	if singleton == nil {
+		singleton = &bvtPlugin{rule: &bvtRule{}}
+	}
+	return singleton
+}

pkg/koordlet/runtimehooks/hooks/groupidentity/bvt_test.go

@@ -0,0 +1,167 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package groupidentity
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/pointer"
+
+	ext "github.com/koordinator-sh/koordinator/apis/extension"
+	runtimeapi "github.com/koordinator-sh/koordinator/apis/runtime/v1alpha1"
+	"github.com/koordinator-sh/koordinator/pkg/util"
+	"github.com/koordinator-sh/koordinator/pkg/util/system"
+)
+
+func initCPUBvt(dirWithKube string, value int64, helper *system.FileTestUtil) {
+	helper.WriteCgroupFileContents(dirWithKube, system.CPUBVTWarpNs, strconv.FormatInt(value, 10))
+}
+
+func getPodCPUBurst(podDirWithKube string, helper *system.FileTestUtil) int64 {
+	valueStr := helper.ReadCgroupFileContents(podDirWithKube, system.CPUBVTWarpNs)
+	value, _ := strconv.ParseInt(valueStr, 10, 64)
+	return value
+}
+
+func Test_bvtPlugin_PreRunPodSandbox(t *testing.T) {
+	defaultRule := &bvtRule{
+		podQOSParams: map[ext.QoSClass]int64{
+			ext.QoSLSR: 2,
+			ext.QoSLS:  2,
+			ext.QoSBE:  -1,
+		},
+		kubeQOSDirParams: map[corev1.PodQOSClass]int64{
+			corev1.PodQOSGuaranteed: 0,
+			corev1.PodQOSBurstable:  2,
+			corev1.PodQOSBestEffort: -1,
+		},
+		kubeQOSPodParams: map[corev1.PodQOSClass]int64{
+			corev1.PodQOSGuaranteed: 2,
+			corev1.PodQOSBurstable:  2,
+			corev1.PodQOSBestEffort: -1,
+		},
+	}
+	type fields struct {
+		systemSupported *bool
+	}
+	type args struct {
+		request  *runtimeapi.RunPodSandboxHookRequest
+		response *runtimeapi.RunPodSandboxHookResponse
+	}
+	type want struct {
+		bvtValue int64
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   want
+	}{
+		{
+			name: "set ls pod bvt",
+			fields: fields{
+				systemSupported: pointer.Bool(true),
+			},
+			args: args{
+				request: &runtimeapi.RunPodSandboxHookRequest{
+					Labels: map[string]string{
+						ext.LabelPodQoS: string(ext.QoSLS),
+					},
+					CgroupParent: "kubepods/pod-guaranteed-test-uid/",
+				},
+				response: &runtimeapi.RunPodSandboxHookResponse{},
+			},
+			want: want{
+				bvtValue: 2,
+			},
+		},
+		{
+			name: "set be pod bvt",
+			fields: fields{
+				systemSupported: pointer.Bool(true),
+			},
+			args: args{
+				request: &runtimeapi.RunPodSandboxHookRequest{
+					Labels: map[string]string{
+						ext.LabelPodQoS: string(ext.QoSBE),
+					},
+					CgroupParent: "kubepods/besteffort/pod-besteffort-test-uid/",
+				},
+				response: &runtimeapi.RunPodSandboxHookResponse{},
+			},
+			want: want{
+				bvtValue: -1,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testHelper := system.NewFileTestUtil(t)
+			initCPUBvt(tt.args.request.CgroupParent, 0, testHelper)
+
+			b := &bvtPlugin{
+				rule:         defaultRule,
+				sysSupported: tt.fields.systemSupported,
+			}
+			err := b.PreRunPodSandbox(tt.args.request, tt.args.response)
+			assert.NoError(t, err)
+
+			gotBvt := getPodCPUBurst(tt.args.request.CgroupParent, testHelper)
+			assert.Equal(t, tt.want.bvtValue, gotBvt, "pod bvt should equal")
+		})
+	}
+}
+
+func Test_bvtPlugin_systemSupported(t *testing.T) {
+	kubeRootDir := util.GetKubeQosRelativePath(corev1.PodQOSGuaranteed)
+	type fields struct {
+		initPath *string
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   bool
+	}{
+		{
+			name: "system support since bvt file exist",
+			fields: fields{
+				initPath: &kubeRootDir,
+			},
+			want: true,
+		},
+		{
+			name:   "system not support since bvt not file exist",
+			fields: fields{},
+			want:   false,
+		},
+	}
+	for _, tt := range tests {
+		testHelper := system.NewFileTestUtil(t)
+		if tt.fields.initPath != nil {
+			initCPUBvt(*tt.fields.initPath, 0, testHelper)
+		}
+		t.Run(tt.name, func(t *testing.T) {
+			b := &bvtPlugin{}
+			if got := b.SystemSupported(); got != tt.want {
+				t.Errorf("SystemSupported() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/koordlet/runtimehooks/hooks/groupidentity/interceptor.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package groupidentity
+
+import (
+	"strconv"
+
+	"k8s.io/klog/v2"
+
+	ext "github.com/koordinator-sh/koordinator/apis/extension"
+	runtimeapi "github.com/koordinator-sh/koordinator/apis/runtime/v1alpha1"
+	"github.com/koordinator-sh/koordinator/pkg/koordlet/audit"
+	"github.com/koordinator-sh/koordinator/pkg/util"
+	sysutil "github.com/koordinator-sh/koordinator/pkg/util/system"
+)
+
+func (b *bvtPlugin) PreRunPodSandbox(requestIf, responseIf interface{}) error {
+	if !b.SystemSupported() {
+		klog.V(5).Infof("plugin %s is not supported by system", name)
+		return nil
+	}
+	r := b.getRule()
+	req := requestIf.(*runtimeapi.RunPodSandboxHookRequest)
+	podQoS := ext.GetQoSClassByLabels(req.Labels)
+	podKubeQoS := util.GetKubeQoSByCgroupParent(req.CgroupParent)
+	podBvt := r.getPodBvtValue(podQoS, podKubeQoS)
+	// CgroupParent e.g. kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod586c1b35_63de_4ee0_9da3_2cebdca672c8.slice
+	klog.V(5).Infof("set pod bvt on cgroup parent %v", req.CgroupParent)
+	if req.PodMeta != nil {
+		audit.V(2).Pod(req.PodMeta.Namespace, req.PodMeta.Name).Reason(name).Message("set bvt to %v", podBvt).Do()
+	}
+	return sysutil.CgroupFileWrite(req.CgroupParent, sysutil.CPUBVTWarpNs, strconv.FormatInt(podBvt, 10))
+}