From 9b4c5b1b0eba36fffb7b8ca240bbfe3f84969e47 Mon Sep 17 00:00:00 2001 From: georgexiang Date: Sun, 28 Apr 2024 14:45:52 +0800 Subject: [PATCH] koordlet: support cri-o container runtime (#1983) Signed-off-by: ocichina001 <165124541+ocichina001@users.noreply.github.com> Signed-off-by: george Signed-off-by: saintube Signed-off-by: wangjianyu.wjy Signed-off-by: xulinfei.xlf Signed-off-by: Siyu Wang Signed-off-by: Fansong Zeng Signed-off-by: acejilam Signed-off-by: lucming <2876757716@qq.com> Signed-off-by: sjtufl Co-authored-by: ocichina001 <165124541+ocichina001@users.noreply.github.com> Co-authored-by: ocichina Co-authored-by: Frame Co-authored-by: wangjianyu Co-authored-by: wangjianyu.wjy Co-authored-by: xulinfei1996 <1187938268@qq.com> Co-authored-by: xulinfei.xlf Co-authored-by: Siyu Wang Co-authored-by: Fansong Zeng Co-authored-by: ls-2018 Co-authored-by: lucming <2876757716@qq.com> Co-authored-by: Liang Fang --- pkg/koordlet/util/pod.go | 9 +- .../util/runtime/handler/crio_runtime.go | 100 ++++++++++++++ .../util/runtime/handler/crio_runtime_test.go | 130 ++++++++++++++++++ pkg/koordlet/util/runtime/runtime.go | 40 ++++++ pkg/koordlet/util/runtime/runtime_test.go | 22 +++ pkg/koordlet/util/system/cgroup_driver.go | 13 +- .../util/system/cgroup_driver_test.go | 18 +++ pkg/koordlet/util/system/config.go | 1 + 8 files changed, 330 insertions(+), 3 deletions(-) create mode 100644 pkg/koordlet/util/runtime/handler/crio_runtime.go create mode 100644 pkg/koordlet/util/runtime/handler/crio_runtime_test.go diff --git a/pkg/koordlet/util/pod.go b/pkg/koordlet/util/pod.go index fc7343b144..10d53d7a13 100644 --- a/pkg/koordlet/util/pod.go +++ b/pkg/koordlet/util/pod.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "path/filepath" + "strings" corev1 "k8s.io/api/core/v1" @@ -79,7 +80,13 @@ func GetPodSandboxContainerID(pod *corev1.Pod) (string, error) { continue } if _, exist := containerSubDirNames[containerDir.Name()]; !exist { - sandboxCandidates = append(sandboxCandidates, containerDir.Name()) + if strings.HasPrefix(containerDir.Name(), "crio-") { + if !strings.HasSuffix(containerDir.Name(), ".scope") { + sandboxCandidates = append(sandboxCandidates, containerDir.Name()) + } + } else { + sandboxCandidates = append(sandboxCandidates, containerDir.Name()) + } } } diff --git a/pkg/koordlet/util/runtime/handler/crio_runtime.go b/pkg/koordlet/util/runtime/handler/crio_runtime.go new file mode 100644 index 0000000000..ad275a5114 --- /dev/null +++ b/pkg/koordlet/util/runtime/handler/crio_runtime.go @@ -0,0 +1,100 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" +) + +func GetCrioEndpoint() string { + return filepath.Join(system.Conf.VarRunRootDir, "crio/crio.sock") +} + +func GetCrioEndpoint2() string { + return filepath.Join(system.Conf.VarRunRootDir, "crio.sock") +} + +type CrioRuntimeHandler struct { + runtimeServiceClient runtimeapi.RuntimeServiceClient + timeout time.Duration + endpoint string +} + +func NewCrioRuntimeHandler(endpoint string) (ContainerRuntimeHandler, error) { + ep := strings.TrimPrefix(endpoint, "unix://") + if _, err := os.Stat(ep); err != nil { + return nil, err + } + + client, err := getRuntimeClient(endpoint) + if err != nil { + return nil, err + } + + return &CrioRuntimeHandler{ + runtimeServiceClient: client, + timeout: defaultConnectionTimeout, + endpoint: endpoint, + }, nil +} + +func (c *CrioRuntimeHandler) StopContainer(containerID string, timeout int64) error { + if containerID == "" { + return fmt.Errorf("containerID cannot be empty") + } + t := c.timeout + time.Duration(timeout) + ctx, cancel := context.WithTimeout(context.Background(), t) + defer cancel() + + request := &runtimeapi.StopContainerRequest{ + ContainerId: containerID, + Timeout: timeout, + } + _, err := c.runtimeServiceClient.StopContainer(ctx, request) + return err +} + +func (c *CrioRuntimeHandler) UpdateContainerResources(containerID string, opts UpdateOptions) error { + if containerID == "" { + return fmt.Errorf("containerID cannot be empty") + } + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + request := &runtimeapi.UpdateContainerResourcesRequest{ + ContainerId: containerID, + Linux: &runtimeapi.LinuxContainerResources{ + CpuPeriod: opts.CPUPeriod, + CpuQuota: opts.CPUQuota, + CpuShares: opts.CPUShares, + CpusetCpus: opts.CpusetCpus, + CpusetMems: opts.CpusetMems, + MemoryLimitInBytes: opts.MemoryLimitInBytes, + OomScoreAdj: opts.OomScoreAdj, + }, + } + _, err := c.runtimeServiceClient.UpdateContainerResources(ctx, request) + return err +} diff --git a/pkg/koordlet/util/runtime/handler/crio_runtime_test.go b/pkg/koordlet/util/runtime/handler/crio_runtime_test.go new file mode 100644 index 0000000000..edc9c64b8a --- /dev/null +++ b/pkg/koordlet/util/runtime/handler/crio_runtime_test.go @@ -0,0 +1,130 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/golang/mock/gomock" + "github.com/prashantv/gostub" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc" + + mockclient "github.com/koordinator-sh/koordinator/pkg/koordlet/util/runtime/handler/mockclient" + "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" +) + +func Test_NewCrioRuntimeHandler(t *testing.T) { + stubs := gostub.Stub(&GrpcDial, func(context context.Context, target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + return &grpc.ClientConn{}, nil + }) + defer stubs.Reset() + + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + helper.WriteFileContents("/var/run/crio/crio.sock", "test") + system.Conf.VarRunRootDir = filepath.Join(helper.TempDir, "/var/run") + CrioEndpoint1 := GetCrioEndpoint() + unixEndPoint := fmt.Sprintf("unix://%s", CrioEndpoint1) + crioRuntime, err := NewCrioRuntimeHandler(unixEndPoint) + assert.NoError(t, err) + assert.NotNil(t, crioRuntime) + + // custom VarRunRootDir + helper.WriteFileContents("/host-var-run/crio/crio.sock", "test1") + system.Conf.VarRunRootDir = filepath.Join(helper.TempDir, "/host-var-run") + CrioEndpoint1 = GetCrioEndpoint() + unixEndPoint = fmt.Sprintf("unix://%s", CrioEndpoint1) + crioRuntime, err = NewCrioRuntimeHandler(unixEndPoint) + assert.NoError(t, err) + assert.NotNil(t, crioRuntime) +} + +func Test_Crio_StopContainer(t *testing.T) { + type args struct { + name string + containerId string + runtimeError error + expectError bool + } + tests := []args{ + { + name: "test_stopContainer_success", + containerId: "test_container_id", + runtimeError: nil, + expectError: false, + }, + { + name: "test_stopContainer_fail", + containerId: "test_container_id", + runtimeError: fmt.Errorf("stopContainer error"), + expectError: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctl := gomock.NewController(t) + defer ctl.Finish() + mockRuntimeClient := mockclient.NewMockRuntimeServiceClient(ctl) + mockRuntimeClient.EXPECT().StopContainer(gomock.Any(), gomock.Any()).Return(nil, tt.runtimeError) + + runtimeHandler := ContainerdRuntimeHandler{runtimeServiceClient: mockRuntimeClient, timeout: 1, endpoint: GetCrioEndpoint()} + gotErr := runtimeHandler.StopContainer(tt.containerId, 1) + assert.Equal(t, gotErr != nil, tt.expectError) + + }) + } +} + +func Test_Crio_UpdateContainerResources(t *testing.T) { + type args struct { + name string + containerId string + runtimeError error + expectError bool + } + tests := []args{ + { + name: "test_UpdateContainerResources_success", + containerId: "test_container_id", + runtimeError: nil, + expectError: false, + }, + { + name: "test_UpdateContainerResources_fail", + containerId: "test_container_id", + runtimeError: fmt.Errorf("UpdateContainerResources error"), + expectError: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctl := gomock.NewController(t) + defer ctl.Finish() + mockRuntimeClient := mockclient.NewMockRuntimeServiceClient(ctl) + mockRuntimeClient.EXPECT().UpdateContainerResources(gomock.Any(), gomock.Any()).Return(nil, tt.runtimeError) + + runtimeHandler := ContainerdRuntimeHandler{runtimeServiceClient: mockRuntimeClient, timeout: 1, endpoint: GetCrioEndpoint()} + gotErr := runtimeHandler.UpdateContainerResources(tt.containerId, UpdateOptions{}) + assert.Equal(t, tt.expectError, gotErr != nil) + }) + } +} diff --git a/pkg/koordlet/util/runtime/runtime.go b/pkg/koordlet/util/runtime/runtime.go index 5db9be1804..db71e6505e 100644 --- a/pkg/koordlet/util/runtime/runtime.go +++ b/pkg/koordlet/util/runtime/runtime.go @@ -32,6 +32,7 @@ var ( DockerHandler handler.ContainerRuntimeHandler ContainerdHandler handler.ContainerRuntimeHandler PouchHandler handler.ContainerRuntimeHandler + CrioHandler handler.ContainerRuntimeHandler mutex = &sync.Mutex{} ) @@ -46,6 +47,8 @@ func GetRuntimeHandler(runtimeType string) (handler.ContainerRuntimeHandler, err return getContainerdHandler() case system.RuntimeTypePouch: return getPouchHandler() + case system.RuntimeTypeCrio: + return getCrioHandler() default: return nil, fmt.Errorf("runtime type %v is not supported", runtimeType) } @@ -152,6 +155,43 @@ func getPouchEndpoint() (string, error) { return "", fmt.Errorf("pouch endpoint does not exist") } +func getCrioHandler() (handler.ContainerRuntimeHandler, error) { + if CrioHandler != nil { + return CrioHandler, nil + } + + unixEndpoint, err := getCrioEndpoint() + if err != nil { + klog.Errorf("failed to get cri-o endpoint, error: %v", err) + return nil, err + } + + CrioHandler, err = handler.NewCrioRuntimeHandler(unixEndpoint) + if err != nil { + klog.Errorf("failed to create cri-o runtime handler, error: %v", err) + return nil, err + } + + return CrioHandler, nil +} + +func getCrioEndpoint() (string, error) { + if crioEndpoint := handler.GetCrioEndpoint(); isFile(crioEndpoint) { + return fmt.Sprintf("unix://%s", crioEndpoint), nil + } + + if crioEndpoint2 := handler.GetCrioEndpoint2(); isFile(crioEndpoint2) { + return fmt.Sprintf("unix://%s", crioEndpoint2), nil + } + + if len(system.Conf.CrioEndPoint) > 0 && isFile(system.Conf.CrioEndPoint) { + klog.Infof("find cri-o Endpoint : %v", system.Conf.CrioEndPoint) + return fmt.Sprintf("unix://%s", system.Conf.CrioEndPoint), nil + } + + return "", fmt.Errorf("cri-o endpoint does not exist") +} + func isFile(path string) bool { s, err := os.Stat(path) if err != nil || s == nil { diff --git a/pkg/koordlet/util/runtime/runtime_test.go b/pkg/koordlet/util/runtime/runtime_test.go index 4e92af885e..0e56994691 100644 --- a/pkg/koordlet/util/runtime/runtime_test.go +++ b/pkg/koordlet/util/runtime/runtime_test.go @@ -95,6 +95,20 @@ func Test_GetRuntimeHandler(t *testing.T) { expectRuntimeHandler: "PouchRuntimeHandler", expectErr: false, }, + { + name: "test_/var/run/crio.sock", + endPoint: "/var/run/crio.sock", + runtimeType: "cri-o", + expectRuntimeHandler: "CrioRuntimeHandler", + expectErr: false, + }, + { + name: "test_/var/run/crio/crio.sock", + endPoint: "/var/run/crio/crio.sock", + runtimeType: "cri-o", + expectRuntimeHandler: "CrioRuntimeHandler", + expectErr: false, + }, { name: "custom containerd", endPoint: "/var/run/test1/containerd.sock", @@ -119,6 +133,14 @@ func Test_GetRuntimeHandler(t *testing.T) { expectRuntimeHandler: "PouchRuntimeHandler", expectErr: false, }, + { + name: "custom crio", + endPoint: "/var/run/test4/crio.sock", + flag: "test4/crio.sock", + runtimeType: "cri-o", + expectRuntimeHandler: "CrioRuntimeHandler", + expectErr: false, + }, } for _, tt := range tests { diff --git a/pkg/koordlet/util/system/cgroup_driver.go b/pkg/koordlet/util/system/cgroup_driver.go index c391fdae9d..1998a5abcc 100644 --- a/pkg/koordlet/util/system/cgroup_driver.go +++ b/pkg/koordlet/util/system/cgroup_driver.go @@ -50,6 +50,7 @@ const ( RuntimeTypeDocker = "docker" RuntimeTypeContainerd = "containerd" RuntimeTypePouch = "pouch" + RuntimeTypeCrio = "cri-o" RuntimeTypeUnknown = "unknown" ) @@ -106,6 +107,8 @@ var cgroupPathFormatterInSystemd = Formatter{ return RuntimeTypeContainerd, fmt.Sprintf("cri-containerd-%s.scope", hashID[1]), nil case RuntimeTypePouch: return RuntimeTypePouch, fmt.Sprintf("pouch-%s.scope", hashID[1]), nil + case RuntimeTypeCrio: + return RuntimeTypeCrio, fmt.Sprintf("crio-%s.scope", hashID[1]), nil default: return RuntimeTypeUnknown, "", fmt.Errorf("unknown container protocol %s", id) } @@ -123,7 +126,6 @@ var cgroupPathFormatterInSystemd = Formatter{ prefix: "kubepods-burstable-pod", suffix: ".slice", }, - { prefix: "kubepods-pod", suffix: ".slice", @@ -150,6 +152,10 @@ var cgroupPathFormatterInSystemd = Formatter{ prefix: "cri-containerd-", suffix: ".scope", }, + { + prefix: "crio-", + suffix: ".scope", + }, } for i := range patterns { @@ -157,6 +163,9 @@ var cgroupPathFormatterInSystemd = Formatter{ return basename[len(patterns[i].prefix) : len(basename)-len(patterns[i].suffix)], nil } } + if strings.HasPrefix(basename, "crio-") { + return basename[len("crio-"):], nil + } return "", fmt.Errorf("fail to parse container id: %v", basename) }, } @@ -182,7 +191,7 @@ var cgroupPathFormatterInCgroupfs = Formatter{ if len(hashID) < 2 { return RuntimeTypeUnknown, "", fmt.Errorf("parse container id %s failed", id) } - if hashID[0] == RuntimeTypeDocker || hashID[0] == RuntimeTypeContainerd || hashID[0] == RuntimeTypePouch { + if hashID[0] == RuntimeTypeDocker || hashID[0] == RuntimeTypeContainerd || hashID[0] == RuntimeTypePouch || hashID[0] == RuntimeTypeCrio { return hashID[0], fmt.Sprintf("%s", hashID[1]), nil } else { return RuntimeTypeUnknown, "", fmt.Errorf("unknown container protocol %s", id) diff --git a/pkg/koordlet/util/system/cgroup_driver_test.go b/pkg/koordlet/util/system/cgroup_driver_test.go index a20e218060..f3d6971f22 100644 --- a/pkg/koordlet/util/system/cgroup_driver_test.go +++ b/pkg/koordlet/util/system/cgroup_driver_test.go @@ -108,6 +108,10 @@ func Test_ParseContainerIDSystemd(t *testing.T) { basename: "cri-containerd-12345.scope", expeceted: "12345", }, + { + basename: "crio-12345.scope", + expeceted: "12345", + }, { basename: "12345", wantError: true, @@ -179,6 +183,13 @@ func Test_SystemdCgroupPathContainerDirFn(t *testing.T) { wantDirName: "pouch-testPouchContainerID.scope", wantError: false, }, + { + name: "cri-o", + containerID: "cri-o://testCrioContainerID", + wantType: RuntimeTypeCrio, + wantDirName: "crio-testCrioContainerID.scope", + wantError: false, + }, { name: "bad-format", containerID: "bad-format-id", @@ -224,6 +235,13 @@ func Test_CgroupfsCgroupPathContainerDirFn(t *testing.T) { wantDirName: "testPouchContainerID", wantError: false, }, + { + name: "cri-o", + containerID: "cri-o://testCrioContainerID", + wantType: RuntimeTypeCrio, + wantDirName: "testCrioContainerID", + wantError: false, + }, { name: "bad-format", containerID: "bad-format-id", diff --git a/pkg/koordlet/util/system/config.go b/pkg/koordlet/util/system/config.go index 09cf16fb1e..7941775e32 100644 --- a/pkg/koordlet/util/system/config.go +++ b/pkg/koordlet/util/system/config.go @@ -47,6 +47,7 @@ type Config struct { ContainerdEndPoint string PouchEndpoint string DockerEndPoint string + CrioEndPoint string DefaultRuntimeType string }