From d70932ac3897441c1beadf484ba72f90f514e068 Mon Sep 17 00:00:00 2001 From: Liu Zhenwei Date: Fri, 18 Mar 2022 13:48:28 +0800 Subject: [PATCH 1/3] support cri-o imageruntime Signed-off-by: Liu Zhenwei --- pkg/daemon/criruntime/factory.go | 21 +++ pkg/daemon/criruntime/imageruntime/crio.go | 177 +++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 pkg/daemon/criruntime/imageruntime/crio.go diff --git a/pkg/daemon/criruntime/factory.go b/pkg/daemon/criruntime/factory.go index 879f153183..5c60279c05 100644 --- a/pkg/daemon/criruntime/factory.go +++ b/pkg/daemon/criruntime/factory.go @@ -48,6 +48,7 @@ const ( ContainerRuntimeDocker = "docker" ContainerRuntimeContainerd = "containerd" ContainerRuntimePouch = "pouch" + ContainerRuntimeCRIO = "cri-o" ) type runtimeConfig struct { @@ -107,6 +108,17 @@ func NewFactory(varRunPath string, accountManager daemonutil.ImagePullAccountMan klog.Warningf("Failed to new image service for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) continue } + case ContainerRuntimeCRIO: + addr, _, err := kubeletutil.GetAddressAndDialer(cfg.runtimeRemoteURI) + if err != nil { + klog.Warningf("Failed to get address for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) + continue + } + imageService, err = runtimeimage.NewCrioImageService(addr, accountManager) + if err != nil { + klog.Warningf("Failed to new image service for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) + continue + } } if _, err = imageService.ListImages(context.TODO()); err != nil { klog.Warningf("Failed to list images for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) @@ -210,5 +222,14 @@ func detectRuntime(varRunPath string) []runtimeConfig { } } + // cri-o + { + if _, err = os.Stat(fmt.Sprintf("%s/crio/crio.sock", varRunPath)); err == nil { + cfgs = append(cfgs, runtimeConfig{ + runtimeType: ContainerRuntimeCRIO, + runtimeRemoteURI: fmt.Sprintf("unix://%s/crio/crio.sock", varRunPath), + }) + } + } return cfgs } diff --git a/pkg/daemon/criruntime/imageruntime/crio.go b/pkg/daemon/criruntime/imageruntime/crio.go new file mode 100644 index 0000000000..d97edb63b0 --- /dev/null +++ b/pkg/daemon/criruntime/imageruntime/crio.go @@ -0,0 +1,177 @@ +/* +Copyright 2021 The Kruise Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imageruntime + +import ( + "context" + daemonutil "github.com/openkruise/kruise/pkg/daemon/util" + "github.com/pkg/errors" + "google.golang.org/grpc" + "io" + v1 "k8s.io/api/core/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/kubelet/cri/remote/util" + "sync" + "time" +) + +const maxMsgSize = 1024 * 1024 * 16 + +// NewCrioImageService create a cri-o runtime +func NewCrioImageService(runtimeURI string, accountManager daemonutil.ImagePullAccountManager) (ImageService, error) { + klog.V(3).InfoS("Connecting to image service", "endpoint", runtimeURI) + addr, dialer, err := util.GetAddressAndDialer(runtimeURI) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure(), grpc.WithContextDialer(dialer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize))) + if err != nil { + klog.ErrorS(err, "Connect remote image service failed", "address", addr) + return nil, err + } + + klog.V(4).InfoS("Finding the CRI API image version") + imageClient := runtimeapi.NewImageServiceClient(conn) + + if _, err := imageClient.ImageFsInfo(ctx, &runtimeapi.ImageFsInfoRequest{}); err == nil { + klog.V(2).InfoS("Using CRI v1 image API") + } + + return &crioImageService{ + accountManager: accountManager, + criImageClient: imageClient, + }, nil +} + +type crioImageService struct { + accountManager daemonutil.ImagePullAccountManager + criImageClient runtimeapi.ImageServiceClient + sync.Mutex +} + +// PullImage implements ImageService.PullImage. +func (c *crioImageService) PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret) (ImagePullStatusReader, error) { + registry := daemonutil.ParseRegistry(imageName) + fullImageName := imageName + ":" + tag + // Reader + pipeR, pipeW := io.Pipe() + defer pipeW.Close() + + var auth *runtimeapi.AuthConfig + pullImageReq := &runtimeapi.PullImageRequest{ + Image: &runtimeapi.ImageSpec{ + Image: fullImageName, + Annotations: make(map[string]string), + }, + Auth: auth, //default is nil + } + var err error + if len(pullSecrets) > 0 { + authInfos, err := convertToRegistryAuths(pullSecrets, registry) + if err == nil { + var pullErrs []error + for _, authInfo := range authInfos { + var pullErr error + klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username) + pullImageReq.Auth = &runtimeapi.AuthConfig{ + Username: authInfo.Username, + Password: authInfo.Password, + } + _, pullErr = c.criImageClient.PullImage(ctx, pullImageReq) + if pullErr == nil { + pipeW.CloseWithError(io.EOF) + return newImagePullStatusReader(pipeR), nil + } + c.handleRuntimeError(pullErr) + klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr) + pullErrs = append(pullErrs, pullErr) + + } + if len(pullErrs) > 0 { + err = utilerrors.NewAggregate(pullErrs) + } + } + } + + // Try the default secret + if c.accountManager != nil { + var authInfo *daemonutil.AuthInfo + var defaultErr error + authInfo, defaultErr = c.accountManager.GetAccountInfo(registry) + if defaultErr != nil { + klog.Warningf("Failed to get account for registry %v, err %v", registry, defaultErr) + // When the default account acquisition fails, try to pull anonymously + } else if authInfo != nil { + klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username) + pullImageReq.Auth = &runtimeapi.AuthConfig{ + Username: authInfo.Username, + Password: authInfo.Password, + } + _, err = c.criImageClient.PullImage(ctx, pullImageReq) + if err == nil { + pipeW.CloseWithError(io.EOF) + return newImagePullStatusReader(pipeR), nil + } + c.handleRuntimeError(err) + klog.Warningf("Failed to pull image %v:%v, err %v", imageName, tag, err) + return nil, err + } + } + if err != nil { + return nil, err + } + + // Anonymous pull + _, err = c.criImageClient.PullImage(ctx, pullImageReq) + if err != nil { + c.handleRuntimeError(err) + return nil, errors.Wrapf(err, "Failed to pull image reference %q", fullImageName) + } + pipeW.CloseWithError(io.EOF) + return newImagePullStatusReader(pipeR), nil +} + +// ListImages implements ImageService.ListImages. +func (c *crioImageService) ListImages(ctx context.Context) ([]ImageInfo, error) { + listImagesReq := &runtimeapi.ListImagesRequest{} + listImagesResp, err := c.criImageClient.ListImages(ctx, listImagesReq) + if err != nil { + c.handleRuntimeError(err) + return nil, err + } + collection := make([]ImageInfo, 0, len(listImagesResp.GetImages())) + for _, img := range listImagesResp.GetImages() { + collection = append(collection, ImageInfo{ + ID: img.GetId(), + RepoTags: img.GetRepoTags(), + RepoDigests: img.GetRepoDigests(), + Size: int64(img.GetSize_()), + }) + } + return collection, nil +} + +func (c *crioImageService) handleRuntimeError(err error) { + if daemonutil.FilterCloseErr(err) { + c.Lock() + defer c.Unlock() + c.criImageClient = nil + } +} From 50bae03a50a200b91b532f8ca66db1b402115bf1 Mon Sep 17 00:00:00 2001 From: Liu Zhenwei Date: Fri, 18 Mar 2022 14:35:51 +0800 Subject: [PATCH 2/3] fix go lint --- pkg/daemon/criruntime/imageruntime/crio.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/daemon/criruntime/imageruntime/crio.go b/pkg/daemon/criruntime/imageruntime/crio.go index d97edb63b0..23f03b759c 100644 --- a/pkg/daemon/criruntime/imageruntime/crio.go +++ b/pkg/daemon/criruntime/imageruntime/crio.go @@ -15,17 +15,18 @@ package imageruntime import ( "context" + "io" + "sync" + "time" + daemonutil "github.com/openkruise/kruise/pkg/daemon/util" "github.com/pkg/errors" "google.golang.org/grpc" - "io" v1 "k8s.io/api/core/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/kubelet/cri/remote/util" - "sync" - "time" ) const maxMsgSize = 1024 * 1024 * 16 @@ -84,7 +85,8 @@ func (c *crioImageService) PullImage(ctx context.Context, imageName, tag string, } var err error if len(pullSecrets) > 0 { - authInfos, err := convertToRegistryAuths(pullSecrets, registry) + var authInfos []daemonutil.AuthInfo + authInfos, err = convertToRegistryAuths(pullSecrets, registry) if err == nil { var pullErrs []error for _, authInfo := range authInfos { @@ -134,6 +136,7 @@ func (c *crioImageService) PullImage(ctx context.Context, imageName, tag string, return nil, err } } + if err != nil { return nil, err } From e3568bc112b142a65b4020d0d30fa1d72f2bc924 Mon Sep 17 00:00:00 2001 From: Liu Zhenwei Date: Fri, 18 Mar 2022 15:05:33 +0800 Subject: [PATCH 3/3] fix remove handleRuntimeError for CRI client --- pkg/daemon/criruntime/imageruntime/crio.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/pkg/daemon/criruntime/imageruntime/crio.go b/pkg/daemon/criruntime/imageruntime/crio.go index 23f03b759c..0d0dc151e7 100644 --- a/pkg/daemon/criruntime/imageruntime/crio.go +++ b/pkg/daemon/criruntime/imageruntime/crio.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Kruise Authors. +Copyright 2022 The Kruise Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -16,7 +16,6 @@ package imageruntime import ( "context" "io" - "sync" "time" daemonutil "github.com/openkruise/kruise/pkg/daemon/util" @@ -64,7 +63,6 @@ func NewCrioImageService(runtimeURI string, accountManager daemonutil.ImagePullA type crioImageService struct { accountManager daemonutil.ImagePullAccountManager criImageClient runtimeapi.ImageServiceClient - sync.Mutex } // PullImage implements ImageService.PullImage. @@ -101,7 +99,6 @@ func (c *crioImageService) PullImage(ctx context.Context, imageName, tag string, pipeW.CloseWithError(io.EOF) return newImagePullStatusReader(pipeR), nil } - c.handleRuntimeError(pullErr) klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr) pullErrs = append(pullErrs, pullErr) @@ -131,7 +128,6 @@ func (c *crioImageService) PullImage(ctx context.Context, imageName, tag string, pipeW.CloseWithError(io.EOF) return newImagePullStatusReader(pipeR), nil } - c.handleRuntimeError(err) klog.Warningf("Failed to pull image %v:%v, err %v", imageName, tag, err) return nil, err } @@ -144,7 +140,6 @@ func (c *crioImageService) PullImage(ctx context.Context, imageName, tag string, // Anonymous pull _, err = c.criImageClient.PullImage(ctx, pullImageReq) if err != nil { - c.handleRuntimeError(err) return nil, errors.Wrapf(err, "Failed to pull image reference %q", fullImageName) } pipeW.CloseWithError(io.EOF) @@ -156,7 +151,6 @@ func (c *crioImageService) ListImages(ctx context.Context) ([]ImageInfo, error) listImagesReq := &runtimeapi.ListImagesRequest{} listImagesResp, err := c.criImageClient.ListImages(ctx, listImagesReq) if err != nil { - c.handleRuntimeError(err) return nil, err } collection := make([]ImageInfo, 0, len(listImagesResp.GetImages())) @@ -170,11 +164,3 @@ func (c *crioImageService) ListImages(ctx context.Context) ([]ImageInfo, error) } return collection, nil } - -func (c *crioImageService) handleRuntimeError(err error) { - if daemonutil.FilterCloseErr(err) { - c.Lock() - defer c.Unlock() - c.criImageClient = nil - } -}