diff --git a/apis/apps/v1alpha1/imagepulljob_types.go b/apis/apps/v1alpha1/imagepulljob_types.go index 0b897a1a74..5de980a6dc 100644 --- a/apis/apps/v1alpha1/imagepulljob_types.go +++ b/apis/apps/v1alpha1/imagepulljob_types.go @@ -61,6 +61,10 @@ type ImagePullJobSpec struct { // CompletionPolicy indicates the completion policy of the job. // Default is Always CompletionPolicyType. CompletionPolicy CompletionPolicy `json:"completionPolicy"` + + // SandboxConfig support attach metadata in PullImage CRI interface during ImagePulljobs + // +optional + SandboxConfig *SandboxConfig `json:"sandboxConfig,omitempty"` } // ImagePullJobPodSelector is a selector over pods diff --git a/apis/apps/v1alpha1/nodeimage_types.go b/apis/apps/v1alpha1/nodeimage_types.go index 535343592c..f53a458d2c 100644 --- a/apis/apps/v1alpha1/nodeimage_types.go +++ b/apis/apps/v1alpha1/nodeimage_types.go @@ -38,6 +38,10 @@ type ImageSpec struct { // Tags is a list of versions of this image Tags []ImageTagSpec `json:"tags"` + + // SandboxConfig support attach metadata in PullImage CRI interface during ImagePulljobs + // +optional + SandboxConfig *SandboxConfig `json:"sandboxConfig,omitempty"` } // ReferenceObject comprises a resource name, with a mandatory namespace, diff --git a/apis/apps/v1alpha1/sandboxconfig_types.go b/apis/apps/v1alpha1/sandboxconfig_types.go new file mode 100644 index 0000000000..3414daf0dd --- /dev/null +++ b/apis/apps/v1alpha1/sandboxconfig_types.go @@ -0,0 +1,25 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// SandboxConfig support attach metadata in PullImage CRI interface during ImagePulljobs +type SandboxConfig struct { + // +optional + Labels map[string]string `json:"labels,omitempty"` + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go index 3055dfbb3d..ffcc088b30 100644 --- a/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -1343,6 +1343,11 @@ func (in *ImagePullJobSpec) DeepCopyInto(out *ImagePullJobSpec) { (*in).DeepCopyInto(*out) } in.CompletionPolicy.DeepCopyInto(&out.CompletionPolicy) + if in.SandboxConfig != nil { + in, out := &in.SandboxConfig, &out.SandboxConfig + *out = new(SandboxConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePullJobSpec. @@ -1398,6 +1403,11 @@ func (in *ImageSpec) DeepCopyInto(out *ImageSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.SandboxConfig != nil { + in, out := &in.SandboxConfig, &out.SandboxConfig + *out = new(SandboxConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageSpec. @@ -2482,6 +2492,35 @@ func (in *RollingUpdateStatefulSetStrategy) DeepCopy() *RollingUpdateStatefulSet return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxConfig) DeepCopyInto(out *SandboxConfig) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxConfig. +func (in *SandboxConfig) DeepCopy() *SandboxConfig { + if in == nil { + return nil + } + out := new(SandboxConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ShareVolumePolicy) DeepCopyInto(out *ShareVolumePolicy) { *out = *in diff --git a/config/crd/bases/apps.kruise.io_imagepulljobs.yaml b/config/crd/bases/apps.kruise.io_imagepulljobs.yaml index f6da130418..9641958484 100644 --- a/config/crd/bases/apps.kruise.io_imagepulljobs.yaml +++ b/config/crd/bases/apps.kruise.io_imagepulljobs.yaml @@ -175,6 +175,19 @@ spec: items: type: string type: array + sandboxConfig: + description: SandboxConfig support attach metadata in PullImage CRI + interface during ImagePulljobs + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object selector: description: Selector is a query over nodes that should match the job. nil to match all nodes. diff --git a/config/crd/bases/apps.kruise.io_nodeimages.yaml b/config/crd/bases/apps.kruise.io_nodeimages.yaml index 49cb2833a2..ba601f929c 100644 --- a/config/crd/bases/apps.kruise.io_nodeimages.yaml +++ b/config/crd/bases/apps.kruise.io_nodeimages.yaml @@ -80,6 +80,19 @@ spec: type: string type: object type: array + sandboxConfig: + description: SandboxConfig support attach metadata in PullImage + CRI interface during ImagePulljobs + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object tags: description: Tags is a list of versions of this image items: diff --git a/pkg/controller/imagepulljob/imagepulljob_controller.go b/pkg/controller/imagepulljob/imagepulljob_controller.go index 3c1f3d6149..ebe6da03be 100644 --- a/pkg/controller/imagepulljob/imagepulljob_controller.go +++ b/pkg/controller/imagepulljob/imagepulljob_controller.go @@ -253,7 +253,6 @@ func (r *ReconcileImagePullJob) syncNodeImages(job *appsv1alpha1.ImagePullJob, n ownerRef := getOwnerRef(job) secrets := getSecrets(job) pullPolicy := getImagePullPolicy(job) - now := metav1.NewTime(r.clock.Now()) imageName, imageTag, _ := daemonutil.NormalizeImageRefToNameTag(job.Spec.Image) for i := 0; i < parallelism; i++ { @@ -267,6 +266,7 @@ func (r *ReconcileImagePullJob) syncNodeImages(job *appsv1alpha1.ImagePullJob, n nodeImage.Spec.Images = make(map[string]appsv1alpha1.ImageSpec, 1) } imageSpec := nodeImage.Spec.Images[imageName] + imageSpec.SandboxConfig = job.Spec.SandboxConfig for _, secret := range secrets { if !containsObject(imageSpec.PullSecrets, secret) { diff --git a/pkg/daemon/criruntime/factory.go b/pkg/daemon/criruntime/factory.go index 707c9fac0c..dd59855aa1 100644 --- a/pkg/daemon/criruntime/factory.go +++ b/pkg/daemon/criruntime/factory.go @@ -102,18 +102,7 @@ func NewFactory(varRunPath string, accountManager daemonutil.ImagePullAccountMan klog.Warningf("Failed to new image service for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) continue } - case ContainerRuntimeContainerd: - addr, _, err := kubeletutil.GetAddressAndDialer(cfg.runtimeRemoteURI) - if err != nil { - klog.Warningf("Failed to get address for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) - continue - } - imageService, err = runtimeimage.NewContainerdImageService(addr, accountManager) - if err != nil { - klog.Warningf("Failed to new image service for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) - continue - } - case ContainerRuntimeCommonCRI: + case ContainerRuntimeContainerd, ContainerRuntimeCommonCRI: addr, _, err := kubeletutil.GetAddressAndDialer(cfg.runtimeRemoteURI) if err != nil { klog.Warningf("Failed to get address for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err) @@ -227,7 +216,7 @@ func detectRuntime(varRunPath string) (cfgs []runtimeConfig) { } } - // containerd + // containerd, with the same behavior of pullImage as commonCRI { if _, err = os.Stat(fmt.Sprintf("%s/containerd.sock", varRunPath)); err == nil { cfgs = append(cfgs, runtimeConfig{ diff --git a/pkg/daemon/criruntime/imageruntime/containerd.go b/pkg/daemon/criruntime/imageruntime/containerd.go index 1844334d67..7fdd154a2d 100644 --- a/pkg/daemon/criruntime/imageruntime/containerd.go +++ b/pkg/daemon/criruntime/imageruntime/containerd.go @@ -27,6 +27,8 @@ import ( "net/url" "time" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/alibaba/pouch/pkg/jsonstream" "github.com/containerd/containerd" "github.com/containerd/containerd/errdefs" @@ -85,7 +87,7 @@ type containerdImageClient struct { } // PullImage implements ImageService.PullImage. -func (d *containerdImageClient) PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret) (ImagePullStatusReader, error) { +func (d *containerdImageClient) PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret, _ *appsv1alpha1.SandboxConfig) (ImagePullStatusReader, error) { ctx = namespaces.WithNamespace(ctx, k8sContainerdNamespace) if tag == "" { diff --git a/pkg/daemon/criruntime/imageruntime/cri.go b/pkg/daemon/criruntime/imageruntime/cri.go index ba317aad8c..2dad7c1cae 100644 --- a/pkg/daemon/criruntime/imageruntime/cri.go +++ b/pkg/daemon/criruntime/imageruntime/cri.go @@ -18,6 +18,8 @@ import ( "io" "time" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + daemonutil "github.com/openkruise/kruise/pkg/daemon/util" "github.com/pkg/errors" "google.golang.org/grpc" @@ -66,7 +68,7 @@ type commonCRIImageService struct { } // PullImage implements ImageService.PullImage. -func (c *commonCRIImageService) PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret) (ImagePullStatusReader, error) { +func (c *commonCRIImageService) PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret, sandboxConfig *appsv1alpha1.SandboxConfig) (ImagePullStatusReader, error) { registry := daemonutil.ParseRegistry(imageName) fullImageName := imageName + ":" + tag // Reader @@ -81,6 +83,12 @@ func (c *commonCRIImageService) PullImage(ctx context.Context, imageName, tag st }, Auth: auth, //default is nil } + if sandboxConfig != nil { + pullImageReq.SandboxConfig = &runtimeapi.PodSandboxConfig{ + Annotations: sandboxConfig.Annotations, + Labels: sandboxConfig.Labels, + } + } var err error if len(pullSecrets) > 0 { var authInfos []daemonutil.AuthInfo diff --git a/pkg/daemon/criruntime/imageruntime/docker.go b/pkg/daemon/criruntime/imageruntime/docker.go index 84e607fca1..39eaa65d12 100644 --- a/pkg/daemon/criruntime/imageruntime/docker.go +++ b/pkg/daemon/criruntime/imageruntime/docker.go @@ -21,6 +21,8 @@ import ( "io" "sync" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + dockertypes "github.com/docker/docker/api/types" dockerapi "github.com/docker/docker/client" daemonutil "github.com/openkruise/kruise/pkg/daemon/util" @@ -68,7 +70,7 @@ func (d *dockerImageService) handleRuntimeError(err error) { } } -func (d *dockerImageService) PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret) (reader ImagePullStatusReader, err error) { +func (d *dockerImageService) PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret, _ *appsv1alpha1.SandboxConfig) (reader ImagePullStatusReader, err error) { if err = d.createRuntimeClientIfNecessary(); err != nil { return nil, err } diff --git a/pkg/daemon/criruntime/imageruntime/interface.go b/pkg/daemon/criruntime/imageruntime/interface.go index 82fb6425ab..ca9eb2b644 100644 --- a/pkg/daemon/criruntime/imageruntime/interface.go +++ b/pkg/daemon/criruntime/imageruntime/interface.go @@ -19,6 +19,8 @@ package imageruntime import ( "context" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + v1 "k8s.io/api/core/v1" ) @@ -46,6 +48,6 @@ type ImagePullStatusReader interface { } type ImageService interface { - PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret) (ImagePullStatusReader, error) + PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret, sandboxConfig *appsv1alpha1.SandboxConfig) (ImagePullStatusReader, error) ListImages(ctx context.Context) ([]ImageInfo, error) } diff --git a/pkg/daemon/criruntime/imageruntime/pouch.go b/pkg/daemon/criruntime/imageruntime/pouch.go index 0e1e4efbcf..7ee06b69b7 100644 --- a/pkg/daemon/criruntime/imageruntime/pouch.go +++ b/pkg/daemon/criruntime/imageruntime/pouch.go @@ -22,6 +22,8 @@ import ( "io" "sync" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + pouchfilters "github.com/alibaba/pouch/apis/filters" pouchtypes "github.com/alibaba/pouch/apis/types" pouchapi "github.com/alibaba/pouch/client" @@ -70,7 +72,7 @@ func (d *pouchImageService) handleRuntimeError(err error) { } } -func (d *pouchImageService) PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret) (reader ImagePullStatusReader, err error) { +func (d *pouchImageService) PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret, _ *appsv1alpha1.SandboxConfig) (reader ImagePullStatusReader, err error) { if err = d.createRuntimeClientIfNecessary(); err != nil { return nil, err } diff --git a/pkg/daemon/imagepuller/imagepuller_worker.go b/pkg/daemon/imagepuller/imagepuller_worker.go index 2d2169b0de..1ce9400cf2 100644 --- a/pkg/daemon/imagepuller/imagepuller_worker.go +++ b/pkg/daemon/imagepuller/imagepuller_worker.go @@ -214,7 +214,7 @@ func (w *realWorkerPool) Sync(spec *appsv1alpha1.ImageSpec, status *appsv1alpha1 _, ok := w.pullWorkers[tagSpec.Tag] if !ok { - worker := newPullWorker(w.name, tagSpec, secrets, w.runtime, w, ref, w.eventRecorder) + worker := newPullWorker(w.name, tagSpec, spec.SandboxConfig, secrets, w.runtime, w, ref, w.eventRecorder) w.pullWorkers[tagSpec.Tag] = worker } } @@ -262,10 +262,11 @@ func (w *realWorkerPool) UpdateStatus(status *appsv1alpha1.ImageTagStatus) { w.tagStatuses[status.Tag] = status } -func newPullWorker(name string, tagSpec appsv1alpha1.ImageTagSpec, secrets []v1.Secret, runtime runtimeimage.ImageService, statusUpdater imageStatusUpdater, ref *v1.ObjectReference, eventRecorder record.EventRecorder) *pullWorker { +func newPullWorker(name string, tagSpec appsv1alpha1.ImageTagSpec, sandboxConfig *appsv1alpha1.SandboxConfig, secrets []v1.Secret, runtime runtimeimage.ImageService, statusUpdater imageStatusUpdater, ref *v1.ObjectReference, eventRecorder record.EventRecorder) *pullWorker { o := &pullWorker{ name: name, tagSpec: tagSpec, + sandboxConfig: sandboxConfig, secrets: secrets, runtime: runtime, statusUpdater: statusUpdater, @@ -283,6 +284,7 @@ type pullWorker struct { name string tagSpec appsv1alpha1.ImageTagSpec + sandboxConfig *appsv1alpha1.SandboxConfig secrets []v1.Secret runtime runtimeimage.ImageService statusUpdater imageStatusUpdater @@ -435,7 +437,7 @@ func (w *pullWorker) doPullImage(ctx context.Context, newStatus *appsv1alpha1.Im var statusReader runtimeimage.ImagePullStatusReader pullChan := make(chan struct{}) go func() { - statusReader, err = w.runtime.PullImage(ctx, w.name, tag, w.secrets) + statusReader, err = w.runtime.PullImage(ctx, w.name, tag, w.secrets, w.sandboxConfig) close(pullChan) }()