Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

optimize the situation where the ImageID remains unchanged but changing the Tag for in-place upgrades #1284

Closed
wants to merge 3 commits into from
Closed

optimize the situation where the ImageID remains unchanged but changing the Tag for in-place upgrades #1284

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1d12e97
record image id into runtime_container_meta
diannaowa May 16, 2023
f64d65c
record image id into runtime_container_meta
diannaowa May 16, 2023
3e28319
record image id into runtime_container_meta
diannaowa May 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 37 additions & 23 deletions pkg/daemon/containermeta/container_meta_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,6 @@ import (
"sync"
"time"

appspub "github.com/openkruise/kruise/apis/apps/pub"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/client"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
daemonruntime "github.com/openkruise/kruise/pkg/daemon/criruntime"
"github.com/openkruise/kruise/pkg/daemon/kuberuntime"
daemonoptions "github.com/openkruise/kruise/pkg/daemon/options"
"github.com/openkruise/kruise/pkg/features"
"github.com/openkruise/kruise/pkg/util"
utilcontainermeta "github.com/openkruise/kruise/pkg/util/containermeta"
"github.com/openkruise/kruise/pkg/util/expectations"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -56,6 +44,20 @@ import (
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
kubeletcontainer "k8s.io/kubernetes/pkg/kubelet/container"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"

appspub "github.com/openkruise/kruise/apis/apps/pub"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/client"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
daemonruntime "github.com/openkruise/kruise/pkg/daemon/criruntime"
runtimeimage "github.com/openkruise/kruise/pkg/daemon/criruntime/imageruntime"
"github.com/openkruise/kruise/pkg/daemon/kuberuntime"
daemonoptions "github.com/openkruise/kruise/pkg/daemon/options"
"github.com/openkruise/kruise/pkg/features"
"github.com/openkruise/kruise/pkg/util"
utilcontainermeta "github.com/openkruise/kruise/pkg/util/containermeta"
"github.com/openkruise/kruise/pkg/util/expectations"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
)

var (
Expand Down Expand Up @@ -261,7 +263,7 @@ func (c *Controller) sync(key string) (retErr error) {
resourceVersionExpectation.Delete(pod)
}

criRuntime, kubeRuntime, err := c.getRuntimeForPod(pod)
criRuntime, imageService, kubeRuntime, err := c.getRuntimeForPod(pod)
if err != nil {
klog.Errorf("Failed to get runtime for Pod %s/%s: %v", namespace, name, err)
return nil
Expand All @@ -287,7 +289,7 @@ func (c *Controller) sync(key string) (retErr error) {
if err != nil {
klog.Warningf("Failed to get old runtime meta from Pod %s/%s: %v", namespace, name, err)
}
newMetaSet := c.manageContainerMetaSet(pod, kubePodStatus, oldMetaSet, criRuntime)
newMetaSet := c.manageContainerMetaSet(pod, kubePodStatus, oldMetaSet, criRuntime, imageService)

return c.reportContainerMetaSet(pod, oldMetaSet, newMetaSet)
}
Expand Down Expand Up @@ -317,7 +319,7 @@ func (c *Controller) reportContainerMetaSet(pod *v1.Pod, oldMetaSet, newMetaSet
return nil
}

func (c *Controller) manageContainerMetaSet(pod *v1.Pod, kubePodStatus *kubeletcontainer.PodStatus, oldMetaSet *appspub.RuntimeContainerMetaSet, criRuntime criapi.RuntimeService) *appspub.RuntimeContainerMetaSet {
func (c *Controller) manageContainerMetaSet(pod *v1.Pod, kubePodStatus *kubeletcontainer.PodStatus, oldMetaSet *appspub.RuntimeContainerMetaSet, criRuntime criapi.RuntimeService, imageService runtimeimage.ImageService) *appspub.RuntimeContainerMetaSet {
var err error
metaSet := appspub.RuntimeContainerMetaSet{Containers: make([]appspub.RuntimeContainerMeta, 0, len(pod.Status.ContainerStatuses))}
for _, cs := range pod.Status.ContainerStatuses {
Expand All @@ -331,10 +333,19 @@ func (c *Controller) manageContainerMetaSet(pod *v1.Pod, kubePodStatus *kubeletc
}

var containerMeta *appspub.RuntimeContainerMeta
var imageID string
if oldMetaSet != nil {
for i := range oldMetaSet.Containers {
if oldMetaSet.Containers[i].ContainerID == status.ID.String() {
containerMeta = oldMetaSet.Containers[i].DeepCopy()
} else {
// container restarted
imageStatus, err := imageService.ImageStatus(context.TODO(), containerSpec.Image)
if err != nil {
klog.Errorf("Failed get image id from CRI fro %s (%s) %v", containerSpec.Name, containerSpec.Image, err)
} else {
imageID = imageStatus.ID
}
}
}
}
Expand All @@ -346,6 +357,9 @@ func (c *Controller) manageContainerMetaSet(pod *v1.Pod, kubePodStatus *kubeletc
Hashes: appspub.RuntimeContainerHashes{PlainHash: status.Hash},
}
}
if imageID != "" {
containerMeta.ImageID = imageID
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

28% of developers fix this issue

typecheck: containerMeta.ImageID undefined (type *pub.RuntimeContainerMeta has no field or method ImageID)

ℹ️ Expand to see all @sonatype-lift commands

You can reply with the following commands. For example, reply with @sonatype-lift ignoreall to leave out all findings.

Command Usage
@sonatype-lift ignore Leave out the above finding from this PR
@sonatype-lift ignoreall Leave out all the existing findings from this PR
@sonatype-lift exclude <file|issue|path|tool> Exclude specified file|issue|path|tool from Lift findings by updating your config.toml file

Note: When talking to LiftBot, you need to refresh the page to see its response.
Click here to add LiftBot to another repo.

}
if utilfeature.DefaultFeatureGate.Enabled(features.InPlaceUpdateEnvFromMetadata) {
envHasher := utilcontainermeta.NewEnvFromMetadataHasher()

Expand Down Expand Up @@ -405,9 +419,9 @@ func wrapEnvGetter(criRuntime criapi.RuntimeService, containerID, logID string)
}
}

func (c *Controller) getRuntimeForPod(pod *v1.Pod) (criapi.RuntimeService, kuberuntime.Runtime, error) {
func (c *Controller) getRuntimeForPod(pod *v1.Pod) (criapi.RuntimeService, runtimeimage.ImageService, kuberuntime.Runtime, error) {
if len(pod.Status.ContainerStatuses) == 0 {
return nil, nil, fmt.Errorf("empty containerStatuses in pod status")
return nil, nil, nil, fmt.Errorf("empty containerStatuses in pod status")
}

var existingID string
Expand All @@ -418,21 +432,21 @@ func (c *Controller) getRuntimeForPod(pod *v1.Pod) (criapi.RuntimeService, kuber
}
}
if existingID == "" {
return nil, nil, nil
return nil, nil, nil, nil
}

containerID := kubeletcontainer.ContainerID{}
if err := containerID.ParseString(pod.Status.ContainerStatuses[0].ContainerID); err != nil {
return nil, nil, fmt.Errorf("failed to parse containerID %s: %v", pod.Status.ContainerStatuses[0].ContainerID, err)
return nil, nil, nil, fmt.Errorf("failed to parse containerID %s: %v", pod.Status.ContainerStatuses[0].ContainerID, err)
} else if containerID.Type == "" {
return nil, nil, fmt.Errorf("no runtime name in containerID %s", pod.Status.ContainerStatuses[0].ContainerID)
return nil, nil, nil, fmt.Errorf("no runtime name in containerID %s", pod.Status.ContainerStatuses[0].ContainerID)
}

runtimeName := containerID.Type
runtimeService := c.runtimeFactory.GetRuntimeServiceByName(runtimeName)
if runtimeService == nil {
return nil, nil, fmt.Errorf("not found runtime service for %s in daemon", runtimeName)
return nil, nil, nil, fmt.Errorf("not found runtime service for %s in daemon", runtimeName)
}

return runtimeService, kuberuntime.NewGenericRuntime(runtimeName, runtimeService, nil, &http.Client{}), nil
imageService := c.runtimeFactory.GetImageService()
return runtimeService, imageService, kuberuntime.NewGenericRuntime(runtimeName, runtimeService, nil, &http.Client{}), nil
}
5 changes: 3 additions & 2 deletions pkg/daemon/criruntime/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,14 @@ import (
"os"
"time"

runtimeimage "github.com/openkruise/kruise/pkg/daemon/criruntime/imageruntime"
daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
criapi "k8s.io/cri-api/pkg/apis"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
"k8s.io/klog/v2"
criremote "k8s.io/kubernetes/pkg/kubelet/cri/remote"
kubeletutil "k8s.io/kubernetes/pkg/kubelet/util"

runtimeimage "github.com/openkruise/kruise/pkg/daemon/criruntime/imageruntime"
daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
)

const (
Expand Down
7 changes: 6 additions & 1 deletion pkg/daemon/criruntime/imageruntime/containerd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,15 @@ import (
"github.com/docker/distribution/reference"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
"github.com/pkg/errors"
"golang.org/x/net/http/httpproxy"
"google.golang.org/grpc"
v1 "k8s.io/api/core/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
"k8s.io/klog/v2"

daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
)

const (
Expand Down Expand Up @@ -126,6 +127,10 @@ func (d *containerdImageClient) ListImages(ctx context.Context) ([]ImageInfo, er
return collection, nil
}

func (d *containerdImageClient) ImageStatus(ctx context.Context, image string) (*ImageInfo, error) {
return nil, fmt.Errorf("not impl")
}

// doPullImage returns pipe reader as ImagePullStatusReader to notify the progressing.
func (d *containerdImageClient) doPullImage(ctx context.Context, ref reference.Named, isSchema1 bool, resolver remotes.Resolver) ImagePullStatusReader {
ongoing := newFetchJobs(ref.String())
Expand Down
24 changes: 23 additions & 1 deletion pkg/daemon/criruntime/imageruntime/cri.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,15 @@ import (

appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"

daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
"github.com/pkg/errors"
"google.golang.org/grpc"
v1 "k8s.io/api/core/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/kubelet/cri/remote/util"

daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
)

const maxMsgSize = 1024 * 1024 * 16
Expand Down Expand Up @@ -172,3 +173,24 @@ func (c *commonCRIImageService) ListImages(ctx context.Context) ([]ImageInfo, er
}
return collection, nil
}

func (c *commonCRIImageService) ImageStatus(ctx context.Context, image string) (*ImageInfo, error) {
imagesReq := &runtimeapi.ImageStatusRequest{
Image: &runtimeapi.ImageSpec{
Image: image,
},
}
img, err := c.criImageClient.ImageStatus(ctx, imagesReq)
if err != nil {
return nil, err
}

info := &ImageInfo{
ID: img.Image.GetId(),
RepoTags: img.Image.GetRepoTags(),
RepoDigests: img.Image.GetRepoDigests(),
Size: int64(img.Image.GetSize_()),
}

return info, nil
}
8 changes: 7 additions & 1 deletion pkg/daemon/criruntime/imageruntime/docker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,19 @@ package imageruntime

import (
"context"
"fmt"
"io"
"sync"

appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"

dockertypes "github.com/docker/docker/api/types"
dockerapi "github.com/docker/docker/client"
daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
v1 "k8s.io/api/core/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/klog/v2"

daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
)

// NewDockerImageService create a docker runtime
Expand Down Expand Up @@ -146,6 +148,10 @@ func (d *dockerImageService) ListImages(ctx context.Context) ([]ImageInfo, error
return newImageCollectionDocker(infos), nil
}

func (d *dockerImageService) ImageStatus(ctx context.Context, image string) (*ImageInfo, error) {
return nil, fmt.Errorf("not impl")
}

func newImageCollectionDocker(infos []dockertypes.ImageSummary) []ImageInfo {
collection := make([]ImageInfo, 0, len(infos))
for _, info := range infos {
Expand Down
1 change: 1 addition & 0 deletions pkg/daemon/criruntime/imageruntime/interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,4 +50,5 @@ type ImagePullStatusReader interface {
type ImageService interface {
PullImage(ctx context.Context, imageName, tag string, pullSecrets []v1.Secret, sandboxConfig *appsv1alpha1.SandboxConfig) (ImagePullStatusReader, error)
ListImages(ctx context.Context) ([]ImageInfo, error)
ImageStatus(ctx context.Context, image string) (*ImageInfo, error)
}
7 changes: 6 additions & 1 deletion pkg/daemon/criruntime/imageruntime/pouch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ import (
pouchfilters "github.com/alibaba/pouch/apis/filters"
pouchtypes "github.com/alibaba/pouch/apis/types"
pouchapi "github.com/alibaba/pouch/client"
daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
v1 "k8s.io/api/core/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/klog/v2"

daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
)

// NewPouchImageService create a pouch runtime client
Expand Down Expand Up @@ -148,6 +149,10 @@ func (d *pouchImageService) ListImages(ctx context.Context) ([]ImageInfo, error)
return newImageCollectionPouch(infos), nil
}

func (d *pouchImageService) ImageStatus(ctx context.Context, image string) (*ImageInfo, error) {
return nil, fmt.Errorf("not impl")
}

func newImageCollectionPouch(infos []pouchtypes.ImageInfo) []ImageInfo {
collection := make([]ImageInfo, 0, len(infos))
for _, info := range infos {
Expand Down