From 000c72550bb6986bef97891c07263ba3079252ab Mon Sep 17 00:00:00 2001 From: "liheng.zms" Date: Fri, 23 Dec 2022 11:21:47 +0800 Subject: [PATCH] move sidecarSet control to openkruise/utils repo Signed-off-by: liheng.zms --- go.mod | 5 +- go.sum | 3 +- pkg/control/pubcontrol/pub_control.go | 2 +- pkg/control/sidecarcontrol/history_control.go | 309 ----- pkg/control/sidecarcontrol/util_test.go | 1061 ----------------- .../sidecarset/sidecarset_controller.go | 27 +- .../sidecarset/sidecarset_controller_test.go | 26 +- .../sidecarset/sidecarset_hotupgrade.go | 15 +- .../sidecarset/sidecarset_hotupgrade_test.go | 10 +- .../sidecarset_pod_event_handler.go | 14 +- .../sidecarset_pod_event_handler_test.go | 8 +- .../sidecarset/sidecarset_processor.go | 117 +- .../sidecarset/sidecarset_processor_test.go | 86 +- .../sidecarset/sidecarset_strategy.go | 42 +- .../sidecarset/sidecarset_strategy_test.go | 18 +- .../container_meta_controller.go | 2 +- .../pod/mutating/persistent_pod_state_test.go | 16 + .../pod/mutating/pod_create_update_handler.go | 24 +- pkg/webhook/pod/mutating/sidecarset.go | 414 +------ .../mutating/sidecarset_hotupgrade_test.go | 79 -- pkg/webhook/pod/mutating/sidecarset_test.go | 1053 ---------------- .../validating/pod_unavailable_budget_test.go | 2 +- .../sidecarset_create_update_handler.go | 2 +- .../mutating/sidecarset_mutating_test.go | 2 +- .../sidecarset_create_update_handler.go | 80 +- .../validating/sidecarset_validating_test.go | 208 +++- pkg/webhook/sidecarset/validating/utils.go | 10 + .../util/controller/webhook_controller.go | 1 - test/e2e/apps/sidecarset.go | 2 +- test/e2e/apps/sidecarset_hotupgrade.go | 4 +- test/e2e/framework/sidecarset_utils.go | 2 +- vendor/github.com/openkruise/utils/.gitignore | 20 + vendor/github.com/openkruise/utils/LICENSE.md | 203 ++++ vendor/github.com/openkruise/utils/Makefile | 31 + vendor/github.com/openkruise/utils/README.md | 1 + vendor/github.com/openkruise/utils/pods.go | 435 +++++++ .../openkruise/utils}/sidecarcontrol/api.go | 26 +- .../openkruise/utils}/sidecarcontrol/hash.go | 9 +- .../utils/sidecarcontrol/history_control.go | 531 +++++++++ .../utils/sidecarcontrol/mutating_pod.go | 334 ++++++ .../utils}/sidecarcontrol/revision_adapter.go | 0 .../sidecarcontrol/sidecarset_control.go | 101 +- .../sidecarcontrol}/sidecarset_hotupgrade.go | 24 +- .../openkruise/utils}/sidecarcontrol/util.go | 96 +- .../utils}/sidecarcontrol/util_hotupgrade.go | 35 - vendor/github.com/openkruise/utils/tools.go | 130 ++ vendor/modules.txt | 4 + 47 files changed, 2353 insertions(+), 3271 deletions(-) delete mode 100644 pkg/control/sidecarcontrol/history_control.go delete mode 100644 pkg/control/sidecarcontrol/util_test.go delete mode 100644 pkg/webhook/pod/mutating/sidecarset_hotupgrade_test.go delete mode 100644 pkg/webhook/pod/mutating/sidecarset_test.go create mode 100644 vendor/github.com/openkruise/utils/.gitignore create mode 100644 vendor/github.com/openkruise/utils/LICENSE.md create mode 100644 vendor/github.com/openkruise/utils/Makefile create mode 100644 vendor/github.com/openkruise/utils/README.md create mode 100644 vendor/github.com/openkruise/utils/pods.go rename {pkg/control => vendor/github.com/openkruise/utils}/sidecarcontrol/api.go (81%) rename {pkg/control => vendor/github.com/openkruise/utils}/sidecarcontrol/hash.go (88%) create mode 100644 vendor/github.com/openkruise/utils/sidecarcontrol/history_control.go create mode 100644 vendor/github.com/openkruise/utils/sidecarcontrol/mutating_pod.go rename {pkg/control => vendor/github.com/openkruise/utils}/sidecarcontrol/revision_adapter.go (100%) rename {pkg/control => vendor/github.com/openkruise/utils}/sidecarcontrol/sidecarset_control.go (71%) rename {pkg/webhook/pod/mutating => vendor/github.com/openkruise/utils/sidecarcontrol}/sidecarset_hotupgrade.go (75%) rename {pkg/control => vendor/github.com/openkruise/utils}/sidecarcontrol/util.go (84%) rename {pkg/control => vendor/github.com/openkruise/utils}/sidecarcontrol/util_hotupgrade.go (68%) create mode 100644 vendor/github.com/openkruise/utils/tools.go diff --git a/go.mod b/go.mod index 36d0b0c6f6..028eaa2106 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/containerd/containerd v1.5.10 github.com/docker/distribution v2.8.0+incompatible github.com/docker/docker v20.10.2+incompatible - github.com/evanphx/json-patch v4.11.0+incompatible github.com/fsnotify/fsnotify v1.4.9 github.com/go-bindata/go-bindata v3.1.2+incompatible github.com/gorilla/mux v1.8.0 @@ -17,6 +16,7 @@ require ( github.com/onsi/gomega v1.15.0 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.2 + github.com/openkruise/utils v0.0.0-20221226104648-0361b39aa451 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 github.com/robfig/cron/v3 v3.0.1 @@ -36,6 +36,7 @@ require ( k8s.io/component-base v0.22.6 k8s.io/component-helpers v0.22.6 k8s.io/cri-api v0.22.6 + k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 k8s.io/klog/v2 v2.9.0 k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c k8s.io/kubernetes v1.22.6 @@ -67,6 +68,7 @@ require ( github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-units v0.4.0 // indirect github.com/emicklei/go-restful v2.9.5+incompatible // indirect + github.com/evanphx/json-patch v4.11.0+incompatible // indirect github.com/go-logr/logr v0.4.0 // indirect github.com/go-openapi/analysis v0.21.2 // indirect github.com/go-openapi/errors v0.20.2 // indirect @@ -130,7 +132,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/cloud-provider v0.22.6 // indirect - k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 // indirect k8s.io/kube-scheduler v0.0.0 // indirect k8s.io/mount-utils v0.22.6 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect diff --git a/go.sum b/go.sum index 0b4b47b8bb..336fb7e4f3 100644 --- a/go.sum +++ b/go.sum @@ -128,7 +128,6 @@ github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -718,6 +717,8 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/openkruise/utils v0.0.0-20221226104648-0361b39aa451 h1:QoGVisvWGhCeOm4hezilxsSlQTA83Yd/HZ7HwEISPwk= +github.com/openkruise/utils v0.0.0-20221226104648-0361b39aa451/go.mod h1:T522/yC4wDMdz1UOX9zOYcuXTPbtZm9zqDvtJcPfZ7M= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= diff --git a/pkg/control/pubcontrol/pub_control.go b/pkg/control/pubcontrol/pub_control.go index 0797f312db..cc139a803d 100644 --- a/pkg/control/pubcontrol/pub_control.go +++ b/pkg/control/pubcontrol/pub_control.go @@ -24,11 +24,11 @@ import ( appspub "github.com/openkruise/kruise/apis/apps/pub" policyv1alpha1 "github.com/openkruise/kruise/apis/policy/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" utilclient "github.com/openkruise/kruise/pkg/util/client" "github.com/openkruise/kruise/pkg/util/controllerfinder" "github.com/openkruise/kruise/pkg/util/inplaceupdate" + "github.com/openkruise/utils/sidecarcontrol" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/sets" diff --git a/pkg/control/sidecarcontrol/history_control.go b/pkg/control/sidecarcontrol/history_control.go deleted file mode 100644 index 1534ef3ca2..0000000000 --- a/pkg/control/sidecarcontrol/history_control.go +++ /dev/null @@ -1,309 +0,0 @@ -/* -Copyright 2021 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sidecarcontrol - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/util" - webhookutil "github.com/openkruise/kruise/pkg/webhook/util" - apps "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/controller/history" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var ( - patchCodec = scheme.Codecs.LegacyCodec(appsv1alpha1.SchemeGroupVersion) -) - -type HistoryControl interface { - CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) - NewRevision(s *appsv1alpha1.SidecarSet, namespace string, revision int64, collisionCount *int32) (*apps.ControllerRevision, error) - NextRevision(revisions []*apps.ControllerRevision) int64 - GetRevisionSelector(s *appsv1alpha1.SidecarSet) labels.Selector - GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error) -} - -type realControl struct { - Client client.Client -} - -func NewHistoryControl(client client.Client) HistoryControl { - return &realControl{ - Client: client, - } -} - -func (r *realControl) NewRevision(s *appsv1alpha1.SidecarSet, namespace string, revision int64, collisionCount *int32) ( - *apps.ControllerRevision, error, -) { - patch, err := r.getPatch(s) - if err != nil { - return nil, err - } - - cr, err := history.NewControllerRevision(s, - s.GetObjectKind().GroupVersionKind(), - s.Labels, - runtime.RawExtension{Raw: patch}, - revision, - collisionCount) - if err != nil { - return nil, err - } - - cr.SetNamespace(namespace) - if cr.Labels == nil { - cr.Labels = make(map[string]string) - } - if cr.ObjectMeta.Annotations == nil { - cr.ObjectMeta.Annotations = make(map[string]string) - } - if s.Annotations[SidecarSetHashAnnotation] != "" { - cr.Annotations[SidecarSetHashAnnotation] = s.Annotations[SidecarSetHashAnnotation] - } - if s.Annotations[SidecarSetHashWithoutImageAnnotation] != "" { - cr.Annotations[SidecarSetHashWithoutImageAnnotation] = s.Annotations[SidecarSetHashWithoutImageAnnotation] - } - if s.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] != "" { - cr.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] = s.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] - } - cr.Labels[SidecarSetKindName] = s.Name - for key, value := range s.Annotations { - cr.ObjectMeta.Annotations[key] = value - } - return cr, nil -} - -// getPatch returns a strategic merge patch that can be applied to restore a SidecarSet to a -// previous version. If the returned error is nil the patch is valid. The current state that we save is just the -// PodSpecTemplate. We can modify this later to encompass more state (or less) and remain compatible with previously -// recorded patches. -func (r *realControl) getPatch(s *appsv1alpha1.SidecarSet) ([]byte, error) { - str, err := runtime.Encode(patchCodec, s) - if err != nil { - return nil, err - } - var raw map[string]interface{} - _ = json.Unmarshal(str, &raw) - - objCopy := make(map[string]interface{}) - specCopy := make(map[string]interface{}) - // only copy some specified fields of s.Spec to specCopy - spec := raw["spec"].(map[string]interface{}) - copySidecarSetSpecRevision(specCopy, spec) - - objCopy["spec"] = specCopy - return json.Marshal(objCopy) -} - -// NextRevision finds the next valid revision number based on revisions. If the length of revisions -// is 0 this is 1. Otherwise, it is 1 greater than the largest revision's Revision. This method -// assumes that revisions has been sorted by Revision. -func (r *realControl) NextRevision(revisions []*apps.ControllerRevision) int64 { - count := len(revisions) - if count <= 0 { - return 1 - } - return revisions[count-1].Revision + 1 -} - -func (r *realControl) GetRevisionSelector(s *appsv1alpha1.SidecarSet) labels.Selector { - labelSelector := &metav1.LabelSelector{ - MatchLabels: map[string]string{ - SidecarSetKindName: s.GetName(), - }, - } - selector, err := util.ValidatedLabelSelectorAsSelector(labelSelector) - if err != nil { - // static error, just panic - panic("Incorrect label selector for ControllerRevision of SidecarSet.") - } - return selector -} - -func (r *realControl) CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) { - if collisionCount == nil { - return nil, fmt.Errorf("collisionCount should not be nil") - } - - // Clone the input - clone := revision.DeepCopy() - - // Continue to attempt to create the revision updating the name with a new hash on each iteration - for { - hash := history.HashControllerRevision(revision, collisionCount) - // Update the revisions name - clone.Name = history.ControllerRevisionName(parent.GetName(), hash) - err := r.Client.Create(context.TODO(), clone) - if errors.IsAlreadyExists(err) { - exists := &apps.ControllerRevision{} - key := types.NamespacedName{ - Namespace: clone.Namespace, - Name: clone.Name, - } - err = r.Client.Get(context.TODO(), key, exists) - if err != nil { - return nil, err - } - if bytes.Equal(exists.Data.Raw, clone.Data.Raw) { - return exists, nil - } - *collisionCount++ - continue - } - return clone, err - } -} - -func (r *realControl) GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error) { - revision, err := r.getControllerRevision(sidecarSet, revisionInfo) - if err != nil || revision == nil { - return nil, err - } - clone := sidecarSet.DeepCopy() - cloneBytes, err := runtime.Encode(patchCodec, clone) - if err != nil { - klog.Errorf("Failed to encode sidecarSet(%v), error: %v", sidecarSet.Name, err) - return nil, err - } - patched, err := strategicpatch.StrategicMergePatch(cloneBytes, revision.Data.Raw, clone) - if err != nil { - klog.Errorf("Failed to merge sidecarSet(%v) and controllerRevision(%v): %v, error: %v", sidecarSet.Name, revision.Name, err) - return nil, err - } - // restore history from patch - restoredSidecarSet := &appsv1alpha1.SidecarSet{} - if err := json.Unmarshal(patched, restoredSidecarSet); err != nil { - return nil, err - } - // restore hash annotation and revision info - if err := restoreRevisionInfo(restoredSidecarSet, revision); err != nil { - return nil, err - } - return restoredSidecarSet, nil -} - -func (r *realControl) getControllerRevision(set *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*apps.ControllerRevision, error) { - if revisionInfo == nil { - return nil, nil - } - switch { - case revisionInfo.RevisionName != nil: - revision := &apps.ControllerRevision{} - revisionKey := types.NamespacedName{ - Namespace: webhookutil.GetNamespace(), - Name: *revisionInfo.RevisionName, - } - if err := r.Client.Get(context.TODO(), revisionKey, revision); err != nil { - klog.Errorf("Failed to get ControllerRevision %v for SidecarSet(%v), err: %v", *revisionInfo.RevisionName, set.Name, err) - return nil, err - } - return revision, nil - - case revisionInfo.CustomVersion != nil: - listOpts := []client.ListOption{ - client.InNamespace(webhookutil.GetNamespace()), - &client.ListOptions{LabelSelector: r.GetRevisionSelector(set)}, - client.MatchingLabels{appsv1alpha1.SidecarSetCustomVersionLabel: *revisionInfo.CustomVersion}, - } - revisionList := &apps.ControllerRevisionList{} - if err := r.Client.List(context.TODO(), revisionList, listOpts...); err != nil { - klog.Errorf("Failed to get ControllerRevision for SidecarSet(%v), custom version: %v, err: %v", set.Name, *revisionInfo.CustomVersion, err) - return nil, err - } - - var revisions []*apps.ControllerRevision - for i := range revisionList.Items { - revisions = append(revisions, &revisionList.Items[i]) - } - - if len(revisions) == 0 { - return nil, generateNotFoundError(set) - } - history.SortControllerRevisions(revisions) - return revisions[len(revisions)-1], nil - } - - klog.Error("Failed to get controllerRevision due to both empty RevisionName and CustomVersion") - return nil, nil -} - -func copySidecarSetSpecRevision(dst, src map[string]interface{}) { - // we will use patch instead of update operation to update pods in the future - // dst["$patch"] = "replace" - // only record these revisions - dst["volumes"] = src["volumes"] - dst["containers"] = src["containers"] - dst["initContainers"] = src["initContainers"] - dst["imagePullSecrets"] = src["imagePullSecrets"] - dst["patchPodMetadata"] = src["patchPodMetadata"] -} - -func restoreRevisionInfo(sidecarSet *appsv1alpha1.SidecarSet, revision *apps.ControllerRevision) error { - if sidecarSet.Annotations == nil { - sidecarSet.Annotations = map[string]string{} - } - if revision.Annotations[SidecarSetHashAnnotation] != "" { - sidecarSet.Annotations[SidecarSetHashAnnotation] = revision.Annotations[SidecarSetHashAnnotation] - } else { - hashCodeWithImage, err := SidecarSetHash(sidecarSet) - if err != nil { - return err - } - sidecarSet.Annotations[SidecarSetHashAnnotation] = hashCodeWithImage - } - if revision.Annotations[SidecarSetHashWithoutImageAnnotation] != "" { - sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = revision.Annotations[SidecarSetHashWithoutImageAnnotation] - } else { - hashCodeWithoutImage, err := SidecarSetHashWithoutImage(sidecarSet) - if err != nil { - return err - } - sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = hashCodeWithoutImage - } - sidecarSet.Status.LatestRevision = revision.Name - return nil -} - -func MockSidecarSetForRevision(set *appsv1alpha1.SidecarSet) metav1.Object { - return &metav1.ObjectMeta{ - UID: set.UID, - Name: set.Name, - Namespace: webhookutil.GetNamespace(), - } -} - -func generateNotFoundError(set *appsv1alpha1.SidecarSet) error { - return errors.NewNotFound(schema.GroupResource{ - Group: apps.GroupName, - Resource: "ControllerRevision", - }, set.Name) -} diff --git a/pkg/control/sidecarcontrol/util_test.go b/pkg/control/sidecarcontrol/util_test.go deleted file mode 100644 index 2e379f2298..0000000000 --- a/pkg/control/sidecarcontrol/util_test.go +++ /dev/null @@ -1,1061 +0,0 @@ -/* -Copyright 2020 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sidecarcontrol - -import ( - "context" - "encoding/json" - "reflect" - "testing" - - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/util" - "github.com/openkruise/kruise/pkg/util/configuration" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func init() { - sch = runtime.NewScheme() - _ = corev1.AddToScheme(sch) -} - -var ( - sch *runtime.Scheme - - // image.Name -> image.Id - ImageIds = map[string]string{ - "main:v1": "4120593193b4", - "cold-sidecar:v1": "docker-pullable://cold-sidecar@sha256:9ead06a1362e", - "cold-sidecar:v2": "docker-pullable://cold-sidecar@sha256:7223aa0f3a7a", - "hot-sidecar:v1": "docker-pullable://hot-sidecar@sha256:86618128c92e", - "hot-sidecar:v2": "docker-pullable://hot-sidecar@sha256:74abd85af1e9", - "hotupgrade:empty": "docker-pullable://hotupgrade@sha256:0e9daf5c02e7", - } - - podDemo = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{}, - Name: "test-pod-1", - Namespace: "default", - Labels: map[string]string{"app": "nginx"}, - ResourceVersion: "495711227", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "main", - Image: "main:v1", - }, - { - Name: "cold-sidecar", - Image: "cold-sidecar:v1", - }, - { - Name: "hot-sidecar-1", - Image: "hot-sidecar:v1", - }, - { - Name: "hot-sidecar-2", - Image: "hotupgrade:empty", - }, - }, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{ - { - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }, - }, - ContainerStatuses: []corev1.ContainerStatus{ - { - Name: "main", - Image: "main:v1", - ImageID: ImageIds["main:v1"], - Ready: true, - }, - { - Name: "cold-sidecar", - Image: "cold-sidecar:v1", - ImageID: ImageIds["cold-sidecar:v1"], - Ready: true, - }, - { - Name: "hot-sidecar-1", - Image: "hot-sidecar:v1", - ImageID: ImageIds["hot-sidecar:v1"], - Ready: true, - }, - { - Name: "hot-sidecar-2", - Image: "hotupgrade:empty", - ImageID: ImageIds["hotupgrade:empty"], - Ready: true, - }, - }, - }, - } - - sidecarSetDemo = &appsv1alpha1.SidecarSet{ - ObjectMeta: metav1.ObjectMeta{ - Generation: 123, - Annotations: map[string]string{ - SidecarSetHashAnnotation: "bbb", - SidecarSetHashWithoutImageAnnotation: "without-image-aaa", - }, - Name: "test-sidecarset", - Labels: map[string]string{ - "app": "sidecar", - }, - }, - Spec: appsv1alpha1.SidecarSetSpec{ - Containers: []appsv1alpha1.SidecarContainer{ - { - Container: corev1.Container{ - Name: "cold-sidecar", - Image: "cold-image:v1", - }, - UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ - UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade, - }, - }, - { - Container: corev1.Container{ - Name: "hot-sidecar", - Image: "hot-image:v1", - }, - UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ - UpgradeType: appsv1alpha1.SidecarContainerHotUpgrade, - HotUpgradeEmptyImage: "hotupgrade:empty", - }, - }, - }, - }, - } -) - -func TestIsSidecarContainerUpdateCompleted(t *testing.T) { - cases := []struct { - name string - getPod func() *corev1.Pod - upgradeSidecars func() (sets.String, sets.String) - expectedCompleted bool - }{ - { - name: "only inject sidecar, not upgrade", - getPod: func() *corev1.Pod { - return podDemo.DeepCopy() - }, - upgradeSidecars: func() (sets.String, sets.String) { - return sets.NewString(sidecarSetDemo.Name), sets.NewString("cold-sidecar", "hot-sidecar-1", "hot-sidecar-2") - }, - expectedCompleted: true, - }, - { - name: "upgrade cold sidecar, upgrade not completed", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - control := New(sidecarSetDemo.DeepCopy()) - pod.Spec.Containers[1].Image = "cold-sidecar:v2" - UpdatePodSidecarSetHash(pod, control.GetSidecarset()) - control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod) - return pod - }, - upgradeSidecars: func() (sets.String, sets.String) { - return sets.NewString(sidecarSetDemo.Name), sets.NewString("cold-sidecar", "hot-sidecar-1", "hot-sidecar-2") - }, - expectedCompleted: false, - }, - { - name: "upgrade cold sidecar, upgrade completed", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - control := New(sidecarSetDemo.DeepCopy()) - pod.Spec.Containers[1].Image = "cold-sidecar:v2" - UpdatePodSidecarSetHash(pod, control.GetSidecarset()) - control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod) - pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"] - return pod - }, - upgradeSidecars: func() (sets.String, sets.String) { - return sets.NewString(sidecarSetDemo.Name), sets.NewString("cold-sidecar", "hot-sidecar-1", "hot-sidecar-2") - }, - expectedCompleted: true, - }, - { - name: "upgrade hot sidecar, upgrade hot-sidecar-2 not completed", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - control := New(sidecarSetDemo.DeepCopy()) - // upgrade cold sidecar completed - pod.Spec.Containers[1].Image = "cold-sidecar:v2" - UpdatePodSidecarSetHash(pod, control.GetSidecarset()) - control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod) - pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"] - // start upgrading hot sidecar - pod.Spec.Containers[3].Image = "hot-sidecar:v2" - control.UpdatePodAnnotationsInUpgrade([]string{"hot-sidecar-2"}, pod) - return pod - }, - upgradeSidecars: func() (sets.String, sets.String) { - return sets.NewString(sidecarSetDemo.Name), sets.NewString("cold-sidecar", "hot-sidecar-1", "hot-sidecar-2") - }, - expectedCompleted: false, - }, - { - name: "upgrade hot sidecar, upgrade hot-sidecar-1 not completed", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - control := New(sidecarSetDemo.DeepCopy()) - // upgrade cold sidecar completed - pod.Spec.Containers[1].Image = "cold-sidecar:v2" - UpdatePodSidecarSetHash(pod, control.GetSidecarset()) - control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod) - pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"] - // start upgrading hot sidecar - pod.Spec.Containers[3].Image = "hot-sidecar:v2" - control.UpdatePodAnnotationsInUpgrade([]string{"hot-sidecar-2"}, pod) - pod.Status.ContainerStatuses[3].ImageID = ImageIds["hot-sidecar:v2"] - pod.Spec.Containers[2].Image = "hotupgrade:empty" - control.UpdatePodAnnotationsInUpgrade([]string{"hot-sidecar-1"}, pod) - return pod - }, - upgradeSidecars: func() (sets.String, sets.String) { - return sets.NewString(sidecarSetDemo.Name), sets.NewString("cold-sidecar", "hot-sidecar-1", "hot-sidecar-2") - }, - expectedCompleted: false, - }, - { - name: "upgrade hot sidecar, upgrade hot-sidecar completed", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - control := New(sidecarSetDemo.DeepCopy()) - // upgrade cold sidecar completed - pod.Spec.Containers[1].Image = "cold-sidecar:v2" - UpdatePodSidecarSetHash(pod, control.GetSidecarset()) - control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod) - pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"] - // start upgrading hot sidecar - pod.Spec.Containers[3].Image = "hot-sidecar:v2" - control.UpdatePodAnnotationsInUpgrade([]string{"hot-sidecar-2"}, pod) - pod.Status.ContainerStatuses[3].ImageID = ImageIds["hot-sidecar:v2"] - pod.Spec.Containers[2].Image = "hotupgrade:empty" - control.UpdatePodAnnotationsInUpgrade([]string{"hot-sidecar-1"}, pod) - pod.Status.ContainerStatuses[2].ImageID = ImageIds["hotupgrade:empty"] - return pod - }, - upgradeSidecars: func() (sets.String, sets.String) { - return sets.NewString(sidecarSetDemo.Name), sets.NewString("cold-sidecar", "hot-sidecar-1", "hot-sidecar-2") - }, - expectedCompleted: true, - }, - } - - for _, cs := range cases { - t.Run(cs.name, func(t *testing.T) { - pod := cs.getPod() - sidecarSets, containers := cs.upgradeSidecars() - if IsSidecarContainerUpdateCompleted(pod, sidecarSets, containers) != cs.expectedCompleted { - t.Fatalf("IsSidecarContainerUpdateCompleted failed: %s", cs.name) - } - }) - } -} - -func TestGetPodSidecarSetRevision(t *testing.T) { - cases := []struct { - name string - getPod func() *corev1.Pod - //sidecarContainer -> sidecarSet.Revision - exceptRevision string - exceptWithoutImageRevision string - }{ - { - name: "normal sidecarSet revision", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}` - pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}` - return pod - }, - exceptRevision: "aaa", - exceptWithoutImageRevision: "without-image-aaa", - }, - { - name: "older sidecarSet revision", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset": "aaa"}` - pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset": "without-image-aaa"}` - return pod - }, - exceptRevision: "aaa", - exceptWithoutImageRevision: "without-image-aaa", - }, - { - name: "failed sidecarSet revision", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - pod.Annotations[SidecarSetHashAnnotation] = "failed-sidecarset-hash" - pod.Annotations[SidecarSetHashWithoutImageAnnotation] = "failed-sidecarset-hash" - return pod - }, - exceptRevision: "", - exceptWithoutImageRevision: "", - }, - } - - for _, cs := range cases { - t.Run(cs.name, func(t *testing.T) { - revison := GetPodSidecarSetRevision("test-sidecarset", cs.getPod()) - if cs.exceptRevision != revison { - t.Fatalf("except sidecar container test-sidecarset revison %s, but get %s", cs.exceptRevision, revison) - } - withoutRevison := GetPodSidecarSetWithoutImageRevision("test-sidecarset", cs.getPod()) - if cs.exceptWithoutImageRevision != withoutRevison { - t.Fatalf("except sidecar container test-sidecarset WithoutImageRevision %s, but get %s", cs.exceptWithoutImageRevision, withoutRevison) - } - }) - } -} - -func TestUpdatePodSidecarSetHash(t *testing.T) { - cases := []struct { - name string - getPod func() *corev1.Pod - getSidecarSet func() *appsv1alpha1.SidecarSet - exceptRevision map[string]SidecarSetUpgradeSpec - exceptWithoutImageRevision map[string]SidecarSetUpgradeSpec - }{ - { - name: "normal sidecarSet revision", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}` - pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}` - return pod - }, - getSidecarSet: func() *appsv1alpha1.SidecarSet { - return sidecarSetDemo.DeepCopy() - }, - exceptRevision: map[string]SidecarSetUpgradeSpec{ - "test-sidecarset": { - SidecarSetHash: "bbb", - }, - }, - exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{ - "test-sidecarset": { - SidecarSetHash: "without-image-aaa", - }, - }, - }, - { - name: "older sidecarSet revision", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset": "aaa"}` - pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset": "without-image-aaa"}` - return pod - }, - getSidecarSet: func() *appsv1alpha1.SidecarSet { - return sidecarSetDemo.DeepCopy() - }, - exceptRevision: map[string]SidecarSetUpgradeSpec{ - "test-sidecarset": { - SidecarSetHash: "bbb", - }, - }, - exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{ - "test-sidecarset": { - SidecarSetHash: "without-image-aaa", - }, - }, - }, - { - name: "failed sidecarSet revision", - getPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - pod.Annotations[SidecarSetHashAnnotation] = "failed-sidecarset-hash" - pod.Annotations[SidecarSetHashWithoutImageAnnotation] = "failed-sidecarset-hash" - return pod - }, - getSidecarSet: func() *appsv1alpha1.SidecarSet { - return sidecarSetDemo.DeepCopy() - }, - exceptRevision: map[string]SidecarSetUpgradeSpec{ - "test-sidecarset": { - SidecarSetHash: "bbb", - }, - }, - exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{}, - }, - } - - for _, cs := range cases { - t.Run(cs.name, func(t *testing.T) { - podInput := cs.getPod() - sidecarSetInput := cs.getSidecarSet() - UpdatePodSidecarSetHash(podInput, sidecarSetInput) - // sidecarSet hash - sidecarSetHash := make(map[string]SidecarSetUpgradeSpec) - err := json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashAnnotation]), &sidecarSetHash) - if err != nil { - t.Fatalf("parse pod sidecarSet hash failed: %s", err.Error()) - } - for k, o := range sidecarSetHash { - eo := cs.exceptRevision[k] - if o.SidecarSetHash != eo.SidecarSetHash { - t.Fatalf("except sidecar container %s revision %s, but get revision %s", k, eo.SidecarSetHash, o.SidecarSetHash) - } - } - if len(cs.exceptWithoutImageRevision) == 0 { - return - } - // without image sidecarSet hash - sidecarSetHash = make(map[string]SidecarSetUpgradeSpec) - err = json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashWithoutImageAnnotation]), &sidecarSetHash) - if err != nil { - t.Fatalf("parse pod sidecarSet hash failed: %s", err.Error()) - } - for k, o := range sidecarSetHash { - eo := cs.exceptWithoutImageRevision[k] - if o.SidecarSetHash != eo.SidecarSetHash { - t.Fatalf("except sidecar container %s revision %s, but get revision %s", k, eo.SidecarSetHash, o.SidecarSetHash) - } - } - }) - } -} - -func TestConvertDownwardAPIFieldLabel(t *testing.T) { - testCases := []struct { - version string - label string - value string - expectedErr bool - expectedLabel string - expectedValue string - }{ - { - version: "v2", - label: "metadata.name", - value: "test-pod", - expectedErr: true, - }, - { - version: "v1", - label: "invalid-label", - value: "value", - expectedErr: true, - }, - { - version: "v1", - label: "metadata.name", - value: "test-pod", - expectedErr: true, - }, - { - version: "v1", - label: "metadata.annotations", - value: "myAnnoValue", - expectedErr: true, - }, - { - version: "v1", - label: "metadata.labels", - value: "myLabelValue", - expectedErr: true, - }, - { - version: "v1", - label: "metadata.annotations['myAnnoKey']", - value: "myAnnoValue", - expectedLabel: "metadata.annotations['myAnnoKey']", - expectedValue: "myAnnoValue", - }, - { - version: "v1", - label: "metadata.labels['myLabelKey']", - value: "myLabelValue", - expectedLabel: "metadata.labels['myLabelKey']", - expectedValue: "myLabelValue", - }, - } - for _, tc := range testCases { - label, value, err := ConvertDownwardAPIFieldLabel(tc.version, tc.label, tc.value) - if err != nil { - if tc.expectedErr { - continue - } - t.Errorf("ConvertDownwardAPIFieldLabel(%s, %s, %s) failed: %s", - tc.version, tc.label, tc.value, err) - } - if tc.expectedLabel != label || tc.expectedValue != value { - t.Errorf("ConvertDownwardAPIFieldLabel(%s, %s, %s) = (%s, %s, nil), expected (%s, %s, nil)", - tc.version, tc.label, tc.value, label, value, tc.expectedLabel, tc.expectedValue) - } - } -} - -func TestExtractContainerNameFromFieldPath(t *testing.T) { - testCases := []struct { - fieldSelector *corev1.ObjectFieldSelector - pod *corev1.Pod - expectedErr bool - expectedName string - }{ - { - fieldSelector: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.labels['test-label']", - }, - pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-label", - Labels: map[string]string{ - "test-label": "test-pod-label", - }, - }}, - expectedName: "test-pod-label", - }, - { - fieldSelector: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.labels['test-label']", - }, - pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-label", - }}, - expectedName: "", - }, - { - fieldSelector: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.annotations['test-anno']", - }, - pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-anno", - Annotations: map[string]string{ - "test-anno": "test-pod-anno", - }, - }}, - expectedName: "test-pod-anno", - }, - } - for _, tc := range testCases { - containerName, err := ExtractContainerNameFromFieldPath(tc.fieldSelector, tc.pod) - if err != nil { - if tc.expectedErr { - continue - } - t.Errorf("ExtractContainerNameFromFieldPath(%s, %s) failed: %s", - tc.fieldSelector.FieldPath, tc.expectedName, err) - } - if tc.expectedName != containerName { - t.Errorf("ExtractContainerNameFromFieldPath (%s, %s), expected (%s)", - tc.fieldSelector.FieldPath, containerName, tc.expectedName) - } - } -} - -func TestGetSidecarTransferEnvs(t *testing.T) { - testCases := []struct { - sidecarContainer *appsv1alpha1.SidecarContainer - pod *corev1.Pod - expectedEnvs []corev1.EnvVar - }{ - { - sidecarContainer: &appsv1alpha1.SidecarContainer{ - Container: corev1.Container{ - Name: "cold-sidecar", - Image: "cold-image:v1", - }, - UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ - UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade, - }, - TransferEnv: []appsv1alpha1.TransferEnvVar{ - { - EnvName: "test-env", - SourceContainerName: "main", - }, - }, - }, - pod: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{}, - Name: "test-pod-1", - Namespace: "default", - Labels: map[string]string{"app": "nginx"}, - ResourceVersion: "495711227", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "main", - Image: "main:v1", - Env: []corev1.EnvVar{ - { - Name: "test-env", - Value: "test-value", - }, - }, - }, - }, - }, - }, - expectedEnvs: []corev1.EnvVar{ - { - Name: "test-env", - Value: "test-value", - }, - }, - }, - { - sidecarContainer: &appsv1alpha1.SidecarContainer{ - Container: corev1.Container{ - Name: "cold-sidecar", - Image: "cold-image:v1", - }, - UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ - UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade, - }, - TransferEnv: []appsv1alpha1.TransferEnvVar{ - { - EnvName: "test-env", - SourceContainerNameFrom: &appsv1alpha1.SourceContainerNameSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.labels['app']", - }, - }, - }, - }, - }, - pod: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{}, - Name: "test-pod-1", - Namespace: "default", - Labels: map[string]string{"app": "main"}, - ResourceVersion: "495711227", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "main", - Image: "main:v1", - Env: []corev1.EnvVar{ - { - Name: "test-env", - Value: "test-value", - }, - }, - }, - }, - }, - }, - expectedEnvs: []corev1.EnvVar{ - { - Name: "test-env", - Value: "test-value", - }, - }, - }, - { - sidecarContainer: &appsv1alpha1.SidecarContainer{ - Container: corev1.Container{ - Name: "cold-sidecar", - Image: "cold-image:v1", - }, - UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ - UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade, - }, - TransferEnv: []appsv1alpha1.TransferEnvVar{ - { - EnvName: "test-env", - SourceContainerNameFrom: &appsv1alpha1.SourceContainerNameSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.annotations['app']", - }, - }, - }, - }, - }, - pod: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{"app": "main"}, - Name: "test-pod-1", - Namespace: "default", - Labels: map[string]string{"app": "main"}, - ResourceVersion: "495711227", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "main", - Image: "main:v1", - Env: []corev1.EnvVar{ - { - Name: "test-env", - Value: "test-value", - }, - }, - }, - }, - }, - }, - expectedEnvs: []corev1.EnvVar{ - { - Name: "test-env", - Value: "test-value", - }, - }, - }, - } - for _, tc := range testCases { - injectedEnvs := GetSidecarTransferEnvs(tc.sidecarContainer, tc.pod) - if len(injectedEnvs) != len(tc.expectedEnvs) { - t.Errorf("GetSidecarTransferEnv failed, expected envs %s, got %s", - tc.expectedEnvs, injectedEnvs) - } - } -} - -func TestPatchPodMetadata(t *testing.T) { - cases := []struct { - name string - getPod func() *corev1.Pod - patches func() []appsv1alpha1.SidecarSetPatchPodMetadata - expectAnnotations map[string]string - expectErr bool - skip bool - }{ - { - name: "add pod annotation", - getPod: func() *corev1.Pod { - demo := &corev1.Pod{} - return demo - }, - patches: func() []appsv1alpha1.SidecarSetPatchPodMetadata { - patch := []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - PatchPolicy: appsv1alpha1.SidecarSetRetainPatchPolicy, - Annotations: map[string]string{ - "key1": "value1", - }, - }, - { - PatchPolicy: appsv1alpha1.SidecarSetOverwritePatchPolicy, - Annotations: map[string]string{ - "key2": "value2", - }, - }, - } - return patch - }, - expectAnnotations: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - skip: false, - expectErr: false, - }, - { - name: "add pod annotation, exist", - getPod: func() *corev1.Pod { - demo := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "key1": "old", - "key2": "old", - }, - }, - } - return demo - }, - patches: func() []appsv1alpha1.SidecarSetPatchPodMetadata { - patch := []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - PatchPolicy: appsv1alpha1.SidecarSetRetainPatchPolicy, - Annotations: map[string]string{ - "key1": "value1", - }, - }, - { - PatchPolicy: appsv1alpha1.SidecarSetOverwritePatchPolicy, - Annotations: map[string]string{ - "key2": "value2", - }, - }, - } - return patch - }, - expectAnnotations: map[string]string{ - "key1": "old", - "key2": "value2", - }, - skip: false, - expectErr: false, - }, - { - name: "json merge pod annotation", - getPod: func() *corev1.Pod { - demo := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "key1": `{"log-agent":1}`, - "key2": `{"log-agent":1}`, - }, - }, - } - return demo - }, - patches: func() []appsv1alpha1.SidecarSetPatchPodMetadata { - patch := []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - PatchPolicy: appsv1alpha1.SidecarSetMergePatchJsonPatchPolicy, - Annotations: map[string]string{ - "key1": `{"log-agent":1}`, - "key2": `{"envoy":2}`, - "key3": `{"probe":5}`, - }, - }, - } - return patch - }, - expectAnnotations: map[string]string{ - "key1": `{"log-agent":1}`, - "key2": `{"envoy":2,"log-agent":1}`, - "key3": `{"probe":5}`, - }, - skip: false, - expectErr: false, - }, - { - name: "json merge pod annotation, skip", - getPod: func() *corev1.Pod { - demo := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "key1": `{"log-agent":1}`, - }, - }, - } - return demo - }, - patches: func() []appsv1alpha1.SidecarSetPatchPodMetadata { - patch := []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - PatchPolicy: appsv1alpha1.SidecarSetMergePatchJsonPatchPolicy, - Annotations: map[string]string{ - "key1": `{"log-agent":1}`, - }, - }, - } - return patch - }, - expectAnnotations: map[string]string{ - "key1": `{"log-agent":1}`, - }, - skip: true, - expectErr: false, - }, - } - - for _, cs := range cases { - t.Run(cs.name, func(t *testing.T) { - pod := cs.getPod() - skip, err := PatchPodMetadata(&pod.ObjectMeta, cs.patches()) - if cs.expectErr && err == nil { - t.Fatalf("PatchPodMetadata failed") - } else if !cs.expectErr && err != nil { - t.Fatalf("PatchPodMetadata failed: %v", err) - } else if skip != cs.skip { - t.Fatalf("expect %v, but get %v", cs.skip, skip) - } else if !reflect.DeepEqual(cs.expectAnnotations, pod.Annotations) { - t.Fatalf("expect %v, but get %v", cs.expectAnnotations, pod.Annotations) - } - }) - } -} - -func TestValidateSidecarSetPatchMetadataWhitelist(t *testing.T) { - cases := []struct { - name string - getSidecarSet func() *appsv1alpha1.SidecarSet - getKruiseCM func() *corev1.ConfigMap - expectErr bool - }{ - { - name: "validate sidecarSet no patch Metadata", - getSidecarSet: func() *appsv1alpha1.SidecarSet { - demo := sidecarSetDemo.DeepCopy() - return demo - }, - getKruiseCM: func() *corev1.ConfigMap { - return nil - }, - expectErr: false, - }, - { - name: "validate sidecarSet whitelist failed-1", - getSidecarSet: func() *appsv1alpha1.SidecarSet { - demo := sidecarSetDemo.DeepCopy() - demo.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - Annotations: map[string]string{ - "key1": "value1", - }, - }, - } - return demo - }, - getKruiseCM: func() *corev1.ConfigMap { - return nil - }, - expectErr: true, - }, - { - name: "validate sidecarSet whitelist success-1", - getSidecarSet: func() *appsv1alpha1.SidecarSet { - demo := sidecarSetDemo.DeepCopy() - demo.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - Annotations: map[string]string{ - "key1": "value1", - }, - }, - } - return demo - }, - getKruiseCM: func() *corev1.ConfigMap { - demo := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configuration.KruiseConfigurationName, - Namespace: util.GetKruiseNamespace(), - }, - Data: map[string]string{ - configuration.SidecarSetPatchPodMetadataWhiteListKey: `{"rules":[{"allowedAnnotationKeyExprs":["key.*"]}]}`, - }, - } - return demo - }, - expectErr: false, - }, - { - name: "validate sidecarSet whitelist failed-2", - getSidecarSet: func() *appsv1alpha1.SidecarSet { - demo := sidecarSetDemo.DeepCopy() - demo.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - Annotations: map[string]string{ - "key1": "value1", - }, - }, - } - return demo - }, - getKruiseCM: func() *corev1.ConfigMap { - demo := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configuration.KruiseConfigurationName, - Namespace: util.GetKruiseNamespace(), - }, - Data: map[string]string{ - configuration.SidecarSetPatchPodMetadataWhiteListKey: `{"rules":[{"allowedAnnotationKeyExprs":["key2"]}]}`, - }, - } - return demo - }, - expectErr: true, - }, - { - name: "validate sidecarSet whitelist failed-3", - getSidecarSet: func() *appsv1alpha1.SidecarSet { - demo := sidecarSetDemo.DeepCopy() - demo.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - Annotations: map[string]string{ - "key1": "value1", - }, - }, - } - return demo - }, - getKruiseCM: func() *corev1.ConfigMap { - demo := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configuration.KruiseConfigurationName, - Namespace: util.GetKruiseNamespace(), - }, - Data: map[string]string{ - configuration.SidecarSetPatchPodMetadataWhiteListKey: `{"rules":[{"allowedAnnotationKeyExprs":["key.*"],"selector":{"matchLabels":{"app":"other"}}}]}`, - }, - } - return demo - }, - expectErr: true, - }, - { - name: "validate sidecarSet whitelist success-2", - getSidecarSet: func() *appsv1alpha1.SidecarSet { - demo := sidecarSetDemo.DeepCopy() - demo.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - Annotations: map[string]string{ - "key1": "value1", - }, - }, - } - return demo - }, - getKruiseCM: func() *corev1.ConfigMap { - demo := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configuration.KruiseConfigurationName, - Namespace: util.GetKruiseNamespace(), - }, - Data: map[string]string{ - configuration.SidecarSetPatchPodMetadataWhiteListKey: `{"rules":[{"allowedAnnotationKeyExprs":["key.*"],"selector":{"matchLabels":{"app":"sidecar"}}}]}`, - }, - } - return demo - }, - expectErr: false, - }, - } - - for _, cs := range cases { - t.Run(cs.name, func(t *testing.T) { - fakeClient := fake.NewClientBuilder().WithScheme(sch).Build() - if cs.getKruiseCM() != nil { - fakeClient.Create(context.TODO(), cs.getKruiseCM()) - } - err := ValidateSidecarSetPatchMetadataWhitelist(fakeClient, cs.getSidecarSet()) - if cs.expectErr && err == nil { - t.Fatalf("ValidateSidecarSetPatchMetadataWhitelist failed") - } else if !cs.expectErr && err != nil { - t.Fatalf("ValidateSidecarSetPatchMetadataWhitelist failed: %s", err.Error()) - } - }) - } -} diff --git a/pkg/controller/sidecarset/sidecarset_controller.go b/pkg/controller/sidecarset/sidecarset_controller.go index 8c662541a1..c0864056c2 100644 --- a/pkg/controller/sidecarset/sidecarset_controller.go +++ b/pkg/controller/sidecarset/sidecarset_controller.go @@ -20,6 +20,16 @@ import ( "context" "flag" + kruiseclient "github.com/openkruise/kruise/pkg/client" + + k8scache "k8s.io/client-go/tools/cache" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + utildiscovery "github.com/openkruise/kruise/pkg/util/discovery" + "github.com/openkruise/kruise/pkg/util/expectations" + "github.com/openkruise/kruise/pkg/util/ratelimiter" + "github.com/openkruise/utils/sidecarcontrol" + apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -32,13 +42,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" - utilclient "github.com/openkruise/kruise/pkg/util/client" - utildiscovery "github.com/openkruise/kruise/pkg/util/discovery" - "github.com/openkruise/kruise/pkg/util/expectations" - "github.com/openkruise/kruise/pkg/util/ratelimiter" ) func init() { @@ -68,11 +71,12 @@ func Add(mgr manager.Manager) error { func newReconciler(mgr manager.Manager) reconcile.Reconciler { expectations := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl) recorder := mgr.GetEventRecorderFor("sidecarset-controller") - cli := utilclient.NewClientFromManager(mgr, "sidecarset-controller") + revInformer, _ := mgr.GetCache().GetInformerForKind(context.TODO(), apps.SchemeGroupVersion.WithKind("ControllerRevision")) + genericClient := kruiseclient.GetGenericClientWithName("sidecarset-controller") return &ReconcileSidecarSet{ - Client: cli, + Client: mgr.GetClient(), scheme: mgr.GetScheme(), - processor: NewSidecarSetProcessor(cli, expectations, recorder), + processor: NewSidecarSetProcessor(mgr.GetClient(), genericClient.KubeClient, revInformer.(k8scache.SharedIndexInformer).GetIndexer(), expectations, recorder), } } @@ -103,7 +107,8 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { } // Watch for changes to Pod - if err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &enqueueRequestForPod{reader: mgr.GetCache()}); err != nil { + if err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &enqueueRequestForPod{reader: mgr.GetCache(), + sidecarSetControl: sidecarcontrol.NewCommonControl(nil, "")}); err != nil { return err } diff --git a/pkg/controller/sidecarset/sidecarset_controller_test.go b/pkg/controller/sidecarset/sidecarset_controller_test.go index 7a3c2448f2..2d660df123 100644 --- a/pkg/controller/sidecarset/sidecarset_controller_test.go +++ b/pkg/controller/sidecarset/sidecarset_controller_test.go @@ -4,9 +4,13 @@ import ( "context" "testing" + k8sfake "k8s.io/client-go/kubernetes/fake" + + "k8s.io/client-go/tools/cache" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util/expectations" + "github.com/openkruise/utils/sidecarcontrol" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -183,10 +187,12 @@ func testUpdateWhenUseNotUpdateStrategy(t *testing.T, sidecarSetInput *appsv1alp fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sidecarSetInput, podInput).Build() exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} reconciler := ReconcileSidecarSet{ Client: fakeClient, updateExpectations: expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl), - processor: NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)), + processor: NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)), } if _, err := reconciler.Reconcile(context.TODO(), request); err != nil { t.Errorf("reconcile failed, err: %v", err) @@ -218,10 +224,12 @@ func testUpdateWhenSidecarSetPaused(t *testing.T, sidecarSetInput *appsv1alpha1. fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sidecarSetInput, podInput).Build() exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} reconciler := ReconcileSidecarSet{ Client: fakeClient, updateExpectations: exps, - processor: NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)), + processor: NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)), } if _, err := reconciler.Reconcile(context.TODO(), request); err != nil { t.Errorf("reconcile failed, err: %v", err) @@ -253,10 +261,12 @@ func testUpdateWhenMaxUnavailableNotZero(t *testing.T, sidecarSetInput *appsv1al fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sidecarSetInput, podInput).Build() exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} reconciler := ReconcileSidecarSet{ Client: fakeClient, updateExpectations: exps, - processor: NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)), + processor: NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)), } if _, err := reconciler.Reconcile(context.TODO(), request); err != nil { t.Errorf("reconcile failed, err: %v", err) @@ -289,10 +299,12 @@ func testUpdateWhenPartitionFinished(t *testing.T, sidecarSetInput *appsv1alpha1 fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sidecarSetInput, podInput).Build() exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} reconciler := ReconcileSidecarSet{ Client: fakeClient, updateExpectations: exps, - processor: NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)), + processor: NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)), } if _, err := reconciler.Reconcile(context.TODO(), request); err != nil { t.Errorf("reconcile failed, err: %v", err) @@ -325,10 +337,12 @@ func testRemoveSidecarSet(t *testing.T, sidecarSetInput *appsv1alpha1.SidecarSet fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sidecarSetInput, podInput).Build() exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} reconciler := ReconcileSidecarSet{ Client: fakeClient, updateExpectations: exps, - processor: NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)), + processor: NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)), } if _, err := reconciler.Reconcile(context.TODO(), request); err != nil { t.Errorf("reconcile failed, err: %v", err) diff --git a/pkg/controller/sidecarset/sidecarset_hotupgrade.go b/pkg/controller/sidecarset/sidecarset_hotupgrade.go index 56f10778ca..aa00ecf3cc 100644 --- a/pkg/controller/sidecarset/sidecarset_hotupgrade.go +++ b/pkg/controller/sidecarset/sidecarset_hotupgrade.go @@ -21,8 +21,8 @@ import ( "fmt" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" + "github.com/openkruise/utils/sidecarcontrol" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -31,9 +31,9 @@ import ( "k8s.io/klog/v2" ) -func (p *Processor) flipHotUpgradingContainers(control sidecarcontrol.SidecarControl, pods []*corev1.Pod) error { +func (p *Processor) flipHotUpgradingContainers(sidecarSet *appsv1alpha1.SidecarSet, pods []*corev1.Pod) error { for _, pod := range pods { - if err := p.flipPodSidecarContainer(control, pod); err != nil { + if err := p.flipPodSidecarContainer(sidecarSet, pod); err != nil { p.recorder.Eventf(pod, corev1.EventTypeWarning, "ResetContainerFailed", fmt.Sprintf("reset sidecar container image empty failed: %s", err.Error())) return err } @@ -42,11 +42,11 @@ func (p *Processor) flipHotUpgradingContainers(control sidecarcontrol.SidecarCon return nil } -func (p *Processor) flipPodSidecarContainer(control sidecarcontrol.SidecarControl, pod *corev1.Pod) error { +func (p *Processor) flipPodSidecarContainer(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) error { podClone := pod.DeepCopy() err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { // sidecar container hot upgrade already complete, and flip container - flipPodSidecarContainerDo(control, podClone) + p.flipPodSidecarContainerDo(sidecarSet, podClone) // update pod in store updateErr := p.Client.Update(context.TODO(), podClone) if updateErr == nil { @@ -66,8 +66,7 @@ func (p *Processor) flipPodSidecarContainer(control sidecarcontrol.SidecarContro return err } -func flipPodSidecarContainerDo(control sidecarcontrol.SidecarControl, pod *corev1.Pod) { - sidecarSet := control.GetSidecarset() +func (p *Processor) flipPodSidecarContainerDo(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) { containersInPod := make(map[string]*corev1.Container) for i := range pod.Spec.Containers { container := &pod.Spec.Containers[i] @@ -93,7 +92,7 @@ func flipPodSidecarContainerDo(control sidecarcontrol.SidecarControl, pod *corev } } // record the updated container status, to determine if the update is complete - control.UpdatePodAnnotationsInUpgrade(changedContainer, pod) + p.sidecarSetControl.UpdatePodAnnotationsInUpgrade(changedContainer, pod, sidecarSet) } func isSidecarSetHasHotUpgradeContainer(sidecarSet *appsv1alpha1.SidecarSet) bool { diff --git a/pkg/controller/sidecarset/sidecarset_hotupgrade_test.go b/pkg/controller/sidecarset/sidecarset_hotupgrade_test.go index eeb2e55cbe..188e664d62 100644 --- a/pkg/controller/sidecarset/sidecarset_hotupgrade_test.go +++ b/pkg/controller/sidecarset/sidecarset_hotupgrade_test.go @@ -20,9 +20,13 @@ import ( "fmt" "testing" + k8sfake "k8s.io/client-go/kubernetes/fake" + + "k8s.io/client-go/tools/cache" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util/expectations" + "github.com/openkruise/utils/sidecarcontrol" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -249,7 +253,9 @@ func testUpdateHotUpgradeSidecar(t *testing.T, hotUpgradeEmptyImage string, side pod := cs.getPods()[0] sidecarset := cs.getSidecarset() fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sidecarset, pod).Build() - processor := NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} + processor := NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)) _, err := processor.UpdateSidecarSet(sidecarset) if err != nil { t.Errorf("processor update sidecarset failed: %s", err.Error()) diff --git a/pkg/controller/sidecarset/sidecarset_pod_event_handler.go b/pkg/controller/sidecarset/sidecarset_pod_event_handler.go index 5a9c36fea7..1cd4e43482 100644 --- a/pkg/controller/sidecarset/sidecarset_pod_event_handler.go +++ b/pkg/controller/sidecarset/sidecarset_pod_event_handler.go @@ -7,7 +7,7 @@ import ( "time" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" + "github.com/openkruise/utils/sidecarcontrol" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -25,7 +25,8 @@ import ( var _ handler.EventHandler = &enqueueRequestForPod{} type enqueueRequestForPod struct { - reader client.Reader + reader client.Reader + sidecarSetControl sidecarcontrol.SidecarControl } func (p *enqueueRequestForPod) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { @@ -87,7 +88,7 @@ func (p *enqueueRequestForPod) updatePod(q workqueue.RateLimitingInterface, old, //check whether pod status is changed if isChanged = isPodStatusChanged(oldPod, newPod); !isChanged { //check whether pod consistent is changed - isChanged, enqueueDelayTime = isPodConsistentChanged(oldPod, newPod, sidecarSet) + isChanged, enqueueDelayTime = p.isPodConsistentChanged(oldPod, newPod, sidecarSet) } if isChanged { q.AddAfter(reconcile.Request{ @@ -167,12 +168,11 @@ func isPodStatusChanged(oldPod, newPod *corev1.Pod) bool { return false } -func isPodConsistentChanged(oldPod, newPod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) (bool, time.Duration) { - control := sidecarcontrol.New(sidecarSet) +func (p *enqueueRequestForPod) isPodConsistentChanged(oldPod, newPod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) (bool, time.Duration) { var enqueueDelayTime time.Duration // contain sidecar empty container - oldConsistent := control.IsPodStateConsistent(oldPod, nil) - newConsistent := control.IsPodStateConsistent(newPod, nil) + oldConsistent := p.sidecarSetControl.IsPodStateConsistent(oldPod, sidecarSet, nil) + newConsistent := p.sidecarSetControl.IsPodStateConsistent(newPod, sidecarSet, nil) if oldConsistent != newConsistent { klog.V(3).Infof("pod(%s/%s) sidecar containers consistent changed(from %v to %v), and reconcile sidecarSet(%s)", newPod.Namespace, newPod.Name, oldConsistent, newConsistent, sidecarSet.Name) diff --git a/pkg/controller/sidecarset/sidecarset_pod_event_handler_test.go b/pkg/controller/sidecarset/sidecarset_pod_event_handler_test.go index ac056a2cb1..833d729de1 100644 --- a/pkg/controller/sidecarset/sidecarset_pod_event_handler_test.go +++ b/pkg/controller/sidecarset/sidecarset_pod_event_handler_test.go @@ -23,7 +23,7 @@ import ( "time" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" + "github.com/openkruise/utils/sidecarcontrol" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/util/workqueue" @@ -34,7 +34,7 @@ import ( func TestPodEventHandler(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() - handler := enqueueRequestForPod{reader: fakeClient} + handler := enqueueRequestForPod{reader: fakeClient, sidecarSetControl: sidecarcontrol.NewCommonControl(nil, "")} err := fakeClient.Create(context.TODO(), sidecarSetDemo.DeepCopy()) if nil != err { @@ -160,9 +160,9 @@ func TestGetPodMatchedSidecarSets(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(pod).Build() sidecarSets := cs.getSidecarSets() for _, sidecarSet := range sidecarSets { - fakeClient.Create(context.TODO(), sidecarSet) + _ = fakeClient.Create(context.TODO(), sidecarSet) } - e := enqueueRequestForPod{fakeClient} + e := enqueueRequestForPod{fakeClient, sidecarcontrol.NewCommonControl(nil, "")} matched, err := e.getPodMatchedSidecarSets(pod) if err != nil { t.Fatalf("getPodMatchedSidecarSets failed: %s", err.Error()) diff --git a/pkg/controller/sidecarset/sidecarset_processor.go b/pkg/controller/sidecarset/sidecarset_processor.go index f4c10a1c8e..83d457c9db 100644 --- a/pkg/controller/sidecarset/sidecarset_processor.go +++ b/pkg/controller/sidecarset/sidecarset_processor.go @@ -24,19 +24,19 @@ import ( "time" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" utilclient "github.com/openkruise/kruise/pkg/util/client" "github.com/openkruise/kruise/pkg/util/expectations" - historyutil "github.com/openkruise/kruise/pkg/util/history" webhookutil "github.com/openkruise/kruise/pkg/webhook/util" - + "github.com/openkruise/utils/sidecarcontrol" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" @@ -50,25 +50,28 @@ import ( type Processor struct { Client client.Client recorder record.EventRecorder - historyController history.Interface updateExpectations expectations.UpdateExpectations + sidecarSetControl sidecarcontrol.SidecarControl + historyControl sidecarcontrol.HistoryControl + strategy Strategy + // controllerRevision namespace + namespace string } -func NewSidecarSetProcessor(cli client.Client, expectations expectations.UpdateExpectations, rec record.EventRecorder) *Processor { - return &Processor{ +func NewSidecarSetProcessor(cli client.Client, kubeClient clientset.Interface, indexer cache.Indexer, expectations expectations.UpdateExpectations, rec record.EventRecorder) *Processor { + p := &Processor{ Client: cli, updateExpectations: expectations, recorder: rec, - historyController: historyutil.NewHistory(cli), + sidecarSetControl: sidecarcontrol.NewCommonControl(indexer, webhookutil.GetNamespace()), + historyControl: sidecarcontrol.NewHistoryControl(kubeClient, indexer, webhookutil.GetNamespace()), + namespace: webhookutil.GetNamespace(), } + p.strategy = NewStrategy(p.sidecarSetControl) + return p } func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (reconcile.Result, error) { - control := sidecarcontrol.New(sidecarSet) - // check whether sidecarSet is active - if !control.IsActiveSidecarSet() { - return reconcile.Result{}, nil - } // 1. get matching pods with the sidecarSet pods, err := p.getMatchingPods(sidecarSet) if err != nil { @@ -85,7 +88,7 @@ func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (recon } // 2. calculate SidecarSet status based on pod and revision information - status := calculateStatus(control, pods, latestRevision, collisionCount) + status := p.calculateStatus(sidecarSet, pods, latestRevision, collisionCount) //update sidecarSet status in store if err := p.updateSidecarSetStatus(sidecarSet, status); err != nil { return reconcile.Result{}, err @@ -119,12 +122,13 @@ func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (recon sidecarContainers.Delete(emptyContainer) } } - if isPodSidecarInHotUpgrading(sidecarSet, pod) && control.IsPodStateConsistent(pod, sidecarContainers) && + if isPodSidecarInHotUpgrading(sidecarSet, pod) && + p.sidecarSetControl.IsPodStateConsistent(pod, sidecarSet, sidecarContainers) && isHotUpgradingReady(sidecarSet, pod) { podsInHotUpgrading = append(podsInHotUpgrading, pod) } } - if err := p.flipHotUpgradingContainers(control, podsInHotUpgrading); err != nil { + if err := p.flipHotUpgradingContainers(sidecarSet, podsInHotUpgrading); err != nil { return reconcile.Result{}, err } } @@ -147,18 +151,17 @@ func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (recon } // 7. upgrade pod sidecar - if err := p.updatePods(control, pods); err != nil { + if err := p.updatePods(sidecarSet, pods); err != nil { return reconcile.Result{}, err } return reconcile.Result{}, nil } -func (p *Processor) updatePods(control sidecarcontrol.SidecarControl, pods []*corev1.Pod) error { - sidecarset := control.GetSidecarset() +func (p *Processor) updatePods(sidecarSet *appsv1alpha1.SidecarSet, pods []*corev1.Pod) error { // compute next updated pods based on the sidecarset upgrade strategy - upgradePods := NewStrategy().GetNextUpgradePods(control, pods) + upgradePods := p.strategy.GetNextUpgradePods(sidecarSet, pods) if len(upgradePods) == 0 { - klog.V(3).Infof("sidecarSet next update is nil, skip this round, name: %s", sidecarset.Name) + klog.V(3).Infof("sidecarSet next update is nil, skip this round, name: %s", sidecarSet.Name) return nil } // mark upgrade pods list @@ -166,26 +169,25 @@ func (p *Processor) updatePods(control sidecarcontrol.SidecarControl, pods []*co // upgrade pod sidecar for _, pod := range upgradePods { podNames = append(podNames, pod.Name) - if err := p.updatePodSidecarAndHash(control, pod); err != nil { - klog.Errorf("updatePodSidecarAndHash error, s:%s, pod:%s, err:%v", sidecarset.Name, pod.Name, err) + if err := p.updatePodSidecarAndHash(sidecarSet, pod); err != nil { + klog.Errorf("updatePodSidecarAndHash error, s:%s, pod:%s, err:%v", sidecarSet.Name, pod.Name, err) return err } - p.updateExpectations.ExpectUpdated(sidecarset.Name, sidecarcontrol.GetSidecarSetRevision(sidecarset), pod) + p.updateExpectations.ExpectUpdated(sidecarSet.Name, sidecarcontrol.GetSidecarSetRevision(sidecarSet), pod) } - klog.V(3).Infof("sidecarSet(%s) updated pods(%s)", sidecarset.Name, strings.Join(podNames, ",")) + klog.V(3).Infof("sidecarSet(%s) updated pods(%s)", sidecarSet.Name, strings.Join(podNames, ",")) return nil } -func (p *Processor) updatePodSidecarAndHash(control sidecarcontrol.SidecarControl, pod *corev1.Pod) error { +func (p *Processor) updatePodSidecarAndHash(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) error { podClone := &corev1.Pod{} - sidecarSet := control.GetSidecarset() err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { if err := p.Client.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, podClone); err != nil { - klog.Errorf("error getting updated pod %s from client", control.GetSidecarset().Name) + klog.Errorf("error getting updated pod %s from client", sidecarSet.Name) } // update pod sidecar container - updatePodSidecarContainer(control, podClone) + p.updatePodSidecarContainer(sidecarSet, podClone) // older pod don't have SidecarSetListAnnotation // which is to improve the performance of the sidecarSet controller sidecarSetNames, ok := podClone.Annotations[sidecarcontrol.SidecarSetListAnnotation] @@ -214,7 +216,7 @@ func (p *Processor) listMatchedSidecarSets(pod *corev1.Pod) string { //matched SidecarSet.Name list sidecarSetNames := make([]string, 0) for _, sidecarSet := range sidecarSetList.Items { - if matched, _ := sidecarcontrol.PodMatchedSidecarSet(pod, sidecarSet); matched { + if matched, _ := sidecarcontrol.PodMatchedSidecarSet(pod, &sidecarSet); matched { sidecarSetNames = append(sidecarSetNames, sidecarSet.Name) } } @@ -303,15 +305,12 @@ func (p *Processor) registerLatestRevision(set *appsv1alpha1.SidecarSet, pods [] latestRevision *apps.ControllerRevision, collisionCount int32, err error, ) { sidecarSet := set.DeepCopy() - // get revision selector of this sidecarSet - hc := sidecarcontrol.NewHistoryControl(p.Client) // list all revisions - revisions, err := p.historyController.ListControllerRevisions(sidecarcontrol.MockSidecarSetForRevision(set), hc.GetRevisionSelector(sidecarSet)) + revisions, err := p.historyControl.ListSidecarSetControllerRevisions(sidecarSet) if err != nil { klog.Errorf("Failed to list history controllerRevisions, err %v, name %v", err, sidecarSet.Name) return nil, collisionCount, err } - // sort revisions by increasing .Revision history.SortControllerRevisions(revisions) revisionCount := len(revisions) @@ -323,7 +322,7 @@ func (p *Processor) registerLatestRevision(set *appsv1alpha1.SidecarSet, pods [] // build a new revision from the current sidecarSet, // the namespace of sidecarset revision must have very strict permissions for average users. // Here use the namespace of kruise-manager. - latestRevision, err = hc.NewRevision(sidecarSet, webhookutil.GetNamespace(), hc.NextRevision(revisions), &collisionCount) + latestRevision, err = sidecarcontrol.NewRevision(sidecarSet, p.namespace, sidecarcontrol.NextRevision(revisions), &collisionCount) if err != nil { return nil, collisionCount, err } @@ -340,7 +339,7 @@ func (p *Processor) registerLatestRevision(set *appsv1alpha1.SidecarSet, pods [] // if the equivalent revision is not immediately prior we will roll back by incrementing the // Revision of the equivalent revision // in case of roll back - latestRevision, err = p.historyController.UpdateControllerRevision(equalRevisions[equalCount-1], latestRevision.Revision) + latestRevision, err = p.historyControl.UpdateControllerRevision(equalRevisions[equalCount-1], latestRevision.Revision) if err != nil { return nil, collisionCount, err } @@ -349,7 +348,7 @@ func (p *Processor) registerLatestRevision(set *appsv1alpha1.SidecarSet, pods [] } else { // if there is no equivalent revision we create a new one // in case of the sidecarSet update - latestRevision, err = hc.CreateControllerRevision(sidecarSet, latestRevision, &collisionCount) + latestRevision, err = p.historyControl.CreateControllerRevision(sidecarSet, latestRevision, &collisionCount) if err != nil { return nil, collisionCount, err } @@ -362,10 +361,16 @@ func (p *Processor) registerLatestRevision(set *appsv1alpha1.SidecarSet, pods [] } // only store limited history revisions - if err = p.truncateHistory(revisions, sidecarSet, pods); err != nil { + deletedRevisions, err := p.truncateHistory(revisions, sidecarSet, pods) + if err != nil { klog.Errorf("Failed to truncate history for %s: err: %v", sidecarSet.Name, err) + return latestRevision, collisionCount, nil + } + for i := range deletedRevisions { + if err = p.historyControl.DeleteControllerRevision(deletedRevisions[i]); err != nil && !errors.IsNotFound(err) { + klog.Errorf("delete sidecarSet(%s) controllerRevision failed: %s", sidecarSet.Name, err.Error()) + } } - return latestRevision, collisionCount, nil } @@ -382,7 +387,7 @@ func (p *Processor) updateCustomVersionLabel(revision *apps.ControllerRevision, return nil } -func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *appsv1alpha1.SidecarSet, pods []*corev1.Pod) error { +func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *appsv1alpha1.SidecarSet, pods []*corev1.Pod) ([]*apps.ControllerRevision, error) { // We do not delete the latest revision because we are using it. // Thus, we must ensure the limitation is bounded, minimum value is 1. limitation := 10 @@ -392,29 +397,26 @@ func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *app // if no need to truncate revisionCount := len(revisions) if revisionCount <= limitation { - return nil + return nil, nil } klog.V(3).Infof("Find %v revisions more than limitation %v, name: %v", revisionCount, limitation, s.Name) - + var deletedRevisions []*apps.ControllerRevision // the number of revisions need to delete deletionCount := revisionCount - limitation // only delete the revisions that no pods use. activeRevisions := filterActiveRevisions(s, pods, revisions) for i := 0; i < revisionCount-1 && deletionCount > 0; i++ { if !activeRevisions.Has(revisions[i].Name) { - if err := p.historyController.DeleteControllerRevision(revisions[i]); err != nil && !errors.IsNotFound(err) { - return err - } + deletedRevisions = append(deletedRevisions, revisions[i]) deletionCount-- } } - // Sometime we cannot ensure the number of stored revisions is within the limitation because of the use by pods. if deletionCount > 0 { - return fmt.Errorf("failed to limit the number of stored revisions, limited: %d, actual: %d, name: %s", limitation, limitation+deletionCount, s.Name) + return nil, fmt.Errorf("failed to limit the number of stored revisions, limited: %d, actual: %d, name: %s", limitation, limitation+deletionCount, s.Name) } - return nil + return deletedRevisions, nil } func filterActiveRevisions(s *appsv1alpha1.SidecarSet, pods []*corev1.Pod, revisions []*apps.ControllerRevision) sets.String { @@ -474,17 +476,16 @@ func replaceRevision(revisions []*apps.ControllerRevision, oldOne, newOne *apps. // ReadyPods: ready pods number // UpdatedReadyPods: updated and ready pods number // UnavailablePods: MatchedPods - UpdatedReadyPods -func calculateStatus(control sidecarcontrol.SidecarControl, pods []*corev1.Pod, latestRevision *apps.ControllerRevision, collisionCount int32, -) *appsv1alpha1.SidecarSetStatus { - sidecarset := control.GetSidecarset() +func (p *Processor) calculateStatus(sidecarSet *appsv1alpha1.SidecarSet, pods []*corev1.Pod, latestRevision *apps.ControllerRevision, collisionCount int32) *appsv1alpha1.SidecarSetStatus { var matchedPods, updatedPods, readyPods, updatedAndReady int32 matchedPods = int32(len(pods)) for _, pod := range pods { - updated := sidecarcontrol.IsPodSidecarUpdated(sidecarset, pod) + updated := sidecarcontrol.IsPodSidecarUpdated(sidecarSet, pod) if updated { updatedPods++ } - if control.IsPodStateConsistent(pod, nil) && control.IsPodReady(pod) { + if p.sidecarSetControl.IsPodStateConsistent(pod, sidecarSet, nil) && + p.sidecarSetControl.IsPodReady(pod, sidecarSet) { readyPods++ if updated { updatedAndReady++ @@ -492,7 +493,7 @@ func calculateStatus(control sidecarcontrol.SidecarControl, pods []*corev1.Pod, } } return &appsv1alpha1.SidecarSetStatus{ - ObservedGeneration: sidecarset.Generation, + ObservedGeneration: sidecarSet.Generation, MatchedPods: matchedPods, UpdatedPods: updatedPods, ReadyPods: readyPods, @@ -519,16 +520,14 @@ func updateContainerInPod(container corev1.Container, pod *corev1.Pod) { } } -func updatePodSidecarContainer(control sidecarcontrol.SidecarControl, pod *corev1.Pod) { - sidecarSet := control.GetSidecarset() - +func (p *Processor) updatePodSidecarContainer(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) { // upgrade sidecar containers var changedContainers []string for _, sidecarContainer := range sidecarSet.Spec.Containers { //sidecarContainer := &sidecarset.Spec.Containers[i] // volumeMounts that injected into sidecar container // when volumeMounts SubPathExpr contains expansions, then need copy container EnvVars(injectEnvs) - injectedMounts, injectedEnvs := sidecarcontrol.GetInjectedVolumeMountsAndEnvs(control, &sidecarContainer, pod) + injectedMounts, injectedEnvs := sidecarcontrol.GetInjectedVolumeMountsAndEnvs(p.sidecarSetControl, &sidecarContainer, pod) // merge VolumeMounts from sidecar.VolumeMounts and shared VolumeMounts sidecarContainer.VolumeMounts = util.MergeVolumeMounts(sidecarContainer.VolumeMounts, injectedMounts) @@ -540,7 +539,7 @@ func updatePodSidecarContainer(control sidecarcontrol.SidecarControl, pod *corev sidecarContainer.Env = util.MergeEnvVar(sidecarContainer.Env, transferEnvs) // upgrade sidecar container to latest - newContainer := control.UpgradeSidecarContainer(&sidecarContainer, pod) + newContainer := p.sidecarSetControl.UpgradeSidecarContainer(&sidecarContainer, pod, sidecarSet) // no change, then continue if newContainer == nil { continue @@ -575,7 +574,7 @@ func updatePodSidecarContainer(control sidecarcontrol.SidecarControl, pod *corev // update pod information in upgrade // UpdatePodAnnotationsInUpgrade needs to be called when Update Container, including hot-upgrade reset empty image. // However, reset empty image should not update pod sidecarSet hash annotation, so UpdatePodSidecarSetHash needs to be called additionally - control.UpdatePodAnnotationsInUpgrade(changedContainers, pod) + p.sidecarSetControl.UpdatePodAnnotationsInUpgrade(changedContainers, pod, sidecarSet) } func inconsistentStatus(sidecarSet *appsv1alpha1.SidecarSet, status *appsv1alpha1.SidecarSetStatus) bool { diff --git a/pkg/controller/sidecarset/sidecarset_processor_test.go b/pkg/controller/sidecarset/sidecarset_processor_test.go index 7f4d2b01a7..7fc1dcb08d 100644 --- a/pkg/controller/sidecarset/sidecarset_processor_test.go +++ b/pkg/controller/sidecarset/sidecarset_processor_test.go @@ -24,14 +24,15 @@ import ( "testing" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" "github.com/openkruise/kruise/pkg/util/expectations" webhookutil "github.com/openkruise/kruise/pkg/webhook/util" - + "github.com/openkruise/utils/sidecarcontrol" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/controller/history" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -178,7 +179,9 @@ func testUpdateColdUpgradeSidecar(t *testing.T, podDemo *corev1.Pod, sidecarSetI pods := cs.getPods() sidecarset := cs.getSidecarset() fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sidecarset, pods[0], pods[1]).Build() - processor := NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} + processor := NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)) _, err := processor.UpdateSidecarSet(sidecarset) if err != nil { t.Errorf("processor update sidecarset failed: %s", err.Error()) @@ -254,7 +257,9 @@ func TestScopeNamespacePods(t *testing.T) { fakeClient.Create(context.TODO(), pod) } exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl) - processor := NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} + processor := NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)) pods, err := processor.getMatchingPods(sidecarSet) if err != nil { t.Fatalf("getMatchingPods failed: %s", err.Error()) @@ -285,8 +290,9 @@ func TestCanUpgradePods(t *testing.T) { } fakeClient.Create(context.TODO(), pods[i]) } - - processor := NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} + processor := NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)) _, err := processor.UpdateSidecarSet(sidecarSet) if err != nil { t.Errorf("processor update sidecarset failed: %s", err.Error()) @@ -319,7 +325,9 @@ func TestGetActiveRevisions(t *testing.T) { kubeSysNs.SetNamespace(webhookutil.GetNamespace()) fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sidecarSet, kubeSysNs).Build() exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl) - processor := NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} + processor := NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)) // case 1 latestRevision, _, err := processor.registerLatestRevision(sidecarSet, nil) @@ -327,7 +335,7 @@ func TestGetActiveRevisions(t *testing.T) { t.Fatalf("in case of create: get active revision failed when the latest revision = 1, err: %v, actual latestRevision: %v", err, latestRevision.Revision) } - + _ = indexer.Add(latestRevision) // case 2 newSidecar := sidecarSet.DeepCopy() newSidecar.Spec.InitContainers = []appsv1alpha1.SidecarContainer{ @@ -358,7 +366,7 @@ func TestGetActiveRevisions(t *testing.T) { t.Fatalf("failed to store revision, err: %v", err) } } - + _ = indexer.Add(latestRevision) // case 3 for i := 0; i < 5; i++ { latestRevision, _, err = processor.registerLatestRevision(sidecarSet, nil) @@ -367,18 +375,35 @@ func TestGetActiveRevisions(t *testing.T) { err, latestRevision.Revision) } } - + _ = indexer.Add(latestRevision) // case 4 for i := 0; i < 100; i++ { sidecarSet.Spec.Containers[0].Image = fmt.Sprintf("%d", i) - if _, _, err = processor.registerLatestRevision(sidecarSet, nil); err != nil { + if latestRevision, _, err = processor.registerLatestRevision(sidecarSet, nil); err != nil { t.Fatalf("unexpected error, err: %v", err) + } else { + _ = indexer.Add(latestRevision) + objList := indexer.List() + var revisions []*apps.ControllerRevision + for _, obj := range objList { + revision := obj.(*apps.ControllerRevision) + revisions = append(revisions, revision) + } + history.SortControllerRevisions(revisions) + revisionCount := len(revisions) + limitation := int(*sidecarSet.Spec.RevisionHistoryLimit) + if revisionCount <= limitation { + continue + } + deletionCount := revisionCount - limitation + for i := 0; i < revisionCount-1 && deletionCount > 0; i++ { + indexer.Delete(revisions[i]) + deletionCount-- + } } } - revisionList := &apps.ControllerRevisionList{} - processor.Client.List(context.TODO(), revisionList) - if len(revisionList.Items) != int(*sidecarSet.Spec.RevisionHistoryLimit) { - t.Fatalf("in case of maxStoredRevisions: get wrong number of revisions, expected %d, actual %d", *sidecarSet.Spec.RevisionHistoryLimit, len(revisionList.Items)) + if len(indexer.List()) != int(*sidecarSet.Spec.RevisionHistoryLimit) { + t.Fatalf("in case of maxStoredRevisions: get wrong number of revisions, expected %d, actual %d", *sidecarSet.Spec.RevisionHistoryLimit, len(indexer.List())) } } @@ -434,7 +459,9 @@ func TestTruncateHistory(t *testing.T) { kubeSysNs.SetNamespace(webhookutil.GetNamespace()) fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sidecarSet, kubeSysNs).Build() exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl) - processor := NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + k8sclient := &k8sfake.Clientset{} + processor := NewSidecarSetProcessor(fakeClient, k8sclient, indexer, exps, record.NewFakeRecorder(10)) getName := func(i int) string { return "sidecar-" + strconv.Itoa(i) @@ -467,7 +494,7 @@ func TestTruncateHistory(t *testing.T) { // check successful cases for i := 1; i <= 9; i++ { pods, revisions := reset(i) - err := processor.truncateHistory(revisions, sidecarSet, pods) + _, err := processor.truncateHistory(revisions, sidecarSet, pods) if err != nil { t.Fatalf("expected revision len: %d, err: %v", 10, err) } @@ -478,7 +505,7 @@ func TestTruncateHistory(t *testing.T) { expectedResults := []int{11, 12, 15, 15, 15} for i := range failedCases { pods, revisions := reset(failedCases[i]) - err := processor.truncateHistory(revisions, sidecarSet, pods) + _, err := processor.truncateHistory(revisions, sidecarSet, pods) if err == nil || err.Error() != stderr(expectedResults[i]) { t.Fatalf("expected revision len: %d, err: %v", expectedResults[i], err) } @@ -486,25 +513,24 @@ func TestTruncateHistory(t *testing.T) { // check revisions exactly pods, revisions := reset(8) - for _, rv := range revisions { - if err := processor.Client.Create(context.TODO(), rv); err != nil { - t.Fatalf("failed to create revisions") - } + for i := range revisions { + _ = indexer.Add(revisions[i]) } - if err := processor.truncateHistory(revisions, sidecarSet, pods); err != nil { + deleteRevisions, err := processor.truncateHistory(revisions, sidecarSet, pods) + if err != nil { t.Fatalf("failed to truncate revisions, err %v", err) } - list := &apps.ControllerRevisionList{} - if err := processor.Client.List(context.TODO(), list); err != nil { - t.Fatalf("failed to list revisions, err %v", err) + for i := range deleteRevisions { + _ = indexer.Delete(deleteRevisions[i]) } - if len(list.Items) != 10 { - t.Fatalf("expected revision len: %d, actual: %d", 10, len(list.Items)) + if len(indexer.List()) != 10 { + t.Fatalf("expected revision len: %d, actual: %d", 10, len(indexer.List())) } // sort history by revision field rvs := make([]*apps.ControllerRevision, 0) - for i := range list.Items { - rvs = append(rvs, &list.Items[i]) + for _, obj := range indexer.List() { + revision := obj.(*apps.ControllerRevision) + rvs = append(rvs, revision) } history.SortControllerRevisions(rvs) diff --git a/pkg/controller/sidecarset/sidecarset_strategy.go b/pkg/controller/sidecarset/sidecarset_strategy.go index 612d8d25ba..4384504170 100644 --- a/pkg/controller/sidecarset/sidecarset_strategy.go +++ b/pkg/controller/sidecarset/sidecarset_strategy.go @@ -4,9 +4,9 @@ import ( "sort" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" "github.com/openkruise/kruise/pkg/util/updatesort" + "github.com/openkruise/utils/sidecarcontrol" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" @@ -22,24 +22,23 @@ type Strategy interface { //2. Sort Pods with default sequence //3. sort waitUpdateIndexes based on the scatter rules //4. calculate max count of pods can update with maxUnavailable - GetNextUpgradePods(control sidecarcontrol.SidecarControl, pods []*corev1.Pod) []*corev1.Pod + GetNextUpgradePods(sidecarSet *appsv1alpha1.SidecarSet, pods []*corev1.Pod) []*corev1.Pod } -type spreadingStrategy struct{} - -var ( - globalSpreadingStrategy = &spreadingStrategy{} -) +type spreadingStrategy struct { + sidecarSetControl sidecarcontrol.SidecarControl +} -func NewStrategy() Strategy { - return globalSpreadingStrategy +func NewStrategy(control sidecarcontrol.SidecarControl) Strategy { + return &spreadingStrategy{ + sidecarSetControl: control, + } } -func (p *spreadingStrategy) GetNextUpgradePods(control sidecarcontrol.SidecarControl, pods []*corev1.Pod) (upgradePods []*corev1.Pod) { - sidecarset := control.GetSidecarset() +func (p *spreadingStrategy) GetNextUpgradePods(sidecarSet *appsv1alpha1.SidecarSet, pods []*corev1.Pod) (upgradePods []*corev1.Pod) { // wait to upgrade pod index var waitUpgradedIndexes []int - strategy := sidecarset.Spec.UpdateStrategy + strategy := sidecarSet.Spec.UpdateStrategy // If selector is not nil, check whether the pods is selected to upgrade isSelected := func(pod *corev1.Pod) bool { @@ -50,7 +49,7 @@ func (p *spreadingStrategy) GetNextUpgradePods(control sidecarcontrol.SidecarCon // if selector failed, always return false selector, err := util.ValidatedLabelSelectorAsSelector(strategy.Selector) if err != nil { - klog.Errorf("sidecarSet(%s) rolling selector error, err: %v", sidecarset.Name, err) + klog.Errorf("sidecarSet(%s) rolling selector error, err: %v", sidecarSet.Name, err) return false } //matched @@ -67,18 +66,18 @@ func (p *spreadingStrategy) GetNextUpgradePods(control sidecarcontrol.SidecarCon // * In kubernetes cluster, when inplace update pod, only fields such as image can be updated for the container. // * It is to determine whether there are other fields that have been modified for pod. for index, pod := range pods { - isUpdated := sidecarcontrol.IsPodSidecarUpdated(sidecarset, pod) - if !isUpdated && isSelected(pod) && control.IsSidecarSetUpgradable(pod) { + isUpdated := sidecarcontrol.IsPodSidecarUpdated(sidecarSet, pod) + if !isUpdated && isSelected(pod) && p.sidecarSetControl.IsSidecarSetUpgradable(pod, sidecarSet) { waitUpgradedIndexes = append(waitUpgradedIndexes, index) } } - klog.V(3).Infof("sidecarSet(%s) matchedPods(%d) waitUpdated(%d)", sidecarset.Name, len(pods), len(waitUpgradedIndexes)) + klog.V(3).Infof("sidecarSet(%s) matchedPods(%d) waitUpdated(%d)", sidecarSet.Name, len(pods), len(waitUpgradedIndexes)) //2. sort Pods with default sequence and scatter waitUpgradedIndexes = SortUpdateIndexes(strategy, pods, waitUpgradedIndexes) //3. calculate to be upgraded pods number for the time - needToUpgradeCount := calculateUpgradeCount(control, waitUpgradedIndexes, pods) + needToUpgradeCount := p.calculateUpgradeCount(sidecarSet, waitUpgradedIndexes, pods) if needToUpgradeCount < len(waitUpgradedIndexes) { waitUpgradedIndexes = waitUpgradedIndexes[:needToUpgradeCount] } @@ -112,9 +111,8 @@ func SortUpdateIndexes(strategy appsv1alpha1.SidecarSetUpdateStrategy, pods []*c return waitUpdateIndexes } -func calculateUpgradeCount(coreControl sidecarcontrol.SidecarControl, waitUpdateIndexes []int, pods []*corev1.Pod) int { +func (p *spreadingStrategy) calculateUpgradeCount(sidecarSet *appsv1alpha1.SidecarSet, waitUpdateIndexes []int, pods []*corev1.Pod) int { totalReplicas := len(pods) - sidecarSet := coreControl.GetSidecarset() strategy := sidecarSet.Spec.UpdateStrategy // default partition = 0, indicates all pods will been upgraded @@ -139,14 +137,16 @@ func calculateUpgradeCount(coreControl sidecarcontrol.SidecarControl, waitUpdate // 1. sidecar containers have been updated to the latest sidecarSet version, for pod.spec.containers // 2. whether pod.spec and pod.status is inconsistent after updating the sidecar containers // 3. whether pod is not ready - if sidecarcontrol.IsPodSidecarUpdated(sidecarSet, pod) && (!coreControl.IsPodStateConsistent(pod, nil) || !coreControl.IsPodReady(pod)) { + if sidecarcontrol.IsPodSidecarUpdated(sidecarSet, pod) && + (!p.sidecarSetControl.IsPodStateConsistent(pod, sidecarSet, nil) || + !p.sidecarSetControl.IsPodReady(pod, sidecarSet)) { upgradeAndNotReadyCount++ } } var needUpgradeCount int for _, i := range waitUpdateIndexes { // If pod is not ready, then not included in the calculation of maxUnavailable - if !coreControl.IsPodReady(pods[i]) { + if !p.sidecarSetControl.IsPodReady(pods[i], sidecarSet) { needUpgradeCount++ continue } diff --git a/pkg/controller/sidecarset/sidecarset_strategy_test.go b/pkg/controller/sidecarset/sidecarset_strategy_test.go index 48a0c9ee42..6fdb88520f 100644 --- a/pkg/controller/sidecarset/sidecarset_strategy_test.go +++ b/pkg/controller/sidecarset/sidecarset_strategy_test.go @@ -24,7 +24,7 @@ import ( "testing" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" + "github.com/openkruise/utils/sidecarcontrol" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,7 +37,7 @@ type FactorySidecarSet func() *appsv1alpha1.SidecarSet type FactoryPods func(int, int, int) []*corev1.Pod func factoryPodsCommon(count, upgraded int, sidecarSet *appsv1alpha1.SidecarSet) []*corev1.Pod { - control := sidecarcontrol.New(sidecarSet) + control := sidecarcontrol.NewCommonControl(nil, "") pods := make([]*corev1.Pod, 0, count) for i := 0; i < count; i++ { pod := &corev1.Pod{ @@ -92,8 +92,8 @@ func factoryPodsCommon(count, upgraded int, sidecarSet *appsv1alpha1.SidecarSet) } for i := 0; i < upgraded; i++ { pods[i].Spec.Containers[1].Image = "test-image:v2" - sidecarcontrol.UpdatePodSidecarSetHash(pods[i], control.GetSidecarset()) - control.UpdatePodAnnotationsInUpgrade([]string{"test-sidecar"}, pods[i]) + sidecarcontrol.UpdatePodSidecarSetHash(pods[i], sidecarSet) + control.UpdatePodAnnotationsInUpgrade([]string{"test-sidecar"}, pods[i], sidecarSet) } return pods } @@ -319,12 +319,11 @@ func testGetNextUpgradePods(t *testing.T, factoryPods FactoryPods, factorySideca exceptNeedUpgradeCount: 30, }, } - strategy := NewStrategy() + strategy := NewStrategy(sidecarcontrol.NewCommonControl(nil, "")) for _, cs := range cases { t.Run(cs.name, func(t *testing.T) { - control := sidecarcontrol.New(cs.getSidecarset()) pods := cs.getPods() - upgradePods := strategy.GetNextUpgradePods(control, pods) + upgradePods := strategy.GetNextUpgradePods(cs.getSidecarset(), pods) if cs.exceptNeedUpgradeCount != len(upgradePods) { t.Fatalf("except NeedUpgradeCount(%d), but get value(%d)", cs.exceptNeedUpgradeCount, len(upgradePods)) } @@ -519,12 +518,11 @@ func testSortNextUpgradePods(t *testing.T, factoryPods FactoryPods, factorySidec }, } - strategy := NewStrategy() + strategy := NewStrategy(sidecarcontrol.NewCommonControl(nil, "")) for _, cs := range cases { t.Run(cs.name, func(t *testing.T) { - control := sidecarcontrol.New(cs.getSidecarset()) pods := cs.getPods() - injectedPods := strategy.GetNextUpgradePods(control, pods) + injectedPods := strategy.GetNextUpgradePods(cs.getSidecarset(), pods) if len(cs.exceptNextUpgradePods) != len(injectedPods) { t.Fatalf("except NeedUpgradeCount(%d), but get value(%d)", len(cs.exceptNextUpgradePods), len(injectedPods)) } diff --git a/pkg/daemon/containermeta/container_meta_controller.go b/pkg/daemon/containermeta/container_meta_controller.go index 93111f0b4e..7947182097 100644 --- a/pkg/daemon/containermeta/container_meta_controller.go +++ b/pkg/daemon/containermeta/container_meta_controller.go @@ -30,7 +30,6 @@ import ( appspub "github.com/openkruise/kruise/apis/apps/pub" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" "github.com/openkruise/kruise/pkg/client" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" daemonruntime "github.com/openkruise/kruise/pkg/daemon/criruntime" "github.com/openkruise/kruise/pkg/daemon/kuberuntime" daemonoptions "github.com/openkruise/kruise/pkg/daemon/options" @@ -39,6 +38,7 @@ import ( utilcontainermeta "github.com/openkruise/kruise/pkg/util/containermeta" "github.com/openkruise/kruise/pkg/util/expectations" utilfeature "github.com/openkruise/kruise/pkg/util/feature" + "github.com/openkruise/utils/sidecarcontrol" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/webhook/pod/mutating/persistent_pod_state_test.go b/pkg/webhook/pod/mutating/persistent_pod_state_test.go index 7964589de8..5fa352dc20 100644 --- a/pkg/webhook/pod/mutating/persistent_pod_state_test.go +++ b/pkg/webhook/pod/mutating/persistent_pod_state_test.go @@ -235,3 +235,19 @@ func TestPersistentPodStateMutatingPod(t *testing.T) { }) } } + +func newAdmission(op admissionv1.Operation, object, oldObject runtime.RawExtension, subResource string) admission.Request { + return admission.Request{ + AdmissionRequest: newAdmissionRequest(op, object, oldObject, subResource), + } +} + +func newAdmissionRequest(op admissionv1.Operation, object, oldObject runtime.RawExtension, subResource string) admissionv1.AdmissionRequest { + return admissionv1.AdmissionRequest{ + Resource: metav1.GroupVersionResource{Group: corev1.SchemeGroupVersion.Group, Version: corev1.SchemeGroupVersion.Version, Resource: "pods"}, + Operation: op, + Object: object, + OldObject: oldObject, + SubResource: subResource, + } +} diff --git a/pkg/webhook/pod/mutating/pod_create_update_handler.go b/pkg/webhook/pod/mutating/pod_create_update_handler.go index 9a8641ae76..7a5a6ff28e 100644 --- a/pkg/webhook/pod/mutating/pod_create_update_handler.go +++ b/pkg/webhook/pod/mutating/pod_create_update_handler.go @@ -24,7 +24,12 @@ import ( "github.com/openkruise/kruise/pkg/features" "github.com/openkruise/kruise/pkg/util/controllerfinder" utilfeature "github.com/openkruise/kruise/pkg/util/feature" + webhookutil "github.com/openkruise/kruise/pkg/webhook/util" + "github.com/openkruise/utils/sidecarcontrol" + apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + k8scache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -41,7 +46,9 @@ type PodCreateHandler struct { // Decoder decodes objects Decoder *admission.Decoder - finder *controllerfinder.ControllerFinder + // sidecarSet control + sidecarSetControl sidecarcontrol.SidecarControl + finder *controllerfinder.ControllerFinder } var _ admission.Handler = &PodCreateHandler{} @@ -73,7 +80,7 @@ func (h *PodCreateHandler) Handle(ctx context.Context, req admission.Request) ad } } - if skip, err := h.sidecarsetMutatingPod(ctx, req, obj); err != nil { + if skip, err := h.sidecarSetMutatingPod(ctx, req, obj); err != nil { return admission.Errored(http.StatusInternalServerError, err) } else if !skip { changed = true @@ -119,6 +126,7 @@ var _ inject.Client = &PodCreateHandler{} func (h *PodCreateHandler) InjectClient(c client.Client) error { h.Client = c h.finder = controllerfinder.Finder + return nil } @@ -129,3 +137,15 @@ func (h *PodCreateHandler) InjectDecoder(d *admission.Decoder) error { h.Decoder = d return nil } + +var _ inject.Cache = &PodCreateHandler{} + +// InjectCache injects the decoder into the PodCreateHandler +func (h *PodCreateHandler) InjectCache(cacher cache.Cache) error { + revInformer, err := cacher.GetInformerForKind(context.TODO(), apps.SchemeGroupVersion.WithKind("ControllerRevision")) + if err != nil { + return err + } + h.sidecarSetControl = sidecarcontrol.NewCommonControl(revInformer.(k8scache.SharedIndexInformer).GetIndexer(), webhookutil.GetNamespace()) + return nil +} diff --git a/pkg/webhook/pod/mutating/sidecarset.go b/pkg/webhook/pod/mutating/sidecarset.go index 45fb0dbcba..c5cfde47c9 100644 --- a/pkg/webhook/pod/mutating/sidecarset.go +++ b/pkg/webhook/pod/mutating/sidecarset.go @@ -18,28 +18,17 @@ package mutating import ( "context" - "encoding/json" - "fmt" - "sort" - "strings" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" - "github.com/openkruise/kruise/pkg/util" utilclient "github.com/openkruise/kruise/pkg/util/client" - "github.com/openkruise/kruise/pkg/util/history" - + "github.com/openkruise/utils/sidecarcontrol" admissionv1 "k8s.io/api/admission/v1" - apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) // mutate pod based on SidecarSet Object -func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admission.Request, pod *corev1.Pod) (skip bool, err error) { +func (h *PodCreateHandler) sidecarSetMutatingPod(ctx context.Context, req admission.Request, pod *corev1.Pod) (skip bool, err error) { if len(req.AdmissionRequest.SubResource) > 0 || (req.AdmissionRequest.Operation != admissionv1.Create && req.AdmissionRequest.Operation != admissionv1.Update) || req.AdmissionRequest.Resource.Resource != "pods" { @@ -51,10 +40,8 @@ func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admiss } var oldPod *corev1.Pod - var isUpdated bool //when Operation is update, decode older object if req.AdmissionRequest.Operation == admissionv1.Update { - isUpdated = true oldPod = new(corev1.Pod) if err = h.Decoder.Decode( admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Object: req.AdmissionRequest.OldObject}}, @@ -64,397 +51,14 @@ func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admiss } // DisableDeepCopy:true, indicates must be deep copy before update sidecarSet objection - sidecarsetList := &appsv1alpha1.SidecarSetList{} - if err = h.Client.List(ctx, sidecarsetList, utilclient.DisableDeepCopy); err != nil { - return false, err - } - - matchedSidecarSets := make([]sidecarcontrol.SidecarControl, 0) - for _, sidecarSet := range sidecarsetList.Items { - if sidecarSet.Spec.InjectionStrategy.Paused { - continue - } - if matched, err := sidecarcontrol.PodMatchedSidecarSet(pod, sidecarSet); err != nil { - return false, err - } else if !matched { - continue - } - // get user-specific revision or the latest revision of SidecarSet - suitableSidecarSet, err := h.getSuitableRevisionSidecarSet(&sidecarSet, oldPod, pod, req.AdmissionRequest.Operation) - if err != nil { - return false, err - } - // check whether sidecarSet is active - // when sidecarSet is not active, it will not perform injections and upgrades process. - control := sidecarcontrol.New(suitableSidecarSet) - if !control.IsActiveSidecarSet() { - continue - } - matchedSidecarSets = append(matchedSidecarSets, control) - } - if len(matchedSidecarSets) == 0 { - return true, nil - } - - // check pod - if isUpdated { - if !matchedSidecarSets[0].IsPodAvailabilityChanged(pod, oldPod) { - klog.V(3).Infof("pod(%s/%s) availability unchanged for sidecarSet, and ignore", pod.Namespace, pod.Name) - return true, nil - } - } - - klog.V(3).Infof("[sidecar inject] begin to operation(%s) pod(%s/%s) resources(%s) subResources(%s)", - req.Operation, req.Namespace, req.Name, req.Resource, req.SubResource) - // patch pod metadata, annotations & labels - // When the Pod main container is upgraded in place, and the sidecarSet configuration does not change at this time, - // at this point, it can also patch pod metadata - if pod.Annotations == nil { - pod.Annotations = make(map[string]string) - } - skip = true - for _, control := range matchedSidecarSets { - sidecarSet := control.GetSidecarset() - sk, err := sidecarcontrol.PatchPodMetadata(&pod.ObjectMeta, sidecarSet.Spec.PatchPodMetadata) - if err != nil { - klog.Errorf("sidecarSet(%s) update pod(%s/%s) metadata failed: %s", sidecarSet.Name, pod.Namespace, pod.Name, err.Error()) - return false, err - } else if !sk { - // skip = false - skip = false - } - } - //build sidecar containers, sidecar initContainers, sidecar volumes, annotations to inject into pod object - sidecarContainers, sidecarInitContainers, sidecarSecrets, volumesInSidecar, injectedAnnotations, err := buildSidecars(isUpdated, pod, oldPod, matchedSidecarSets) - if err != nil { + sidecarSetList := &appsv1alpha1.SidecarSetList{} + if err = h.Client.List(ctx, sidecarSetList, utilclient.DisableDeepCopy); err != nil { return false, err - } else if len(sidecarContainers) == 0 && len(sidecarInitContainers) == 0 { - klog.V(3).Infof("[sidecar inject] pod(%s/%s) don't have injected containers", pod.Namespace, pod.Name) - return skip, nil } - - klog.V(3).Infof("[sidecar inject] begin inject sidecarContainers(%v) sidecarInitContainers(%v) sidecarSecrets(%v), volumes(%s)"+ - "annotations(%v) into pod(%s/%s)", sidecarContainers, sidecarInitContainers, sidecarSecrets, volumesInSidecar, injectedAnnotations, - pod.Namespace, pod.Name) - klog.V(4).Infof("[sidecar inject] before mutating: %v", util.DumpJSON(pod)) - // apply sidecar set info into pod - // 1. inject init containers, sort by their name, after the original init containers - sort.SliceStable(sidecarInitContainers, func(i, j int) bool { - return sidecarInitContainers[i].Name < sidecarInitContainers[j].Name - }) - for _, initContainer := range sidecarInitContainers { - pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer.Container) - } - // 2. inject containers - pod.Spec.Containers = mergeSidecarContainers(pod.Spec.Containers, sidecarContainers) - // 3. inject volumes - pod.Spec.Volumes = util.MergeVolumes(pod.Spec.Volumes, volumesInSidecar) - // 4. inject imagePullSecrets - pod.Spec.ImagePullSecrets = mergeSidecarSecrets(pod.Spec.ImagePullSecrets, sidecarSecrets) - // 5. apply annotations - for k, v := range injectedAnnotations { - pod.Annotations[k] = v - } - klog.V(4).Infof("[sidecar inject] after mutating: %v", util.DumpJSON(pod)) - return false, nil -} - -func (h *PodCreateHandler) getSuitableRevisionSidecarSet(sidecarSet *appsv1alpha1.SidecarSet, oldPod, newPod *corev1.Pod, operation admissionv1.Operation) (*appsv1alpha1.SidecarSet, error) { - switch operation { - case admissionv1.Update: - // optimization: quickly return if newPod matched the latest sidecarSet - if sidecarcontrol.GetPodSidecarSetRevision(sidecarSet.Name, newPod) == sidecarcontrol.GetSidecarSetRevision(sidecarSet) { - return sidecarSet.DeepCopy(), nil - } - - hc := sidecarcontrol.NewHistoryControl(h.Client) - revisions, err := history.NewHistory(h.Client).ListControllerRevisions(sidecarcontrol.MockSidecarSetForRevision(sidecarSet), hc.GetRevisionSelector(sidecarSet)) - if err != nil { - klog.Errorf("Failed to list history controllerRevisions, err %v, name %v", err, sidecarSet.Name) - return nil, err - } - - suitableSidecarSet, err := h.getSpecificRevisionSidecarSetForPod(sidecarSet, revisions, newPod) - if err != nil { - return nil, err - } else if suitableSidecarSet != nil { - return suitableSidecarSet, nil - } - - suitableSidecarSet, err = h.getSpecificRevisionSidecarSetForPod(sidecarSet, revisions, oldPod) - if err != nil { - return nil, err - } else if suitableSidecarSet != nil { - return suitableSidecarSet, nil - } - - return sidecarSet.DeepCopy(), nil - - default: - revisionInfo := sidecarSet.Spec.InjectionStrategy.Revision - if revisionInfo == nil || (revisionInfo.RevisionName == nil && revisionInfo.CustomVersion == nil) { - return sidecarSet.DeepCopy(), nil - } - - // TODO: support 'PartitionBased' policy to inject old/new revision according to Partition - switch sidecarSet.Spec.InjectionStrategy.Revision.Policy { - case "", appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy: - return h.getSpecificHistorySidecarSet(sidecarSet, revisionInfo) - } - - return h.getSpecificHistorySidecarSet(sidecarSet, revisionInfo) - } -} - -func (h *PodCreateHandler) getSpecificRevisionSidecarSetForPod(sidecarSet *appsv1alpha1.SidecarSet, revisions []*apps.ControllerRevision, pod *corev1.Pod) (*appsv1alpha1.SidecarSet, error) { - var err error - var matchedSidecarSet *appsv1alpha1.SidecarSet - for _, revision := range revisions { - if sidecarcontrol.GetPodSidecarSetControllerRevision(sidecarSet.Name, pod) == revision.Name { - matchedSidecarSet, err = h.getSpecificHistorySidecarSet(sidecarSet, &appsv1alpha1.SidecarSetInjectRevision{RevisionName: &revision.Name}) - if err != nil { - return nil, err - } - break - } - } - return matchedSidecarSet, nil -} - -func (h *PodCreateHandler) getSpecificHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error) { - // else return its corresponding history revision - hc := sidecarcontrol.NewHistoryControl(h.Client) - historySidecarSet, err := hc.GetHistorySidecarSet(sidecarSet, revisionInfo) - if err != nil { - klog.Warningf("Failed to restore history revision for SidecarSet %v, ControllerRevision name %v:, error: %v", - sidecarSet.Name, sidecarSet.Spec.InjectionStrategy.Revision, err) - return nil, err - } - if historySidecarSet == nil { - historySidecarSet = sidecarSet.DeepCopy() - klog.Warningf("Failed to restore history revision for SidecarSet %v, will use the latest", sidecarSet.Name) - } - return historySidecarSet, nil -} - -func mergeSidecarSecrets(secretsInPod, secretsInSidecar []corev1.LocalObjectReference) (allSecrets []corev1.LocalObjectReference) { - secretFilter := make(map[string]bool) - for _, podSecret := range secretsInPod { - if _, ok := secretFilter[podSecret.Name]; !ok { - secretFilter[podSecret.Name] = true - allSecrets = append(allSecrets, podSecret) - } - } - for _, sidecarSecret := range secretsInSidecar { - if _, ok := secretFilter[sidecarSecret.Name]; !ok { - secretFilter[sidecarSecret.Name] = true - allSecrets = append(allSecrets, sidecarSecret) - } - } - return allSecrets -} - -func mergeSidecarContainers(origins []corev1.Container, injected []*appsv1alpha1.SidecarContainer) []corev1.Container { - //format: pod.spec.containers[index].name -> index(the index of container in pod) - containersInPod := make(map[string]int) - for index, container := range origins { - containersInPod[container.Name] = index - } - var beforeAppContainers []corev1.Container - var afterAppContainers []corev1.Container - for _, sidecar := range injected { - //sidecar container already exist in pod - //keep the order of pod's original containers unchanged - if index, ok := containersInPod[sidecar.Name]; ok { - origins[index] = sidecar.Container - continue - } - - switch sidecar.PodInjectPolicy { - case appsv1alpha1.BeforeAppContainerType: - beforeAppContainers = append(beforeAppContainers, sidecar.Container) - case appsv1alpha1.AfterAppContainerType: - afterAppContainers = append(afterAppContainers, sidecar.Container) - default: - beforeAppContainers = append(beforeAppContainers, sidecar.Container) - } - } - origins = append(beforeAppContainers, origins...) - origins = append(origins, afterAppContainers...) - return origins -} - -func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedSidecarSets []sidecarcontrol.SidecarControl) ( - sidecarContainers, sidecarInitContainers []*appsv1alpha1.SidecarContainer, sidecarSecrets []corev1.LocalObjectReference, - volumesInSidecars []corev1.Volume, injectedAnnotations map[string]string, err error) { - - // injected annotations - injectedAnnotations = make(map[string]string) - // get sidecarSet annotations from pods - // sidecarSet.name -> sidecarSet hash struct - sidecarSetHash := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec) - // sidecarSet.name -> sidecarSet hash(without image) struct - sidecarSetHashWithoutImage := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec) - // parse sidecar hash in pod annotations - if oldHashStr := pod.Annotations[sidecarcontrol.SidecarSetHashAnnotation]; len(oldHashStr) > 0 { - if err = json.Unmarshal([]byte(oldHashStr), &sidecarSetHash); err != nil { - // to be compatible with older sidecarSet hash struct, map[string]string - olderSidecarSetHash := make(map[string]string) - if err = json.Unmarshal([]byte(oldHashStr), &olderSidecarSetHash); err != nil { - return nil, nil, nil, nil, nil, - fmt.Errorf("pod(%s/%s) invalid annotations[%s] value %v, unmarshal failed: %v", pod.Namespace, pod.Name, sidecarcontrol.SidecarSetHashAnnotation, oldHashStr, err) - } - for k, v := range olderSidecarSetHash { - sidecarSetHash[k] = sidecarcontrol.SidecarSetUpgradeSpec{ - SidecarSetHash: v, - SidecarSetName: k, - } - } - } - } - if oldHashStr := pod.Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation]; len(oldHashStr) > 0 { - if err = json.Unmarshal([]byte(oldHashStr), &sidecarSetHashWithoutImage); err != nil { - // to be compatible with older sidecarSet hash struct, map[string]string - olderSidecarSetHash := make(map[string]string) - if err = json.Unmarshal([]byte(oldHashStr), &olderSidecarSetHash); err != nil { - return nil, nil, nil, nil, nil, - fmt.Errorf("pod(%s/%s) invalid annotations[%s] value %v, unmarshal failed: %v", pod.Namespace, pod.Name, sidecarcontrol.SidecarSetHashWithoutImageAnnotation, oldHashStr, err) - } - for k, v := range olderSidecarSetHash { - sidecarSetHashWithoutImage[k] = sidecarcontrol.SidecarSetUpgradeSpec{ - SidecarSetHash: v, - SidecarSetName: k, - } - } - } - } - // hotUpgrade work info, sidecarSet.spec.container[x].name -> pod.spec.container[x].name - // for example: mesh -> mesh-1, envoy -> envoy-2 - hotUpgradeWorkInfo := sidecarcontrol.GetPodHotUpgradeInfoInAnnotations(pod) - // SidecarSet Name List, for example: log-sidecarset,envoy-sidecarset - sidecarSetNames := sets.NewString() - if sidecarSetListStr := pod.Annotations[sidecarcontrol.SidecarSetListAnnotation]; sidecarSetListStr != "" { - sidecarSetNames.Insert(strings.Split(sidecarSetListStr, ",")...) - } - - for _, control := range matchedSidecarSets { - sidecarSet := control.GetSidecarset() - klog.V(3).Infof("build pod(%s/%s) sidecar containers for sidecarSet(%s)", pod.Namespace, pod.Name, sidecarSet.Name) - // sidecarSet List - sidecarSetNames.Insert(sidecarSet.Name) - // pre-process volumes only in sidecar - volumesMap := getVolumesMapInSidecarSet(sidecarSet) - // process sidecarset hash - setUpgrade1 := sidecarcontrol.SidecarSetUpgradeSpec{ - UpdateTimestamp: metav1.Now(), - SidecarSetHash: sidecarcontrol.GetSidecarSetRevision(sidecarSet), - SidecarSetName: sidecarSet.Name, - SidecarSetControllerRevision: sidecarSet.Status.LatestRevision, - } - setUpgrade2 := sidecarcontrol.SidecarSetUpgradeSpec{ - UpdateTimestamp: metav1.Now(), - SidecarSetHash: sidecarcontrol.GetSidecarSetWithoutImageRevision(sidecarSet), - SidecarSetName: sidecarSet.Name, - } - - //process initContainers - //only when created pod, inject initContainer and pullSecrets - if !isUpdated { - for i := range sidecarSet.Spec.InitContainers { - initContainer := &sidecarSet.Spec.InitContainers[i] - //add "IS_INJECTED" env in initContainer's envs - initContainer.Env = append(initContainer.Env, corev1.EnvVar{Name: sidecarcontrol.SidecarEnvKey, Value: "true"}) - transferEnvs := sidecarcontrol.GetSidecarTransferEnvs(initContainer, pod) - initContainer.Env = append(initContainer.Env, transferEnvs...) - sidecarInitContainers = append(sidecarInitContainers, initContainer) - // insert volumes that initContainers used - for _, mount := range initContainer.VolumeMounts { - volumesInSidecars = append(volumesInSidecars, *volumesMap[mount.Name]) - } - } - //process imagePullSecrets - sidecarSecrets = append(sidecarSecrets, sidecarSet.Spec.ImagePullSecrets...) - } - - sidecarList := sets.NewString() - isInjecting := false - //process containers - for i := range sidecarSet.Spec.Containers { - sidecarContainer := &sidecarSet.Spec.Containers[i] - sidecarList.Insert(sidecarContainer.Name) - // volumeMounts that injected into sidecar container - // when volumeMounts SubPathExpr contains expansions, then need copy container EnvVars(injectEnvs) - injectedMounts, injectedEnvs := sidecarcontrol.GetInjectedVolumeMountsAndEnvs(control, sidecarContainer, pod) - // get injected env & mounts explicitly so that can be compared with old ones in pod - transferEnvs := sidecarcontrol.GetSidecarTransferEnvs(sidecarContainer, pod) - // append volumeMounts SubPathExpr environments - transferEnvs = util.MergeEnvVar(transferEnvs, injectedEnvs) - klog.Infof("try to inject sidecar %v@%v/%v, with injected envs: %v, volumeMounts: %v", - sidecarContainer.Name, pod.Namespace, pod.Name, transferEnvs, injectedMounts) - //when update pod object - if isUpdated { - // judge whether inject sidecar container into pod - needInject, existSidecars, existVolumes := control.NeedToInjectInUpdatedPod(pod, oldPod, sidecarContainer, transferEnvs, injectedMounts) - if !needInject { - sidecarContainers = append(sidecarContainers, existSidecars...) - volumesInSidecars = append(volumesInSidecars, existVolumes...) - continue - } - - klog.V(3).Infof("upgrade or insert sidecar container %v during upgrade in pod %v/%v", - sidecarContainer.Name, pod.Namespace, pod.Name) - //when created pod object, need inject sidecar container into pod - } else { - klog.V(3).Infof("inject new sidecar container %v during creation in pod %v/%v", - sidecarContainer.Name, pod.Namespace, pod.Name) - } - isInjecting = true - // insert volume that sidecar container used - for _, mount := range sidecarContainer.VolumeMounts { - volumesInSidecars = append(volumesInSidecars, *volumesMap[mount.Name]) - } - // merge VolumeMounts from sidecar.VolumeMounts and shared VolumeMounts - sidecarContainer.VolumeMounts = util.MergeVolumeMounts(sidecarContainer.VolumeMounts, injectedMounts) - // add the "Injected" env to the sidecar container - sidecarContainer.Env = append(sidecarContainer.Env, corev1.EnvVar{Name: sidecarcontrol.SidecarEnvKey, Value: "true"}) - // merged Env from sidecar.Env and transfer envs - sidecarContainer.Env = util.MergeEnvVar(sidecarContainer.Env, transferEnvs) - - // when sidecar container UpgradeStrategy is HotUpgrade - if sidecarcontrol.IsHotUpgradeContainer(sidecarContainer) { - hotContainers, annotations := injectHotUpgradeContainers(hotUpgradeWorkInfo, sidecarContainer) - sidecarContainers = append(sidecarContainers, hotContainers...) - for k, v := range annotations { - injectedAnnotations[k] = v - } - } else { - sidecarContainers = append(sidecarContainers, sidecarContainer) - } - } - // the container was (re)injected and the annotations need to be updated - if isInjecting { - setUpgrade1.SidecarList = sidecarList.List() - setUpgrade2.SidecarList = sidecarList.List() - sidecarSetHash[sidecarSet.Name] = setUpgrade1 - sidecarSetHashWithoutImage[sidecarSet.Name] = setUpgrade2 - } - } - - // store sidecarset hash in pod annotations - by, _ := json.Marshal(sidecarSetHash) - injectedAnnotations[sidecarcontrol.SidecarSetHashAnnotation] = string(by) - by, _ = json.Marshal(sidecarSetHashWithoutImage) - injectedAnnotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = string(by) - sidecarSetNameList := strings.Join(sidecarSetNames.List(), ",") - // store matched sidecarset list in pod annotations - injectedAnnotations[sidecarcontrol.SidecarSetListAnnotation] = sidecarSetNameList - return sidecarContainers, sidecarInitContainers, sidecarSecrets, volumesInSidecars, injectedAnnotations, nil -} - -func getVolumesMapInSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) map[string]*corev1.Volume { - volumesMap := make(map[string]*corev1.Volume) - for idx, volume := range sidecarSet.Spec.Volumes { - volumesMap[volume.Name] = &sidecarSet.Spec.Volumes[idx] + sidecarSets := make([]*appsv1alpha1.SidecarSet, 0) + for i := range sidecarSetList.Items { + sidecarSet := &sidecarSetList.Items[i] + sidecarSets = append(sidecarSets, sidecarSet) } - return volumesMap + return sidecarcontrol.SidecarSetMutatingPod(pod, oldPod, sidecarSets, h.sidecarSetControl) } diff --git a/pkg/webhook/pod/mutating/sidecarset_hotupgrade_test.go b/pkg/webhook/pod/mutating/sidecarset_hotupgrade_test.go deleted file mode 100644 index fc01d62bff..0000000000 --- a/pkg/webhook/pod/mutating/sidecarset_hotupgrade_test.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2020 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mutating - -import ( - "context" - "testing" - - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" - - admissionv1 "k8s.io/api/admission/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -func TestInjectHotUpgradeSidecar(t *testing.T) { - sidecarSetIn := sidecarSet1.DeepCopy() - sidecarSetIn.Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = "without-c4k2dbb95d" - sidecarSetIn.Spec.Containers[0].UpgradeStrategy.UpgradeType = appsv1alpha1.SidecarContainerHotUpgrade - sidecarSetIn.Spec.Containers[0].UpgradeStrategy.HotUpgradeEmptyImage = "busy:hotupgrade-empty" - testInjectHotUpgradeSidecar(t, sidecarSetIn) -} - -func testInjectHotUpgradeSidecar(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) { - podIn := pod1.DeepCopy() - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(sidecarSetIn).Build() - podOut := podIn.DeepCopy() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, err := podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - if err != nil { - t.Fatalf("inject sidecar into pod failed, err: %v", err) - } - if len(podOut.Spec.Containers) != 4 { - t.Fatalf("expect 4 containers but got %v", len(podOut.Spec.Containers)) - } - if podOut.Spec.Containers[0].Image != sidecarSetIn.Spec.Containers[0].Image { - t.Fatalf("expect image %v but got %v", sidecarSetIn.Spec.Containers[0].Image, podOut.Spec.Containers[0].Image) - } - if podOut.Spec.Containers[1].Image != sidecarSetIn.Spec.Containers[0].UpgradeStrategy.HotUpgradeEmptyImage { - t.Fatalf("expect image busy:hotupgrade-empty but got %v", podOut.Spec.Containers[1].Image) - } - if sidecarcontrol.GetPodSidecarSetRevision("sidecarset1", podOut) != sidecarcontrol.GetSidecarSetRevision(sidecarSetIn) { - t.Fatalf("pod sidecarset revision(%s) error", sidecarcontrol.GetPodSidecarSetRevision("sidecarset1", podOut)) - } - if sidecarcontrol.GetPodSidecarSetWithoutImageRevision("sidecarset1", podOut) != sidecarcontrol.GetSidecarSetWithoutImageRevision(sidecarSetIn) { - t.Fatalf("pod sidecarset without image revision(%s) error", sidecarcontrol.GetPodSidecarSetWithoutImageRevision("sidecarset1", podOut)) - } - if podOut.Annotations[sidecarcontrol.SidecarSetListAnnotation] != "sidecarset1" { - t.Fatalf("pod annotations[%s]=%s error", sidecarcontrol.SidecarSetListAnnotation, podOut.Annotations[sidecarcontrol.SidecarSetListAnnotation]) - } - if sidecarcontrol.GetPodHotUpgradeInfoInAnnotations(podOut)["dns-f"] != "dns-f-1" { - t.Fatalf("pod annotations[%s]=%s error", sidecarcontrol.SidecarSetWorkingHotUpgradeContainer, podOut.Annotations[sidecarcontrol.SidecarSetWorkingHotUpgradeContainer]) - } - if podOut.Annotations[sidecarcontrol.GetPodSidecarSetVersionAnnotation("dns-f-1")] != "1" { - t.Fatalf("pod annotations dns-f-1 version=%s", podOut.Annotations[sidecarcontrol.GetPodSidecarSetVersionAnnotation("dns-f-1")]) - } - if podOut.Annotations[sidecarcontrol.GetPodSidecarSetVersionAnnotation("dns-f-2")] != "0" { - t.Fatalf("pod annotations dns-f-2 version=%s", podOut.Annotations[sidecarcontrol.GetPodSidecarSetVersionAnnotation("dns-f-2")]) - } -} diff --git a/pkg/webhook/pod/mutating/sidecarset_test.go b/pkg/webhook/pod/mutating/sidecarset_test.go deleted file mode 100644 index 7fe7451672..0000000000 --- a/pkg/webhook/pod/mutating/sidecarset_test.go +++ /dev/null @@ -1,1053 +0,0 @@ -/* -Copyright 2020 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mutating - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/openkruise/kruise/apis" - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" - "github.com/openkruise/kruise/pkg/util" - webhookutil "github.com/openkruise/kruise/pkg/webhook/util" - - admissionv1 "k8s.io/api/admission/v1" - apps "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -const ( - defaultNs = "default" -) - -func TestMain(m *testing.M) { - t := &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, - } - apis.AddToScheme(scheme.Scheme) - - code := m.Run() - t.Stop() - os.Exit(code) -} - -var ( - sidecarSet1 = &appsv1alpha1.SidecarSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sidecarset1", - Annotations: map[string]string{ - sidecarcontrol.SidecarSetHashAnnotation: "c4k2dbb95d", - }, - Labels: map[string]string{}, - }, - Spec: appsv1alpha1.SidecarSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "suxing-test", - }, - }, - InitContainers: []appsv1alpha1.SidecarContainer{ - { - Container: corev1.Container{ - Name: "init-2", - Image: "busybox:1.0.0", - }, - }, - { - Container: corev1.Container{ - Name: "init-1", - Image: "busybox:1.0.0", - }, - }, - }, - Containers: []appsv1alpha1.SidecarContainer{ - { - Container: corev1.Container{ - Name: "dns-f", - Image: "dns-f-image:1.0", - }, - PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, - ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ - Type: appsv1alpha1.ShareVolumePolicyDisabled, - }, - }, - { - Container: corev1.Container{ - Name: "log-agent", - Image: "log-agent-image:1.0", - }, - PodInjectPolicy: appsv1alpha1.AfterAppContainerType, - ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ - Type: appsv1alpha1.ShareVolumePolicyDisabled, - }, - }, - }, - }, - } - - sidecarsetWithTransferEnv = &appsv1alpha1.SidecarSet{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - sidecarcontrol.SidecarSetHashAnnotation: "c4k2dbb95d", - }, - Name: "sidecarset2", - Labels: map[string]string{}, - }, - Spec: appsv1alpha1.SidecarSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "suxing-test", - }, - }, - InitContainers: []appsv1alpha1.SidecarContainer{ - { - Container: corev1.Container{ - Name: "dns-e", - Image: "dns-e-image:1.0", - VolumeMounts: []corev1.VolumeMount{ - {Name: "volume-1"}, - }, - }, - PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, - TransferEnv: []appsv1alpha1.TransferEnvVar{ - { - SourceContainerName: "nginx", - EnvName: "hello2", - }, - }, - }, - }, - Containers: []appsv1alpha1.SidecarContainer{ - { - Container: corev1.Container{ - Name: "dns-f", - Image: "dns-f-image:1.0", - VolumeMounts: []corev1.VolumeMount{ - {Name: "volume-1"}, - }, - }, - PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, - ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ - Type: appsv1alpha1.ShareVolumePolicyDisabled, - }, - TransferEnv: []appsv1alpha1.TransferEnvVar{ - { - SourceContainerName: "nginx", - EnvName: "hello2", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - {Name: "volume-1"}, - {Name: "volume-2"}, - }, - }, - } - - sidecarSet3 = &appsv1alpha1.SidecarSet{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - sidecarcontrol.SidecarSetHashAnnotation: "gm967682cm", - }, - Name: "sidecarset3", - Labels: map[string]string{}, - }, - Spec: appsv1alpha1.SidecarSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "suxing-test", - }, - }, - Containers: []appsv1alpha1.SidecarContainer{ - { - Container: corev1.Container{ - Name: "dns-f", - Image: "dns-f-image:1.0", - VolumeMounts: []corev1.VolumeMount{ - { - Name: "volume-1", - MountPath: "/a/b/c", - }, - { - Name: "volume-2", - MountPath: "/d/e/f", - }, - }, - }, - PodInjectPolicy: appsv1alpha1.AfterAppContainerType, - ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ - Type: appsv1alpha1.ShareVolumePolicyEnabled, - }, - }, - { - Container: corev1.Container{ - Name: "log-agent", - Image: "log-agent-image:1.0", - }, - PodInjectPolicy: appsv1alpha1.AfterAppContainerType, - ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ - Type: appsv1alpha1.ShareVolumePolicyDisabled, - }, - }, - }, - Volumes: []corev1.Volume{ - {Name: "volume-1"}, - {Name: "volume-2"}, - }, - }, - } - - sidecarSetWithStaragent = &appsv1alpha1.SidecarSet{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - sidecarcontrol.SidecarSetHashAnnotation: "gm967682cm", - }, - Name: "sidecarset3", - Labels: map[string]string{}, - }, - Spec: appsv1alpha1.SidecarSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "suxing-test", - }, - }, - Containers: []appsv1alpha1.SidecarContainer{ - { - Container: corev1.Container{ - Name: "dns-f", - Image: "dns-f-image:1.0", - VolumeMounts: []corev1.VolumeMount{ - { - Name: "volume-1", - MountPath: "/a/b/c", - }, - { - Name: "volume-2", - MountPath: "/d/e/f", - }, - { - Name: "volume-staragent", - MountPath: "/staragent", - }, - }, - }, - PodInjectPolicy: appsv1alpha1.AfterAppContainerType, - ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ - Type: appsv1alpha1.ShareVolumePolicyEnabled, - }, - }, - { - Container: corev1.Container{ - Name: "staragent", - Image: "staragent-image:1.0", - }, - PodInjectPolicy: appsv1alpha1.AfterAppContainerType, - ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ - Type: appsv1alpha1.ShareVolumePolicyEnabled, - }, - }, - { - Container: corev1.Container{ - Name: "log-agent", - Image: "log-agent-image:1.0", - }, - PodInjectPolicy: appsv1alpha1.AfterAppContainerType, - ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ - Type: appsv1alpha1.ShareVolumePolicyDisabled, - }, - }, - }, - Volumes: []corev1.Volume{ - {Name: "volume-1"}, - {Name: "volume-2"}, - {Name: "volume-staragent"}, - }, - }, - } - - pod1 = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: defaultNs, - Labels: map[string]string{"app": "suxing-test"}, - }, - Spec: corev1.PodSpec{ - InitContainers: []corev1.Container{ - { - Name: "init-0", - Image: "busybox:1.0.0", - }, - }, - Containers: []corev1.Container{ - { - Name: "nginx", - Image: "nginx:1.15.1", - Env: []corev1.EnvVar{ - { - Name: "hello1", - Value: "world1", - }, - { - Name: "hello2", - Value: "world2", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "volume-a", - MountPath: "/a/b", - }, - { - Name: "volume-b", - MountPath: "/e/f", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - {Name: "volume-a"}, - {Name: "volume-b"}, - }, - }, - } - - podWithStaragent = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: defaultNs, - Labels: map[string]string{"app": "suxing-test"}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "nginx", - Image: "nginx:1.15.1", - Env: []corev1.EnvVar{ - { - Name: "hello1", - Value: "world1", - }, - { - Name: "hello2", - Value: "world2", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "volume-a", - MountPath: "/a/b", - }, - { - Name: "volume-b", - MountPath: "/e/f", - }, - { - Name: "volume-staragent", - MountPath: "/staragent", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - {Name: "volume-a"}, - {Name: "volume-b"}, - {Name: "volume-staragent"}, - }, - }, - } -) - -func TestPodHasNoMatchedSidecarSet(t *testing.T) { - sidecarSetIn := sidecarSet1.DeepCopy() - testPodHasNoMatchedSidecarSet(t, sidecarSetIn) -} - -func testPodHasNoMatchedSidecarSet(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) { - podIn := pod1.DeepCopy() - podIn.Labels["app"] = "doesnt-match" - podOut := podIn.DeepCopy() - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(sidecarSetIn).Build() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, _ = podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - - if len(podOut.Spec.Containers) != len(podIn.Spec.Containers) { - t.Fatalf("expect %v containers but got %v", len(podIn.Spec.Containers), len(podOut.Spec.Containers)) - } -} - -func TestMergeSidecarSecrets(t *testing.T) { - sidecarSetIn := sidecarSet1.DeepCopy() - testMergeSidecarSecrets(t, sidecarSetIn) -} - -func testMergeSidecarSecrets(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) { - sidecarImagePullSecrets := []corev1.LocalObjectReference{ - {Name: "s0"}, {Name: "s1"}, {Name: "s2"}, - } - podImagePullSecrets := []corev1.LocalObjectReference{ - {Name: "s2"}, {Name: "s3"}, {Name: "s3"}, - } - // 3 + 3 - 2 = 4 - sidecarSetIn.Spec.ImagePullSecrets = sidecarImagePullSecrets - doMergeSidecarSecretsTest(t, sidecarSetIn, podImagePullSecrets, 2) - // 3 + 3 - 0 = 6 - podImagePullSecrets = []corev1.LocalObjectReference{ - {Name: "s3"}, {Name: "s4"}, {Name: "s5"}, - } - doMergeSidecarSecretsTest(t, sidecarSetIn, podImagePullSecrets, 0) - // 3 + 0 - 0 = 3 - doMergeSidecarSecretsTest(t, sidecarSetIn, nil, 0) - // 0 + 3 - 0 = 3 - sidecarSetIn.Spec.ImagePullSecrets = nil - doMergeSidecarSecretsTest(t, sidecarSetIn, podImagePullSecrets, 0) -} - -func doMergeSidecarSecretsTest(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet, podImagePullSecrets []corev1.LocalObjectReference, repeat int) { - podIn := pod1.DeepCopy() - podIn.Spec.ImagePullSecrets = podImagePullSecrets - podOut := podIn.DeepCopy() - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(sidecarSetIn).Build() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, _ = podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - - if len(podOut.Spec.ImagePullSecrets) != len(podIn.Spec.ImagePullSecrets)+len(sidecarSetIn.Spec.ImagePullSecrets)-repeat { - t.Fatalf("expect %v secrets but got %v", len(podIn.Spec.ImagePullSecrets)+len(sidecarSetIn.Spec.ImagePullSecrets)-repeat, len(podOut.Spec.ImagePullSecrets)) - } -} - -func TestInjectionStrategyPaused(t *testing.T) { - sidecarSetIn := sidecarSet1.DeepCopy() - testInjectionStrategyPaused(t, sidecarSetIn) -} - -func testInjectionStrategyPaused(t *testing.T, sidecarIn *appsv1alpha1.SidecarSet) { - podIn := pod1.DeepCopy() - podOut := podIn.DeepCopy() - sidecarPaused := sidecarIn - sidecarPaused.Spec.InjectionStrategy.Paused = true - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(sidecarPaused).Build() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, _ = podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - - if len(podOut.Spec.Containers) != len(podIn.Spec.Containers) { - t.Fatalf("expect %v containers but got %v", len(podIn.Spec.Containers), len(podOut.Spec.Containers)) - } -} - -func TestInjectMetadata(t *testing.T) { - podIn := pod1.DeepCopy() - demo1 := sidecarSet1.DeepCopy() - demo1.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - PatchPolicy: appsv1alpha1.SidecarSetMergePatchJsonPatchPolicy, - Annotations: map[string]string{ - "key1": `{"log-agent":1}`, - }, - }, - { - PatchPolicy: appsv1alpha1.SidecarSetRetainPatchPolicy, - Annotations: map[string]string{ - "key": "envoy=1,log=2", - }, - }, - } - demo2 := sidecarSet1.DeepCopy() - demo2.Name = "sidecarset2" - demo2.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ - { - PatchPolicy: appsv1alpha1.SidecarSetMergePatchJsonPatchPolicy, - Annotations: map[string]string{ - "key1": `{"envoy":2}`, - }, - }, - } - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(demo1, demo2).Build() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - podHandler.sidecarsetMutatingPod(context.Background(), req, podIn) - expect := map[string]string{ - "key1": `{"envoy":2,"log-agent":1}`, - "key": "envoy=1,log=2", - } - if expect["key1"] != podIn.Annotations["key1"] || expect["key"] != podIn.Annotations["key"] { - t.Fatalf("sidecarSet inject annotations failed, expect %v, but get %v", expect, podIn.Annotations) - } -} - -func TestInjectionStrategyRevision(t *testing.T) { - spec := map[string]interface{}{ - "spec": map[string]interface{}{ - "initContainers": []appsv1alpha1.SidecarContainer{ - { - Container: corev1.Container{ - Name: "init-2", - Image: "busybox:1.0.0", - }, - }, - }, - "containers": []appsv1alpha1.SidecarContainer{ - { - Container: corev1.Container{ - Name: "dns-f", - Image: "dns-f-image:1.0", - }, - PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, - ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{ - Type: appsv1alpha1.ShareVolumePolicyDisabled, - }, - }, - }, - }, - } - - raw, _ := json.Marshal(spec) - revisionID := fmt.Sprintf("%s-12345", sidecarSet1.Name) - sidecarSetIn := sidecarSet1.DeepCopy() - sidecarSetIn.Spec.InjectionStrategy.Revision = &appsv1alpha1.SidecarSetInjectRevision{ - CustomVersion: &revisionID, - Policy: appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy, - } - historyInjection := []client.Object{ - sidecarSetIn, - &apps.ControllerRevision{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: webhookutil.GetNamespace(), - Name: revisionID, - Labels: map[string]string{ - appsv1alpha1.SidecarSetCustomVersionLabel: revisionID, - }, - }, - Data: runtime.RawExtension{ - Raw: raw, - }, - }, - } - testInjectionStrategyRevision(t, historyInjection) -} - -func testInjectionStrategyRevision(t *testing.T, env []client.Object) { - podIn := pod1.DeepCopy() - podOut := podIn.DeepCopy() - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(env...).Build() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, err := podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - if err != nil { - t.Fatalf("failed to mutating pod, err: %v", err) - } - - if len(podIn.Spec.Containers)+len(podIn.Spec.InitContainers)+2 != len(podOut.Spec.Containers)+len(podOut.Spec.InitContainers) { - t.Fatalf("expect %v containers but got %v", len(podIn.Spec.Containers)+2, len(podOut.Spec.Containers)) - } -} - -func TestSidecarSetPodInjectPolicy(t *testing.T) { - sidecarSetIn := sidecarSet1.DeepCopy() - testSidecarSetPodInjectPolicy(t, sidecarSetIn) -} - -func testSidecarSetPodInjectPolicy(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) { - podIn := pod1.DeepCopy() - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(sidecarSetIn).Build() - podOut := podIn.DeepCopy() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, err := podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - if err != nil { - t.Fatalf("inject sidecar into pod failed, err: %v", err) - } - - expectLen := len(podIn.Spec.Containers) + len(sidecarSetIn.Spec.Containers) - if len(podOut.Spec.Containers) != expectLen { - t.Fatalf("expect %v containers but got %v", expectLen, len(podOut.Spec.Containers)) - } - - for i, container := range podOut.Spec.Containers { - switch i { - case 0: - if container.Name != "dns-f" { - t.Fatalf("expect dns-f but got %v", container.Name) - } - case 1: - if container.Name != "nginx" { - t.Fatalf("expect nginx but got %v", container.Name) - } - case 2: - if container.Name != "log-agent" { - t.Fatalf("expect log-agent but got %v", container.Name) - } - } - } - - expectInitLen := len(podIn.Spec.InitContainers) + len(sidecarSetIn.Spec.InitContainers) - if len(podOut.Spec.InitContainers) != expectInitLen { - t.Fatalf("expect %v initContainers but got %v", expectLen, len(podOut.Spec.InitContainers)) - } - - for i, container := range podOut.Spec.InitContainers { - //injected container must be contain env "IS_INJECTED" - if i > 0 { - exist := false - for _, env := range container.Env { - if env.Name == sidecarcontrol.SidecarEnvKey { - exist = true - break - } - } - if !exist { - t.Fatalf("Injected initContainer %v don't contain env(%v)", container.Name, sidecarcontrol.SidecarEnvKey) - } - } - - switch i { - case 0: - if container.Name != "init-0" { - t.Fatalf("expect dns-f but got %v", container.Name) - } - case 1: - if container.Name != "init-1" { - t.Fatalf("expect nginx but got %v", container.Name) - } - case 2: - if container.Name != "init-2" { - t.Fatalf("expect log-agent but got %v", container.Name) - } - } - } -} - -func TestSidecarVolumesAppend(t *testing.T) { - sidecarSetIn := sidecarsetWithTransferEnv.DeepCopy() - testSidecarVolumesAppend(t, sidecarSetIn) -} - -func testSidecarVolumesAppend(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) { - podIn := pod1.DeepCopy() - - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(sidecarSetIn).Build() - podOut := podIn.DeepCopy() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, err := podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - if err != nil { - t.Fatalf("inject sidecar into pod failed, err: %v", err) - } - - expectLen := len(podIn.Spec.Volumes) + 1 - if len(podOut.Spec.Volumes) != expectLen { - t.Fatalf("expect %v volumes but got %v", expectLen, len(podOut.Spec.Volumes)) - } - - for i, volume := range podOut.Spec.Volumes { - switch i { - case 0: - if volume.Name != "volume-a" { - t.Fatalf("expect volume-a but got %v", volume.Name) - } - case 1: - if volume.Name != "volume-b" { - t.Fatalf("expect volume-b but got %v", volume.Name) - } - case 2: - if volume.Name != "volume-1" { - t.Fatalf("expect volume-1 but got %v", volume.Name) - } - } - } -} - -func TestPodSidecarSetHashCompatibility(t *testing.T) { - podIn := pod1.DeepCopy() - podIn.Annotations = map[string]string{} - podIn.Annotations[sidecarcontrol.SidecarSetHashAnnotation] = `{"sidecarset-test":"bv6d2fbw97wz8xx5x4v4wddwbd5z744wcf7c786dd4dvxvd5w6w424df7vx47989"}` - podIn.Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = `{"sidecarset-test":"54x5977vf9zz4248w7v44456zf655b8bcffv7x74w88f6dwb994fw48b8f9b8959"}` - _, _, _, _, annotations, err := buildSidecars(false, podIn, nil, nil) - if err != nil { - t.Fatalf("compatible pod sidecarSet Hash failed: %s", err.Error()) - } - // format: sidecarset.name -> sidecarset hash - sidecarSetHash := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec) - // format: sidecarset.name -> sidecarset hash(without image) - sidecarSetHashWithoutImage := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec) - // parse sidecar hash in pod annotations - if oldHashStr := annotations[sidecarcontrol.SidecarSetHashAnnotation]; len(oldHashStr) > 0 { - if err := json.Unmarshal([]byte(oldHashStr), &sidecarSetHash); err != nil { - t.Fatalf("compatible pod sidecarSet Hash failed: %s", err.Error()) - } - } - if oldHashStr := annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation]; len(oldHashStr) > 0 { - if err := json.Unmarshal([]byte(oldHashStr), &sidecarSetHashWithoutImage); err != nil { - t.Fatalf("compatible pod sidecarSet Hash failed: %s", err.Error()) - } - } -} - -func TestPodVolumeMountsAppend(t *testing.T) { - sidecarSetIn := sidecarSetWithStaragent.DeepCopy() - // /a/b/c, /d/e/f, /staragent - sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1] - testPodVolumeMountsAppend(t, sidecarSetIn) -} - -func testPodVolumeMountsAppend(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) { - // /a/b、/e/f - podIn := podWithStaragent.DeepCopy() - cases := []struct { - name string - getPod func() *corev1.Pod - getSidecarSets func() *appsv1alpha1.SidecarSet - exceptVolumeMounts []string - exceptEnvs []string - }{ - { - name: "append normal volumeMounts", - getPod: func() *corev1.Pod { - return podIn.DeepCopy() - }, - getSidecarSets: func() *appsv1alpha1.SidecarSet { - return sidecarSetIn.DeepCopy() - }, - exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent"}, - }, - { - name: "append volumeMounts SubPathExpr, volumes with expanded subpath", - getPod: func() *corev1.Pod { - podOut := podIn.DeepCopy() - podOut.Spec.Containers[0].VolumeMounts = append(podOut.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "volume-expansion", - MountPath: "/e/expansion", - SubPathExpr: "foo/$(POD_NAME)/$(OD_NAME)/conf", - }) - podOut.Spec.Containers[0].Env = append(podOut.Spec.Containers[0].Env, corev1.EnvVar{ - Name: "POD_NAME", - Value: "bar", - }) - podOut.Spec.Containers[0].Env = append(podOut.Spec.Containers[0].Env, corev1.EnvVar{ - Name: "OD_NAME", - Value: "od_name", - }) - return podOut - }, - getSidecarSets: func() *appsv1alpha1.SidecarSet { - return sidecarSetIn.DeepCopy() - }, - exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"}, - exceptEnvs: []string{"POD_NAME", "OD_NAME"}, - }, - { - name: "append volumeMounts SubPathExpr, subpath with no expansion", - getPod: func() *corev1.Pod { - podOut := podIn.DeepCopy() - podOut.Spec.Containers[0].VolumeMounts = append(podOut.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "volume-expansion", - MountPath: "/e/expansion", - SubPathExpr: "foo", - }) - return podOut - }, - getSidecarSets: func() *appsv1alpha1.SidecarSet { - return sidecarSetIn.DeepCopy() - }, - exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"}, - }, - { - name: "append volumeMounts SubPathExpr, volumes expanded with empty subpath", - getPod: func() *corev1.Pod { - podOut := podIn.DeepCopy() - podOut.Spec.Containers[0].VolumeMounts = append(podOut.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "volume-expansion", - MountPath: "/e/expansion", - SubPathExpr: "", - }) - return podOut - }, - getSidecarSets: func() *appsv1alpha1.SidecarSet { - return sidecarSetIn.DeepCopy() - }, - exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"}, - }, - } - - for _, cs := range cases { - t.Run(cs.name, func(t *testing.T) { - podIn := cs.getPod() - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(cs.getSidecarSets()).Build() - podOut := podIn.DeepCopy() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, err := podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - if err != nil { - t.Fatalf("inject sidecar into pod failed, err: %v", err) - } - - for _, mount := range cs.exceptVolumeMounts { - if util.GetContainerVolumeMount(&podOut.Spec.Containers[1], mount) == nil { - t.Fatalf("expect volume mounts %s but got nil", mount) - } - } - - for _, env := range cs.exceptEnvs { - if util.GetContainerEnvVar(&podOut.Spec.Containers[1], env) == nil { - t.Fatalf("expect env %s but got nil", env) - } - } - }) - } -} - -func TestSidecarSetTransferEnv(t *testing.T) { - sidecarSetIn := sidecarsetWithTransferEnv.DeepCopy() - testSidecarSetTransferEnv(t, sidecarSetIn) -} - -func testSidecarSetTransferEnv(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) { - podIn := pod1.DeepCopy() - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(sidecarSetIn).Build() - podOut := podIn.DeepCopy() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, err := podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - if err != nil { - t.Fatalf("inject sidecar into pod failed, err: %v", err) - } - if len(podOut.Spec.InitContainers[1].Env) != 2 { - t.Fatalf("expect 2 envs but got %v", len(podOut.Spec.InitContainers[0].Env)) - } - if podOut.Spec.InitContainers[1].Env[1].Value != "world2" { - t.Fatalf("expect env with value 'world2' but got %v", podOut.Spec.Containers[0].Env[1].Value) - } - if len(podOut.Spec.Containers[0].Env) != 2 { - t.Fatalf("expect 2 envs but got %v", len(podOut.Spec.Containers[0].Env)) - } - if podOut.Spec.Containers[0].Env[1].Value != "world2" { - t.Fatalf("expect env with value 'world2' but got %v", podOut.Spec.Containers[0].Env[1].Value) - } -} - -func TestSidecarSetHashInject(t *testing.T) { - sidecarSetIn1 := sidecarSet1.DeepCopy() - testSidecarSetHashInject(t, sidecarSetIn1) -} - -func testSidecarSetHashInject(t *testing.T, sidecarSetIn1 *appsv1alpha1.SidecarSet) { - podIn := pod1.DeepCopy() - sidecarSetIn1.Spec.Selector.MatchLabels["app"] = "doesnt-match" - sidecarSetIn2 := sidecarsetWithTransferEnv.DeepCopy() - sidecarSetIn3 := sidecarSet3.DeepCopy() - - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(sidecarSetIn1, sidecarSetIn2, sidecarSetIn3).Build() - podOut := podIn.DeepCopy() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, err := podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - if err != nil { - t.Fatalf("inject sidecar into pod failed, err: %v", err) - } - - hashKey := sidecarcontrol.SidecarSetHashAnnotation - //expectedAnnotation := `{"sidecarset2":"c4k2dbb95d","sidecarset3":"gm967682cm"}` - expectedRevision := map[string]string{ - "sidecarset2": "c4k2dbb95d", - "sidecarset3": "gm967682cm", - } - for k, v := range expectedRevision { - if sidecarcontrol.GetPodSidecarSetRevision(k, podOut) != v { - t.Errorf("except sidecarset(%s:%s), but get in pod annotations(%s)", k, v, podOut.Annotations[hashKey]) - } - } -} - -func TestSidecarSetNameInject(t *testing.T) { - sidecarSetIn1 := sidecarSet1.DeepCopy() - sidecarSetIn3 := sidecarSet3.DeepCopy() - testSidecarSetNameInject(t, sidecarSetIn1, sidecarSetIn3) -} - -func testSidecarSetNameInject(t *testing.T, sidecarSetIn1, sidecarSetIn3 *appsv1alpha1.SidecarSet) { - podIn := pod1.DeepCopy() - decoder, _ := admission.NewDecoder(scheme.Scheme) - client := fake.NewClientBuilder().WithObjects(sidecarSetIn1, sidecarSetIn3).Build() - podOut := podIn.DeepCopy() - podHandler := &PodCreateHandler{Decoder: decoder, Client: client} - req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "") - _, err := podHandler.sidecarsetMutatingPod(context.Background(), req, podOut) - if err != nil { - t.Fatalf("inject sidecar into pod failed, err: %v", err) - } - sidecarSetListKey := sidecarcontrol.SidecarSetListAnnotation - expectedAnnotation := "sidecarset1,sidecarset3" - if podOut.Annotations[sidecarSetListKey] != expectedAnnotation { - t.Errorf("expect annotation %v but got %v", expectedAnnotation, podOut.Annotations[sidecarSetListKey]) - } -} - -func TestMergeSidecarContainers(t *testing.T) { - podContainers := []corev1.Container{ - { - Name: "sidecar-1", - }, - { - Name: "app-container", - }, - { - Name: "sidecar-2", - }, - } - - sidecarContainers := []*appsv1alpha1.SidecarContainer{ - { - Container: corev1.Container{ - Name: "sidecar-1", - }, - PodInjectPolicy: appsv1alpha1.AfterAppContainerType, - }, - { - Container: corev1.Container{ - Name: "sidecar-2", - }, - PodInjectPolicy: appsv1alpha1.AfterAppContainerType, - }, - { - Container: corev1.Container{ - Name: "new-sidecar-1", - }, - PodInjectPolicy: appsv1alpha1.BeforeAppContainerType, - }, - } - - cases := []struct { - name string - getOrigins func() []corev1.Container - getInjected func() []*appsv1alpha1.SidecarContainer - expectContainerLen int - expectedContainers []string - }{ - { - name: "origins not sidecar, and inject new sidecar", - getOrigins: func() []corev1.Container { - return podContainers[1:2] - }, - getInjected: func() []*appsv1alpha1.SidecarContainer { - return sidecarContainers - }, - expectContainerLen: 4, - expectedContainers: []string{"new-sidecar-1", "app-container", "sidecar-1", "sidecar-2"}, - }, - { - name: "origins not sidecar, and inject new sidecar, only before app container", - getOrigins: func() []corev1.Container { - return podContainers[1:2] - }, - getInjected: func() []*appsv1alpha1.SidecarContainer { - return sidecarContainers[2:] - }, - expectContainerLen: 2, - expectedContainers: []string{"new-sidecar-1", "app-container"}, - }, - { - name: "origins not sidecar, and inject new sidecar, only after app container", - getOrigins: func() []corev1.Container { - return podContainers[1:2] - }, - getInjected: func() []*appsv1alpha1.SidecarContainer { - return sidecarContainers[:2] - }, - expectContainerLen: 3, - expectedContainers: []string{"app-container", "sidecar-1", "sidecar-2"}, - }, - { - name: "origin have sidecars, sidecar no new containers", - getOrigins: func() []corev1.Container { - return podContainers - }, - getInjected: func() []*appsv1alpha1.SidecarContainer { - return sidecarContainers[:2] - }, - expectContainerLen: 3, - expectedContainers: []string{"sidecar-1", "app-container", "sidecar-2"}, - }, - { - name: "origin have sidecars, sidecar have new containers", - getOrigins: func() []corev1.Container { - return podContainers - }, - getInjected: func() []*appsv1alpha1.SidecarContainer { - return sidecarContainers - }, - expectContainerLen: 4, - expectedContainers: []string{"new-sidecar-1", "sidecar-1", "app-container", "sidecar-2"}, - }, - } - - for _, cs := range cases { - t.Run(cs.name, func(t *testing.T) { - origins := cs.getOrigins() - injected := cs.getInjected() - finals := mergeSidecarContainers(origins, injected) - if len(finals) != cs.expectContainerLen { - t.Fatalf("expect %d containers but got %v", cs.expectContainerLen, len(finals)) - } - for index, cName := range cs.expectedContainers { - if finals[index].Name != cName { - t.Fatalf("expect index(%d) container(%s) but got %s", index, cName, finals[index].Name) - } - } - }) - } -} - -func newAdmission(op admissionv1.Operation, object, oldObject runtime.RawExtension, subResource string) admission.Request { - return admission.Request{ - AdmissionRequest: newAdmissionRequest(op, object, oldObject, subResource), - } -} - -func newAdmissionRequest(op admissionv1.Operation, object, oldObject runtime.RawExtension, subResource string) admissionv1.AdmissionRequest { - return admissionv1.AdmissionRequest{ - Resource: metav1.GroupVersionResource{Group: corev1.SchemeGroupVersion.Group, Version: corev1.SchemeGroupVersion.Version, Resource: "pods"}, - Operation: op, - Object: object, - OldObject: oldObject, - SubResource: subResource, - } -} diff --git a/pkg/webhook/pod/validating/pod_unavailable_budget_test.go b/pkg/webhook/pod/validating/pod_unavailable_budget_test.go index a502e0aad7..6469491df1 100644 --- a/pkg/webhook/pod/validating/pod_unavailable_budget_test.go +++ b/pkg/webhook/pod/validating/pod_unavailable_budget_test.go @@ -25,8 +25,8 @@ import ( appspub "github.com/openkruise/kruise/apis/apps/pub" policyv1alpha1 "github.com/openkruise/kruise/apis/policy/v1alpha1" "github.com/openkruise/kruise/pkg/control/pubcontrol" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" + "github.com/openkruise/utils/sidecarcontrol" admissionv1 "k8s.io/api/admission/v1" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go b/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go index 8f2ef3ccb0..3af8389386 100644 --- a/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go +++ b/pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go @@ -24,8 +24,8 @@ import ( "github.com/openkruise/kruise/apis/apps/defaults" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" + "github.com/openkruise/utils/sidecarcontrol" admissionv1 "k8s.io/api/admission/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" diff --git a/pkg/webhook/sidecarset/mutating/sidecarset_mutating_test.go b/pkg/webhook/sidecarset/mutating/sidecarset_mutating_test.go index 8905c0e0db..50b2ca7456 100644 --- a/pkg/webhook/sidecarset/mutating/sidecarset_mutating_test.go +++ b/pkg/webhook/sidecarset/mutating/sidecarset_mutating_test.go @@ -5,7 +5,7 @@ import ( "github.com/openkruise/kruise/apis/apps/defaults" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" + "github.com/openkruise/utils/sidecarcontrol" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go b/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go index 01811c0e6b..0dd054d0fa 100644 --- a/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go +++ b/pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go @@ -24,10 +24,19 @@ import ( "regexp" "strings" + "github.com/openkruise/kruise/pkg/features" + "github.com/openkruise/kruise/pkg/util/configuration" + utilfeature "github.com/openkruise/kruise/pkg/util/feature" + "k8s.io/apimachinery/pkg/labels" + + k8scache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" webhookutil "github.com/openkruise/kruise/pkg/webhook/util" + "github.com/openkruise/utils/sidecarcontrol" + apps "k8s.io/api/apps/v1" admissionv1 "k8s.io/api/admission/v1" v1 "k8s.io/api/core/v1" @@ -71,7 +80,8 @@ type SidecarSetCreateUpdateHandler struct { Client client.Client // Decoder decodes objects - Decoder *admission.Decoder + Decoder *admission.Decoder + historyControl sidecarcontrol.HistoryControl } func (h *SidecarSetCreateUpdateHandler) validatingSidecarSetFn(ctx context.Context, obj *appsv1alpha1.SidecarSet, older *appsv1alpha1.SidecarSet) (bool, string, error) { @@ -136,7 +146,7 @@ func (h *SidecarSetCreateUpdateHandler) validateSidecarSetSpec(obj *appsv1alpha1 } // validating metadata annotationKeys := sets.NewString() - if err := sidecarcontrol.ValidateSidecarSetPatchMetadataWhitelist(h.Client, obj); err != nil { + if err := h.validateSidecarSetPatchMetadataWhitelist(obj); err != nil { allErrs = append(allErrs, field.Required(fldPath.Child("patchPodMetadata"), err.Error())) } for _, patch := range spec.PatchPodMetadata { @@ -161,6 +171,56 @@ func (h *SidecarSetCreateUpdateHandler) validateSidecarSetSpec(obj *appsv1alpha1 return allErrs } +func (h *SidecarSetCreateUpdateHandler) validateSidecarSetPatchMetadataWhitelist(sidecarSet *appsv1alpha1.SidecarSet) error { + if len(sidecarSet.Spec.PatchPodMetadata) == 0 { + return nil + } + + regAnnotations := make([]*regexp.Regexp, 0) + whitelist, err := configuration.GetSidecarSetPatchMetadataWhiteList(h.Client) + if err != nil { + return err + } else if whitelist == nil { + if utilfeature.DefaultFeatureGate.Enabled(features.SidecarSetPatchPodMetadataDefaultsAllowed) { + return nil + } + return fmt.Errorf("SidecarSet patch metadata whitelist not found") + } + + for _, rule := range whitelist.Rules { + if rule.Selector != nil { + selector, err := util.ValidatedLabelSelectorAsSelector(rule.Selector) + if err != nil { + return err + } + if !selector.Matches(labels.Set(sidecarSet.Labels)) { + continue + } + } + for _, key := range rule.AllowedAnnotationKeyExprs { + reg, err := regexp.Compile(key) + if err != nil { + return err + } + regAnnotations = append(regAnnotations, reg) + } + } + if len(regAnnotations) == 0 { + if utilfeature.DefaultFeatureGate.Enabled(features.SidecarSetPatchPodMetadataDefaultsAllowed) { + return nil + } + return fmt.Errorf("sidecarSet patch metadata annotation is not allowed") + } + for _, patch := range sidecarSet.Spec.PatchPodMetadata { + for key := range patch.Annotations { + if !matchRegKey(key, regAnnotations) { + return fmt.Errorf("sidecarSet patch metadata annotation(%s) is not allowed", key) + } + } + } + return nil +} + func validateSelector(selector *metav1.LabelSelector, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, metavalidation.ValidateLabelSelector(selector, fldPath)...) @@ -183,7 +243,7 @@ func (h *SidecarSetCreateUpdateHandler) validateSidecarSetInjectionStrategy(obj case revisionInfo.RevisionName == nil && revisionInfo.CustomVersion == nil: errList = append(errList, field.Invalid(field.NewPath("revision"), revisionInfo, "revisionName and customVersion cannot be empty simultaneously")) default: - revision, err := sidecarcontrol.NewHistoryControl(h.Client).GetHistorySidecarSet(obj, revisionInfo) + revision, err := h.historyControl.GetHistorySidecarSet(obj, revisionInfo) if err != nil || revision == nil { errList = append(errList, field.Invalid(field.NewPath("revision"), revision, fmt.Sprintf("Cannot find specific ControllerRevision, err: %v", err))) } @@ -498,3 +558,15 @@ func (h *SidecarSetCreateUpdateHandler) InjectDecoder(d *admission.Decoder) erro h.Decoder = d return nil } + +var _ inject.Cache = &SidecarSetCreateUpdateHandler{} + +// InjectCache injects the decoder into the PodCreateHandler +func (h *SidecarSetCreateUpdateHandler) InjectCache(cacher cache.Cache) error { + revInformer, err := cacher.GetInformerForKind(context.TODO(), apps.SchemeGroupVersion.WithKind("ControllerRevision")) + if err != nil { + return err + } + h.historyControl = sidecarcontrol.NewHistoryControl(nil, revInformer.(k8scache.SharedIndexInformer).GetIndexer(), webhookutil.GetNamespace()) + return nil +} diff --git a/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go b/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go index 5a687ce9e9..c58fb77c5d 100644 --- a/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go +++ b/pkg/webhook/sidecarset/validating/sidecarset_validating_test.go @@ -1,19 +1,23 @@ package validating import ( + "context" "fmt" "testing" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" "github.com/openkruise/kruise/pkg/features" "github.com/openkruise/kruise/pkg/util" + "github.com/openkruise/kruise/pkg/util/configuration" utilfeature "github.com/openkruise/kruise/pkg/util/feature" - + webhookutil "github.com/openkruise/kruise/pkg/webhook/util" + "github.com/openkruise/utils/sidecarcontrol" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/client-go/tools/cache" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -40,6 +44,39 @@ var ( }, }, } + + sidecarSetDemo = &appsv1alpha1.SidecarSet{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 123, + Name: "test-sidecarset", + Labels: map[string]string{ + "app": "sidecar", + }, + }, + Spec: appsv1alpha1.SidecarSetSpec{ + Containers: []appsv1alpha1.SidecarContainer{ + { + Container: corev1.Container{ + Name: "cold-sidecar", + Image: "cold-image:v1", + }, + UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ + UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade, + }, + }, + { + Container: corev1.Container{ + Name: "hot-sidecar", + Image: "hot-image:v1", + }, + UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{ + UpgradeType: appsv1alpha1.SidecarContainerHotUpgrade, + HotUpgradeEmptyImage: "hotupgrade:empty", + }, + }, + }, + }, + } ) var ( @@ -367,6 +404,8 @@ func TestValidateSidecarSet(t *testing.T) { for name, sidecarSet := range errorCases { fakeClient := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(SidecarSetRevisions...).Build() handler.Client = fakeClient + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + handler.historyControl = sidecarcontrol.NewHistoryControl(nil, indexer, webhookutil.GetNamespace()) allErrs := handler.validateSidecarSetSpec(&sidecarSet, field.NewPath("spec")) if len(allErrs) != 1 { t.Errorf("%v: expect errors len 1, but got: len %d %v", name, len(allErrs), util.DumpJSON(allErrs)) @@ -374,6 +413,7 @@ func TestValidateSidecarSet(t *testing.T) { fmt.Printf("%v: %v\n", name, allErrs) } } + _ = utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=false", features.SidecarSetPatchPodMetadataDefaultsAllowed)) } func TestSidecarSetNameConflict(t *testing.T) { @@ -816,3 +856,169 @@ func TestSidecarSetVolumeConflict(t *testing.T) { }) } } + +func TestValidateSidecarSetPatchMetadataWhitelist(t *testing.T) { + cases := []struct { + name string + getSidecarSet func() *appsv1alpha1.SidecarSet + getKruiseCM func() *corev1.ConfigMap + expectErr bool + }{ + { + name: "validate sidecarSet no patch Metadata", + getSidecarSet: func() *appsv1alpha1.SidecarSet { + demo := sidecarSetDemo.DeepCopy() + return demo + }, + getKruiseCM: func() *corev1.ConfigMap { + return nil + }, + expectErr: false, + }, + { + name: "validate sidecarSet whitelist failed-1", + getSidecarSet: func() *appsv1alpha1.SidecarSet { + demo := sidecarSetDemo.DeepCopy() + demo.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ + { + Annotations: map[string]string{ + "key1": "value1", + }, + }, + } + return demo + }, + getKruiseCM: func() *corev1.ConfigMap { + return nil + }, + expectErr: true, + }, + { + name: "validate sidecarSet whitelist success-1", + getSidecarSet: func() *appsv1alpha1.SidecarSet { + demo := sidecarSetDemo.DeepCopy() + demo.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ + { + Annotations: map[string]string{ + "key1": "value1", + }, + }, + } + return demo + }, + getKruiseCM: func() *corev1.ConfigMap { + demo := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configuration.KruiseConfigurationName, + Namespace: util.GetKruiseNamespace(), + }, + Data: map[string]string{ + configuration.SidecarSetPatchPodMetadataWhiteListKey: `{"rules":[{"allowedAnnotationKeyExprs":["key.*"]}]}`, + }, + } + return demo + }, + expectErr: false, + }, + { + name: "validate sidecarSet whitelist failed-2", + getSidecarSet: func() *appsv1alpha1.SidecarSet { + demo := sidecarSetDemo.DeepCopy() + demo.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ + { + Annotations: map[string]string{ + "key1": "value1", + }, + }, + } + return demo + }, + getKruiseCM: func() *corev1.ConfigMap { + demo := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configuration.KruiseConfigurationName, + Namespace: util.GetKruiseNamespace(), + }, + Data: map[string]string{ + configuration.SidecarSetPatchPodMetadataWhiteListKey: `{"rules":[{"allowedAnnotationKeyExprs":["key2"]}]}`, + }, + } + return demo + }, + expectErr: true, + }, + { + name: "validate sidecarSet whitelist failed-3", + getSidecarSet: func() *appsv1alpha1.SidecarSet { + demo := sidecarSetDemo.DeepCopy() + demo.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ + { + Annotations: map[string]string{ + "key1": "value1", + }, + }, + } + return demo + }, + getKruiseCM: func() *corev1.ConfigMap { + demo := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configuration.KruiseConfigurationName, + Namespace: util.GetKruiseNamespace(), + }, + Data: map[string]string{ + configuration.SidecarSetPatchPodMetadataWhiteListKey: `{"rules":[{"allowedAnnotationKeyExprs":["key.*"],"selector":{"matchLabels":{"app":"other"}}}]}`, + }, + } + return demo + }, + expectErr: true, + }, + { + name: "validate sidecarSet whitelist success-2", + getSidecarSet: func() *appsv1alpha1.SidecarSet { + demo := sidecarSetDemo.DeepCopy() + demo.Spec.PatchPodMetadata = []appsv1alpha1.SidecarSetPatchPodMetadata{ + { + Annotations: map[string]string{ + "key1": "value1", + }, + }, + } + return demo + }, + getKruiseCM: func() *corev1.ConfigMap { + demo := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configuration.KruiseConfigurationName, + Namespace: util.GetKruiseNamespace(), + }, + Data: map[string]string{ + configuration.SidecarSetPatchPodMetadataWhiteListKey: `{"rules":[{"allowedAnnotationKeyExprs":["key.*"],"selector":{"matchLabels":{"app":"sidecar"}}}]}`, + }, + } + return demo + }, + expectErr: false, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(testScheme).Build() + if cs.getKruiseCM() != nil { + err := fakeClient.Create(context.TODO(), cs.getKruiseCM()) + if err != nil { + fmt.Println(err.Error()) + } + } + handler.Client = fakeClient + err := handler.validateSidecarSetPatchMetadataWhitelist(cs.getSidecarSet()) + if cs.expectErr && err == nil { + t.Fatalf("ValidateSidecarSetPatchMetadataWhitelist failed") + } else if !cs.expectErr && err != nil { + t.Fatalf("ValidateSidecarSetPatchMetadataWhitelist failed: %s", err.Error()) + } + }) + } +} diff --git a/pkg/webhook/sidecarset/validating/utils.go b/pkg/webhook/sidecarset/validating/utils.go index 37aadfed66..6a63c2c182 100644 --- a/pkg/webhook/sidecarset/validating/utils.go +++ b/pkg/webhook/sidecarset/validating/utils.go @@ -18,6 +18,7 @@ package validating import ( "fmt" + "regexp" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" @@ -48,3 +49,12 @@ func isSidecarSetNamespaceDiff(origin *appsv1alpha1.SidecarSet, other *appsv1alp otherNamespace := other.Spec.Namespace return originNamespace != "" && otherNamespace != "" && originNamespace != otherNamespace } + +func matchRegKey(key string, regs []*regexp.Regexp) bool { + for _, reg := range regs { + if reg.MatchString(key) { + return true + } + } + return false +} diff --git a/pkg/webhook/util/controller/webhook_controller.go b/pkg/webhook/util/controller/webhook_controller.go index 882844e5dd..7b06d89092 100644 --- a/pkg/webhook/util/controller/webhook_controller.go +++ b/pkg/webhook/util/controller/webhook_controller.go @@ -87,7 +87,6 @@ func New(cfg *rest.Config, handlers map[string]admission.Handler) (*Controller, handlers: handlers, queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "webhook-controller"), } - c.informerFactory = informers.NewSharedInformerFactory(c.kubeClient, 0) secretInformer := coreinformers.New(c.informerFactory, namespace, nil).Secrets() diff --git a/test/e2e/apps/sidecarset.go b/test/e2e/apps/sidecarset.go index b0a2f0b0fa..7aa9a4a7ca 100644 --- a/test/e2e/apps/sidecarset.go +++ b/test/e2e/apps/sidecarset.go @@ -26,10 +26,10 @@ import ( appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" "github.com/openkruise/kruise/pkg/util/configuration" "github.com/openkruise/kruise/test/e2e/framework" + "github.com/openkruise/utils/sidecarcontrol" "github.com/onsi/ginkgo" "github.com/onsi/gomega" diff --git a/test/e2e/apps/sidecarset_hotupgrade.go b/test/e2e/apps/sidecarset_hotupgrade.go index b9573eef80..fdd1931328 100644 --- a/test/e2e/apps/sidecarset_hotupgrade.go +++ b/test/e2e/apps/sidecarset_hotupgrade.go @@ -22,9 +22,9 @@ import ( appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" "github.com/openkruise/kruise/test/e2e/framework" + "github.com/openkruise/utils/sidecarcontrol" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -33,7 +33,7 @@ import ( utilpointer "k8s.io/utils/pointer" ) -var _ = SIGDescribe("SidecarSet", func() { +var _ = SIGDescribe("SidecarSet HotUpgrade", func() { f := framework.NewDefaultFramework("sidecarset") var ns string var c clientset.Interface diff --git a/test/e2e/framework/sidecarset_utils.go b/test/e2e/framework/sidecarset_utils.go index ec38759c24..1c59edfb76 100644 --- a/test/e2e/framework/sidecarset_utils.go +++ b/test/e2e/framework/sidecarset_utils.go @@ -22,9 +22,9 @@ import ( appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" "github.com/openkruise/kruise/pkg/util" webhookutil "github.com/openkruise/kruise/pkg/webhook/util" + "github.com/openkruise/utils/sidecarcontrol" "github.com/onsi/gomega" apps "k8s.io/api/apps/v1" diff --git a/vendor/github.com/openkruise/utils/.gitignore b/vendor/github.com/openkruise/utils/.gitignore new file mode 100644 index 0000000000..e78615823e --- /dev/null +++ b/vendor/github.com/openkruise/utils/.gitignore @@ -0,0 +1,20 @@ +# OSX leaves these everywhere on SMB shares +._* + +# OSX trash +.DS_Store + +# Eclipse files +.classpath +.project +.settings/** + +# Files generated by JetBrains IDEs, e.g. IntelliJ IDEA +.idea/ +*.iml + +/vendor +/testbin +# Vscode files +.vscode +cover.out diff --git a/vendor/github.com/openkruise/utils/LICENSE.md b/vendor/github.com/openkruise/utils/LICENSE.md new file mode 100644 index 0000000000..f5c11e92e2 --- /dev/null +++ b/vendor/github.com/openkruise/utils/LICENSE.md @@ -0,0 +1,203 @@ +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` \ No newline at end of file diff --git a/vendor/github.com/openkruise/utils/Makefile b/vendor/github.com/openkruise/utils/Makefile new file mode 100644 index 0000000000..90c991644a --- /dev/null +++ b/vendor/github.com/openkruise/utils/Makefile @@ -0,0 +1,31 @@ +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# This is a requirement for 'setup-envtest.sh' in the test target. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +all: test + +##@ Development + +go_check: + @scripts/check_go_version "1.18.0" + +fmt: go_check ## Run go fmt against code. + go fmt $(shell go list ./... | grep -v /vendor/) + +vet: ## Run go vet against code. + go vet $(shell go list ./... | grep -v /vendor/) + +ENVTEST_ASSETS_DIR=$(shell pwd)/testbin +test: fmt vet ## Run tests + mkdir -p ${ENVTEST_ASSETS_DIR} + source ./scripts/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out + diff --git a/vendor/github.com/openkruise/utils/README.md b/vendor/github.com/openkruise/utils/README.md new file mode 100644 index 0000000000..06588bfb77 --- /dev/null +++ b/vendor/github.com/openkruise/utils/README.md @@ -0,0 +1 @@ +# utils diff --git a/vendor/github.com/openkruise/utils/pods.go b/vendor/github.com/openkruise/utils/pods.go new file mode 100644 index 0000000000..54b8bae8f2 --- /dev/null +++ b/vendor/github.com/openkruise/utils/pods.go @@ -0,0 +1,435 @@ +/* +Copyright 2020 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "github.com/docker/distribution/reference" + "k8s.io/klog/v2" + "k8s.io/utils/integer" + "strings" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +func MergeVolumeMounts(original, additional []v1.VolumeMount) []v1.VolumeMount { + mountpoints := sets.NewString() + for _, mount := range original { + mountpoints.Insert(mount.MountPath) + } + + for _, mount := range additional { + if mountpoints.Has(mount.MountPath) { + continue + } + original = append(original, mount) + mountpoints.Insert(mount.MountPath) + } + return original +} + +func MergeEnvVar(original []v1.EnvVar, additional []v1.EnvVar) []v1.EnvVar { + exists := sets.NewString() + for _, env := range original { + exists.Insert(env.Name) + } + + for _, env := range additional { + if exists.Has(env.Name) { + continue + } + original = append(original, env) + exists.Insert(env.Name) + } + + return original +} + +func MergeVolumes(original []v1.Volume, additional []v1.Volume) []v1.Volume { + exists := sets.NewString() + for _, volume := range original { + exists.Insert(volume.Name) + } + + for _, volume := range additional { + if exists.Has(volume.Name) { + continue + } + original = append(original, volume) + exists.Insert(volume.Name) + } + + return original +} + +func GetContainerEnvVar(container *v1.Container, key string) *v1.EnvVar { + if container == nil { + return nil + } + for i, e := range container.Env { + if e.Name == key { + return &container.Env[i] + } + } + return nil +} + +func GetContainerEnvValue(container *v1.Container, key string) string { + if container == nil { + return "" + } + for i, e := range container.Env { + if e.Name == key { + return container.Env[i].Value + } + } + return "" +} + +func GetContainerVolumeMount(container *v1.Container, key string) *v1.VolumeMount { + if container == nil { + return nil + } + for i, m := range container.VolumeMounts { + if m.MountPath == key { + return &container.VolumeMounts[i] + } + } + return nil +} + +func GetContainer(name string, pod *v1.Pod) *v1.Container { + if pod == nil { + return nil + } + for i := range pod.Spec.InitContainers { + v := &pod.Spec.InitContainers[i] + if v.Name == name { + return v + } + } + + for i := range pod.Spec.Containers { + v := &pod.Spec.Containers[i] + if v.Name == name { + return v + } + } + return nil +} + +func GetContainerStatus(name string, pod *v1.Pod) *v1.ContainerStatus { + if pod == nil { + return nil + } + for i := range pod.Status.ContainerStatuses { + v := &pod.Status.ContainerStatuses[i] + if v.Name == name { + return v + } + } + return nil +} + +func GetPodVolume(pod *v1.Pod, volumeName string) *v1.Volume { + for idx, v := range pod.Spec.Volumes { + if v.Name == volumeName { + return &pod.Spec.Volumes[idx] + } + } + return nil +} + +func IsRunningAndReady(pod *v1.Pod) bool { + return pod.Status.Phase == v1.PodRunning && IsPodReady(pod) && pod.DeletionTimestamp.IsZero() +} + +func GetPodContainerImageIDs(pod *v1.Pod) map[string]string { + cImageIDs := make(map[string]string, len(pod.Status.ContainerStatuses)) + for i := range pod.Status.ContainerStatuses { + c := &pod.Status.ContainerStatuses[i] + //ImageID format: docker-pullable://busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d + imageID := c.ImageID + if strings.Contains(imageID, "://") { + imageID = strings.Split(imageID, "://")[1] + } + cImageIDs[c.Name] = imageID + } + return cImageIDs +} + +func IsPodContainerDigestEqual(containers sets.String, pod *v1.Pod) bool { + cImageIDs := GetPodContainerImageIDs(pod) + + for _, container := range pod.Spec.Containers { + if !containers.Has(container.Name) { + continue + } + // image must be digest format + if !IsImageDigest(container.Image) { + return false + } + imageID, ok := cImageIDs[container.Name] + if !ok { + return false + } + if !IsContainerImageEqual(container.Image, imageID) { + return false + } + } + return true +} + +// parse container images, +// 1. docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d +// repo=docker.io/busybox, tag="", digest=sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d +// 2. docker.io/busybox:latest +// repo=docker.io/busybox, tag=latest, digest="" +func ParseImage(image string) (repo, tag, digest string, err error) { + refer, err := reference.Parse(image) + if err != nil { + return "", "", "", err + } + + if named, ok := refer.(reference.Named); ok { + repo = named.Name() + } + if tagged, ok := refer.(reference.Tagged); ok { + tag = tagged.Tag() + } + if digested, ok := refer.(reference.Digested); ok { + digest = digested.Digest().String() + } + return +} + +//whether image is digest format, +//for example: docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d +func IsImageDigest(image string) bool { + _, _, digest, _ := ParseImage(image) + return digest != "" +} + +// 1. image1, image2 are digest image, compare repo+digest +// 2. image1, image2 are normal image, compare repo+tag +// 3. image1, image2 are digest+normal image, don't support compare it, return false +func IsContainerImageEqual(image1, image2 string) bool { + repo1, tag1, digest1, err := ParseImage(image1) + if err != nil { + klog.Errorf("parse image %s failed: %s", image1, err.Error()) + return false + } + + repo2, tag2, digest2, err := ParseImage(image2) + if err != nil { + klog.Errorf("parse image %s failed: %s", image2, err.Error()) + return false + } + + if IsImageDigest(image1) && IsImageDigest(image2) { + return repo1 == repo2 && digest1 == digest2 + } + + return repo1 == repo2 && tag1 == tag2 +} + +func MergeVolumeMountsInContainer(origin *v1.Container, other v1.Container) { + mountExist := make(map[string]bool) + for _, volume := range origin.VolumeMounts { + mountExist[volume.MountPath] = true + + } + + for _, volume := range other.VolumeMounts { + if mountExist[volume.MountPath] { + continue + } + + origin.VolumeMounts = append(origin.VolumeMounts, volume) + } +} + +// GetPodCondition extracts the provided condition from the given status and returns that. +// Returns nil and -1 if the condition is not present, and the index of the located condition. +func GetPodCondition(status *v1.PodStatus, conditionType v1.PodConditionType) (int, *v1.PodCondition) { + if status == nil { + return -1, nil + } + return GetPodConditionFromList(status.Conditions, conditionType) +} + +// GetPodConditionFromList extracts the provided condition from the given list of condition and +// returns the index of the condition and the condition. Returns -1 and nil if the condition is not present. +func GetPodConditionFromList(conditions []v1.PodCondition, conditionType v1.PodConditionType) (int, *v1.PodCondition) { + if conditions == nil { + return -1, nil + } + for i := range conditions { + if conditions[i].Type == conditionType { + return i, &conditions[i] + } + } + return -1, nil +} + +func SetPodCondition(pod *v1.Pod, condition v1.PodCondition) { + for i, c := range pod.Status.Conditions { + if c.Type == condition.Type { + if c.Status != condition.Status { + pod.Status.Conditions[i] = condition + } + return + } + } + pod.Status.Conditions = append(pod.Status.Conditions, condition) +} + +func SetPodReadyCondition(pod *v1.Pod) { + _, podReady := GetPodCondition(&pod.Status, v1.PodReady) + if podReady == nil { + return + } + + _, containersReady := GetPodCondition(&pod.Status, v1.ContainersReady) + if containersReady == nil || containersReady.Status != v1.ConditionTrue { + return + } + + var unreadyMessages []string + for _, rg := range pod.Spec.ReadinessGates { + _, c := GetPodCondition(&pod.Status, rg.ConditionType) + if c == nil { + unreadyMessages = append(unreadyMessages, fmt.Sprintf("corresponding condition of pod readiness gate %q does not exist.", string(rg.ConditionType))) + } else if c.Status != v1.ConditionTrue { + unreadyMessages = append(unreadyMessages, fmt.Sprintf("the status of pod readiness gate %q is not \"True\", but %v", string(rg.ConditionType), c.Status)) + } + } + + newPodReady := v1.PodCondition{ + Type: v1.PodReady, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.Now(), + } + // Set "Ready" condition to "False" if any readiness gate is not ready. + if len(unreadyMessages) != 0 { + unreadyMessage := strings.Join(unreadyMessages, ", ") + newPodReady = v1.PodCondition{ + Type: v1.PodReady, + Status: v1.ConditionFalse, + Reason: "ReadinessGatesNotReady", + Message: unreadyMessage, + } + } + + SetPodCondition(pod, newPodReady) +} + +// GetPodReadyCondition extracts the pod ready condition from the given status and returns that. +// Returns nil if the condition is not present. +func GetPodReadyCondition(status v1.PodStatus) *v1.PodCondition { + _, condition := GetPodCondition(&status, v1.PodReady) + return condition +} + +// IsPodReadyConditionTrue returns true if a pod is ready; false otherwise. +func IsPodReadyConditionTrue(status v1.PodStatus) bool { + condition := GetPodReadyCondition(status) + return condition != nil && condition.Status == v1.ConditionTrue +} + +// IsPodReady returns true if a pod is ready; false otherwise. +func IsPodReady(pod *v1.Pod) bool { + return IsPodReadyConditionTrue(pod.Status) +} + +var ( + podPhaseToOrdinal = map[v1.PodPhase]int{v1.PodPending: 0, v1.PodUnknown: 1, v1.PodRunning: 2} +) + +// ActivePods type allows custom sorting of pods so a controller can pick the best ones to delete. +type ActivePods []*v1.Pod + +func (s ActivePods) Len() int { return len(s) } +func (s ActivePods) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func (s ActivePods) Less(i, j int) bool { + // 1. Unassigned < assigned + // If only one of the pods is unassigned, the unassigned one is smaller + if s[i].Spec.NodeName != s[j].Spec.NodeName && (len(s[i].Spec.NodeName) == 0 || len(s[j].Spec.NodeName) == 0) { + return len(s[i].Spec.NodeName) == 0 + } + // 2. PodPending < PodUnknown < PodRunning + if podPhaseToOrdinal[s[i].Status.Phase] != podPhaseToOrdinal[s[j].Status.Phase] { + return podPhaseToOrdinal[s[i].Status.Phase] < podPhaseToOrdinal[s[j].Status.Phase] + } + // 3. Not ready < ready + // If only one of the pods is not ready, the not ready one is smaller + if IsPodReady(s[i]) != IsPodReady(s[j]) { + return !IsPodReady(s[i]) + } + // TODO: take availability into account when we push minReadySeconds information from deployment into pods, + // see https://github.com/kubernetes/kubernetes/issues/22065 + // 4. Been ready for empty time < less time < more time + // If both pods are ready, the latest ready one is smaller + if IsPodReady(s[i]) && IsPodReady(s[j]) { + readyTime1 := podReadyTime(s[i]) + readyTime2 := podReadyTime(s[j]) + if !readyTime1.Equal(readyTime2) { + return afterOrZero(readyTime1, readyTime2) + } + } + // 5. Pods with containers with higher restart counts < lower restart counts + if maxContainerRestarts(s[i]) != maxContainerRestarts(s[j]) { + return maxContainerRestarts(s[i]) > maxContainerRestarts(s[j]) + } + // 6. Empty creation time pods < newer pods < older pods + if !s[i].CreationTimestamp.Equal(&s[j].CreationTimestamp) { + return afterOrZero(&s[i].CreationTimestamp, &s[j].CreationTimestamp) + } + return false +} + +func maxContainerRestarts(pod *v1.Pod) int { + maxRestarts := 0 + for _, c := range pod.Status.ContainerStatuses { + maxRestarts = integer.IntMax(maxRestarts, int(c.RestartCount)) + } + return maxRestarts +} + +// afterOrZero checks if time t1 is after time t2; if one of them +// is zero, the zero time is seen as after non-zero time. +func afterOrZero(t1, t2 *metav1.Time) bool { + if t1.Time.IsZero() || t2.Time.IsZero() { + return t1.Time.IsZero() + } + return t1.After(t2.Time) +} + +func podReadyTime(pod *v1.Pod) *metav1.Time { + if IsPodReady(pod) { + for _, c := range pod.Status.Conditions { + // we only care about pod ready conditions + if c.Type == v1.PodReady && c.Status == v1.ConditionTrue { + return &c.LastTransitionTime + } + } + } + return &metav1.Time{} +} diff --git a/pkg/control/sidecarcontrol/api.go b/vendor/github.com/openkruise/utils/sidecarcontrol/api.go similarity index 81% rename from pkg/control/sidecarcontrol/api.go rename to vendor/github.com/openkruise/utils/sidecarcontrol/api.go index f0b5d3823f..ac357c03d2 100644 --- a/pkg/control/sidecarcontrol/api.go +++ b/vendor/github.com/openkruise/utils/sidecarcontrol/api.go @@ -18,20 +18,11 @@ package sidecarcontrol import ( appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" ) type SidecarControl interface { - //*****common*****// - // get sidecarset - GetSidecarset() *appsv1alpha1.SidecarSet - // when sidecarSet is not active, it will not perform injections and upgrades process. - // You can re-implement the function IsActiveSidecarSet to indicate that this sidecarSet is no longer working by adding some sidecarSet flags, - // for example: sidecarSet.Annotations[sidecarset.kruise.io/disabled] = "true" - IsActiveSidecarSet() bool - //*****inject portion*****// // whether need inject the volumeMount into container // when ShareVolumePolicy is enabled, the sidecar container will share the other container's VolumeMounts in the pod(don't contains the injected sidecar container). @@ -47,25 +38,26 @@ type SidecarControl interface { //*****upgrade portion*****// // IsPodStateConsistent indicates whether pod.spec and pod.status are consistent after updating the sidecar containers - IsPodStateConsistent(pod *v1.Pod, sidecarContainers sets.String) bool + IsPodStateConsistent(pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet, sidecarContainers sets.String) bool // IsPodReady indicates whether pod is fully ready // 1. pod.Status.Phase == v1.PodRunning // 2. pod.condition PodReady == true // 3. whether empty sidecar container is HotUpgradeEmptyImage - IsPodReady(pod *v1.Pod) bool + IsPodReady(pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet) bool // upgrade pod sidecar container to sidecarSet latest version // if container==nil means no change, no need to update, otherwise need to update - UpgradeSidecarContainer(sidecarContainer *appsv1alpha1.SidecarContainer, pod *v1.Pod) *v1.Container + UpgradeSidecarContainer(sidecarContainer *appsv1alpha1.SidecarContainer, pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet) *v1.Container // When upgrading the pod sidecar container, you need to record some in-place upgrade information in pod annotations, // which is needed by the sidecarset controller to determine whether the upgrade is completed. - UpdatePodAnnotationsInUpgrade(changedContainers []string, pod *v1.Pod) + UpdatePodAnnotationsInUpgrade(changedContainers []string, pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet) // Is sidecarset can upgrade pods, // In Kubernetes native scenarios, only Container Image upgrades are allowed // When modifying other fields of the container, e.g. volumemounts, the sidecarSet will not depart to upgrade the sidecar container logic in-place, // and needs to be done by rebuilding the pod - IsSidecarSetUpgradable(pod *v1.Pod) bool -} + IsSidecarSetUpgradable(pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet) bool + + // FindContainerToHotUpgrade + FindContainerToHotUpgrade(sidecarContainer *appsv1alpha1.SidecarContainer, pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet) (string, string) -func New(cs *appsv1alpha1.SidecarSet) SidecarControl { - return &commonControl{SidecarSet: cs} + GetSuitableRevisionSidecarSet(sidecarSet *appsv1alpha1.SidecarSet, oldPod, newPod *v1.Pod) (*appsv1alpha1.SidecarSet, error) } diff --git a/pkg/control/sidecarcontrol/hash.go b/vendor/github.com/openkruise/utils/sidecarcontrol/hash.go similarity index 88% rename from pkg/control/sidecarcontrol/hash.go rename to vendor/github.com/openkruise/utils/sidecarcontrol/hash.go index 85451dd1f7..15fdbf8590 100644 --- a/pkg/control/sidecarcontrol/hash.go +++ b/vendor/github.com/openkruise/utils/sidecarcontrol/hash.go @@ -32,7 +32,7 @@ func SidecarSetHash(sidecarSet *appsv1alpha1.SidecarSet) (string, error) { if err != nil { return "", err } - h := rand.SafeEncodeString(hash(encoded)) + h := rand.SafeEncodeString(fmt.Sprintf("%x", sha256.Sum256([]byte(encoded)))) return h, nil } @@ -47,7 +47,7 @@ func SidecarSetHashWithoutImage(sidecarSet *appsv1alpha1.SidecarSet) (string, er if err != nil { return "", err } - return rand.SafeEncodeString(hash(encoded)), nil + return rand.SafeEncodeString(fmt.Sprintf("%x", sha256.Sum256([]byte(encoded)))), nil } func encodeSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (string, error) { @@ -60,8 +60,3 @@ func encodeSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (string, error) { } return string(data), nil } - -// hash hashes `data` with sha256 and returns the hex string -func hash(data string) string { - return fmt.Sprintf("%x", sha256.Sum256([]byte(data))) -} diff --git a/vendor/github.com/openkruise/utils/sidecarcontrol/history_control.go b/vendor/github.com/openkruise/utils/sidecarcontrol/history_control.go new file mode 100644 index 0000000000..42b17ce481 --- /dev/null +++ b/vendor/github.com/openkruise/utils/sidecarcontrol/history_control.go @@ -0,0 +1,531 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecarcontrol + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/openkruise/utils" + "hash" + "hash/fnv" + "sort" + "strconv" + + "github.com/davecgh/go-spew/spew" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/strategicpatch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + appslisters "k8s.io/client-go/listers/apps/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" +) + +var ( + patchCodec = scheme.Codecs.LegacyCodec(appsv1alpha1.SchemeGroupVersion) +) + +func GetRevisionSelector(s *appsv1alpha1.SidecarSet) labels.Selector { + labelSelector := &metav1.LabelSelector{ + MatchLabels: map[string]string{ + SidecarSetKindName: s.GetName(), + }, + } + selector, err := utils.ValidatedLabelSelectorAsSelector(labelSelector) + if err != nil { + // static error, just panic + panic("Incorrect label selector for ControllerRevision of SidecarSet.") + } + return selector +} + +func NewRevision(s *appsv1alpha1.SidecarSet, namespace string, revision int64, collisionCount *int32) ( + *apps.ControllerRevision, error, +) { + patch, err := getPatch(s) + if err != nil { + return nil, err + } + + cr, err := NewControllerRevision(s, + s.GetObjectKind().GroupVersionKind(), + s.Labels, + runtime.RawExtension{Raw: patch}, + revision, + collisionCount) + if err != nil { + return nil, err + } + + cr.SetNamespace(namespace) + if cr.Labels == nil { + cr.Labels = make(map[string]string) + } + if cr.ObjectMeta.Annotations == nil { + cr.ObjectMeta.Annotations = make(map[string]string) + } + if s.Annotations[SidecarSetHashAnnotation] != "" { + cr.Annotations[SidecarSetHashAnnotation] = s.Annotations[SidecarSetHashAnnotation] + } + if s.Annotations[SidecarSetHashWithoutImageAnnotation] != "" { + cr.Annotations[SidecarSetHashWithoutImageAnnotation] = s.Annotations[SidecarSetHashWithoutImageAnnotation] + } + if s.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] != "" { + cr.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] = s.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] + } + cr.Labels[SidecarSetKindName] = s.Name + for key, value := range s.Annotations { + cr.ObjectMeta.Annotations[key] = value + } + return cr, nil +} + +// getPatch returns a strategic merge patch that can be applied to restore a SidecarSet to a +// previous version. If the returned error is nil the patch is valid. The current state that we save is just the +// PodSpecTemplate. We can modify this later to encompass more state (or less) and remain compatible with previously +// recorded patches. +func getPatch(s *appsv1alpha1.SidecarSet) ([]byte, error) { + str, err := runtime.Encode(patchCodec, s) + if err != nil { + return nil, err + } + var raw map[string]interface{} + _ = json.Unmarshal(str, &raw) + + objCopy := make(map[string]interface{}) + specCopy := make(map[string]interface{}) + // only copy some specified fields of s.Spec to specCopy + spec := raw["spec"].(map[string]interface{}) + copySidecarSetSpecRevision(specCopy, spec) + + objCopy["spec"] = specCopy + return json.Marshal(objCopy) +} + +// NextRevision finds the next valid revision number based on revisions. If the length of revisions +// is 0 this is 1. Otherwise, it is 1 greater than the largest revision's Revision. This method +// assumes that revisions has been sorted by Revision. +func NextRevision(revisions []*apps.ControllerRevision) int64 { + count := len(revisions) + if count <= 0 { + return 1 + } + return revisions[count-1].Revision + 1 +} + +type HistoryControl interface { + CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) + ListSidecarSetControllerRevisions(sidecarSet *appsv1alpha1.SidecarSet) ([]*apps.ControllerRevision, error) + UpdateControllerRevision(revision *apps.ControllerRevision, newRevision int64) (*apps.ControllerRevision, error) + DeleteControllerRevision(revision *apps.ControllerRevision) error + GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error) + GetSuitableRevisionSidecarSet(sidecarSet *appsv1alpha1.SidecarSet, oldPod, newPod *v1.Pod) (*appsv1alpha1.SidecarSet, error) +} + +type realControl struct { + // client for k8s + client clientset.Interface + // sidecarSet controllerRevision namespace, default is openKruise namespace + namespace string + // historyLister get list/get history from the shared informers's store + historyLister appslisters.ControllerRevisionLister +} + +// NewHistoryControl new history control +// indexer is controllerRevision indexer +// If you need CreateControllerRevision function, you must set parameter client +// If you need GetHistorySidecarSet and GetSuitableRevisionSidecarSet function, you must set parameter indexer +// Parameter namespace is required +func NewHistoryControl(client clientset.Interface, indexer cache.Indexer, namespace string) HistoryControl { + return &realControl{ + client: client, + namespace: namespace, + historyLister: appslisters.NewControllerRevisionLister(indexer), + } +} + +func (r *realControl) CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) { + if collisionCount == nil { + return nil, fmt.Errorf("collisionCount should not be nil") + } + // Clone the input + clone := revision.DeepCopy() + + // Continue to attempt to create the revision updating the name with a new hash on each iteration + for { + hash := HashControllerRevision(revision, collisionCount) + // Update the revisions name + clone.Name = ControllerRevisionName(parent.GetName(), hash) + _, err := r.client.AppsV1().ControllerRevisions(clone.Namespace).Create(context.TODO(), clone, metav1.CreateOptions{}) + if errors.IsAlreadyExists(err) { + exists, err := r.client.AppsV1().ControllerRevisions(clone.Namespace).Get(context.TODO(), clone.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + if bytes.Equal(exists.Data.Raw, clone.Data.Raw) { + return exists, nil + } + *collisionCount++ + continue + } + return clone, err + } +} + +func (r *realControl) ListSidecarSetControllerRevisions(sidecarSet *appsv1alpha1.SidecarSet) ([]*apps.ControllerRevision, error) { + selector := GetRevisionSelector(sidecarSet) + revisions, err := r.historyLister.ControllerRevisions(r.namespace).List(selector) + if err != nil { + klog.Errorf("Failed to get ControllerRevision for SidecarSet(%v), err: %v", sidecarSet.Name, err) + return nil, err + } + return revisions, nil +} + +func (r *realControl) UpdateControllerRevision(revision *apps.ControllerRevision, newRevision int64) (*apps.ControllerRevision, error) { + clone := revision.DeepCopy() + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if clone.Revision == newRevision { + return nil + } + clone.Revision = newRevision + updated, updateErr := r.client.AppsV1().ControllerRevisions(clone.Namespace).Update(context.TODO(), clone, metav1.UpdateOptions{}) + if updateErr == nil { + return nil + } + if updated != nil { + clone = updated + } + if updated, err := r.historyLister.ControllerRevisions(clone.Namespace).Get(clone.Name); err == nil { + // make a copy so we don't mutate the shared cache + clone = updated.DeepCopy() + } + return updateErr + }) + return clone, err +} + +func (r *realControl) DeleteControllerRevision(revision *apps.ControllerRevision) error { + return r.client.AppsV1().ControllerRevisions(r.namespace).Delete(context.TODO(), revision.Name, metav1.DeleteOptions{}) +} + +func (r *realControl) GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error) { + revision, err := r.getControllerRevision(sidecarSet, revisionInfo) + if err != nil || revision == nil { + return nil, err + } + clone := sidecarSet.DeepCopy() + cloneBytes, err := runtime.Encode(patchCodec, clone) + if err != nil { + klog.Errorf("Failed to encode sidecarSet(%v), error: %v", sidecarSet.Name, err) + return nil, err + } + patched, err := strategicpatch.StrategicMergePatch(cloneBytes, revision.Data.Raw, clone) + if err != nil { + klog.Errorf("Failed to merge sidecarSet(%v) and controllerRevision(%v): %v, error: %v", sidecarSet.Name, revision.Name, err) + return nil, err + } + // restore history from patch + restoredSidecarSet := &appsv1alpha1.SidecarSet{} + if err := json.Unmarshal(patched, restoredSidecarSet); err != nil { + return nil, err + } + // restore hash annotation and revision info + if err := restoreRevisionInfo(restoredSidecarSet, revision); err != nil { + return nil, err + } + return restoredSidecarSet, nil +} + +func (r *realControl) getControllerRevision(set *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*apps.ControllerRevision, error) { + if revisionInfo == nil { + return nil, nil + } + switch { + case revisionInfo.RevisionName != nil: + revision, err := r.historyLister.ControllerRevisions(r.namespace).Get(*revisionInfo.RevisionName) + if err != nil { + klog.Errorf("Failed to get ControllerRevision %v for SidecarSet(%v), err: %v", *revisionInfo.RevisionName, set.Name, err) + return nil, err + } + return revision, nil + + case revisionInfo.CustomVersion != nil: + selector := GetRevisionSelector(set) + req, _ := labels.NewRequirement(appsv1alpha1.SidecarSetCustomVersionLabel, selection.Equals, []string{*revisionInfo.CustomVersion}) + selector = selector.Add(*req) + revisionList, err := r.historyLister.ControllerRevisions(r.namespace).List(selector) + if err != nil { + klog.Errorf("Failed to get ControllerRevision for SidecarSet(%v), custom version: %v, err: %v", set.Name, *revisionInfo.CustomVersion, err) + return nil, err + } + + var revisions []*apps.ControllerRevision + for i := range revisionList { + obj := revisionList[i] + revisions = append(revisions, obj) + } + + if len(revisions) == 0 { + return nil, generateNotFoundError(set) + } + SortControllerRevisions(revisions) + return revisions[len(revisions)-1], nil + } + + klog.Error("Failed to get controllerRevision due to both empty RevisionName and CustomVersion") + return nil, nil +} + +func copySidecarSetSpecRevision(dst, src map[string]interface{}) { + // we will use patch instead of update operation to update pods in the future + // dst["$patch"] = "replace" + // only record these revisions + dst["volumes"] = src["volumes"] + dst["containers"] = src["containers"] + dst["initContainers"] = src["initContainers"] + dst["imagePullSecrets"] = src["imagePullSecrets"] + dst["patchPodMetadata"] = src["patchPodMetadata"] +} + +func restoreRevisionInfo(sidecarSet *appsv1alpha1.SidecarSet, revision *apps.ControllerRevision) error { + if sidecarSet.Annotations == nil { + sidecarSet.Annotations = map[string]string{} + } + if revision.Annotations[SidecarSetHashAnnotation] != "" { + sidecarSet.Annotations[SidecarSetHashAnnotation] = revision.Annotations[SidecarSetHashAnnotation] + } else { + hashCodeWithImage, err := SidecarSetHash(sidecarSet) + if err != nil { + return err + } + sidecarSet.Annotations[SidecarSetHashAnnotation] = hashCodeWithImage + } + if revision.Annotations[SidecarSetHashWithoutImageAnnotation] != "" { + sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = revision.Annotations[SidecarSetHashWithoutImageAnnotation] + } else { + hashCodeWithoutImage, err := SidecarSetHashWithoutImage(sidecarSet) + if err != nil { + return err + } + sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = hashCodeWithoutImage + } + sidecarSet.Status.LatestRevision = revision.Name + return nil +} + +func MockSidecarSetForRevision(set *appsv1alpha1.SidecarSet) metav1.Object { + return &metav1.ObjectMeta{ + UID: set.UID, + Name: set.Name, + Namespace: utils.GetNamespace(), + } +} + +func generateNotFoundError(set *appsv1alpha1.SidecarSet) error { + return errors.NewNotFound(schema.GroupResource{ + Group: apps.GroupName, + Resource: "ControllerRevision", + }, set.Name) +} + +func (r *realControl) GetSuitableRevisionSidecarSet(sidecarSet *appsv1alpha1.SidecarSet, oldPod, newPod *v1.Pod) (*appsv1alpha1.SidecarSet, error) { + // operation == update + if oldPod != nil { + // optimization: quickly return if newPod matched the latest sidecarSet + if GetPodSidecarSetRevision(sidecarSet.Name, newPod) == GetSidecarSetRevision(sidecarSet) { + return sidecarSet.DeepCopy(), nil + } + selector := GetRevisionSelector(sidecarSet) + revisions, err := r.historyLister.ControllerRevisions(r.namespace).List(selector) + if err != nil { + klog.Errorf("Failed to get ControllerRevision for SidecarSet(%v), err: %v", sidecarSet.Name, err) + return nil, err + } + + suitableSidecarSet, err := r.getSpecificRevisionSidecarSetForPod(sidecarSet, revisions, newPod) + if err != nil { + return nil, err + } else if suitableSidecarSet != nil { + return suitableSidecarSet, nil + } + + suitableSidecarSet, err = r.getSpecificRevisionSidecarSetForPod(sidecarSet, revisions, oldPod) + if err != nil { + return nil, err + } else if suitableSidecarSet != nil { + return suitableSidecarSet, nil + } + + return sidecarSet.DeepCopy(), nil + } + + revisionInfo := sidecarSet.Spec.InjectionStrategy.Revision + if revisionInfo == nil || (revisionInfo.RevisionName == nil && revisionInfo.CustomVersion == nil) { + return sidecarSet.DeepCopy(), nil + } + + // TODO: support 'PartitionBased' policy to inject old/new revision according to Partition + switch sidecarSet.Spec.InjectionStrategy.Revision.Policy { + case "", appsv1alpha1.AlwaysSidecarSetInjectRevisionPolicy: + return r.getSpecificHistorySidecarSet(sidecarSet, revisionInfo) + } + + return r.getSpecificHistorySidecarSet(sidecarSet, revisionInfo) +} + +func (r *realControl) getSpecificRevisionSidecarSetForPod(sidecarSet *appsv1alpha1.SidecarSet, revisions []*apps.ControllerRevision, pod *corev1.Pod) (*appsv1alpha1.SidecarSet, error) { + var err error + var matchedSidecarSet *appsv1alpha1.SidecarSet + for _, revision := range revisions { + if GetPodSidecarSetControllerRevision(sidecarSet.Name, pod) == revision.Name { + matchedSidecarSet, err = r.getSpecificHistorySidecarSet(sidecarSet, &appsv1alpha1.SidecarSetInjectRevision{RevisionName: &revision.Name}) + if err != nil { + return nil, err + } + break + } + } + return matchedSidecarSet, nil +} + +func (r *realControl) getSpecificHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionInfo *appsv1alpha1.SidecarSetInjectRevision) (*appsv1alpha1.SidecarSet, error) { + // else return its corresponding history revision + historySidecarSet, err := r.GetHistorySidecarSet(sidecarSet, revisionInfo) + if err != nil { + klog.Warningf("Failed to restore history revision for SidecarSet %v, ControllerRevision name %v:, error: %v", + sidecarSet.Name, sidecarSet.Spec.InjectionStrategy.Revision, err) + return nil, err + } + if historySidecarSet == nil { + historySidecarSet = sidecarSet.DeepCopy() + klog.Warningf("Failed to restore history revision for SidecarSet %v, will use the latest", sidecarSet.Name) + } + return historySidecarSet, nil +} + +// ControllerRevisionHashLabel is the label used to indicate the hash value of a ControllerRevision's Data. +const ControllerRevisionHashLabel = "controller.kubernetes.io/hash" + +// ControllerRevisionName returns the Name for a ControllerRevision in the form prefix-hash. If the length +// of prefix is greater than 223 bytes, it is truncated to allow for a name that is no larger than 253 bytes. +func ControllerRevisionName(prefix string, hash string) string { + if len(prefix) > 223 { + prefix = prefix[:223] + } + + return fmt.Sprintf("%s-%s", prefix, hash) +} + +// NewControllerRevision returns a ControllerRevision with a ControllerRef pointing to parent and indicating that +// parent is of parentKind. The ControllerRevision has labels matching template labels, contains Data equal to data, and +// has a Revision equal to revision. The collisionCount is used when creating the name of the ControllerRevision +// so the name is likely unique. If the returned error is nil, the returned ControllerRevision is valid. If the +// returned error is not nil, the returned ControllerRevision is invalid for use. +func NewControllerRevision(parent metav1.Object, + parentKind schema.GroupVersionKind, + templateLabels map[string]string, + data runtime.RawExtension, + revision int64, + collisionCount *int32) (*apps.ControllerRevision, error) { + labelMap := make(map[string]string) + for k, v := range templateLabels { + labelMap[k] = v + } + cr := &apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labelMap, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(parent, parentKind)}, + }, + Data: data, + Revision: revision, + } + hash := HashControllerRevision(cr, collisionCount) + cr.Name = ControllerRevisionName(parent.GetName(), hash) + cr.Labels[ControllerRevisionHashLabel] = hash + return cr, nil +} + +// HashControllerRevision hashes the contents of revision's Data using FNV hashing. If probe is not nil, the byte value +// of probe is added written to the hash as well. The returned hash will be a safe encoded string to avoid bad words. +func HashControllerRevision(revision *apps.ControllerRevision, probe *int32) string { + hf := fnv.New32() + if len(revision.Data.Raw) > 0 { + hf.Write(revision.Data.Raw) + } + if revision.Data.Object != nil { + DeepHashObject(hf, revision.Data.Object) + } + if probe != nil { + hf.Write([]byte(strconv.FormatInt(int64(*probe), 10))) + } + return rand.SafeEncodeString(fmt.Sprint(hf.Sum32())) +} + +// DeepHashObject writes specified object to hash using the spew library +// which follows pointers and prints actual values of the nested objects +// ensuring the hash does not change when a pointer changes. +func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { + hasher.Reset() + printer := spew.ConfigState{ + Indent: " ", + SortKeys: true, + DisableMethods: true, + SpewKeys: true, + } + printer.Fprintf(hasher, "%#v", objectToWrite) +} + +// SortControllerRevisions sorts revisions by their Revision. +func SortControllerRevisions(revisions []*apps.ControllerRevision) { + sort.Stable(byRevision(revisions)) +} + +// byRevision implements sort.Interface to allow ControllerRevisions to be sorted by Revision. +type byRevision []*apps.ControllerRevision + +func (br byRevision) Len() int { + return len(br) +} + +// Less breaks ties first by creation timestamp, then by name +func (br byRevision) Less(i, j int) bool { + if br[i].Revision == br[j].Revision { + if br[j].CreationTimestamp.Equal(&br[i].CreationTimestamp) { + return br[i].Name < br[j].Name + } + return br[j].CreationTimestamp.After(br[i].CreationTimestamp.Time) + } + return br[i].Revision < br[j].Revision +} + +func (br byRevision) Swap(i, j int) { + br[i], br[j] = br[j], br[i] +} diff --git a/vendor/github.com/openkruise/utils/sidecarcontrol/mutating_pod.go b/vendor/github.com/openkruise/utils/sidecarcontrol/mutating_pod.go new file mode 100644 index 0000000000..d520106bd9 --- /dev/null +++ b/vendor/github.com/openkruise/utils/sidecarcontrol/mutating_pod.go @@ -0,0 +1,334 @@ +/* +Copyright 2020 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecarcontrol + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/utils" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" +) + +// mutate pod based on SidecarSet Object +func SidecarSetMutatingPod(pod, oldPod *corev1.Pod, sidecarSets []*appsv1alpha1.SidecarSet, control SidecarControl) (skip bool, err error) { + matchedSidecarSets := make([]*appsv1alpha1.SidecarSet, 0) + for _, sidecarSet := range sidecarSets { + if sidecarSet.Spec.InjectionStrategy.Paused { + continue + } + if matched, err := PodMatchedSidecarSet(pod, sidecarSet); err != nil { + return false, err + } else if !matched { + continue + } + // get user-specific revision or the latest revision of SidecarSet + suitableSidecarSet, err := control.GetSuitableRevisionSidecarSet(sidecarSet, oldPod, pod) + if err != nil { + return false, err + } + // check whether sidecarSet is active + // when sidecarSet is not active, it will not perform injections and upgrades process. + matchedSidecarSets = append(matchedSidecarSets, suitableSidecarSet) + } + if len(matchedSidecarSets) == 0 { + return true, nil + } + + // check pod + if oldPod != nil && !control.IsPodAvailabilityChanged(pod, oldPod) { + klog.V(3).Infof("pod(%s/%s) availability unchanged for sidecarSet, and ignore", pod.Namespace, pod.Name) + return true, nil + } + // patch pod metadata, annotations & labels + // When the Pod main container is upgraded in place, and the sidecarSet configuration does not change at this time, + // at this point, it can also patch pod metadata + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + skip = true + for _, sidecarSet := range matchedSidecarSets { + sk, err := PatchPodMetadata(&pod.ObjectMeta, sidecarSet.Spec.PatchPodMetadata) + if err != nil { + klog.Errorf("sidecarSet(%s) update pod(%s/%s) metadata failed: %s", sidecarSet.Name, pod.Namespace, pod.Name, err.Error()) + return false, err + } else if !sk { + // skip = false + skip = false + } + } + //build sidecar containers, sidecar initContainers, sidecar volumes, annotations to inject into pod object + sidecarContainers, sidecarInitContainers, sidecarSecrets, volumesInSidecar, injectedAnnotations, err := buildSidecars(control, pod, oldPod, matchedSidecarSets) + if err != nil { + return false, err + } else if len(sidecarContainers) == 0 && len(sidecarInitContainers) == 0 { + klog.V(3).Infof("[sidecar inject] pod(%s/%s) don't have injected containers", pod.Namespace, pod.Name) + return skip, nil + } + + klog.V(3).Infof("[sidecar inject] begin inject sidecarContainers(%v) sidecarInitContainers(%v) sidecarSecrets(%v), volumes(%s)"+ + "annotations(%v) into pod(%s/%s)", sidecarContainers, sidecarInitContainers, sidecarSecrets, volumesInSidecar, injectedAnnotations, + pod.Namespace, pod.Name) + klog.V(4).Infof("[sidecar inject] before mutating: %v", utils.DumpJSON(pod)) + // apply sidecar set info into pod + // 1. inject init containers, sort by their name, after the original init containers + sort.SliceStable(sidecarInitContainers, func(i, j int) bool { + return sidecarInitContainers[i].Name < sidecarInitContainers[j].Name + }) + for _, initContainer := range sidecarInitContainers { + pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer.Container) + } + // 2. inject containers + pod.Spec.Containers = mergeSidecarContainers(pod.Spec.Containers, sidecarContainers) + // 3. inject volumes + pod.Spec.Volumes = utils.MergeVolumes(pod.Spec.Volumes, volumesInSidecar) + // 4. inject imagePullSecrets + pod.Spec.ImagePullSecrets = mergeSidecarSecrets(pod.Spec.ImagePullSecrets, sidecarSecrets) + // 5. apply annotations + for k, v := range injectedAnnotations { + pod.Annotations[k] = v + } + klog.V(4).Infof("[sidecar inject] after mutating: %v", utils.DumpJSON(pod)) + return false, nil +} + +func mergeSidecarSecrets(secretsInPod, secretsInSidecar []corev1.LocalObjectReference) (allSecrets []corev1.LocalObjectReference) { + secretFilter := make(map[string]bool) + for _, podSecret := range secretsInPod { + if _, ok := secretFilter[podSecret.Name]; !ok { + secretFilter[podSecret.Name] = true + allSecrets = append(allSecrets, podSecret) + } + } + for _, sidecarSecret := range secretsInSidecar { + if _, ok := secretFilter[sidecarSecret.Name]; !ok { + secretFilter[sidecarSecret.Name] = true + allSecrets = append(allSecrets, sidecarSecret) + } + } + return allSecrets +} + +func mergeSidecarContainers(origins []corev1.Container, injected []*appsv1alpha1.SidecarContainer) []corev1.Container { + //format: pod.spec.containers[index].name -> index(the index of container in pod) + containersInPod := make(map[string]int) + for index, container := range origins { + containersInPod[container.Name] = index + } + var beforeAppContainers []corev1.Container + var afterAppContainers []corev1.Container + for _, sidecar := range injected { + //sidecar container already exist in pod + //keep the order of pod's original containers unchanged + if index, ok := containersInPod[sidecar.Name]; ok { + origins[index] = sidecar.Container + continue + } + + switch sidecar.PodInjectPolicy { + case appsv1alpha1.BeforeAppContainerType: + beforeAppContainers = append(beforeAppContainers, sidecar.Container) + case appsv1alpha1.AfterAppContainerType: + afterAppContainers = append(afterAppContainers, sidecar.Container) + default: + beforeAppContainers = append(beforeAppContainers, sidecar.Container) + } + } + origins = append(beforeAppContainers, origins...) + origins = append(origins, afterAppContainers...) + return origins +} + +func buildSidecars(control SidecarControl, pod *corev1.Pod, oldPod *corev1.Pod, matchedSidecarSets []*appsv1alpha1.SidecarSet) ( + sidecarContainers, sidecarInitContainers []*appsv1alpha1.SidecarContainer, sidecarSecrets []corev1.LocalObjectReference, + volumesInSidecars []corev1.Volume, injectedAnnotations map[string]string, err error) { + + // injected annotations + injectedAnnotations = make(map[string]string) + // get sidecarSet annotations from pods + // sidecarSet.name -> sidecarSet hash struct + sidecarSetHash := make(map[string]SidecarSetUpgradeSpec) + // sidecarSet.name -> sidecarSet hash(without image) struct + sidecarSetHashWithoutImage := make(map[string]SidecarSetUpgradeSpec) + // parse sidecar hash in pod annotations + if oldHashStr := pod.Annotations[SidecarSetHashAnnotation]; len(oldHashStr) > 0 { + if err = json.Unmarshal([]byte(oldHashStr), &sidecarSetHash); err != nil { + // to be compatible with older sidecarSet hash struct, map[string]string + olderSidecarSetHash := make(map[string]string) + if err = json.Unmarshal([]byte(oldHashStr), &olderSidecarSetHash); err != nil { + return nil, nil, nil, nil, nil, + fmt.Errorf("pod(%s/%s) invalid annotations[%s] value %v, unmarshal failed: %v", pod.Namespace, pod.Name, SidecarSetHashAnnotation, oldHashStr, err) + } + for k, v := range olderSidecarSetHash { + sidecarSetHash[k] = SidecarSetUpgradeSpec{ + SidecarSetHash: v, + SidecarSetName: k, + } + } + } + } + if oldHashStr := pod.Annotations[SidecarSetHashWithoutImageAnnotation]; len(oldHashStr) > 0 { + if err = json.Unmarshal([]byte(oldHashStr), &sidecarSetHashWithoutImage); err != nil { + // to be compatible with older sidecarSet hash struct, map[string]string + olderSidecarSetHash := make(map[string]string) + if err = json.Unmarshal([]byte(oldHashStr), &olderSidecarSetHash); err != nil { + return nil, nil, nil, nil, nil, + fmt.Errorf("pod(%s/%s) invalid annotations[%s] value %v, unmarshal failed: %v", pod.Namespace, pod.Name, SidecarSetHashWithoutImageAnnotation, oldHashStr, err) + } + for k, v := range olderSidecarSetHash { + sidecarSetHashWithoutImage[k] = SidecarSetUpgradeSpec{ + SidecarSetHash: v, + SidecarSetName: k, + } + } + } + } + // hotUpgrade work info, sidecarSet.spec.container[x].name -> pod.spec.container[x].name + // for example: mesh -> mesh-1, envoy -> envoy-2 + hotUpgradeWorkInfo := GetPodHotUpgradeInfoInAnnotations(pod) + // SidecarSet Name List, for example: log-sidecarset,envoy-sidecarset + sidecarSetNames := sets.NewString() + if sidecarSetListStr := pod.Annotations[SidecarSetListAnnotation]; sidecarSetListStr != "" { + sidecarSetNames.Insert(strings.Split(sidecarSetListStr, ",")...) + } + isUpdated := oldPod != nil + for _, sidecarSet := range matchedSidecarSets { + klog.V(3).Infof("build pod(%s/%s) sidecar containers for sidecarSet(%s)", pod.Namespace, pod.Name, sidecarSet.Name) + // sidecarSet List + sidecarSetNames.Insert(sidecarSet.Name) + // pre-process volumes only in sidecar + volumesMap := getVolumesMapInSidecarSet(sidecarSet) + // process sidecarset hash + setUpgrade1 := SidecarSetUpgradeSpec{ + UpdateTimestamp: metav1.Now(), + SidecarSetHash: GetSidecarSetRevision(sidecarSet), + SidecarSetName: sidecarSet.Name, + SidecarSetControllerRevision: sidecarSet.Status.LatestRevision, + } + setUpgrade2 := SidecarSetUpgradeSpec{ + UpdateTimestamp: metav1.Now(), + SidecarSetHash: GetSidecarSetWithoutImageRevision(sidecarSet), + SidecarSetName: sidecarSet.Name, + } + + //process initContainers + //only when created pod, inject initContainer and pullSecrets + if !isUpdated { + for i := range sidecarSet.Spec.InitContainers { + initContainer := &sidecarSet.Spec.InitContainers[i] + //add "IS_INJECTED" env in initContainer's envs + initContainer.Env = append(initContainer.Env, corev1.EnvVar{Name: SidecarEnvKey, Value: "true"}) + transferEnvs := GetSidecarTransferEnvs(initContainer, pod) + initContainer.Env = append(initContainer.Env, transferEnvs...) + sidecarInitContainers = append(sidecarInitContainers, initContainer) + // insert volumes that initContainers used + for _, mount := range initContainer.VolumeMounts { + volumesInSidecars = append(volumesInSidecars, *volumesMap[mount.Name]) + } + } + //process imagePullSecrets + sidecarSecrets = append(sidecarSecrets, sidecarSet.Spec.ImagePullSecrets...) + } + + sidecarList := sets.NewString() + isInjecting := false + //process containers + for i := range sidecarSet.Spec.Containers { + sidecarContainer := &sidecarSet.Spec.Containers[i] + sidecarList.Insert(sidecarContainer.Name) + // volumeMounts that injected into sidecar container + // when volumeMounts SubPathExpr contains expansions, then need copy container EnvVars(injectEnvs) + injectedMounts, injectedEnvs := GetInjectedVolumeMountsAndEnvs(control, sidecarContainer, pod) + // get injected env & mounts explicitly so that can be compared with old ones in pod + transferEnvs := GetSidecarTransferEnvs(sidecarContainer, pod) + // append volumeMounts SubPathExpr environments + transferEnvs = utils.MergeEnvVar(transferEnvs, injectedEnvs) + klog.Infof("try to inject sidecar %v@%v/%v, with injected envs: %v, volumeMounts: %v", + sidecarContainer.Name, pod.Namespace, pod.Name, transferEnvs, injectedMounts) + //when update pod object + if isUpdated { + // judge whether inject sidecar container into pod + needInject, existSidecars, existVolumes := control.NeedToInjectInUpdatedPod(pod, oldPod, sidecarContainer, transferEnvs, injectedMounts) + if !needInject { + sidecarContainers = append(sidecarContainers, existSidecars...) + volumesInSidecars = append(volumesInSidecars, existVolumes...) + continue + } + + klog.V(3).Infof("upgrade or insert sidecar container %v during upgrade in pod %v/%v", + sidecarContainer.Name, pod.Namespace, pod.Name) + //when created pod object, need inject sidecar container into pod + } else { + klog.V(3).Infof("inject new sidecar container %v during creation in pod %v/%v", + sidecarContainer.Name, pod.Namespace, pod.Name) + } + isInjecting = true + // insert volume that sidecar container used + for _, mount := range sidecarContainer.VolumeMounts { + volumesInSidecars = append(volumesInSidecars, *volumesMap[mount.Name]) + } + // merge VolumeMounts from sidecar.VolumeMounts and shared VolumeMounts + sidecarContainer.VolumeMounts = utils.MergeVolumeMounts(sidecarContainer.VolumeMounts, injectedMounts) + // add the "Injected" env to the sidecar container + sidecarContainer.Env = append(sidecarContainer.Env, corev1.EnvVar{Name: SidecarEnvKey, Value: "true"}) + // merged Env from sidecar.Env and transfer envs + sidecarContainer.Env = utils.MergeEnvVar(sidecarContainer.Env, transferEnvs) + + // when sidecar container UpgradeStrategy is HotUpgrade + if IsHotUpgradeContainer(sidecarContainer) { + hotContainers, annotations := injectHotUpgradeContainers(hotUpgradeWorkInfo, sidecarContainer) + sidecarContainers = append(sidecarContainers, hotContainers...) + for k, v := range annotations { + injectedAnnotations[k] = v + } + } else { + sidecarContainers = append(sidecarContainers, sidecarContainer) + } + } + // the container was (re)injected and the annotations need to be updated + if isInjecting { + setUpgrade1.SidecarList = sidecarList.List() + setUpgrade2.SidecarList = sidecarList.List() + sidecarSetHash[sidecarSet.Name] = setUpgrade1 + sidecarSetHashWithoutImage[sidecarSet.Name] = setUpgrade2 + } + } + + // store sidecarset hash in pod annotations + by, _ := json.Marshal(sidecarSetHash) + injectedAnnotations[SidecarSetHashAnnotation] = string(by) + by, _ = json.Marshal(sidecarSetHashWithoutImage) + injectedAnnotations[SidecarSetHashWithoutImageAnnotation] = string(by) + sidecarSetNameList := strings.Join(sidecarSetNames.List(), ",") + // store matched sidecarset list in pod annotations + injectedAnnotations[SidecarSetListAnnotation] = sidecarSetNameList + return sidecarContainers, sidecarInitContainers, sidecarSecrets, volumesInSidecars, injectedAnnotations, nil +} + +func getVolumesMapInSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) map[string]*corev1.Volume { + volumesMap := make(map[string]*corev1.Volume) + for idx, volume := range sidecarSet.Spec.Volumes { + volumesMap[volume.Name] = &sidecarSet.Spec.Volumes[idx] + } + return volumesMap +} diff --git a/pkg/control/sidecarcontrol/revision_adapter.go b/vendor/github.com/openkruise/utils/sidecarcontrol/revision_adapter.go similarity index 100% rename from pkg/control/sidecarcontrol/revision_adapter.go rename to vendor/github.com/openkruise/utils/sidecarcontrol/revision_adapter.go diff --git a/pkg/control/sidecarcontrol/sidecarset_control.go b/vendor/github.com/openkruise/utils/sidecarcontrol/sidecarset_control.go similarity index 71% rename from pkg/control/sidecarcontrol/sidecarset_control.go rename to vendor/github.com/openkruise/utils/sidecarcontrol/sidecarset_control.go index b2c6816710..45d5307601 100644 --- a/pkg/control/sidecarcontrol/sidecarset_control.go +++ b/vendor/github.com/openkruise/utils/sidecarcontrol/sidecarset_control.go @@ -21,40 +21,44 @@ import ( "github.com/openkruise/kruise/apis/apps/pub" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/util" - + "github.com/openkruise/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" ) -type commonControl struct { - *appsv1alpha1.SidecarSet -} - -func (c *commonControl) GetSidecarset() *appsv1alpha1.SidecarSet { - return c.SidecarSet +// NewCommonControl new sidecarSet control +// indexer is controllerRevision indexer +// If you need GetSuitableRevisionSidecarSet function, you must set indexer, namespace parameters +// otherwise you don't need to set any parameters +func NewCommonControl(indexer cache.Indexer, namespace string) SidecarControl { + c := &commonControl{} + if indexer != nil { + c.historyControl = NewHistoryControl(nil, indexer, namespace) + } + return c } -func (c *commonControl) IsActiveSidecarSet() bool { - return true +type commonControl struct { + historyControl HistoryControl } -func (c *commonControl) UpgradeSidecarContainer(sidecarContainer *appsv1alpha1.SidecarContainer, pod *v1.Pod) *v1.Container { +func (c *commonControl) UpgradeSidecarContainer(sidecarContainer *appsv1alpha1.SidecarContainer, pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet) *v1.Container { var nameToUpgrade, otherContainer, oldImage string if IsHotUpgradeContainer(sidecarContainer) { - nameToUpgrade, otherContainer = findContainerToHotUpgrade(sidecarContainer, pod, c) - oldImage = util.GetContainer(otherContainer, pod).Image + nameToUpgrade, otherContainer = c.FindContainerToHotUpgrade(sidecarContainer, pod, sidecarSet) + oldImage = utils.GetContainer(otherContainer, pod).Image } else { nameToUpgrade = sidecarContainer.Name - oldImage = util.GetContainer(nameToUpgrade, pod).Image + oldImage = utils.GetContainer(nameToUpgrade, pod).Image } // community in-place upgrades are only allowed to update image if sidecarContainer.Image == oldImage { return nil } - container := util.GetContainer(nameToUpgrade, pod) + container := utils.GetContainer(nameToUpgrade, pod) container.Image = sidecarContainer.Image klog.V(3).Infof("upgrade pod(%s/%s) container(%s) Image from(%s) -> to(%s)", pod.Namespace, pod.Name, nameToUpgrade, oldImage, container.Image) return container @@ -70,8 +74,7 @@ func (c *commonControl) NeedToInjectInUpdatedPod(pod, oldPod *v1.Pod, sidecarCon return false, nil, nil } -func (c *commonControl) IsPodReady(pod *v1.Pod) bool { - sidecarSet := c.GetSidecarset() +func (c *commonControl) IsPodReady(pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet) bool { // check whether hot upgrade is complete // map[string]string: {empty container name}->{sidecarSet.spec.containers[x].upgradeStrategy.HotUpgradeEmptyImage} emptyContainers := map[string]string{} @@ -92,11 +95,10 @@ func (c *commonControl) IsPodReady(pod *v1.Pod) bool { // 1. pod.Status.Phase == v1.PodRunning // 2. pod.condition PodReady == true - return util.IsRunningAndReady(pod) + return utils.IsRunningAndReady(pod) } -func (c *commonControl) UpdatePodAnnotationsInUpgrade(changedContainers []string, pod *v1.Pod) { - sidecarSet := c.GetSidecarset() +func (c *commonControl) UpdatePodAnnotationsInUpgrade(changedContainers []string, pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet) { // record the ImageID, before update pod sidecar container // if it is changed, indicates the update is complete. //format: sidecarset.name -> appsv1alpha1.InPlaceUpdateState @@ -140,18 +142,16 @@ func (c *commonControl) UpdatePodAnnotationsInUpgrade(changedContainers []string } // only check sidecar container is consistent -func (c *commonControl) IsPodStateConsistent(pod *v1.Pod, sidecarContainers sets.String) bool { +func (c *commonControl) IsPodStateConsistent(pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet, sidecarContainers sets.String) bool { if len(pod.Spec.Containers) != len(pod.Status.ContainerStatuses) { return false } - - sidecarset := c.GetSidecarset() if sidecarContainers.Len() == 0 { - sidecarContainers = GetSidecarContainersInPod(sidecarset) + sidecarContainers = GetSidecarContainersInPod(sidecarSet) } allDigestImage := true - cImageIDs := util.GetPodContainerImageIDs(pod) + cImageIDs := utils.GetPodContainerImageIDs(pod) for _, container := range pod.Spec.Containers { // only check whether sidecar container is consistent if !sidecarContainers.Has(container.Name) { @@ -160,7 +160,7 @@ func (c *commonControl) IsPodStateConsistent(pod *v1.Pod, sidecarContainers sets //whether image is digest format, //for example: docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d - if !util.IsImageDigest(container.Image) { + if !utils.IsImageDigest(container.Image) { allDigestImage = false break } @@ -169,7 +169,7 @@ func (c *commonControl) IsPodStateConsistent(pod *v1.Pod, sidecarContainers sets if !ok { return false } - if !util.IsContainerImageEqual(container.Image, imageID) { + if !utils.IsContainerImageEqual(container.Image, imageID) { return false } } @@ -179,17 +179,15 @@ func (c *commonControl) IsPodStateConsistent(pod *v1.Pod, sidecarContainers sets } // check container InpalceUpdate status - return IsSidecarContainerUpdateCompleted(pod, sets.NewString(sidecarset.Name), sidecarContainers) + return IsSidecarContainerUpdateCompleted(pod, sets.NewString(sidecarSet.Name), sidecarContainers) } // k8s only allow modify pod.spec.container[x].image, // only when annotations[SidecarSetHashWithoutImageAnnotation] is the same, sidecarSet can upgrade pods -func (c *commonControl) IsSidecarSetUpgradable(pod *v1.Pod) bool { - sidecarSet := c.GetSidecarset() +func (c *commonControl) IsSidecarSetUpgradable(pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet) bool { if GetPodSidecarSetWithoutImageRevision(sidecarSet.Name, pod) != GetSidecarSetWithoutImageRevision(sidecarSet) { return false } - // cStatus: container.name -> containerStatus.Ready cStatus := map[string]bool{} for _, status := range pod.Status.ContainerStatuses { @@ -200,7 +198,7 @@ func (c *commonControl) IsSidecarSetUpgradable(pod *v1.Pod) bool { // when containerStatus.Ready == true and container non-consistent, // indicates that sidecar container is in the process of being upgraded // wait for the last upgrade to complete before performing this upgrade - if cStatus[sidecar] && !c.IsPodStateConsistent(pod, sets.NewString(sidecar)) { + if cStatus[sidecar] && !c.IsPodStateConsistent(pod, sidecarSet, sets.NewString(sidecar)) { return false } } @@ -212,6 +210,37 @@ func (c *commonControl) IsPodAvailabilityChanged(pod, oldPod *v1.Pod) bool { return false } +// para1: nameToUpgrade, para2: otherContainer +func (c *commonControl) FindContainerToHotUpgrade(sidecarContainer *appsv1alpha1.SidecarContainer, pod *v1.Pod, sidecarSet *appsv1alpha1.SidecarSet) (string, string) { + containerInPods := make(map[string]v1.Container) + for _, containerInPod := range pod.Spec.Containers { + containerInPods[containerInPod.Name] = containerInPod + } + name1, name2 := GetHotUpgradeContainerName(sidecarContainer.Name) + c1, c2 := containerInPods[name1], containerInPods[name2] + + // First, empty hot sidecar container will be upgraded with the latest sidecarSet specification + if c1.Image == sidecarContainer.UpgradeStrategy.HotUpgradeEmptyImage { + return c1.Name, c2.Name + } else if c2.Image == sidecarContainer.UpgradeStrategy.HotUpgradeEmptyImage { + return c2.Name, c1.Name + } + + // Second, Not ready sidecar container will be upgraded + c1Ready := utils.GetContainerStatus(c1.Name, pod).Ready && c.IsPodStateConsistent(pod, sidecarSet, sets.NewString(c1.Name)) + c2Ready := utils.GetContainerStatus(c2.Name, pod).Ready && c.IsPodStateConsistent(pod, sidecarSet, sets.NewString(c2.Name)) + klog.V(3).Infof("pod(%s/%s) container(%s) ready(%v) container(%s) ready(%v)", pod.Namespace, pod.Name, c1.Name, c1Ready, c2.Name, c2Ready) + if c1Ready && !c2Ready { + return c2.Name, c1.Name + } else if !c1Ready && c2Ready { + return c1.Name, c2.Name + } + + // Third, the older sidecar container will be upgraded + workContianer, olderContainer := GetPodHotUpgradeContainers(sidecarContainer.Name, pod) + return olderContainer, workContianer +} + // isContainerInplaceUpdateCompleted checks whether imageID in container status has been changed since in-place update. // If the imageID in containerStatuses has not been changed, we assume that kubelet has not updated containers in Pod. func IsSidecarContainerUpdateCompleted(pod *v1.Pod, sidecarSets, containers sets.String) bool { @@ -265,3 +294,11 @@ func IsSidecarContainerUpdateCompleted(pod *v1.Pod, sidecarSets, containers sets return true } + +func (c *commonControl) GetSuitableRevisionSidecarSet(sidecarSet *appsv1alpha1.SidecarSet, oldPod, newPod *v1.Pod) (*appsv1alpha1.SidecarSet, error) { + // If historyControl is nil, then don't support get suitable revision sidecarSet, and return current obj. + if c.historyControl == nil { + return sidecarSet.DeepCopy(), nil + } + return c.historyControl.GetSuitableRevisionSidecarSet(sidecarSet, oldPod, newPod) +} diff --git a/pkg/webhook/pod/mutating/sidecarset_hotupgrade.go b/vendor/github.com/openkruise/utils/sidecarcontrol/sidecarset_hotupgrade.go similarity index 75% rename from pkg/webhook/pod/mutating/sidecarset_hotupgrade.go rename to vendor/github.com/openkruise/utils/sidecarcontrol/sidecarset_hotupgrade.go index 453cb1df41..75840a36da 100644 --- a/pkg/webhook/pod/mutating/sidecarset_hotupgrade.go +++ b/vendor/github.com/openkruise/utils/sidecarcontrol/sidecarset_hotupgrade.go @@ -14,15 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mutating +package sidecarcontrol import ( "encoding/json" "fmt" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/control/sidecarcontrol" - corev1 "k8s.io/api/core/v1" ) @@ -37,23 +35,23 @@ func injectHotUpgradeContainers(hotUpgradeWorkInfo map[string]string, sidecarCon sidecarContainers = append(sidecarContainers, container2) //mark sidecarset.version in annotations // "1" indicates sidecar container is first injected into pod, and not upgrade process - injectedAnnotations[sidecarcontrol.GetPodSidecarSetVersionAnnotation(container1.Name)] = "1" - injectedAnnotations[sidecarcontrol.GetPodSidecarSetVersionAltAnnotation(container1.Name)] = "0" + injectedAnnotations[GetPodSidecarSetVersionAnnotation(container1.Name)] = "1" + injectedAnnotations[GetPodSidecarSetVersionAltAnnotation(container1.Name)] = "0" // "0" indicates sidecar container is hot upgrade empty container - injectedAnnotations[sidecarcontrol.GetPodSidecarSetVersionAnnotation(container2.Name)] = "0" - injectedAnnotations[sidecarcontrol.GetPodSidecarSetVersionAltAnnotation(container2.Name)] = "1" + injectedAnnotations[GetPodSidecarSetVersionAnnotation(container2.Name)] = "0" + injectedAnnotations[GetPodSidecarSetVersionAltAnnotation(container2.Name)] = "1" // used to mark which container is currently working, first is container1 // format: map[container.name] = pod.spec.container[x].name hotUpgradeWorkInfo[sidecarContainer.Name] = container1.Name // store working HotUpgrade container in pod annotations by, _ := json.Marshal(hotUpgradeWorkInfo) - injectedAnnotations[sidecarcontrol.SidecarSetWorkingHotUpgradeContainer] = string(by) + injectedAnnotations[SidecarSetWorkingHotUpgradeContainer] = string(by) return sidecarContainers, injectedAnnotations } func generateHotUpgradeContainers(container *appsv1alpha1.SidecarContainer) (*appsv1alpha1.SidecarContainer, *appsv1alpha1.SidecarContainer) { - name1, name2 := sidecarcontrol.GetHotUpgradeContainerName(container.Name) + name1, name2 := GetHotUpgradeContainerName(container.Name) container1, container2 := container.DeepCopy(), container.DeepCopy() container1.Name = name1 container2.Name = name2 @@ -70,19 +68,19 @@ func generateHotUpgradeContainers(container *appsv1alpha1.SidecarContainer) (*ap func setSidecarContainerVersionEnv(container *corev1.Container) { // inject SIDECARSET_VERSION container.Env = append(container.Env, corev1.EnvVar{ - Name: sidecarcontrol.SidecarSetVersionEnvKey, + Name: SidecarSetVersionEnvKey, ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: fmt.Sprintf("metadata.annotations['%s']", sidecarcontrol.GetPodSidecarSetVersionAnnotation(container.Name)), + FieldPath: fmt.Sprintf("metadata.annotations['%s']", GetPodSidecarSetVersionAnnotation(container.Name)), }, }, }) // inject SIDECARSET_VERSION_ALT container.Env = append(container.Env, corev1.EnvVar{ - Name: sidecarcontrol.SidecarSetVersionAltEnvKey, + Name: SidecarSetVersionAltEnvKey, ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: fmt.Sprintf("metadata.annotations['%s']", sidecarcontrol.GetPodSidecarSetVersionAltAnnotation(container.Name)), + FieldPath: fmt.Sprintf("metadata.annotations['%s']", GetPodSidecarSetVersionAltAnnotation(container.Name)), }, }, }) diff --git a/pkg/control/sidecarcontrol/util.go b/vendor/github.com/openkruise/utils/sidecarcontrol/util.go similarity index 84% rename from pkg/control/sidecarcontrol/util.go rename to vendor/github.com/openkruise/utils/sidecarcontrol/util.go index 984a1466c0..025617ba91 100644 --- a/pkg/control/sidecarcontrol/util.go +++ b/vendor/github.com/openkruise/utils/sidecarcontrol/util.go @@ -25,10 +25,7 @@ import ( jsonpatch "github.com/evanphx/json-patch" appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/features" - "github.com/openkruise/kruise/pkg/util" - "github.com/openkruise/kruise/pkg/util/configuration" - utilfeature "github.com/openkruise/kruise/pkg/util/feature" + "github.com/openkruise/utils" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,9 +33,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/klog/v2" - kubecontroller "k8s.io/kubernetes/pkg/controller" - "k8s.io/kubernetes/pkg/fieldpath" - "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -76,13 +70,13 @@ type SidecarSetUpgradeSpec struct { } // PodMatchSidecarSet determines if pod match Selector of sidecar. -func PodMatchedSidecarSet(pod *corev1.Pod, sidecarSet appsv1alpha1.SidecarSet) (bool, error) { +func PodMatchedSidecarSet(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) (bool, error) { //If matchedNamespace is not empty, sidecarSet will only match the pods in the namespace if sidecarSet.Spec.Namespace != "" && sidecarSet.Spec.Namespace != pod.Namespace { return false, nil } // if selector not matched, then continue - selector, err := util.ValidatedLabelSelectorAsSelector(sidecarSet.Spec.Selector) + selector, err := utils.ValidatedLabelSelectorAsSelector(sidecarSet.Spec.Selector) if err != nil { return false, err } @@ -95,12 +89,9 @@ func PodMatchedSidecarSet(pod *corev1.Pod, sidecarSet appsv1alpha1.SidecarSet) ( // IsActivePod determines the pod whether need be injected and updated func IsActivePod(pod *corev1.Pod) bool { - /*for _, namespace := range SidecarIgnoredNamespaces { - if pod.Namespace == namespace { - return false - } - }*/ - return kubecontroller.IsPodActive(pod) + return corev1.PodSucceeded != pod.Status.Phase && + corev1.PodFailed != pod.Status.Phase && + pod.DeletionTimestamp == nil } func GetSidecarSetRevision(sidecarSet *appsv1alpha1.SidecarSet) string { @@ -223,7 +214,7 @@ func GetSidecarContainersInPod(sidecarSet *appsv1alpha1.SidecarSet) sets.String func GetPodsSortFunc(pods []*corev1.Pod, waitUpdateIndexes []int) func(i, j int) bool { // not-ready < ready, unscheduled < scheduled, and pending < running return func(i, j int) bool { - return kubecontroller.ActivePods(pods).Less(waitUpdateIndexes[i], waitUpdateIndexes[j]) + return utils.ActivePods(pods).Less(waitUpdateIndexes[i], waitUpdateIndexes[j]) } } @@ -242,12 +233,12 @@ func IsPodConsistentWithSidecarSet(pod *corev1.Pod, sidecarSet *appsv1alpha1.Sid switch container.UpgradeStrategy.UpgradeType { case appsv1alpha1.SidecarContainerHotUpgrade: _, exist := GetPodHotUpgradeInfoInAnnotations(pod)[container.Name] - if !exist || util.GetContainer(fmt.Sprintf("%v-1", container.Name), pod) == nil || - util.GetContainer(fmt.Sprintf("%v-2", container.Name), pod) == nil { + if !exist || utils.GetContainer(fmt.Sprintf("%v-1", container.Name), pod) == nil || + utils.GetContainer(fmt.Sprintf("%v-2", container.Name), pod) == nil { return false } default: - if util.GetContainer(container.Name, pod) == nil { + if utils.GetContainer(container.Name, pod) == nil { return false } } @@ -257,7 +248,7 @@ func IsPodConsistentWithSidecarSet(pod *corev1.Pod, sidecarSet *appsv1alpha1.Sid } func IsInjectedSidecarContainerInPod(container *corev1.Container) bool { - return util.GetContainerEnvValue(container, SidecarEnvKey) == "true" + return utils.GetContainerEnvValue(container, SidecarEnvKey) == "true" } func IsSharePodVolumeMounts(container *appsv1alpha1.SidecarContainer) bool { @@ -292,7 +283,7 @@ func GetInjectedVolumeMountsAndEnvs(control SidecarControl, sidecarContainer *ap // $(ODD_NAME) -> ODD_NAME envName := env[2 : len(env)-1] // get envVar in container - eVar := util.GetContainerEnvVar(&appContainer, envName) + eVar := utils.GetContainerEnvVar(&appContainer, envName) if eVar == nil { klog.Warningf("pod(%s/%s) container(%s) get env(%s) is nil", pod.Namespace, pod.Name, appContainer.Name, envName) continue @@ -354,7 +345,7 @@ func ExtractContainerNameFromFieldPath(fs *corev1.ObjectFieldSelector, pod *core if err != nil { return "", err } - path, subscript, ok := fieldpath.SplitMaybeSubscriptedPath(fieldPath) + path, subscript, ok := utils.SplitMaybeSubscriptedPath(fieldPath) if ok { switch path { case "metadata.annotations": @@ -383,7 +374,7 @@ func ConvertDownwardAPIFieldLabel(version, label, value string) (string, string, if version != "v1" { return "", "", fmt.Errorf("unsupported pod version: %s", version) } - path, _, ok := fieldpath.SplitMaybeSubscriptedPath(label) + path, _, ok := utils.SplitMaybeSubscriptedPath(label) if ok { switch path { case "metadata.annotations", "metadata.labels": @@ -452,62 +443,3 @@ func mergePatchJsonPodMetadata(originMetadata *metav1.ObjectMeta, patchPodField } return nil } - -func ValidateSidecarSetPatchMetadataWhitelist(c client.Client, sidecarSet *appsv1alpha1.SidecarSet) error { - if len(sidecarSet.Spec.PatchPodMetadata) == 0 { - return nil - } - - regAnnotations := make([]*regexp.Regexp, 0) - whitelist, err := configuration.GetSidecarSetPatchMetadataWhiteList(c) - if err != nil { - return err - } else if whitelist == nil { - if utilfeature.DefaultFeatureGate.Enabled(features.SidecarSetPatchPodMetadataDefaultsAllowed) { - return nil - } - return fmt.Errorf("SidecarSet patch metadata whitelist not found") - } - - for _, rule := range whitelist.Rules { - if rule.Selector != nil { - selector, err := util.ValidatedLabelSelectorAsSelector(rule.Selector) - if err != nil { - return err - } - if !selector.Matches(labels.Set(sidecarSet.Labels)) { - continue - } - } - for _, key := range rule.AllowedAnnotationKeyExprs { - reg, err := regexp.Compile(key) - if err != nil { - return err - } - regAnnotations = append(regAnnotations, reg) - } - } - if len(regAnnotations) == 0 { - if utilfeature.DefaultFeatureGate.Enabled(features.SidecarSetPatchPodMetadataDefaultsAllowed) { - return nil - } - return fmt.Errorf("sidecarSet patch metadata annotation is not allowed") - } - for _, patch := range sidecarSet.Spec.PatchPodMetadata { - for key := range patch.Annotations { - if !matchRegKey(key, regAnnotations) { - return fmt.Errorf("sidecarSet patch metadata annotation(%s) is not allowed", key) - } - } - } - return nil -} - -func matchRegKey(key string, regs []*regexp.Regexp) bool { - for _, reg := range regs { - if reg.MatchString(key) { - return true - } - } - return false -} diff --git a/pkg/control/sidecarcontrol/util_hotupgrade.go b/vendor/github.com/openkruise/utils/sidecarcontrol/util_hotupgrade.go similarity index 68% rename from pkg/control/sidecarcontrol/util_hotupgrade.go rename to vendor/github.com/openkruise/utils/sidecarcontrol/util_hotupgrade.go index f94cbceb32..9172babaa8 100644 --- a/pkg/control/sidecarcontrol/util_hotupgrade.go +++ b/vendor/github.com/openkruise/utils/sidecarcontrol/util_hotupgrade.go @@ -20,12 +20,8 @@ import ( "encoding/json" "fmt" - podutil "k8s.io/kubernetes/pkg/api/v1/pod" - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" ) @@ -100,34 +96,3 @@ func GetPodHotUpgradeContainers(sidecarName string, pod *corev1.Pod) (workContai return } - -// para1: nameToUpgrade, para2: otherContainer -func findContainerToHotUpgrade(sidecarContainer *appsv1alpha1.SidecarContainer, pod *corev1.Pod, control SidecarControl) (string, string) { - containerInPods := make(map[string]corev1.Container) - for _, containerInPod := range pod.Spec.Containers { - containerInPods[containerInPod.Name] = containerInPod - } - name1, name2 := GetHotUpgradeContainerName(sidecarContainer.Name) - c1, c2 := containerInPods[name1], containerInPods[name2] - - // First, empty hot sidecar container will be upgraded with the latest sidecarSet specification - if c1.Image == sidecarContainer.UpgradeStrategy.HotUpgradeEmptyImage { - return c1.Name, c2.Name - } else if c2.Image == sidecarContainer.UpgradeStrategy.HotUpgradeEmptyImage { - return c2.Name, c1.Name - } - - // Second, Not ready sidecar container will be upgraded - c1Ready := podutil.GetExistingContainerStatus(pod.Status.ContainerStatuses, c1.Name).Ready && control.IsPodStateConsistent(pod, sets.NewString(c1.Name)) - c2Ready := podutil.GetExistingContainerStatus(pod.Status.ContainerStatuses, c2.Name).Ready && control.IsPodStateConsistent(pod, sets.NewString(c2.Name)) - klog.V(3).Infof("pod(%s/%s) container(%s) ready(%v) container(%s) ready(%v)", pod.Namespace, pod.Name, c1.Name, c1Ready, c2.Name, c2Ready) - if c1Ready && !c2Ready { - return c2.Name, c1.Name - } else if !c1Ready && c2Ready { - return c1.Name, c2.Name - } - - // Third, the older sidecar container will be upgraded - workContianer, olderContainer := GetPodHotUpgradeContainers(sidecarContainer.Name, pod) - return olderContainer, workContianer -} diff --git a/vendor/github.com/openkruise/utils/tools.go b/vendor/github.com/openkruise/utils/tools.go new file mode 100644 index 0000000000..8d2d3cac68 --- /dev/null +++ b/vendor/github.com/openkruise/utils/tools.go @@ -0,0 +1,130 @@ +/* +Copyright 2019 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "encoding/json" + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "os" + "reflect" + "strings" + "unsafe" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" +) + +// DumpJSON returns the JSON encoding +func DumpJSON(o interface{}) string { + j, _ := json.Marshal(o) + return string(j) +} + +// SplitMaybeSubscriptedPath checks whether the specified fieldPath is +// subscripted, and +// - if yes, this function splits the fieldPath into path and subscript, and +// returns (path, subscript, true). +// - if no, this function returns (fieldPath, "", false). +// +// Example inputs and outputs: +// - "metadata.annotations['myKey']" --> ("metadata.annotations", "myKey", true) +// - "metadata.annotations['a[b]c']" --> ("metadata.annotations", "a[b]c", true) +// - "metadata.labels['']" --> ("metadata.labels", "", true) +// - "metadata.labels" --> ("metadata.labels", "", false) +func SplitMaybeSubscriptedPath(fieldPath string) (string, string, bool) { + if !strings.HasSuffix(fieldPath, "']") { + return fieldPath, "", false + } + s := strings.TrimSuffix(fieldPath, "']") + parts := strings.SplitN(s, "['", 2) + if len(parts) < 2 { + return fieldPath, "", false + } + if len(parts[0]) == 0 { + return fieldPath, "", false + } + return parts[0], parts[1], true +} + +func ValidatedLabelSelectorAsSelector(ps *metav1.LabelSelector) (labels.Selector, error) { + if ps == nil { + return labels.Nothing(), nil + } + if len(ps.MatchLabels)+len(ps.MatchExpressions) == 0 { + return labels.Everything(), nil + } + + selector := labels.NewSelector() + for k, v := range ps.MatchLabels { + r, err := newRequirement(k, selection.Equals, []string{v}) + if err != nil { + return nil, err + } + selector = selector.Add(*r) + } + for _, expr := range ps.MatchExpressions { + var op selection.Operator + switch expr.Operator { + case metav1.LabelSelectorOpIn: + op = selection.In + case metav1.LabelSelectorOpNotIn: + op = selection.NotIn + case metav1.LabelSelectorOpExists: + op = selection.Exists + case metav1.LabelSelectorOpDoesNotExist: + op = selection.DoesNotExist + default: + return nil, fmt.Errorf("%q is not a valid pod selector operator", expr.Operator) + } + r, err := newRequirement(expr.Key, op, append([]string(nil), expr.Values...)) + if err != nil { + return nil, err + } + selector = selector.Add(*r) + } + return selector, nil +} + +func newRequirement(key string, op selection.Operator, vals []string) (*labels.Requirement, error) { + sel := &labels.Requirement{} + selVal := reflect.ValueOf(sel) + val := reflect.Indirect(selVal) + + keyField := val.FieldByName("key") + keyFieldPtr := (*string)(unsafe.Pointer(keyField.UnsafeAddr())) + *keyFieldPtr = key + + opField := val.FieldByName("operator") + opFieldPtr := (*selection.Operator)(unsafe.Pointer(opField.UnsafeAddr())) + *opFieldPtr = op + + if len(vals) > 0 { + valuesField := val.FieldByName("strValues") + valuesFieldPtr := (*[]string)(unsafe.Pointer(valuesField.UnsafeAddr())) + *valuesFieldPtr = vals + } + + return sel, nil +} + +func GetNamespace() string { + if ns := os.Getenv("POD_NAMESPACE"); len(ns) > 0 { + return ns + } + return "kruise-system" +} diff --git a/vendor/modules.txt b/vendor/modules.txt index eb1d8c80b3..448f922a51 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -410,6 +410,10 @@ github.com/opencontainers/runtime-spec/specs-go github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label github.com/opencontainers/selinux/pkg/pwalk +# github.com/openkruise/utils v0.0.0-20221226104648-0361b39aa451 +## explicit; go 1.18 +github.com/openkruise/utils +github.com/openkruise/utils/sidecarcontrol # github.com/pkg/errors v0.9.1 ## explicit github.com/pkg/errors