sidecarset inject history revision

Signed-off-by: mingzhou.swx <mingzhou.swx@alibaba-inc.com>

apis/apps/defaults/v1alpha1.go

@@ -43,6 +43,15 @@ func SetDefaultsSidecarSet(obj *v1alpha1.SidecarSet) {
 
 	//default setting history revision limitation
 	SetDefaultRevisionHistoryLimit(&obj.Spec.RevisionHistoryLimit)
+
+	//default setting injectRevisionStrategy
+	SetDefaultInjectRevision(&obj.Spec.InjectionStrategy)
+}
+
+func SetDefaultInjectRevision(strategy *v1alpha1.SidecarSetInjectionStrategy) {
+	if strategy.Revision != nil && strategy.Revision.Policy == "" {
+		strategy.Revision.Policy = v1alpha1.AlwaysSidecarSetInjectRevisionPolicy
+	}
 }
 
 func SetDefaultRevisionHistoryLimit(revisionHistoryLimit **int32) {

apis/apps/v1alpha1/sidecarset_types.go

@@ -142,8 +142,34 @@ type SidecarSetInjectionStrategy struct {
 	// but the injected sidecar container remains updating and running.
 	// default is false
 	Paused bool `json:"paused,omitempty"`
+
+	// Revision can help users rolling update SidecarSet safely. If users set
+	// this filed, SidecarSet will try to inject specific revision according to
+	// different policies.
+	Revision *SidecarSetInjectRevision `json:"revision,omitempty"`
+}
+
+type SidecarSetInjectRevision struct {
+	// ID corresponds to label 'apps.kruise.io/sidecarset-revision-id' of (History) SidecarSet.
+	// SidecarSet will select the specific ControllerRevision via this ID, and then restore the
+	// history SidecarSet to inject specific version of the sidecar to pods.
+	ID string `json:"id"`
+	// Policy describes the behavior of revision injection.
+	// Defaults to Always.
+	Policy SidecarSetInjectRevisionPolicy `json:"policy,omitempty"`
 }
 
+type SidecarSetInjectRevisionPolicy string
+
+const (
+	// AlwaysSidecarSetInjectRevisionPolicy means the SidecarSet will always inject
+	// the specific revision to Pods when pod creating, except matching UpdateStrategy.Selector.
+	AlwaysSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Always"
+	// AdaptiveSidecarSetInjectRevisionPolicy means the SidecarSet will inject the
+	// specific or the latest revision according to Partition.
+	//AdaptiveSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Adaptive"
+)
+
 // SidecarSetUpdateStrategy indicates the strategy that the SidecarSet
 // controller will use to perform updates. It includes any additional parameters
 // necessary to perform the update for the indicated strategy.

config/crd/bases/apps.kruise.io_sidecarsets.yaml

@@ -229,6 +229,25 @@ spec:
                       to newly created Pods, but the injected sidecar container remains
                       updating and running. default is false
                     type: boolean
+                  revision:
+                    description: Revision can help users rolling update SidecarSet
+                      safely. If users set this filed, SidecarSet will try to inject
+                      specific revision according to different policies.
+                    properties:
+                      id:
+                        description: ID corresponds to label 'apps.kruise.io/sidecarset-revision-id'
+                          of (History) SidecarSet. SidecarSet will select the specific
+                          ControllerRevision via this ID, and then restore the history
+                          SidecarSet to inject specific version of the sidecar to
+                          pods.
+                        type: string
+                      policy:
+                        description: Policy describes the behavior of revision injection.
+                          Defaults to Always.
+                        type: string
+                    required:
+                    - id
+                    type: object
                 type: object
               namespace:
                 description: Namespace sidecarSet will only match the pods in the

pkg/webhook/sidecarset/mutating/hash.go → pkg/control/sidecarcontrol/hash.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package mutating
+package sidecarcontrol
 
 import (
 	"crypto/sha256"
@@ -66,3 +66,17 @@ func encodeSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (string, error) {
 func hash(data string) string {
 	return fmt.Sprintf("%x", sha256.Sum256([]byte(data)))
 }
+
+func CalculateSidecarSetHash(sidecarSet *appsv1alpha1.SidecarSet) error {
+	hashCodeWithImage, err := SidecarSetHash(sidecarSet)
+	if err != nil {
+		return err
+	}
+	hashCodeWithoutImage, err := SidecarSetHashWithoutImage(sidecarSet)
+	if err != nil {
+		return err
+	}
+	sidecarSet.Annotations[SidecarSetHashAnnotation] = hashCodeWithImage
+	sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = hashCodeWithoutImage
+	return nil
+}

pkg/control/sidecarcontrol/history_control.go

@@ -23,13 +23,16 @@ import (
 	"fmt"
 
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
-
+	webhookutil "github.com/openkruise/kruise/pkg/webhook/util"
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/controller/history"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -43,6 +46,7 @@ type HistoryControl interface {
 	NewRevision(s *appsv1alpha1.SidecarSet, namespace string, revision int64, collisionCount *int32) (*apps.ControllerRevision, error)
 	NextRevision(revisions []*apps.ControllerRevision) int64
 	GetRevisionLabelSelector(s *appsv1alpha1.SidecarSet) *metav1.LabelSelector
+	GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionName string) (*appsv1alpha1.SidecarSet, error)
 }
 
 type realControl struct {
@@ -80,6 +84,15 @@ func (r *realControl) NewRevision(s *appsv1alpha1.SidecarSet, namespace string,
 	if cr.ObjectMeta.Annotations == nil {
 		cr.ObjectMeta.Annotations = make(map[string]string)
 	}
+	if s.Annotations[SidecarSetHashAnnotation] != "" {
+		cr.Annotations[SidecarSetHashAnnotation] = s.Annotations[SidecarSetHashAnnotation]
+	}
+	if s.Annotations[SidecarSetHashWithoutImageAnnotation] != "" {
+		cr.Annotations[SidecarSetHashWithoutImageAnnotation] = s.Annotations[SidecarSetHashWithoutImageAnnotation]
+	}
+	if s.Labels[SidecarSetRevisionIDLabel] != "" {
+		cr.Labels[SidecarSetRevisionIDLabel] = s.Labels[SidecarSetRevisionIDLabel]
+	}
 	cr.Labels[SidecarSetKindName] = s.Name
 	for key, value := range s.Annotations {
 		cr.ObjectMeta.Annotations[key] = value
@@ -162,6 +175,55 @@ func (r *realControl) CreateControllerRevision(parent metav1.Object, revision *a
 	}
 }
 
+func (r *realControl) GetHistorySidecarSet(sidecarSet *appsv1alpha1.SidecarSet, revisionID string) (*appsv1alpha1.SidecarSet, error) {
+	if revisionID == "" {
+		return nil, generateNotFoundError(sidecarSet)
+	}
+
+	listOpts := []client.ListOption{
+		client.InNamespace(webhookutil.GetNamespace()),
+		client.MatchingLabels{SidecarSetRevisionIDLabel: revisionID},
+	}
+	revisionList := &apps.ControllerRevisionList{}
+	if err := r.Client.List(context.TODO(), revisionList, listOpts...); err != nil {
+		klog.Errorf("Failed to get ControllerRevision ID: %s, err %v", revisionID, err)
+		return nil, err
+	}
+
+	var revisions []*apps.ControllerRevision
+	for i := range revisionList.Items {
+		revisions = append(revisions, &revisionList.Items[i])
+	}
+
+	if len(revisions) == 0 {
+		return nil, generateNotFoundError(sidecarSet)
+	}
+	history.SortControllerRevisions(revisions)
+	revision := revisions[len(revisions)-1]
+
+	// calculate patch
+	clone := sidecarSet.DeepCopy()
+	cloneBytes, err := runtime.Encode(patchCodec, clone)
+	if err != nil {
+		klog.Errorf("Failed to encode sidecarSet(%v), error: %v", sidecarSet.Name, err)
+		return nil, err
+	}
+	patched, err := strategicpatch.StrategicMergePatch(cloneBytes, revision.Data.Raw, clone)
+	if err != nil {
+		klog.Errorf("Failed to merge sidecarSet(%v) and controllerRevision, ID: %v, error: %v", sidecarSet.Name, revisionID, err)
+		return nil, err
+	}
+	// restore history from patch
+	restoredSidecarSet := &appsv1alpha1.SidecarSet{}
+	if err := json.Unmarshal(patched, restoredSidecarSet); err != nil {
+		return nil, err
+	}
+	if err := restoreHashAnnotations(restoredSidecarSet, revision); err != nil {
+		return nil, err
+	}
+	return restoredSidecarSet, nil
+}
+
 func copySidecarSetSpecRevision(dst, src map[string]interface{}) {
 	// we will use patch instead of update operation to update pods in the future
 	// dst["$patch"] = "replace"
@@ -171,3 +233,35 @@ func copySidecarSetSpecRevision(dst, src map[string]interface{}) {
 	dst["initContainers"] = src["initContainers"]
 	dst["imagePullSecrets"] = src["imagePullSecrets"]
 }
+
+func restoreHashAnnotations(sidecarSet *appsv1alpha1.SidecarSet, revision *apps.ControllerRevision) error {
+	if sidecarSet.Annotations == nil {
+		sidecarSet.Annotations = map[string]string{}
+	}
+	if revision.Annotations[SidecarSetHashAnnotation] != "" {
+		sidecarSet.Annotations[SidecarSetHashAnnotation] = revision.Annotations[SidecarSetHashAnnotation]
+	} else {
+		hashCodeWithImage, err := SidecarSetHash(sidecarSet)
+		if err != nil {
+			return err
+		}
+		sidecarSet.Annotations[SidecarSetHashAnnotation] = hashCodeWithImage
+	}
+	if revision.Annotations[SidecarSetHashWithoutImageAnnotation] != "" {
+		sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = revision.Annotations[SidecarSetHashWithoutImageAnnotation]
+	} else {
+		hashCodeWithoutImage, err := SidecarSetHashWithoutImage(sidecarSet)
+		if err != nil {
+			return err
+		}
+		sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = hashCodeWithoutImage
+	}
+	return nil
+}
+
+func generateNotFoundError(set *appsv1alpha1.SidecarSet) error {
+	return errors.NewNotFound(schema.GroupResource{
+		Group:    apps.GroupName,
+		Resource: "ControllerRevision",
+	}, set.Name)
+}

pkg/control/sidecarcontrol/util.go

@@ -54,6 +54,8 @@ const (
 	// SidecarsetInplaceUpdateStateKey records the state of inplace-update.
 	// The value of annotation is SidecarsetInplaceUpdateStateKey.
 	SidecarsetInplaceUpdateStateKey string = "kruise.io/sidecarset-inplace-update-state"
+
+	SidecarSetRevisionIDLabel = "apps.kruise.io/sidecarset-revision-id"
 )
 
 var (

pkg/controller/sidecarset/sidecarset_pod_event_handler.go

@@ -2,6 +2,7 @@ package sidecarset
 
 import (
 	"context"
+	"reflect"
 	"strings"
 	"time"
 
@@ -179,5 +180,12 @@ func isPodConsistentChanged(oldPod, newPod *corev1.Pod, sidecarSet *appsv1alpha1
 		return true, enqueueDelayTime
 	}
 
+	// If the pod's labels changed, and sidecarSet enable selector updateStrategy, should reconcile.
+	if !reflect.DeepEqual(oldPod.Labels, newPod.Labels) && sidecarSet.Spec.UpdateStrategy.Selector != nil {
+		klog.V(3).Infof("pod(%s/%s) Labels changed and sidecarSet (%s) enable selector upgrade strategy, "+
+			"and reconcile sidecarSet", newPod.Namespace, newPod.Name, sidecarSet.Name)
+		return true, 0
+	}
+
 	return false, enqueueDelayTime
 }

pkg/controller/sidecarset/sidecarset_processor.go

@@ -369,6 +369,11 @@ func (p *Processor) registerLatestRevision(sidecarSet *appsv1alpha1.SidecarSet,
 		revisions = append(revisions, latestRevision)
 	}
 
+	// update revision id for the latest controller revision
+	if err = p.updateRevisionID(latestRevision, sidecarSet.Labels[sidecarcontrol.SidecarSetRevisionIDLabel]); err != nil {
+		return nil, collisionCount, err
+	}
+
 	// only store limited history revisions
 	if err = p.truncateHistory(revisions, sidecarSet, pods); err != nil {
 		klog.Errorf("Failed to truncate history for %s: err: %v", sidecarSet.Name, err)
@@ -377,6 +382,19 @@ func (p *Processor) registerLatestRevision(sidecarSet *appsv1alpha1.SidecarSet,
 	return latestRevision, collisionCount, nil
 }
 
+func (p *Processor) updateRevisionID(revision *apps.ControllerRevision, updateRevisionID string) error {
+	if updateRevisionID != "" && updateRevisionID != revision.Labels[sidecarcontrol.SidecarSetRevisionIDLabel] {
+		revisionCLone := revision.DeepCopy()
+		patchBody := fmt.Sprintf(`{"metadata":{"labels":{"%v":"%v"}}}`, sidecarcontrol.SidecarSetRevisionIDLabel, updateRevisionID)
+		err := p.Client.Patch(context.TODO(), revisionCLone, client.RawPatch(types.StrategicMergePatchType, []byte(patchBody)))
+		if err != nil {
+			klog.Errorf("Failed to patch revision id to latest revision %v, err: %v", revision.Name, err)
+			return err
+		}
+	}
+	return nil
+}
+
 func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *appsv1alpha1.SidecarSet, pods []*corev1.Pod) error {
 	// We do not delete the latest revision because we are using it.
 	// Thus, we must ensure the limitation is bounded, minimum value is 1.
@@ -395,9 +413,9 @@ func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *app
 	// the number of revisions need to delete
 	deletionCount := revisionCount - limitation
 	// only delete the revisions that no pods use.
-	activeRevisions := filterActiveRevisions(s, pods)
+	activeRevisions := filterActiveRevisions(s, pods, revisions)
 	for i := 0; i < revisionCount-1 && deletionCount > 0; i++ {
-		if !activeRevisions.Has(revisions[i].Name) { // && revision.InjectionStrategy.ControllerRevision != revisions[i].Name
+		if !activeRevisions.Has(revisions[i].Name) {
 			if err := p.historyController.DeleteControllerRevision(revisions[i]); err != nil && !errors.IsNotFound(err) {
 				return err
 			}
@@ -412,13 +430,28 @@ func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *app
 	return nil
 }
 
-func filterActiveRevisions(s *appsv1alpha1.SidecarSet, pods []*corev1.Pod) sets.String {
+func filterActiveRevisions(s *appsv1alpha1.SidecarSet, pods []*corev1.Pod, revisions []*apps.ControllerRevision) sets.String {
 	activeRevisions := sets.NewString()
 	for _, pod := range pods {
 		if revision := sidecarcontrol.GetPodSidecarSetControllerRevision(s.Name, pod); revision != "" {
 			activeRevisions.Insert(revision)
 		}
 	}
+
+	if s.Spec.InjectionStrategy.Revision != nil {
+		equalRevisions := make([]*apps.ControllerRevision, 0)
+		for i := range revisions {
+			revision := revisions[i]
+			if revision.Labels[sidecarcontrol.SidecarSetRevisionIDLabel] == s.Spec.InjectionStrategy.Revision.ID {
+				equalRevisions = append(equalRevisions, revision)
+			}
+		}
+		if len(equalRevisions) > 0 {
+			history.SortControllerRevisions(equalRevisions)
+			activeRevisions.Insert(equalRevisions[len(equalRevisions)-1].Name)
+		}
+	}
+
 	return activeRevisions
 }