From 40103b661f4af9014f10b5da81c53b5215a1248b Mon Sep 17 00:00:00 2001 From: pingjiang Date: Sun, 20 Aug 2023 23:13:22 +0800 Subject: [PATCH] SidecarSet add upgrade state in pod annotation Signed-off-by: pingjiang --- pkg/control/sidecarcontrol/util.go | 54 +++++++++++++++++++ .../sidecarset/sidecarset_processor.go | 54 +++++++++++++++++++ pkg/webhook/pod/mutating/sidecarset.go | 8 +++ 3 files changed, 116 insertions(+) diff --git a/pkg/control/sidecarcontrol/util.go b/pkg/control/sidecarcontrol/util.go index 4801e26f29..d694ae717a 100644 --- a/pkg/control/sidecarcontrol/util.go +++ b/pkg/control/sidecarcontrol/util.go @@ -64,6 +64,10 @@ const ( // SidecarSetUpgradable is a pod condition to indicate whether the pod's sidecarset is upgradable SidecarSetUpgradable corev1.PodConditionType = "SidecarSetUpgradable" + + // SidecarSetUpgradeSpec State + SidecarSetHashStateUpdating = "Updating" + SidecarSetHashStateNormal = "Normal" ) var ( @@ -81,6 +85,7 @@ type SidecarSetUpgradeSpec struct { SidecarSetName string `json:"sidecarSetName"` SidecarList []string `json:"sidecarList"` // sidecarSet container list SidecarSetControllerRevision string `json:"controllerRevision,omitempty"` // sidecarSet controllerRevision name + State string `json:"state"` // enum: Normal, Updating } // PodMatchSidecarSet determines if pod match Selector of sidecar. @@ -253,11 +258,60 @@ func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSe SidecarSetName: sidecarSet.Name, SidecarList: sidecarList.List(), SidecarSetControllerRevision: sidecarSet.Status.LatestRevision, + State: SidecarSetHashStateUpdating, } newHash, _ := json.Marshal(sidecarSetHash) pod.Annotations[hashKey] = string(newHash) } +func IsSiderCarContainersReady(pod *corev1.Pod, containers sets.String) bool { + for _, cs := range pod.Status.ContainerStatuses { + // only check containers set + if !containers.Has(cs.Name) { + continue + } + if !cs.Ready { + return false + } + } + return true +} + +func IsPodFinishSiderCarContainersUpdate(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) bool { + sidecars := GetSidecarContainersInPod(sidecarSet) + if _, ok := pod.Annotations[SidecarsetInplaceUpdateStateKey]; ok { + if IsSidecarContainerUpdateCompleted(pod, sets.NewString(sidecarSet.Name), sidecars) && IsSiderCarContainersReady(pod, sidecars) { + return true + } + } else { + if IsSiderCarContainersReady(pod, sidecars) { + return true + } + } + return false +} + +func GetPodSidecarSetHashState(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) string { + hashKey := SidecarSetHashAnnotation + sidecarSetHash := make(map[string]SidecarSetUpgradeSpec) + if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil { + klog.Errorf("unmarshal pod(%s/%s) annotations[%s] failed: %s", pod.Namespace, pod.Name, hashKey, err.Error()) + + // to be compatible with older sidecarSet hash struct, map[string]string + olderSidecarSetHash := make(map[string]string) + if err = json.Unmarshal([]byte(pod.Annotations[hashKey]), &olderSidecarSetHash); err == nil { + for k, v := range olderSidecarSetHash { + sidecarSetHash[k] = SidecarSetUpgradeSpec{ + SidecarSetHash: v, + UpdateTimestamp: metav1.Now(), + SidecarSetName: sidecarSet.Name, + } + } + } + } + return sidecarSetHash[sidecarSet.Name].State +} + func GetSidecarContainersInPod(sidecarSet *appsv1alpha1.SidecarSet) sets.String { names := sets.NewString() for _, sidecarContainer := range sidecarSet.Spec.Containers { diff --git a/pkg/controller/sidecarset/sidecarset_processor.go b/pkg/controller/sidecarset/sidecarset_processor.go index 54233be433..76d5a2d337 100644 --- a/pkg/controller/sidecarset/sidecarset_processor.go +++ b/pkg/controller/sidecarset/sidecarset_processor.go @@ -103,6 +103,13 @@ func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (recon return reconcile.Result{RequeueAfter: time.Second}, nil } + // check if pod finish sidecar container upgrade + for _, pod := range pods { + if sidecarcontrol.IsPodFinishSiderCarContainersUpdate(pod, sidecarSet) && sidecarcontrol.GetPodSidecarSetHashState(pod, sidecarSet) != "Normal" { + p.updatePodSidecarHashState(control, pod) + } + } + // 3. If sidecar container hot upgrade complete, then set the other one(empty sidecar container) image to HotUpgradeEmptyImage if isSidecarSetHasHotUpgradeContainer(sidecarSet) { var podsInHotUpgrading []*corev1.Pod @@ -189,6 +196,53 @@ func (p *Processor) updatePods(control sidecarcontrol.SidecarControl, pods []*co return nil } +func (p *Processor) updatePodSidecarHashState(control sidecarcontrol.SidecarControl, pod *corev1.Pod) error { + podClone := &corev1.Pod{} + sidecarSet := control.GetSidecarset() + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err := p.Client.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, podClone); err != nil { + klog.Errorf("sidecarset(%s) error getting updated pod %s/%s from client", control.GetSidecarset().Name, pod.Namespace, pod.Name) + } + + hashKey := sidecarcontrol.SidecarSetHashAnnotation + sidecarSetHash := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec) + if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil { + klog.Errorf("unmarshal pod(%s/%s) annotations[%s] failed: %s", pod.Namespace, pod.Name, hashKey, err.Error()) + + // to be compatible with older sidecarSet hash struct, map[string]string + olderSidecarSetHash := make(map[string]string) + if err = json.Unmarshal([]byte(pod.Annotations[hashKey]), &olderSidecarSetHash); err == nil { + for k, v := range olderSidecarSetHash { + sidecarSetHash[k] = sidecarcontrol.SidecarSetUpgradeSpec{ + SidecarSetHash: v, + UpdateTimestamp: metav1.Now(), + SidecarSetName: sidecarSet.Name, + } + } + } + } + + sidecarSetHash[sidecarSet.Name] = sidecarcontrol.SidecarSetUpgradeSpec{ + UpdateTimestamp: metav1.Now(), + SidecarSetHash: sidecarSetHash[sidecarSet.Name].SidecarSetHash, + SidecarSetName: sidecarSetHash[sidecarSet.Name].SidecarSetName, + SidecarList: sidecarSetHash[sidecarSet.Name].SidecarList, + SidecarSetControllerRevision: sidecarSetHash[sidecarSet.Name].SidecarSetControllerRevision, + State: sidecarcontrol.SidecarSetHashStateNormal, + } + newHash, _ := json.Marshal(sidecarSetHash) + podClone.Annotations[hashKey] = string(newHash) + // update pod in store + return p.Client.Update(context.TODO(), podClone) + }) + + if err != nil { + return err + } + + return nil +} + func (p *Processor) updatePodSidecarAndHash(control sidecarcontrol.SidecarControl, pod *corev1.Pod) error { podClone := &corev1.Pod{} sidecarSet := control.GetSidecarset() diff --git a/pkg/webhook/pod/mutating/sidecarset.go b/pkg/webhook/pod/mutating/sidecarset.go index df85689b04..87fd9dfb4b 100644 --- a/pkg/webhook/pod/mutating/sidecarset.go +++ b/pkg/webhook/pod/mutating/sidecarset.go @@ -450,6 +450,14 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS sidecarSetHash[sidecarSet.Name] = setUpgrade1 sidecarSetHashWithoutImage[sidecarSet.Name] = setUpgrade2 } + sidecarSetHash[sidecarSet.Name] = sidecarcontrol.SidecarSetUpgradeSpec{ + UpdateTimestamp: metav1.Now(), + SidecarSetHash: sidecarSetHash[sidecarSet.Name].SidecarSetHash, + SidecarSetName: sidecarSet.Name, + SidecarList: sidecarList.List(), + SidecarSetControllerRevision: sidecarSetHash[sidecarSet.Name].SidecarSetControllerRevision, + State: sidecarcontrol.SidecarSetHashStateUpdating, + } } // store sidecarset hash in pod annotations