Skip to content

Commit

Permalink
SidecarSet add upgrade state in pod annotation
Browse files Browse the repository at this point in the history 
Signed-off-by: pingjiang <xiangpingjiang1998@gmail.com>
  • Loading branch information
@xiangpingjiang
xiangpingjiang committed Aug 4, 2023
1 parent 4e35a1d commit 5088706
Show file tree
Hide file tree
Showing 4 changed files with 201 additions and 0 deletions.
71 changes: 71 additions & 0 deletions pkg/control/sidecarcontrol/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ type SidecarSetUpgradeSpec struct {
SidecarSetName string `json:"sidecarSetName"`
SidecarList []string `json:"sidecarList"` // sidecarSet container list
SidecarSetControllerRevision string `json:"controllerRevision,omitempty"` // sidecarSet controllerRevision name
State string `json:"state"` // enum: Normal, Updating
}

// PodMatchSidecarSet determines if pod match Selector of sidecar.
Expand Down Expand Up @@ -253,11 +254,81 @@ func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSe
SidecarSetName: sidecarSet.Name,
SidecarList: sidecarList.List(),
SidecarSetControllerRevision: sidecarSet.Status.LatestRevision,
State: "Updating",
}
newHash, _ := json.Marshal(sidecarSetHash)
pod.Annotations[hashKey] = string(newHash)
}

func UpdatePodSidecarSetHashState(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) {
hashKey := SidecarSetHashAnnotation
sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil {
klog.Errorf("unmarshal pod(%s/%s) annotations[%s] failed: %s", pod.Namespace, pod.Name, hashKey, err.Error())

// to be compatible with older sidecarSet hash struct, map[string]string
olderSidecarSetHash := make(map[string]string)
if err = json.Unmarshal([]byte(pod.Annotations[hashKey]), &olderSidecarSetHash); err == nil {
for k, v := range olderSidecarSetHash {
sidecarSetHash[k] = SidecarSetUpgradeSpec{
SidecarSetHash: v,
UpdateTimestamp: metav1.Now(),
SidecarSetName: sidecarSet.Name,
}
}
}
}

sidecarList := sets.NewString()
for _, sidecar := range sidecarSet.Spec.Containers {
sidecarList.Insert(sidecar.Name)
}

sidecarSetHash[sidecarSet.Name] = SidecarSetUpgradeSpec{
UpdateTimestamp: metav1.Now(),
SidecarSetHash: sidecarSetHash[sidecarSet.Name].SidecarSetHash,
SidecarSetName: sidecarSet.Name,
SidecarList: sidecarList.List(),
SidecarSetControllerRevision: sidecarSetHash[sidecarSet.Name].SidecarSetControllerRevision,
State: "Normal",
}
newHash, _ := json.Marshal(sidecarSetHash)
pod.Annotations[hashKey] = string(newHash)
}

func UpdateSidecarSetHashState(sidecarSetHash map[string]SidecarSetUpgradeSpec, sidecarSetName string, State string) {
upgradeSpec := sidecarSetHash[sidecarSetName]
upgradeSpec.State = State
sidecarSetHash[sidecarSetName] = upgradeSpec
}

func IsSiderCarContainersReady(pod *corev1.Pod, containers sets.String) bool {
for _, cs := range pod.Status.ContainerStatuses {
// only check containers set
if !containers.Has(cs.Name) {
continue
}
if !cs.Ready {
return false
}
}
return true
}

func IsPodFinishSiderCarContainersUpdate(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) bool {
sidecars := GetSidecarContainersInPod(sidecarSet)
if _, ok := pod.Annotations[SidecarsetInplaceUpdateStateKey]; ok {
if IsSidecarContainerUpdateCompleted(pod, sets.NewString(sidecarSet.Name), sidecars) && IsSiderCarContainersReady(pod, sidecars) {
return true
}
} else {
if IsSiderCarContainersReady(pod, sidecars) {
return true
}
}
return false
}

func GetSidecarContainersInPod(sidecarSet *appsv1alpha1.SidecarSet) sets.String {
names := sets.NewString()
for _, sidecarContainer := range sidecarSet.Spec.Containers {
Expand Down
122 changes: 122 additions & 0 deletions pkg/control/sidecarcontrol/util_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,128 @@ func TestUpdatePodSidecarSetHash(t *testing.T) {
}
}

func TestUpdatePodSidecarSetHashState(t *testing.T) {
cases := []struct {
name string
getPod func() *corev1.Pod
getSidecarSet func() *appsv1alpha1.SidecarSet
exceptRevision map[string]SidecarSetUpgradeSpec
exceptWithoutImageRevision map[string]SidecarSetUpgradeSpec
}{
{
name: "sidecarContainer inplaceUpdate, not ready",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
pod.Annotations[SidecarsetInplaceUpdateStateKey] = `{"test-sidecarset": {"revision":"new-revision","lastContainerStatuses":{"sidecar-mesh":{"imageID":"envoy@sha256:1ba0da74b20aad52b091877b0e0ece503c563f39e37aa6b0e46777c4d820a2ae"}}}}`
pod.Status.ContainerStatuses[1].Ready = false
return pod
},
getSidecarSet: func() *appsv1alpha1.SidecarSet {
return sidecarSetDemo.DeepCopy()
},
exceptRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
SidecarSetHash: "bbb",
},
},
exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
State: "Updating",
},
},
},
{
name: "sidecarContainer inplaceUpdate, ready",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
pod.Annotations[SidecarsetInplaceUpdateStateKey] = `{"test-sidecarset": {"revision":"new-revision","lastContainerStatuses":{"sidecar-mesh":{"imageID":"envoy@sha256:1ba0da74b20aad52b091877b0e0ece503c563f39e37aa6b0e46777c4d820a2ae"}}}}`
return pod
},
getSidecarSet: func() *appsv1alpha1.SidecarSet {
return sidecarSetDemo.DeepCopy()
},
exceptRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
SidecarSetHash: "bbb",
},
},
exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
State: "Normal",
},
},
},
{
name: "create pod, sidecarContainer not ready",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset": "without-image-aaa"}`
pod.Status.ContainerStatuses[1].Ready = false
return pod
},
getSidecarSet: func() *appsv1alpha1.SidecarSet {
return sidecarSetDemo.DeepCopy()
},
exceptRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
SidecarSetHash: "bbb",
},
},
exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
State: "Updating",
},
},
},
{
name: "create pod, sidecarContainer ready",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
return pod
},
getSidecarSet: func() *appsv1alpha1.SidecarSet {
return sidecarSetDemo.DeepCopy()
},
exceptRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
SidecarSetHash: "bbb",
},
},
exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
State: "Normal",
},
},
},
}

for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
podInput := cs.getPod()
sidecarSetInput := cs.getSidecarSet()
UpdatePodSidecarSetHash(podInput, sidecarSetInput)
sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
err := json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashAnnotation]), &sidecarSetHash)
if err != nil {
t.Fatalf("parse pod sidecarSet hash failed: %s", err.Error())
}
for k, o := range sidecarSetHash {
eo := cs.exceptRevision[k]
if o.SidecarSetHash != eo.SidecarSetHash {
t.Fatalf("except sidecar container %s revision %s, but get revision %s", k, eo.SidecarSetHash, o.SidecarSetHash)
}
}
})
}
}

func TestConvertDownwardAPIFieldLabel(t *testing.T) {
testCases := []struct {
version string
Expand Down
7 changes: 7 additions & 0 deletions pkg/controller/sidecarset/sidecarset_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,13 @@ func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (recon
klog.V(3).Infof("sidecarset %s matched pods has some update in flight: %v, will sync later", sidecarSet.Name, inflightPods)
return reconcile.Result{RequeueAfter: time.Second}, nil
}
// check sideCar finish upgrade, update PodSidecarSetHashState
for _, pod := range pods {
if sidecarcontrol.IsPodFinishSiderCarContainersUpdate(pod, sidecarSet) {
sidecarcontrol.UpdatePodSidecarSetHashState(pod, sidecarSet)
p.Client.Update(context.TODO(), pod)
}
}

// 3. If sidecar container hot upgrade complete, then set the other one(empty sidecar container) image to HotUpgradeEmptyImage
if isSidecarSetHasHotUpgradeContainer(sidecarSet) {
Expand Down
1 change: 1 addition & 0 deletions pkg/webhook/pod/mutating/sidecarset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,7 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
sidecarSetHash[sidecarSet.Name] = setUpgrade1
sidecarSetHashWithoutImage[sidecarSet.Name] = setUpgrade2
}
sidecarcontrol.UpdateSidecarSetHashState(sidecarSetHash, sidecarSet.Name, "Updating")
}

// store sidecarset hash in pod annotations
Expand Down

0 comments on commit 5088706

Please sign in to comment.