diff --git a/pkg/controllers/alibabacloudslb/alibabacloudslb_controller.go b/pkg/controllers/alibabacloudslb/alibabacloudslb_controller.go index 15376969..d2aeaec8 100644 --- a/pkg/controllers/alibabacloudslb/alibabacloudslb_controller.go +++ b/pkg/controllers/alibabacloudslb/alibabacloudslb_controller.go @@ -28,6 +28,7 @@ import ( ) var _ resourceconsist.ReconcileAdapter = &ReconcileAdapter{} +var _ resourceconsist.ReconcileLifecycleOptions = &ReconcileAdapter{} type ReconcileAdapter struct { client.Client @@ -49,12 +50,16 @@ func NewReconcileAdapter(c client.Client) (*ReconcileAdapter, error) { }, nil } -func (r *ReconcileAdapter) GetControllerName() string { - return "alibaba-cloud-slb-controller" +func (r *ReconcileAdapter) FollowPodOpsLifeCycle() bool { + return true +} + +func (r *ReconcileAdapter) NeedRecordEmployees() bool { + return true } -func (r *ReconcileAdapter) NotFollowPodOpsLifeCycle() bool { - return false +func (r *ReconcileAdapter) GetControllerName() string { + return "alibaba-cloud-slb-controller" } func (r *ReconcileAdapter) GetExpectedEmployer(ctx context.Context, employer client.Object) ([]resourceconsist.IEmployer, error) { diff --git a/pkg/controllers/resourceconsist/consister.go b/pkg/controllers/resourceconsist/consister.go index 028782b0..bfbf7934 100644 --- a/pkg/controllers/resourceconsist/consister.go +++ b/pkg/controllers/resourceconsist/consister.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "strings" corev1 "k8s.io/api/core/v1" @@ -201,6 +202,26 @@ func (r *Consist) syncEmployees(ctx context.Context, employer client.Object, exp toAddLifecycleFlzEmployees, toDeleteLifecycleFlzEmployees := r.getToAddDeleteLifecycleFlzEmployees( succCreate, succDelete, succUpdate, toCudEmployees.Unchanged) + recordedNotSelected := make(map[string]bool) + lifecycleOptions, lifecycleOptionsImplemented := r.adapter.(ReconcileLifecycleOptions) + needRecordEmployees := lifecycleOptionsImplemented && lifecycleOptions.FollowPodOpsLifeCycle() && lifecycleOptions.NeedRecordEmployees() + if needRecordEmployees { + if employer.GetAnnotations()[lifecycleFinalizerRecordedAnnoKey] != "" { + selectedEmployees, err := r.adapter.GetSelectedEmployeeNames(ctx, employer) + if err != nil { + return false, false, fmt.Errorf("GetSelectedEmployeeNames failed, err: %s", err.Error()) + } + recordedEmployees := strings.Split(employer.GetAnnotations()[lifecycleFinalizerRecordedAnnoKey], ",") + selectedSet := sets.NewString(selectedEmployees...) + for _, recordedEmployee := range recordedEmployees { + if !selectedSet.Has(recordedEmployee) { + recordedNotSelected[recordedEmployee] = true + toDeleteLifecycleFlzEmployees = append(toDeleteLifecycleFlzEmployees, recordedEmployee) + } + } + } + } + ns := employer.GetNamespace() lifecycleFlz := GenerateLifecycleFinalizer(employer.GetName()) err = r.ensureLifecycleFinalizer(ctx, ns, lifecycleFlz, toAddLifecycleFlzEmployees, toDeleteLifecycleFlzEmployees) @@ -208,6 +229,33 @@ func (r *Consist) syncEmployees(ctx context.Context, employer client.Object, exp return false, false, fmt.Errorf("ensureLifecycleFinalizer failed, err: %s", err.Error()) } + if needRecordEmployees { + needUpdate := false + if employer.GetAnnotations()[lifecycleFinalizerRecordedAnnoKey] == "" { + if len(toAddLifecycleFlzEmployees) != 0 { + needUpdate = true + } + } else { + recordedEmployees := strings.Split(employer.GetAnnotations()[lifecycleFinalizerRecordedAnnoKey], ",") + if !reflect.DeepEqual(recordedEmployees, toAddLifecycleFlzEmployees) { + needUpdate = true + } + } + if needUpdate { + patch := client.MergeFrom(employer.DeepCopyObject().(client.Object)) + annos := employer.GetAnnotations() + if annos == nil { + annos = make(map[string]string) + } + annos[lifecycleFinalizerRecordedAnnoKey] = strings.Join(toAddLifecycleFlzEmployees, ",") + employer.SetAnnotations(annos) + err = r.Client.Patch(ctx, employer, patch) + if err != nil { + return false, false, fmt.Errorf("patch lifecycleFinalizerRecordedAnno failed, err: %s", err.Error()) + } + } + } + isClean := len(toCudEmployees.ToCreate) == 0 && len(toCudEmployees.ToUpdate) == 0 && len(toCudEmployees.Unchanged) == 0 && len(failDelete) == 0 cudFailedExist := len(failCreate) > 0 || len(failUpdate) > 0 || len(failDelete) > 0 return isClean, cudFailedExist, nil @@ -217,7 +265,8 @@ func (r *Consist) syncEmployees(ctx context.Context, employer client.Object, exp func (r *Consist) ensureExpectedFinalizer(ctx context.Context, employer client.Object) (bool, error) { // employee is not pod or not follow PodOpsLifecycle watchOptions, watchOptionsImplemented := r.adapter.(ReconcileWatchOptions) - if r.adapter.NotFollowPodOpsLifeCycle() || (watchOptionsImplemented && !isPod(watchOptions.NewEmployee())) { + lifecycleOptions, lifecycleOptionsImplemented := r.adapter.(ReconcileLifecycleOptions) + if (lifecycleOptionsImplemented && !lifecycleOptions.FollowPodOpsLifeCycle()) || (watchOptionsImplemented && !isPod(watchOptions.NewEmployee())) { return true, nil } @@ -550,7 +599,8 @@ func (r *Consist) getToAddDeleteLifecycleFlzEmployees(succCreate, succDelete, su watchOptions, watchOptionsImplemented := r.adapter.(ReconcileWatchOptions) - if r.adapter.NotFollowPodOpsLifeCycle() || (watchOptionsImplemented && !isPod(watchOptions.NewEmployee())) { + lifecycleOptions, lifecycleOptionsImplemented := r.adapter.(ReconcileLifecycleOptions) + if (lifecycleOptionsImplemented && !lifecycleOptions.FollowPodOpsLifeCycle()) || (watchOptionsImplemented && !isPod(watchOptions.NewEmployee())) { return toAddLifecycleFlz[:toAddIdx], toDeleteLifecycleFlz[:toDeleteIdx] } diff --git a/pkg/controllers/resourceconsist/consts.go b/pkg/controllers/resourceconsist/consts.go index 267d66b0..8d341b96 100644 --- a/pkg/controllers/resourceconsist/consts.go +++ b/pkg/controllers/resourceconsist/consts.go @@ -17,7 +17,8 @@ limitations under the License. package resourceconsist const ( - defaultMaxConcurrentReconciles = 5 - expectedFinalizerAddedAnnoKey = "resource-consist.kusionstack.io/employees-expected-finalizer-added" - cleanFinalizerPrefix = "resource-consist.kusionstack.io/clean-" + defaultMaxConcurrentReconciles = 5 + expectedFinalizerAddedAnnoKey = "resource-consist.kusionstack.io/employees-expected-finalizer-added" + lifecycleFinalizerRecordedAnnoKey = "resource-consist.kusionstack.io/employees-lifecycle-finalizer-recorded" + cleanFinalizerPrefix = "resource-consist.kusionstack.io/clean-" ) diff --git a/pkg/controllers/resourceconsist/resourceconsist_controller_suite_test.go b/pkg/controllers/resourceconsist/resourceconsist_controller_suite_test.go index 4b4c7b6a..e93f6184 100644 --- a/pkg/controllers/resourceconsist/resourceconsist_controller_suite_test.go +++ b/pkg/controllers/resourceconsist/resourceconsist_controller_suite_test.go @@ -34,6 +34,9 @@ type DemoReconcile struct { } var _ ReconcileAdapter = &DemoReconcile{} +var _ ReconcileLifecycleOptions = &DemoReconcile{} + +var needRecordEmployees = false func NewDemoReconcileAdapter(c client.Client, rc *DemoResourceProviderClient) *DemoReconcile { return &DemoReconcile{ @@ -42,6 +45,14 @@ func NewDemoReconcileAdapter(c client.Client, rc *DemoResourceProviderClient) *D } } +func (r *DemoReconcile) FollowPodOpsLifeCycle() bool { + return true +} + +func (r *DemoReconcile) NeedRecordEmployees() bool { + return needRecordEmployees +} + func (r *DemoReconcile) GetSelectedEmployeeNames(ctx context.Context, employer client.Object) ([]string, error) { svc, ok := employer.(*corev1.Service) if !ok { @@ -66,10 +77,6 @@ func (r *DemoReconcile) GetControllerName() string { return "demo-controller" } -func (r *DemoReconcile) NotFollowPodOpsLifeCycle() bool { - return false -} - func (r *DemoReconcile) GetExpectedEmployer(ctx context.Context, employer client.Object) ([]IEmployer, error) { if !employer.GetDeletionTimestamp().IsZero() { return nil, nil diff --git a/pkg/controllers/resourceconsist/resourceconsist_controller_test.go b/pkg/controllers/resourceconsist/resourceconsist_controller_test.go index d476fe00..a5d869a0 100644 --- a/pkg/controllers/resourceconsist/resourceconsist_controller_test.go +++ b/pkg/controllers/resourceconsist/resourceconsist_controller_test.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/kubernetes" "os" "path/filepath" + "strings" "testing" "time" @@ -83,12 +84,12 @@ var _ = Describe("resource-consist-controller", func() { err := mgr.GetClient().Create(context.Background(), &svc0) Expect(err).NotTo(HaveOccurred()) Eventually(func() bool { - service1 := corev1.Service{} + svcTmp := corev1.Service{} Expect(mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: svc0.Name, Namespace: svc0.Namespace, - }, &service1)).Should(BeNil()) - for _, flz := range service1.GetFinalizers() { + }, &svcTmp)).Should(BeNil()) + for _, flz := range svcTmp.GetFinalizers() { if flz == cleanFinalizerPrefix+svc0.GetName() { return true } @@ -115,16 +116,16 @@ var _ = Describe("resource-consist-controller", func() { }) // trigger reconcile - service1 := corev1.Service{} + svcTmp := corev1.Service{} Expect(mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: svc0.Name, Namespace: svc0.Namespace, - }, &service1)).Should(BeNil()) - if service1.Labels == nil { - service1.Labels = make(map[string]string) + }, &svcTmp)).Should(BeNil()) + if svcTmp.Labels == nil { + svcTmp.Labels = make(map[string]string) } - service1.Labels["demo-controller-trigger-reconcile"] = fmt.Sprintf("%d", time.Now().Unix()) - Expect(mgr.GetClient().Update(context.TODO(), &service1)).Should(BeNil()) + svcTmp.Labels["demo-controller-trigger-reconcile"] = fmt.Sprintf("%d", time.Now().Unix()) + Expect(mgr.GetClient().Update(context.TODO(), &svcTmp)).Should(BeNil()) Eventually(func() bool { details, exist := demoResourceVipStatusInProvider.Load(svc0.Name) @@ -134,25 +135,25 @@ var _ = Describe("resource-consist-controller", func() { It("employer deleted", func() { Eventually(func() bool { - service1 := corev1.Service{} + svcTmp := corev1.Service{} Expect(mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: svc0.Name, Namespace: svc0.Namespace, - }, &service1)).Should(BeNil()) - flzs := service1.GetFinalizers() + }, &svcTmp)).Should(BeNil()) + flzs := svcTmp.GetFinalizers() flzs = append(flzs, "kusionstack.io/ut-block-finalizer") - service1.SetFinalizers(flzs) - return mgr.GetClient().Update(context.TODO(), &service1) == nil + svcTmp.SetFinalizers(flzs) + return mgr.GetClient().Update(context.TODO(), &svcTmp) == nil }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) Expect(mgr.GetClient().Delete(context.TODO(), &svc0)).Should(BeNil()) Eventually(func() bool { - service1 := corev1.Service{} + svcTmp := corev1.Service{} Expect(mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: svc0.Name, Namespace: svc0.Namespace, - }, &service1)).Should(BeNil()) - return !service1.GetDeletionTimestamp().IsZero() + }, &svcTmp)).Should(BeNil()) + return !svcTmp.GetDeletionTimestamp().IsZero() }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) Eventually(func() bool { _, exist := demoResourceVipStatusInProvider.Load(svc0.Name) @@ -160,28 +161,28 @@ var _ = Describe("resource-consist-controller", func() { }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) Eventually(func() bool { - service1 := corev1.Service{} + svcTmp := corev1.Service{} Expect(mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: svc0.Name, Namespace: svc0.Namespace, - }, &service1)).Should(BeNil()) + }, &svcTmp)).Should(BeNil()) var flzs []string - for _, flz := range service1.GetFinalizers() { + for _, flz := range svcTmp.GetFinalizers() { if flz == "kusionstack.io/ut-block-finalizer" { continue } flzs = append(flzs, flz) } - service1.SetFinalizers(flzs) - return mgr.GetClient().Update(context.TODO(), &service1) == nil + svcTmp.SetFinalizers(flzs) + return mgr.GetClient().Update(context.TODO(), &svcTmp) == nil }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) Eventually(func() bool { - service1 := corev1.Service{} + svcTmp := corev1.Service{} err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: svc0.Name, Namespace: svc0.Namespace, - }, &service1) + }, &svcTmp) return errors.IsNotFound(err) }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) }) @@ -250,15 +251,15 @@ var _ = Describe("resource-consist-controller", func() { It("employee synced, employees created", func() { Expect(mgr.GetClient().Create(context.TODO(), &pod)).Should(BeNil()) Eventually(func() bool { - pod1 := corev1.Pod{} + podTmp := corev1.Pod{} err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: pod.Name, Namespace: pod.Namespace, - }, &pod1) + }, &podTmp) if err != nil { return false } - pod1.Status = corev1.PodStatus{ + podTmp.Status = corev1.PodStatus{ PodIP: "1.2.3.4", Conditions: []corev1.PodCondition{ { @@ -271,7 +272,7 @@ var _ = Describe("resource-consist-controller", func() { }, }, } - return mgr.GetClient().Status().Update(context.TODO(), &pod1) == nil + return mgr.GetClient().Status().Update(context.TODO(), &podTmp) == nil }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) Eventually(func() bool { @@ -282,13 +283,13 @@ var _ = Describe("resource-consist-controller", func() { }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) Eventually(func() bool { - pod1 := corev1.Pod{} + podTmp := corev1.Pod{} _ = mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: pod.Name, Namespace: pod.Namespace, - }, &pod1) + }, &podTmp) containsLifecycleFlz := false - for _, flz := range pod1.GetFinalizers() { + for _, flz := range podTmp.GetFinalizers() { if flz == GenerateLifecycleFinalizer(svc1.Name) { containsLifecycleFlz = true break @@ -298,29 +299,29 @@ var _ = Describe("resource-consist-controller", func() { }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) Eventually(func() bool { - service1 := corev1.Service{} + svcTmp := corev1.Service{} err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: svc1.Name, Namespace: svc1.Namespace, - }, &service1) + }, &svcTmp) if err != nil { return false } - return service1.GetAnnotations()[expectedFinalizerAddedAnnoKey] == pod.Name + return svcTmp.GetAnnotations()[expectedFinalizerAddedAnnoKey] == pod.Name }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) }) It("employee synced, employees updated", func() { Eventually(func() bool { - pod1 := corev1.Pod{} + podTmp := corev1.Pod{} err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: pod.Name, Namespace: pod.Namespace, - }, &pod1) + }, &podTmp) if err != nil { return false } - pod1.Status.Conditions = []corev1.PodCondition{ + podTmp.Status.Conditions = []corev1.PodCondition{ { Type: corev1.PodReady, Status: corev1.ConditionTrue, @@ -330,7 +331,7 @@ var _ = Describe("resource-consist-controller", func() { Status: corev1.ConditionFalse, }, } - return mgr.GetClient().Status().Update(context.TODO(), &pod1) == nil + return mgr.GetClient().Status().Update(context.TODO(), &podTmp) == nil }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) Eventually(func() bool { @@ -341,15 +342,15 @@ var _ = Describe("resource-consist-controller", func() { }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) Eventually(func() bool { - pod1 := corev1.Pod{} + podTmp := corev1.Pod{} err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: pod.Name, Namespace: pod.Namespace, - }, &pod1) + }, &podTmp) if err != nil { return false } - pod1.Status.Conditions = []corev1.PodCondition{ + podTmp.Status.Conditions = []corev1.PodCondition{ { Type: corev1.PodReady, Status: corev1.ConditionTrue, @@ -359,7 +360,7 @@ var _ = Describe("resource-consist-controller", func() { Status: corev1.ConditionTrue, }, } - return mgr.GetClient().Status().Update(context.TODO(), &pod1) == nil + return mgr.GetClient().Status().Update(context.TODO(), &podTmp) == nil }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) Eventually(func() bool { @@ -371,20 +372,20 @@ var _ = Describe("resource-consist-controller", func() { }) It("employee synced, employees deleted", func() { - pod1 := corev1.Pod{} + podTmp := corev1.Pod{} Expect(mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: pod.Name, Namespace: pod.Namespace, - }, &pod1)).Should(BeNil()) + }, &podTmp)).Should(BeNil()) - Expect(mgr.GetClient().Delete(context.TODO(), &pod1)).Should(BeNil()) + Expect(mgr.GetClient().Delete(context.TODO(), &podTmp)).Should(BeNil()) Eventually(func() bool { - pod1 := corev1.Pod{} + podTmp := corev1.Pod{} err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: pod.Name, Namespace: pod.Namespace, - }, &pod1) + }, &podTmp) return errors.IsNotFound(err) }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) @@ -396,11 +397,11 @@ var _ = Describe("resource-consist-controller", func() { Expect(mgr.GetClient().Delete(context.TODO(), &svc1)).Should(BeNil()) Eventually(func() bool { - service1 := corev1.Service{} + svcTmp := corev1.Service{} err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ Name: svc1.Name, Namespace: svc1.Namespace, - }, &service1) + }, &svcTmp) return errors.IsNotFound(err) }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) @@ -412,7 +413,6 @@ var _ = Describe("resource-consist-controller", func() { }) Context("cases with unexpected err", func() { - svc2 := corev1.Service{ ObjectMeta: v1.ObjectMeta{ Name: "resource-consist-ut-svc-2", @@ -611,6 +611,287 @@ var _ = Describe("resource-consist-controller", func() { }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) }) }) + + Context("label/selector change", func() { + svc3 := corev1.Service{ + ObjectMeta: v1.ObjectMeta{ + Name: "resource-consist-ut-svc-3", + Namespace: "default", + Labels: map[string]string{ + v1alpha1.ControlledByKusionStackLabelKey: "true", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "tcp-80", + Port: 80, + Protocol: corev1.ProtocolTCP, + }, + }, + Selector: map[string]string{ + "resource-consist-ut": "resource-consist-ut-3", + }, + }, + } + + pod3 := corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: "resource-consist-ut-pod", + Namespace: "default", + Labels: map[string]string{ + v1alpha1.ControlledByKusionStackLabelKey: "true", + "resource-consist-ut": "resource-consist-ut-3", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:latest", + }, + }, + ReadinessGates: []corev1.PodReadinessGate{ + { + ConditionType: v1alpha1.ReadinessGatePodServiceReady, + }, + }, + }, + } + + It("employee synced, employer created", func() { + rc.ExpectedCalls = nil + rc.On("QueryVip", mock.Anything).Return(&DemoResourceVipOps{}, nil) + rc.On("CreateVip", mock.Anything).Return(&DemoResourceVipOps{}, nil) + rc.On("UpdateVip", mock.Anything).Return(&DemoResourceVipOps{}, nil) + rc.On("DeleteVip", mock.Anything).Return(&DemoResourceVipOps{}, nil) + rc.On("QueryRealServer", mock.Anything).Return(&DemoResourceRsOps{}, nil) + rc.On("CreateRealServer", mock.Anything).Return(&DemoResourceRsOps{}, nil) + rc.On("UpdateRealServer", mock.Anything).Return(&DemoResourceRsOps{}, nil) + rc.On("DeleteRealServer", mock.Anything).Return(&DemoResourceRsOps{}, nil) + + Expect(mgr.GetClient().Create(context.Background(), &svc3)).Should(BeNil()) + Eventually(func() bool { + details, exist := demoResourceVipStatusInProvider.Load(svc3.Name) + return exist && details.(DemoServiceDetails).RemoteVIP == "demo-remote-VIP" && details.(DemoServiceDetails).RemoteVIPQPS == 100 + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + }) + + It("employee synced, employees created", func() { + Expect(mgr.GetClient().Create(context.TODO(), &pod3)).Should(BeNil()) + Eventually(func() bool { + podTmp := corev1.Pod{} + err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: pod3.Name, + Namespace: pod3.Namespace, + }, &podTmp) + if err != nil { + return false + } + podTmp.Status = corev1.PodStatus{ + PodIP: "1.2.3.4", + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + { + Type: v1alpha1.ReadinessGatePodServiceReady, + Status: corev1.ConditionTrue, + }, + }, + } + return mgr.GetClient().Status().Update(context.TODO(), &podTmp) == nil + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + + Eventually(func() bool { + details, exist := demoResourceRsStatusInProvider.Load(pod3.Name) + return exist && details.(DemoPodStatus).GetEmployeeName() == pod3.Name && + details.(DemoPodStatus).GetEmployeeStatuses().(PodEmployeeStatuses).ExtraStatus.(PodExtraStatus).TrafficWeight == 100 && + details.(DemoPodStatus).GetEmployeeStatuses().(PodEmployeeStatuses).ExtraStatus.(PodExtraStatus).TrafficOn == true + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + + Eventually(func() bool { + podTmp := corev1.Pod{} + _ = mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: pod3.Name, + Namespace: pod3.Namespace, + }, &podTmp) + containsLifecycleFlz := false + for _, flz := range podTmp.GetFinalizers() { + if flz == GenerateLifecycleFinalizer(svc3.Name) { + containsLifecycleFlz = true + break + } + } + return containsLifecycleFlz + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + + Eventually(func() bool { + svcTmp := corev1.Service{} + err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: svc3.Name, + Namespace: svc3.Namespace, + }, &svcTmp) + if err != nil { + return false + } + return svcTmp.GetAnnotations()[expectedFinalizerAddedAnnoKey] == pod3.Name + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + }) + + It("label/selector change and resources in backend provider might changed by others", func() { + // make pod not selected, resource in backend provider not changed by others + podTmp := corev1.Pod{} + Expect(mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: pod3.Name, + Namespace: pod3.Namespace, + }, &podTmp)).Should(BeNil()) + podTmp.Labels["resource-consist-ut"] = "not-selected" + Expect(mgr.GetClient().Update(context.TODO(), &podTmp)).Should(BeNil()) + + Eventually(func() bool { + podTmp := corev1.Pod{} + err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: pod3.Name, + Namespace: pod3.Namespace, + }, &podTmp) + if err != nil { + return false + } + + lifecycleFlzExist := false + for _, flz := range podTmp.Finalizers { + if flz == GenerateLifecycleFinalizer(svc3.Name) { + lifecycleFlzExist = true + break + } + } + + _, pod3RsExist := demoResourceRsStatusInProvider.Load(pod3.Name) + return !strings.Contains(podTmp.GetAnnotations()[v1alpha1.PodAvailableConditionsAnnotation], + "Service/default/resource-consist-ut-svc-3") && !lifecycleFlzExist && !pod3RsExist + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + + // make demo controller need record employees + needRecordEmployees = true + // make pod selected again + Expect(mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: pod3.Name, + Namespace: pod3.Namespace, + }, &podTmp)).Should(BeNil()) + podTmp.Labels["resource-consist-ut"] = "resource-consist-ut-3" + Expect(mgr.GetClient().Update(context.TODO(), &podTmp)).Should(BeNil()) + + Eventually(func() bool { + podTmp := corev1.Pod{} + err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: pod3.Name, + Namespace: pod3.Namespace, + }, &podTmp) + if err != nil { + return false + } + + lifecycleFlzExist := false + for _, flz := range podTmp.Finalizers { + if flz == GenerateLifecycleFinalizer(svc3.Name) { + lifecycleFlzExist = true + break + } + } + + _, pod3RsExist := demoResourceRsStatusInProvider.Load(pod3.Name) + return strings.Contains(podTmp.GetAnnotations()[v1alpha1.PodAvailableConditionsAnnotation], + "Service/default/resource-consist-ut-svc-3") && lifecycleFlzExist && pod3RsExist + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + + // delete pod3 rs + demoResourceRsStatusInProvider.Delete(pod3.Name) + // make pod not selected + Expect(mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: pod3.Name, + Namespace: pod3.Namespace, + }, &podTmp)).Should(BeNil()) + podTmp.Labels["resource-consist-ut"] = "not-selected" + Expect(mgr.GetClient().Update(context.TODO(), &podTmp)).Should(BeNil()) + + Eventually(func() bool { + podTmp := corev1.Pod{} + err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: pod3.Name, + Namespace: pod3.Namespace, + }, &podTmp) + if err != nil { + return false + } + + lifecycleFlzExist := false + for _, flz := range podTmp.Finalizers { + if flz == GenerateLifecycleFinalizer(svc3.Name) { + lifecycleFlzExist = true + break + } + } + + _, pod3RsExist := demoResourceRsStatusInProvider.Load(pod3.Name) + + svcTmp := corev1.Service{} + err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: svc3.Name, + Namespace: svc3.Namespace, + }, &svcTmp) + if err != nil { + return false + } + + return !strings.Contains(podTmp.GetAnnotations()[v1alpha1.PodAvailableConditionsAnnotation], + "Service/default/resource-consist-ut-svc-3") && !lifecycleFlzExist && !pod3RsExist && + !strings.Contains(svcTmp.GetAnnotations()[lifecycleFinalizerRecordedAnnoKey], pod3.Name) + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + }) + + It("employee synced, employees deleted", func() { + podTmp := corev1.Pod{} + Expect(mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: pod3.Name, + Namespace: pod3.Namespace, + }, &podTmp)).Should(BeNil()) + + Expect(mgr.GetClient().Delete(context.TODO(), &podTmp)).Should(BeNil()) + + Eventually(func() bool { + podTmp := corev1.Pod{} + err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: pod3.Name, + Namespace: pod3.Namespace, + }, &podTmp) + return errors.IsNotFound(err) + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + + Eventually(func() bool { + _, exist := demoResourceRsStatusInProvider.Load(pod3.Name) + return !exist + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + + Expect(mgr.GetClient().Delete(context.TODO(), &svc3)).Should(BeNil()) + + Eventually(func() bool { + svcTmp := corev1.Service{} + err := mgr.GetClient().Get(context.TODO(), types.NamespacedName{ + Name: svc3.Name, + Namespace: svc3.Namespace, + }, &svcTmp) + return errors.IsNotFound(err) + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + + Eventually(func() bool { + _, exist := demoResourceVipStatusInProvider.Load(svc3.Name) + return !exist + }, 3*time.Second, 100*time.Millisecond).Should(BeTrue()) + }) + + }) }) var _ = BeforeSuite(func() { diff --git a/pkg/controllers/resourceconsist/types.go b/pkg/controllers/resourceconsist/types.go index e5923dc3..00c53ece 100644 --- a/pkg/controllers/resourceconsist/types.go +++ b/pkg/controllers/resourceconsist/types.go @@ -34,6 +34,10 @@ type ReconcileOptions interface { // ReconcileWatchOptions defines what employer and employee is and how controller watch // default employer: Service, default employee: Pod +// Recommend: +// implement ReconcileWatchOptions if Employer resource might be reconciled by other controller, +// different Predicates make an employer won't be reconciled by more than one controller so that LifecycleFinalizer won't +// be solved incorrectly. type ReconcileWatchOptions interface { NewEmployer() client.Object NewEmployee() client.Object @@ -43,10 +47,19 @@ type ReconcileWatchOptions interface { EmployeePredicates() predicate.Funcs } +// ReconcileLifecycleOptions defines whether PodOpsLifecycle followed and +// whether employees' LifecycleFinalizer conditions need to be Record/Erase to employer's anno +// actually NeedRecordEmployees only needed for those adapters that follow PodOpsLifecycle, +// in the case of employment relationship might change and resources in backend provider might be changed by others. +// if not implemented, the two options would be FollowPodOpsLifeCycle: true and NeedRecordEmployees: false +type ReconcileLifecycleOptions interface { + FollowPodOpsLifeCycle() bool + NeedRecordEmployees() bool +} + // ReconcileAdapter is the interface that customized controllers should implement. type ReconcileAdapter interface { GetControllerName() string - NotFollowPodOpsLifeCycle() bool GetSelectedEmployeeNames(ctx context.Context, employer client.Object) ([]string, error)