🌱 honor the settings of AppliedManifestWorkEvictionGracePeriod in Klu…

…sterlet API

Signed-off-by: Yang Le <yangle@redhat.com>

deploy/klusterlet/config/crds/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml

@@ -115,7 +115,7 @@ spec:
                   like service accounts, roles and rolebindings, while the agent is
                   deployed to the namespace with the same name as klusterlet on the
                   management cluster.
-                maxLength: 63
+                maxLength: 57
                 pattern: ^open-cluster-management-[-a-z0-9]*[a-z0-9]$
                 type: string
               nodePlacement:
@@ -358,6 +358,14 @@ spec:
               workConfiguration:
                 description: WorkConfiguration contains the configuration of work
                 properties:
+                  appliedManifestWorkEvictionGracePeriod:
+                    description: AppliedManifestWorkEvictionGracePeriod is the eviction
+                      grace period the work agent will wait before evicting the AppliedManifestWorks,
+                      whose corresponding ManifestWorks are missing on the hub cluster,
+                      from the managed cluster. If not present, the default value
+                      of the work agent will be used.
+                    pattern: ^([0-9]+(s|m|h))+$
+                    type: string
                   featureGates:
                     description: 'FeatureGates represents the list of feature gates
                       for work If it is set empty, default feature gates will be used.

go.mod

@@ -35,7 +35,7 @@ require (
 	k8s.io/kube-aggregator v0.29.3
 	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
 	open-cluster-management.io/addon-framework v0.9.1-0.20240419070222-e703fc5a2556
-	open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad
+	open-cluster-management.io/api v0.13.1-0.20240506072237-800b00d9f0db
 	open-cluster-management.io/sdk-go v0.13.1-0.20240416062924-20307e6fe090
 	sigs.k8s.io/controller-runtime v0.17.3
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96

go.sum

@@ -425,8 +425,8 @@ k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0g
 k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 open-cluster-management.io/addon-framework v0.9.1-0.20240419070222-e703fc5a2556 h1:X3vJEx9agC94l7SitpWZFDshISdL1niqVH0+diyqfJo=
 open-cluster-management.io/addon-framework v0.9.1-0.20240419070222-e703fc5a2556/go.mod h1:HayKCznnlyW+0dUJQGj5sNR6i3tvylSySD3YnvZkBtY=
-open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad h1:DB3GpK5vzbGu9ss13bfodi8pGTkPcpdcLvOPEPMptTk=
-open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad/go.mod h1:yrNuMMpciXjXPnj2yznb6LTyrGliiTrFZAJDp/Ck3c4=
+open-cluster-management.io/api v0.13.1-0.20240506072237-800b00d9f0db h1:puVfabidvMj0phg34e5PqAmC0jzFiVN5LCNlZIEk+CA=
+open-cluster-management.io/api v0.13.1-0.20240506072237-800b00d9f0db/go.mod h1:yrNuMMpciXjXPnj2yznb6LTyrGliiTrFZAJDp/Ck3c4=
 open-cluster-management.io/sdk-go v0.13.1-0.20240416062924-20307e6fe090 h1:zFmHuW+ztdfUUNslqNW+H1WEcfdEUQHoRDbmdajX340=
 open-cluster-management.io/sdk-go v0.13.1-0.20240416062924-20307e6fe090/go.mod h1:w2OaxtCyegxeyFLU42UQ3oxUz01QdsBQkcHI17T/l48=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I=

manifests/klusterlet/management/klusterlet-agent-deployment.yaml

@@ -90,6 +90,9 @@ spec:
           {{if gt .AgentKubeAPIBurst 0}}
           - "--kube-api-burst={{ .AgentKubeAPIBurst }}"
           {{end}}
+          {{if .AppliedManifestWorkEvictionGracePeriod}}
+          - "--appliedmanifestwork-eviction-grace-period={{ .AppliedManifestWorkEvictionGracePeriod }}"
+          {{end}}
         env:
           - name: POD_NAME
             valueFrom:

manifests/klusterlet/management/klusterlet-work-deployment.yaml

@@ -76,6 +76,9 @@ spec:
           {{if gt .WorkKubeAPIBurst 0}}
           - "--kube-api-burst={{ .WorkKubeAPIBurst }}"
           {{end}}
+          {{if .AppliedManifestWorkEvictionGracePeriod}}
+          - "--appliedmanifestwork-eviction-grace-period={{ .AppliedManifestWorkEvictionGracePeriod }}"
+          {{end}}
         env:
           - name: POD_NAME
             valueFrom:

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go

@@ -143,6 +143,7 @@ type klusterletConfig struct {
 	ExternalManagedKubeConfigWorkSecret         string
 	ExternalManagedKubeConfigAgentSecret        string
 	InstallMode                                 operatorapiv1.InstallMode
+	AppliedManifestWorkEvictionGracePeriod      string
 
 	// PriorityClassName is the name of the PriorityClass used by the deployed agents
 	PriorityClassName string
@@ -179,20 +180,21 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
 	}
 
 	config := klusterletConfig{
-		KlusterletName:            klusterlet.Name,
-		KlusterletNamespace:       helpers.KlusterletNamespace(klusterlet),
-		AgentNamespace:            helpers.AgentNamespace(klusterlet),
-		AgentID:                   string(klusterlet.UID),
-		RegistrationImage:         klusterlet.Spec.RegistrationImagePullSpec,
-		WorkImage:                 klusterlet.Spec.WorkImagePullSpec,
-		ClusterName:               klusterlet.Spec.ClusterName,
-		SingletonImage:            klusterlet.Spec.ImagePullSpec,
-		BootStrapKubeConfigSecret: helpers.BootstrapHubKubeConfig,
-		HubKubeConfigSecret:       helpers.HubKubeConfig,
-		ExternalServerURL:         getServersFromKlusterlet(klusterlet),
-		OperatorNamespace:         n.operatorNamespace,
-		Replica:                   helpers.DetermineReplica(ctx, n.kubeClient, klusterlet.Spec.DeployOption.Mode, n.kubeVersion),
-		PriorityClassName:         helpers.AgentPriorityClassName(klusterlet, n.kubeVersion),
+		KlusterletName:                         klusterlet.Name,
+		KlusterletNamespace:                    helpers.KlusterletNamespace(klusterlet),
+		AgentNamespace:                         helpers.AgentNamespace(klusterlet),
+		AgentID:                                string(klusterlet.UID),
+		RegistrationImage:                      klusterlet.Spec.RegistrationImagePullSpec,
+		WorkImage:                              klusterlet.Spec.WorkImagePullSpec,
+		ClusterName:                            klusterlet.Spec.ClusterName,
+		SingletonImage:                         klusterlet.Spec.ImagePullSpec,
+		BootStrapKubeConfigSecret:              helpers.BootstrapHubKubeConfig,
+		HubKubeConfigSecret:                    helpers.HubKubeConfig,
+		ExternalServerURL:                      getServersFromKlusterlet(klusterlet),
+		OperatorNamespace:                      n.operatorNamespace,
+		Replica:                                helpers.DetermineReplica(ctx, n.kubeClient, klusterlet.Spec.DeployOption.Mode, n.kubeVersion),
+		PriorityClassName:                      helpers.AgentPriorityClassName(klusterlet, n.kubeVersion),
+		AppliedManifestWorkEvictionGracePeriod: getAppliedManifestWorkEvictionGracePeriod(klusterlet),
 
 		ExternalManagedKubeConfigSecret:             helpers.ExternalManagedKubeConfig,
 		ExternalManagedKubeConfigRegistrationSecret: helpers.ExternalManagedKubeConfigRegistration,
@@ -362,6 +364,22 @@ func getServersFromKlusterlet(klusterlet *operatorapiv1.Klusterlet) string {
 	return strings.Join(serverString, ",")
 }
 
+func getAppliedManifestWorkEvictionGracePeriod(klusterlet *operatorapiv1.Klusterlet) string {
+	if klusterlet == nil {
+		return ""
+	}
+
+	if klusterlet.Spec.WorkConfiguration == nil {
+		return ""
+	}
+
+	if klusterlet.Spec.WorkConfiguration.AppliedManifestWorkEvictionGracePeriod == nil {
+		return ""
+	}
+
+	return klusterlet.Spec.WorkConfiguration.AppliedManifestWorkEvictionGracePeriod.Duration.String()
+}
+
 // getManagedKubeConfig is a helper func for Hosted mode, it will retrieve managed cluster
 // kubeconfig from "external-managed-kubeconfig" secret.
 func getManagedKubeConfig(ctx context.Context, kubeClient kubernetes.Interface, namespace, secretName string) (*rest.Config, error) {

pkg/work/spoke/controllers/finalizercontroller/unmanaged_appliedmanifestwork_controller.go

@@ -91,6 +91,11 @@ func (m *unmanagedAppliedWorkController) sync(ctx context.Context, controllerCon
 		return err
 	}
 
+	// Do NOT evict the AppliedManifestWorks if the eviction grace period is longer than 100 years
+	if m.evictionGracePeriod >= 100*365*24*time.Hour {
+		return m.stopToEvictAppliedManifestWork(ctx, appliedManifestWork)
+	}
+
 	_, err = m.manifestWorkLister.Get(appliedManifestWork.Spec.ManifestWorkName)
 	if errors.IsNotFound(err) {
 		// evict the current appliedmanifestwork when its relating manifestwork is missing on the hub

pkg/work/spoke/controllers/finalizercontroller/unmanaged_appliedmanifestwork_controller_test.go

@@ -186,6 +186,55 @@ func TestSyncUnamanagedAppliedWork(t *testing.T) {
 			expectedQueueLen:                   1,
 			validateAppliedManifestWorkActions: testingcommon.AssertNoActions,
 		},
+		{
+			name:                    "disable the eviction",
+			appliedManifestWorkName: "hubhash-test",
+			hubHash:                 "hubhash",
+			agentID:                 "test-agent",
+			evictionGracePeriod:     100 * 365 * 24 * time.Hour,
+			works:                   []runtime.Object{},
+			appliedWorks: []runtime.Object{
+				&workapiv1.AppliedManifestWork{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "hubhash-test",
+					},
+					Spec: workapiv1.AppliedManifestWorkSpec{
+						ManifestWorkName: "test",
+						HubHash:          "hubhash",
+						AgentID:          "test-agent",
+					},
+				},
+			},
+			validateAppliedManifestWorkActions: testingcommon.AssertNoActions,
+		},
+		{
+			name:                    "disable the eviction and remove the EvictionStartTime",
+			appliedManifestWorkName: "hubhash-test",
+			hubHash:                 "hubhash",
+			agentID:                 "test-agent",
+			evictionGracePeriod:     100 * 365 * 24 * time.Hour,
+			works:                   []runtime.Object{},
+			appliedWorks: []runtime.Object{
+				&workapiv1.AppliedManifestWork{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "hubhash-test",
+					},
+					Spec: workapiv1.AppliedManifestWorkSpec{
+						ManifestWorkName: "test",
+						HubHash:          "hubhash",
+						AgentID:          "test-agent",
+					},
+					Status: workapiv1.AppliedManifestWorkStatus{
+						EvictionStartTime: &metav1.Time{
+							Time: time.Now().Add(-5 * time.Minute),
+						},
+					},
+				},
+			},
+			validateAppliedManifestWorkActions: func(t *testing.T, actions []clienttesting.Action) {
+				testingcommon.AssertActions(t, actions, "patch")
+			},
+		},
 	}
 
 	for _, c := range cases {

vendor/modules.txt

@@ -1507,7 +1507,7 @@ open-cluster-management.io/addon-framework/pkg/basecontroller/events
 open-cluster-management.io/addon-framework/pkg/basecontroller/factory
 open-cluster-management.io/addon-framework/pkg/index
 open-cluster-management.io/addon-framework/pkg/utils
-# open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad
+# open-cluster-management.io/api v0.13.1-0.20240506072237-800b00d9f0db
 ## explicit; go 1.21
 open-cluster-management.io/api/addon/v1alpha1
 open-cluster-management.io/api/client/addon/clientset/versioned