configure QPS and Burst for the flags of klusterlet agents

Signed-off-by: Zhiwei Yin <zyin@redhat.com>

deploy/cluster-manager/olm-catalog/cluster-manager/manifests/cluster-manager.clusterserviceversion.yaml

@@ -59,7 +59,7 @@ metadata:
     categories: Integration & Delivery,OpenShift Optional
     certified: "false"
     containerImage: quay.io/open-cluster-management/registration-operator:latest
-    createdAt: "2023-12-07T15:16:47Z"
+    createdAt: "2024-01-15T16:05:25Z"
     description: Manages the installation and upgrade of the ClusterManager.
     operators.operatorframework.io/builder: operator-sdk-v1.32.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3

deploy/klusterlet/config/crds/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml

@@ -223,6 +223,20 @@ spec:
                       - feature
                       type: object
                     type: array
+                  kubeAPIBurst:
+                    default: 100
+                    description: 'KubeAPIBurst indicates the maximum burst of the
+                      throttle while talking with apiserver of hub cluster from the
+                      spoke cluster. If it is set empty, use the default value: 100'
+                    format: int32
+                    type: integer
+                  kubeAPIQPS:
+                    default: 50
+                    description: 'KubeAPIQPS indicates the maximum QPS while talking
+                      with apiserver of hub cluster from the spoke cluster. If it
+                      is set empty, use the default value: 50'
+                    format: int32
+                    type: integer
                 type: object
               registrationImagePullSpec:
                 description: RegistrationImagePullSpec represents the desired image
@@ -274,6 +288,20 @@ spec:
                       - feature
                       type: object
                     type: array
+                  kubeAPIBurst:
+                    default: 100
+                    description: 'KubeAPIBurst indicates the maximum burst of the
+                      throttle while talking with apiserver of hub cluster from the
+                      spoke cluster. If it is set empty, use the default value: 100'
+                    format: int32
+                    type: integer
+                  kubeAPIQPS:
+                    default: 50
+                    description: 'KubeAPIQPS indicates the maximum QPS while talking
+                      with apiserver of hub cluster from the spoke cluster. If it
+                      is set empty, use the default value: 50'
+                    format: int32
+                    type: integer
                 type: object
               workImagePullSpec:
                 description: WorkImagePullSpec represents the desired image configuration

deploy/klusterlet/olm-catalog/klusterlet/manifests/klusterlet.clusterserviceversion.yaml

@@ -31,7 +31,7 @@ metadata:
     categories: Integration & Delivery,OpenShift Optional
     certified: "false"
     containerImage: quay.io/open-cluster-management/registration-operator:latest
-    createdAt: "2023-12-07T15:16:47Z"
+    createdAt: "2024-01-15T16:05:25Z"
     description: Manages the installation and upgrade of the Klusterlet.
     operators.operatorframework.io/builder: operator-sdk-v1.32.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3

deploy/klusterlet/olm-catalog/klusterlet/manifests/operator.open-cluster-management.io_klusterlets.yaml

@@ -223,6 +223,20 @@ spec:
                       - feature
                       type: object
                     type: array
+                  kubeAPIBurst:
+                    default: 100
+                    description: 'KubeAPIBurst indicates the maximum burst of the
+                      throttle while talking with apiserver of hub cluster from the
+                      spoke cluster. If it is set empty, use the default value: 100'
+                    format: int32
+                    type: integer
+                  kubeAPIQPS:
+                    default: 50
+                    description: 'KubeAPIQPS indicates the maximum QPS while talking
+                      with apiserver of hub cluster from the spoke cluster. If it
+                      is set empty, use the default value: 50'
+                    format: int32
+                    type: integer
                 type: object
               registrationImagePullSpec:
                 description: RegistrationImagePullSpec represents the desired image
@@ -274,6 +288,20 @@ spec:
                       - feature
                       type: object
                     type: array
+                  kubeAPIBurst:
+                    default: 100
+                    description: 'KubeAPIBurst indicates the maximum burst of the
+                      throttle while talking with apiserver of hub cluster from the
+                      spoke cluster. If it is set empty, use the default value: 100'
+                    format: int32
+                    type: integer
+                  kubeAPIQPS:
+                    default: 50
+                    description: 'KubeAPIQPS indicates the maximum QPS while talking
+                      with apiserver of hub cluster from the spoke cluster. If it
+                      is set empty, use the default value: 50'
+                    format: int32
+                    type: integer
                 type: object
               workImagePullSpec:
                 description: WorkImagePullSpec represents the desired image configuration

go.mod

@@ -34,7 +34,7 @@ require (
 	k8s.io/kube-aggregator v0.28.1
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
 	open-cluster-management.io/addon-framework v0.8.1-0.20240102072130-44852ea0722f
-	open-cluster-management.io/api v0.12.1-0.20240103082609-f6404f30c42c
+	open-cluster-management.io/api v0.12.1-0.20240115071352-3d94ce8f3499
 	sigs.k8s.io/controller-runtime v0.16.2
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96
 )

go.sum

@@ -767,8 +767,8 @@ k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSn
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 open-cluster-management.io/addon-framework v0.8.1-0.20240102072130-44852ea0722f h1:R5IbFdM4g+kDyONZ1/q67/czRwgDkq9Q7ETZ3RhGaAg=
 open-cluster-management.io/addon-framework v0.8.1-0.20240102072130-44852ea0722f/go.mod h1:aj97pgpGJ0/LpQzBVtU2oDFqqIiZLOPnsjLKG/sVkFw=
-open-cluster-management.io/api v0.12.1-0.20240103082609-f6404f30c42c h1:Z/xoxtc5lDw0j+SqTbePgm5xGNxAmC+Res6dTe0xhM4=
-open-cluster-management.io/api v0.12.1-0.20240103082609-f6404f30c42c/go.mod h1:K3Rza3qN/W1+E1a+wbLtFatvdI8UlQWkSqBlpeRHMPw=
+open-cluster-management.io/api v0.12.1-0.20240115071352-3d94ce8f3499 h1:Odh200i57Z9V44eJak98VqdKMCAlj3m9dtnUp8285nE=
+open-cluster-management.io/api v0.12.1-0.20240115071352-3d94ce8f3499/go.mod h1:K3Rza3qN/W1+E1a+wbLtFatvdI8UlQWkSqBlpeRHMPw=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

manifests/klusterlet/management/klusterlet-agent-deployment.yaml

@@ -84,6 +84,12 @@ spec:
           - "--spoke-kubeconfig=/spoke/config/kubeconfig"
           - "--terminate-on-files=/spoke/config/kubeconfig"
           {{end}}
+          {{if gt .AgentKubeAPIQPS 0.0}}
+          - "--kube-api-qps={{ .AgentKubeAPIQPS }}"
+          {{end}}
+          {{if gt .AgentKubeAPIBurst 0}}
+          - "--kube-api-burst={{ .AgentKubeAPIBurst }}"
+          {{end}}
         env:
           - name: POD_NAME
             valueFrom:

manifests/klusterlet/management/klusterlet-registration-deployment.yaml

@@ -75,6 +75,12 @@ spec:
           {{if .ClusterAnnotationsString}}
           - "--cluster-annotations={{ .ClusterAnnotationsString }}"
           {{end}}
+          {{if gt .RegistrationKubeAPIQPS 0.0}}
+          - "--kube-api-qps={{ .RegistrationKubeAPIQPS }}"
+          {{end}}
+          {{if gt .RegistrationKubeAPIBurst 0}}
+          - "--kube-api-burst={{ .RegistrationKubeAPIBurst }}"
+          {{end}}
         env:
           - name: POD_NAME
             valueFrom:

manifests/klusterlet/management/klusterlet-work-deployment.yaml

@@ -70,6 +70,12 @@ spec:
           - "--disable-leader-election"
           - "--status-sync-interval=60s"
           {{end}}
+          {{if gt .WorkKubeAPIQPS 0.0}}
+          - "--kube-api-qps={{ .WorkKubeAPIQPS }}"
+          {{end}}
+          {{if gt .WorkKubeAPIBurst 0}}
+          - "--kube-api-burst={{ .WorkKubeAPIBurst }}"
+          {{end}}
         env:
           - name: POD_NAME
             valueFrom:

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go

@@ -120,22 +120,27 @@ type klusterletConfig struct {
 	//     namespace as KlusterletNamespace;
 	// 2). In the Hosted mode, it is on the management cluster and has the same name as
 	//     the klusterlet.
-	AgentNamespace              string
-	AgentID                     string
-	RegistrationImage           string
-	WorkImage                   string
-	SingletonImage              string
-	RegistrationServiceAccount  string
-	WorkServiceAccount          string
-	ClusterName                 string
-	ExternalServerURL           string
-	HubKubeConfigSecret         string
-	BootStrapKubeConfigSecret   string
-	OperatorNamespace           string
-	Replica                     int32
-	ClientCertExpirationSeconds int32
-	ClusterAnnotationsString    string
-
+	AgentNamespace                              string
+	AgentID                                     string
+	RegistrationImage                           string
+	WorkImage                                   string
+	SingletonImage                              string
+	RegistrationServiceAccount                  string
+	WorkServiceAccount                          string
+	ClusterName                                 string
+	ExternalServerURL                           string
+	HubKubeConfigSecret                         string
+	BootStrapKubeConfigSecret                   string
+	OperatorNamespace                           string
+	Replica                                     int32
+	ClientCertExpirationSeconds                 int32
+	ClusterAnnotationsString                    string
+	RegistrationKubeAPIQPS                      float32
+	RegistrationKubeAPIBurst                    int32
+	WorkKubeAPIQPS                              float32
+	WorkKubeAPIBurst                            int32
+	AgentKubeAPIQPS                             float32
+	AgentKubeAPIBurst                           int32
 	ExternalManagedKubeConfigSecret             string
 	ExternalManagedKubeConfigRegistrationSecret string
 	ExternalManagedKubeConfigWorkSecret         string
@@ -241,6 +246,8 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
 	if klusterlet.Spec.RegistrationConfiguration != nil {
 		registrationFeatureGates = klusterlet.Spec.RegistrationConfiguration.FeatureGates
 		config.ClientCertExpirationSeconds = klusterlet.Spec.RegistrationConfiguration.ClientCertExpirationSeconds
+		config.RegistrationKubeAPIQPS = float32(klusterlet.Spec.RegistrationConfiguration.KubeAPIQPS)
+		config.RegistrationKubeAPIBurst = klusterlet.Spec.RegistrationConfiguration.KubeAPIBurst
 
 		// construct cluster annotations string, the final format is "key1=value1,key2=value2"
 		var annotationsArray []string
@@ -255,10 +262,22 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
 	var workFeatureGates []operatorapiv1.FeatureGate
 	if klusterlet.Spec.WorkConfiguration != nil {
 		workFeatureGates = klusterlet.Spec.WorkConfiguration.FeatureGates
+		config.WorkKubeAPIQPS = float32(klusterlet.Spec.WorkConfiguration.KubeAPIQPS)
+		config.WorkKubeAPIBurst = klusterlet.Spec.WorkConfiguration.KubeAPIBurst
 	}
 	config.WorkFeatureGates, workFeatureMsgs = helpers.ConvertToFeatureGateFlags("Work", workFeatureGates, ocmfeature.DefaultSpokeWorkFeatureGates)
 	meta.SetStatusCondition(&klusterlet.Status.Conditions, helpers.BuildFeatureCondition(registrationFeatureMsgs, workFeatureMsgs))
 
+	// for singleton agent, the QPS and Burst use the max one between the configurations of registration and work
+	config.AgentKubeAPIQPS = config.RegistrationKubeAPIQPS
+	if config.AgentKubeAPIQPS < config.WorkKubeAPIQPS {
+		config.AgentKubeAPIQPS = config.WorkKubeAPIQPS
+	}
+	config.AgentKubeAPIBurst = config.RegistrationKubeAPIBurst
+	if config.AgentKubeAPIBurst < config.WorkKubeAPIBurst {
+		config.AgentKubeAPIBurst = config.WorkKubeAPIBurst
+	}
+
 	reconcilers := []klusterletReconcile{
 		&crdReconcile{
 			managedClusterClients: managedClusterClients,

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go

@@ -111,6 +111,12 @@ func newKlusterlet(name, namespace, clustername string) *operatorapiv1.Klusterle
 						Mode:    "Enable",
 					},
 				},
+				KubeAPIQPS:   10,
+				KubeAPIBurst: 60,
+			},
+			WorkConfiguration: &operatorapiv1.WorkAgentConfiguration{
+				KubeAPIQPS:   20,
+				KubeAPIBurst: 50,
 			},
 			HubApiServerHostAlias: &operatorapiv1.HubApiServerHostAlias{
 				IP:       "11.22.33.44",
@@ -388,6 +394,8 @@ func assertRegistrationDeployment(t *testing.T, actions []clienttesting.Action,
 		expectedArgs = append(expectedArgs, "--disable-leader-election")
 	}
 
+	expectedArgs = append(expectedArgs, "--kube-api-qps=10", "--kube-api-burst=60")
+
 	if !equality.Semantic.DeepEqual(args, expectedArgs) {
 		t.Errorf("Expect args %v, but got %v", expectedArgs, args)
 		return
@@ -430,6 +438,8 @@ func assertWorkDeployment(t *testing.T, actions []clienttesting.Action, verb, cl
 		expectArgs = append(expectArgs, "--disable-leader-election", "--status-sync-interval=60s")
 	}
 
+	expectArgs = append(expectArgs, "--kube-api-qps=20", "--kube-api-burst=50")
+
 	if !equality.Semantic.DeepEqual(args, expectArgs) {
 		t.Errorf("Expect args %v, but got %v", expectArgs, args)
 		return

test/integration/operator/klusterlet_test.go

@@ -821,7 +821,7 @@ var _ = ginkgo.Describe("Klusterlet", func() {
 
 		ginkgo.It("feature gates configuration is nil or empty", func() {
 			klusterlet.Spec.RegistrationConfiguration = nil
-			klusterlet.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{}
+			klusterlet.Spec.WorkConfiguration = &operatorapiv1.WorkAgentConfiguration{}
 
 			ginkgo.By("Create the klusterlet with RegistrationConfiguration nil and WorkConfiguration empty")
 			_, err := operatorClient.OperatorV1().Klusterlets().Create(context.Background(),
@@ -944,7 +944,7 @@ var _ = ginkgo.Describe("Klusterlet", func() {
 					"agent.open-cluster-management.io/foo": "bar",
 				},
 			}
-			klusterlet.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{
+			klusterlet.Spec.WorkConfiguration = &operatorapiv1.WorkAgentConfiguration{
 				FeatureGates: []operatorapiv1.FeatureGate{
 					{
 						Feature: string(ocmfeature.ExecutorValidatingCaches),
@@ -1026,7 +1026,7 @@ var _ = ginkgo.Describe("Klusterlet", func() {
 					},
 				},
 			}
-			klusterlet.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{
+			klusterlet.Spec.WorkConfiguration = &operatorapiv1.WorkAgentConfiguration{
 				FeatureGates: []operatorapiv1.FeatureGate{
 					{
 						Feature: "Bar",

vendor/modules.txt

@@ -1496,7 +1496,7 @@ open-cluster-management.io/addon-framework/pkg/index
 open-cluster-management.io/addon-framework/pkg/manager/controllers/addonconfiguration
 open-cluster-management.io/addon-framework/pkg/manager/controllers/addonowner
 open-cluster-management.io/addon-framework/pkg/utils
-# open-cluster-management.io/api v0.12.1-0.20240103082609-f6404f30c42c
+# open-cluster-management.io/api v0.12.1-0.20240115071352-3d94ce8f3499
 ## explicit; go 1.20
 open-cluster-management.io/api/addon/v1alpha1
 open-cluster-management.io/api/client/addon/clientset/versioned