✨ Configurable qos resources for containers managed by cluster-manager and klusterlet

manifests/cluster-manager/management/cluster-manager-addon-manager-deployment.yaml

@@ -78,10 +78,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp

manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml

@@ -75,10 +75,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp

manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml

@@ -74,10 +74,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp

manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml

@@ -82,10 +82,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp

manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml

@@ -56,10 +56,19 @@ spec:
         {{ if .HostedMode }}
         - "--kubeconfig=/var/run/secrets/hub/kubeconfig"
         {{ end }}
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:

manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml

@@ -75,10 +75,19 @@ spec:
             scheme: HTTP
             port: 8000
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         ports:
         - containerPort: 9443
           protocol: TCP

manifests/config.go

@@ -1,5 +1,7 @@
 package manifests
 
+import operatorapiv1 "open-cluster-management.io/api/operator/v1"
+
 type HubConfig struct {
 	ClusterManagerName             string
 	ClusterManagerNamespace        string
@@ -18,6 +20,11 @@ type HubConfig struct {
 	AddOnManagerEnabled            bool
 	MWReplicaSetEnabled            bool
 	AutoApproveUsers               string
+	// ResourceRequirementResourceType is the resource requirement resource type for the cluster manager managed containers.
+	ResourceRequirementResourceType operatorapiv1.ResourceQosClass
+	// ResourceRequirements is the resource requirements for the cluster manager managed containers.
+	// The type has to be []byte to use "indent" template function.
+	ResourceRequirements []byte
 }
 
 type Webhook struct {

manifests/klusterlet/management/klusterlet-agent-deployment.yaml

@@ -129,12 +129,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
-        {{if ne .ResourceRequirement "BestEffort"}}
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
-        {{end}}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
       volumes:
       - name: bootstrap-secret
         secret:

manifests/klusterlet/management/klusterlet-registration-deployment.yaml

@@ -120,12 +120,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
-        {{if ne .ResourceRequirement "BestEffort"}}
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
-        {{end}}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
       volumes:
       - name: bootstrap-secret
         secret:

manifests/klusterlet/management/klusterlet-work-deployment.yaml

@@ -113,12 +113,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
-        {{if ne .ResourceRequirement "BestEffort"}}
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
-        {{end}}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
       volumes:
       - name: hub-kubeconfig-secret
         secret:

pkg/operator/helpers/helpers.go

@@ -6,6 +6,7 @@ import (
 	"reflect"
 	"strings"
 
+	"github.com/ghodss/yaml"
 	"github.com/openshift/api"
 	"github.com/openshift/library-go/pkg/operator/events"
 	"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
@@ -605,11 +606,26 @@ func AgentNamespace(klusterlet *operatorapiv1.Klusterlet) string {
 }
 
 // ResourceType set default and return resource requirements override by user
-func ResourceType(klusterlet *operatorapiv1.Klusterlet) operatorapiv1.ResourceQosClass {
-	if klusterlet.Spec.ResourceRequirement == nil {
+func ResourceType(resourceRequirementAcquirer operatorapiv1.ResourceRequirementAcquirer) operatorapiv1.ResourceQosClass {
+	r := resourceRequirementAcquirer.GetResourceRequirement()
+	if r == nil {
 		return operatorapiv1.ResourceQosClassDefault
 	}
-	return klusterlet.Spec.ResourceRequirement.Type
+	return r.Type
+}
+
+// ResourceRequirements get resource requirements overridden by user for ResourceQosClassResourceRequirement type
+func ResourceRequirements(resourceRequirementAcquirer operatorapiv1.ResourceRequirementAcquirer) ([]byte, error) {
+	r := resourceRequirementAcquirer.GetResourceRequirement()
+	if r == nil || r.Type == operatorapiv1.ResourceQosClassBestEffort {
+		return nil, nil
+	}
+	marshal, err := yaml.Marshal(r.ResourceRequirements)
+	if err != nil {
+		klog.Errorf("failed to marshal resource requirement: %v", err)
+		return nil, err
+	}
+	return marshal, nil
 }
 
 // SyncSecret forked from:

pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go

@@ -128,6 +128,12 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 	clusterManagerMode := clusterManager.Spec.DeployOption.Mode
 	clusterManagerNamespace := helpers.ClusterManagerNamespace(clusterManagerName, clusterManagerMode)
 
+	resourceRequirements, err := helpers.ResourceRequirements(clusterManager)
+	if err != nil {
+		klog.Errorf("failed to parse resource requirements for cluster manager %s: %v", clusterManager.Name, err)
+		return err
+	}
+
 	// This config is used to render template of manifests.
 	config := manifests.HubConfig{
 		ClusterManagerName:      clusterManager.Name,
@@ -144,6 +150,8 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 		WorkWebhook: manifests.Webhook{
 			Port: defaultWebhookPort,
 		},
+		ResourceRequirementResourceType: helpers.ResourceType(clusterManager),
+		ResourceRequirements:            resourceRequirements,
 	}
 
 	var registrationFeatureMsgs, workFeatureMsgs, addonFeatureMsgs string