support work driver config for cluster manager.

Signed-off-by: morvencao <lcao@redhat.com>

deploy/cluster-manager/config/rbac/cluster_role.yaml

@@ -24,6 +24,7 @@ rules:
     - "work-controller-sa-kubeconfig"
     - "addon-manager-controller-sa-kubeconfig"
     - "external-hub-kubeconfig"
+    - "work-driver-config"
 - apiGroups: [""]
   resources: ["secrets"]
   verbs: ["create"]

deploy/cluster-manager/olm-catalog/latest/manifests/cluster-manager.clusterserviceversion.yaml

@@ -59,7 +59,7 @@ metadata:
     categories: Integration & Delivery,OpenShift Optional
     certified: "false"
     containerImage: quay.io/open-cluster-management/registration-operator:latest
-    createdAt: "2024-04-10T15:46:14Z"
+    createdAt: "2024-04-17T07:51:41Z"
     description: Manages the installation and upgrade of the ClusterManager.
     operators.operatorframework.io/builder: operator-sdk-v1.32.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
@@ -146,6 +146,7 @@ spec:
           - work-controller-sa-kubeconfig
           - addon-manager-controller-sa-kubeconfig
           - external-hub-kubeconfig
+          - work-driver-config
           resources:
           - secrets
           verbs:

manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml

@@ -46,10 +46,15 @@ spec:
         args:
           - "/work"
           - "manager"
-          - "--work-driver=kube"
+          - "--work-driver={{ .WorkDriver }}"
+          {{ if eq .WorkDriver "kube" }}
           {{ if .HostedMode }}
           - "--kubeconfig=/var/run/secrets/hub/kubeconfig"
           {{ end }}
+          {{ else }}
+          - "--cloudevents-client-id=work-controller-$(POD_NAME)"
+          - "--work-driver-config=/var/run/secrets/hub/config.yaml"
+          {{ end }}
         env:
           - name: POD_NAME
             valueFrom:
@@ -92,16 +97,28 @@ spec:
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp
+        {{ if eq .WorkDriver "kube" }}
         {{ if .HostedMode }}
         - mountPath: /var/run/secrets/hub
           name: kubeconfig
           readOnly: true
         {{ end }}
+        {{ else }}
+        - mountPath: /var/run/secrets/hub
+          name: workdriverconfig
+          readOnly: true
+        {{ end }}
       volumes:
       - name: tmpdir
         emptyDir: { }
+      {{ if eq .WorkDriver "kube" }}
       {{ if .HostedMode }}
       - name: kubeconfig
         secret:
           secretName: work-controller-sa-kubeconfig
       {{ end }}
+      {{ else }}
+      - name: workdriverconfig
+        secret:
+          secretName: work-driver-config
+      {{ end }}

manifests/config.go

@@ -19,6 +19,7 @@ type HubConfig struct {
 	AddOnManagerImage              string
 	AddOnManagerEnabled            bool
 	MWReplicaSetEnabled            bool
+	WorkDriver                     string
 	AutoApproveUsers               string
 	// ResourceRequirementResourceType is the resource requirement resource type for the cluster manager managed containers.
 	ResourceRequirementResourceType operatorapiv1.ResourceQosClass

pkg/operator/helpers/queuekey.go

@@ -36,6 +36,9 @@ const (
 	// ExternalManagedKubeConfigAgent is the secret name of kubeconfig secret to connecting to the managed cluster
 	// Only applicable to SingletonHosted mode, agent uses it to connect to the managed cluster.
 	ExternalManagedKubeConfigAgent = "external-managed-kubeconfig-agent"
+	// WorkDriverConfig is the secret name of work driver config to connect to work driver.
+	// Only applicable to non-kube work driver, hub controllers connect to work driver using this config.
+	WorkDriverConfig = "work-driver-config"
 
 	RegistrationWebhookSecret  = "registration-webhook-serving-cert"
 	RegistrationWebhookService = "cluster-manager-registration-webhook"

pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go

@@ -38,7 +38,8 @@ import (
 )
 
 const (
-	clusterManagerFinalizer = "operator.open-cluster-management.io/cluster-manager-cleanup"
+	clusterManagerFinalizer   = "operator.open-cluster-management.io/cluster-manager-cleanup"
+	secretSyncedConditionType = "SecretSynced"
 
 	defaultWebhookPort       = int32(9443)
 	clusterManagerReSyncTime = 5 * time.Second
@@ -58,7 +59,8 @@ type clusterManagerController struct {
 		mwctrEnabled, addonManagerEnabled bool) error
 	generateHubClusterClients func(hubConfig *rest.Config) (kubernetes.Interface, apiextensionsclient.Interface,
 		migrationclient.StorageVersionMigrationsGetter, error)
-	skipRemoveCRDs bool
+	skipRemoveCRDs    bool
+	operatorNamespace string
 }
 
 type clusterManagerReconcile interface {
@@ -83,6 +85,7 @@ func NewClusterManagerController(
 	configMapInformer corev1informers.ConfigMapInformer,
 	recorder events.Recorder,
 	skipRemoveCRDs bool,
+	operatorNamespace string,
 ) factory.Controller {
 	controller := &clusterManagerController{
 		operatorKubeClient: operatorKubeClient,
@@ -97,6 +100,7 @@ func NewClusterManagerController(
 		ensureSAKubeconfigs:       ensureSAKubeconfigs,
 		cache:                     resourceapply.NewResourceCache(),
 		skipRemoveCRDs:            skipRemoveCRDs,
+		operatorNamespace:         operatorNamespace,
 	}
 
 	return factory.New().WithSync(controller.sync).
@@ -132,6 +136,12 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 		return err
 	}
 
+	// default driver is kube
+	workDriver := operatorapiv1.WorkDriverTypeKube
+	if clusterManager.Spec.WorkConfiguration.WorkDriver != "" {
+		workDriver = clusterManager.Spec.WorkConfiguration.WorkDriver
+	}
+
 	// This config is used to render template of manifests.
 	config := manifests.HubConfig{
 		ClusterManagerName:      clusterManager.Name,
@@ -150,6 +160,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 		},
 		ResourceRequirementResourceType: helpers.ResourceType(clusterManager),
 		ResourceRequirements:            resourceRequirements,
+		WorkDriver:                      string(workDriver),
 	}
 
 	var registrationFeatureMsgs, workFeatureMsgs, addonFeatureMsgs string
@@ -207,6 +218,52 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 	}
 	managementClient := n.operatorKubeClient // We assume that operator is always running on the management cluster.
 
+	// If the work driver is not kube, we need to sync the work driver config secret
+	if workDriver != operatorapiv1.WorkDriverTypeKube {
+		// check the secret containing work driver config explicitly because
+		// resourceapply.SyncSecret below won't return err if the source secret is not found
+		_, err = n.operatorKubeClient.CoreV1().Secrets(n.operatorNamespace).Get(ctx, helpers.WorkDriverConfig, metav1.GetOptions{})
+		if err != nil {
+			if errors.IsNotFound(err) {
+				klog.Errorf("Secret %s not found in namespace %s", helpers.WorkDriverConfig, n.operatorNamespace)
+				conditionChanged := meta.SetStatusCondition(&clusterManager.Status.Conditions, metav1.Condition{
+					Type:    secretSyncedConditionType,
+					Status:  metav1.ConditionFalse,
+					Reason:  "WorkDriverConfigSecretNotFound",
+					Message: "Work driver config secret is not found",
+				})
+				if conditionChanged {
+					clusterManager.Status.ObservedGeneration = clusterManager.Generation
+					_, err := n.patcher.PatchStatus(ctx, clusterManager, clusterManager.Status, originalClusterManager.Status)
+					if err != nil {
+						return err
+					}
+				}
+			}
+			return err
+		} else {
+			_, _, err = resourceapply.SyncSecret(ctx, n.operatorKubeClient.CoreV1(), n.recorder,
+				n.operatorNamespace, helpers.WorkDriverConfig, clusterManagerNamespace, helpers.WorkDriverConfig,
+				[]metav1.OwnerReference{})
+			if err != nil {
+				return err
+			}
+			conditionChanged := meta.SetStatusCondition(&clusterManager.Status.Conditions, metav1.Condition{
+				Type:    secretSyncedConditionType,
+				Status:  metav1.ConditionTrue,
+				Reason:  "WorkDriverConfigSecretSynced",
+				Message: "Work driver config secret is synced",
+			})
+			if conditionChanged {
+				clusterManager.Status.ObservedGeneration = clusterManager.Generation
+				_, err := n.patcher.PatchStatus(ctx, clusterManager, clusterManager.Status, originalClusterManager.Status)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
 	var errs []error
 	reconcilers := []clusterManagerReconcile{
 		&crdReconcile{cache: n.cache, recorder: n.recorder, hubAPIExtensionClient: hubApiExtensionClient,

pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go

@@ -75,6 +75,7 @@ func newClusterManager(name string) *operatorapiv1.ClusterManager {
 			},
 			WorkConfiguration: &operatorapiv1.WorkConfiguration{
 				FeatureGates: []operatorapiv1.FeatureGate{featureGate},
+				WorkDriver:   operatorapiv1.WorkDriverTypeKube,
 			},
 		},
 	}
@@ -299,6 +300,68 @@ func ensureObject(t *testing.T, object runtime.Object, hubCore *operatorapiv1.Cl
 	}
 }
 
+func TestSyncSecret(t *testing.T) {
+	operatorNamespace := metav1.NamespaceDefault
+	clusterManager := newClusterManager("testhub")
+	clusterManager.Spec.WorkConfiguration.WorkDriver = operatorapiv1.WorkDriverTypeGrpc
+	tc := newTestController(t, clusterManager)
+	tc.clusterManagerController.operatorNamespace = operatorNamespace
+	clusterManagerNamespace := helpers.ClusterManagerNamespace(clusterManager.Name, clusterManager.Spec.DeployOption.Mode)
+	setup(t, tc, nil)
+
+	syncContext := testingcommon.NewFakeSyncContext(t, "testhub")
+
+	err := tc.clusterManagerController.sync(ctx, syncContext)
+	if err == nil {
+		t.Fatalf("Expected error when sync, %v", err)
+	}
+
+	clusterManager, err = tc.operatorClient.OperatorV1().ClusterManagers().Get(ctx, clusterManager.Name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("Failed to get cluster manager: %v", err)
+	}
+
+	if !meta.IsStatusConditionFalse(clusterManager.Status.Conditions, secretSyncedConditionType) {
+		t.Fatalf("Expected secretSyncedConditionType to be false")
+	}
+
+	workDriverConfig := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "work-driver-config",
+		},
+		Data: map[string][]byte{
+			"config.yaml": []byte("url: grpc.example.com:8443"),
+		},
+	}
+
+	if _, err = tc.managementKubeClient.CoreV1().Secrets(operatorNamespace).Create(ctx, workDriverConfig, metav1.CreateOptions{}); err != nil {
+		t.Fatalf("Failed to create work driver config secret: %v", err)
+	}
+
+	err = tc.clusterManagerController.sync(ctx, syncContext)
+	if err != nil {
+		t.Fatalf("Expected no error when sync, %v", err)
+	}
+
+	clusterManager, err = tc.operatorClient.OperatorV1().ClusterManagers().Get(ctx, clusterManager.Name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("Failed to get cluster manager: %v", err)
+	}
+
+	if !meta.IsStatusConditionTrue(clusterManager.Status.Conditions, secretSyncedConditionType) {
+		t.Fatalf("Expected secretSyncedConditionType to be true")
+	}
+
+	syncedSecret, err := tc.managementKubeClient.CoreV1().Secrets(clusterManagerNamespace).Get(ctx, "work-driver-config", metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("Failed to get synced work driver config secret: %v", err)
+	}
+
+	if string(syncedSecret.Data["config.yaml"]) != "url: grpc.example.com:8443" {
+		t.Fatalf("Expected secret data to be url: grpc.example.com:8443")
+	}
+}
+
 // TestSyncDeploy tests sync manifests of hub component
 func TestSyncDeploy(t *testing.T) {
 	clusterManager := newClusterManager("testhub")

pkg/operator/operators/clustermanager/options.go

@@ -74,7 +74,9 @@ func (o *Options) RunClusterManagerOperator(ctx context.Context, controllerConte
 		kubeInformer.Apps().V1().Deployments(),
 		kubeInformer.Core().V1().ConfigMaps(),
 		controllerContext.EventRecorder,
-		o.SkipRemoveCRDs)
+		o.SkipRemoveCRDs,
+		controllerContext.OperatorNamespace,
+	)
 
 	statusController := clustermanagerstatuscontroller.NewClusterManagerStatusController(
 		operatorClient.OperatorV1().ClusterManagers(),