support work driver config for cluster manager.

Signed-off-by: morvencao <lcao@redhat.com>

deploy/cluster-manager/config/crds/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml

@@ -323,6 +323,8 @@ spec:
                     type: string
                 type: object
               workConfiguration:
+                default:
+                  workDriver: kube
                 description: WorkConfiguration contains the configuration of work
                 properties:
                   featureGates:
@@ -356,6 +358,22 @@ spec:
                       - feature
                       type: object
                     type: array
+                  workDriver:
+                    default: kube
+                    description: "WorkDriver represents the type of work driver. Possible
+                      values are \"kube\", \"mqtt\", or \"grpc\". If not provided,
+                      the default value is \"kube\". If set to non-\"kube\" drivers,
+                      the klusterlet need to use the same driver. and the driver configuration
+                      must be provided in a secret named \"work-driver-config\" in
+                      the namespace where the cluster manager is running, adhering
+                      to the following structure: config.yaml: | <driver-config-in-yaml>
+                      \n For detailed driver configuration, please refer to the sdk-go
+                      documentation: https://github.com/open-cluster-management-io/sdk-go/blob/main/pkg/cloudevents/README.md#supported-protocols-and-drivers"
+                    enum:
+                    - kube
+                    - mqtt
+                    - grpc
+                    type: string
                 type: object
               workImagePullSpec:
                 default: quay.io/open-cluster-management/work

deploy/cluster-manager/config/rbac/cluster_role.yaml

@@ -24,6 +24,7 @@ rules:
     - "work-controller-sa-kubeconfig"
     - "addon-manager-controller-sa-kubeconfig"
     - "external-hub-kubeconfig"
+    - "work-driver-config"
 - apiGroups: [""]
   resources: ["secrets"]
   verbs: ["create"]

deploy/cluster-manager/olm-catalog/cluster-manager/manifests/cluster-manager.clusterserviceversion.yaml

@@ -146,6 +146,7 @@ spec:
           - work-controller-sa-kubeconfig
           - addon-manager-controller-sa-kubeconfig
           - external-hub-kubeconfig
+          - work-driver-config
           resources:
           - secrets
           verbs:

deploy/cluster-manager/olm-catalog/cluster-manager/manifests/operator.open-cluster-management.io_clustermanagers.yaml

@@ -323,6 +323,8 @@ spec:
                     type: string
                 type: object
               workConfiguration:
+                default:
+                  workDriver: kube
                 description: WorkConfiguration contains the configuration of work
                 properties:
                   featureGates:
@@ -356,6 +358,22 @@ spec:
                       - feature
                       type: object
                     type: array
+                  workDriver:
+                    default: kube
+                    description: "WorkDriver represents the type of work driver. Possible
+                      values are \"kube\", \"mqtt\", or \"grpc\". If not provided,
+                      the default value is \"kube\". If set to non-\"kube\" drivers,
+                      the klusterlet need to use the same driver. and the driver configuration
+                      must be provided in a secret named \"work-driver-config\" in
+                      the namespace where the cluster manager is running, adhering
+                      to the following structure: config.yaml: | <driver-config-in-yaml>
+                      \n For detailed driver configuration, please refer to the sdk-go
+                      documentation: https://github.com/open-cluster-management-io/sdk-go/blob/main/pkg/cloudevents/README.md#supported-protocols-and-drivers"
+                    enum:
+                    - kube
+                    - mqtt
+                    - grpc
+                    type: string
                 type: object
               workImagePullSpec:
                 default: quay.io/open-cluster-management/work

go.mod

@@ -35,7 +35,7 @@ require (
 	k8s.io/kube-aggregator v0.29.0
 	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
 	open-cluster-management.io/addon-framework v0.9.1-0.20240311065811-974d3377ecbd
-	open-cluster-management.io/api v0.13.0
+	open-cluster-management.io/api v0.13.1-0.20240320091526-19eb7b587540
 	open-cluster-management.io/sdk-go v0.13.1-0.20240313075541-00a94671ced1
 	sigs.k8s.io/controller-runtime v0.17.2
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96

go.sum

@@ -425,8 +425,8 @@ k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0g
 k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 open-cluster-management.io/addon-framework v0.9.1-0.20240311065811-974d3377ecbd h1:wKAzvMuzSuaheaLw6iiDx7sl5vsPiGm8+ZvZEHrPIDU=
 open-cluster-management.io/addon-framework v0.9.1-0.20240311065811-974d3377ecbd/go.mod h1:ZZvGRA/zmIjIww0reatmOxcXy0Eoghnat/Opszgo2NA=
-open-cluster-management.io/api v0.13.0 h1:dlcJEZlNlE0DmSDctK2s7iWKg9l+Tgb0V78Z040nMuk=
-open-cluster-management.io/api v0.13.0/go.mod h1:CuCPEzXDvOyxBB0H1d1eSeajbHqaeGEKq9c63vQc63w=
+open-cluster-management.io/api v0.13.1-0.20240320091526-19eb7b587540 h1:cx+9dB1V9yAmPW3urZwEG1RQOImoVoTy++u8Vch4HAY=
+open-cluster-management.io/api v0.13.1-0.20240320091526-19eb7b587540/go.mod h1:CuCPEzXDvOyxBB0H1d1eSeajbHqaeGEKq9c63vQc63w=
 open-cluster-management.io/sdk-go v0.13.1-0.20240313075541-00a94671ced1 h1:s3dJdi1eol+/8ek6JQuaEuoGPkK/wRyM9zowqzKHPDY=
 open-cluster-management.io/sdk-go v0.13.1-0.20240313075541-00a94671ced1/go.mod h1:sq+amR9Ls9JzMP5dypvlCx4jIGfDg45gicS67Z/MnlI=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I=

manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml

@@ -46,10 +46,15 @@ spec:
         args:
           - "/work"
           - "manager"
-          - "--work-driver=kube"
+          - "--work-driver={{ .WorkDriver }}"
+          {{ if eq .WorkDriver "kube" }}
           {{ if .HostedMode }}
           - "--kubeconfig=/var/run/secrets/hub/kubeconfig"
           {{ end }}
+          {{ else }}
+          - "--cloudevents-client-id=$(POD_NAME)"
+          - "--work-driver-config=/var/run/secrets/hub/config.yaml"
+          {{ end }}
         env:
           - name: POD_NAME
             valueFrom:
@@ -92,16 +97,28 @@ spec:
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp
+        {{ if eq .WorkDriver "kube" }}
         {{ if .HostedMode }}
         - mountPath: /var/run/secrets/hub
           name: kubeconfig
           readOnly: true
         {{ end }}
+        {{ else }}
+        - mountPath: /var/run/secrets/hub
+          name: workdriverconfig
+          readOnly: true
+        {{ end }}
       volumes:
       - name: tmpdir
         emptyDir: { }
+      {{ if eq .WorkDriver "kube" }}
       {{ if .HostedMode }}
       - name: kubeconfig
         secret:
           secretName: work-controller-sa-kubeconfig
       {{ end }}
+      {{ else }}
+      - name: workdriverconfig
+        secret:
+          secretName: work-driver-config
+      {{ end }}

manifests/config.go

@@ -19,6 +19,7 @@ type HubConfig struct {
 	AddOnManagerImage              string
 	AddOnManagerEnabled            bool
 	MWReplicaSetEnabled            bool
+	WorkDriver                     string
 	AutoApproveUsers               string
 	// ResourceRequirementResourceType is the resource requirement resource type for the cluster manager managed containers.
 	ResourceRequirementResourceType operatorapiv1.ResourceQosClass

pkg/operator/helpers/helpers.go

@@ -3,6 +3,7 @@ package helpers
 import (
 	"context"
 	"fmt"
+	"os"
 	"reflect"
 	"strings"
 
@@ -724,6 +725,78 @@ func GetHubKubeconfig(ctx context.Context,
 	}
 }
 
+// SyncWorkConfigSecret is used to sync the secret of work config from operator namespace to the cluster manager namespace.
+// For both Default and Hosted mode, the original secret of work config should be in the operator namespace.
+func SyncWorkConfigSecret(ctx context.Context,
+	operatorClient kubernetes.Interface,
+	clusterManagerNamespace string,
+) error {
+	// get the namespace of the operator
+	operatorNamespace, err := getOperatorNamespace()
+	if err != nil {
+		return err
+	}
+
+	// get secret of work config
+	workDriverSecret, err := operatorClient.CoreV1().Secrets(operatorNamespace).Get(ctx, WorkDriverConfig, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	// copy the secret of work config to new secret in the cluster manager namespace
+	newSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      workDriverSecret.Name,
+			Namespace: clusterManagerNamespace,
+		},
+		Data: workDriverSecret.Data,
+	}
+	// create or update secret of work config in the cluster manager namespace
+	_, err = operatorClient.CoreV1().Secrets(clusterManagerNamespace).Create(ctx, newSecret, metav1.CreateOptions{})
+	if err != nil {
+		if errors.IsAlreadyExists(err) {
+			_, err = operatorClient.CoreV1().Secrets(clusterManagerNamespace).Update(ctx, newSecret, metav1.UpdateOptions{})
+			return err
+		}
+		return err
+	}
+
+	return nil
+}
+
+// RemoveWorkConfigSecret is used to remove the secret of work config from the cluster manager namespace.
+func RemoveWorkConfigSecret(ctx context.Context,
+	operatorClient kubernetes.Interface,
+	clusterManagerNamespace string,
+) error {
+	if _, err := operatorClient.CoreV1().Namespaces().Get(ctx, clusterManagerNamespace, metav1.GetOptions{}); err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	err := operatorClient.CoreV1().Secrets(clusterManagerNamespace).Delete(ctx, WorkDriverConfig, metav1.DeleteOptions{})
+	if err != nil && errors.IsNotFound(err) {
+		return nil
+	}
+
+	return err
+}
+
+// getOperatorNamespace is used to get the namespace where the operator is running.
+func getOperatorNamespace() (string, error) {
+	podNamespace, exists := os.LookupEnv("POD_NAMESPACE")
+	if exists {
+		return podNamespace, nil
+	}
+	nsBytes, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+	if err != nil {
+		return "", err
+	}
+
+	return string(nsBytes), nil
+}
+
 func BuildFeatureCondition(invalidMsgs ...string) metav1.Condition {
 	if len(strings.Join(invalidMsgs, "")) == 0 {
 		return metav1.Condition{

pkg/operator/helpers/queuekey.go

@@ -36,6 +36,9 @@ const (
 	// ExternalManagedKubeConfigAgent is the secret name of kubeconfig secret to connecting to the managed cluster
 	// Only applicable to SingletonHosted mode, agent uses it to connect to the managed cluster.
 	ExternalManagedKubeConfigAgent = "external-managed-kubeconfig-agent"
+	// WorkDriverConfig is the secret name of work driver config to connect to work driver.
+	// Only applicable to non-kube work driver, hub controllers connect to work driver using this config.
+	WorkDriverConfig = "work-driver-config"
 
 	RegistrationWebhookSecret  = "registration-webhook-serving-cert"
 	RegistrationWebhookService = "cluster-manager-registration-webhook"

pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go

@@ -134,6 +134,12 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 		return err
 	}
 
+	// default driver is kube
+	workDriver := operatorapiv1.WorkDriverTypeKube
+	if clusterManager.Spec.WorkConfiguration.WorkDriver != "" {
+		workDriver = clusterManager.Spec.WorkConfiguration.WorkDriver
+	}
+
 	// This config is used to render template of manifests.
 	config := manifests.HubConfig{
 		ClusterManagerName:      clusterManager.Name,
@@ -152,6 +158,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 		},
 		ResourceRequirementResourceType: helpers.ResourceType(clusterManager),
 		ResourceRequirements:            resourceRequirements,
+		WorkDriver:                      string(workDriver),
 	}
 
 	var registrationFeatureMsgs, workFeatureMsgs, addonFeatureMsgs string
@@ -209,6 +216,17 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 	}
 	managementClient := n.operatorKubeClient // We assume that operator is always running on the management cluster.
 
+	// If the work driver is not kube, we need to sync the work driver config secret
+	if workDriver != operatorapiv1.WorkDriverTypeKube {
+		if err := helpers.SyncWorkConfigSecret(ctx, n.operatorKubeClient, clusterManagerNamespace); err != nil {
+			return err
+		}
+	} else {
+		if err := helpers.RemoveWorkConfigSecret(ctx, n.operatorKubeClient, clusterManagerNamespace); err != nil {
+			return err
+		}
+	}
+
 	var errs []error
 	reconcilers := []clusterManagerReconcile{
 		&crdReconcile{cache: n.cache, recorder: n.recorder, hubAPIExtensionClient: hubApiExtensionClient,

pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go

@@ -75,6 +75,7 @@ func newClusterManager(name string) *operatorapiv1.ClusterManager {
 			},
 			WorkConfiguration: &operatorapiv1.WorkConfiguration{
 				FeatureGates: []operatorapiv1.FeatureGate{featureGate},
+				WorkDriver:   operatorapiv1.WorkDriverTypeKube,
 			},
 		},
 	}
@@ -407,7 +408,7 @@ func TestSyncDelete(t *testing.T) {
 			deleteKubeActions = append(deleteKubeActions, deleteKubeAction)
 		}
 	}
-	testingcommon.AssertEqualNumber(t, len(deleteKubeActions), 28) // delete namespace both from the hub cluster and the mangement cluster
+	testingcommon.AssertEqualNumber(t, len(deleteKubeActions), 29) // delete namespace both from the hub cluster and the mangement cluster
 
 	var deleteCRDActions []clienttesting.DeleteActionImpl
 	crdActions := tc.apiExtensionClient.Actions()