support work driver config for cluster manager.

Signed-off-by: morvencao <lcao@redhat.com>

deploy/cluster-manager/config/rbac/cluster_role.yaml

@@ -24,6 +24,7 @@ rules:
     - "work-controller-sa-kubeconfig"
     - "addon-manager-controller-sa-kubeconfig"
     - "external-hub-kubeconfig"
+    - "work-driver-config"
 - apiGroups: [""]
   resources: ["secrets"]
   verbs: ["create"]

deploy/cluster-manager/olm-catalog/latest/manifests/cluster-manager.clusterserviceversion.yaml

@@ -59,7 +59,7 @@ metadata:
     categories: Integration & Delivery,OpenShift Optional
     certified: "false"
     containerImage: quay.io/open-cluster-management/registration-operator:latest
-    createdAt: "2024-04-10T15:46:14Z"
+    createdAt: "2024-04-17T07:51:41Z"
     description: Manages the installation and upgrade of the ClusterManager.
     operators.operatorframework.io/builder: operator-sdk-v1.32.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
@@ -146,6 +146,7 @@ spec:
           - work-controller-sa-kubeconfig
           - addon-manager-controller-sa-kubeconfig
           - external-hub-kubeconfig
+          - work-driver-config
           resources:
           - secrets
           verbs:

deploy/klusterlet/config/crds/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml

@@ -181,6 +181,51 @@ spec:
                 description: RegistrationConfiguration contains the configuration
                   of registration
                 properties:
+                  bootstrapKubeConfigs:
+                    description: "BootstrapKubeConfigs defines the ordered list of
+                      bootstrap kubeconfigs. The order decides which bootstrap kubeconfig
+                      to use first when rebootstrap. \n When the agent loses the connection
+                      to the current hub over HubConnectionTimeoutSeconds, or the
+                      managedcluster CR is set `hubAcceptsClient=false` on the hub,
+                      the controller marks the related bootstrap kubeconfig as \"failed\".
+                      \n A failed bootstrapkubeconfig won't be used for the duration
+                      specified by SkipFailedBootstrapKubeConfigSeconds. But if the
+                      user updates the content of a failed bootstrapkubeconfig, the
+                      \"failed\" mark will be cleared."
+                    properties:
+                      localSecretsConfig:
+                        description: LocalSecretsConfig include a list of secrets
+                          that contains the kubeconfigs for ordered bootstrap kubeconifigs.
+                          The secrets must be in the same namespace where the agent
+                          controller runs.
+                        properties:
+                          hubConnectionTimeoutSeconds:
+                            default: 600
+                            description: HubConnectionTimeoutSeconds is used to set
+                              the timeout of connecting to the hub cluster. When agent
+                              loses the connection to the hub over the timeout seconds,
+                              the agent do a rebootstrap. By default is 10 mins.
+                            format: int32
+                            minimum: 180
+                            type: integer
+                          secretNames:
+                            description: SecretNames is a list of secret names. The
+                              secrets are in the same namespace where the agent controller
+                              runs.
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      type:
+                        default: None
+                        description: Type specifies the type of priority bootstrap
+                          kubeconfigs. By default, it is set to None, representing
+                          no priority bootstrap kubeconfigs are set.
+                        enum:
+                        - None
+                        - LocalSecrets
+                        type: string
+                    type: object
                   clientCertExpirationSeconds:
                     description: clientCertExpirationSeconds represents the seconds
                       of a client certificate to expire. If it is not set or 0, the

deploy/klusterlet/olm-catalog/latest/manifests/operator.open-cluster-management.io_klusterlets.yaml

@@ -181,6 +181,51 @@ spec:
                 description: RegistrationConfiguration contains the configuration
                   of registration
                 properties:
+                  bootstrapKubeConfigs:
+                    description: "BootstrapKubeConfigs defines the ordered list of
+                      bootstrap kubeconfigs. The order decides which bootstrap kubeconfig
+                      to use first when rebootstrap. \n When the agent loses the connection
+                      to the current hub over HubConnectionTimeoutSeconds, or the
+                      managedcluster CR is set `hubAcceptsClient=false` on the hub,
+                      the controller marks the related bootstrap kubeconfig as \"failed\".
+                      \n A failed bootstrapkubeconfig won't be used for the duration
+                      specified by SkipFailedBootstrapKubeConfigSeconds. But if the
+                      user updates the content of a failed bootstrapkubeconfig, the
+                      \"failed\" mark will be cleared."
+                    properties:
+                      localSecretsConfig:
+                        description: LocalSecretsConfig include a list of secrets
+                          that contains the kubeconfigs for ordered bootstrap kubeconifigs.
+                          The secrets must be in the same namespace where the agent
+                          controller runs.
+                        properties:
+                          hubConnectionTimeoutSeconds:
+                            default: 600
+                            description: HubConnectionTimeoutSeconds is used to set
+                              the timeout of connecting to the hub cluster. When agent
+                              loses the connection to the hub over the timeout seconds,
+                              the agent do a rebootstrap. By default is 10 mins.
+                            format: int32
+                            minimum: 180
+                            type: integer
+                          secretNames:
+                            description: SecretNames is a list of secret names. The
+                              secrets are in the same namespace where the agent controller
+                              runs.
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      type:
+                        default: None
+                        description: Type specifies the type of priority bootstrap
+                          kubeconfigs. By default, it is set to None, representing
+                          no priority bootstrap kubeconfigs are set.
+                        enum:
+                        - None
+                        - LocalSecrets
+                        type: string
+                    type: object
                   clientCertExpirationSeconds:
                     description: clientCertExpirationSeconds represents the seconds
                       of a client certificate to expire. If it is not set or 0, the

go.mod

@@ -35,7 +35,7 @@ require (
 	k8s.io/kube-aggregator v0.29.3
 	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
 	open-cluster-management.io/addon-framework v0.9.1-0.20240419070222-e703fc5a2556
-	open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c
+	open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad
 	open-cluster-management.io/sdk-go v0.13.1-0.20240416062924-20307e6fe090
 	sigs.k8s.io/controller-runtime v0.17.3
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96

go.sum

@@ -425,8 +425,8 @@ k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0g
 k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 open-cluster-management.io/addon-framework v0.9.1-0.20240419070222-e703fc5a2556 h1:X3vJEx9agC94l7SitpWZFDshISdL1niqVH0+diyqfJo=
 open-cluster-management.io/addon-framework v0.9.1-0.20240419070222-e703fc5a2556/go.mod h1:HayKCznnlyW+0dUJQGj5sNR6i3tvylSySD3YnvZkBtY=
-open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c h1:/iUoY6/PqBmcBq3v0+UBFvIcI39k/QPRGqpOv9XtDIc=
-open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c/go.mod h1:CuCPEzXDvOyxBB0H1d1eSeajbHqaeGEKq9c63vQc63w=
+open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad h1:DB3GpK5vzbGu9ss13bfodi8pGTkPcpdcLvOPEPMptTk=
+open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad/go.mod h1:yrNuMMpciXjXPnj2yznb6LTyrGliiTrFZAJDp/Ck3c4=
 open-cluster-management.io/sdk-go v0.13.1-0.20240416062924-20307e6fe090 h1:zFmHuW+ztdfUUNslqNW+H1WEcfdEUQHoRDbmdajX340=
 open-cluster-management.io/sdk-go v0.13.1-0.20240416062924-20307e6fe090/go.mod h1:w2OaxtCyegxeyFLU42UQ3oxUz01QdsBQkcHI17T/l48=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I=

manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml

@@ -46,7 +46,13 @@ spec:
         args:
           - "/work"
           - "manager"
-          - "--work-driver=kube"
+          {{ if .CloudEventsDriverEnabled }}
+          - "--work-driver={{ .WorkDriver }}"
+          {{ if ne .WorkDriver "kube" }}
+          - "--cloudevents-client-id=work-controller-$(POD_NAME)"
+          - "--work-driver-config=/var/run/secrets/work/config.yaml"
+          {{ end }}
+          {{ end }}
           {{ if .HostedMode }}
           - "--kubeconfig=/var/run/secrets/hub/kubeconfig"
           {{ end }}
@@ -97,6 +103,11 @@ spec:
           name: kubeconfig
           readOnly: true
         {{ end }}
+        {{ if and .CloudEventsDriverEnabled (ne .WorkDriver "kube") }}
+        - mountPath: /var/run/secrets/work
+          name: workdriverconfig
+          readOnly: true
+        {{ end }}
       volumes:
       - name: tmpdir
         emptyDir: { }
@@ -105,3 +116,8 @@ spec:
         secret:
           secretName: work-controller-sa-kubeconfig
       {{ end }}
+      {{ if and .CloudEventsDriverEnabled (ne .WorkDriver "kube") }}
+      - name: workdriverconfig
+        secret:
+          secretName: work-driver-config
+      {{ end }}

manifests/config.go

@@ -19,6 +19,8 @@ type HubConfig struct {
 	AddOnManagerImage              string
 	AddOnManagerEnabled            bool
 	MWReplicaSetEnabled            bool
+	CloudEventsDriverEnabled       bool
+	WorkDriver                     string
 	AutoApproveUsers               string
 	// ResourceRequirementResourceType is the resource requirement resource type for the cluster manager managed containers.
 	ResourceRequirementResourceType operatorapiv1.ResourceQosClass

pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go

@@ -58,7 +58,8 @@ type clusterManagerController struct {
 		mwctrEnabled, addonManagerEnabled bool) error
 	generateHubClusterClients func(hubConfig *rest.Config) (kubernetes.Interface, apiextensionsclient.Interface,
 		migrationclient.StorageVersionMigrationsGetter, error)
-	skipRemoveCRDs bool
+	skipRemoveCRDs    bool
+	operatorNamespace string
 }
 
 type clusterManagerReconcile interface {
@@ -83,6 +84,7 @@ func NewClusterManagerController(
 	configMapInformer corev1informers.ConfigMapInformer,
 	recorder events.Recorder,
 	skipRemoveCRDs bool,
+	operatorNamespace string,
 ) factory.Controller {
 	controller := &clusterManagerController{
 		operatorKubeClient: operatorKubeClient,
@@ -97,6 +99,7 @@ func NewClusterManagerController(
 		ensureSAKubeconfigs:       ensureSAKubeconfigs,
 		cache:                     resourceapply.NewResourceCache(),
 		skipRemoveCRDs:            skipRemoveCRDs,
+		operatorNamespace:         operatorNamespace,
 	}
 
 	return factory.New().WithSync(controller.sync).
@@ -132,6 +135,12 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 		return err
 	}
 
+	// default driver is kube
+	workDriver := operatorapiv1.WorkDriverTypeKube
+	if clusterManager.Spec.WorkConfiguration != nil && clusterManager.Spec.WorkConfiguration.WorkDriver != "" {
+		workDriver = clusterManager.Spec.WorkConfiguration.WorkDriver
+	}
+
 	// This config is used to render template of manifests.
 	config := manifests.HubConfig{
 		ClusterManagerName:      clusterManager.Name,
@@ -150,6 +159,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 		},
 		ResourceRequirementResourceType: helpers.ResourceType(clusterManager),
 		ResourceRequirements:            resourceRequirements,
+		WorkDriver:                      string(workDriver),
 	}
 
 	var registrationFeatureMsgs, workFeatureMsgs, addonFeatureMsgs string
@@ -169,6 +179,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 	}
 	config.WorkFeatureGates, workFeatureMsgs = helpers.ConvertToFeatureGateFlags("Work", workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates)
 	config.MWReplicaSetEnabled = helpers.FeatureGateEnabled(workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates, ocmfeature.ManifestWorkReplicaSet)
+	config.CloudEventsDriverEnabled = helpers.FeatureGateEnabled(workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates, ocmfeature.CloudEventsDrivers)
 
 	var addonFeatureGates []operatorapiv1.FeatureGate
 	if clusterManager.Spec.AddOnManagerConfiguration != nil {
@@ -211,6 +222,8 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 	reconcilers := []clusterManagerReconcile{
 		&crdReconcile{cache: n.cache, recorder: n.recorder, hubAPIExtensionClient: hubApiExtensionClient,
 			hubMigrationClient: hubMigrationClient, skipRemoveCRDs: n.skipRemoveCRDs},
+		&secretReconcile{cache: n.cache, recorder: n.recorder, operatorKubeClient: n.operatorKubeClient,
+			hubKubeClient: hubClient, operatorNamespace: n.operatorNamespace},
 		&hubReoncile{cache: n.cache, recorder: n.recorder, hubKubeClient: hubClient},
 		&runtimeReconcile{cache: n.cache, recorder: n.recorder, hubKubeConfig: hubKubeConfig, hubKubeClient: hubClient,
 			kubeClient: managementClient, ensureSAKubeconfigs: n.ensureSAKubeconfigs},

pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go

@@ -31,6 +31,7 @@ import (
 
 	fakeoperatorlient "open-cluster-management.io/api/client/operator/clientset/versioned/fake"
 	operatorinformers "open-cluster-management.io/api/client/operator/informers/externalversions"
+	ocmfeature "open-cluster-management.io/api/feature"
 	operatorapiv1 "open-cluster-management.io/api/operator/v1"
 	"open-cluster-management.io/sdk-go/pkg/patcher"
 
@@ -75,6 +76,7 @@ func newClusterManager(name string) *operatorapiv1.ClusterManager {
 			},
 			WorkConfiguration: &operatorapiv1.WorkConfiguration{
 				FeatureGates: []operatorapiv1.FeatureGate{featureGate},
+				WorkDriver:   operatorapiv1.WorkDriverTypeKube,
 			},
 		},
 	}
@@ -299,6 +301,56 @@ func ensureObject(t *testing.T, object runtime.Object, hubCore *operatorapiv1.Cl
 	}
 }
 
+func TestSyncSecret(t *testing.T) {
+	operatorNamespace := metav1.NamespaceDefault
+	clusterManager := newClusterManager("testhub")
+	clusterManager.Spec.WorkConfiguration.FeatureGates = append(clusterManager.Spec.WorkConfiguration.FeatureGates,
+		operatorapiv1.FeatureGate{
+			Feature: string(ocmfeature.CloudEventsDrivers),
+			Mode:    operatorapiv1.FeatureGateModeTypeEnable,
+		})
+	clusterManager.Spec.WorkConfiguration.WorkDriver = operatorapiv1.WorkDriverTypeGrpc
+	tc := newTestController(t, clusterManager)
+	tc.clusterManagerController.operatorNamespace = operatorNamespace
+	clusterManagerNamespace := helpers.ClusterManagerNamespace(clusterManager.Name, clusterManager.Spec.DeployOption.Mode)
+	setup(t, tc, nil)
+
+	syncContext := testingcommon.NewFakeSyncContext(t, "testhub")
+
+	err := tc.clusterManagerController.sync(ctx, syncContext)
+	if err == nil || err.Error() != "failed to sync secret as the source secret work-driver-config not found" {
+		t.Fatalf("Expected error 'failed to sync secret as the source secret work-driver-config not found' when sync but got, %v", err)
+	}
+
+	workDriverConfig := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "work-driver-config",
+		},
+		Data: map[string][]byte{
+			"config.yaml": []byte("url: grpc.example.com:8443"),
+		},
+	}
+
+	if _, err = tc.managementKubeClient.CoreV1().Secrets(operatorNamespace).Create(ctx, workDriverConfig, metav1.CreateOptions{}); err != nil {
+		t.Fatalf("Failed to create work driver config secret: %v", err)
+	}
+
+	err = tc.clusterManagerController.sync(ctx, syncContext)
+	if err != nil {
+		t.Fatalf("Expected no error when sync, %v", err)
+	}
+
+	// TODO: add test for secret sync condition
+	syncedSecret, err := tc.hubKubeClient.CoreV1().Secrets(clusterManagerNamespace).Get(ctx, "work-driver-config", metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("Failed to get synced work driver config secret: %v", err)
+	}
+
+	if string(syncedSecret.Data["config.yaml"]) != "url: grpc.example.com:8443" {
+		t.Fatalf("Expected secret data to be url: grpc.example.com:8443")
+	}
+}
+
 // TestSyncDeploy tests sync manifests of hub component
 func TestSyncDeploy(t *testing.T) {
 	clusterManager := newClusterManager("testhub")