add cloudevents driver feature gate.

Signed-off-by: morvencao <lcao@redhat.com>

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/valyala/fasttemplate v1.2.2
 	go.uber.org/zap v1.27.0
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
-	golang.org/x/net v0.21.0
+	golang.org/x/net v0.23.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.29.3
 	k8s.io/apiextensions-apiserver v0.29.3
@@ -35,7 +35,7 @@ require (
 	k8s.io/kube-aggregator v0.29.3
 	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
 	open-cluster-management.io/addon-framework v0.9.1
-	open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c
+	open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad
 	open-cluster-management.io/sdk-go v0.13.1-0.20240415030117-612344aae744
 	sigs.k8s.io/controller-runtime v0.17.3
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96
@@ -123,11 +123,11 @@ require (
 	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/oauth2 v0.16.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.17.0 // indirect

go.sum

@@ -303,8 +303,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -317,8 +317,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
 golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -338,13 +338,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -425,8 +425,8 @@ k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0g
 k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 open-cluster-management.io/addon-framework v0.9.1 h1:m6n/W29G/4KzMx+8mgC9P/ybuiyNHVy+O6AHEaWbfQg=
 open-cluster-management.io/addon-framework v0.9.1/go.mod h1:OEIFCEXhZKO/Grv08CB0T+TGzS0bLshw4G9u7Vw8dw0=
-open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c h1:/iUoY6/PqBmcBq3v0+UBFvIcI39k/QPRGqpOv9XtDIc=
-open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c/go.mod h1:CuCPEzXDvOyxBB0H1d1eSeajbHqaeGEKq9c63vQc63w=
+open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad h1:DB3GpK5vzbGu9ss13bfodi8pGTkPcpdcLvOPEPMptTk=
+open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad/go.mod h1:yrNuMMpciXjXPnj2yznb6LTyrGliiTrFZAJDp/Ck3c4=
 open-cluster-management.io/sdk-go v0.13.1-0.20240415030117-612344aae744 h1:dBO6eK3gHSoRpl8OckW1zyOp35BOI48rYgoCznrPn40=
 open-cluster-management.io/sdk-go v0.13.1-0.20240415030117-612344aae744/go.mod h1:w2OaxtCyegxeyFLU42UQ3oxUz01QdsBQkcHI17T/l48=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I=

manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml

@@ -46,15 +46,16 @@ spec:
         args:
           - "/work"
           - "manager"
+          {{ if .CloudEventsDriverEnabled }}
           - "--work-driver={{ .WorkDriver }}"
-          {{ if eq .WorkDriver "kube" }}
+          {{ if ne .WorkDriver "kube" }}
+          - "--cloudevents-client-id=work-controller-$(POD_NAME)"
+          - "--work-driver-config=/var/run/secrets/work/config.yaml"
+          {{ end }}
+          {{ end }}
           {{ if .HostedMode }}
           - "--kubeconfig=/var/run/secrets/hub/kubeconfig"
           {{ end }}
-          {{ else }}
-          - "--cloudevents-client-id=work-controller-$(POD_NAME)"
-          - "--work-driver-config=/var/run/secrets/hub/config.yaml"
-          {{ end }}
         env:
           - name: POD_NAME
             valueFrom:
@@ -97,27 +98,25 @@ spec:
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp
-        {{ if eq .WorkDriver "kube" }}
         {{ if .HostedMode }}
         - mountPath: /var/run/secrets/hub
           name: kubeconfig
           readOnly: true
         {{ end }}
-        {{ else }}
-        - mountPath: /var/run/secrets/hub
+        {{ if and .CloudEventsDriverEnabled (ne .WorkDriver "kube") }}
+        - mountPath: /var/run/secrets/work
           name: workdriverconfig
           readOnly: true
         {{ end }}
       volumes:
       - name: tmpdir
         emptyDir: { }
-      {{ if eq .WorkDriver "kube" }}
       {{ if .HostedMode }}
       - name: kubeconfig
         secret:
           secretName: work-controller-sa-kubeconfig
       {{ end }}
-      {{ else }}
+      {{ if and .CloudEventsDriverEnabled (ne .WorkDriver "kube") }}
       - name: workdriverconfig
         secret:
           secretName: work-driver-config

manifests/config.go

@@ -19,6 +19,7 @@ type HubConfig struct {
 	AddOnManagerImage              string
 	AddOnManagerEnabled            bool
 	MWReplicaSetEnabled            bool
+	CloudEventsDriverEnabled       bool
 	WorkDriver                     string
 	AutoApproveUsers               string
 	// ResourceRequirementResourceType is the resource requirement resource type for the cluster manager managed containers.

pkg/operator/helpers/queuekey.go

@@ -36,14 +36,12 @@ const (
 	// ExternalManagedKubeConfigAgent is the secret name of kubeconfig secret to connecting to the managed cluster
 	// Only applicable to SingletonHosted mode, agent uses it to connect to the managed cluster.
 	ExternalManagedKubeConfigAgent = "external-managed-kubeconfig-agent"
-	// WorkDriverConfig is the secret name of work driver config to connect to work driver.
-	// Only applicable to non-kube work driver, hub controllers connect to work driver using this config.
-	WorkDriverConfig = "work-driver-config"
 
 	RegistrationWebhookSecret  = "registration-webhook-serving-cert"
 	RegistrationWebhookService = "cluster-manager-registration-webhook"
 	WorkWebhookSecret          = "work-webhook-serving-cert" // #nosec G101
 	WorkWebhookService         = "cluster-manager-work-webhook"
+	WorkDriverConfig = "work-driver-config"
 
 	SignerSecret      = "signer-secret"
 	CaBundleConfigmap = "ca-bundle-configmap"

pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go

@@ -180,6 +180,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 	}
 	config.WorkFeatureGates, workFeatureMsgs = helpers.ConvertToFeatureGateFlags("Work", workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates)
 	config.MWReplicaSetEnabled = helpers.FeatureGateEnabled(workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates, ocmfeature.ManifestWorkReplicaSet)
+	config.CloudEventsDriverEnabled = helpers.FeatureGateEnabled(workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates, ocmfeature.CloudEventsDrivers)
 
 	var addonFeatureGates []operatorapiv1.FeatureGate
 	if clusterManager.Spec.AddOnManagerConfiguration != nil {

pkg/work/hub/options.go

@@ -13,7 +13,9 @@ type WorkHubManagerOptions struct {
 }
 
 func NewWorkHubManagerOptions() *WorkHubManagerOptions {
-	return &WorkHubManagerOptions{}
+	return &WorkHubManagerOptions{
+		WorkDriver: "kube",
+	}
 }
 
 // AddFlags register and binds the default flags

test/integration/operator/clustermanager_hosted_test.go

@@ -497,7 +497,28 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
 				if err != nil {
 					return err
 				}
-				clusterManager.Spec.WorkConfiguration.WorkDriver = work.ConfigTypeGRPC
+				featureGates := []operatorapiv1.FeatureGate{
+					{
+						Feature: string(feature.ManifestWorkReplicaSet),
+						Mode:    operatorapiv1.FeatureGateModeTypeEnable,
+					},
+					{
+						Feature: string(feature.CloudEventsDrivers),
+						Mode:    operatorapiv1.FeatureGateModeTypeEnable,
+					},
+				}
+				if clusterManager.Spec.WorkConfiguration != nil {
+					for _, fg := range clusterManager.Spec.WorkConfiguration.FeatureGates {
+						if fg.Feature != string(feature.ManifestWorkReplicaSet) &&
+							fg.Feature != string(feature.CloudEventsDrivers) {
+							featureGates = append(featureGates, fg)
+						}
+					}
+				}
+				clusterManager.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{
+					FeatureGates: featureGates,
+					WorkDriver:   work.ConfigTypeGRPC,
+				}
 				_, err = hostedOperatorClient.OperatorV1().ClusterManagers().Update(
 					context.Background(), clusterManager, metav1.UpdateOptions{})
 				return err

test/integration/operator/clustermanager_test.go

@@ -472,7 +472,28 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
 				if err != nil {
 					return err
 				}
-				clusterManager.Spec.WorkConfiguration.WorkDriver = work.ConfigTypeGRPC
+				featureGates := []operatorapiv1.FeatureGate{
+					{
+						Feature: string(feature.ManifestWorkReplicaSet),
+						Mode:    operatorapiv1.FeatureGateModeTypeEnable,
+					},
+					{
+						Feature: string(feature.CloudEventsDrivers),
+						Mode:    operatorapiv1.FeatureGateModeTypeEnable,
+					},
+				}
+				if clusterManager.Spec.WorkConfiguration != nil {
+					for _, fg := range clusterManager.Spec.WorkConfiguration.FeatureGates {
+						if fg.Feature != string(feature.ManifestWorkReplicaSet) &&
+							fg.Feature != string(feature.CloudEventsDrivers) {
+							featureGates = append(featureGates, fg)
+						}
+					}
+				}
+				clusterManager.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{
+					FeatureGates: featureGates,
+					WorkDriver:   work.ConfigTypeGRPC,
+				}
 				_, err = operatorClient.OperatorV1().ClusterManagers().Update(
 					context.Background(), clusterManager, metav1.UpdateOptions{})
 				return err