From 6b86d67455e23ee8b94c36b4d89857a01477ff49 Mon Sep 17 00:00:00 2001 From: morvencao Date: Wed, 17 Apr 2024 08:01:34 +0000 Subject: [PATCH] support work driver config for cluster manager. Signed-off-by: morvencao --- .../config/rbac/cluster_role.yaml | 1 + ...cluster-manager.clusterserviceversion.yaml | 3 +- ...ger-manifestworkreplicaset-deployment.yaml | 19 ++- manifests/config.go | 1 + pkg/operator/helpers/queuekey.go | 3 + .../clustermanager_controller.go | 61 ++++++++- .../clustermanager_controller_test.go | 63 +++++++++ .../operators/clustermanager/options.go | 4 +- .../operator/clustermanager_hosted_test.go | 122 +++++++++++++++++ .../operator/clustermanager_test.go | 127 +++++++++++++++++- .../operator/integration_suite_test.go | 6 +- 11 files changed, 401 insertions(+), 9 deletions(-) diff --git a/deploy/cluster-manager/config/rbac/cluster_role.yaml b/deploy/cluster-manager/config/rbac/cluster_role.yaml index ce37bcd7a..5bf5203a5 100644 --- a/deploy/cluster-manager/config/rbac/cluster_role.yaml +++ b/deploy/cluster-manager/config/rbac/cluster_role.yaml @@ -24,6 +24,7 @@ rules: - "work-controller-sa-kubeconfig" - "addon-manager-controller-sa-kubeconfig" - "external-hub-kubeconfig" + - "work-driver-config" - apiGroups: [""] resources: ["secrets"] verbs: ["create"] diff --git a/deploy/cluster-manager/olm-catalog/latest/manifests/cluster-manager.clusterserviceversion.yaml b/deploy/cluster-manager/olm-catalog/latest/manifests/cluster-manager.clusterserviceversion.yaml index d4b866d84..3ce24d5bc 100644 --- a/deploy/cluster-manager/olm-catalog/latest/manifests/cluster-manager.clusterserviceversion.yaml +++ b/deploy/cluster-manager/olm-catalog/latest/manifests/cluster-manager.clusterserviceversion.yaml @@ -59,7 +59,7 @@ metadata: categories: Integration & Delivery,OpenShift Optional certified: "false" containerImage: quay.io/open-cluster-management/registration-operator:latest - createdAt: "2024-04-10T15:46:14Z" + createdAt: "2024-04-17T07:51:41Z" description: Manages the installation and upgrade of the ClusterManager. operators.operatorframework.io/builder: operator-sdk-v1.32.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 @@ -146,6 +146,7 @@ spec: - work-controller-sa-kubeconfig - addon-manager-controller-sa-kubeconfig - external-hub-kubeconfig + - work-driver-config resources: - secrets verbs: diff --git a/manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml index 54b8d0a87..da46af2ad 100644 --- a/manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml @@ -46,10 +46,15 @@ spec: args: - "/work" - "manager" - - "--work-driver=kube" + - "--work-driver={{ .WorkDriver }}" + {{ if eq .WorkDriver "kube" }} {{ if .HostedMode }} - "--kubeconfig=/var/run/secrets/hub/kubeconfig" {{ end }} + {{ else }} + - "--cloudevents-client-id=work-controller-$(POD_NAME)" + - "--work-driver-config=/var/run/secrets/hub/config.yaml" + {{ end }} env: - name: POD_NAME valueFrom: @@ -92,16 +97,28 @@ spec: volumeMounts: - name: tmpdir mountPath: /tmp + {{ if eq .WorkDriver "kube" }} {{ if .HostedMode }} - mountPath: /var/run/secrets/hub name: kubeconfig readOnly: true {{ end }} + {{ else }} + - mountPath: /var/run/secrets/hub + name: workdriverconfig + readOnly: true + {{ end }} volumes: - name: tmpdir emptyDir: { } + {{ if eq .WorkDriver "kube" }} {{ if .HostedMode }} - name: kubeconfig secret: secretName: work-controller-sa-kubeconfig {{ end }} + {{ else }} + - name: workdriverconfig + secret: + secretName: work-driver-config + {{ end }} diff --git a/manifests/config.go b/manifests/config.go index fddd15d95..d92e9cb0b 100644 --- a/manifests/config.go +++ b/manifests/config.go @@ -19,6 +19,7 @@ type HubConfig struct { AddOnManagerImage string AddOnManagerEnabled bool MWReplicaSetEnabled bool + WorkDriver string AutoApproveUsers string // ResourceRequirementResourceType is the resource requirement resource type for the cluster manager managed containers. ResourceRequirementResourceType operatorapiv1.ResourceQosClass diff --git a/pkg/operator/helpers/queuekey.go b/pkg/operator/helpers/queuekey.go index 7c59f9b00..90b8c3bd0 100644 --- a/pkg/operator/helpers/queuekey.go +++ b/pkg/operator/helpers/queuekey.go @@ -36,6 +36,9 @@ const ( // ExternalManagedKubeConfigAgent is the secret name of kubeconfig secret to connecting to the managed cluster // Only applicable to SingletonHosted mode, agent uses it to connect to the managed cluster. ExternalManagedKubeConfigAgent = "external-managed-kubeconfig-agent" + // WorkDriverConfig is the secret name of work driver config to connect to work driver. + // Only applicable to non-kube work driver, hub controllers connect to work driver using this config. + WorkDriverConfig = "work-driver-config" RegistrationWebhookSecret = "registration-webhook-serving-cert" RegistrationWebhookService = "cluster-manager-registration-webhook" diff --git a/pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go b/pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go index 15d9e808e..ad598377a 100644 --- a/pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go +++ b/pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go @@ -38,7 +38,8 @@ import ( ) const ( - clusterManagerFinalizer = "operator.open-cluster-management.io/cluster-manager-cleanup" + clusterManagerFinalizer = "operator.open-cluster-management.io/cluster-manager-cleanup" + secretSyncedConditionType = "SecretSynced" defaultWebhookPort = int32(9443) clusterManagerReSyncTime = 5 * time.Second @@ -58,7 +59,8 @@ type clusterManagerController struct { mwctrEnabled, addonManagerEnabled bool) error generateHubClusterClients func(hubConfig *rest.Config) (kubernetes.Interface, apiextensionsclient.Interface, migrationclient.StorageVersionMigrationsGetter, error) - skipRemoveCRDs bool + skipRemoveCRDs bool + operatorNamespace string } type clusterManagerReconcile interface { @@ -83,6 +85,7 @@ func NewClusterManagerController( configMapInformer corev1informers.ConfigMapInformer, recorder events.Recorder, skipRemoveCRDs bool, + operatorNamespace string, ) factory.Controller { controller := &clusterManagerController{ operatorKubeClient: operatorKubeClient, @@ -97,6 +100,7 @@ func NewClusterManagerController( ensureSAKubeconfigs: ensureSAKubeconfigs, cache: resourceapply.NewResourceCache(), skipRemoveCRDs: skipRemoveCRDs, + operatorNamespace: operatorNamespace, } return factory.New().WithSync(controller.sync). @@ -132,6 +136,12 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f return err } + // default driver is kube + workDriver := operatorapiv1.WorkDriverTypeKube + if clusterManager.Spec.WorkConfiguration.WorkDriver != "" { + workDriver = clusterManager.Spec.WorkConfiguration.WorkDriver + } + // This config is used to render template of manifests. config := manifests.HubConfig{ ClusterManagerName: clusterManager.Name, @@ -150,6 +160,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f }, ResourceRequirementResourceType: helpers.ResourceType(clusterManager), ResourceRequirements: resourceRequirements, + WorkDriver: string(workDriver), } var registrationFeatureMsgs, workFeatureMsgs, addonFeatureMsgs string @@ -207,6 +218,52 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f } managementClient := n.operatorKubeClient // We assume that operator is always running on the management cluster. + // If the work driver is not kube, we need to sync the work driver config secret + if workDriver != operatorapiv1.WorkDriverTypeKube { + // check the secret containing work driver config explicitly because + // resourceapply.SyncSecret below won't return err if the source secret is not found + _, err = n.operatorKubeClient.CoreV1().Secrets(n.operatorNamespace).Get(ctx, helpers.WorkDriverConfig, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + klog.Errorf("Secret %s not found in namespace %s", helpers.WorkDriverConfig, n.operatorNamespace) + conditionChanged := meta.SetStatusCondition(&clusterManager.Status.Conditions, metav1.Condition{ + Type: secretSyncedConditionType, + Status: metav1.ConditionFalse, + Reason: "WorkDriverConfigSecretNotFound", + Message: "Work driver config secret is not found", + }) + if conditionChanged { + clusterManager.Status.ObservedGeneration = clusterManager.Generation + _, err := n.patcher.PatchStatus(ctx, clusterManager, clusterManager.Status, originalClusterManager.Status) + if err != nil { + return err + } + } + } + return err + } else { + _, _, err = resourceapply.SyncSecret(ctx, n.operatorKubeClient.CoreV1(), n.recorder, + n.operatorNamespace, helpers.WorkDriverConfig, clusterManagerNamespace, helpers.WorkDriverConfig, + []metav1.OwnerReference{}) + if err != nil { + return err + } + conditionChanged := meta.SetStatusCondition(&clusterManager.Status.Conditions, metav1.Condition{ + Type: secretSyncedConditionType, + Status: metav1.ConditionTrue, + Reason: "WorkDriverConfigSecretSynced", + Message: "Work driver config secret is synced", + }) + if conditionChanged { + clusterManager.Status.ObservedGeneration = clusterManager.Generation + _, err := n.patcher.PatchStatus(ctx, clusterManager, clusterManager.Status, originalClusterManager.Status) + if err != nil { + return err + } + } + } + } + var errs []error reconcilers := []clusterManagerReconcile{ &crdReconcile{cache: n.cache, recorder: n.recorder, hubAPIExtensionClient: hubApiExtensionClient, diff --git a/pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go b/pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go index 6ef5bbe0f..dc226cd4b 100644 --- a/pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go +++ b/pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go @@ -75,6 +75,7 @@ func newClusterManager(name string) *operatorapiv1.ClusterManager { }, WorkConfiguration: &operatorapiv1.WorkConfiguration{ FeatureGates: []operatorapiv1.FeatureGate{featureGate}, + WorkDriver: operatorapiv1.WorkDriverTypeKube, }, }, } @@ -299,6 +300,68 @@ func ensureObject(t *testing.T, object runtime.Object, hubCore *operatorapiv1.Cl } } +func TestSyncSecret(t *testing.T) { + operatorNamespace := metav1.NamespaceDefault + clusterManager := newClusterManager("testhub") + clusterManager.Spec.WorkConfiguration.WorkDriver = operatorapiv1.WorkDriverTypeGrpc + tc := newTestController(t, clusterManager) + tc.clusterManagerController.operatorNamespace = operatorNamespace + clusterManagerNamespace := helpers.ClusterManagerNamespace(clusterManager.Name, clusterManager.Spec.DeployOption.Mode) + setup(t, tc, nil) + + syncContext := testingcommon.NewFakeSyncContext(t, "testhub") + + err := tc.clusterManagerController.sync(ctx, syncContext) + if err == nil { + t.Fatalf("Expected error when sync, %v", err) + } + + clusterManager, err = tc.operatorClient.OperatorV1().ClusterManagers().Get(ctx, clusterManager.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Failed to get cluster manager: %v", err) + } + + if !meta.IsStatusConditionFalse(clusterManager.Status.Conditions, secretSyncedConditionType) { + t.Fatalf("Expected secretSyncedConditionType to be false") + } + + workDriverConfig := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "work-driver-config", + }, + Data: map[string][]byte{ + "config.yaml": []byte("url: grpc.example.com:8443"), + }, + } + + if _, err = tc.managementKubeClient.CoreV1().Secrets(operatorNamespace).Create(ctx, workDriverConfig, metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create work driver config secret: %v", err) + } + + err = tc.clusterManagerController.sync(ctx, syncContext) + if err != nil { + t.Fatalf("Expected no error when sync, %v", err) + } + + clusterManager, err = tc.operatorClient.OperatorV1().ClusterManagers().Get(ctx, clusterManager.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Failed to get cluster manager: %v", err) + } + + if !meta.IsStatusConditionTrue(clusterManager.Status.Conditions, secretSyncedConditionType) { + t.Fatalf("Expected secretSyncedConditionType to be true") + } + + syncedSecret, err := tc.managementKubeClient.CoreV1().Secrets(clusterManagerNamespace).Get(ctx, "work-driver-config", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Failed to get synced work driver config secret: %v", err) + } + + if string(syncedSecret.Data["config.yaml"]) != "url: grpc.example.com:8443" { + t.Fatalf("Expected secret data to be url: grpc.example.com:8443") + } +} + // TestSyncDeploy tests sync manifests of hub component func TestSyncDeploy(t *testing.T) { clusterManager := newClusterManager("testhub") diff --git a/pkg/operator/operators/clustermanager/options.go b/pkg/operator/operators/clustermanager/options.go index d929596e6..faf213e26 100644 --- a/pkg/operator/operators/clustermanager/options.go +++ b/pkg/operator/operators/clustermanager/options.go @@ -74,7 +74,9 @@ func (o *Options) RunClusterManagerOperator(ctx context.Context, controllerConte kubeInformer.Apps().V1().Deployments(), kubeInformer.Core().V1().ConfigMaps(), controllerContext.EventRecorder, - o.SkipRemoveCRDs) + o.SkipRemoveCRDs, + controllerContext.OperatorNamespace, + ) statusController := clustermanagerstatuscontroller.NewClusterManagerStatusController( operatorClient.OperatorV1().ClusterManagers(), diff --git a/test/integration/operator/clustermanager_hosted_test.go b/test/integration/operator/clustermanager_hosted_test.go index 7c3238072..f584fd90b 100644 --- a/test/integration/operator/clustermanager_hosted_test.go +++ b/test/integration/operator/clustermanager_hosted_test.go @@ -11,12 +11,14 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/cert" "open-cluster-management.io/api/feature" operatorapiv1 "open-cluster-management.io/api/operator/v1" + "open-cluster-management.io/sdk-go/pkg/cloudevents/work" "open-cluster-management.io/ocm/pkg/operator/helpers" "open-cluster-management.io/ocm/test/integration/util" @@ -487,6 +489,126 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() { }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) }) + ginkgo.It("should have expected work driver when work driver is updated", func() { + ginkgo.By("Update work driver to grpc") + gomega.Eventually(func() error { + clusterManager, err := hostedOperatorClient.OperatorV1().ClusterManagers().Get( + context.Background(), clusterManagerName, metav1.GetOptions{}) + if err != nil { + return err + } + clusterManager.Spec.WorkConfiguration.WorkDriver = work.ConfigTypeGRPC + _, err = hostedOperatorClient.OperatorV1().ClusterManagers().Update( + context.Background(), clusterManager, metav1.UpdateOptions{}) + return err + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + gomega.Eventually(func() error { + actual, err := hostedOperatorClient.OperatorV1().ClusterManagers().Get( + context.Background(), clusterManagerName, metav1.GetOptions{}) + if err != nil { + return err + } + if !meta.IsStatusConditionFalse(actual.Status.Conditions, "SecretSynced") { + return fmt.Errorf("should get WorkDriverConfigSecretSynced condition false") + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + _, err := hostedKubeClient.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: helpers.WorkDriverConfig, + Namespace: "default", + }, + Data: map[string][]byte{ + "config.yaml": []byte("url: grpc.example.com:8443"), + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + actual, err := hostedOperatorClient.OperatorV1().ClusterManagers().Get( + context.Background(), clusterManagerName, metav1.GetOptions{}) + if err != nil { + return err + } + if !meta.IsStatusConditionTrue(actual.Status.Conditions, "SecretSynced") { + return fmt.Errorf("should get WorkDriverConfigSecretSynced condition true") + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + actual, err := hostedKubeClient.AppsV1().Deployments(hubNamespaceHosted).Get(context.Background(), + hubWorkControllerDeployment, metav1.GetOptions{}) + if err != nil { + return err + } + foundArg := false + for _, arg := range actual.Spec.Template.Spec.Containers[0].Args { + if arg == "--work-driver=grpc" { + foundArg = true + } + } + if !foundArg { + return fmt.Errorf("do not find the --work-driver=grpc args, got %v", actual.Spec.Template.Spec.Containers[0].Args) + } + foundVol := false + for _, vol := range actual.Spec.Template.Spec.Volumes { + if vol.Name == "workdriverconfig" && vol.Secret.SecretName == helpers.WorkDriverConfig { + foundVol = true + } + } + if !foundVol { + return fmt.Errorf("do not find the workdriverconfig volume, got %v", actual.Spec.Template.Spec.Volumes) + } + return nil + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + gomega.Eventually(func() error { + workConfigSecret, err := hostedKubeClient.CoreV1().Secrets(hubNamespaceHosted).Get(context.Background(), + helpers.WorkDriverConfig, metav1.GetOptions{}) + if err != nil { + return err + } + if string(workConfigSecret.Data["config.yaml"]) != "url: grpc.example.com:8443" { + return fmt.Errorf("do not find the expected config.yaml, got %v", string(workConfigSecret.Data["config.yaml"])) + } + return nil + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + ginkgo.By("Revert work driver back to kube") + gomega.Eventually(func() error { + clusterManager, err := hostedOperatorClient.OperatorV1().ClusterManagers().Get( + context.Background(), clusterManagerName, metav1.GetOptions{}) + if err != nil { + return err + } + clusterManager.Spec.WorkConfiguration.WorkDriver = operatorapiv1.WorkDriverTypeKube + _, err = hostedOperatorClient.OperatorV1().ClusterManagers().Update( + context.Background(), clusterManager, metav1.UpdateOptions{}) + return err + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + gomega.Eventually(func() error { + actual, err := hostedKubeClient.AppsV1().Deployments(hubNamespaceHosted).Get(context.Background(), + hubWorkControllerDeployment, metav1.GetOptions{}) + if err != nil { + return err + } + for _, arg := range actual.Spec.Template.Spec.Containers[0].Args { + if arg == "--work-driver=grpc" { + return err + } + } + return nil + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + err = hostedKubeClient.CoreV1().Secrets("default").Delete(context.Background(), + helpers.WorkDriverConfig, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("should have expected resource created/deleted successfully when feature gates AddOnManager enabled/disabled", func() { ginkgo.By("Disable AddOnManager feature gate") // Check addon manager disable mode diff --git a/test/integration/operator/clustermanager_test.go b/test/integration/operator/clustermanager_test.go index cf0b4a6d6..59f4b94c1 100644 --- a/test/integration/operator/clustermanager_test.go +++ b/test/integration/operator/clustermanager_test.go @@ -11,12 +11,14 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/client-go/util/cert" "open-cluster-management.io/api/feature" operatorapiv1 "open-cluster-management.io/api/operator/v1" + "open-cluster-management.io/sdk-go/pkg/cloudevents/work" "open-cluster-management.io/ocm/pkg/operator/helpers" "open-cluster-management.io/ocm/pkg/operator/operators/clustermanager" @@ -44,8 +46,9 @@ func startHubOperator(ctx context.Context, mode operatorapiv1.InstallMode) { o := &clustermanager.Options{} err := o.RunClusterManagerOperator(ctx, &controllercmd.ControllerContext{ - KubeConfig: config, - EventRecorder: util.NewIntegrationTestEventRecorder("integration"), + KubeConfig: config, + EventRecorder: util.NewIntegrationTestEventRecorder("integration"), + OperatorNamespace: metav1.NamespaceDefault, }) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -461,6 +464,126 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() { }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) }) + ginkgo.It("should have expected work driver when work driver is updated", func() { + ginkgo.By("Update work driver to grpc") + gomega.Eventually(func() error { + clusterManager, err := operatorClient.OperatorV1().ClusterManagers().Get( + context.Background(), clusterManagerName, metav1.GetOptions{}) + if err != nil { + return err + } + clusterManager.Spec.WorkConfiguration.WorkDriver = work.ConfigTypeGRPC + _, err = operatorClient.OperatorV1().ClusterManagers().Update( + context.Background(), clusterManager, metav1.UpdateOptions{}) + return err + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + gomega.Eventually(func() error { + actual, err := operatorClient.OperatorV1().ClusterManagers().Get( + context.Background(), clusterManagerName, metav1.GetOptions{}) + if err != nil { + return err + } + if !meta.IsStatusConditionFalse(actual.Status.Conditions, "SecretSynced") { + return fmt.Errorf("should get WorkDriverConfigSecretSynced condition false") + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + _, err := kubeClient.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: helpers.WorkDriverConfig, + Namespace: "default", + }, + Data: map[string][]byte{ + "config.yaml": []byte("url: grpc.example.com:8443"), + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + actual, err := operatorClient.OperatorV1().ClusterManagers().Get( + context.Background(), clusterManagerName, metav1.GetOptions{}) + if err != nil { + return err + } + if !meta.IsStatusConditionTrue(actual.Status.Conditions, "SecretSynced") { + return fmt.Errorf("should get WorkDriverConfigSecretSynced condition true") + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + actual, err := kubeClient.AppsV1().Deployments(hubNamespace).Get(context.Background(), + hubWorkControllerDeployment, metav1.GetOptions{}) + if err != nil { + return err + } + foundArg := false + for _, arg := range actual.Spec.Template.Spec.Containers[0].Args { + if arg == "--work-driver=grpc" { + foundArg = true + } + } + if !foundArg { + return fmt.Errorf("do not find the --work-driver=grpc args, got %v", actual.Spec.Template.Spec.Containers[0].Args) + } + foundVol := false + for _, vol := range actual.Spec.Template.Spec.Volumes { + if vol.Name == "workdriverconfig" && vol.Secret.SecretName == helpers.WorkDriverConfig { + foundVol = true + } + } + if !foundVol { + return fmt.Errorf("do not find the workdriverconfig volume, got %v", actual.Spec.Template.Spec.Volumes) + } + return nil + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + gomega.Eventually(func() error { + workConfigSecret, err := kubeClient.CoreV1().Secrets(hubNamespace).Get(context.Background(), + helpers.WorkDriverConfig, metav1.GetOptions{}) + if err != nil { + return err + } + if string(workConfigSecret.Data["config.yaml"]) != "url: grpc.example.com:8443" { + return fmt.Errorf("do not find the expected config.yaml, got %v", string(workConfigSecret.Data["config.yaml"])) + } + return nil + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + ginkgo.By("Revert work driver back to kube") + gomega.Eventually(func() error { + clusterManager, err := operatorClient.OperatorV1().ClusterManagers().Get( + context.Background(), clusterManagerName, metav1.GetOptions{}) + if err != nil { + return err + } + clusterManager.Spec.WorkConfiguration.WorkDriver = operatorapiv1.WorkDriverTypeKube + _, err = operatorClient.OperatorV1().ClusterManagers().Update( + context.Background(), clusterManager, metav1.UpdateOptions{}) + return err + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + gomega.Eventually(func() error { + actual, err := kubeClient.AppsV1().Deployments(hubNamespace).Get(context.Background(), + hubWorkControllerDeployment, metav1.GetOptions{}) + if err != nil { + return err + } + for _, arg := range actual.Spec.Template.Spec.Containers[0].Args { + if arg == "--work-driver=grpc" { + return err + } + } + return nil + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + err = kubeClient.CoreV1().Secrets("default").Delete(context.Background(), + helpers.WorkDriverConfig, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("should have expected resource created/deleted successfully when feature gates AddOnManager enabled/disabled", func() { ginkgo.By("Check addon manager disable mode") gomega.Eventually(func() error { diff --git a/test/integration/operator/integration_suite_test.go b/test/integration/operator/integration_suite_test.go index 605ef73c0..bea6e7b9a 100644 --- a/test/integration/operator/integration_suite_test.go +++ b/test/integration/operator/integration_suite_test.go @@ -139,6 +139,7 @@ var _ = ginkgo.BeforeSuite(func() { Mode: operatorapiv1.InstallModeDefault, }, WorkConfiguration: &operatorapiv1.WorkConfiguration{ + WorkDriver: operatorapiv1.WorkDriverTypeKube, FeatureGates: []operatorapiv1.FeatureGate{ { Feature: string(feature.NilExecutorValidating), @@ -177,10 +178,11 @@ var _ = ginkgo.BeforeSuite(func() { }, }, WorkConfiguration: &operatorapiv1.WorkConfiguration{ + WorkDriver: operatorapiv1.WorkDriverTypeKube, FeatureGates: []operatorapiv1.FeatureGate{ { - Feature: "ManifestWorkReplicaSet", - Mode: "Enable", + Feature: string(feature.ManifestWorkReplicaSet), + Mode: operatorapiv1.FeatureGateModeTypeEnable, }, }, },