From 5b7e80e11f21e604de4ba8fb78fa3dbc1fa2e803 Mon Sep 17 00:00:00 2001 From: Danil Grigorev Date: Wed, 4 Oct 2023 13:20:10 +0200 Subject: [PATCH] Tests: apply embedded capi ff after rancher install Signed-off-by: Danil Grigorev --- Makefile | 7 +- charts/rancher-turtles/values.yaml | 4 +- test/e2e/config/operator.yaml | 5 +- test/e2e/const.go | 23 ++++--- test/e2e/suites/import-gitops/suite_test.go | 70 ++++++++++---------- test/e2e/suites/v2prov/suite_test.go | 30 +++++---- test/testenv/rancher.go | 72 ++++++++++++++++----- 7 files changed, 130 insertions(+), 81 deletions(-) diff --git a/Makefile b/Makefile index 1cdd2fb3c..30de51730 100644 --- a/Makefile +++ b/Makefile @@ -180,7 +180,7 @@ help: ## Display this help. ##@ Development .PHONY: manifests manifests: vendor controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./..." output:crd:artifacts:config=hack/crd/bases + $(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./internal/rancher/..." output:crd:artifacts:config=hack/crd/bases $(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./vendor/sigs.k8s.io/cluster-api/..." output:crd:artifacts:config=hack/crd/bases # Vendor is only required for pulling latest CRDs from the dependencies $(MAKE) vendor-clean @@ -198,7 +198,6 @@ vendor-clean: .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." .PHONY: generate-modules generate-modules: ## Run go mod tidy to ensure modules are up to date @@ -249,11 +248,11 @@ test: $(SETUP_ENVTEST) manifests ## Run tests. ##@ Build .PHONY: build -build: manifests generate fmt vet ## Build manager binary. +build: generate fmt vet ## Build manager binary. go build -o bin/manager main.go .PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. +run: generate fmt vet ## Run a controller from your host. go run ./main.go ## -------------------------------------- diff --git a/charts/rancher-turtles/values.yaml b/charts/rancher-turtles/values.yaml index ea5ddfd7b..4cef8c968 100644 --- a/charts/rancher-turtles/values.yaml +++ b/charts/rancher-turtles/values.yaml @@ -3,7 +3,7 @@ rancherTurtles: tag: v0.0.0 imagePullPolicy: Never namespace: rancher-turtles-system - managerArguments: {} + managerArguments: [] imagePullSecrets: [] features: embedded-capi: @@ -14,7 +14,7 @@ rancherTurtles: cluster-api-operator: enabled: true cert-manager: - enabled: true + enabled: false cluster-api: enabled: true version: v1.4.6 diff --git a/test/e2e/config/operator.yaml b/test/e2e/config/operator.yaml index fe6a4ec85..430868c3f 100644 --- a/test/e2e/config/operator.yaml +++ b/test/e2e/config/operator.yaml @@ -21,13 +21,16 @@ intervals: variables: RANCHER_VERSION: "v2.7.6" RANCHER_HOSTNAME: "localhost" - RANCHER_FEATURES: "embedded-cluster-api=false" + RANCHER_FEATURES: "" RANCHER_PATH: "rancher-stable/rancher" KUBERNETES_VERSION: "v1.26.3" RKE2_VERSION: "v1.26.8+rke2r1" CAPI_CORE: "cluster-api:v1.4.6" RANCHER_REPO_NAME: "rancher-stable" RANCHER_URL: "https://releases.rancher.com/server-charts/stable" + CERT_MANAGER_URL: "https://charts.jetstack.io" + CERT_MANAGER_REPO_NAME: "jetstack" + CERT_MANAGER_PATH: "jetstack/cert-manager" RANCHER_PASSWORD: "rancheradmin" # Additional setup for establishing rancher ingress NGROK_REPO_NAME: "ngrok" diff --git a/test/e2e/const.go b/test/e2e/const.go index 7c267db06..aff87a2da 100644 --- a/test/e2e/const.go +++ b/test/e2e/const.go @@ -71,16 +71,19 @@ const ( ) const ( - KubernetesVersionVar = "KUBERNETES_VERSION" - RancherFeaturesVar = "RANCHER_FEATURES" - RancherHostnameVar = "RANCHER_HOSTNAME" - RancherVersionVar = "RANCHER_VERSION" - RancherPathVar = "RANCHER_PATH" - RancherUrlVar = "RANCHER_URL" - RancherRepoNameVar = "RANCHER_REPO_NAME" - RancherPasswordVar = "RANCHER_PASSWORD" - CapiInfrastructureVar = "CAPI_INFRASTRUCTURE" - CapiCoreVar = "CAPI_CORE" + KubernetesVersionVar = "KUBERNETES_VERSION" + RancherFeaturesVar = "RANCHER_FEATURES" + RancherHostnameVar = "RANCHER_HOSTNAME" + RancherVersionVar = "RANCHER_VERSION" + RancherPathVar = "RANCHER_PATH" + RancherUrlVar = "RANCHER_URL" + RancherRepoNameVar = "RANCHER_REPO_NAME" + RancherPasswordVar = "RANCHER_PASSWORD" + CertManagerUrlVar = "CERT_MANAGER_URL" + CertManagerRepoNameVar = "CERT_MANAGER_REPO_NAME" + CertManagerPathVar = "CERT_MANAGER_PATH" + CapiInfrastructureVar = "CAPI_INFRASTRUCTURE" + CapiCoreVar = "CAPI_CORE" NgrokRepoNameVar = "NGROK_REPO_NAME" NgrokUrlVar = "NGROK_URL" diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go index 3981ee15d..3f4c87476 100644 --- a/test/e2e/suites/import-gitops/suite_test.go +++ b/test/e2e/suites/import-gitops/suite_test.go @@ -104,6 +104,42 @@ var _ = BeforeSuite(func() { hostName = setupClusterResult.IsolatedHostName } + testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + HelmBinaryPath: flagVals.HelmBinaryPath, + IsolatedMode: flagVals.IsolatedMode, + NginxIngress: e2e.NginxIngress, + NginxIngressNamespace: e2e.NginxIngressNamespace, + IngressWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"), + NgrokApiKey: e2eConfig.GetVariable(e2e.NgrokApiKeyVar), + NgrokAuthToken: e2eConfig.GetVariable(e2e.NgrokAuthTokenVar), + NgrokPath: e2eConfig.GetVariable(e2e.NgrokPathVar), + NgrokRepoName: e2eConfig.GetVariable(e2e.NgrokRepoNameVar), + NgrokRepoURL: e2eConfig.GetVariable(e2e.NgrokUrlVar), + DefaultIngressClassPatch: e2e.IngressClassPatch, + }) + + testenv.DeployRancher(ctx, testenv.DeployRancherInput{ + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + HelmBinaryPath: flagVals.HelmBinaryPath, + CertManagerChartPath: e2eConfig.GetVariable(e2e.CertManagerPathVar), + CertManagerUrl: e2eConfig.GetVariable(e2e.CertManagerUrlVar), + CertManagerRepoName: e2eConfig.GetVariable(e2e.CertManagerRepoNameVar), + RancherChartRepoName: e2eConfig.GetVariable(e2e.RancherRepoNameVar), + RancherChartURL: e2eConfig.GetVariable(e2e.RancherUrlVar), + RancherChartPath: e2eConfig.GetVariable(e2e.RancherPathVar), + RancherVersion: e2eConfig.GetVariable(e2e.RancherVersionVar), + RancherHost: hostName, + RancherNamespace: e2e.RancherNamespace, + RancherPassword: e2eConfig.GetVariable(e2e.RancherPasswordVar), + RancherPatches: [][]byte{e2e.RancherSettingPatch}, + RancherWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"), + ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), + IsolatedMode: flagVals.IsolatedMode, + RancherIngressConfig: e2e.IngressConfig, + RancherServicePatch: e2e.RancherServicePatch, + }) + testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, @@ -146,40 +182,6 @@ var _ = BeforeSuite(func() { }) } - testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ - BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - HelmBinaryPath: flagVals.HelmBinaryPath, - IsolatedMode: flagVals.IsolatedMode, - NginxIngress: e2e.NginxIngress, - NginxIngressNamespace: e2e.NginxIngressNamespace, - IngressWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"), - NgrokApiKey: e2eConfig.GetVariable(e2e.NgrokApiKeyVar), - NgrokAuthToken: e2eConfig.GetVariable(e2e.NgrokAuthTokenVar), - NgrokPath: e2eConfig.GetVariable(e2e.NgrokPathVar), - NgrokRepoName: e2eConfig.GetVariable(e2e.NgrokRepoNameVar), - NgrokRepoURL: e2eConfig.GetVariable(e2e.NgrokUrlVar), - DefaultIngressClassPatch: e2e.IngressClassPatch, - }) - - testenv.DeployRancher(ctx, testenv.DeployRancherInput{ - BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - HelmBinaryPath: flagVals.HelmBinaryPath, - RancherChartRepoName: e2eConfig.GetVariable(e2e.RancherRepoNameVar), - RancherChartURL: e2eConfig.GetVariable(e2e.RancherUrlVar), - RancherChartPath: e2eConfig.GetVariable(e2e.RancherPathVar), - RancherVersion: e2eConfig.GetVariable(e2e.RancherVersionVar), - RancherHost: hostName, - RancherNamespace: e2e.RancherNamespace, - RancherPassword: e2eConfig.GetVariable(e2e.RancherPasswordVar), - RancherFeatures: e2eConfig.GetVariable(e2e.RancherFeaturesVar), - RancherSettingsPatch: e2e.RancherSettingPatch, - RancherWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"), - ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), - IsolatedMode: flagVals.IsolatedMode, - RancherIngressConfig: e2e.IngressConfig, - RancherServicePatch: e2e.RancherServicePatch, - }) - giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, diff --git a/test/e2e/suites/v2prov/suite_test.go b/test/e2e/suites/v2prov/suite_test.go index aed861389..e953529ed 100644 --- a/test/e2e/suites/v2prov/suite_test.go +++ b/test/e2e/suites/v2prov/suite_test.go @@ -103,18 +103,6 @@ var _ = BeforeSuite(func() { hostName = setupClusterResult.IsolatedHostName } - testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{ - BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - HelmBinaryPath: flagVals.HelmBinaryPath, - ChartPath: flagVals.ChartPath, - CAPIProvidersSecretYAML: e2e.CapiProvidersSecret, - CAPIProvidersYAML: e2e.CapiProviders, - Namespace: turtlesframework.DefaultRancherTurtlesNamespace, - Image: "ghcr.io/rancher-sandbox/rancher-turtles-amd64", - Tag: "v0.0.1", - WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), - }) - testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, @@ -133,6 +121,9 @@ var _ = BeforeSuite(func() { testenv.DeployRancher(ctx, testenv.DeployRancherInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, + CertManagerChartPath: e2eConfig.GetVariable(e2e.CertManagerPathVar), + CertManagerUrl: e2eConfig.GetVariable(e2e.CertManagerUrlVar), + CertManagerRepoName: e2eConfig.GetVariable(e2e.CertManagerRepoNameVar), RancherChartRepoName: "rancher-latest", RancherChartURL: "https://releases.rancher.com/server-charts/latest", RancherChartPath: "rancher-latest/rancher", @@ -143,13 +134,26 @@ var _ = BeforeSuite(func() { RancherNamespace: e2e.RancherNamespace, RancherPassword: e2eConfig.GetVariable(e2e.RancherPasswordVar), RancherFeatures: e2eConfig.GetVariable(e2e.RancherFeaturesVar), - RancherSettingsPatch: e2e.RancherSettingPatch, + RancherPatches: [][]byte{e2e.RancherSettingPatch}, RancherWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"), ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), IsolatedMode: flagVals.IsolatedMode, RancherIngressConfig: e2e.IngressConfig, RancherServicePatch: e2e.RancherServicePatch, }) + + testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{ + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + HelmBinaryPath: flagVals.HelmBinaryPath, + ChartPath: flagVals.ChartPath, + CAPIProvidersSecretYAML: e2e.CapiProvidersSecret, + CAPIProvidersYAML: e2e.CapiProviders, + Namespace: turtlesframework.DefaultRancherTurtlesNamespace, + Image: "ghcr.io/rancher-sandbox/rancher-turtles-amd64", + Tag: "v0.0.1", + WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), + }) + }) var _ = AfterSuite(func() { diff --git a/test/testenv/rancher.go b/test/testenv/rancher.go index d0655886e..de9012f97 100644 --- a/test/testenv/rancher.go +++ b/test/testenv/rancher.go @@ -22,20 +22,27 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework" "github.com/drone/envsubst/v2" + "github.com/rancher-sandbox/rancher-turtles/test/e2e" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" opframework "sigs.k8s.io/cluster-api-operator/test/framework" "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest/komega" ) type DeployRancherInput struct { BootstrapClusterProxy framework.ClusterProxy + E2EConfig *clusterctl.E2EConfig HelmBinaryPath string + CertManagerChartPath string + CertManagerUrl string + CertManagerRepoName string RancherChartRepoName string RancherChartURL string RancherChartPath string @@ -45,7 +52,7 @@ type DeployRancherInput struct { RancherHost string RancherPassword string RancherFeatures string - RancherSettingsPatch []byte + RancherPatches [][]byte RancherWaitInterval []interface{} ControllerWaitInterval []interface{} IsolatedMode bool @@ -59,6 +66,9 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) { Expect(ctx).NotTo(BeNil(), "ctx is required for DeployRancher") Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy is required for DeployRancher") Expect(input.HelmBinaryPath).ToNot(BeEmpty(), "HelmBinaryPath is required for DeployRancher") + Expect(input.CertManagerRepoName).ToNot(BeEmpty(), "CertManagerRepoName is required for DeployRancher") + Expect(input.CertManagerUrl).ToNot(BeEmpty(), "CertManagerUrl is required for DeployRancher") + Expect(input.CertManagerChartPath).ToNot(BeEmpty(), "CertManagerChartPath is required for DeployRancher") Expect(input.RancherChartRepoName).ToNot(BeEmpty(), "RancherChartRepoName is required for DeployRancher") Expect(input.RancherChartURL).ToNot(BeEmpty(), "RancherChartURL is required for DeployRancher") Expect(input.RancherChartPath).ToNot(BeEmpty(), "RancherChartPath is required for DeployRancher") @@ -75,8 +85,20 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) { Fail("Only one of RancherVersion or RancherImageTag cen be used") } - By("Adding Rancher chart repo") + By("Add cert manager chart repo") addChart := &opframework.HelmChart{ + BinaryPath: input.HelmBinaryPath, + Name: input.CertManagerRepoName, + Path: input.CertManagerUrl, + Commands: opframework.Commands(opframework.Repo, opframework.Add), + AdditionalFlags: opframework.Flags("--force-update"), + Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(), + } + _, err := addChart.Run(nil) + Expect(err).ToNot(HaveOccurred()) + + By("Adding Rancher chart repo") + addChart = &opframework.HelmChart{ BinaryPath: input.HelmBinaryPath, Name: input.RancherChartRepoName, Path: input.RancherChartURL, @@ -84,7 +106,7 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) { AdditionalFlags: opframework.Flags("--force-update"), Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(), } - _, err := addChart.Run(nil) + _, err = addChart.Run(nil) Expect(err).ToNot(HaveOccurred()) updateChart := &opframework.HelmChart{ @@ -95,11 +117,28 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) { _, err = updateChart.Run(nil) Expect(err).ToNot(HaveOccurred()) + By("Installing cert-manager") + chart := &opframework.HelmChart{ + BinaryPath: input.HelmBinaryPath, + Path: input.CertManagerChartPath, + Name: "cert-manager", + Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(), + AdditionalFlags: opframework.Flags( + "--namespace", "cert-manager", + "--version", "v1.12.0", + "--create-namespace", + ), + Wait: true, + } + _, err = chart.Run(map[string]string{ + "installCRDs": "true", + }) + Expect(err).ToNot(HaveOccurred()) + By("Installing Rancher") installFlags := opframework.Flags( "--namespace", input.RancherNamespace, "--create-namespace", - "--wait", ) if input.RancherVersion != "" { installFlags = append(installFlags, "--version", input.RancherVersion) @@ -108,12 +147,13 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) { installFlags = append(installFlags, "--devel") } - chart := &opframework.HelmChart{ + chart = &opframework.HelmChart{ BinaryPath: input.HelmBinaryPath, Path: input.RancherChartPath, Name: "rancher", Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(), AdditionalFlags: installFlags, + Wait: true, } values := map[string]string{ "bootstrapPassword": input.RancherPassword, @@ -131,18 +171,16 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) { _, err = chart.Run(values) Expect(err).ToNot(HaveOccurred()) - if len(input.RancherSettingsPatch) > 0 { - By("Updating rancher settings") - settingPatch, err := envsubst.Eval(string(input.RancherSettingsPatch), func(s string) string { - switch s { - case "RANCHER_HOSTNAME": - return input.RancherHost - default: - return os.Getenv(s) - } - }) - Expect(err).ToNot(HaveOccurred()) - Expect(input.BootstrapClusterProxy.Apply(ctx, []byte(settingPatch))).To(Succeed()) + By("Updating rancher configuration") + for _, patch := range input.RancherPatches { + Expect(turtlesframework.ApplyFromTemplate(ctx, turtlesframework.ApplyFromTemplateInput{ + Proxy: input.BootstrapClusterProxy, + Template: patch, + Getter: input.E2EConfig.GetVariable, + AddtionalEnvironmentVariables: map[string]string{ + e2e.RancherHostnameVar: input.RancherHost, + }, + })).To(Succeed()) } if !input.IsolatedMode {