diff --git a/test/e2e/const.go b/test/e2e/const.go index 30b6a0bd3..fe70b245b 100644 --- a/test/e2e/const.go +++ b/test/e2e/const.go @@ -127,4 +127,8 @@ const ( FullTestLabel = "full" DontRunLabel = "dontrun" LocalTestLabel = "local" + + CapiClusterOwnerLabel = "cluster-api.cattle.io/capi-cluster-owner" + CapiClusterOwnerNamespaceLabel = "cluster-api.cattle.io/capi-cluster-owner-ns" + OwnedLabelName = "cluster-api.cattle.io/owned" ) diff --git a/test/e2e/specs/import_gitops.go b/test/e2e/specs/import_gitops.go index e7e9bcbcc..3d9b66001 100644 --- a/test/e2e/specs/import_gitops.go +++ b/test/e2e/specs/import_gitops.go @@ -28,6 +28,7 @@ import ( "strconv" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -274,7 +275,6 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi }, rancherConnectRes) Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig") Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code") - }) AfterEach(func() { diff --git a/test/e2e/specs/import_gitops_mgmtv3.go b/test/e2e/specs/import_gitops_mgmtv3.go new file mode 100644 index 000000000..5b2592bb7 --- /dev/null +++ b/test/e2e/specs/import_gitops_mgmtv3.go @@ -0,0 +1,315 @@ +//go:build e2e +// +build e2e + +/* +Copyright © 2023 - 2024 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package specs + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + + managementv3 "github.com/rancher-sandbox/rancher-turtles/internal/rancher/management/v3" + "github.com/rancher-sandbox/rancher-turtles/test/e2e" + turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework" + "github.com/rancher-sandbox/rancher-turtles/test/testenv" +) + +type CreateMgmtV3UsingGitOpsSpecInput struct { + E2EConfig *clusterctl.E2EConfig + BootstrapClusterProxy framework.ClusterProxy + ClusterctlConfigPath string + ArtifactFolder string + RancherServerURL string + + ClusterctlBinaryPath string + ClusterTemplate []byte + ClusterName string + AdditionalTemplateVariables map[string]string + + CAPIClusterCreateWaitName string + DeleteClusterWaitName string + + // ControlPlaneMachineCount defines the number of control plane machines to be added to the workload cluster. + // If not specified, 1 will be used. + ControlPlaneMachineCount *int + + // WorkerMachineCount defines number of worker machines to be added to the workload cluster. + // If not specified, 1 will be used. + WorkerMachineCount *int + + GitAddr string + GitAuthSecretName string + + SkipCleanup bool + SkipDeletionTest bool + + LabelNamespace bool + + // management.cattle.io specifc + CapiClusterOwnerLabel string + CapiClusterOwnerNamespaceLabel string + OwnedLabelName string +} + +// CreateMgmtV3UsingGitOpsSpec implements a spec that will create a cluster via Fleet and test that it +// automatically imports into Rancher Manager. +func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateMgmtV3UsingGitOpsSpecInput) { + var ( + specName = "creategitops" + input CreateMgmtV3UsingGitOpsSpecInput + namespace *corev1.Namespace + repoName string + cancelWatches context.CancelFunc + capiCluster *types.NamespacedName + rancherKubeconfig *turtlesframework.RancherGetClusterKubeconfigResult + originalKubeconfig *turtlesframework.RancherGetClusterKubeconfigResult + rancherConnectRes *turtlesframework.RunCommandResult + rancherCluster *managementv3.Cluster + capiClusterCreateWait []interface{} + deleteClusterWait []interface{} + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(e2e.KubernetesManagementVersionVar)) + namespace, cancelWatches = e2e.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + repoName = e2e.CreateRepoName(specName) + + capiClusterCreateWait = input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), input.CAPIClusterCreateWaitName) + Expect(capiClusterCreateWait).ToNot(BeNil(), "Failed to get wait intervals %s", input.CAPIClusterCreateWaitName) + + deleteClusterWait = input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), input.DeleteClusterWaitName) + Expect(capiClusterCreateWait).ToNot(BeNil(), "Failed to get wait intervals %s", input.CAPIClusterCreateWaitName) + + capiCluster = &types.NamespacedName{ + Namespace: namespace.Name, + Name: input.ClusterName, + } + + rancherKubeconfig = new(turtlesframework.RancherGetClusterKubeconfigResult) + originalKubeconfig = new(turtlesframework.RancherGetClusterKubeconfigResult) + rancherConnectRes = new(turtlesframework.RunCommandResult) + + komega.SetClient(input.BootstrapClusterProxy.GetClient()) + komega.SetContext(ctx) + }) + + It("Should import a cluster using gitops", func() { + controlPlaneMachineCount := 1 + if input.ControlPlaneMachineCount != nil { + controlPlaneMachineCount = *input.ControlPlaneMachineCount + } + + workerMachineCount := 1 + if input.WorkerMachineCount != nil { + workerMachineCount = *input.WorkerMachineCount + } + + if input.LabelNamespace { + turtlesframework.AddLabelsToNamespace(ctx, turtlesframework.AddLabelsToNamespaceInput{ + ClusterProxy: input.BootstrapClusterProxy, + Name: namespace.Name, + Labels: map[string]string{ + "cluster-api.cattle.io/rancher-auto-import": "true", + }, + }) + } + + By("Create Git repository") + + repoCloneAddr := turtlesframework.GiteaCreateRepo(ctx, turtlesframework.GiteaCreateRepoInput{ + ServerAddr: input.GitAddr, + RepoName: repoName, + Username: input.E2EConfig.GetVariable(e2e.GiteaUserNameVar), + Password: input.E2EConfig.GetVariable(e2e.GiteaUserPasswordVar), + }) + repoDir := turtlesframework.GitCloneRepo(ctx, turtlesframework.GitCloneRepoInput{ + Address: repoCloneAddr, + Username: input.E2EConfig.GetVariable(e2e.GiteaUserNameVar), + Password: input.E2EConfig.GetVariable(e2e.GiteaUserPasswordVar), + }) + + By("Create fleet repository structure") + + clustersDir := filepath.Join(repoDir, "clusters") + os.MkdirAll(clustersDir, os.ModePerm) + + additionalVars := map[string]string{ + "CLUSTER_NAME": input.ClusterName, + "WORKER_MACHINE_COUNT": strconv.Itoa(workerMachineCount), + "CONTROL_PLANE_MACHINE_COUNT": strconv.Itoa(controlPlaneMachineCount), + } + for k, v := range input.AdditionalTemplateVariables { + additionalVars[k] = v + } + + clusterPath := filepath.Join(clustersDir, fmt.Sprintf("%s.yaml", input.ClusterName)) + Expect(turtlesframework.ApplyFromTemplate(ctx, turtlesframework.ApplyFromTemplateInput{ + Getter: input.E2EConfig.GetVariable, + Template: input.ClusterTemplate, + OutputFilePath: clusterPath, + AddtionalEnvironmentVariables: additionalVars, + })).To(Succeed()) + + fleetPath := filepath.Join(clustersDir, "fleet.yaml") + turtlesframework.FleetCreateFleetFile(ctx, turtlesframework.FleetCreateFleetFileInput{ + Namespace: namespace.Name, + FilePath: fleetPath, + }) + + By("Committing changes to fleet repo and pushing") + + turtlesframework.GitCommitAndPush(ctx, turtlesframework.GitCommitAndPushInput{ + CloneLocation: repoDir, + Username: input.E2EConfig.GetVariable(e2e.GiteaUserNameVar), + Password: input.E2EConfig.GetVariable(e2e.GiteaUserPasswordVar), + CommitMessage: "ci: add clusters bundle", + }) + + By("Applying GitRepo") + + turtlesframework.FleetCreateGitRepo(ctx, turtlesframework.FleetCreateGitRepoInput{ + Name: repoName, + Namespace: turtlesframework.FleetLocalNamespace, + Branch: turtlesframework.DefaultBranchName, + Repo: repoCloneAddr, + FleetGeneration: 1, + Paths: []string{"clusters"}, + ClientSecretName: input.GitAuthSecretName, + ClusterProxy: input.BootstrapClusterProxy, + }) + + By("Waiting for the CAPI cluster to appear") + capiCluster := &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace.Name, + Name: input.ClusterName, + }} + Eventually( + komega.Get(capiCluster), + input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...). + Should(Succeed(), "Failed to apply CAPI cluster definition to cluster via Fleet") + + By("Waiting for cluster control plane to be Ready") + Eventually(komega.Object(capiCluster), capiClusterCreateWait...).Should(HaveField("Status.ControlPlaneReady", BeTrue())) + + By("Waiting for the CAPI cluster to be connectable") + Eventually(func() error { + remoteClient := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, capiCluster.Namespace, capiCluster.Name).GetClient() + namespaces := &corev1.NamespaceList{} + + return remoteClient.List(ctx, namespaces) + }, capiClusterCreateWait...).Should(Succeed(), "Failed to connect to workload cluster using CAPI kubeconfig") + + By("Storing the original CAPI cluster kubeconfig") + turtlesframework.RancherGetOriginalKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{ + Getter: input.BootstrapClusterProxy.GetClient(), + SecretName: fmt.Sprintf("%s-kubeconfig", capiCluster.Name), + Namespace: capiCluster.Namespace, + WriteToTempFile: true, + }, originalKubeconfig) + + By("Waiting for the rancher cluster record to appear") + rancherClusters := &managementv3.ClusterList{} + selectors := []client.ListOption{ + client.MatchingLabels{ + input.CapiClusterOwnerLabel: capiCluster.Name, + input.CapiClusterOwnerNamespaceLabel: capiCluster.Namespace, + input.OwnedLabelName: "", + }, + } + Eventually(komega.List(rancherClusters, selectors...)).Should(Succeed()) + Expect(rancherClusters.Items).To(HaveLen(1)) + rancherCluster = &rancherClusters.Items[0] + Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + + By("Waiting for the rancher cluster to have a deployed agent") + Eventually(func() bool { + Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + return managementv3.ClusterConditionAgentDeployed.IsTrue(rancherCluster) + }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue()) + + By("Waiting for the rancher cluster to be ready") + Eventually(func() bool { + Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + return managementv3.ClusterConditionReady.IsTrue(rancherCluster) + }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue()) + + By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig") + turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{ + Getter: input.BootstrapClusterProxy.GetClient(), + SecretName: fmt.Sprintf("%s-kubeconfig", rancherCluster.Name), + Namespace: rancherCluster.Spec.FleetWorkspaceName, + RancherServerURL: input.RancherServerURL, + WriteToTempFile: true, + }, rancherKubeconfig) + + turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + Command: "kubectl", + Args: []string{ + "--kubeconfig", + rancherKubeconfig.TempFilePath, + "get", + "nodes", + "--insecure-skip-tls-verify", + }, + }, rancherConnectRes) + Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig") + Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code") + }) + + AfterEach(func() { + err := testenv.CollectArtifacts(ctx, originalKubeconfig.TempFilePath, path.Join(input.ArtifactFolder, input.BootstrapClusterProxy.GetName(), input.ClusterName)) + if err != nil { + fmt.Printf("Failed to collect artifacts for the child cluster: %v\n", err) + } + + By("Deleting GitRepo from Rancher") + turtlesframework.FleetDeleteGitRepo(ctx, turtlesframework.FleetDeleteGitRepoInput{ + Name: repoName, + Namespace: turtlesframework.FleetLocalNamespace, + ClusterProxy: input.BootstrapClusterProxy, + }) + + By("Waiting for the rancher cluster record to be removed") + Eventually(komega.Get(rancherCluster), deleteClusterWait...).Should(MatchError(ContainSubstring("not found")), "Rancher cluster should be deleted") + + e2e.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, capiCluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + }) +} diff --git a/test/e2e/suites/import-gitops/import_gitops_test.go b/test/e2e/suites/import-gitops/import_gitops_test.go index 602df09aa..1c6643211 100644 --- a/test/e2e/suites/import-gitops/import_gitops_test.go +++ b/test/e2e/suites/import-gitops/import_gitops_test.go @@ -33,7 +33,6 @@ import ( ) var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel, e2e.FullTestLabel), func() { - BeforeEach(func() { SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) SetContext(ctx) @@ -63,7 +62,6 @@ var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionalit }) var _ = Describe("[AWS] [EKS] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel), func() { - BeforeEach(func() { komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) komega.SetContext(ctx) @@ -93,7 +91,6 @@ var _ = Describe("[AWS] [EKS] Create and delete CAPI cluster functionality shoul }) var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.DontRunLabel), func() { - BeforeEach(func() { SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) SetContext(ctx) @@ -122,7 +119,6 @@ var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster functionality sho }) var _ = Describe("[vSphere] [Kubeadm] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.LocalTestLabel), func() { - BeforeEach(func() { SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) SetContext(ctx) diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go index e4f5daf56..e65140933 100644 --- a/test/e2e/suites/import-gitops/suite_test.go +++ b/test/e2e/suites/import-gitops/suite_test.go @@ -24,9 +24,8 @@ import ( "fmt" "os" "path/filepath" - "testing" - "runtime" + "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" diff --git a/test/e2e/suites/managementv3/managementv3_test.go b/test/e2e/suites/managementv3/managementv3_test.go new file mode 100644 index 000000000..d383bd36a --- /dev/null +++ b/test/e2e/suites/managementv3/managementv3_test.go @@ -0,0 +1,125 @@ +//go:build e2e +// +build e2e + +/* +Copyright © 2023 - 2024 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package managementv3 + +import ( + . "github.com/onsi/ginkgo/v2" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + + "k8s.io/utils/ptr" + + "github.com/rancher-sandbox/rancher-turtles/test/e2e" + "github.com/rancher-sandbox/rancher-turtles/test/e2e/specs" +) + +var _ = Describe("[Docker] [Kubeadm] - [management.cattle.io/v3] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel, e2e.FullTestLabel), func() { + BeforeEach(func() { + komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) + komega.SetContext(ctx) + }) + + specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput { + return specs.CreateMgmtV3UsingGitOpsSpecInput{ + E2EConfig: e2eConfig, + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + ClusterctlConfigPath: flagVals.ConfigPath, + ClusterctlBinaryPath: flagVals.ClusterctlBinaryPath, + ArtifactFolder: flagVals.ArtifactFolder, + ClusterTemplate: e2e.CAPIDockerKubeadm, + ClusterName: "highlander-e2e-clusterv3-1", + ControlPlaneMachineCount: ptr.To[int](1), + WorkerMachineCount: ptr.To[int](1), + GitAddr: giteaResult.GitAddress, + GitAuthSecretName: e2e.AuthSecretName, + SkipCleanup: false, + SkipDeletionTest: false, + LabelNamespace: true, + RancherServerURL: hostName, + CAPIClusterCreateWaitName: "wait-rancher", + DeleteClusterWaitName: "wait-controllers", + CapiClusterOwnerLabel: e2e.CapiClusterOwnerLabel, + CapiClusterOwnerNamespaceLabel: e2e.CapiClusterOwnerNamespaceLabel, + OwnedLabelName: e2e.OwnedLabelName, + } + }) +}) + +var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.DontRunLabel), func() { + BeforeEach(func() { + komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) + komega.SetContext(ctx) + }) + + specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput { + return specs.CreateMgmtV3UsingGitOpsSpecInput{ + E2EConfig: e2eConfig, + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + ClusterctlConfigPath: flagVals.ConfigPath, + ArtifactFolder: flagVals.ArtifactFolder, + ClusterTemplate: e2e.CAPIAzureAKSMMP, + ClusterName: "highlander-e2e-clusterv3-2", + ControlPlaneMachineCount: ptr.To[int](1), + WorkerMachineCount: ptr.To[int](1), + GitAddr: giteaResult.GitAddress, + GitAuthSecretName: e2e.AuthSecretName, + SkipCleanup: false, + SkipDeletionTest: false, + LabelNamespace: true, + RancherServerURL: hostName, + CAPIClusterCreateWaitName: "wait-capz-create-cluster", + DeleteClusterWaitName: "wait-aks-delete", + CapiClusterOwnerLabel: e2e.CapiClusterOwnerLabel, + CapiClusterOwnerNamespaceLabel: e2e.CapiClusterOwnerNamespaceLabel, + OwnedLabelName: e2e.OwnedLabelName, + } + }) +}) + +var _ = Describe("[AWS] [EKS] - [management.cattle.io/v3] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel), func() { + BeforeEach(func() { + komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) + komega.SetContext(ctx) + }) + + specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput { + return specs.CreateMgmtV3UsingGitOpsSpecInput{ + E2EConfig: e2eConfig, + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + ClusterctlConfigPath: flagVals.ConfigPath, + ClusterctlBinaryPath: flagVals.ClusterctlBinaryPath, + ArtifactFolder: flagVals.ArtifactFolder, + ClusterTemplate: e2e.CAPIAwsEKSMMP, + ClusterName: "highlander-e2e-clusterv3-3", + ControlPlaneMachineCount: ptr.To[int](1), + WorkerMachineCount: ptr.To[int](1), + GitAddr: giteaResult.GitAddress, + GitAuthSecretName: e2e.AuthSecretName, + SkipCleanup: false, + SkipDeletionTest: false, + LabelNamespace: true, + RancherServerURL: hostName, + CAPIClusterCreateWaitName: "wait-capa-create-cluster", + DeleteClusterWaitName: "wait-eks-delete", + CapiClusterOwnerLabel: e2e.CapiClusterOwnerLabel, + CapiClusterOwnerNamespaceLabel: e2e.CapiClusterOwnerNamespaceLabel, + OwnedLabelName: e2e.OwnedLabelName, + } + }) +}) diff --git a/test/e2e/suites/managementv3/suite_test.go b/test/e2e/suites/managementv3/suite_test.go new file mode 100644 index 000000000..42a8c18fa --- /dev/null +++ b/test/e2e/suites/managementv3/suite_test.go @@ -0,0 +1,224 @@ +//go:build e2e +// +build e2e + +/* +Copyright © 2023 - 2024 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package managementv3 + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/klog/v2" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/rancher-sandbox/rancher-turtles/test/e2e" + turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework" + "github.com/rancher-sandbox/rancher-turtles/test/testenv" +) + +// Test suite flags. +var ( + flagVals *e2e.FlagValues +) + +// Test suite global vars. +var ( + // e2eConfig to be used for this test, read from configPath. + e2eConfig *clusterctl.E2EConfig + + // clusterctlConfigPath to be used for this test, created by generating a clusterctl local repository + // with the providers specified in the configPath. + clusterctlConfigPath string + + // hostName is the host name for the Rancher Manager server. + hostName string + + ctx = context.Background() + + setupClusterResult *testenv.SetupTestClusterResult + giteaResult *testenv.DeployGiteaResult +) + +func init() { + flagVals = &e2e.FlagValues{} + e2e.InitFlags(flagVals) +} + +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + + ctrl.SetLogger(klog.Background()) + + RunSpecs(t, "rancher-turtles-e2e-managementv3") +} + +var _ = BeforeSuite(func() { + Expect(flagVals.ConfigPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.") + Expect(os.MkdirAll(flagVals.ArtifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", flagVals.ArtifactFolder) + Expect(flagVals.HelmBinaryPath).To(BeAnExistingFile(), "Invalid test suite argument. helm-binary-path should be an existing file.") + Expect(flagVals.ChartPath).To(BeAnExistingFile(), "Invalid test suite argument. chart-path should be an existing file.") + + By(fmt.Sprintf("Loading the e2e test configuration from %q", flagVals.ConfigPath)) + e2eConfig = e2e.LoadE2EConfig(flagVals.ConfigPath) + + By(fmt.Sprintf("Creating a clusterctl config into %q", flagVals.ArtifactFolder)) + clusterctlConfigPath = e2e.CreateClusterctlLocalRepository(ctx, e2eConfig, filepath.Join(flagVals.ArtifactFolder, "repository")) + + hostName = e2eConfig.GetVariable(e2e.RancherHostnameVar) + + setupClusterResult = testenv.SetupTestCluster(ctx, testenv.SetupTestClusterInput{ + UseExistingCluster: flagVals.UseExistingCluster, + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + Scheme: e2e.InitScheme(), + ArtifactFolder: flagVals.ArtifactFolder, + Hostname: hostName, + KubernetesVersion: e2eConfig.GetVariable(e2e.KubernetesManagementVersionVar), + IsolatedMode: flagVals.IsolatedMode, + HelmBinaryPath: flagVals.HelmBinaryPath, + }) + + if flagVals.IsolatedMode { + hostName = setupClusterResult.IsolatedHostName + } + + testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + HelmBinaryPath: flagVals.HelmBinaryPath, + HelmExtraValuesPath: filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher-ingress.yaml"), + IsolatedMode: flagVals.IsolatedMode, + NginxIngress: e2e.NginxIngress, + NginxIngressNamespace: e2e.NginxIngressNamespace, + IngressWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"), + NgrokApiKey: e2eConfig.GetVariable(e2e.NgrokApiKeyVar), + NgrokAuthToken: e2eConfig.GetVariable(e2e.NgrokAuthTokenVar), + NgrokPath: e2eConfig.GetVariable(e2e.NgrokPathVar), + NgrokRepoName: e2eConfig.GetVariable(e2e.NgrokRepoNameVar), + NgrokRepoURL: e2eConfig.GetVariable(e2e.NgrokUrlVar), + DefaultIngressClassPatch: e2e.IngressClassPatch, + }) + + testenv.DeployRancher(ctx, testenv.DeployRancherInput{ + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + HelmBinaryPath: flagVals.HelmBinaryPath, + HelmExtraValuesPath: filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher.yaml"), + InstallCertManager: true, + CertManagerChartPath: e2eConfig.GetVariable(e2e.CertManagerPathVar), + CertManagerUrl: e2eConfig.GetVariable(e2e.CertManagerUrlVar), + CertManagerRepoName: e2eConfig.GetVariable(e2e.CertManagerRepoNameVar), + RancherChartRepoName: e2eConfig.GetVariable(e2e.RancherRepoNameVar), + RancherChartURL: e2eConfig.GetVariable(e2e.RancherUrlVar), + RancherChartPath: e2eConfig.GetVariable(e2e.RancherPathVar), + RancherVersion: e2eConfig.GetVariable(e2e.RancherVersionVar), + RancherHost: hostName, + RancherNamespace: e2e.RancherNamespace, + RancherPassword: e2eConfig.GetVariable(e2e.RancherPasswordVar), + RancherPatches: [][]byte{e2e.RancherSettingPatch}, + RancherWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"), + ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), + IsolatedMode: flagVals.IsolatedMode, + RancherIngressConfig: e2e.IngressConfig, + RancherServicePatch: e2e.RancherServicePatch, + Variables: e2eConfig.Variables, + }) + + testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{ + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + HelmBinaryPath: flagVals.HelmBinaryPath, + ChartPath: flagVals.ChartPath, + CAPIProvidersYAML: e2e.CapiProviders, + Namespace: turtlesframework.DefaultRancherTurtlesNamespace, + Image: fmt.Sprintf("ghcr.io/rancher-sandbox/rancher-turtles-%s", runtime.GOARCH), + Tag: "v0.0.1", + WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), + AdditionalValues: map[string]string{ + "rancherTurtles.features.managementv3-cluster.enabled": "true", // enable management.cattle.io/v3 controller + }, + }) + + if !shortTestOnly() && !localTestOnly() { + By("Running full tests, deploying additional infrastructure providers") + awsCreds := e2eConfig.GetVariable(e2e.CapaEncodedCredentialsVar) + Expect(awsCreds).ToNot(BeEmpty(), "AWS creds required for full test") + + testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{ + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + CAPIProvidersSecretsYAML: [][]byte{ + e2e.AWSProviderSecret, + e2e.AzureIdentitySecret, + }, + CAPIProvidersYAML: e2e.FullProviders, + TemplateData: map[string]string{ + "AWSEncodedCredentials": e2eConfig.GetVariable(e2e.CapaEncodedCredentialsVar), + }, + WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), + WaitForDeployments: []testenv.NamespaceName{ + { + Name: "capa-controller-manager", + Namespace: "capa-system", + }, + { + Name: "capz-controller-manager", + Namespace: "capz-system", + }, + }, + }) + } + + giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{ + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + HelmBinaryPath: flagVals.HelmBinaryPath, + ChartRepoName: e2eConfig.GetVariable(e2e.GiteaRepoNameVar), + ChartRepoURL: e2eConfig.GetVariable(e2e.GiteaRepoURLVar), + ChartName: e2eConfig.GetVariable(e2e.GiteaChartNameVar), + ChartVersion: e2eConfig.GetVariable(e2e.GiteaChartVersionVar), + ValuesFilePath: "../../data/gitea/values.yaml", + Values: map[string]string{ + "gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar), + "gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar), + }, + RolloutWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-gitea"), + ServiceWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-getservice"), + AuthSecretName: e2e.AuthSecretName, + Username: e2eConfig.GetVariable(e2e.GiteaUserNameVar), + Password: e2eConfig.GetVariable(e2e.GiteaUserPasswordVar), + }) +}) + +var _ = AfterSuite(func() { + testenv.CleanupTestCluster(ctx, testenv.CleanupTestClusterInput{ + SetupTestClusterResult: *setupClusterResult, + SkipCleanup: flagVals.SkipCleanup, + ArtifactFolder: flagVals.ArtifactFolder, + }) +}) + +func shortTestOnly() bool { + return GinkgoLabelFilter() == e2e.ShortTestLabel +} + +func localTestOnly() bool { + return GinkgoLabelFilter() == e2e.LocalTestLabel +}