From 717d614ed1ec6b3547eba8ba8ea4e915dc13d67f Mon Sep 17 00:00:00 2001 From: Alexandr Demicev Date: Mon, 2 Oct 2023 11:49:41 +0200 Subject: [PATCH] Add RKE2 boostrap provider to e2e Signed-off-by: Alexandr Demicev --- test/e2e/config/operator.yaml | 3 +- .../data/capi-operator/full-providers.yaml | 32 ++++- .../cluster-templates/docker-rke2.yaml | 124 ++++++++++++++++++ .../e2e/suites/import-gitops/import_gitops.go | 25 ++-- .../import-gitops/import_gitops_test.go | 33 +++++ test/e2e/suites/import-gitops/suite_test.go | 17 ++- 6 files changed, 222 insertions(+), 12 deletions(-) create mode 100644 test/e2e/suites/import-gitops/cluster-templates/docker-rke2.yaml diff --git a/test/e2e/config/operator.yaml b/test/e2e/config/operator.yaml index fe6a4ec8..954b0ad0 100644 --- a/test/e2e/config/operator.yaml +++ b/test/e2e/config/operator.yaml @@ -17,6 +17,7 @@ intervals: default/wait-eks-delete: ["20m", "30s"] default/wait-aks-delete: ["20m", "30s"] default/wait-azure: ["30m", "30s"] + default/wait-rke2-docker: ["15m", "30s"] variables: RANCHER_VERSION: "v2.7.6" @@ -40,4 +41,4 @@ variables: GITEA_CHART_NAME: "gitea" GITEA_CHART_VERSION: "9.4.0" GITEA_USER_NAME: "gitea_admin" - GITEA_USER_PWD: "password" \ No newline at end of file + GITEA_USER_PWD: "password" diff --git a/test/e2e/data/capi-operator/full-providers.yaml b/test/e2e/data/capi-operator/full-providers.yaml index 1491539d..e5d9fa3e 100644 --- a/test/e2e/data/capi-operator/full-providers.yaml +++ b/test/e2e/data/capi-operator/full-providers.yaml @@ -9,6 +9,16 @@ kind: Namespace metadata: name: capz-system --- +apiVersion: v1 +kind: Namespace +metadata: + name: rke2-bootstrap-system +--- +apiVersion: v1 +kind: Namespace +metadata: + name: rke2-control-plane-system +--- apiVersion: operator.cluster.x-k8s.io/v1alpha1 kind: InfrastructureProvider metadata: @@ -25,4 +35,24 @@ metadata: namespace: capz-system spec: secretName: azure-variables - secretNamespace: default \ No newline at end of file + secretNamespace: default +--- +apiVersion: operator.cluster.x-k8s.io/v1alpha1 +kind: BootstrapProvider +metadata: + name: bootstrap-caprke2 + namespace: rke2-bootstrap-system +spec: + fetchConfig: + url: https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/v0.1.1/bootstrap-components.yaml + version: v0.1.1 +--- +apiVersion: operator.cluster.x-k8s.io/v1alpha1 +kind: ControlPlaneProvider +metadata: + name: control-plane-caprke2 + namespace: rke2-control-plane-system +spec: + fetchConfig: + url: https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/v0.1.1/control-plane-components.yaml + version: v0.1.1 diff --git a/test/e2e/suites/import-gitops/cluster-templates/docker-rke2.yaml b/test/e2e/suites/import-gitops/cluster-templates/docker-rke2.yaml new file mode 100644 index 00000000..ecd92229 --- /dev/null +++ b/test/e2e/suites/import-gitops/cluster-templates/docker-rke2.yaml @@ -0,0 +1,124 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: ${CLUSTER_NAME} +spec: + loadBalancer: + customHAProxyConfigTemplateRef: + name: rke2-ha-proxy-config +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.45.0.0/16 + services: + cidrBlocks: + - 10.46.0.0/16 + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + kind: RKE2ControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + customImage: ghcr.io/rancher-sandbox/ubuntu-22.04-rke2:${KUBERNETES_VERSION} + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 +kind: RKE2ControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + agentConfig: + version: ${KUBERNETES_VERSION}+rke2r1 + airGapped: true + serverConfig: + cni: calico + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + nodeDrainTimeout: 2m +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + customImage: ghcr.io/rancher-sandbox/ubuntu-22.04-rke2:${KUBERNETES_VERSION} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1 +kind: RKE2ConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + agentConfig: + version: ${KUBERNETES_VERSION}+rke2r1 + airGapped: true +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: worker-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + template: + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1 + kind: RKE2ConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-md-0 +--- +apiVersion: v1 +data: + value: | + # addtional config + frontend additional-join + bind *:9345 + {{ if .IPv6 -}} + bind :::9345; + {{- end }} + default_backend additional-servers + backend additional-servers + option httpchk GET /v1-additional/readyz + http-check expect status 403 + {{range $server, $address := .BackendServers}} + server {{ $server }} {{ JoinHostPort $address "9345" }} check check-ssl verify none resolvers docker resolve-prefer {{ if $.IPv6 -}} ipv6 {{- else -}} ipv4 {{- end }} + {{- end}} +kind: ConfigMap +metadata: + name: rke2-ha-proxy-config + diff --git a/test/e2e/suites/import-gitops/import_gitops.go b/test/e2e/suites/import-gitops/import_gitops.go index 60aea06c..02ca4ee2 100644 --- a/test/e2e/suites/import-gitops/import_gitops.go +++ b/test/e2e/suites/import-gitops/import_gitops.go @@ -65,6 +65,9 @@ type CreateUsingGitOpsSpecInput struct { // If not specified, 1 will be used. WorkerMachineCount *int + // OverrideKubernetesVersion if specified will override the Kubernetes version used in the cluster template. + OverrideKubernetesVersion string + GitAddr string GitAuthSecretName string @@ -160,16 +163,22 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi clustersDir := filepath.Join(repoDir, "clusters") os.MkdirAll(clustersDir, os.ModePerm) + additionalVariables := map[string]string{ + "CLUSTER_NAME": input.ClusterName, + "WORKER_MACHINE_COUNT": strconv.Itoa(workerMachineCount), + "CONTROL_PLANE_MACHINE_COUNT": strconv.Itoa(controlPlaneMachineCount), + } + + if input.OverrideKubernetesVersion != "" { + additionalVariables["KUBERNETES_VERSION"] = input.OverrideKubernetesVersion + } + clusterPath := filepath.Join(clustersDir, fmt.Sprintf("%s.yaml", input.ClusterName)) Expect(turtlesframework.ApplyFromTemplate(ctx, turtlesframework.ApplyFromTemplateInput{ - Getter: input.E2EConfig.GetVariable, - Template: input.ClusterTemplate, - OutputFilePath: clusterPath, - AddtionalEnvironmentVariables: map[string]string{ - "CLUSTER_NAME": input.ClusterName, - "WORKER_MACHINE_COUNT": strconv.Itoa(workerMachineCount), - "CONTROL_PLANE_MACHINE_COUNT": strconv.Itoa(controlPlaneMachineCount), - }, + Getter: input.E2EConfig.GetVariable, + Template: input.ClusterTemplate, + OutputFilePath: clusterPath, + AddtionalEnvironmentVariables: additionalVariables, })).To(Succeed()) fleetPath := filepath.Join(clustersDir, "fleet.yaml") diff --git a/test/e2e/suites/import-gitops/import_gitops_test.go b/test/e2e/suites/import-gitops/import_gitops_test.go index bead33bd..31551133 100644 --- a/test/e2e/suites/import-gitops/import_gitops_test.go +++ b/test/e2e/suites/import-gitops/import_gitops_test.go @@ -39,6 +39,9 @@ var ( //go:embed cluster-templates/azure-aks-mmp.yaml azureAKSMMP []byte + + //go:embed cluster-templates/docker-rke2.yaml + rke2Docker []byte ) var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel, e2e.FullTestLabel), func() { @@ -129,3 +132,33 @@ var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster functionality sho } }) }) + +var _ = Describe("[Docker] [RKE2] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel), func() { + + BeforeEach(func() { + SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) + SetContext(ctx) + }) + + CreateUsingGitOpsSpec(ctx, func() CreateUsingGitOpsSpecInput { + return CreateUsingGitOpsSpecInput{ + E2EConfig: e2eConfig, + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + ClusterctlConfigPath: flagVals.ConfigPath, + ArtifactFolder: flagVals.ArtifactFolder, + ClusterTemplate: rke2Docker, + ClusterName: "cluster-rke2-docker", + OverrideKubernetesVersion: "v1.28.2", + ControlPlaneMachineCount: ptr.To[int](1), + WorkerMachineCount: ptr.To[int](1), + GitAddr: giteaResult.GitAddress, + GitAuthSecretName: e2e.AuthSecretName, + SkipCleanup: false, + SkipDeletionTest: false, + LabelNamespace: true, + RancherServerURL: hostName, + CAPIClusterCreateWaitName: "wait-rke2-docker", + DeleteClusterWaitName: "wait-rke2-docker", + } + }) +}) diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go index 3981ee15..c784b94f 100644 --- a/test/e2e/suites/import-gitops/suite_test.go +++ b/test/e2e/suites/import-gitops/suite_test.go @@ -24,6 +24,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" . "github.com/onsi/ginkgo/v2" @@ -104,6 +105,10 @@ var _ = BeforeSuite(func() { hostName = setupClusterResult.IsolatedHostName } + Expect(e2eConfig.Images).To(HaveLen(1)) + imageNameTag := strings.Split(e2eConfig.Images[0].Name, ":") + Expect(imageNameTag).To(HaveLen(2)) + testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, HelmBinaryPath: flagVals.HelmBinaryPath, @@ -111,8 +116,8 @@ var _ = BeforeSuite(func() { CAPIProvidersSecretYAML: e2e.CapiProvidersSecret, CAPIProvidersYAML: e2e.CapiProviders, Namespace: turtlesframework.DefaultRancherTurtlesNamespace, - Image: "ghcr.io/rancher-sandbox/rancher-turtles-amd64", - Tag: "v0.0.1", + Image: imageNameTag[0], + Tag: imageNameTag[1], WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), }) @@ -142,6 +147,14 @@ var _ = BeforeSuite(func() { Name: "capz-controller-manager", Namespace: "capz-system", }, + { + Name: "rke2-bootstrap-controller-manager", + Namespace: "rke2-bootstrap-system", + }, + { + Name: "rke2-control-plane-controller-manager", + Namespace: "rke2-control-plane-system", + }, }, }) }