Add RKE2 boostrap provider to e2e

Signed-off-by: Alexandr Demicev <alexandr.demicev@suse.com>
rancher · Oct 6, 2023 · 717d614 · 717d614
1 parent 56a7603
commit 717d614
Show file tree

Hide file tree

Showing 6 changed files with 222 additions and 12 deletions.
diff --git a/test/e2e/config/operator.yaml b/test/e2e/config/operator.yaml
@@ -17,6 +17,7 @@ intervals:
   default/wait-eks-delete: ["20m", "30s"]
   default/wait-aks-delete: ["20m", "30s"]
   default/wait-azure: ["30m", "30s"]
+  default/wait-rke2-docker: ["15m", "30s"]
 
 variables:
   RANCHER_VERSION: "v2.7.6"
@@ -40,4 +41,4 @@ variables:
   GITEA_CHART_NAME: "gitea"
   GITEA_CHART_VERSION: "9.4.0"
   GITEA_USER_NAME: "gitea_admin"
-  GITEA_USER_PWD: "password"
+  GITEA_USER_PWD: "password"
diff --git a/test/e2e/data/capi-operator/full-providers.yaml b/test/e2e/data/capi-operator/full-providers.yaml
@@ -9,6 +9,16 @@ kind: Namespace
 metadata:
   name: capz-system
 ---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: rke2-bootstrap-system
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: rke2-control-plane-system
+---
 apiVersion: operator.cluster.x-k8s.io/v1alpha1
 kind: InfrastructureProvider
 metadata:
@@ -25,4 +35,24 @@ metadata:
   namespace: capz-system
 spec:
   secretName: azure-variables
-  secretNamespace: default
+  secretNamespace: default
+---
+apiVersion: operator.cluster.x-k8s.io/v1alpha1
+kind: BootstrapProvider
+metadata:
+  name: bootstrap-caprke2
+  namespace: rke2-bootstrap-system
+spec:
+  fetchConfig:
+    url: https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/v0.1.1/bootstrap-components.yaml
+  version: v0.1.1
+---
+apiVersion: operator.cluster.x-k8s.io/v1alpha1
+kind: ControlPlaneProvider
+metadata:
+  name: control-plane-caprke2
+  namespace: rke2-control-plane-system
+spec:
+  fetchConfig:
+    url: https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/v0.1.1/control-plane-components.yaml
+  version: v0.1.1
diff --git a/test/e2e/suites/import-gitops/cluster-templates/docker-rke2.yaml b/test/e2e/suites/import-gitops/cluster-templates/docker-rke2.yaml
@@ -0,0 +1,124 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  loadBalancer:
+    customHAProxyConfigTemplateRef:
+      name: rke2-ha-proxy-config
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster 
+metadata:
+  name: ${CLUSTER_NAME} 
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 10.45.0.0/16
+    services:
+      cidrBlocks:
+      - 10.46.0.0/16
+    serviceDomain: cluster.local
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+    kind: RKE2ControlPlane
+    name: ${CLUSTER_NAME}-control-plane
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerCluster
+    name: ${CLUSTER_NAME}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec: 
+      customImage: ghcr.io/rancher-sandbox/ubuntu-22.04-rke2:${KUBERNETES_VERSION}
+      extraMounts:
+      - containerPath: /var/run/docker.sock
+        hostPath: /var/run/docker.sock
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+kind: RKE2ControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec: 
+  agentConfig:
+    version: ${KUBERNETES_VERSION}+rke2r1
+    airGapped: true
+  serverConfig:
+    cni: calico
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerMachineTemplate
+    name: ${CLUSTER_NAME}-control-plane
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  nodeDrainTimeout: 2m
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec: 
+      customImage: ghcr.io/rancher-sandbox/ubuntu-22.04-rke2:${KUBERNETES_VERSION}
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
+kind: RKE2ConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec: 
+  template:
+    spec:
+      agentConfig:
+        version: ${KUBERNETES_VERSION}+rke2r1
+        airGapped: true
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: worker-md-0
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+  template:
+    spec:
+      version: ${KUBERNETES_VERSION}
+      clusterName: ${CLUSTER_NAME}
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
+          kind: RKE2ConfigTemplate
+          name: ${CLUSTER_NAME}-md-0
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: DockerMachineTemplate
+        name: ${CLUSTER_NAME}-md-0
+---
+apiVersion: v1
+data:
+  value: |
+    # addtional config
+    frontend additional-join
+      bind *:9345
+      {{ if .IPv6 -}}
+      bind :::9345;
+      {{- end }}
+      default_backend additional-servers
+    backend additional-servers
+      option httpchk GET /v1-additional/readyz
+      http-check expect status 403
+      {{range $server, $address := .BackendServers}}
+      server {{ $server }} {{ JoinHostPort $address "9345" }} check check-ssl verify none resolvers docker resolve-prefer {{ if $.IPv6 -}} ipv6 {{- else -}} ipv4 {{- end }}
+      {{- end}}
+kind: ConfigMap
+metadata:
+  name: rke2-ha-proxy-config
+
diff --git a/test/e2e/suites/import-gitops/import_gitops.go b/test/e2e/suites/import-gitops/import_gitops.go
@@ -65,6 +65,9 @@ type CreateUsingGitOpsSpecInput struct {
 	// If not specified, 1 will be used.
 	WorkerMachineCount *int
 
+	// OverrideKubernetesVersion if specified will override the Kubernetes version used in the cluster template.
+	OverrideKubernetesVersion string
+
 	GitAddr           string
 	GitAuthSecretName string
 
@@ -160,16 +163,22 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
 		clustersDir := filepath.Join(repoDir, "clusters")
 		os.MkdirAll(clustersDir, os.ModePerm)
 
+		additionalVariables := map[string]string{
+			"CLUSTER_NAME":                input.ClusterName,
+			"WORKER_MACHINE_COUNT":        strconv.Itoa(workerMachineCount),
+			"CONTROL_PLANE_MACHINE_COUNT": strconv.Itoa(controlPlaneMachineCount),
+		}
+
+		if input.OverrideKubernetesVersion != "" {
+			additionalVariables["KUBERNETES_VERSION"] = input.OverrideKubernetesVersion
+		}
+
 		clusterPath := filepath.Join(clustersDir, fmt.Sprintf("%s.yaml", input.ClusterName))
 		Expect(turtlesframework.ApplyFromTemplate(ctx, turtlesframework.ApplyFromTemplateInput{
-			Getter:         input.E2EConfig.GetVariable,
-			Template:       input.ClusterTemplate,
-			OutputFilePath: clusterPath,
-			AddtionalEnvironmentVariables: map[string]string{
-				"CLUSTER_NAME":                input.ClusterName,
-				"WORKER_MACHINE_COUNT":        strconv.Itoa(workerMachineCount),
-				"CONTROL_PLANE_MACHINE_COUNT": strconv.Itoa(controlPlaneMachineCount),
-			},
+			Getter:                        input.E2EConfig.GetVariable,
+			Template:                      input.ClusterTemplate,
+			OutputFilePath:                clusterPath,
+			AddtionalEnvironmentVariables: additionalVariables,
 		})).To(Succeed())
 
 		fleetPath := filepath.Join(clustersDir, "fleet.yaml")

diff --git a/test/e2e/suites/import-gitops/import_gitops_test.go b/test/e2e/suites/import-gitops/import_gitops_test.go
@@ -39,6 +39,9 @@ var (
 
 	//go:embed cluster-templates/azure-aks-mmp.yaml
 	azureAKSMMP []byte
+
+	//go:embed cluster-templates/docker-rke2.yaml
+	rke2Docker []byte
 )
 
 var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel, e2e.FullTestLabel), func() {
@@ -129,3 +132,33 @@ var _ = Describe("[Azure] [AKS] Create and delete CAPI cluster functionality sho
 		}
 	})
 })
+
+var _ = Describe("[Docker] [RKE2] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel), func() {
+
+	BeforeEach(func() {
+		SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
+		SetContext(ctx)
+	})
+
+	CreateUsingGitOpsSpec(ctx, func() CreateUsingGitOpsSpecInput {
+		return CreateUsingGitOpsSpecInput{
+			E2EConfig:                 e2eConfig,
+			BootstrapClusterProxy:     setupClusterResult.BootstrapClusterProxy,
+			ClusterctlConfigPath:      flagVals.ConfigPath,
+			ArtifactFolder:            flagVals.ArtifactFolder,
+			ClusterTemplate:           rke2Docker,
+			ClusterName:               "cluster-rke2-docker",
+			OverrideKubernetesVersion: "v1.28.2",
+			ControlPlaneMachineCount:  ptr.To[int](1),
+			WorkerMachineCount:        ptr.To[int](1),
+			GitAddr:                   giteaResult.GitAddress,
+			GitAuthSecretName:         e2e.AuthSecretName,
+			SkipCleanup:               false,
+			SkipDeletionTest:          false,
+			LabelNamespace:            true,
+			RancherServerURL:          hostName,
+			CAPIClusterCreateWaitName: "wait-rke2-docker",
+			DeleteClusterWaitName:     "wait-rke2-docker",
+		}
+	})
+})
diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go
@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -104,15 +105,19 @@ var _ = BeforeSuite(func() {
 		hostName = setupClusterResult.IsolatedHostName
 	}
 
+	Expect(e2eConfig.Images).To(HaveLen(1))
+	imageNameTag := strings.Split(e2eConfig.Images[0].Name, ":")
+	Expect(imageNameTag).To(HaveLen(2))
+
 	testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
 		BootstrapClusterProxy:        setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:               flagVals.HelmBinaryPath,
 		ChartPath:                    flagVals.ChartPath,
 		CAPIProvidersSecretYAML:      e2e.CapiProvidersSecret,
 		CAPIProvidersYAML:            e2e.CapiProviders,
 		Namespace:                    turtlesframework.DefaultRancherTurtlesNamespace,
-		Image:                        "ghcr.io/rancher-sandbox/rancher-turtles-amd64",
-		Tag:                          "v0.0.1",
+		Image:                        imageNameTag[0],
+		Tag:                          imageNameTag[1],
 		WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
 	})
 
@@ -142,6 +147,14 @@ var _ = BeforeSuite(func() {
 					Name:      "capz-controller-manager",
 					Namespace: "capz-system",
 				},
+				{
+					Name:      "rke2-bootstrap-controller-manager",
+					Namespace: "rke2-bootstrap-system",
+				},
+				{
+					Name:      "rke2-control-plane-controller-manager",
+					Namespace: "rke2-control-plane-system",
+				},
 			},
 		})
 	}