Merge pull request #608 from alexander-demicev/giteaingress

E2E: Add custom ingress for gitea

Makefile

@@ -76,6 +76,7 @@ SKIP_RESOURCE_CLEANUP ?= false
 USE_EXISTING_CLUSTER ?= false
 USE_EKS ?= true
 ISOLATED_MODE ?= false
+GITEA_CUSTOM_INGRESS ?= false
 GINKGO_NOCOLOR ?= false
 GINKGO_LABEL_FILTER ?= short
 GINKGO_TESTS ?= $(ROOT_DIR)/$(TEST_DIR)/e2e/suites/...
@@ -535,7 +536,8 @@ test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-en
 	    -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
 		-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \
 		-e2e.isolated-mode=$(ISOLATED_MODE) \
-		-e2e.use-eks=$(USE_EKS)
+		-e2e.use-eks=$(USE_EKS) \
+		-e2e.gitea-custom-ingress=$(GITEA_CUSTOM_INGRESS)
 
 .PHONY: e2e-image
 e2e-image: ## Build the image for e2e tests

test/e2e/const.go

@@ -80,6 +80,9 @@ var (
 
 	//go:embed data/cluster-templates/vsphere-kubeadm.yaml
 	CAPIvSphereKubeadm []byte
+
+	//go:embed data/gitea/ingress.yaml
+	GiteaIngress []byte
 )
 
 const (

test/e2e/data/cluster-templates/vsphere-kubeadm.yaml

@@ -57,6 +57,7 @@ spec:
       numCPUs: 2
       os: Linux
       powerOffMode: hard
+      resourcePool: '${VSPHERE_RESOURCE_POOL}'
       server: '${VSPHERE_SERVER}'
       template: '${VSPHERE_TEMPLATE}'
       thumbprint: '${VSPHERE_TLS_THUMBPRINT}'
@@ -186,6 +187,7 @@ spec:
       server: '${VSPHERE_SERVER}'
       template: '${VSPHERE_TEMPLATE}'
       thumbprint: '${VSPHERE_TLS_THUMBPRINT}'
+      resourcePool: '${VSPHERE_RESOURCE_POOL}'
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfigTemplate

test/e2e/data/gitea/ingress.yaml

@@ -0,0 +1,18 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: gitea-http
+  namespace: default
+spec:
+  ingressClassName: ngrok # This ingress in intended to be used only with ngrok
+  rules:
+  - host: gitea.${RANCHER_HOSTNAME}
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: gitea-http
+            port:
+              number: 3000

test/e2e/flags.go

@@ -54,6 +54,9 @@ type FlagValues struct {
 
 	// ClusterctlBinaryPath is the path to the clusterctl binary to use.
 	ClusterctlBinaryPath string
+
+	// GiteaCustomIngress is the flag to enable custom ingress for Gitea.
+	GiteaCustomIngress bool
 }
 
 // InitFlags is used to specify the standard flags for the e2e tests.
@@ -68,4 +71,5 @@ func InitFlags(values *FlagValues) {
 	flag.StringVar(&values.ClusterctlBinaryPath, "e2e.clusterctl-binary-path", "helm", "path to the clusterctl binary")
 	flag.StringVar(&values.ChartPath, "e2e.chart-path", "", "path to the operator chart")
 	flag.BoolVar(&values.IsolatedMode, "e2e.isolated-mode", false, "if true, the test will run without ngrok and exposing the cluster to the internet. This setup will only work with CAPD or other providers that run in the same network as the bootstrap cluster.")
+	flag.BoolVar(&values.GiteaCustomIngress, "e2e.gitea-custom-ingress", false, "if true, the test will use a custom ingress for Gitea")
 }

test/e2e/helpers.go

@@ -41,6 +41,7 @@ import (
 	managementv3 "github.com/rancher/turtles/internal/rancher/management/v3"
 	provisioningv1 "github.com/rancher/turtles/internal/rancher/provisioning/v1"
 	turtlesframework "github.com/rancher/turtles/test/framework"
+	networkingv1 "k8s.io/api/networking/v1"
 )
 
 func SetupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) {
@@ -86,6 +87,7 @@ func InitScheme() *runtime.Scheme {
 	Expect(clusterv1.AddToScheme(scheme)).To(Succeed())
 	Expect(provisioningv1.AddToScheme(scheme)).To(Succeed())
 	Expect(managementv3.AddToScheme(scheme)).To(Succeed())
+	Expect(networkingv1.AddToScheme(scheme)).To(Succeed())
 	return scheme
 }
 

test/e2e/specs/import_gitops.go

@@ -277,6 +277,7 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
 		turtlesframework.RancherGetOriginalKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
 			Getter:          input.BootstrapClusterProxy.GetClient(),
 			SecretName:      fmt.Sprintf("%s-kubeconfig", capiCluster.Name),
+			ClusterName:     capiCluster.Name,
 			Namespace:       capiCluster.Namespace,
 			WriteToTempFile: true,
 		}, originalKubeconfig)

test/e2e/specs/import_gitops_mgmtv3.go

@@ -297,6 +297,7 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM
 		turtlesframework.RancherGetOriginalKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
 			Getter:          input.BootstrapClusterProxy.GetClient(),
 			SecretName:      fmt.Sprintf("%s-kubeconfig", capiCluster.Name),
+			ClusterName:     capiCluster.Name,
 			Namespace:       capiCluster.Namespace,
 			WriteToTempFile: true,
 		}, originalKubeconfig)

test/e2e/specs/migrate_gitops_provv1_mgmtv3.go

@@ -341,6 +341,7 @@ func MigrateToV3UsingGitOpsSpec(ctx context.Context, inputGetter func() MigrateT
 		turtlesframework.RancherGetOriginalKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
 			Getter:          input.BootstrapClusterProxy.GetClient(),
 			SecretName:      fmt.Sprintf("%s-kubeconfig", capiCluster.Name),
+			ClusterName:     capiCluster.Name,
 			Namespace:       capiCluster.Namespace,
 			WriteToTempFile: true,
 		}, originalKubeconfig)

test/e2e/suites/embedded-capi-disabled/suite_test.go

@@ -29,6 +29,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -243,10 +244,15 @@ var _ = BeforeSuite(func() {
 	giteaValues := map[string]string{
 		"gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		"gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
-		"service.http.type":    "NodePort",
 	}
+
+	giteaServiceType := corev1.ServiceTypeNodePort
 	if flagVals.UseEKS {
-		giteaValues["service.http.type"] = "LoadBalancer"
+		giteaServiceType = corev1.ServiceTypeLoadBalancer
+	}
+
+	if flagVals.GiteaCustomIngress {
+		giteaServiceType = corev1.ServiceTypeClusterIP
 	}
 
 	giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
@@ -263,6 +269,9 @@ var _ = BeforeSuite(func() {
 		AuthSecretName:        e2e.AuthSecretName,
 		Username:              e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		Password:              e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
+		ServiceType:           giteaServiceType,
+		CustomIngressConfig:   e2e.GiteaIngress,
+		Variables:             e2eConfig.Variables,
 	})
 })
 

test/e2e/suites/import-gitops-v3/suite_test.go

@@ -29,14 +29,14 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
-	ctrl "sigs.k8s.io/controller-runtime"
-
 	"github.com/rancher/turtles/test/e2e"
 	"github.com/rancher/turtles/test/framework"
 	turtlesframework "github.com/rancher/turtles/test/framework"
 	"github.com/rancher/turtles/test/testenv"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 // Test suite flags.
@@ -237,10 +237,15 @@ var _ = BeforeSuite(func() {
 	giteaValues := map[string]string{
 		"gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		"gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
-		"service.http.type":    "NodePort",
 	}
+
+	giteaServiceType := corev1.ServiceTypeNodePort
 	if flagVals.UseEKS {
-		giteaValues["service.http.type"] = "LoadBalancer"
+		giteaServiceType = corev1.ServiceTypeLoadBalancer
+	}
+
+	if flagVals.GiteaCustomIngress {
+		giteaServiceType = corev1.ServiceTypeClusterIP
 	}
 
 	giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
@@ -257,6 +262,9 @@ var _ = BeforeSuite(func() {
 		AuthSecretName:        e2e.AuthSecretName,
 		Username:              e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		Password:              e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
+		ServiceType:           giteaServiceType,
+		CustomIngressConfig:   e2e.GiteaIngress,
+		Variables:             e2eConfig.Variables,
 	})
 })
 

test/e2e/suites/import-gitops/suite_test.go

@@ -29,14 +29,14 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
-	ctrl "sigs.k8s.io/controller-runtime"
-
 	"github.com/rancher/turtles/test/e2e"
 	"github.com/rancher/turtles/test/framework"
 	turtlesframework "github.com/rancher/turtles/test/framework"
 	"github.com/rancher/turtles/test/testenv"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 // Test suite flags.
@@ -301,10 +301,15 @@ var _ = BeforeSuite(func() {
 	giteaValues := map[string]string{
 		"gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		"gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
-		"service.http.type":    "NodePort",
 	}
+
+	giteaServiceType := corev1.ServiceTypeNodePort
 	if flagVals.UseEKS {
-		giteaValues["service.http.type"] = "LoadBalancer"
+		giteaServiceType = corev1.ServiceTypeLoadBalancer
+	}
+
+	if flagVals.GiteaCustomIngress {
+		giteaServiceType = corev1.ServiceTypeClusterIP
 	}
 
 	giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
@@ -321,6 +326,9 @@ var _ = BeforeSuite(func() {
 		AuthSecretName:        e2e.AuthSecretName,
 		Username:              e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		Password:              e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
+		ServiceType:           giteaServiceType,
+		CustomIngressConfig:   e2e.GiteaIngress,
+		Variables:             e2eConfig.Variables,
 	})
 })
 

test/e2e/suites/migrate-gitops/suite_test.go

@@ -29,13 +29,13 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
-	ctrl "sigs.k8s.io/controller-runtime"
-
 	"github.com/rancher/turtles/test/e2e"
 	turtlesframework "github.com/rancher/turtles/test/framework"
 	"github.com/rancher/turtles/test/testenv"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 // Test suite flags.
@@ -188,7 +188,15 @@ var _ = BeforeSuite(func() {
 	giteaValues := map[string]string{
 		"gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		"gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
-		"service.http.type":    "NodePort",
+	}
+
+	giteaServiceType := corev1.ServiceTypeNodePort
+	if flagVals.UseEKS {
+		giteaServiceType = corev1.ServiceTypeLoadBalancer
+	}
+
+	if flagVals.GiteaCustomIngress {
+		giteaServiceType = corev1.ServiceTypeClusterIP
 	}
 
 	giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
@@ -205,6 +213,9 @@ var _ = BeforeSuite(func() {
 		AuthSecretName:        e2e.AuthSecretName,
 		Username:              e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		Password:              e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
+		ServiceType:           giteaServiceType,
+		CustomIngressConfig:   e2e.GiteaIngress,
+		Variables:             e2eConfig.Variables,
 	})
 })
 

test/framework/helper.go

@@ -38,7 +38,7 @@ type VariableLookupFunc func(key string) string
 // GetVariable is used to get the value for a variable. The expectation is that the variable exists in one of
 // the sources. Assertion will fail if its not found. The order of precedence when checking for variables is:
 // 1. Environment variables
-// 2. Base variablesß
+// 2. Base variables
 // This is a re-implementation of the CAPI function to add additional logging.
 func GetVariable(vars VariableCollection) VariableLookupFunc {
 	Expect(vars).ToNot(BeNil(), "Variable should not be nil")

test/framework/kube_helper.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
@@ -32,7 +33,7 @@ import (
 
 const (
 	retryableOperationInterval = 3 * time.Second
-	retryableOperationTimeout  = 3 * time.Minute
+	retryableOperationTimeout  = 99 * time.Minute
 )
 
 // GetNodeAddressInput is th einput to GetNodeAddress.
@@ -233,3 +234,28 @@ func CreateDockerRegistrySecret(ctx context.Context, input CreateDockerRegistryS
 	Expect(cmdCreateSecret.Error).NotTo(HaveOccurred(), "Failed creating docker registry k8s secret")
 	Expect(cmdCreateSecret.ExitCode).To(Equal(0), "Creating secret return non-zero exit code")
 }
+
+// GetIngressHostInput is the input to GetIngressHost.
+type GetIngressHostInput struct {
+	GetLister        framework.GetLister
+	IngressName      string
+	IngressNamespace string
+	IngressRuleIndex int
+}
+
+// GetIngressHost gets the host from an ingress object.
+func GetIngressHost(ctx context.Context, input GetIngressHostInput) string {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for GetNodeAddress")
+	Expect(input.GetLister).ToNot(BeNil(), "Invalid argument. input.GetLister can't be nil when calling GetIngressHost")
+
+	ingress := &networkingv1.Ingress{}
+	Eventually(func() error {
+		return input.GetLister.Get(ctx, client.ObjectKey{Namespace: input.IngressNamespace, Name: input.IngressName}, ingress)
+	}).Should(Succeed(), "Failed to get ingress")
+
+	Expect(ingress.Spec.Rules).NotTo(HaveLen(0), "Expected ingress to have at least 1 rule")
+	Expect(len(ingress.Spec.Rules) >= input.IngressRuleIndex).To(BeTrue(), "Ingress rule index is greater than number of rules")
+
+	rule := ingress.Spec.Rules[input.IngressRuleIndex]
+	return rule.Host
+}

test/framework/rancher_helpers.go

@@ -43,6 +43,7 @@ type RancherGetClusterKubeconfigInput struct {
 	Getter           framework.Getter
 	SecretName       string
 	Namespace        string
+	ClusterName      string
 	RancherServerURL string
 	WriteToTempFile  bool
 }
@@ -160,7 +161,7 @@ func RancherGetOriginalKubeconfig(ctx context.Context, input RancherGetClusterKu
 func (i *RancherGetClusterKubeconfigInput) isDockerCluster(ctx context.Context) bool {
 	cluster := &clusterv1.Cluster{}
 	key := client.ObjectKey{
-		Name:      i.SecretName,
+		Name:      i.ClusterName,
 		Namespace: i.Namespace,
 	}