tests: introduce new isolated suite for running integration tests

.github/workflows/_integration_tests.yaml

@@ -87,7 +87,7 @@ jobs:
           - name: dbless-expression-router
             test: dbless
             feature_gates: "ExpressionRoutes=true,GatewayAlpha=true"
-            router-flavor: "expressions"
+            router-flavor: expressions
           - name: dbless-gateway-alpha
             test: dbless
             feature_gates: "GatewayAlpha=true"
@@ -104,6 +104,14 @@ jobs:
             test: dbless
             run_invalid_config: "true"
             go_test_flags: -run=TestIngressRecoverFromInvalidPath
+          #
+          - name: isolated-dbless
+            test: isolated.dbless
+            feature_gates: "GatewayAlpha=true"
+          - name: isolated-dbless-expression-router
+            router-flavor: expressions
+            test: isolated.dbless
+            feature_gates: "ExpressionRoutes=true,GatewayAlpha=true"
 
     steps:
       - uses: Kong/kong-license@master

Makefile

@@ -426,6 +426,31 @@ _test.integration: _check.container.environment go-junit-report
 		./test/integration | \
 	$(GOJUNIT) -iocopy -out $(JUNIT_REPORT) -parser gotest
 
+.PHONY: _test.integration.isolated
+_test.integration.isolated: _check.container.environment go-junit-report
+	KONG_CLUSTER_VERSION="$(KONG_CLUSTER_VERSION)" \
+		TEST_KONG_HELM_CHART_VERSION="$(TEST_KONG_HELM_CHART_VERSION)" \
+		TEST_DATABASE_MODE="$(DBMODE)" \
+		GOFLAGS="-tags=$(GOTAGS)" \
+		KONG_CONTROLLER_FEATURE_GATES="$(KONG_CONTROLLER_FEATURE_GATES)" \
+		go test $(GOTESTFLAGS) \
+		-v \
+		-timeout $(INTEGRATION_TEST_TIMEOUT) \
+		-parallel $(NCPU) \
+		-race \
+		-covermode=atomic \
+		-coverpkg=$(PKG_LIST) \
+		-coverprofile=$(COVERAGE_OUT) \
+		./test/integration/isolated -args --parallel $(E2E_FRAMEWORK_FLAGS) | \
+	$(GOJUNIT) -iocopy -out $(JUNIT_REPORT) -parser gotest
+
+.PHONY: test.integration.isolated.dbles.
+test.integration.isolated.dbless:
+	@$(MAKE) _test.integration.isolated \
+		GOTAGS="integration_tests" \
+		DBMODE=off \
+		COVERAGE_OUT=coverage.dbless.out
+
 .PHONY: test.integration.dbless
 test.integration.dbless:
 	@$(MAKE) _test.integration \

TESTING.md

@@ -10,6 +10,8 @@ In KIC, we use several levels of testing:
 - [unit tests](#unit-tests)
 - [`envtest` based tests](#envtest-based-tests)
 - [integration tests](#integration-tests)
+- [isolated integration tests](#isolated-integration-tests)
+- [Kong integration tests](#kong-integration-tests)
 - [end to end (E2E) tests](#end-to-end-e2e-tests)
 
 ### Unit tests
@@ -149,6 +151,105 @@ make test.integration.dbless GOTESTFLAGS="-count 1 -run TestUDPRouteEssentials"
 [integration_test_suite]: https://github.com/Kong/kubernetes-ingress-controller/blob/61e06ee64ff913aa9952816121125fca7ed59ba5/test/integration/suite_test.go#L36
 [manager_run]: https://github.com/Kong/kubernetes-ingress-controller/blob/5abc699aeee552945a76c82e3f7abb3e1b2fabf1/internal/cmd/rootcmd/run.go#L14-L22
 
+### Isolated integration tests
+
+Similarly to KIC's integration tests, isolated integration tests rely on its
+controller manager being run in the same process as the tests via [`manager.Run()`][manager_run].
+
+These tests rely on [kubernetes-sigs/e2e-framework][github-e2e-framework] for setup,
+teardown, tests filtering etc.
+
+Said setup will either create a new [kind](https://kind.sigs.k8s.io) cluster or use an existing one to run the
+tests against.
+
+Most of the setup - and cleanup after the tests have run - is being done using [ktf][ktf].
+
+Currently, the cluster is being shared across all the tests that are a part
+of the test suite so special care needs to be taken in order to clean up after the tests
+that run.
+The typical approach is to run them in a dedicated, disposable Kubernetes namespace created just for the purposes of that test.
+
+#### Difference between isolated and regular integration tests
+
+There are a couple of key differences between isolated and regular integration tests:
+
+- In isolated tests each test's [`Feature`][pkggodev-e2e-framework-feature] gets
+  its own kong deployment through the means of [ktf][ktf]'s kong addon.
+- In isolated tests each test's [`Feature`][pkggodev-e2e-framework-feature] gets
+  its own controller manager instance started which is configured against the above
+  mentioned kong instance.
+  Said controller manager instance will be configured to only watch `Feature`'s
+  dedicated namespace (more info below) via `--watch-namespace`.
+  You can add more namespaces for the controller manager to watch via
+  `ControllerManagerOptAdditionalWatchNamespace` `featureSetup` option:
+
+  ```go
+  ...
+  WithSetup("deploy kong addon into cluster", featureSetup(
+    helpers.ControllerManagerOptAdditionalWatchNamespace("my-additional-namespace"),
+  )).
+  ...
+  ```
+
+- In regular integration tests each test gets it's own Kubernetes namespace through
+  manual creation via helper functions like:
+
+  ```go
+  ns, cleaner := helpers.Setup(ctx, t, env)
+  ```
+
+  Which will automatically remove the namspace when the test is finished.
+
+  In isolated tests each test's [`Feature`][pkggodev-e2e-framework-feature] will
+  get its own Kubernetes namespace which you can get via:
+
+  ```go
+  namespace := GetNamespaceForT(ctx, t)
+  ```
+
+#### Room for improvement
+
+- Eventually the whole integration suite could be migrate to use this setup.
+  When that happens, we can add logic into setup which would make tests that don't 
+  need this level of separation to reuse a common installation of Kong (e.g. to
+  its default - `kong` - namespace).
+
+  This way we'll have the best of both worlds:
+
+  - seprate tests where it's needed
+  - shared, when it's not and where speed is the priority
+
+#### How to run
+
+Tests are located under `tests/integration/isolated` and use `integration_tests`
+[build tag][go_build_tag].
+
+You can run them using one of the dedicated Makefile targets:
+
+- `test.integration.isolated.dbless` run all dbless isolated integration tests with standard
+  verbose output on stderr. The output will also include controllers' logs.
+
+Through `GOTESTFLAGS` you can specify custom flags that will be passed to `go test`.
+This can allow you to run a subset of all the tests for faster feedback times, e.g.:
+
+```
+make test.integration.isolated.dbless GOTESTFLAGS="-count 1 -run TestUDPRouteEssentials"
+```
+
+You can also specify e2e-framework's flags e.g. to filter tests via [labels][github-e2e-framework-labels].
+
+```
+make test.integration.isolated.dbless E2E_FRAMEWORK_FLAGS="-labels=kind=UDPRoute,example=true"
+```
+
+[github-e2e-framework]: https://github.com/kubernetes-sigs/e2e-framework
+[github-e2e-framework-labels]: https://github.com/kubernetes-sigs/e2e-framework/tree/main/examples/skip_flags#use-labels-in-your-tests
+[pkggodev-e2e-framework-feature]: https://pkg.go.dev/sigs.k8s.io/e2e-framework/pkg/features
+[ktf]: https://github.com/Kong/kubernetes-testing-framework
+[pkggodev_testmain]: https://pkg.go.dev/testing#hdr-Main
+[integration_test_suite]: https://github.com/Kong/kubernetes-ingress-controller/blob/61e06ee64ff913aa9952816121125fca7ed59ba5/test/integration/suite_test.go#L36
+[manager_run]: https://github.com/Kong/kubernetes-ingress-controller/blob/5abc699aeee552945a76c82e3f7abb3e1b2fabf1/internal/cmd/rootcmd/run.go#L14-L22
+
 ### Kong integration tests
 
 Tests located under `test/kongintegration/` allow verifying individual components of KIC

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/kong/deck v1.27.1
 	github.com/kong/go-kong v0.47.0
 	github.com/kong/kubernetes-telemetry v0.1.1
-	github.com/kong/kubernetes-testing-framework v0.40.0
+	github.com/kong/kubernetes-testing-framework v0.39.2-0.20231011203028-366e9fe9e610
 	github.com/lithammer/dedent v1.1.0
 	github.com/miekg/dns v1.1.56
 	github.com/mitchellh/mapstructure v1.5.0
@@ -212,6 +212,7 @@ require (
 	k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect
 	k8s.io/kubectl v0.28.2
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	sigs.k8s.io/e2e-framework v0.3.0
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kind v0.20.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect

go.sum

@@ -258,8 +258,8 @@ github.com/kong/go-kong v0.47.0 h1:fuZjRnUChITogljM4mY4HGGTKTaZM50970Wwg6EvQgs=
 github.com/kong/go-kong v0.47.0/go.mod h1:QaLx1EsT+roe5V6h7aKghwLn0bXg7l/Bg8wzlcCmygc=
 github.com/kong/kubernetes-telemetry v0.1.1 h1:q8iABCuonO16p5q6QVygr7J39G5o5jK5Y3w3ZlE8JC4=
 github.com/kong/kubernetes-telemetry v0.1.1/go.mod h1:0yzRZVPwKeOQUFMWbROEJzUE1z8WWngWiujoYtdEEPA=
-github.com/kong/kubernetes-testing-framework v0.40.0 h1:dZsofs0V2F1DepjQeZkEZ7SE5AzuNJjgrttCIlt2yIo=
-github.com/kong/kubernetes-testing-framework v0.40.0/go.mod h1:ymhaj4J6qrKzrdWJchdbOCASObZoqCfdGtPy2URQpgo=
+github.com/kong/kubernetes-testing-framework v0.39.2-0.20231011203028-366e9fe9e610 h1:yns94g328EsmKwTOpWI3Ra1rodUSB7t+DvBJSUdhp4c=
+github.com/kong/kubernetes-testing-framework v0.39.2-0.20231011203028-366e9fe9e610/go.mod h1:lkKmrPeHHjAl2unrElIP0/mOO5VPaPWrBi1hZ5rzz9Y=
 github.com/kong/semver/v4 v4.0.1 h1:DIcNR8W3gfx0KabFBADPalxxsp+q/5COwIFkkhrFQ2Y=
 github.com/kong/semver/v4 v4.0.1/go.mod h1:LImQ0oT15pJvSns/hs2laLca2zcYoHu5EsSNY0J6/QA=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -446,6 +446,8 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY=
+github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -688,6 +690,8 @@ k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSn
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU=
 sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU=
+sigs.k8s.io/e2e-framework v0.3.0 h1:eqQALBtPCth8+ulTs6lcPK7ytV5rZSSHJzQHZph4O7U=
+sigs.k8s.io/e2e-framework v0.3.0/go.mod h1:C+ef37/D90Dc7Xq1jQnNbJYscrUGpxrWog9bx2KIa+c=
 sigs.k8s.io/gateway-api v0.8.1 h1:Bo4NMAQFYkQZnHXOfufbYwbPW7b3Ic5NjpbeW6EJxuU=
 sigs.k8s.io/gateway-api v0.8.1/go.mod h1:0PteDrsrgkRmr13nDqFWnev8tOysAVrwnvfFM55tSVg=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=

internal/cmd/rootcmd/run.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"os/signal"
 
 	"github.com/go-logr/logr"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -19,16 +20,17 @@ func Run(ctx context.Context, c *manager.Config, output io.Writer) error {
 	}
 	ctx = ctrl.LoggerInto(ctx, logger)
 
+	ctx, err = SetupSignalHandler(ctx, c, logger)
+	if err != nil {
+		return fmt.Errorf("failed to setup signal handler: %w", err)
+	}
+	defer signal.Ignore(shutdownSignals...)
+
 	return RunWithLogger(ctx, c, logger)
 }
 
 // RunWithLogger starts the controller manager with a provided logger.
 func RunWithLogger(ctx context.Context, c *manager.Config, logger logr.Logger) error {
-	ctx, err := SetupSignalHandler(ctx, c, logger)
-	if err != nil {
-		return fmt.Errorf("failed to setup signal handler: %w", err)
-	}
-
 	if err := c.Validate(); err != nil {
 		return fmt.Errorf("config invalid: %w", err)
 	}

internal/cmd/rootcmd/signal.go

@@ -26,6 +26,7 @@ var (
 func SetupSignalHandler(ctx context.Context, cfg *manager.Config, logger logr.Logger) (context.Context, error) {
 	// This will prevent multiple signal handlers from being created
 	if ok := mutex.TryLock(); !ok {
+		// return ctx, nil
 		return nil, errors.New("signal handler can only be setup once")
 	}
 

internal/util/test/controller_manager.go

@@ -8,6 +8,9 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters"
 	ktfkong "github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/kong"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/cmd/rootcmd"
@@ -22,65 +25,88 @@ var logOutput = os.Getenv("TEST_KONG_KIC_MANAGER_LOG_OUTPUT")
 // Testing Utility Functions - Controller Manager
 // -----------------------------------------------------------------------------
 
+// PrepareClusterForRunningControllerManager prepares the provided cluster for running
+// the controller manager.
+// It creates kong's namespace, deploys its RBAC manifests and CRDs.
+func PrepareClusterForRunningControllerManager(
+	ctx context.Context,
+	cluster clusters.Cluster,
+) error {
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: consts.ControllerNamespace,
+		},
+	}
+	nsClient := cluster.Client().CoreV1().Namespaces()
+	if _, err := nsClient.Create(ctx, ns, metav1.CreateOptions{}); err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("failed creating %s namespace: %w", ns.Name, err)
+		}
+	}
+
+	// deploy all RBACs required for testing to the cluster
+	if err := DeployRBACsForCluster(ctx, cluster); err != nil {
+		return fmt.Errorf("failed to deploy RBACs: %w", err)
+	}
+
+	// deploy all CRDs required for testing to the cluster
+	if err := DeployCRDsForCluster(ctx, cluster); err != nil {
+		return fmt.Errorf("failed to deploy CRDs: %w", err)
+	}
+
+	return nil
+}
+
 // DeployControllerManagerForCluster deploys all the base CRDs needed for the
 // controller manager to function, and also runs a copy of the controller
 // manager on a provided test cluster.
 //
 // Controller managers started this way will run in the background in a goroutine:
 // The caller must use the provided context.Context to stop the controller manager
 // from running when they're done with it.
+//
+// It returns a context cancellation func which will stop the manager and an error.
 func DeployControllerManagerForCluster(
 	ctx context.Context,
 	logger logr.Logger,
 	cluster clusters.Cluster,
+	kongAddon *ktfkong.Addon,
 	additionalFlags ...string,
-) error {
-	// ensure that the provided test cluster has a kongAddon deployed to it
-	var kongAddon *ktfkong.Addon
-	for _, addon := range cluster.ListAddons() {
-		if addon.Name() == ktfkong.AddonName {
-			var ok bool
-			kongAddon, ok = addon.(*ktfkong.Addon)
-			if !ok {
-				return fmt.Errorf("an invalid kong addon was present in test environment")
+) (func(), error) {
+	if kongAddon == nil {
+		// ensure that the provided test cluster has a kongAddon deployed to it
+		for _, addon := range cluster.ListAddons() {
+			a, ok := addon.(*ktfkong.Addon)
+			if ok {
+				kongAddon = a
 			}
 		}
 	}
 	if kongAddon == nil {
-		return fmt.Errorf("no kong addon found loaded in cluster %s", cluster.Name())
+		return nil, fmt.Errorf("no kong addon found loaded in cluster %s", cluster.Name())
 	}
 
 	// determine the proxy admin URL for the Kong Gateway on this cluster:
 	proxyAdminURL, err := kongAddon.ProxyAdminURL(ctx, cluster)
 	if err != nil {
-		return fmt.Errorf("couldn't determine Kong Gateway Admin URL on cluster %s: %w", cluster.Name(), err)
-	}
-
-	// deploy all RBACs required for testing to the cluster
-	if err := DeployRBACsForCluster(ctx, cluster); err != nil {
-		return fmt.Errorf("failed to deploy RBACs: %w", err)
-	}
-
-	// deploy all CRDs required for testing to the cluster
-	if err := DeployCRDsForCluster(ctx, cluster); err != nil {
-		return fmt.Errorf("failed to deploy CRDs: %w", err)
+		return nil, fmt.Errorf("couldn't determine Kong Gateway Admin URL on cluster %s: %w", cluster.Name(), err)
 	}
 
 	// create a tempfile to hold the cluster kubeconfig that will be used for the controller
 	// generate a temporary kubeconfig since we're going to be using the helm CLI
 	kubeconfig, err := clusters.TempKubeconfig(cluster)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	// render all controller manager flag options
 	controllerManagerFlags := []string{
 		fmt.Sprintf("--kong-admin-url=http://%s:8001", proxyAdminURL.Hostname()),
 		fmt.Sprintf("--kubeconfig=%s", kubeconfig.Name()),
 		"--election-id=integrationtests.konghq.com",
-		fmt.Sprintf("--ingress-service=%s/ingress-controller-kong-proxy", consts.ControllerNamespace),
-		fmt.Sprintf("--ingress-service-udp=%s/ingress-controller-kong-udp-proxy", consts.ControllerNamespace),
 		"--log-format=text",
+		fmt.Sprintf("--ingress-service=%s/ingress-controller-kong-proxy", kongAddon.Namespace()),
+		fmt.Sprintf("--ingress-service-udp=%s/ingress-controller-kong-udp-proxy", kongAddon.Namespace()),
 	}
 	controllerManagerFlags = append(controllerManagerFlags, additionalFlags...)
 
@@ -91,9 +117,10 @@ func DeployControllerManagerForCluster(
 	flags := config.FlagSet()
 	if err := flags.Parse(controllerManagerFlags); err != nil {
 		os.Remove(kubeconfig.Name())
-		return fmt.Errorf("failed to parse manager flags: %w", err)
+		return nil, fmt.Errorf("failed to parse manager flags: %w", err)
 	}
 
+	ctx, cancel := context.WithCancel(ctx)
 	// run the controller in the background
 	go func() {
 		defer os.Remove(kubeconfig.Name())
@@ -103,7 +130,7 @@ func DeployControllerManagerForCluster(
 		}
 	}()
 
-	return nil
+	return cancel, nil
 }
 
 // SetupLoggers sets up the loggers for the controller manager.