Support --skip-delete to skip deletion of resources created during tests

CONTRIBUTING.md

@@ -55,6 +55,26 @@ make integration-test
 
 See [the testing documentation](https://kudo.dev/docs/testing) for more details.
 
+##### Debugging KUDO integration tests
+
+To debug the integration tests, run them locally:
+
+```
+go run ./cmd/kubectl-kudo test
+```
+
+By default, the test harness starts a local etcd and kube-apiserver for use in the tests. When debugging, it can be helpful to run your tests against a live cluster to make access easier:
+
+```
+go run ./cmd/kubectl-kudo test --start-control-plane=false
+```
+
+It also defaults to deleting any resources created during tests. You can configure the test harness to skip cleanup:
+
+```
+go run ./cmd/kubectl-kudo test --start-control-plane=false --skip-delete
+```
+
 ### Build and run tests using Docker
 If you don't want to install kubebuilder and other dependencies of KUDO locally, you can build KUDO and run the tests inside a Docker container.
 

pkg/apis/kudo/v1alpha1/test_types.go

@@ -24,6 +24,8 @@ type TestSuite struct {
 	StartControlPlane bool `json:"startControlPlane"`
 	// Whether or not to start the KUDO controller for the tests.
 	StartKUDO bool `json:"startKUDO"`
+	// If set, do not delete the resources after running the tests.
+	SkipDelete bool `json:"skipDelete"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/kudoctl/cmd/test.go

@@ -4,14 +4,14 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"reflect"
 	"testing"
 
 	kudo "github.com/kudobuilder/kudo/pkg/apis/kudo/v1alpha1"
 	"github.com/kudobuilder/kudo/pkg/test"
 	testutils "github.com/kudobuilder/kudo/pkg/test/utils"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
 var (
@@ -36,10 +36,14 @@ var (
 // newTestCmd creates the test command for the CLI
 func newTestCmd() *cobra.Command {
 	configPath := ""
+	crdDir := ""
+	manifestsDir := ""
+	testToRun := ""
+	startControlPlane := false
+	startKUDO := false
+	skipDelete := false
+
 	options := kudo.TestSuite{}
-	defaults := kudo.TestSuite{
-		TestDirs: []string{},
-	}
 
 	testCmd := &cobra.Command{
 		Use:   "test [flags]... [test directories]...",
@@ -55,20 +59,20 @@ If no arguments are provided, the test harness will attempt to load the test con
 For more detailed documentation, visit: https://kudo.dev/docs/testing`,
 		Example: testExample,
 		PreRunE: func(cmd *cobra.Command, args []string) error {
+			flags := cmd.Flags()
+
 			options.TestDirs = args
 
-			if configPath == "" && reflect.DeepEqual(options, defaults) {
+			// If a config is not set and kudo-test.yaml exists, set configPath to kudo-test.yaml.
+			if configPath == "" {
 				if _, err := os.Stat("kudo-test.yaml"); err == nil {
 					configPath = "kudo-test.yaml"
 				} else {
 					return fmt.Errorf("kudo-test.yaml not found, provide either --config or arguments indicating the tests to load")
 				}
 			}
 
-			if configPath != "" && !reflect.DeepEqual(options, defaults) {
-				return fmt.Errorf("provide either --config or other arguments, but not both")
-			}
-
+			// Load the configuration YAML into options.
 			if configPath != "" {
 				objects, err := testutils.LoadYAML(configPath)
 				if err != nil {
@@ -86,14 +90,40 @@ For more detailed documentation, visit: https://kudo.dev/docs/testing`,
 				}
 			}
 
+			// Override configuration file options with any command line flags if they are set.
+
+			if isSet(flags, "crd-dir") {
+				options.CRDDir = crdDir
+			}
+
+			if isSet(flags, "manifests-dir") {
+				options.ManifestsDir = manifestsDir
+			}
+
+			if isSet(flags, "start-control-plane") {
+				options.StartControlPlane = startControlPlane
+			}
+
+			if isSet(flags, "start-kudo") {
+				options.StartKUDO = startKUDO
+			}
+
+			if isSet(flags, "skip-delete") {
+				options.SkipDelete = skipDelete
+			}
+
+			if len(args) != 0 {
+				options.TestDirs = args
+			}
+
 			if len(options.TestDirs) == 0 {
 				return fmt.Errorf("no test directories provided, please provide either --config or test directories on the command line")
 			}
 
 			return nil
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			testutils.RunTests("kudo", func(t *testing.T) {
+			testutils.RunTests("kudo", testToRun, func(t *testing.T) {
 				harness := test.Harness{
 					TestSuite: options,
 					T:         t,
@@ -105,10 +135,25 @@ For more detailed documentation, visit: https://kudo.dev/docs/testing`,
 	}
 
 	testCmd.Flags().StringVar(&configPath, "config", "", "Path to file to load test settings from (must not be set with any other arguments).")
-	testCmd.Flags().StringVar(&options.CRDDir, "crd-dir", "", "Directory to load CustomResourceDefinitions from prior to running the tests.")
-	testCmd.Flags().StringVar(&options.ManifestsDir, "manifests-dir", "", "A directory containing manifests to apply before running the tests.")
-	testCmd.Flags().BoolVar(&options.StartControlPlane, "start-control-plane", false, "Start a local Kubernetes control plane for the tests (requires etcd and kube-apiserver binaries, implies --start-kudo).")
-	testCmd.Flags().BoolVar(&options.StartKUDO, "start-kudo", false, "Start KUDO during the test run.")
+	testCmd.Flags().StringVar(&crdDir, "crd-dir", "", "Directory to load CustomResourceDefinitions from prior to running the tests.")
+	testCmd.Flags().StringVar(&manifestsDir, "manifests-dir", "", "A directory containing manifests to apply before running the tests.")
+	testCmd.Flags().StringVar(&testToRun, "test", "", "If set, the specific test case to run.")
+	testCmd.Flags().BoolVar(&startControlPlane, "start-control-plane", false, "Start a local Kubernetes control plane for the tests (requires etcd and kube-apiserver binaries, implies --start-kudo).")
+	testCmd.Flags().BoolVar(&startKUDO, "start-kudo", false, "Start KUDO during the test run.")
+	testCmd.Flags().BoolVar(&skipDelete, "skip-delete", false, "If set, do not delete resources created during tests (helpful for debugging test failures).")
 
 	return testCmd
 }
+
+// isSet returns true if a flag is set on the command line.
+func isSet(flagSet *pflag.FlagSet, name string) bool {
+	found := false
+
+	flagSet.Visit(func(flag *pflag.Flag) {
+		if flag.Name == name {
+			found = true
+		}
+	})
+
+	return found
+}

pkg/test/case.go

@@ -24,9 +24,10 @@ var testStepRegex = regexp.MustCompile(`^(\d+)-([^.]+)(.yaml)?$`)
 // Case contains all of the test steps and the Kubernetes client and other global configuration
 // for a test.
 type Case struct {
-	Steps []*Step
-	Name  string
-	Dir   string
+	Steps      []*Step
+	Name       string
+	Dir        string
+	SkipDelete bool
 
 	Client          client.Client
 	DiscoveryClient discovery.DiscoveryInterface
@@ -72,14 +73,18 @@ func (t *Case) TestCaseFactory() func(*testing.T) {
 			test.Fatal(err)
 		}
 
-		defer t.DeleteNamespace(ns)
+		if !t.SkipDelete {
+			defer t.DeleteNamespace(ns)
+		}
 
 		for _, testStep := range t.Steps {
 			testStep.Client = t.Client
 			testStep.DiscoveryClient = t.DiscoveryClient
 			testStep.Logger = t.Logger.WithPrefix(testStep.String())
 
-			defer testStep.Clean(ns)
+			if !t.SkipDelete {
+				defer testStep.Clean(ns)
+			}
 
 			if errs := testStep.Run(ns); len(errs) > 0 {
 				for _, err := range errs {

pkg/test/harness.go

@@ -52,9 +52,10 @@ func (h *Harness) LoadTests(dir string) ([]*Case, error) {
 		}
 
 		tests = append(tests, &Case{
-			Steps: []*Step{},
-			Name:  file.Name(),
-			Dir:   filepath.Join(dir, file.Name()),
+			Steps:      []*Step{},
+			Name:       file.Name(),
+			Dir:        filepath.Join(dir, file.Name()),
+			SkipDelete: h.TestSuite.SkipDelete,
 		})
 	}
 

pkg/test/utils/testing.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"flag"
+	"fmt"
 	"io"
 	"os"
 	"regexp"
@@ -11,10 +12,15 @@ import (
 
 // RunTests runs a Go test method without requiring the Go compiler.
 // This does not currently support test caching.
-func RunTests(testName string, testFunc func(*testing.T)) {
+// If testToRun is set to a non-empty string, it is passed as a `-run` argument to the go test harness.
+func RunTests(testName string, testToRun string, testFunc func(*testing.T)) {
 	// Set the verbose test flag to true since we are not using the regular go test CLI.
 	flag.Lookup("test.v").Value.Set("true")
 
+	if testToRun != "" {
+		flag.Lookup("test.run").Value.Set(fmt.Sprintf("//%s", testToRun))
+	}
+
 	os.Exit(testing.MainStart(&testDeps{}, []testing.InternalTest{
 		{
 			Name: testName,