Implement test commands feature (#788)

This implements the test commands feature to support running commands that are not kubectl plugins, see #754 for more details.

To use, set:

```
apiVersion: kudo.dev/v1alpha1
kind: TestSuite
commands:
- command: kubectl kudo init
```

Or:

```
apiVersion: kudo.dev/v1alpha1
kind: TestStep
commands:
- command: kubectl label mypod label=true
  namespaced: true
```

Also fixes integration-test output and test infrastructure requirements 

```release-note
The test harness now supports running non-kubectl-based commands as a part of tests.
```

.gitignore

@@ -4,6 +4,7 @@ vendor/
 hack/code-gen
 hack/controller-gen
 test/.git-credentials
+reports/
 
 ### mac
 .DS_Store

Makefile

@@ -27,9 +27,10 @@ test:
 .PHONY: integration-test
 # Run integration tests
 integration-test: cli-fast
+	mkdir -p reports/
 	go get github.com/jstemmer/go-junit-report
-	go test -tags integration ./pkg/... ./cmd/... -v -mod=readonly -coverprofile cover-integration.out 2>&1 | go-junit-report -set-exit-code > reports/integration_report.xml
-	go run ./cmd/kubectl-kudo test 2>&1 | go-junit-report -set-exit-code > reports/kudo_test_report.xml
+	go test -tags integration ./pkg/... ./cmd/... -v -mod=readonly -coverprofile cover-integration.out 2>&1 |tee /dev/fd/2 |go-junit-report -set-exit-code > reports/integration_report.xml
+	go run ./cmd/kubectl-kudo test 2>&1 |tee /dev/fd/2 |go-junit-report -set-exit-code > reports/kudo_test_report.xml
 
 .PHONY: test-clean
 # Clean test reports

go.sum

@@ -172,6 +172,7 @@ github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswD
 github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=

keps/0008-operator-testing.md

@@ -176,6 +176,8 @@ type Command struct {
 	Command string `json:"command"`
 	// If set, the `--namespace` flag will be appended to the command with the namespace to use.
 	Namespaced bool `json:"namespaced"`
+	// If set, failures will be ignored.
+	IgnoreFailure bool `json:"ignoreFailure"`
 }
 ```
 

pkg/apis/kudo/v1alpha1/test_types.go

@@ -42,6 +42,8 @@ type TestSuite struct {
 	ArtifactsDir string `json:"artifactsDir"`
 	// Kubectl commands to run before running any tests.
 	Kubectl []string `json:"kubectl"`
+	// Commands to run prior to running the tests.
+	Commands []Command `json:"commands"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -63,6 +65,9 @@ type TestStep struct {
 	// Kubectl commands to run at the start of the test
 	Kubectl []string `json:"kubectl"`
 
+	// Commands to run prior at the beginning of the test step.
+	Commands []Command `json:"commands"`
+
 	// Allowed environment labels
 	// Disallowed environment labels
 }
@@ -84,3 +89,13 @@ type ObjectReference struct {
 	// Labels to match on.
 	Labels map[string]string
 }
+
+// Command describes a command to run as a part of a test step or suite.
+type Command struct {
+	// The command and argument to run as a string.
+	Command string `json:"command"`
+	// If set, the `--namespace` flag will be appended to the command with the namespace to use.
+	Namespaced bool `json:"namespaced"`
+	// If set, failures will be ignored.
+	IgnoreFailure bool `json:"ignoreFailure"`
+}

pkg/test/harness.go

@@ -325,6 +325,10 @@ func (h *Harness) Run() {
 		}
 	}
 
+	if err := testutils.RunCommands(h.GetLogger(), "default", "", h.TestSuite.Commands, ""); err != nil {
+		h.T.Fatal(err)
+	}
+
 	if err := testutils.RunKubectlCommands(h.GetLogger(), "default", h.TestSuite.Kubectl, ""); err != nil {
 		h.T.Fatal(err)
 	}

pkg/test/step.go

@@ -304,6 +304,10 @@ func (s *Step) Run(namespace string) []error {
 	testErrors := []error{}
 
 	if s.Step != nil {
+		if errors := testutils.RunCommands(s.Logger, namespace, "", s.Step.Commands, s.Dir); errors != nil {
+			testErrors = append(testErrors, errors...)
+		}
+
 		if errors := testutils.RunKubectlCommands(s.Logger, namespace, s.Step.Kubectl, s.Dir); errors != nil {
 			testErrors = append(testErrors, errors...)
 		}

pkg/test/test_data/cli-test/01-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cli-test-pod
+  labels:
+    test: "true"

pkg/test/test_data/cli-test/01-patch.yaml

@@ -0,0 +1,7 @@
+apiVersion: kudo.dev/v1alpha1
+kind: TestStep
+commands:
+- command: kubectl label pod cli-test-pod test=true
+  namespaced: true
+- command: kubectl command is bad
+  ignoreFailure: true

pkg/test/utils/kubernetes.go

@@ -809,64 +809,74 @@ func StartTestEnvironment() (env TestEnvironment, err error) {
 	return
 }
 
-// GetKubectlArgs parses a kubectl command line string into its arguments and appends a namespace if it is not already set.
-func GetKubectlArgs(args string, namespace string) ([]string, error) {
+// GetArgs parses a command line string into its arguments and appends a namespace if it is not already set.
+func GetArgs(ctx context.Context, command string, cmd kudo.Command, namespace string) (*exec.Cmd, error) {
 	argSlice := []string{}
 
-	argSplit, err := shlex.Split(args)
+	argSplit, err := shlex.Split(cmd.Command)
 	if err != nil {
 		return nil, err
 	}
 
-	fs := pflag.NewFlagSet("", pflag.ContinueOnError)
-	fs.ParseErrorsWhitelist.UnknownFlags = true
-
-	namespaceParsed := fs.StringP("namespace", "n", "", "")
-	if err := fs.Parse(argSplit); err != nil {
-		return nil, err
-	}
-
-	if argSplit[0] != "kubectl" {
-		argSlice = append(argSlice, "kubectl")
+	if command != "" && argSplit[0] != command {
+		argSlice = append(argSlice, command)
 	}
 
 	argSlice = append(argSlice, argSplit...)
 
-	if *namespaceParsed == "" {
-		argSlice = append(argSlice, "--namespace", namespace)
+	if cmd.Namespaced {
+		fs := pflag.NewFlagSet("", pflag.ContinueOnError)
+		fs.ParseErrorsWhitelist.UnknownFlags = true
+
+		namespaceParsed := fs.StringP("namespace", "n", "", "")
+		if err := fs.Parse(argSplit); err != nil {
+			return nil, err
+		}
+
+		if *namespaceParsed == "" {
+			argSlice = append(argSlice, "--namespace", namespace)
+		}
 	}
 
-	return argSlice, nil
+	builtCmd := exec.Command(argSlice[0])
+	builtCmd.Args = argSlice
+	return builtCmd, nil
 }
 
-// Kubectl runs a kubectl command (or plugin) with args.
+// RunCommand runs a command with args.
 // args gets split on spaces (respecting quoted strings).
-func Kubectl(ctx context.Context, namespace string, args string, cwd string, stdout io.Writer, stderr io.Writer) error {
+func RunCommand(ctx context.Context, namespace string, command string, cmd kudo.Command, cwd string, stdout io.Writer, stderr io.Writer) error {
 	actualDir, err := os.Getwd()
 	if err != nil {
 		return err
 	}
 
-	argSlice, err := GetKubectlArgs(args, namespace)
+	builtCmd, err := GetArgs(ctx, command, cmd, namespace)
 	if err != nil {
 		return err
 	}
 
-	cmd := exec.Command("kubectl")
-	cmd.Args = argSlice
-	cmd.Dir = cwd
-	cmd.Stdout = stdout
-	cmd.Stderr = stderr
-	cmd.Env = []string{
+	builtCmd.Dir = cwd
+	builtCmd.Stdout = stdout
+	builtCmd.Stderr = stderr
+	builtCmd.Env = []string{
 		fmt.Sprintf("KUBECONFIG=%s/kubeconfig", actualDir),
 		fmt.Sprintf("PATH=%s/bin/:%s", actualDir, os.Getenv("PATH")),
 	}
 
-	return cmd.Run()
+	err = builtCmd.Run()
+	if err != nil {
+		if _, ok := err.(*exec.ExitError); ok && cmd.IgnoreFailure {
+			return nil
+		}
+	}
+
+	return err
 }
 
-// RunKubectlCommands runs a set of kubectl commands, returning any errors.
-func RunKubectlCommands(logger Logger, namespace string, commands []string, workdir string) []error {
+// RunCommands runs a set of commands, returning any errors.
+// If `command` is set, then `command` will be the command that is invoked (if a command specifies it already, it will not be prepended again).
+func RunCommands(logger Logger, namespace string, command string, commands []kudo.Command, workdir string) []error {
 	errs := []error{}
 
 	if commands == nil {
@@ -877,9 +887,9 @@ func RunKubectlCommands(logger Logger, namespace string, commands []string, work
 		stdout := &bytes.Buffer{}
 		stderr := &bytes.Buffer{}
 
-		logger.Log("Running kubectl:", cmd)
+		logger.Log("Running command:", cmd)
 
-		err := Kubectl(context.TODO(), namespace, cmd, workdir, stdout, stderr)
+		err := RunCommand(context.TODO(), namespace, command, cmd, workdir, stdout, stderr)
 		if err != nil {
 			errs = append(errs, err)
 		}
@@ -895,6 +905,20 @@ func RunKubectlCommands(logger Logger, namespace string, commands []string, work
 	return errs
 }
 
+// RunKubectlCommands runs a set of kubectl commands, returning any errors.
+func RunKubectlCommands(logger Logger, namespace string, commands []string, workdir string) []error {
+	apiCommands := []kudo.Command{}
+
+	for _, cmd := range commands {
+		apiCommands = append(apiCommands, kudo.Command{
+			Command:    cmd,
+			Namespaced: true,
+		})
+	}
+
+	return RunCommands(logger, namespace, "kubectl", apiCommands, workdir)
+}
+
 // Kubeconfig converts a rest.Config into a YAML kubeconfig and writes it to w
 func Kubeconfig(cfg *rest.Config, w io.Writer) error {
 	var authProvider *api.AuthProviderConfig

pkg/test/utils/kubernetes_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 	"time"
 
+	kudo "github.com/kudobuilder/kudo/pkg/apis/kudo/v1alpha1"
+
 	"github.com/stretchr/testify/assert"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -396,9 +398,12 @@ func TestGetKubectlArgs(t *testing.T) {
 		},
 	} {
 		t.Run(test.testName, func(t *testing.T) {
-			args, err := GetKubectlArgs(test.args, test.namespace)
+			cmd, err := GetArgs(context.TODO(), "kubectl", kudo.Command{
+				Command:    test.args,
+				Namespaced: true,
+			}, test.namespace)
 			assert.Nil(t, err)
-			assert.Equal(t, test.expected, args)
+			assert.Equal(t, test.expected, cmd.Args)
 		})
 	}
 }