Fix default test timeout and make default test timeout configurable a…

…t the test suite level. (#534)

* Fix default test timeout and make default test timeout configurable at the test suite level.

The KEP defined the default assertion timeout as 300 seconds, but we actually implemented it as 10 seconds. I think 300 seconds is much too high for the default, so I've adjusted it to 30 in both the implementation and KEP.

This also makes the timeout configurable at the test suite level so that it can be configured for an entire test suite instead of just a specific assert.

* appease linter

keps/0008-operator-testing.md

@@ -153,6 +153,8 @@ type TestSuite struct {
 	StartControlPlane bool
 	// Whether or not to start the KUDO controller for the tests.
 	StartKUDO         bool
+	// Override the default assertion timeout of 30 seconds (in seconds).
+	Timeout int
 }
 
 ```
@@ -295,7 +297,7 @@ When searching the assertion file for a test step, if a `TestAssert` object is f
 type TestAssert struct {
     // The type meta object, should always be a GVK of kudo.k8s.io/v1alpha1/TestAssert.
     TypeMeta
-    // Override the default timeout of 300 seconds (in seconds).
+    // Override the default timeout of 30 seconds (in seconds).
     Timeout int
 }
 ```

pkg/apis/kudo/v1alpha1/test_types.go

@@ -26,6 +26,8 @@ type TestSuite struct {
 	StartKUDO bool `json:"startKUDO"`
 	// If set, do not delete the resources after running the tests.
 	SkipDelete bool `json:"skipDelete"`
+	// Override the default timeout of 30 seconds (in seconds).
+	Timeout int `json:"timeout"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -54,7 +56,7 @@ type TestStep struct {
 type TestAssert struct {
 	// The type meta object, should always be a GVK of kudo.k8s.io/v1alpha1/TestAssert.
 	metav1.TypeMeta `json:",inline"`
-	// Override the default timeout of 300 seconds (in seconds).
+	// Override the default timeout of 30 seconds (in seconds).
 	Timeout int `json:"timeout"`
 }
 

pkg/test/case.go

@@ -28,6 +28,7 @@ type Case struct {
 	Name       string
 	Dir        string
 	SkipDelete bool
+	Timeout    int
 
 	Client          client.Client
 	DiscoveryClient discovery.DiscoveryInterface
@@ -152,6 +153,7 @@ func (t *Case) LoadTestSteps() error {
 
 	for index, files := range testStepFiles {
 		testStep := &Step{
+			Timeout: t.Timeout,
 			Index:   int(index),
 			Asserts: []runtime.Object{},
 			Apply:   []runtime.Object{},

pkg/test/harness.go

@@ -52,6 +52,7 @@ func (h *Harness) LoadTests(dir string) ([]*Case, error) {
 		}
 
 		tests = append(tests, &Case{
+			Timeout:    h.GetTimeout(),
 			Steps:      []*Step{},
 			Name:       file.Name(),
 			Dir:        filepath.Join(dir, file.Name()),
@@ -62,6 +63,15 @@ func (h *Harness) LoadTests(dir string) ([]*Case, error) {
 	return tests, nil
 }
 
+// GetTimeout returns the configured timeout for the test suite.
+func (h *Harness) GetTimeout() int {
+	timeout := 30
+	if h.TestSuite.Timeout != 0 {
+		timeout = 30
+	}
+	return timeout
+}
+
 // RunTestEnv starts a Kubernetes API server and etcd server for use in the
 // tests and returns the Kubernetes configuration.
 func (h *Harness) RunTestEnv() (*rest.Config, error) {

pkg/test/step.go

@@ -33,6 +33,7 @@ type Step struct {
 	Apply   []runtime.Object
 	Errors  []runtime.Object
 
+	Timeout         int
 	DiscoveryClient discovery.DiscoveryInterface
 	Client          client.Client
 	Logger          testutils.Logger
@@ -159,7 +160,7 @@ func (s *Step) Create(namespace string) []error {
 
 // GetTimeout gets the timeout defined for the test step.
 func (s *Step) GetTimeout() int {
-	timeout := 10
+	timeout := s.Timeout
 	if s.Assert != nil && s.Assert.Timeout != 0 {
 		timeout = s.Assert.Timeout
 	}