Merge pull request #6 from gdt-dev/nest-exec-asserts

nest `exec` plugin assertions under `assert` field

README.md

@@ -489,24 +489,26 @@ the base `Spec` fields listed above):
 * `shell`: (optional) a string with the specific shell to use in executing the
   command. If empty (the default), no shell is used to execute the command and
   instead the operating system's `exec` family of calls is used.
-* `exit_code`: (optional) an integer with the expected exit code from the
+* `assert`: (optional) an object describing the conditions that will be
+  asserted about the test action.
+* `assert.exit_code`: (optional) an integer with the expected exit code from the
   executed command. The default successful exit code is 0 and therefore you do
   not need to specify this if you expect a successful exit code.
-* `out`: (optional) a [`PipeExpect`][pipeexpect] object containing
+* `assert.out`: (optional) a [`PipeExpect`][pipeexpect] object containing
   assertions about content in `stdout`.
-* `out.is`: (optional) a string with the exact contents of `stdout` you expect
+* `assert.out.is`: (optional) a string with the exact contents of `stdout` you expect
   to get.
-* `out.contains`: (optional) a list of one or more strings that *all* must be
+* `assert.out.contains`: (optional) a list of one or more strings that *all* must be
   present in `stdout`.
-* `out.contains_one_of`: (optional) a list of one or more strings of which *at
+* `assert.out.contains_one_of`: (optional) a list of one or more strings of which *at
   least one* must be present in `stdout`.
-* `err`: (optional) a [`PipeAssertions`][pipeexpect] object containing
+* `assert.err`: (optional) a [`PipeAssertions`][pipeexpect] object containing
   assertions about content in `stderr`.
-* `err.is`: (optional) a string with the exact contents of `stderr` you expect
+* `assert.err.is`: (optional) a string with the exact contents of `stderr` you expect
   to get.
-* `err.contains`: (optional) a list of one or more strings that *all* must be
+* `assert.err.contains`: (optional) a list of one or more strings that *all* must be
   present in `stderr`.
-* `err.contains_one_of`: (optional) a list of one or more strings of which *at
+* `assert.err.contains_one_of`: (optional) a list of one or more strings of which *at
   least one* must be present in `stderr`.
 
 [execspec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/spec.go#L11-L34

plugin/exec/assertions.go

@@ -12,6 +12,18 @@ import (
 	gdttypes "github.com/gdt-dev/gdt/types"
 )
 
+// Expect contains the assertions about an Exec Spec's actions
+type Expect struct {
+	// ExitCode is the expected exit code for the executed command. The default
+	// (0) is the universal successful exit code, so you only need to set this
+	// if you expect a non-successful result from executing the command.
+	ExitCode int `yaml:"exit_code,omitempty"`
+	// Out has things that are expected in the stdout response
+	Out *PipeExpect `yaml:"out,omitempty"`
+	// Err has things that are expected in the stderr response
+	Err *PipeExpect `yaml:"err,omitempty"`
+}
+
 // PipeExpect contains assertions about the contents of a pipe
 type PipeExpect struct {
 	// Is contains the exact match (minus whitespace) of the contents of the
@@ -165,30 +177,30 @@ func (a *assertions) OK() bool {
 // newAssertions returns an assertions object populated with the supplied exec
 // spec assertions
 func newAssertions(
-	expExitCode int,
+	e *Expect,
 	exitCode int,
-	expOutPipe *PipeExpect,
 	outPipe *bytes.Buffer,
-	expErrPipe *PipeExpect,
 	errPipe *bytes.Buffer,
 ) gdttypes.Assertions {
 	a := &assertions{
 		failures:    []error{},
 		expExitCode: exitCode,
 		exitCode:    exitCode,
 	}
-	if expOutPipe != nil {
-		a.expOutPipe = &pipeAssertions{
-			PipeExpect: *expOutPipe,
-			name:       "stdout",
-			pipe:       outPipe,
+	if e != nil {
+		if e.Out != nil {
+			a.expOutPipe = &pipeAssertions{
+				PipeExpect: *e.Out,
+				name:       "stdout",
+				pipe:       outPipe,
+			}
 		}
-	}
-	if expErrPipe != nil {
-		a.expErrPipe = &pipeAssertions{
-			PipeExpect: *expErrPipe,
-			name:       "stderr",
-			pipe:       errPipe,
+		if e.Err != nil {
+			a.expErrPipe = &pipeAssertions{
+				PipeExpect: *e.Err,
+				name:       "stderr",
+				pipe:       errPipe,
+			}
 		}
 	}
 	return a

plugin/exec/eval.go

@@ -71,8 +71,6 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
 		eerr, _ := err.(*exec.ExitError)
 		ec = eerr.ExitCode()
 	}
-	assertions := newAssertions(
-		s.ExitCode, ec, s.Out, outbuf, s.Err, errbuf,
-	)
+	assertions := newAssertions(s.Assert, ec, outbuf, errbuf)
 	return result.New(result.WithFailures(assertions.Failures()...))
 }

plugin/exec/parse.go

@@ -63,6 +63,48 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			if s.Exec == "" {
 				return ExecEmpty(valNode)
 			}
+		case "assert":
+			if valNode.Kind != yaml.MappingNode {
+				return errors.ExpectedMapAt(valNode)
+			}
+			var e *Expect
+			if err := valNode.Decode(&e); err != nil {
+				return err
+			}
+			s.Assert = e
+		default:
+			if lo.Contains(gdttypes.BaseSpecFields, key) {
+				continue
+			}
+			return errors.UnknownFieldAt(key, keyNode)
+		}
+	}
+	if s.Exec == "" {
+		return ExecEmpty(node)
+	}
+	if s.Shell != "" {
+		_, err := shlex.Split(s.Exec)
+		if err != nil {
+			return ExecInvalidShellParse(err)
+		}
+	}
+	return nil
+}
+
+func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.MappingNode {
+		return errors.ExpectedMapAt(node)
+	}
+	// maps/structs are stored in a top-level Node.Content field which is a
+	// concatenated slice of Node pointers in pairs of key/values.
+	for i := 0; i < len(node.Content); i += 2 {
+		keyNode := node.Content[i]
+		if keyNode.Kind != yaml.ScalarNode {
+			return errors.ExpectedScalarAt(keyNode)
+		}
+		key := keyNode.Value
+		valNode := node.Content[i+1]
+		switch key {
 		case "exit_code":
 			if valNode.Kind != yaml.ScalarNode {
 				return errors.ExpectedScalarAt(valNode)
@@ -71,7 +113,7 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			if err != nil {
 				return err
 			}
-			s.ExitCode = ec
+			e.ExitCode = ec
 		case "out":
 			if valNode.Kind != yaml.MappingNode {
 				return errors.ExpectedMapAt(valNode)
@@ -80,7 +122,7 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			if err := valNode.Decode(&pe); err != nil {
 				return err
 			}
-			s.Out = pe
+			e.Out = pe
 		case "err":
 			if valNode.Kind != yaml.MappingNode {
 				return errors.ExpectedMapAt(valNode)
@@ -89,22 +131,10 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			if err := valNode.Decode(&pe); err != nil {
 				return err
 			}
-			s.Err = pe
+			e.Err = pe
 		default:
-			if lo.Contains(gdttypes.BaseSpecFields, key) {
-				continue
-			}
 			return errors.UnknownFieldAt(key, keyNode)
 		}
 	}
-	if s.Exec == "" {
-		return ExecEmpty(node)
-	}
-	if s.Shell != "" {
-		_, err := shlex.Split(s.Exec)
-		if err != nil {
-			return ExecInvalidShellParse(err)
-		}
-	}
 	return nil
 }

plugin/exec/spec.go

@@ -23,14 +23,8 @@ type Spec struct {
 	// (the default), no shell is used to execute the command and instead the
 	// operating system's `exec` family of calls is used.
 	Shell string `yaml:"shell,omitempty"`
-	// ExitCode is the expected exit code for the executed command. The default
-	// (0) is the universal successful exit code, so you only need to set this
-	// if you expect a non-successful result from executing the command.
-	ExitCode int `yaml:"exit_code,omitempty"`
-	// Out has things that are expected in the stdout response
-	Out *PipeExpect `yaml:"out,omitempty"`
-	// Err has things that are expected in the stderr response
-	Err *PipeExpect `yaml:"err,omitempty"`
+	// Assert is an object containing the conditions that the Spec will assert.
+	Assert *Expect `yaml:"assert,omitempty"`
 }
 
 func (s *Spec) SetBase(b gdttypes.Spec) {

plugin/exec/testdata/echo-cat.yaml

@@ -2,11 +2,13 @@ name: echo-cat
 description: a scenario that echoes the word "cat" and expects that output.
 tests:
   - exec: echo "cat"
-    out:
-      is: cat
+    assert:
+      out:
+        is: cat
   # To test the stderr assertions, we redirect stdout to stderr in a shell
   # command...
   - exec: "echo cat 1>&2"
     shell: sh
-    err:
-      is: cat
+    assert:
+      err:
+        is: cat

plugin/exec/testdata/ls-contains-one-of.yaml

@@ -2,17 +2,19 @@ name: ls-contains-one-of
 description: a scenario that runs the `ls` command and checks the output contains one of a set of strings
 tests:
   - exec: ls -l
-    out:
-      contains_one_of:
-       - thisdoesnotexist
-       - neitherdoesthisexist
-       - parse.go
+    assert:
+      out:
+        contains_one_of:
+         - thisdoesnotexist
+         - neitherdoesthisexist
+         - parse.go
   # To test the stderr assertions, we redirect stdout to stderr in a shell
   # command...
   - exec: "ls -l 1>&2"
     shell: sh
-    err:
-      contains_one_of:
-       - thisdoesnotexist
-       - neitherdoesthisexist
-       - parse.go
+    assert:
+      err:
+        contains_one_of:
+         - thisdoesnotexist
+         - neitherdoesthisexist
+         - parse.go

plugin/exec/testdata/ls-contains.yaml

@@ -2,13 +2,15 @@ name: ls-contains
 description: a scenario that runs the `ls` command and checks the output contains a string
 tests:
   - exec: ls -l
-    out:
-      contains:
-       - parse.go
+    assert:
+      out:
+        contains:
+         - parse.go
   # To test the stderr assertions, we redirect stdout to stderr in a shell
   # command...
   - exec: "ls -l 1>&2"
     shell: sh
-    err:
-      contains:
-        - parse.go
+    assert:
+      err:
+        contains:
+         - parse.go

plugin/exec/testdata/ls-with-exit-code.yaml

@@ -2,4 +2,5 @@ name: ls-with-exit-code
 description: a scenario that runs the `ls` command expecting a non-0 exit code
 tests:
   - exec: ls /this/dir/does/not/exist
-    exit_code: 2
+    assert:
+      exit_code: 2

plugin/exec/testdata/mac-ls-with-exit-code.yaml

@@ -2,4 +2,5 @@ name: mac-ls-with-exit-code
 description: a scenario that runs the `ls` command expecting a non-0 exit code on Mac
 tests:
   - exec: ls /this/dir/does/not/exist
-    exit_code: 1
+    assert:
+      exit_code: 1

plugin/exec/testdata/windows-sleep-timeout.yaml

@@ -10,4 +10,5 @@ tests:
       expected: true
     # the context's deadline cancels the pipe and results in a 1 result
     # code on Windows...
-    exit_code: 1
+    assert:
+      exit_code: 1