E2E: move example test to use golangs stdlib test runner (#12383)

Our E2E "framework" has a bunch of features around test discovery and
standing up infra that were never completed or fully used, and we
ended up building out a large test suite that ignored all that in lieu
of Terraform-provided infrastructure for the last couple years.

This changeset is a proposal (and demonstration) for gradually
migrating our E2E tests off the framework code so that developers can
write fairly ordinary golang stdlib testing tests.

e2e/README.md

@@ -41,7 +41,7 @@ environment for `NOMAD_ADDR` and run the tests as shown below:
 $(terraform output environment)
 
 cd ..
-go test -v .
+go test -v ./...
 ```
 
 If you want to run a specific suite, you can specify the `-suite` flag as
@@ -65,6 +65,16 @@ go test -v . -run 'TestE2E/Consul/\*consul\.ScriptChecksE2ETest/TestGroup'
                                   Go Package      Struct
 ```
 
+We're also in the process of migrating to "stdlib-style" tests that
+use the standard go `testing` package without a notion of "suite". You
+can run these with `-run` regexes the same way you would any other go
+test:
+
+```sh
+go test -v . -run TestExample/TestExample_Simple
+```
+
+
 ## I Want To...
 
 ### ...SSH Into One Of The Test Machines

e2e/e2e_test.go

@@ -18,7 +18,6 @@ import (
 	_ "github.com/hashicorp/nomad/e2e/disconnectedclients"
 	_ "github.com/hashicorp/nomad/e2e/eval_priority"
 	_ "github.com/hashicorp/nomad/e2e/events"
-	_ "github.com/hashicorp/nomad/e2e/example"
 	_ "github.com/hashicorp/nomad/e2e/isolation"
 	_ "github.com/hashicorp/nomad/e2e/lifecycle"
 	_ "github.com/hashicorp/nomad/e2e/metrics"

e2e/e2eutil/client.go

@@ -0,0 +1,35 @@
+package e2eutil
+
+import (
+	"testing"
+
+	capi "github.com/hashicorp/consul/api"
+	napi "github.com/hashicorp/nomad/api"
+	vapi "github.com/hashicorp/vault/api"
+
+	"github.com/stretchr/testify/require"
+)
+
+// NomadClient creates a default Nomad client based on the env vars
+// from the test environment. Fails the test if it can't be created
+func NomadClient(t *testing.T) *napi.Client {
+	client, err := napi.NewClient(napi.DefaultConfig())
+	require.NoError(t, err, "could not create Nomad client")
+	return client
+}
+
+// ConsulClient creates a default Consul client based on the env vars
+// from the test environment. Fails the test if it can't be created
+func ConsulClient(t *testing.T) *capi.Client {
+	client, err := capi.NewClient(capi.DefaultConfig())
+	require.NoError(t, err, "could not create Consul client")
+	return client
+}
+
+// VaultClient creates a default Vault client based on the env vars
+// from the test environment. Fails the test if it can't be created
+func VaultClient(t *testing.T) *vapi.Client {
+	client, err := vapi.NewClient(vapi.DefaultConfig())
+	require.NoError(t, err, "could not create Vault client")
+	return client
+}

e2e/e2eutil/job.go

@@ -8,7 +8,10 @@ import (
 	"os/exec"
 	"regexp"
 	"strings"
+	"testing"
 	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
 // Register registers a jobspec from a file but with a unique ID.
@@ -231,3 +234,17 @@ func StopJob(jobID string, args ...string) error {
 	}
 	return err
 }
+
+// CleanupJobsAndGC stops and purges the list of jobIDs and runs a
+// system gc. Returns a func so that the return value can be used
+// in t.Cleanup
+func CleanupJobsAndGC(t *testing.T, jobIDs *[]string) func() {
+	return func() {
+		for _, jobID := range *jobIDs {
+			err := StopJob(jobID, "-purge", "-detach")
+			assert.NoError(t, err)
+		}
+		_, err := Command("nomad", "system", "gc")
+		assert.NoError(t, err)
+	}
+}

e2e/example/doc.go

@@ -0,0 +1,4 @@
+package example
+
+// This package contains only tests, so this is a placeholder file to
+// make sure builds don't fail with "no non-test Go files in" errors

e2e/example/example_test.go

@@ -0,0 +1,54 @@
+package example
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/helper/uuid"
+)
+
+func TestExample(t *testing.T) {
+	nomad := e2eutil.NomadClient(t)
+
+	e2eutil.WaitForLeader(t, nomad)
+	e2eutil.WaitForNodesReady(t, nomad, 2)
+
+	t.Run("TestExample_Simple", testExample_Simple)
+	t.Run("TestExample_WithCleanup", testExample_WithCleanup)
+}
+
+func testExample_Simple(t *testing.T) {
+	t.Logf("Logging %s", t.Name())
+	out, err := e2eutil.Command("nomad", "node", "status")
+	require.NoError(t, err, "failed to run `nomad node status`")
+
+	rows, err := e2eutil.ParseColumns(out)
+	require.NoError(t, err, "failed to parse `nomad node status`")
+	for _, row := range rows {
+		require.Equal(t, "ready", row["Status"])
+	}
+}
+
+func testExample_WithCleanup(t *testing.T) {
+
+	t.Logf("Logging %s", t.Name())
+	nomad := e2eutil.NomadClient(t)
+
+	_, err := e2eutil.Command("nomad", "job", "init", "-short", "./input/example.nomad")
+	require.NoError(t, err, "failed to run `nomad job init -short`")
+	t.Cleanup(func() { os.Remove("input/example.nomad") })
+
+	jobIDs := []string{}
+	t.Cleanup(e2eutil.CleanupJobsAndGC(t, &jobIDs))
+
+	jobID := "example-" + uuid.Short()
+	jobIDs = append(jobIDs, jobID)
+	e2eutil.RegisterAndWaitForAllocs(t, nomad, "./input/example.nomad", jobID, "")
+
+	jobs, _, err := nomad.Jobs().List(nil)
+	require.NoError(t, err)
+	require.NotEmpty(t, jobs)
+}