Allow e2e results to refer to one another

By allowing reference to other test results (and adding/removing
a few tests from them) we can drastically reduce the amount
of space we dedicate to this feature.

Partially addresses: #1650

Signed-off-by: John Schnake <jschnake@vmware.com>

cmd/sonobuoy/app/e2e.go

@@ -49,6 +49,19 @@ type e2eFlags struct {
 	mode string
 }
 
+// testListStubError is the type of error returned when we try and read tests from a file/url
+// but it references another version. Rather than handle it in the reader method, we bubble this
+// error up so that we have more context on the initial user request.
+type testListStubError struct {
+	ReferenceVersion string
+	AddTests         []string
+	RemoveTests      []string
+}
+
+func (e *testListStubError) Error() string {
+	return fmt.Sprintf("test list stub: attempted to load tests but the list references version %v", e.ReferenceVersion)
+}
+
 func NewCmdE2E() *cobra.Command {
 	f := e2eFlags{}
 	cmd := &cobra.Command{
@@ -184,15 +197,43 @@ func filterTests(list []string, focus, skip *regexp.Regexp) []string {
 }
 
 func getTests(input, baseURL, version string) ([]string, error) {
+	var tests []string
+	var err error
+
 	switch input {
 	case e2eInputOnline:
-		return getTestsOnline(baseURL, version)
+		tests, err = getTestsOnline(baseURL, version)
 	case e2eInputOffline:
-		return getTestsOffline(version)
+		tests, err = getTestsOffline(version)
 	case e2eInputStdin:
-		return getTestsStdin()
+		tests, err = getTestsStdin()
+	default:
+		err = fmt.Errorf("unknown input option set: %q, expected one of [%v, %v, %v]", input, e2eInputOnline, e2eInputOffline, e2eInputStdin)
+	}
+	var stubErr *testListStubError
+	if errors.As(err, &stubErr) {
+		logrus.Tracef("Attempted to load tests for version %v but it referenced version %v. Loading that version with other options the same.", version, stubErr.ReferenceVersion)
+
+		// Recurse with all the same options. Return on errors (they'll be non-testListStubError).
+		tests, err = getTests(input, baseURL, stubErr.ReferenceVersion)
+		if err != nil {
+			return nil, err
+		}
+		tests = mergeResults(tests, stubErr.AddTests, stubErr.RemoveTests)
 	}
-	return nil, fmt.Errorf("unknown input option set: %q, expected one of [%v, %v, %v]", input, e2eInputOnline, e2eInputOffline, e2eInputStdin)
+	return tests, err
+}
+
+func mergeResults(tests, addTests, removeTests []string) []string {
+	result := append(tests, addTests...)
+	sort.Strings(result)
+	for _, removeVal := range removeTests {
+		i := sort.SearchStrings(result, removeVal)
+		if result[i] == removeVal {
+			result = append(result[:i], result[i+1:]...)
+		}
+	}
+	return result
 }
 
 func getTestsStdin() ([]string, error) {
@@ -216,8 +257,27 @@ func getTestsOffline(version string) ([]string, error) {
 }
 
 func testsFromReader(r io.Reader) ([]string, error) {
-	tests := []string{}
 	scanner := bufio.NewScanner(r)
+
+	// Scan once to check if reader starts with the prefix.
+	scanner.Scan()
+	if strings.HasPrefix(scanner.Text(), "#") {
+		version := strings.TrimPrefix(scanner.Text(), "#")
+		err := &testListStubError{ReferenceVersion: version}
+
+		for scanner.Scan() {
+			switch {
+			case strings.HasPrefix(scanner.Text(), "+"):
+				err.AddTests = append(err.RemoveTests, strings.TrimPrefix(scanner.Text(), "+"))
+			case strings.HasPrefix(scanner.Text(), "-"):
+				err.RemoveTests = append(err.RemoveTests, strings.TrimPrefix(scanner.Text(), "-"))
+			}
+		}
+		return nil, err
+	}
+
+	// Normal file handling, no prefixes to handle.
+	tests := []string{scanner.Text()}
 	for scanner.Scan() {
 		tests = append(tests, scanner.Text())
 	}

cmd/sonobuoy/app/e2e_test.go

@@ -2,12 +2,16 @@ package app
 
 import (
 	"bytes"
+	"compress/gzip"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"regexp"
 	"strings"
 	"testing"
 
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/sirupsen/logrus"
 )
 
 func TestFilterTests(t *testing.T) {
@@ -128,3 +132,67 @@ func TestTagCountsFromList(t *testing.T) {
 		})
 	}
 }
+
+func TestGetTestsRedirects(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		getVersion string
+		expect     []string
+	}{
+		{
+			desc:       "Can get static data",
+			getVersion: "v1",
+			expect:     []string{"v1 data line 1", "v1 data line 2"},
+		}, {
+			desc:       "Can get redirected data",
+			getVersion: "v0",
+			expect:     []string{"v1 data line 1", "v1 data line 2"},
+		}, {
+			desc:       "Can get modified redirected data",
+			getVersion: "v2",
+			expect:     []string{"v1 data line 2", "v2 data line 1"},
+		}, {
+			desc:       "Can handle multiple redirects with modifications and it remains sorted",
+			getVersion: "v3",
+			expect:     []string{"_v3 data line 1", "v1 data line 2", "v2 data line 1"},
+		},
+	}
+
+	logrus.SetLevel(logrus.TraceLevel)
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/v0.gz", func(w http.ResponseWriter, req *http.Request) {
+		gw := gzip.NewWriter(w)
+		gw.Write([]byte("#v1\n"))
+		gw.Close()
+	})
+	mux.HandleFunc("/v1.gz", func(w http.ResponseWriter, req *http.Request) {
+		gw := gzip.NewWriter(w)
+		gw.Write([]byte("v1 data line 1\nv1 data line 2"))
+		gw.Close()
+	})
+	mux.HandleFunc("/v2.gz", func(w http.ResponseWriter, req *http.Request) {
+		gw := gzip.NewWriter(w)
+		gw.Write([]byte("#v1\n+v2 data line 1\n-v1 data line 1"))
+		gw.Close()
+	})
+	mux.HandleFunc("/v3.gz", func(w http.ResponseWriter, req *http.Request) {
+		gw := gzip.NewWriter(w)
+		gw.Write([]byte("#v2\n+_v3 data line 1"))
+		gw.Close()
+	})
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			output, err := getTests(e2eInputOnline, ts.URL, tc.getVersion)
+			if err != nil {
+				t.Fatalf("Unexpected error %v", err)
+			}
+			if diff := pretty.Compare(tc.expect, output); diff != "" {
+				t.Fatalf("Expected %v but got diff:\n\n%s\n", tc.expect, diff)
+			}
+		})
+	}
+}