[BCF-2374] Flakey test runner

tools/flakeytests/cmd/runner/main.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"flag"
+	"io"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/smartcontractkit/chainlink/v2/tools/flakeytests"
+)
+
+const numReruns = 2
+
+func main() {
+	flag.Parse()
+	args := flag.Args()
+
+	log.Printf("Parsing output at: %v", strings.Join(args, ", "))
+	readers := []io.Reader{}
+	for _, f := range args {
+		r, err := os.Open(f)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		readers = append(readers, r)
+	}
+
+	r := flakeytests.NewRunner(readers, numReruns)
+	flakes, err := r.Run()
+	if err != nil {
+		log.Fatalf("Error re-running flakey tests: %s", err)
+	}
+
+	if len(flakes) > 0 {
+		log.Printf("ERROR: Suspected flakes found: %+v\n", flakes)
+	} else {
+		log.Print("SUCCESS: No suspected flakes detected")
+	}
+}

tools/flakeytests/runner.go

@@ -0,0 +1,122 @@
+package flakeytests
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+var (
+	failedTestRe = regexp.MustCompile("^--- FAIL: (Test\\w+)")
+	failedPkgRe  = regexp.MustCompile("^FAIL\\s+github\\.com\\/smartcontractkit\\/chainlink\\/v2\\/(\\S+)")
+)
+
+type Runner struct {
+	readers   []io.Reader
+	numReruns int
+	runTestFn runTestCmd
+	parse     parseFn
+}
+
+type runTestCmd func(pkg string, testNames []string, numReruns int, w io.Writer) error
+type parseFn func(readers ...io.Reader) (map[string]map[string]int, error)
+
+func NewRunner(readers []io.Reader, numReruns int) *Runner {
+	return &Runner{
+		readers:   readers,
+		numReruns: numReruns,
+		runTestFn: runGoTest,
+		parse:     parseOutput,
+	}
+}
+
+func runGoTest(pkg string, tests []string, numReruns int, w io.Writer) error {
+	testFilter := strings.Join(tests, "|")
+	cmd := exec.Command("go", "test", "-count", fmt.Sprintf("%d", numReruns), "-run", testFilter, "-tags", "test", fmt.Sprintf("./%s/...", pkg))
+	cmd.Stdout = io.MultiWriter(os.Stdout, w)
+	cmd.Stderr = io.MultiWriter(os.Stderr, w)
+	return cmd.Run()
+}
+
+func parseOutput(readers ...io.Reader) (map[string]map[string]int, error) {
+	testsWithoutPackage := []string{}
+	tests := map[string]map[string]int{}
+	for _, r := range readers {
+		s := bufio.NewScanner(r)
+		for s.Scan() {
+			t := s.Text()
+			switch {
+			case failedTestRe.MatchString(t):
+				m := failedTestRe.FindStringSubmatch(t)
+				testsWithoutPackage = append(testsWithoutPackage, m[1])
+			case failedPkgRe.MatchString(t):
+				p := failedPkgRe.FindStringSubmatch(t)
+				for _, t := range testsWithoutPackage {
+					if tests[p[1]] == nil {
+						tests[p[1]] = map[string]int{}
+					}
+					tests[p[1]][t] += 1
+				}
+				testsWithoutPackage = []string{}
+			}
+		}
+
+		if err := s.Err(); err != nil {
+			return nil, err
+		}
+	}
+
+	return tests, nil
+}
+
+func (r *Runner) runTests(failedTests map[string]map[string]int) io.Reader {
+	var out bytes.Buffer
+	for pkg, tests := range failedTests {
+		ts := []string{}
+		for test := range tests {
+			ts = append(ts, test)
+		}
+
+		log.Printf("Executing test command with parameters: pkg=%s, tests=%+v, numReruns=%d\n", pkg, ts, r.numReruns)
+		err := r.runTestFn(pkg, ts, r.numReruns, &out)
+		if err != nil {
+			log.Printf("Test command errored: %s\n", err)
+		}
+	}
+
+	return &out
+}
+
+func (r *Runner) Run() (map[string]string, error) {
+	failedTests, err := r.parse(r.readers...)
+	if err != nil {
+		return nil, err
+	}
+
+	output := r.runTests(failedTests)
+
+	failedReruns, err := r.parse(output)
+	if err != nil {
+		return nil, err
+	}
+
+	suspectedFlakes := map[string]string{}
+	// A test is flakey if it appeared in the list of original flakey tests
+	// and doesn't appear in the reruns, or if it hasn't failed each additional
+	// run, i.e. if it hasn't twice after being re-run.
+	for pkg, t := range failedTests {
+		for test := range t {
+			if failedReruns[pkg][test] != r.numReruns {
+				suspectedFlakes[pkg] = test
+			}
+		}
+	}
+
+	return suspectedFlakes, nil
+}

tools/flakeytests/runner_test.go

@@ -0,0 +1,204 @@
+package flakeytests
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParser(t *testing.T) {
+	output := `
+--- FAIL: TestLink (0.00s)
+    --- FAIL: TestLink/1.1_link#01 (0.00s)
+        currencies_test.go:325:
+                Error Trace:    /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
+                Error:          Not equal:
+                                expected: "1.2 link"
+                                actual  : "1.1 link"
+
+                                Diff:
+                                --- Expected
+                                +++ Actual
+                                @@ -1 +1 @@
+                                -1.2 link
+                                +1.1 link
+                Test:           TestLink/1.1_link#01
+FAIL
+FAIL    github.com/smartcontractkit/chainlink/v2/core/assets    0.338s
+FAIL
+`
+
+	r := strings.NewReader(output)
+	ts, err := parseOutput(r)
+	require.NoError(t, err)
+
+	assert.Len(t, ts, 1)
+	assert.Len(t, ts["core/assets"], 1)
+	assert.Equal(t, ts["core/assets"]["TestLink"], 1)
+}
+
+func TestParser_SuccessfulOutput(t *testing.T) {
+	output := `
+?       github.com/smartcontractkit/chainlink/v2/tools/flakeytests/cmd/runner   [no test files]
+ok      github.com/smartcontractkit/chainlink/v2/tools/flakeytests      0.320s
+`
+
+	r := strings.NewReader(output)
+	ts, err := parseOutput(r)
+	require.NoError(t, err)
+	assert.Len(t, ts, 0)
+}
+
+func TestRunner_WithFlake(t *testing.T) {
+	output := `
+--- FAIL: TestLink (0.00s)
+    --- FAIL: TestLink/1.1_link#01 (0.00s)
+        currencies_test.go:325:
+                Error Trace:    /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
+                Error:          Not equal:
+                                expected: "1.2 link"
+                                actual  : "1.1 link"
+
+                                Diff:
+                                --- Expected
+                                +++ Actual
+                                @@ -1 +1 @@
+                                -1.2 link
+                                +1.1 link
+                Test:           TestLink/1.1_link#01
+FAIL
+FAIL    github.com/smartcontractkit/chainlink/v2/core/assets    0.338s
+FAIL
+`
+	r := &Runner{
+		numReruns: 2,
+		readers:   []io.Reader{strings.NewReader(output)},
+		runTestFn: func(pkg string, testNames []string, numReruns int, w io.Writer) error {
+			_, err := w.Write([]byte(output))
+			return err
+		},
+		parse: parseOutput,
+	}
+
+	// This will report a flake since we've mocked the rerun
+	// to only report one failure (not two as expected).
+	sf, err := r.Run()
+	require.NoError(t, err)
+	assert.Len(t, sf, 1)
+	assert.Equal(t, sf["core/assets"], "TestLink")
+}
+
+func TestRunner_AllFailures(t *testing.T) {
+	output := `
+--- FAIL: TestLink (0.00s)
+    --- FAIL: TestLink/1.1_link#01 (0.00s)
+        currencies_test.go:325:
+                Error Trace:    /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
+                Error:          Not equal:
+                                expected: "1.2 link"
+                                actual  : "1.1 link"
+
+                                Diff:
+                                --- Expected
+                                +++ Actual
+                                @@ -1 +1 @@
+                                -1.2 link
+                                +1.1 link
+                Test:           TestLink/1.1_link#01
+FAIL
+FAIL    github.com/smartcontractkit/chainlink/v2/core/assets    0.338s
+FAIL
+`
+
+	rerunOutput := `
+--- FAIL: TestLink (0.00s)
+    --- FAIL: TestLink/1.1_link#01 (0.00s)
+        currencies_test.go:325:
+                Error Trace:    /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
+                Error:          Not equal:
+                                expected: "1.2 link"
+                                actual  : "1.1 link"
+
+                                Diff:
+                                --- Expected
+                                +++ Actual
+                                @@ -1 +1 @@
+                                -1.2 link
+                                +1.1 link
+                Test:           TestLink/1.1_link#01
+--- FAIL: TestLink (0.00s)
+    --- FAIL: TestLink/1.1_link#01 (0.00s)
+        currencies_test.go:325:
+                Error Trace:    /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
+                Error:          Not equal:
+                                expected: "1.2 link"
+                                actual  : "1.1 link"
+
+                                Diff:
+                                --- Expected
+                                +++ Actual
+                                @@ -1 +1 @@
+                                -1.2 link
+                                +1.1 link
+                Test:           TestLink/1.1_link#01
+FAIL
+FAIL    github.com/smartcontractkit/chainlink/v2/core/assets    0.315s
+FAIL
+`
+	r := &Runner{
+		numReruns: 2,
+		readers:   []io.Reader{strings.NewReader(output)},
+		runTestFn: func(pkg string, testNames []string, numReruns int, w io.Writer) error {
+			_, err := w.Write([]byte(rerunOutput))
+			return err
+		},
+		parse: parseOutput,
+	}
+
+	sf, err := r.Run()
+	require.NoError(t, err)
+	assert.Len(t, sf, 0)
+}
+
+func TestRunner_RerunSuccessful(t *testing.T) {
+	output := `
+--- FAIL: TestLink (0.00s)
+    --- FAIL: TestLink/1.1_link#01 (0.00s)
+        currencies_test.go:325:
+                Error Trace:    /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
+                Error:          Not equal:
+                                expected: "1.2 link"
+                                actual  : "1.1 link"
+
+                                Diff:
+                                --- Expected
+                                +++ Actual
+                                @@ -1 +1 @@
+                                -1.2 link
+                                +1.1 link
+                Test:           TestLink/1.1_link#01
+FAIL
+FAIL    github.com/smartcontractkit/chainlink/v2/core/assets    0.338s
+FAIL
+`
+
+	rerunOutput := `
+ok      github.com/smartcontractkit/chainlink/v2/core/assets      0.320s
+`
+	r := &Runner{
+		numReruns: 2,
+		readers:   []io.Reader{strings.NewReader(output)},
+		runTestFn: func(pkg string, testNames []string, numReruns int, w io.Writer) error {
+			_, err := w.Write([]byte(rerunOutput))
+			return err
+		},
+		parse: parseOutput,
+	}
+
+	sf, err := r.Run()
+	require.NoError(t, err)
+	assert.Equal(t, sf["core/assets"], "TestLink")
+}