kola: support tags for --allow-rerun-success

New format is:
```
--allow-rerun-success tags=tag1[,tag2]
```

To allow all tests simply:
```
--allow-rerun-success tags=all
```

Co-authored-by: Dusty Mabe <dusty@dustymabe.com>

Issue: coreos/fedora-coreos-pipeline#842

mantle/cmd/kola/kola.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strings"
 	"text/tabwriter"
 	"time"
 
@@ -126,7 +127,7 @@ This can be useful for e.g. serving locally built OSTree repos to qemu.
 	runExternals      []string
 	runMultiply       int
 	runRerunFlag      bool
-	allowRerunSuccess bool
+	allowRerunSuccess string
 
 	nonexclusiveWrapperMatch = regexp.MustCompile(`^non-exclusive-test-bucket-[0-9]$`)
 )
@@ -136,7 +137,7 @@ func init() {
 	cmdRun.Flags().StringArrayVarP(&runExternals, "exttest", "E", nil, "Externally defined tests (will be found in DIR/tests/kola)")
 	cmdRun.Flags().IntVar(&runMultiply, "multiply", 0, "Run the provided tests N times (useful to find race conditions)")
 	cmdRun.Flags().BoolVar(&runRerunFlag, "rerun", false, "re-run failed tests once")
-	cmdRun.Flags().BoolVar(&allowRerunSuccess, "allow-rerun-success", false, "Allow kola test run to be successful when tests pass during re-run")
+	cmdRun.Flags().StringVar(&allowRerunSuccess, "allow-rerun-success", "", "Allow kola test run to be successful when tests with given 'tags=...[,...]' pass during re-run")
 
 	root.AddCommand(cmdList)
 	cmdList.Flags().StringArrayVarP(&runExternals, "exttest", "E", nil, "Externally defined tests in directory")
@@ -237,6 +238,27 @@ func runRerun(cmd *cobra.Command, args []string) error {
 	return kolaRunPatterns(patterns, false)
 }
 
+// parseRerunSuccess converts rerun specification into a tags
+func parseRerunSuccess() ([]string, error) {
+	// In the future we may extend format to something like: <SELECTOR>[:<OPTIONS>]
+	//   SELECTOR
+	//     tags=...,...,...
+	//     tests=...,...,...
+	//   OPTIONS
+	//     n=...
+	//     allow-single=..
+	tags := []string{}
+	if len(allowRerunSuccess) == 0 {
+		return tags, nil
+	}
+	if !strings.HasPrefix(allowRerunSuccess, "tags=") {
+		return nil, fmt.Errorf("invalid rerun spec %s", allowRerunSuccess)
+	}
+	split := strings.TrimPrefix(allowRerunSuccess, "tags=")
+	tags = append(tags, strings.Split(split, ",")...)
+	return tags, nil
+}
+
 func kolaRunPatterns(patterns []string, rerun bool) error {
 	var err error
 	outputDir, err = kola.SetupOutputDir(outputDir, kolaPlatform)
@@ -248,7 +270,12 @@ func kolaRunPatterns(patterns []string, rerun bool) error {
 		return err
 	}
 
-	runErr := kola.RunTests(patterns, runMultiply, rerun, allowRerunSuccess, kolaPlatform, outputDir, !kola.Options.NoTestExitError)
+	rerunSuccessTags, err := parseRerunSuccess()
+	if err != nil {
+		return err
+	}
+
+	runErr := kola.RunTests(patterns, runMultiply, rerun, rerunSuccessTags, kolaPlatform, outputDir, !kola.Options.NoTestExitError)
 
 	// needs to be after RunTests() because harness empties the directory
 	if err := writeProps(); err != nil {

mantle/kola/harness.go

@@ -658,7 +658,7 @@ func filterTests(tests map[string]*register.Test, patterns []string, pltfrm stri
 // register tests in their init() function.  outputDir is where various test
 // logs and data will be written for analysis after the test run. If it already
 // exists it will be erased!
-func runProvidedTests(testsBank map[string]*register.Test, patterns []string, multiply int, rerun bool, allowRerunSuccess bool, pltfrm, outputDir string, propagateTestErrors bool) error {
+func runProvidedTests(testsBank map[string]*register.Test, patterns []string, multiply int, rerun bool, rerunSuccessTags []string, pltfrm, outputDir string, propagateTestErrors bool) error {
 	var versionStr string
 
 	// Avoid incurring cost of starting machine in getClusterSemver when
@@ -804,16 +804,48 @@ func runProvidedTests(testsBank map[string]*register.Test, patterns []string, mu
 	if len(testsToRerun) > 0 && rerun {
 		newOutputDir := filepath.Join(outputDir, "rerun")
 		fmt.Printf("\n\n======== Re-running failed tests (flake detection) ========\n\n")
-		reRunErr := runProvidedTests(testsBank, testsToRerun, multiply, false, allowRerunSuccess, pltfrm, newOutputDir, propagateTestErrors)
-		if allowRerunSuccess {
+		reRunErr := runProvidedTests(testsBank, testsToRerun, multiply, false, rerunSuccessTags, pltfrm, newOutputDir, propagateTestErrors)
+
+		// Return the results from the rerun if rerun success allowed
+		if allTestsAllowRerunSuccess(testsBank, testsToRerun, rerunSuccessTags) {
 			return reRunErr
 		}
 	}
-
 	// If the intial run failed and the rerun passed, we still return an error
 	return firstRunErr
 }
 
+func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun, rerunSuccessTags []string) bool {
+	// No tags, we can return early
+	if len(rerunSuccessTags) == 0 {
+		return false
+	}
+	// Build up a map of rerunSuccessTags so that we can easily check
+	// if a given tag is in the map.
+	rerunSuccessTagMap := make(map[string]bool)
+	for _, tag := range rerunSuccessTags {
+		if tag == "all" || tag == "*" {
+			// If `all` is in rerunSuccessTags then we can return early
+			return true
+		}
+		rerunSuccessTagMap[tag] = true
+	}
+	// Iterate over the tests that were re-ran. If any of them don't
+	// allow rerun success then just exit early.
+	for _, test := range testsToRerun {
+		testAllowsRerunSuccess := false
+		for _, tag := range testsBank[test].Tags {
+			if rerunSuccessTagMap[tag] {
+				testAllowsRerunSuccess = true
+			}
+		}
+		if !testAllowsRerunSuccess {
+			return false
+		}
+	}
+	return true
+}
+
 func GetRerunnableTestName(testName string) (string, bool) {
 	// The current nonexclusive test wrapper would rerun all non-exclusive tests.
 	// Instead, we only want to rerun the one(s) that failed, so we will not consider
@@ -859,12 +891,12 @@ func getRerunnable(tests []*harness.H) []string {
 	return testsToRerun
 }
 
-func RunTests(patterns []string, multiply int, rerun bool, allowRerunSuccess bool, pltfrm, outputDir string, propagateTestErrors bool) error {
-	return runProvidedTests(register.Tests, patterns, multiply, rerun, allowRerunSuccess, pltfrm, outputDir, propagateTestErrors)
+func RunTests(patterns []string, multiply int, rerun bool, rerunSuccessTags []string, pltfrm, outputDir string, propagateTestErrors bool) error {
+	return runProvidedTests(register.Tests, patterns, multiply, rerun, rerunSuccessTags, pltfrm, outputDir, propagateTestErrors)
 }
 
 func RunUpgradeTests(patterns []string, rerun bool, pltfrm, outputDir string, propagateTestErrors bool) error {
-	return runProvidedTests(register.UpgradeTests, patterns, 0, rerun, false, pltfrm, outputDir, propagateTestErrors)
+	return runProvidedTests(register.UpgradeTests, patterns, 0, rerun, nil, pltfrm, outputDir, propagateTestErrors)
 }
 
 // externalTestMeta is parsed from kola.json in external tests