kola: support tags for --allow-rerun-success

New format is:
```
--allow-rerun-success tags=tag1[,tag2]
```

To allow all tests simply:
```
--allow-rerun-success tags=all
```

Issue: coreos/fedora-coreos-pipeline#842

mantle/cmd/kola/kola.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strings"
 	"text/tabwriter"
 	"time"
 
@@ -126,7 +127,7 @@ This can be useful for e.g. serving locally built OSTree repos to qemu.
 	runExternals      []string
 	runMultiply       int
 	runRerunFlag      bool
-	allowRerunSuccess bool
+	allowRerunSuccess string
 
 	nonexclusiveWrapperMatch = regexp.MustCompile(`^non-exclusive-test-bucket-[0-9]$`)
 )
@@ -136,7 +137,7 @@ func init() {
 	cmdRun.Flags().StringArrayVarP(&runExternals, "exttest", "E", nil, "Externally defined tests (will be found in DIR/tests/kola)")
 	cmdRun.Flags().IntVar(&runMultiply, "multiply", 0, "Run the provided tests N times (useful to find race conditions)")
 	cmdRun.Flags().BoolVar(&runRerunFlag, "rerun", false, "re-run failed tests once")
-	cmdRun.Flags().BoolVar(&allowRerunSuccess, "allow-rerun-success", false, "Allow kola test run to be successful when tests pass during re-run")
+	cmdRun.Flags().StringVar(&allowRerunSuccess, "allow-rerun-success", "", "Allow kola test run to be successful when tests with given 'tags=...[,...]' pass during re-run")
 
 	root.AddCommand(cmdList)
 	cmdList.Flags().StringArrayVarP(&runExternals, "exttest", "E", nil, "Externally defined tests in directory")
@@ -237,6 +238,27 @@ func runRerun(cmd *cobra.Command, args []string) error {
 	return kolaRunPatterns(patterns, false)
 }
 
+// parseRerunSuccess converts rerun specification into a tags
+func parseRerunSuccess() ([]string, error) {
+	// In the future we may extend format to something like: <SELECTOR>[:<OPTIONS>]
+	//   SELECTOR
+	//     tags=...,...,...
+	//     tests=...,...,...
+	//   OPTIONS
+	//     n=...
+	//     allow-single=..
+	tags := []string{}
+	if len(allowRerunSuccess) == 0 {
+		return tags, nil
+	}
+	if !strings.HasPrefix(allowRerunSuccess, "tags=") {
+		return nil, fmt.Errorf("invalid rerun spec %s", allowRerunSuccess)
+	}
+	split := strings.TrimPrefix(allowRerunSuccess, "tags=")
+	tags = append(tags, strings.Split(split, ",")...)
+	return tags, nil
+}
+
 func kolaRunPatterns(patterns []string, rerun bool) error {
 	var err error
 	outputDir, err = kola.SetupOutputDir(outputDir, kolaPlatform)
@@ -248,7 +270,12 @@ func kolaRunPatterns(patterns []string, rerun bool) error {
 		return err
 	}
 
-	runErr := kola.RunTests(patterns, runMultiply, rerun, allowRerunSuccess, kolaPlatform, outputDir, !kola.Options.NoTestExitError)
+	rerunSuccessTags, err := parseRerunSuccess()
+	if err != nil {
+		return err
+	}
+
+	runErr := kola.RunTests(patterns, runMultiply, rerun, rerunSuccessTags, kolaPlatform, outputDir, !kola.Options.NoTestExitError)
 
 	// needs to be after RunTests() because harness empties the directory
 	if err := writeProps(); err != nil {

mantle/kola/harness.go

@@ -658,7 +658,7 @@ func filterTests(tests map[string]*register.Test, patterns []string, pltfrm stri
 // register tests in their init() function.  outputDir is where various test
 // logs and data will be written for analysis after the test run. If it already
 // exists it will be erased!
-func runProvidedTests(testsBank map[string]*register.Test, patterns []string, multiply int, rerun bool, allowRerunSuccess bool, pltfrm, outputDir string, propagateTestErrors bool) error {
+func runProvidedTests(testsBank map[string]*register.Test, patterns []string, multiply int, rerun bool, rerunSuccessTags []string, pltfrm, outputDir string, propagateTestErrors bool) error {
 	var versionStr string
 
 	// Avoid incurring cost of starting machine in getClusterSemver when
@@ -678,6 +678,14 @@ func runProvidedTests(testsBank map[string]*register.Test, patterns []string, mu
 		plog.Fatal(err)
 	}
 
+	// In case of testfailure we allow to return True when
+	// - all failed tests are tagged with `tag` (all by default)
+	// - rerun was successful
+	var tagged = make(map[string]bool)
+	if len(rerunSuccessTags) > 0 {
+		tagged = getTaggedTests(testsBank, rerunSuccessTags)
+	}
+
 	flight, err := NewFlight(pltfrm)
 	if err != nil {
 		plog.Fatalf("Flight failed: %v", err)
@@ -804,16 +812,55 @@ func runProvidedTests(testsBank map[string]*register.Test, patterns []string, mu
 	if len(testsToRerun) > 0 && rerun {
 		newOutputDir := filepath.Join(outputDir, "rerun")
 		fmt.Printf("\n\n======== Re-running failed tests (flake detection) ========\n\n")
-		reRunErr := runProvidedTests(testsBank, testsToRerun, multiply, false, allowRerunSuccess, pltfrm, newOutputDir, propagateTestErrors)
-		if allowRerunSuccess {
+		reRunErr := runProvidedTests(testsBank, testsToRerun, multiply, false, rerunSuccessTags, pltfrm, newOutputDir, propagateTestErrors)
+
+		// Let's see which tests have failed
+		if len(rerunSuccessTags) > 0 && len(filterTaggedTests(testsToRerun, tagged)) == 0 {
 			return reRunErr
 		}
 	}
-
 	// If the intial run failed and the rerun passed, we still return an error
 	return firstRunErr
 }
 
+func getTaggedTests(tests map[string]*register.Test, tests_tags []string) map[string]bool {
+	var tags = make(map[string]bool)
+	for _, t := range tests_tags {
+		tags[t] = true
+	}
+
+	// by default all tags
+	var tagged = make(map[string]bool)
+	_, all := tags["all"]
+	_, asteriks := tags["*"]
+	if asteriks || all {
+		tagged["*"] = true
+		return tagged
+	}
+
+	for _, test := range tests {
+		for _, t := range test.Tags {
+			if _, exists := tags[t]; exists {
+				tagged[test.Name] = true
+			}
+		}
+	}
+	return tagged
+}
+
+func filterTaggedTests(tests []string, tagged map[string]bool) []string {
+	var out []string
+	if _, all := tagged["*"]; all {
+		return out
+	}
+	for _, test := range tests {
+		if !tagged[test] {
+			out = append(out, test)
+		}
+	}
+	return out
+}
+
 func GetRerunnableTestName(testName string) (string, bool) {
 	// The current nonexclusive test wrapper would rerun all non-exclusive tests.
 	// Instead, we only want to rerun the one(s) that failed, so we will not consider
@@ -859,12 +906,12 @@ func getRerunnable(tests []*harness.H) []string {
 	return testsToRerun
 }
 
-func RunTests(patterns []string, multiply int, rerun bool, allowRerunSuccess bool, pltfrm, outputDir string, propagateTestErrors bool) error {
-	return runProvidedTests(register.Tests, patterns, multiply, rerun, allowRerunSuccess, pltfrm, outputDir, propagateTestErrors)
+func RunTests(patterns []string, multiply int, rerun bool, rerunSuccessTags []string, pltfrm, outputDir string, propagateTestErrors bool) error {
+	return runProvidedTests(register.Tests, patterns, multiply, rerun, rerunSuccessTags, pltfrm, outputDir, propagateTestErrors)
 }
 
 func RunUpgradeTests(patterns []string, rerun bool, pltfrm, outputDir string, propagateTestErrors bool) error {
-	return runProvidedTests(register.UpgradeTests, patterns, 0, rerun, false, pltfrm, outputDir, propagateTestErrors)
+	return runProvidedTests(register.UpgradeTests, patterns, 0, rerun, nil, pltfrm, outputDir, propagateTestErrors)
 }
 
 // externalTestMeta is parsed from kola.json in external tests