From 3996b71b0ab819f6755e6b20f47393ec0aa8614c Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 13 Apr 2023 15:40:28 +0200 Subject: [PATCH] kola: support tags for --allow-rerun-success New format is: ``` --allow-rerun-success tags=tag1[,tag2] ``` To allow all tests simply: ``` --allow-rerun-success tags=all ``` Co-authored-by: Dusty Mabe Issue: https://github.com/coreos/fedora-coreos-pipeline/issues/842 (cherry picked from commit 864cece33f7ad2287f2dd2a1d7639fcf6a3efbbe) --- mantle/cmd/kola/kola.go | 33 ++++++++++++++++++++++++++--- mantle/kola/harness.go | 46 ++++++++++++++++++++++++++++++++++------- 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/mantle/cmd/kola/kola.go b/mantle/cmd/kola/kola.go index ace7d0ef17..35a91d46bf 100644 --- a/mantle/cmd/kola/kola.go +++ b/mantle/cmd/kola/kola.go @@ -23,6 +23,7 @@ import ( "path/filepath" "regexp" "sort" + "strings" "text/tabwriter" "time" @@ -126,7 +127,7 @@ This can be useful for e.g. serving locally built OSTree repos to qemu. runExternals []string runMultiply int runRerunFlag bool - allowRerunSuccess bool + allowRerunSuccess string nonexclusiveWrapperMatch = regexp.MustCompile(`^non-exclusive-test-bucket-[0-9]$`) ) @@ -136,7 +137,7 @@ func init() { cmdRun.Flags().StringArrayVarP(&runExternals, "exttest", "E", nil, "Externally defined tests (will be found in DIR/tests/kola)") cmdRun.Flags().IntVar(&runMultiply, "multiply", 0, "Run the provided tests N times (useful to find race conditions)") cmdRun.Flags().BoolVar(&runRerunFlag, "rerun", false, "re-run failed tests once") - cmdRun.Flags().BoolVar(&allowRerunSuccess, "allow-rerun-success", false, "Allow kola test run to be successful when tests pass during re-run") + cmdRun.Flags().StringVar(&allowRerunSuccess, "allow-rerun-success", "", "Allow kola test run to be successful when tests with given 'tags=...[,...]' pass during re-run") root.AddCommand(cmdList) cmdList.Flags().StringArrayVarP(&runExternals, "exttest", "E", nil, "Externally defined tests in directory") @@ -237,6 +238,27 @@ func runRerun(cmd *cobra.Command, args []string) error { return kolaRunPatterns(patterns, false) } +// parseRerunSuccess converts rerun specification into a tags +func parseRerunSuccess() ([]string, error) { + // In the future we may extend format to something like: [:] + // SELECTOR + // tags=...,...,... + // tests=...,...,... + // OPTIONS + // n=... + // allow-single=.. + tags := []string{} + if len(allowRerunSuccess) == 0 { + return tags, nil + } + if !strings.HasPrefix(allowRerunSuccess, "tags=") { + return nil, fmt.Errorf("invalid rerun spec %s", allowRerunSuccess) + } + split := strings.TrimPrefix(allowRerunSuccess, "tags=") + tags = append(tags, strings.Split(split, ",")...) + return tags, nil +} + func kolaRunPatterns(patterns []string, rerun bool) error { var err error outputDir, err = kola.SetupOutputDir(outputDir, kolaPlatform) @@ -248,7 +270,12 @@ func kolaRunPatterns(patterns []string, rerun bool) error { return err } - runErr := kola.RunTests(patterns, runMultiply, rerun, allowRerunSuccess, kolaPlatform, outputDir, !kola.Options.NoTestExitError) + rerunSuccessTags, err := parseRerunSuccess() + if err != nil { + return err + } + + runErr := kola.RunTests(patterns, runMultiply, rerun, rerunSuccessTags, kolaPlatform, outputDir, !kola.Options.NoTestExitError) // needs to be after RunTests() because harness empties the directory if err := writeProps(); err != nil { diff --git a/mantle/kola/harness.go b/mantle/kola/harness.go index 4490b27e0b..1134b02590 100644 --- a/mantle/kola/harness.go +++ b/mantle/kola/harness.go @@ -658,7 +658,7 @@ func filterTests(tests map[string]*register.Test, patterns []string, pltfrm stri // register tests in their init() function. outputDir is where various test // logs and data will be written for analysis after the test run. If it already // exists it will be erased! -func runProvidedTests(testsBank map[string]*register.Test, patterns []string, multiply int, rerun bool, allowRerunSuccess bool, pltfrm, outputDir string, propagateTestErrors bool) error { +func runProvidedTests(testsBank map[string]*register.Test, patterns []string, multiply int, rerun bool, rerunSuccessTags []string, pltfrm, outputDir string, propagateTestErrors bool) error { var versionStr string // Avoid incurring cost of starting machine in getClusterSemver when @@ -804,16 +804,48 @@ func runProvidedTests(testsBank map[string]*register.Test, patterns []string, mu if len(testsToRerun) > 0 && rerun { newOutputDir := filepath.Join(outputDir, "rerun") fmt.Printf("\n\n======== Re-running failed tests (flake detection) ========\n\n") - reRunErr := runProvidedTests(testsBank, testsToRerun, multiply, false, allowRerunSuccess, pltfrm, newOutputDir, propagateTestErrors) - if allowRerunSuccess { + reRunErr := runProvidedTests(testsBank, testsToRerun, multiply, false, rerunSuccessTags, pltfrm, newOutputDir, propagateTestErrors) + + // Return the results from the rerun if rerun success allowed + if allTestsAllowRerunSuccess(testsBank, testsToRerun, rerunSuccessTags) { return reRunErr } } - // If the intial run failed and the rerun passed, we still return an error return firstRunErr } +func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun, rerunSuccessTags []string) bool { + // No tags, we can return early + if len(rerunSuccessTags) == 0 { + return false + } + // Build up a map of rerunSuccessTags so that we can easily check + // if a given tag is in the map. + rerunSuccessTagMap := make(map[string]bool) + for _, tag := range rerunSuccessTags { + if tag == "all" || tag == "*" { + // If `all` or `*` is in rerunSuccessTags then we can return early + return true + } + rerunSuccessTagMap[tag] = true + } + // Iterate over the tests that were re-ran. If any of them don't + // allow rerun success then just exit early. + for _, test := range testsToRerun { + testAllowsRerunSuccess := false + for _, tag := range testsBank[test].Tags { + if rerunSuccessTagMap[tag] { + testAllowsRerunSuccess = true + } + } + if !testAllowsRerunSuccess { + return false + } + } + return true +} + func GetRerunnableTestName(testName string) (string, bool) { // The current nonexclusive test wrapper would rerun all non-exclusive tests. // Instead, we only want to rerun the one(s) that failed, so we will not consider @@ -859,12 +891,12 @@ func getRerunnable(tests []*harness.H) []string { return testsToRerun } -func RunTests(patterns []string, multiply int, rerun bool, allowRerunSuccess bool, pltfrm, outputDir string, propagateTestErrors bool) error { - return runProvidedTests(register.Tests, patterns, multiply, rerun, allowRerunSuccess, pltfrm, outputDir, propagateTestErrors) +func RunTests(patterns []string, multiply int, rerun bool, rerunSuccessTags []string, pltfrm, outputDir string, propagateTestErrors bool) error { + return runProvidedTests(register.Tests, patterns, multiply, rerun, rerunSuccessTags, pltfrm, outputDir, propagateTestErrors) } func RunUpgradeTests(patterns []string, rerun bool, pltfrm, outputDir string, propagateTestErrors bool) error { - return runProvidedTests(register.UpgradeTests, patterns, 0, rerun, false, pltfrm, outputDir, propagateTestErrors) + return runProvidedTests(register.UpgradeTests, patterns, 0, rerun, nil, pltfrm, outputDir, propagateTestErrors) } // externalTestMeta is parsed from kola.json in external tests