[regression-test](framework) Support running tests multiple times and…

… reporting correctly to TeamCity (#26606) (#26871)
apache · Nov 13, 2023 · be2dd64 · be2dd64
1 parent c23b674
commit be2dd64
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 6 deletions.
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
@@ -30,6 +30,7 @@ import org.apache.doris.regression.suite.event.StackEventListeners
 import org.apache.doris.regression.suite.SuiteScript
 import org.apache.doris.regression.suite.event.TeamcityEventListener
 import org.apache.doris.regression.util.Recorder
+import org.apache.doris.regression.util.TeamcityUtils
 import groovy.util.logging.Slf4j
 import org.apache.commons.cli.*
 import org.apache.commons.lang3.concurrent.BasicThreadFactory;
@@ -65,10 +66,28 @@ class RegressionTest {
         Config config = Config.fromCommandLine(cmd)
         initGroovyEnv(config)
         boolean success = true
+        Integer totalFailure = 0
+        Integer failureLimit = Integer.valueOf(config.otherConfigs.getOrDefault("max_failure_num", "-1").toString())
+        if (failureLimit <= 0) {
+            failureLimit = Integer.MAX_VALUE
+        }
+
         for (int i = 0; i < config.times; i++) {
             log.info("=== run ${i} time ===")
+            if (config.times > 1) {
+                TeamcityUtils.postfix = i.toString()
+            }
+
             Recorder recorder = runScripts(config)
-            success = printResult(config, recorder)
+            success = (success && printResult(config, recorder))
+
+            if (recorder.getFatalNum() > 0) {
+                break
+            }
+            totalFailure += recorder.getFailureOrFatalNum()
+            if (totalFailure > failureLimit) {
+                break
+            }
         }
         actionExecutors.shutdown()
         suiteExecutors.shutdown()

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy
@@ -45,6 +45,10 @@ class Recorder {
         return failureCounter.get()
     }
 
+    public int getFatalNum() {
+        return fatalScriptList.size()
+    }
+
     void onSuccess(SuiteInfo suiteInfo) {
         successList.add(suiteInfo)
     }

diff --git a/...sion-test/framework/src/main/groovy/org/apache/doris/regression/util/TeamcityUtils.groovy b/...sion-test/framework/src/main/groovy/org/apache/doris/regression/util/TeamcityUtils.groovy
@@ -24,34 +24,49 @@ import org.apache.tools.ant.util.DateUtils
 
 @CompileStatic
 class TeamcityUtils {
+    static String postfix = ""
+
+    static String getSuiteName(String name) {
+        if (postfix == "") {
+            return name
+        } else {
+            return name+"-"+postfix
+        }
+    }
+
     static String formatNow() {
         return DateUtils.format(System.currentTimeMillis(), "yyyy-MM-dd'T'HH:mm:ss.SSSZ")
     }
 
     static String formatStdOut(SuiteContext suiteContext, String msg) {
         String timestamp = formatNow()
-        return "##teamcity[testStdOut name='${suiteContext.flowName}' out='${escape(msg)}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']"
+        String name = getSuiteName(suiteContext.flowName)
+        return "##teamcity[testStdOut name='${name}' out='${escape(msg)}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']"
     }
 
     static String formatStdErr(SuiteContext suiteContext, String msg) {
         String timestamp = formatNow()
-        return "##teamcity[testStdErr name='${suiteContext.flowName}' out='${escape(msg)}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']"
+        String name = getSuiteName(suiteContext.flowName)
+        return "##teamcity[testStdErr name='${name}' out='${escape(msg)}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']"
     }
 
     static void testStarted(SuiteContext suiteContext) {
         String timestamp = formatNow()
+        String name = getSuiteName(suiteContext.flowName)
         println("##teamcity[flowStarted flowId='${suiteContext.flowId}' timestamp='${timestamp}']")
-        println("##teamcity[testStarted name='${suiteContext.flowName}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']")
+        println("##teamcity[testStarted name='${name}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']")
     }
 
     static void testFailed(SuiteContext suiteContext, String msg, String details) {
         String timestamp = formatNow()
-        println("##teamcity[testFailed name='${suiteContext.flowName}' message='${escape(msg)}' flowId='${suiteContext.flowId}' details='${escape(details)}' timestamp='${timestamp}']")
+        String name = getSuiteName(suiteContext.flowName)
+        println("##teamcity[testFailed name='${name}' message='${escape(msg)}' flowId='${suiteContext.flowId}' details='${escape(details)}' timestamp='${timestamp}']")
     }
 
     static void testFinished(SuiteContext suiteContext, long elapsed) {
         String timestamp = formatNow()
-        println("##teamcity[testFinished name='${suiteContext.flowName}' flowId='${suiteContext.flowId}' duration='${elapsed}' timestamp='${timestamp}']")
+        String name = getSuiteName(suiteContext.flowName)
+        println("##teamcity[testFinished name='${name}' flowId='${suiteContext.flowId}' duration='${elapsed}' timestamp='${timestamp}']")
         println("##teamcity[flowFinished flowId='${suiteContext.flowId}' timestamp='${timestamp}']")
     }