Test retry

test_runner/src/main/kotlin/ftl/reports/util/JUnitDedupe.kt

@@ -0,0 +1,29 @@
+package ftl.reports.util
+
+import ftl.reports.xml.model.JUnitTestCase
+import ftl.reports.xml.model.JUnitTestResult
+
+// Read in JUnitReport.xml and remove duplicate results when `flaky-test-attempts` is > 0
+// for each test `name="testFails" classname="com.example.app.ExampleUiTest"`
+// Keep first result. If next result for the same test is successful, keep last successful result.
+object JUnitDedupe {
+
+    private fun JUnitTestCase.key(): String {
+        return "${this.classname}#${this.name}"
+    }
+
+    fun modify(testResult: JUnitTestResult?) {
+        testResult?.testsuites?.forEach { suite ->
+            val testCaseMap = mutableMapOf<String, JUnitTestCase>()
+
+            suite.testcases?.forEach { testcase ->
+                if (testCaseMap[testcase.key()] == null || testcase.successful()) {
+                    testCaseMap[testcase.key()] = testcase
+                }
+            }
+
+            suite.testcases = testCaseMap.values
+            suite.updateTestStats()
+        }
+    }
+}

test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt

@@ -46,14 +46,23 @@ object ReportManager {
         return webLink
     }
 
+    private val deviceStringRgx = Regex("([^-]+-[^-]+-[^-]+-[^-]+).*")
+    // NexusLowRes-28-en-portrait-rerun_1 =>  NexusLowRes-28-en-portrait
+    fun getDeviceString(deviceString: String): String {
+        val matchResult = deviceStringRgx.find(deviceString)
+        return matchResult?.groupValues?.last().orEmpty()
+    }
+
     private fun processXml(matrices: MatrixMap, process: (file: File) -> JUnitTestResult): JUnitTestResult? {
         var mergedXml: JUnitTestResult? = null
 
         findXmlFiles(matrices).forEach { xmlFile ->
             val parsedXml = process(xmlFile)
             val webLink = getWebLink(matrices, xmlFile)
+            val suiteName = getDeviceString(xmlFile.parentFile.name)
 
             parsedXml.testsuites?.forEach { testSuite ->
+                testSuite.name = suiteName
                 testSuite.testcases?.forEach { testCase ->
                     testCase.webLink = webLink
                 }
@@ -77,7 +86,11 @@ object ReportManager {
     /** Returns true if there were no test failures */
     fun generate(matrices: MatrixMap, args: IArgs): Int {
         val testSuite = parseTestSuite(matrices, args)
-        val testSuccessful = matrices.allSuccessful()
+
+        val useFlakyTests = args.flakyTestAttempts > 0
+        if (useFlakyTests) JUnitDedupe.modify(testSuite)
+
+        val testSuccessful = if (useFlakyTests) testSuite?.successful() ?: false else matrices.allSuccessful()
 
         listOf(
             CostReport,
@@ -95,7 +108,14 @@ object ReportManager {
         JUnitReport.run(matrices, testSuite)
         processJunitXml(testSuite, args)
 
-        return matrices.exitCode()
+        // FTL has a bug with matrix roll-up when using flakyTestAttempts
+        // as a work around, we calculate the success based on JUnit XML results.
+        val exitCode = if (useFlakyTests) {
+            if (testSuccessful) 0 else 1
+        } else {
+            matrices.exitCode()
+        }
+        return exitCode
     }
 
     data class ShardEfficiency(

test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestCase.kt

@@ -10,15 +10,21 @@ private class FilterNotNull {
         // other is not null = absent (default value)
         return other != null
     }
+
+    override fun hashCode(): Int {
+        return javaClass.hashCode()
+    }
 }
 
 // https://android.googlesource.com/platform/tools/base/+/tools_r22/ddmlib/src/main/java/com/android/ddmlib/testrunner/XmlTestRunListener.java#256
 data class JUnitTestCase(
     // name, classname, and time are always present except for empty test cases <testcase/>
     @JacksonXmlProperty(isAttribute = true)
     val name: String?,
+
     @JacksonXmlProperty(isAttribute = true)
     val classname: String?,
+
     @JacksonXmlProperty(isAttribute = true)
     val time: String?,
 
@@ -43,6 +49,7 @@ data class JUnitTestCase(
         return name == null || classname == null || time == null
     }
 
+    /** Failed means there was a failure or an error. */
     fun failed(): Boolean {
         return failures?.isNotEmpty() == true || errors?.isNotEmpty() == true
     }

test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestResult.kt

@@ -11,6 +11,15 @@ data class JUnitTestResult(
     @JacksonXmlProperty(localName = "testsuite")
     var testsuites: MutableList<JUnitTestSuite>?
 ) {
+    fun successful(): Boolean {
+        var successful = true
+        testsuites?.forEach { suite ->
+            if (suite.failed()) successful = false
+        }
+
+        return successful
+    }
+
     fun mergeTestTimes(other: JUnitTestResult?): JUnitTestResult {
         if (other == null) return this
         if (this.testsuites == null) this.testsuites = mutableListOf()

test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestSuite.kt

@@ -5,7 +5,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty
 
 data class JUnitTestSuite(
     @JacksonXmlProperty(isAttribute = true)
-    val name: String,
+    var name: String,
 
     @JacksonXmlProperty(isAttribute = true)
     var tests: String, // Int
@@ -31,7 +31,7 @@ data class JUnitTestSuite(
     val hostname: String, // String.
 
     @JacksonXmlProperty(localName = "testcase")
-    var testcases: MutableList<JUnitTestCase>?,
+    var testcases: MutableCollection<JUnitTestCase>?,
 
     // not used
     @JsonInclude(JsonInclude.Include.NON_NULL)
@@ -46,6 +46,23 @@ data class JUnitTestSuite(
     val systemErr: Any? // <system-err />
 ) {
 
+    fun successful(): Boolean {
+        return failures == "0" && errors == "0"
+    }
+
+    fun failed(): Boolean {
+        return successful().not()
+    }
+
+    /** Call after setting testcases manually to update the statistics (error count, skip count, etc.) */
+    fun updateTestStats() {
+        this.tests = testcases?.size.toString()
+        this.failures = testcases?.count { it.failures?.isNotEmpty() == true }.toString()
+        this.errors = testcases?.count { it.errors?.isNotEmpty() == true }.toString()
+        this.skipped = testcases?.count { it.skipped() }.toString()
+        this.time = testcases?.fold("0") { acc, test -> mergeDouble(acc, test.time) } ?: "0"
+    }
+
     /**
      * Strips all characters except numbers and a period
      * Returns 0 when the string is null

test_runner/src/test/kotlin/ftl/reports/utils/JUnitDedupeTest.kt

@@ -0,0 +1,70 @@
+package ftl.reports.utils
+
+import com.google.common.truth.Truth.assertThat
+import ftl.reports.util.JUnitDedupe
+import ftl.reports.xml.parseAllSuitesXml
+import ftl.reports.xml.xmlToString
+import ftl.test.util.FlankTestRunner
+import org.junit.Test
+import org.junit.runner.RunWith
+
+@RunWith(FlankTestRunner::class)
+class JUnitDedupeTest {
+
+    @Test
+    fun `Dedupes multiple tests in a suite`() {
+        val inputXml = """
+<?xml version='1.0' encoding='UTF-8' ?>
+<testsuites>
+  <testsuite name="" tests="6" failures="3" errors="0" skipped="0" time="2.1" timestamp="2019-03-14T19:21:26" hostname="localhost">
+    <testcase name="testFails" classname="com.example.app.ExampleUiTest" time="0.6">
+      <failure>junit.framework.AssertionFailedError</failure>
+      <webLink>matrices/7494574344413871385</webLink>
+    </testcase>
+    <testcase name="testPasses" classname="com.example.app.ExampleUiTest" time="0.5">
+      <webLink>matrices/7494574344413871385</webLink>
+    </testcase>
+    <testcase name="testFails" classname="com.example.app.ExampleUiTest" time="0.4">
+      <failure>junit.framework.AssertionFailedError</failure>
+      <webLink>matrices/7494574344413871385</webLink>
+    </testcase>
+    <testcase name="testPasses" classname="com.example.app.ExampleUiTest" time="0.3">
+      <webLink>matrices/7494574344413871385</webLink>
+    </testcase>
+    <testcase name="testFlaky" classname="com.example.app.ExampleUiTest" time="0.2">
+      <webLink>matrices/7494574344413871385</webLink>
+    </testcase>
+    <testcase name="testFlaky" classname="com.example.app.ExampleUiTest" time="0.1">
+      <failure>junit.framework.AssertionFailedError</failure>
+      <webLink>matrices/7494574344413871385</webLink>
+    </testcase>
+  </testsuite>
+</testsuites>
+
+        """.trimIndent()
+
+        val expectedXml = """
+<?xml version='1.0' encoding='UTF-8' ?>
+<testsuites>
+  <testsuite name="" tests="3" failures="1" errors="0" skipped="0" time="1.100" timestamp="2019-03-14T19:21:26" hostname="localhost">
+    <testcase name="testFails" classname="com.example.app.ExampleUiTest" time="0.6">
+      <failure>junit.framework.AssertionFailedError</failure>
+      <webLink>matrices/7494574344413871385</webLink>
+    </testcase>
+    <testcase name="testPasses" classname="com.example.app.ExampleUiTest" time="0.3">
+      <webLink>matrices/7494574344413871385</webLink>
+    </testcase>
+    <testcase name="testFlaky" classname="com.example.app.ExampleUiTest" time="0.2">
+      <webLink>matrices/7494574344413871385</webLink>
+    </testcase>
+  </testsuite>
+</testsuites>
+
+        """.trimIndent()
+
+        val suites = parseAllSuitesXml(inputXml)
+        JUnitDedupe.modify(suites)
+
+        assertThat(suites.xmlToString()).isEqualTo(expectedXml)
+    }
+}

test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt

@@ -74,4 +74,16 @@ class ReportManagerTest {
 
         assertThat(result).isEqualTo(expected)
     }
+
+    @Test
+    fun `Test getDeviceString`() {
+        assertThat(ReportManager.getDeviceString("NexusLowRes-28-en-portrait-rerun_1"))
+            .isEqualTo("NexusLowRes-28-en-portrait")
+
+        assertThat(ReportManager.getDeviceString("NexusLowRes-28-en-portrait"))
+            .isEqualTo("NexusLowRes-28-en-portrait")
+
+        assertThat(ReportManager.getDeviceString(""))
+            .isEqualTo("")
+    }
 }