Use matrix rollup

release_notes.md

@@ -1,5 +1,7 @@
 ## next (unreleased)
 
+- [#608](https://github.com/TestArmada/flank/pull/608) Use MatrixRollupOutcome to set exit code value. ([bootstraponline](https://github.com/bootstraponline))
+
 ## v8.0.0
 
 - [#595](https://github.com/TestArmada/flank/pull/595) Rename `flaky-test-attempts` to `num-flaky-test-attempts`. Rename `repeat-tests` to `num-test-runs`. ([bootstraponline](https://github.com/bootstraponline))

test_runner/src/main/kotlin/ftl/gc/GcToolResults.kt

@@ -1,8 +1,10 @@
 package ftl.gc
 
+import com.google.api.services.testing.model.TestExecution
 import com.google.api.services.testing.model.ToolResultsHistory
 import com.google.api.services.testing.model.ToolResultsStep
 import com.google.api.services.toolresults.ToolResults
+import com.google.api.services.toolresults.model.Execution
 import com.google.api.services.toolresults.model.History
 import com.google.api.services.toolresults.model.ListTestCasesResponse
 import com.google.api.services.toolresults.model.Step
@@ -27,7 +29,7 @@ object GcToolResults {
 
     // https://github.com/bootstraponline/gcloud_cli/blob/0752e88b155a417a18d244c242b4ab3fb9aa1c1f/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/history_picker.py#L45
     private fun listHistoriesByName(args: IArgs): List<History> {
-        val result = GcToolResults.service
+        val result = service
             .projects()
             .histories()
             .list(args.project)
@@ -40,7 +42,7 @@ object GcToolResults {
         val history = History()
             .setName(args.resultsHistoryName)
             .setDisplayName(args.resultsHistoryName)
-        return GcToolResults.service
+        return service
             .projects()
             .histories()
             .create(args.project, history)
@@ -61,8 +63,28 @@ object GcToolResults {
         return createHistory(args).historyId
     }
 
-    fun getResults(toolResultsStep: ToolResultsStep): Step {
-        return GcToolResults.service
+    // FetchMatrixRollupOutcome - https://github.com/bootstraponline/gcloud_cli/blob/137d864acd5928baf25434cf59b0225c4d1f9319/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/results_summary.py#L122
+    // Get the rolled-up outcome for a test execution from the Tool Results service.
+    // Prefer execution rollUp outcome over individual step outcome
+    // https://github.com/bootstraponline/gcloud_cli/blob/137d864acd5928baf25434cf59b0225c4d1f9319/google-cloud-sdk/lib/googlecloudsdk/third_party/apis/toolresults_v1beta3.json#L992
+    fun getExecutionResult(testExecution: TestExecution): Execution {
+        val toolResultsStep = testExecution.toolResultsStep
+
+        // Toolresults.Projects.Histories.Executions.GetRequest
+        return service
+            .projects()
+            .histories()
+            .executions()
+            .get(
+                toolResultsStep.projectId,
+                toolResultsStep.historyId,
+                toolResultsStep.executionId
+            )
+            .executeWithRetry()
+    }
+
+    fun getStepResult(toolResultsStep: ToolResultsStep): Step {
+        return service
             .projects()
             .histories()
             .executions()
@@ -78,7 +100,7 @@ object GcToolResults {
 
     // Lists Test Cases attached to a Step
     fun listTestCases(toolResultsStep: ToolResultsStep): ListTestCasesResponse {
-        return GcToolResults.service
+        return service
             .projects()
             .histories()
             .executions()
@@ -93,7 +115,7 @@ object GcToolResults {
     }
 
     fun getDefaultBucket(projectId: String): String? {
-        val response = GcToolResults.service.Projects().initializeSettings(projectId).executeWithRetry()
+        val response = service.Projects().initializeSettings(projectId).executeWithRetry()
         return response.defaultBucket
     }
 }

test_runner/src/main/kotlin/ftl/json/MatrixMap.kt

@@ -50,6 +50,6 @@ class MatrixMap(
     }
 
     private fun SavedMatrix.logError(message: String) {
-        println("Error: Matrix $message: ${this.matrixId} ${this.state} ${this.webLink}")
+        println("Error: Matrix $message: ${this.matrixId} ${this.state} ${this.outcome} ${this.outcomeDetails} ${this.webLink}")
     }
 }

test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt

@@ -75,12 +75,13 @@ class SavedMatrix(matrix: TestMatrix) {
         if (matrix.testExecutions == null) return
 
         matrix.testExecutions.forEach {
-            val step = GcToolResults.getResults(it.toolResultsStep)
-            updateOutcome(step.outcome)
+            val executionResult = GcToolResults.getExecutionResult(it)
+            updateOutcome(executionResult.outcome)
 
             // testExecutionStep, testTiming, etc. can all be null.
             // sometimes testExecutionStep is present and testTiming is null
-            val testTimeSeconds = step.testExecutionStep?.testTiming?.testProcessDuration?.seconds ?: return
+            val stepResult = GcToolResults.getStepResult(it.toolResultsStep)
+            val testTimeSeconds = stepResult.testExecutionStep?.testTiming?.testProcessDuration?.seconds ?: return
 
             val billableMinutes = Billing.billableMinutes(testTimeSeconds)
 

test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt

@@ -3,7 +3,6 @@ package ftl.reports
 import ftl.args.IArgs
 import ftl.config.FtlConstants.indent
 import ftl.json.MatrixMap
-import ftl.json.SavedMatrix
 import ftl.reports.util.IReport
 import ftl.reports.xml.model.JUnitTestResult
 import ftl.util.Utils.println
@@ -29,13 +28,11 @@ object MatrixResultsReport : IReport {
     private fun generate(matrices: MatrixMap): String {
         var total = 0
         var success = 0
-        val failedMatrices = mutableListOf<SavedMatrix>()
         matrices.map.values.forEach { matrix ->
             total += 1
 
-            if (matrix.failed()) {
-                failedMatrices.add(matrix)
-            } else {
+            // unfinished matrix will not be reported as failed since it's still running
+            if (matrix.failed().not()) {
                 success += 1
             }
         }
@@ -52,11 +49,6 @@ object MatrixResultsReport : IReport {
             if (failed > 0) {
                 writer.println("$indent$failed matrices failed")
                 writer.println()
-                failedMatrices.forEach {
-                    writer.println("$indent${it.matrixId} ${it.outcomeDetails}")
-                    writer.println("$indent${it.webLink}")
-                    writer.println()
-                }
             }
 
             return writer.toString()

test_runner/src/main/kotlin/ftl/util/StepOutcome.kt

@@ -2,7 +2,7 @@ package ftl.util
 
 // ToolResults API step outcome values
 object StepOutcome {
-    // https://github.com/bootstraponline/gcloud_cli/blob/master/google-cloud-sdk/lib/googlecloudsdk/third_party/apis/toolresults_v1beta3.json#L755
+    // https://github.com/bootstraponline/gcloud_cli/blob/137d864acd5928baf25434cf59b0225c4d1f9319/google-cloud-sdk/lib/googlecloudsdk/third_party/apis/toolresults_v1beta3.json#L610
     const val failure = "failure"
     const val flaky = "flaky"
     const val inconclusive = "inconclusive"

test_runner/src/test/kotlin/Tmp.kt

@@ -87,7 +87,7 @@ object Tmp {
         val toolResult = gson.fromJson(content, ToolResultsStep::class.java)
 
         val tests = GcToolResults.listTestCases(toolResult)
-        val result = GcToolResults.getResults(toolResult)
+        val result = GcToolResults.getStepResult(toolResult)
 
         // todo: handle multiple overviews (iOS only)
         val overview = result.testExecutionStep.testSuiteOverviews.first()

test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt

@@ -25,7 +25,7 @@ class SavedMatrixTest {
             val toolResultsStep = ToolResultsStep()
             toolResultsStep.projectId = "1"
             toolResultsStep.historyId = "2"
-            toolResultsStep.executionId = "3"
+            toolResultsStep.executionId = stepId.toString()
             toolResultsStep.stepId = stepId.toString()
 
             val testExecution = TestExecution()
@@ -75,7 +75,7 @@ class SavedMatrixTest {
         assertThat(savedMatrix.matrixId).isEqualTo(matrixId)
         assertThat(savedMatrix.state).isEqualTo(matrixState)
         assertThat(savedMatrix.gcsPath).isEqualTo(mockGcsPath)
-        assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/3")
+        assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/-1")
         assertThat(savedMatrix.downloaded).isFalse()
         assertThat(savedMatrix.billableVirtualMinutes).isEqualTo(0)
         assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(2)
@@ -107,7 +107,7 @@ class SavedMatrixTest {
         assertThat(savedMatrix.matrixId).isEqualTo(matrixId)
         assertThat(savedMatrix.state).isEqualTo(matrixState)
         assertThat(savedMatrix.gcsPath).isEqualTo(mockGcsPath)
-        assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/3/executions/-3")
+        assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/-3/executions/-3")
         assertThat(savedMatrix.downloaded).isFalse()
         assertThat(savedMatrix.billableVirtualMinutes).isEqualTo(0)
         assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1)

test_runner/src/test/kotlin/ftl/test/util/MockServer.kt

@@ -67,6 +67,44 @@ object MockServer {
     private val androidCatalog by lazy { loadCatalog<AndroidDeviceCatalog>("android_catalog.json") }
     private val iosCatalog by lazy { loadCatalog<IosDeviceCatalog>("ios_catalog.json") }
 
+    private fun fakeStep(stringId: String): Step {
+        val oneSecond = Duration().setSeconds(1)
+
+        val testTiming = TestTiming()
+            .setTestProcessDuration(oneSecond)
+
+        val testExecutionStep = TestExecutionStep()
+            .setTestTiming(testTiming)
+
+        val outcome = Outcome()
+        when (stringId) {
+            "-1" -> {
+                outcome.summary = failure
+                val failureDetail = FailureDetail()
+                failureDetail.timedOut = true
+                outcome.failureDetail = failureDetail
+            }
+            "-2" -> {
+                outcome.summary = inconclusive
+                val inconclusiveDetail = InconclusiveDetail()
+                inconclusiveDetail.abortedByUser = true
+                outcome.inconclusiveDetail = inconclusiveDetail
+            }
+            "-3" -> {
+                outcome.summary = skipped
+                val skippedDetail = SkippedDetail()
+                skippedDetail.incompatibleAppVersion = true
+                outcome.skippedDetail = skippedDetail
+            }
+            else -> outcome.summary = success
+        }
+
+        return Step()
+            .setTestExecutionStep(testExecutionStep)
+            .setRunDuration(oneSecond)
+            .setOutcome(outcome)
+    }
+
     val application by lazy {
         embeddedServer(Netty, port) {
             install(ContentNegotiation) {
@@ -144,49 +182,20 @@ object MockServer {
                     call.respond(matrix)
                 }
 
-                // GcToolResults.getResults(toolResultsStep)
+                // GcToolResults.getStepResult(toolResultsStep)
                 // GET /toolresults/v1beta3/projects/delta-essence-114723/histories/1/executions/1/steps/1
                 get("/toolresults/v1beta3/projects/{project}/histories/{historyId}/executions/{executionId}/steps/{stepId}") {
                     println("Responding to GET ${call.request.uri}")
                     val stepId = call.parameters["stepId"] ?: ""
+                    call.respond(fakeStep(stepId))
+                }
 
-                    val oneSecond = Duration().setSeconds(1)
-
-                    val testTiming = TestTiming()
-                        .setTestProcessDuration(oneSecond)
-
-                    val testExecutionStep = TestExecutionStep()
-                        .setTestTiming(testTiming)
-
-                    val outcome = Outcome()
-                    when (stepId) {
-                        "-1" -> {
-                            outcome.summary = failure
-                            val failureDetail = FailureDetail()
-                            failureDetail.timedOut = true
-                            outcome.failureDetail = failureDetail
-                        }
-                        "-2" -> {
-                        outcome.summary = inconclusive
-                        val inconclusiveDetail = InconclusiveDetail()
-                        inconclusiveDetail.abortedByUser = true
-                        outcome.inconclusiveDetail = inconclusiveDetail
-                        }
-                        "-3" -> {
-                            outcome.summary = skipped
-                            val skippedDetail = SkippedDetail()
-                            skippedDetail.incompatibleAppVersion = true
-                            outcome.skippedDetail = skippedDetail
-                        }
-                        else -> outcome.summary = success
-                    }
-
-                    val step = Step()
-                        .setTestExecutionStep(testExecutionStep)
-                        .setRunDuration(oneSecond)
-                        .setOutcome(outcome)
-
-                    call.respond(step)
+                // GcToolResults.getExecutionResult(toolResultsStep)
+                // GET /toolresults/v1beta3/projects/delta-essence-114723/histories/1/executions/1
+                get("/toolresults/v1beta3/projects/{project}/histories/{historyId}/executions/{executionId}") {
+                    println("Responding to GET ${call.request.uri}")
+                    val executionId = call.parameters["executionId"] ?: ""
+                    call.respond(fakeStep(executionId))
                 }
 
                 // GcToolResults.getDefaultBucket(project)