From cdcf5b577ae3902a865197bcced0d0b946fa4e5b Mon Sep 17 00:00:00 2001 From: bootstraponline Date: Thu, 10 Oct 2019 15:30:39 -0700 Subject: [PATCH 1/2] Use matrix rollup --- .../src/main/kotlin/ftl/gc/GcToolResults.kt | 34 ++++++-- .../src/main/kotlin/ftl/json/MatrixMap.kt | 2 +- .../src/main/kotlin/ftl/json/SavedMatrix.kt | 7 +- .../kotlin/ftl/reports/MatrixResultsReport.kt | 12 +-- .../src/main/kotlin/ftl/util/StepOutcome.kt | 2 +- test_runner/src/test/kotlin/Tmp.kt | 2 +- .../test/kotlin/ftl/json/SavedMatrixTest.kt | 6 +- .../test/kotlin/ftl/test/util/MockServer.kt | 85 ++++++++++--------- 8 files changed, 87 insertions(+), 63 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/gc/GcToolResults.kt b/test_runner/src/main/kotlin/ftl/gc/GcToolResults.kt index e0fb5f5a94..1a1d4f7153 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcToolResults.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcToolResults.kt @@ -1,8 +1,10 @@ package ftl.gc +import com.google.api.services.testing.model.TestExecution import com.google.api.services.testing.model.ToolResultsHistory import com.google.api.services.testing.model.ToolResultsStep import com.google.api.services.toolresults.ToolResults +import com.google.api.services.toolresults.model.Execution import com.google.api.services.toolresults.model.History import com.google.api.services.toolresults.model.ListTestCasesResponse import com.google.api.services.toolresults.model.Step @@ -27,7 +29,7 @@ object GcToolResults { // https://github.com/bootstraponline/gcloud_cli/blob/0752e88b155a417a18d244c242b4ab3fb9aa1c1f/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/history_picker.py#L45 private fun listHistoriesByName(args: IArgs): List { - val result = GcToolResults.service + val result = service .projects() .histories() .list(args.project) @@ -40,7 +42,7 @@ object GcToolResults { val history = History() .setName(args.resultsHistoryName) .setDisplayName(args.resultsHistoryName) - return GcToolResults.service + return service .projects() .histories() .create(args.project, history) @@ -61,8 +63,28 @@ object GcToolResults { return createHistory(args).historyId } - fun getResults(toolResultsStep: ToolResultsStep): Step { - return GcToolResults.service + // FetchMatrixRollupOutcome - https://github.com/bootstraponline/gcloud_cli/blob/137d864acd5928baf25434cf59b0225c4d1f9319/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/results_summary.py#L122 + // Get the rolled-up outcome for a test execution from the Tool Results service. + // Prefer execution rollUp outcome over individual step outcome + // https://github.com/bootstraponline/gcloud_cli/blob/137d864acd5928baf25434cf59b0225c4d1f9319/google-cloud-sdk/lib/googlecloudsdk/third_party/apis/toolresults_v1beta3.json#L992 + fun getExecutionResult(testExecution: TestExecution): Execution { + val toolResultsStep = testExecution.toolResultsStep + + // Toolresults.Projects.Histories.Executions.GetRequest + return service + .projects() + .histories() + .executions() + .get( + toolResultsStep.projectId, + toolResultsStep.historyId, + toolResultsStep.executionId + ) + .executeWithRetry() + } + + fun getStepResult(toolResultsStep: ToolResultsStep): Step { + return service .projects() .histories() .executions() @@ -78,7 +100,7 @@ object GcToolResults { // Lists Test Cases attached to a Step fun listTestCases(toolResultsStep: ToolResultsStep): ListTestCasesResponse { - return GcToolResults.service + return service .projects() .histories() .executions() @@ -93,7 +115,7 @@ object GcToolResults { } fun getDefaultBucket(projectId: String): String? { - val response = GcToolResults.service.Projects().initializeSettings(projectId).executeWithRetry() + val response = service.Projects().initializeSettings(projectId).executeWithRetry() return response.defaultBucket } } diff --git a/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt b/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt index 7f3f33b03a..9d4024f5b6 100644 --- a/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt +++ b/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt @@ -50,6 +50,6 @@ class MatrixMap( } private fun SavedMatrix.logError(message: String) { - println("Error: Matrix $message: ${this.matrixId} ${this.state} ${this.webLink}") + println("Error: Matrix $message: ${this.matrixId} ${this.state} ${this.outcome} ${this.outcomeDetails} ${this.webLink}") } } diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt index 3039710046..8eaa7ac4de 100644 --- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt @@ -75,12 +75,13 @@ class SavedMatrix(matrix: TestMatrix) { if (matrix.testExecutions == null) return matrix.testExecutions.forEach { - val step = GcToolResults.getResults(it.toolResultsStep) - updateOutcome(step.outcome) + val executionResult = GcToolResults.getExecutionResult(it) + updateOutcome(executionResult.outcome) // testExecutionStep, testTiming, etc. can all be null. // sometimes testExecutionStep is present and testTiming is null - val testTimeSeconds = step.testExecutionStep?.testTiming?.testProcessDuration?.seconds ?: return + val stepResult = GcToolResults.getStepResult(it.toolResultsStep) + val testTimeSeconds = stepResult.testExecutionStep?.testTiming?.testProcessDuration?.seconds ?: return val billableMinutes = Billing.billableMinutes(testTimeSeconds) diff --git a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt index bd6c507347..3dc7a1f4a6 100644 --- a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt @@ -3,7 +3,6 @@ package ftl.reports import ftl.args.IArgs import ftl.config.FtlConstants.indent import ftl.json.MatrixMap -import ftl.json.SavedMatrix import ftl.reports.util.IReport import ftl.reports.xml.model.JUnitTestResult import ftl.util.Utils.println @@ -29,13 +28,11 @@ object MatrixResultsReport : IReport { private fun generate(matrices: MatrixMap): String { var total = 0 var success = 0 - val failedMatrices = mutableListOf() matrices.map.values.forEach { matrix -> total += 1 - if (matrix.failed()) { - failedMatrices.add(matrix) - } else { + // unfinished matrix will not be reported as failed since it's still running + if (matrix.failed().not()) { success += 1 } } @@ -52,11 +49,6 @@ object MatrixResultsReport : IReport { if (failed > 0) { writer.println("$indent$failed matrices failed") writer.println() - failedMatrices.forEach { - writer.println("$indent${it.matrixId} ${it.outcomeDetails}") - writer.println("$indent${it.webLink}") - writer.println() - } } return writer.toString() diff --git a/test_runner/src/main/kotlin/ftl/util/StepOutcome.kt b/test_runner/src/main/kotlin/ftl/util/StepOutcome.kt index 184c3702f2..a22087f521 100644 --- a/test_runner/src/main/kotlin/ftl/util/StepOutcome.kt +++ b/test_runner/src/main/kotlin/ftl/util/StepOutcome.kt @@ -2,7 +2,7 @@ package ftl.util // ToolResults API step outcome values object StepOutcome { - // https://github.com/bootstraponline/gcloud_cli/blob/master/google-cloud-sdk/lib/googlecloudsdk/third_party/apis/toolresults_v1beta3.json#L755 + // https://github.com/bootstraponline/gcloud_cli/blob/137d864acd5928baf25434cf59b0225c4d1f9319/google-cloud-sdk/lib/googlecloudsdk/third_party/apis/toolresults_v1beta3.json#L610 const val failure = "failure" const val flaky = "flaky" const val inconclusive = "inconclusive" diff --git a/test_runner/src/test/kotlin/Tmp.kt b/test_runner/src/test/kotlin/Tmp.kt index 88404f3dca..6fc81449b4 100644 --- a/test_runner/src/test/kotlin/Tmp.kt +++ b/test_runner/src/test/kotlin/Tmp.kt @@ -87,7 +87,7 @@ object Tmp { val toolResult = gson.fromJson(content, ToolResultsStep::class.java) val tests = GcToolResults.listTestCases(toolResult) - val result = GcToolResults.getResults(toolResult) + val result = GcToolResults.getStepResult(toolResult) // todo: handle multiple overviews (iOS only) val overview = result.testExecutionStep.testSuiteOverviews.first() diff --git a/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt b/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt index 7d583acbd0..1985f020a3 100644 --- a/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt @@ -25,7 +25,7 @@ class SavedMatrixTest { val toolResultsStep = ToolResultsStep() toolResultsStep.projectId = "1" toolResultsStep.historyId = "2" - toolResultsStep.executionId = "3" + toolResultsStep.executionId = stepId.toString() toolResultsStep.stepId = stepId.toString() val testExecution = TestExecution() @@ -75,7 +75,7 @@ class SavedMatrixTest { assertThat(savedMatrix.matrixId).isEqualTo(matrixId) assertThat(savedMatrix.state).isEqualTo(matrixState) assertThat(savedMatrix.gcsPath).isEqualTo(mockGcsPath) - assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/3") + assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/-1") assertThat(savedMatrix.downloaded).isFalse() assertThat(savedMatrix.billableVirtualMinutes).isEqualTo(0) assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(2) @@ -107,7 +107,7 @@ class SavedMatrixTest { assertThat(savedMatrix.matrixId).isEqualTo(matrixId) assertThat(savedMatrix.state).isEqualTo(matrixState) assertThat(savedMatrix.gcsPath).isEqualTo(mockGcsPath) - assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/3/executions/-3") + assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/-3/executions/-3") assertThat(savedMatrix.downloaded).isFalse() assertThat(savedMatrix.billableVirtualMinutes).isEqualTo(0) assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1) diff --git a/test_runner/src/test/kotlin/ftl/test/util/MockServer.kt b/test_runner/src/test/kotlin/ftl/test/util/MockServer.kt index 91d8a6a057..ae1be700dc 100644 --- a/test_runner/src/test/kotlin/ftl/test/util/MockServer.kt +++ b/test_runner/src/test/kotlin/ftl/test/util/MockServer.kt @@ -67,6 +67,44 @@ object MockServer { private val androidCatalog by lazy { loadCatalog("android_catalog.json") } private val iosCatalog by lazy { loadCatalog("ios_catalog.json") } + private fun fakeStep(stringId: String): Step { + val oneSecond = Duration().setSeconds(1) + + val testTiming = TestTiming() + .setTestProcessDuration(oneSecond) + + val testExecutionStep = TestExecutionStep() + .setTestTiming(testTiming) + + val outcome = Outcome() + when (stringId) { + "-1" -> { + outcome.summary = failure + val failureDetail = FailureDetail() + failureDetail.timedOut = true + outcome.failureDetail = failureDetail + } + "-2" -> { + outcome.summary = inconclusive + val inconclusiveDetail = InconclusiveDetail() + inconclusiveDetail.abortedByUser = true + outcome.inconclusiveDetail = inconclusiveDetail + } + "-3" -> { + outcome.summary = skipped + val skippedDetail = SkippedDetail() + skippedDetail.incompatibleAppVersion = true + outcome.skippedDetail = skippedDetail + } + else -> outcome.summary = success + } + + return Step() + .setTestExecutionStep(testExecutionStep) + .setRunDuration(oneSecond) + .setOutcome(outcome) + } + val application by lazy { embeddedServer(Netty, port) { install(ContentNegotiation) { @@ -144,49 +182,20 @@ object MockServer { call.respond(matrix) } - // GcToolResults.getResults(toolResultsStep) + // GcToolResults.getStepResult(toolResultsStep) // GET /toolresults/v1beta3/projects/delta-essence-114723/histories/1/executions/1/steps/1 get("/toolresults/v1beta3/projects/{project}/histories/{historyId}/executions/{executionId}/steps/{stepId}") { println("Responding to GET ${call.request.uri}") val stepId = call.parameters["stepId"] ?: "" + call.respond(fakeStep(stepId)) + } - val oneSecond = Duration().setSeconds(1) - - val testTiming = TestTiming() - .setTestProcessDuration(oneSecond) - - val testExecutionStep = TestExecutionStep() - .setTestTiming(testTiming) - - val outcome = Outcome() - when (stepId) { - "-1" -> { - outcome.summary = failure - val failureDetail = FailureDetail() - failureDetail.timedOut = true - outcome.failureDetail = failureDetail - } - "-2" -> { - outcome.summary = inconclusive - val inconclusiveDetail = InconclusiveDetail() - inconclusiveDetail.abortedByUser = true - outcome.inconclusiveDetail = inconclusiveDetail - } - "-3" -> { - outcome.summary = skipped - val skippedDetail = SkippedDetail() - skippedDetail.incompatibleAppVersion = true - outcome.skippedDetail = skippedDetail - } - else -> outcome.summary = success - } - - val step = Step() - .setTestExecutionStep(testExecutionStep) - .setRunDuration(oneSecond) - .setOutcome(outcome) - - call.respond(step) + // GcToolResults.getExecutionResult(toolResultsStep) + // GET /toolresults/v1beta3/projects/delta-essence-114723/histories/1/executions/1 + get("/toolresults/v1beta3/projects/{project}/histories/{historyId}/executions/{executionId}") { + println("Responding to GET ${call.request.uri}") + val executionId = call.parameters["executionId"] ?: "" + call.respond(fakeStep(executionId)) } // GcToolResults.getDefaultBucket(project) From 8a8d8f40f1dc2f725af4106ac471c27e66dfb0f3 Mon Sep 17 00:00:00 2001 From: bootstraponline Date: Thu, 10 Oct 2019 20:29:21 -0700 Subject: [PATCH 2/2] Update release_notes.md --- release_notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release_notes.md b/release_notes.md index 13d5df9072..288b43898a 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,7 @@ ## next (unreleased) +- [#608](https://github.com/TestArmada/flank/pull/608) Use MatrixRollupOutcome to set exit code value. ([bootstraponline](https://github.com/bootstraponline)) + ## v8.0.0 - [#595](https://github.com/TestArmada/flank/pull/595) Rename `flaky-test-attempts` to `num-flaky-test-attempts`. Rename `repeat-tests` to `num-test-runs`. ([bootstraponline](https://github.com/bootstraponline))