v7 (#574)

* Update Kotlin

* Add suite time

* Add iOS example. Remove old JSON

* Update gradle

* Improve shard debugging

* Update catalog to use projectId

release_notes.md

@@ -1,6 +1,6 @@
 ## next (unreleased)
 
-- 
+- [#574](https://github.com/TestArmada/flank/pull/574) Improve test shard error reporting. Update device catalog to use projectId. ([bootstraponline](https://github.com/bootstraponline))
 
 ## v6.2.3
 

test_runner/buildSrc/src/main/kotlin/Deps.kt

@@ -1,6 +1,6 @@
 object Versions {
     // match to Tools -> Kotlin -> Configure Kotlin Plugin Updates -> Update Channel: Stable
-    val KOTLIN = "1.3.40"
+    val KOTLIN = "1.3.41"
     // https://github.com/Kotlin/kotlinx.coroutines/releases
     val KOTLIN_COROUTINES = "1.3.0-M2"
     // https://github.com/FasterXML/jackson-core/releases

test_runner/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.5-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

test_runner/src/main/kotlin/ftl/android/AndroidCatalog.kt

@@ -1,6 +1,7 @@
 package ftl.android
 
 import com.google.api.services.testing.model.AndroidDevice
+import com.google.api.services.testing.model.AndroidDeviceCatalog
 import ftl.gc.GcTesting
 import ftl.http.executeWithRetry
 
@@ -9,16 +10,43 @@ import ftl.http.executeWithRetry
  * to validate device configuration
  */
 object AndroidCatalog {
-    private val androidDeviceCatalog by lazy {
-        GcTesting.get.testEnvironmentCatalog().get("android").executeWithRetry().androidDeviceCatalog
+    private val catalogMap: MutableMap<String, AndroidDeviceCatalog> = mutableMapOf()
+    private val modelMap: MutableMap<String, List<String>> = mutableMapOf()
+    private val versionMap: MutableMap<String, List<String>> = mutableMapOf()
+
+    private fun androidDeviceCatalog(projectId: String): AndroidDeviceCatalog {
+        val cached = catalogMap[projectId]
+        if (cached != null) return cached
+
+        val newCatalog = GcTesting.get.testEnvironmentCatalog()
+            .get("android")
+            .setProjectId(projectId)
+            .executeWithRetry().androidDeviceCatalog
+        catalogMap[projectId] = newCatalog
+        return newCatalog
     }
 
-    val androidModelIds by lazy { androidDeviceCatalog.models.map { it.id } }
-    val androidVersionIds by lazy { androidDeviceCatalog.versions.map { it.id } }
+    fun androidModelIds(projectId: String): List<String> {
+        val cached = modelMap[projectId]
+        if (cached != null) return cached
+
+        val newModels = androidDeviceCatalog(projectId).models.map { it.id }
+        modelMap[projectId] = newModels
+        return newModels
+    }
+
+    fun androidVersionIds(projectId: String): List<String> {
+        val cached = versionMap[projectId]
+        if (cached != null) return cached
+
+        val newVersions = androidDeviceCatalog(projectId).versions.map { it.id }
+        versionMap[projectId] = newVersions
+        return newVersions
+    }
 
-    fun supportedDeviceConfig(modelId: String, versionId: String): DeviceConfigCheck {
-        val foundModel = androidDeviceCatalog.models.find { it.id == modelId } ?: return UnsupportedModelId
-        if (!androidVersionIds.contains(versionId)) return UnsupportedVersionId
+    fun supportedDeviceConfig(modelId: String, versionId: String, projectId: String): DeviceConfigCheck {
+        val foundModel = androidDeviceCatalog(projectId).models.find { it.id == modelId } ?: return UnsupportedModelId
+        if (!androidVersionIds(projectId).contains(versionId)) return UnsupportedVersionId
 
         val supportedVersionIds = foundModel.supportedVersionIds
         supportedVersionIds?.let {
@@ -28,9 +56,9 @@ object AndroidCatalog {
         return SupportedDeviceConfig
     }
 
-    fun isVirtualDevice(device: AndroidDevice?): Boolean {
+    fun isVirtualDevice(device: AndroidDevice?, projectId: String): Boolean {
         val modelId = device?.androidModelId ?: return false
-        val form = androidDeviceCatalog.models.find { it.id == modelId }?.form ?: "PHYSICAL"
+        val form = androidDeviceCatalog(projectId).models.find { it.id == modelId }?.form ?: "PHYSICAL"
         return form == "VIRTUAL"
     }
 }

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -101,11 +101,11 @@ class AndroidArgs(
     }
 
     private fun assertDeviceSupported(device: Device) {
-        when (val deviceConfigTest = AndroidCatalog.supportedDeviceConfig(device.model, device.version)) {
+        when (val deviceConfigTest = AndroidCatalog.supportedDeviceConfig(device.model, device.version, this.project)) {
             SupportedDeviceConfig -> {
             }
-            UnsupportedModelId -> throw RuntimeException("Unsupported model id, '${device.model}'\nSupported model ids: ${AndroidCatalog.androidModelIds}")
-            UnsupportedVersionId -> throw RuntimeException("Unsupported version id, '${device.version}'\nSupported Version ids: ${AndroidCatalog.androidVersionIds}")
+            UnsupportedModelId -> throw RuntimeException("Unsupported model id, '${device.model}'\nSupported model ids: ${AndroidCatalog.androidModelIds(this.project)}")
+            UnsupportedVersionId -> throw RuntimeException("Unsupported version id, '${device.version}'\nSupported Version ids: ${AndroidCatalog.androidVersionIds(this.project)}")
             is IncompatibleModelVersion -> throw RuntimeException("Incompatible model, '${device.model}', and version, '${device.version}'\nSupported version ids for '${device.model}': $deviceConfigTest")
         }
     }

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -20,7 +20,6 @@ import ftl.config.Device
 import ftl.config.FtlConstants
 import ftl.ios.IosCatalog
 import ftl.ios.Xctestrun
-import ftl.util.Utils
 import ftl.util.Utils.fatalError
 import java.nio.file.Files
 import java.nio.file.Path
@@ -95,14 +94,14 @@ class IosArgs(
 
     private fun assertXcodeSupported(xcodeVersion: String?) {
         if (xcodeVersion == null) return
-        if (!IosCatalog.supportedXcode(xcodeVersion)) {
-            Utils.fatalError(("Xcode $xcodeVersion is not a supported Xcode version"))
+        if (!IosCatalog.supportedXcode(xcodeVersion, this.project)) {
+            fatalError(("Xcode $xcodeVersion is not a supported Xcode version"))
         }
     }
 
     private fun assertDeviceSupported(device: Device) {
-        if (!IosCatalog.supportedDevice(device.model, device.version)) {
-            Utils.fatalError("iOS ${device.version} on ${device.model} is not a supported device")
+        if (!IosCatalog.supportedDevice(device.model, device.version, this.project)) {
+            fatalError("iOS ${device.version} on ${device.model} is not a supported device")
         }
     }
 

test_runner/src/main/kotlin/ftl/ios/IosCatalog.kt

@@ -1,5 +1,6 @@
 package ftl.ios
 
+import com.google.api.services.testing.model.IosDeviceCatalog
 import ftl.gc.GcTesting
 import ftl.http.executeWithRetry
 
@@ -9,17 +10,34 @@ import ftl.http.executeWithRetry
  * note:  500 Internal Server Error is returned on invalid model id/version
  **/
 object IosCatalog {
-    private val xcodeVersions by lazy {
-        iosDeviceCatalog.xcodeVersions.map { it.version }
+    private val catalogMap: MutableMap<String, IosDeviceCatalog> = mutableMapOf()
+    private val xcodeMap: MutableMap<String, List<String>> = mutableMapOf()
+
+    fun xcodeVersions(projectId: String): List<String> {
+        val cached = xcodeMap[projectId]
+        if (cached != null) return cached
+
+        val newVersions = iosDeviceCatalog(projectId).xcodeVersions.map { it.version }
+        xcodeMap[projectId] = newVersions
+        return newVersions
     }
 
-    private val iosDeviceCatalog by lazy {
+    // Device catalogMap is different depending on the project id
+    fun iosDeviceCatalog(projectId: String): IosDeviceCatalog {
+        val cached = catalogMap[projectId]
+        if (cached != null) return cached
+
         try {
-            GcTesting.get.testEnvironmentCatalog().get("ios").executeWithRetry().iosDeviceCatalog
+            val newCatalog = GcTesting.get.testEnvironmentCatalog()
+                .get("ios")
+                .setProjectId(projectId)
+                .executeWithRetry().iosDeviceCatalog
+            catalogMap[projectId] = newCatalog
+            return newCatalog
         } catch (e: java.lang.Exception) {
             throw java.lang.RuntimeException(
                 """
-Unable to access the test environment catalog. Firebase Test Lab for iOS is currently in beta.
+Unable to access the test environment catalogMap. Firebase Test Lab for iOS is currently in beta.
 Request access for your project via the following form:
   https://goo.gl/forms/wAxbiNEP2pxeIRG82
 
@@ -32,12 +50,12 @@ If you are still having issues, please email ftl-ios-feedback@google.com for sup
         }
     }
 
-    fun supportedXcode(version: String): Boolean {
-        return xcodeVersions.contains(version)
+    fun supportedXcode(version: String, projectId: String): Boolean {
+        return xcodeVersions(projectId).contains(version)
     }
 
-    fun supportedDevice(modelId: String, versionId: String): Boolean {
-        val model = iosDeviceCatalog.models.find { it.id == modelId }
+    fun supportedDevice(modelId: String, versionId: String, projectId: String): Boolean {
+        val model = iosDeviceCatalog(projectId).models.find { it.id == modelId }
         return model?.supportedVersionIds?.contains(versionId) ?: false
     }
 }

test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt

@@ -84,7 +84,7 @@ class SavedMatrix(matrix: TestMatrix) {
 
             val billableMinutes = Billing.billableMinutes(testTimeSeconds)
 
-            if (AndroidCatalog.isVirtualDevice(it.environment?.androidDevice)) {
+            if (AndroidCatalog.isVirtualDevice(it.environment?.androidDevice, matrix.projectId ?: "")) {
                 billableVirtualMinutes += billableMinutes
             } else {
                 billablePhysicalMinutes += billableMinutes

test_runner/src/main/kotlin/ftl/shard/Shard.kt

@@ -2,8 +2,10 @@ package ftl.shard
 
 import ftl.args.AndroidArgs
 import ftl.args.IArgs
+import ftl.args.IosArgs
 import ftl.reports.xml.model.JUnitTestCase
 import ftl.reports.xml.model.JUnitTestResult
+import ftl.util.Utils.fatalError
 import kotlin.math.roundToInt
 
 data class TestMethod(
@@ -104,6 +106,17 @@ object Shard {
         val shardsCount = if (maxShards == -1 || maxShards > testCount) testCount else maxShards
 
         // Create the list of shards we will return
+        if (shardsCount <= 0) {
+            val platform = if (args is IosArgs) "ios" else "android"
+            fatalError(
+                """Invalid shard count. To debug try: flank $platform run --dump-shards
+                    | args.maxTestShards: ${args.maxTestShards}
+                    | forcedShardCount: $forcedShardCount 
+                    | testCount: $testCount 
+                    | maxShards: $maxShards 
+                    | shardsCount: $shardsCount""".trimMargin()
+            )
+        }
         var shards = List(shardsCount) { TestShard(0.0, mutableListOf()) }
 
         // We want to iterate over testcase going from slowest to fastest

test_runner/src/test/kotlin/Tmp.kt

@@ -2,14 +2,19 @@ import com.google.api.services.testing.model.ToolResultsStep
 import com.google.api.services.toolresults.model.Step
 import com.google.api.services.toolresults.model.TestSuiteOverview
 import com.google.gson.GsonBuilder
+import ftl.args.AndroidArgs
+import ftl.gc.GcTestMatrix
 import ftl.gc.GcToolResults
 import ftl.reports.xml.model.JUnitTestCase
 import ftl.reports.xml.model.JUnitTestResult
 import ftl.reports.xml.model.JUnitTestSuite
 import ftl.reports.xml.xmlToString
+import ftl.util.firstToolResults
 import java.nio.file.Files
 import java.nio.file.Paths
+import kotlin.system.exitProcess
 
+@Suppress("UnusedPrivateMember")
 object Tmp {
 
     fun Step.testSuiteName(): String {
@@ -34,6 +39,14 @@ object Tmp {
         return (this.skippedCount ?: 0).toString()
     }
 
+    fun TestSuiteOverview.time(): String {
+        val time = this.elapsedTime ?: return "0"
+        val seconds = time.seconds ?: 0
+        val nanos = nanosToSeconds(time.nanos)
+
+        return (seconds + nanos).toString()
+    }
+
     // TODO: TestMatrix.ResultStorage.resultsUrl -- link to web console
     private fun ToolResultsStep.webLink(): String {
         return "https://console.firebase.google.com/project/${this.projectId}/" +
@@ -42,23 +55,48 @@ object Tmp {
                 "executions/${this.stepId}"
     }
 
+    private val gson = GsonBuilder().setPrettyPrinting().create()!!
+    private const val androidSuccessStep = "android_matrix-27s4d0h0p53da_success.json"
+    private const val iosSuccessStep = "ios_matrix-3vg7fnfansppa_success.json"
+
+    private fun stepFromMatrixId(matrixId: String) {
+        val matrix = GcTestMatrix.refresh(matrixId, AndroidArgs.default())
+        val toolResult = matrix.firstToolResults()
+        println(toolResult?.webLink())
+
+        Files.write(Paths.get("tool_results_step_${matrix.testMatrixId}.json"), gson.toJson(toolResult).toByteArray())
+    }
+
     @JvmStatic
     fun main(args: Array<String>) {
-        val gson = GsonBuilder().setPrettyPrinting().create()!!
-//        Files.write(Paths.get("./tool_results_step${it.matrixId}.json"), gson.toJson(it.toolResultsStep).toByteArray())
+//        stepFromMatrixId("matrix-3vg7fnfansppa")
+
+        resultToXml(iosSuccessStep)
+//        resultToXml(androidSuccessStep)
+        exitProcess(0)
+    }
+
+    fun nanosToSeconds(seconds: Int?): Double {
+        // manually divide to keep fractional precision
+        if (seconds == null) return 0.0
+        return seconds / 1E9
+    }
+
+    private fun resultToXml(step: String) {
+        val content = String(Files.readAllBytes(Paths.get(step)))
+        val toolResult = gson.fromJson(content, ToolResultsStep::class.java)
 
-        val failedStep = "./tool_results_step_matrix-14eytaygn7sis_fail.json"
-//        val successStep = "./tool_results_stepmatrix-3l3x3k8qzjmgg_success.json"
-        val content = String(Files.readAllBytes(Paths.get(failedStep)))
-        val toolResult = gson.fromJson<ToolResultsStep>(content, ToolResultsStep::class.java)
         val tests = GcToolResults.listTestCases(toolResult)
         val result = GcToolResults.getResults(toolResult)
 
-        // todo: handle multiple overviews
+        // todo: handle multiple overviews (iOS only)
         val overview = result.testExecutionStep.testSuiteOverviews.first()
+        println("overview name: ${overview.name}") // null on android
 
         val testCases = mutableListOf<JUnitTestCase>()
         tests.testCases.forEach { testCase ->
+            // null on android, EarlGreyExampleSwiftTests on iOS
+            println("test case suite name: ${testCase.testCaseReference.testSuiteName}")
             // TODO: time doesn't match real JUnit XML
             var failures: List<String>? = null
             var errors: List<String>? = null
@@ -70,13 +108,15 @@ object Tmp {
                 "failed" -> failures = testCase.stackTraces?.map { it.exception }
                 "passed" -> {
                 }
-                null -> {} // null status == passed
+                null -> {
+                } // null status == passed
                 "skipped" -> skipped = null
                 else -> throw RuntimeException("Unknown TestCase status ${testCase.status}")
             }
 
-            // manually divide to keep fractional precision
-            val timeSeconds = (testCase.endTime.nanos - testCase.startTime.nanos) / 1E9
+            // TODO: On iOS testCase.endTime & testCase.startTime are always null.
+            // TODO: On iOS map test cases back to test suites testCase.testCaseReference.testSuiteName
+            val timeSeconds = nanosToSeconds(testCase.endTime.nanos - testCase.startTime.nanos)
             testCases.add(
                 JUnitTestCase(
                     name = testCase.testCaseReference.name,
@@ -95,7 +135,7 @@ object Tmp {
             failures = overview.failures(),
             errors = overview.errors(),
             skipped = overview.skipped(),
-            time = "",
+            time = overview.time(),
             timestamp = "",
             hostname = "localhost",
             testcases = testCases