Smart Flank (#385)

* Smart Flank * SmartFlank algo (#397) * SmartFlank * SmarkFlank code review * Fix iOS tests, add testsAlwaysRun * Print cache hit rate and shard times * Lint fix * Round times when printing to stdout * Fix tests * Fixed int division * iOS now working * Fix iOS sharding * Update catalog fixtures * Fix ShardTest * Add performance test * Rename junitGcsPath to smartFlankGcsPath * Fix lint issues * Update release_notes.md * Continue on failed visit file
Flank · Nov 27, 2018 · fcbaf02 · fcbaf02
1 parent bd8dd92
commit fcbaf02
Show file tree

Hide file tree

Showing 34 changed files with 801 additions and 317 deletions.
diff --git a/release_notes.md b/release_notes.md
@@ -1,6 +1,7 @@
 ## v4.1 (unreleased)
 - `app`, `test`, and `xctestrun-file` now support `~`, environment variables, and globs (`*`, `**`) when resolving paths. [#386](https://github.com/TestArmada/flank/pull/386)
 - Update `flank android run` to support `--app`, `--test`, `--test-targets`, `--use-orchestrator` and `--no-use-orchestrator`.
+- Add `smartFlankGcsPath` to shard iOS and Android tests by time using historical run data. The amount of shards used is set by `testShards`.
 
 ## v4.0.0
 

diff --git a/test_runner/build.gradle.kts b/test_runner/build.gradle.kts
@@ -38,7 +38,9 @@ tasks.withType<JacocoReport> {
 }
 
 tasks.withType<KotlinCompile>().configureEach {
-    kotlinOptions.allWarningsAsErrors = true
+    // https://devcenter.bitrise.io/builds/available-environment-variables/
+    val runningOnBitrise = System.getenv("BITRISE_IO") != null
+    kotlinOptions.allWarningsAsErrors = runningOnBitrise
 }
 
 apply {

diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml
@@ -11,24 +11,33 @@ gcloud:
   # test and xctestrun-file are the only required args
   test: ./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExample.zip
   xctestrun-file: ./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun
-  xcode-version: 9.2
+  xcode-version: 10.1
   device:
     - model: iphone8
       version: 11.2
       locale: en
       orientation: portrait
 
 flank:
+  # # Google cloud storage path to store the JUnit XML results from the last run.
+  #
+  # smartFlankGcsPath: gs://tmp_flank/flank/test_app_ios.xml
+
   # test shards - the amount of groups to split the test suite into
   # set to -1 to use one shard per test.
-  testShards: 1
+  testShards: 2
+
   # repeat tests - the amount of times to run the tests.
   # 1 runs the tests once. 10 runs all the tests 10x
   repeatTests: 1
-  # always run - these tests are inserted at the beginning of every shard
-  # useful if you need to grant permissions or login before other tests run
+
+  # # always run - these tests are inserted at the beginning of every shard
+  # # useful if you need to grant permissions or login before other tests run
+  #
   # test-targets-always-run:
   #   - a/testGrantPermissions
-  # test targets - a list of tests to run. omit to run all tests.
+
+  # # test targets - a list of tests to run. omit to run all tests.
+  #
   # test-targets:
   #   - b/testBasicSelection
diff --git a/test_runner/flank.yml b/test_runner/flank.yml
@@ -27,13 +27,22 @@ gcloud:
       version: 28
 
 flank:
+  # # Google cloud storage path to store the JUnit XML results from the last run.
+  #
+  # smartFlankGcsPath: gs://tmp_flank/flank/test_app_android.xml
+
   # test shards - the amount of groups to split the test suite into
   # set to -1 to use one shard per test.
+  #
   testShards: 1
+
   # repeat tests - the amount of times to run the tests.
   # 1 runs the tests once. 10 runs all the tests 10x
+  #
   repeatTests: 1
-  # always run - these tests are inserted at the beginning of every shard
-  # useful if you need to grant permissions or login before other tests run
+
+  # # always run - these tests are inserted at the beginning of every shard
+  # # useful if you need to grant permissions or login before other tests run
+  #
   # test-targets-always-run:
-    # - class com.example.app.ExampleUiTest#testPasses
+  #   - class com.example.app.ExampleUiTest#testPasses
diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt
@@ -10,8 +10,9 @@ import ftl.android.UnsupportedVersionId
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
 import ftl.args.ArgsHelper.calculateShards
+import ftl.args.ArgsHelper.createGcsBucket
+import ftl.args.ArgsHelper.createJunitBucket
 import ftl.args.ArgsHelper.evaluateFilePath
-import ftl.args.ArgsHelper.getGcsBucket
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.yamlMapper
 import ftl.args.ArgsToString.devicesToString
@@ -63,6 +64,7 @@ class AndroidArgs(
     private val flank = flankYml.flank
     override val testShards = flank.testShards
     override val repeatTests = flank.repeatTests
+    override val smartFlankGcsPath = flank.smartFlankGcsPath
     override val testTargetsAlwaysRun = flank.testTargetsAlwaysRun
 
     // computed properties not specified in yaml
@@ -76,16 +78,12 @@ class AndroidArgs(
         }
 
         val filteredTests = getTestMethods(testLocalApk)
-
-        calculateShards(
-            filteredTests,
-            testTargetsAlwaysRun,
-            testShards
-            )
+        calculateShards(filteredTests, this)
     }
 
     init {
-        resultsBucket = getGcsBucket(projectId, gcloud.resultsBucket)
+        resultsBucket = createGcsBucket(projectId, gcloud.resultsBucket)
+        createJunitBucket(projectId, flank.smartFlankGcsPath)
 
         if (appApk.startsWith(FtlConstants.GCS_PREFIX)) {
             assertGcsFileExists(appApk)
@@ -110,6 +108,7 @@ class AndroidArgs(
         val testFilter = TestFilters.fromTestTargets(testTargets)
         val filteredTests = allTestMethods
             .asSequence()
+            .distinct()
             .filter(testFilter.shouldRun)
             .map(TestMethod::testName)
             .map { "class $it" }
@@ -157,6 +156,7 @@ ${devicesToString(devices)}
     flank:
       testShards: $testShards
       repeatTests: $repeatTests
+      smartFlankGcsPath: $smartFlankGcsPath
       test-targets-always-run:
 ${listToString(testTargetsAlwaysRun)}
    """.trimIndent()
@@ -167,7 +167,8 @@ ${listToString(testTargetsAlwaysRun)}
             mergeYmlMaps(GcloudYml, AndroidGcloudYml, FlankYml)
         }
 
-        fun load(data: Path, cli: AndroidRunCommand = AndroidRunCommand()): AndroidArgs = load(String(Files.readAllBytes(data)), cli)
+        fun load(data: Path, cli: AndroidRunCommand = AndroidRunCommand()): AndroidArgs =
+            load(String(Files.readAllBytes(data)), cli)
 
         fun load(data: String, cli: AndroidRunCommand = AndroidRunCommand()): AndroidArgs {
             val flankYml = yamlMapper.readValue(data, FlankYml::class.java)

diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsFileVisitor.kt b/test_runner/src/main/kotlin/ftl/args/ArgsFileVisitor.kt
@@ -1,6 +1,5 @@
 package ftl.args
 
-import ftl.util.Utils.fatalError
 import java.io.IOException
 import java.nio.file.FileSystems
 import java.nio.file.FileVisitOption
@@ -26,7 +25,8 @@ class ArgsFileVisitor(glob: String) : SimpleFileVisitor<Path>() {
     }
 
     override fun visitFileFailed(file: Path?, exc: IOException?): FileVisitResult {
-        fatalError("Failed to visit $file $exc")
+        // java.nio.file.AccessDeniedException: /tmp/systemd-private-2bc4cd4c824142ab95fb18cbb14165f5-systemd-timesyncd.service-epYUoK
+        System.err.println("Failed to visit $file ${exc?.message}")
         return FileVisitResult.CONTINUE
     }
 

diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt
@@ -11,16 +11,18 @@ import com.google.cloud.storage.BucketInfo
 import com.google.cloud.storage.Storage
 import com.google.cloud.storage.StorageClass
 import com.google.cloud.storage.StorageOptions
-import com.google.common.math.IntMath
 import ftl.args.yml.IYmlMap
 import ftl.config.FtlConstants
 import ftl.config.FtlConstants.GCS_PREFIX
 import ftl.config.FtlConstants.JSON_FACTORY
 import ftl.config.FtlConstants.defaultCredentialPath
 import ftl.gc.GcStorage
+import ftl.reports.xml.model.JUnitTestResult
+import ftl.shard.Shard
+import ftl.shard.StringShards
+import ftl.shard.stringShards
 import ftl.util.Utils
 import java.io.File
-import java.math.RoundingMode
 import java.net.URI
 import java.nio.file.Files
 import java.nio.file.Path
@@ -95,54 +97,31 @@ object ArgsHelper {
         if (validTestMethods.isEmpty()) Utils.fatalError("$from has no tests")
     }
 
-    fun calculateShards(
-        testMethodsToShard: Collection<String>,
-        testMethodsAlwaysRun: Collection<String>,
-        testShards: Int
-    ): List<List<String>> {
-        val testShardMethods = testMethodsToShard.distinct().toMutableList()
-        testShardMethods.removeAll(testMethodsAlwaysRun)
-
-        val oneTestPerChunk = testShards == -1
-        var chunkSize = IntMath.divide(testShardMethods.size, testShards, RoundingMode.UP)
-
-        if (oneTestPerChunk || chunkSize < 1) {
-            chunkSize = 1
-        }
-
-        val testShardChunks = testShardMethods.asSequence()
-            .chunked(chunkSize)
-            .map { testMethodsAlwaysRun + it }
-            .toList()
-
-        // Ensure we don't create more VMs than requested. VM count per run should be <= testShards
-        if (!oneTestPerChunk && testShardChunks.size > testShards) {
-            Utils.fatalError("Calculated chunks $testShardChunks is > requested $testShards testShards.")
-        }
-        if (testShardChunks.isEmpty()) Utils.fatalError("Failed to populate test shard chunks")
-
-        return testShardChunks
+    fun createJunitBucket(projectId: String, junitGcsPath: String) {
+        if (FtlConstants.useMock || junitGcsPath.isEmpty()) return
+        val bucket = junitGcsPath.drop(GCS_PREFIX.length).substringBefore('/')
+            createGcsBucket(projectId, bucket)
     }
 
-    fun getGcsBucket(projectId: String, resultsBucket: String): String {
+    fun createGcsBucket(projectId: String, bucket: String): String {
         // com.google.cloud.storage.contrib.nio.testing.FakeStorageRpc doesn't support list
         // when testing, use a hard coded results bucket instead.
-        if (FtlConstants.useMock) return resultsBucket
+        if (FtlConstants.useMock) return bucket
         // test lab supports using a special free storage bucket
         // because we don't have access to the root account, it won't show up in the storage list.
-        if (resultsBucket.startsWith("test-lab-")) return resultsBucket
+        if (bucket.startsWith("test-lab-")) return bucket
 
         val storage = StorageOptions.newBuilder().setProjectId(projectId).build().service
         val bucketLabel = mapOf(Pair("flank", ""))
         val storageLocation = "us-central1"
 
-        val bucketListOption = Storage.BucketListOption.prefix(resultsBucket)
+        val bucketListOption = Storage.BucketListOption.prefix(bucket)
         val storageList = storage.list(bucketListOption).values?.map { it.name } ?: emptyList()
-        val bucket = storageList.find { it == resultsBucket }
-        if (bucket != null) return bucket
+        val targetBucket = storageList.find { it == bucket }
+        if (targetBucket != null) return targetBucket
 
         return storage.create(
-            BucketInfo.newBuilder(resultsBucket)
+            BucketInfo.newBuilder(targetBucket)
                 .setStorageClass(StorageClass.REGIONAL)
                 .setLocation(storageLocation)
                 .setLabels(bucketLabel)
@@ -176,6 +155,7 @@ object ArgsHelper {
 
     // https://stackoverflow.com/a/2821201/2450315
     private val envRegex = Pattern.compile("\\$([a-zA-Z_]+[a-zA-Z0-9_]*)")
+
     private fun evaluateEnvVars(text: String): String {
         val buffer = StringBuffer()
         val matcher = envRegex.matcher(text)
@@ -196,4 +176,22 @@ object ArgsHelper {
 
         return ArgsFileVisitor("glob:$filePath").walk(searchDir)
     }
+
+    fun calculateShards(filteredTests: List<String>, args: IArgs): List<List<String>> {
+        val oldTestResult = GcStorage.downloadJunitXml(args) ?: JUnitTestResult(mutableListOf())
+        val shardsByTime = Shard.calculateShardsByTime(filteredTests, oldTestResult, args)
+
+        return testMethodsAlwaysRun(shardsByTime.stringShards(), args)
+    }
+
+    private fun testMethodsAlwaysRun(shards: StringShards, args: IArgs): StringShards {
+        val alwaysRun = args.testTargetsAlwaysRun
+
+        shards.forEach { shard ->
+            shard.removeAll(alwaysRun)
+            shard.addAll(0, alwaysRun)
+        }
+
+        return shards
+    }
 }
diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt
@@ -15,6 +15,7 @@ interface IArgs {
     // FlankYml
     val testShards: Int
     val repeatTests: Int
+    val smartFlankGcsPath: String
     val testTargetsAlwaysRun: List<String>
 
     // computed property

diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt
@@ -2,6 +2,8 @@ package ftl.args
 
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
+import ftl.args.ArgsHelper.createGcsBucket
+import ftl.args.ArgsHelper.createJunitBucket
 import ftl.args.ArgsHelper.evaluateFilePath
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.validateTestMethods
@@ -29,7 +31,7 @@ class IosArgs(
 ) : IArgs {
 
     private val gcloud = gcloudYml.gcloud
-    override val resultsBucket = gcloud.resultsBucket
+    override val resultsBucket: String
     override val recordVideo = gcloud.recordVideo
     override val testTimeout = gcloud.timeout
     override val async = gcloud.async
@@ -45,6 +47,7 @@ class IosArgs(
     private val flank = flankYml.flank
     override val testShards = flank.testShards
     override val repeatTests = flank.repeatTests
+    override val smartFlankGcsPath = flank.smartFlankGcsPath
     override val testTargetsAlwaysRun = flank.testTargetsAlwaysRun
 
     private val iosFlank = iosFlankYml.flank
@@ -58,16 +61,15 @@ class IosArgs(
             validTestMethods
         } else {
             testTargets
-        }
+        }.distinct()
 
-        ArgsHelper.calculateShards(
-            testMethodsToShard = testsToShard,
-            testMethodsAlwaysRun = testTargetsAlwaysRun,
-            testShards = testShards
-            )
+        ArgsHelper.calculateShards(testsToShard, this)
     }
 
     init {
+        resultsBucket = createGcsBucket(projectId, gcloud.resultsBucket)
+        createJunitBucket(projectId, flank.smartFlankGcsPath)
+
         if (xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) {
             assertGcsFileExists(xctestrunZip)
         } else {
@@ -114,6 +116,7 @@ ${devicesToString(devices)}
     flank:
       testShards: $testShards
       repeatTests: $repeatTests
+      smartFlankGcsPath: $smartFlankGcsPath
       test-targets-always-run:
 ${listToString(testTargetsAlwaysRun)}
       # iOS flank

diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt
@@ -2,24 +2,35 @@ package ftl.args.yml
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
+import ftl.config.FtlConstants.GCS_PREFIX
 import ftl.util.Utils.fatalError
 
 /** Flank specific parameters for both iOS and Android */
 @JsonIgnoreProperties(ignoreUnknown = true)
 class FlankYmlParams(
     val testShards: Int = 1,
     val repeatTests: Int = 1,
+    val smartFlankGcsPath: String = "",
 
     @field:JsonProperty("test-targets-always-run")
     val testTargetsAlwaysRun: List<String> = emptyList()
 ) {
     companion object : IYmlKeys {
-        override val keys = listOf("testShards", "repeatTests", "test-targets-always-run")
+        override val keys = listOf("testShards", "repeatTests", "smartFlankGcsPath", "test-targets-always-run")
     }
 
     init {
         if (testShards <= 0 && testShards != -1) fatalError("testShards must be >= 1 or -1")
         if (repeatTests < 1) fatalError("repeatTests must be >= 1")
+
+        if (smartFlankGcsPath.isNotEmpty()) {
+            if (!smartFlankGcsPath.startsWith(GCS_PREFIX)) {
+                fatalError("smartFlankGcsPath must start with gs://")
+            }
+            if (smartFlankGcsPath.count { it == '/' } <= 2 || !smartFlankGcsPath.endsWith(".xml")) {
+                fatalError("smartFlankGcsPath must be in the format gs://bucket/foo.xml")
+            }
+        }
     }
 }