Add support for additional-apks option

release_notes.md

@@ -1,4 +1,5 @@
 ## next (unreleased)
+- [#695](https://github.com/Flank/flank/pull/695) Add support for additional-apks option. ([jan-gogo](https://github.com/jan-gogo))
 - [#683](https://github.com/Flank/flank/pull/683) Print web link. ([pawelpasterz](https://github.com/pawelpasterz))
 - [#692](https://github.com/Flank/flank/pull/692) Add support for network-profiles list command & --network-profile option. ([jan-gogo](https://github.com/jan-gogo))
 - [#689](https://github.com/Flank/flank/pull/689) Add support for client-details option. ([jan-gogo](https://github.com/jan-gogo))

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -20,6 +20,7 @@ import ftl.args.ArgsToString.mapToString
 import ftl.args.yml.AndroidFlankYml
 import ftl.args.yml.AndroidGcloudYml
 import ftl.args.yml.AndroidGcloudYmlParams
+import ftl.args.yml.AppTestPair
 import ftl.args.yml.FlankYml
 import ftl.args.yml.GcloudYml
 import ftl.args.yml.YamlDeprecated
@@ -49,8 +50,9 @@ class AndroidArgs(
     override val flakyTestAttempts = cli?.flakyTestAttempts ?: gcloud.flakyTestAttempts
 
     private val androidGcloud = androidGcloudYml.gcloud
-    var appApk = cli?.app ?: androidGcloud.app ?: fatalError("app is not set")
-    var testApk = cli?.test ?: androidGcloud.test ?: fatalError("test is not set")
+    val appApk = (cli?.app ?: androidGcloud.app ?: fatalError("app is not set")).processApkPath("from app")
+    val testApk = (cli?.test ?: androidGcloud.test ?: fatalError("test is not set")).processApkPath("from test")
+    val additionalApks = (cli?.additionalApks ?: androidGcloud.additionalApks).map { it.processApkPath("from additional-apks") }
     val autoGoogleLogin = cli?.autoGoogleLogin ?: cli?.noAutoGoogleLogin?.not() ?: androidGcloud.autoGoogleLogin
 
     // We use not() on noUseOrchestrator because if the flag is on, useOrchestrator needs to be false
@@ -79,27 +81,18 @@ class AndroidArgs(
     override val networkProfile = cli?.networkProfile ?: gcloud.networkProfile
 
     private val androidFlank = androidFlankYml.flank
-    val additionalAppTestApks = cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks
+    val additionalAppTestApks = (cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks).map { (app, test) ->
+        AppTestPair(
+            app = app?.processApkPath("from additional-app-test-apks.app"),
+            test = test.processApkPath("from additional-app-test-apks.test")
+        )
+    }
     val keepFilePath = cli?.keepFilePath ?: androidFlank.keepFilePath
 
     init {
         resultsBucket = createGcsBucket(project, cli?.resultsBucket ?: gcloud.resultsBucket)
         createJunitBucket(project, flank.smartFlankGcsPath)
 
-        if (appApk.startsWith(FtlConstants.GCS_PREFIX)) {
-            assertGcsFileExists(appApk)
-        } else {
-            appApk = evaluateFilePath(appApk)
-            assertFileExists(appApk, "appApk")
-        }
-
-        if (testApk.startsWith(FtlConstants.GCS_PREFIX)) {
-            assertGcsFileExists(testApk)
-        } else {
-            testApk = evaluateFilePath(testApk)
-            assertFileExists(testApk, "testApk")
-        }
-
         devices.forEach { device -> assertDeviceSupported(device) }
 
         assertCommonProps(this)
@@ -131,6 +124,7 @@ AndroidArgs
       # Android gcloud
       app: $appApk
       test: $testApk
+      additional-apks: ${listToString(additionalApks)}
       auto-google-login: $autoGoogleLogin
       use-orchestrator: $useOrchestrator
       directories-to-pull:${listToString(directoriesToPull)}
@@ -196,3 +190,8 @@ AndroidArgs
         }
     }
 }
+
+private fun String.processApkPath(name: String): String =
+    if (startsWith(FtlConstants.GCS_PREFIX))
+        this.also { assertGcsFileExists(it) } else
+        evaluateFilePath(this).also { assertFileExists(it, name) }

test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt

@@ -8,14 +8,16 @@ data class AppTestPair(
     val test: String
 )
 
-data class ResolvedTestPair(
+data class ResolvedTestApks(
     val app: String,
-    val test: String
+    val test: String,
+    val additionalApks: List<String> = emptyList()
 )
 
-data class UploadedTestPair(
+data class UploadedTestApks(
     val app: String,
-    val test: String
+    val test: String,
+    val additionalApks: List<String> = emptyList()
 )
 
 /** Flank specific parameters for Android */

test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt

@@ -17,6 +17,9 @@ class AndroidGcloudYmlParams(
     val app: String? = null,
     val test: String? = null,
 
+    @field:JsonProperty("additional-apks")
+    val additionalApks: List<String> = emptyList(),
+
     @field:JsonProperty("auto-google-login")
     val autoGoogleLogin: Boolean = FlankDefaults.DISABLE_AUTO_LOGIN,
 
@@ -42,8 +45,17 @@ class AndroidGcloudYmlParams(
 ) {
     companion object : IYmlKeys {
         override val keys = listOf(
-            "app", "test", "auto-google-login", "use-orchestrator", "environment-variables",
-            "directories-to-pull", "performance-metrics", "test-runner-class", "test-targets", "device"
+            "app",
+            "test",
+            "additional-apks",
+            "auto-google-login",
+            "use-orchestrator",
+            "environment-variables",
+            "directories-to-pull",
+            "performance-metrics",
+            "test-runner-class",
+            "test-targets",
+            "device"
         )
     }
 }

test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt

@@ -92,6 +92,16 @@ class AndroidRunCommand : CommonRunCommand(), Runnable {
     )
     var test: String? = null
 
+    @Option(
+        names = ["--additional-apks"],
+        split = ",",
+        description = [
+            "A list of up to 100 additional APKs to install, in addition to those being directly tested.",
+            "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. "
+        ]
+    )
+    var additionalApks: List<String>? = null
+
     @Option(
         names = ["--auto-google-login"],
         description = ["Automatically log into the test device using a preconfigured " +

test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt

@@ -4,6 +4,7 @@ import com.google.api.services.testing.Testing
 import com.google.api.services.testing.model.Account
 import com.google.api.services.testing.model.AndroidDeviceList
 import com.google.api.services.testing.model.AndroidInstrumentationTest
+import com.google.api.services.testing.model.Apk
 import com.google.api.services.testing.model.ClientInfo
 import com.google.api.services.testing.model.EnvironmentMatrix
 import com.google.api.services.testing.model.EnvironmentVariable
@@ -38,7 +39,8 @@ object GcAndroidTestMatrix {
         androidDeviceList: AndroidDeviceList,
         testShards: ShardChunks,
         args: AndroidArgs,
-        toolResultsHistory: ToolResultsHistory
+        toolResultsHistory: ToolResultsHistory,
+        additionalApkGcsPaths: List<String>
     ): Testing.Projects.TestMatrices.Create {
 
         // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120
@@ -81,6 +83,7 @@ object GcAndroidTestMatrix {
             .setAccount(account)
             .setNetworkProfile(args.networkProfile)
             .setDirectoriesToPull(args.directoriesToPull)
+            .setAdditionalApks(additionalApkGcsPaths.mapGcsPathsToApks())
 
         if (args.environmentVariables.isNotEmpty()) {
             testSetup.environmentVariables =
@@ -118,3 +121,7 @@ object GcAndroidTestMatrix {
         throw RuntimeException("Failed to create test matrix")
     }
 }
+
+private fun List<String>?.mapGcsPathsToApks(): List<Apk>? = this
+    ?.takeIf { it.isNotEmpty() }
+    ?.map { gcsPath -> Apk().setLocation(FileReference().setGcsPath(gcsPath)) }

test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt

@@ -1,17 +1,18 @@
 package ftl.run.platform
 
+import com.google.api.services.testing.model.AndroidDeviceList
 import com.google.api.services.testing.model.TestMatrix
+import com.google.api.services.testing.model.ToolResultsHistory
 import ftl.args.AndroidArgs
 import ftl.args.AndroidTestShard
-import ftl.args.yml.AppTestPair
+import ftl.args.ShardChunks
+import ftl.args.yml.ResolvedTestApks
+import ftl.args.yml.UploadedTestApks
 import ftl.gc.GcAndroidDevice
 import ftl.gc.GcAndroidTestMatrix
 import ftl.gc.GcStorage
 import ftl.gc.GcToolResults
 import ftl.http.executeWithRetry
-import ftl.args.ShardChunks
-import ftl.args.yml.ResolvedTestPair
-import ftl.args.yml.UploadedTestPair
 import ftl.run.model.TestResult
 import ftl.run.platform.common.afterRunTests
 import ftl.run.platform.common.beforeRunMessage
@@ -29,60 +30,96 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS
     // GcAndroidTestMatrix.execute() 3x retry => matrix id (string)
     val androidDeviceList = GcAndroidDevice.build(args.devices)
 
-    val jobs = arrayListOf<Deferred<TestMatrix>>()
+    val testMatrices = arrayListOf<Deferred<TestMatrix>>()
     val runCount = args.repeatTests
     val history = GcToolResults.createToolResultsHistory(args)
-    val apkPairsInArgs = listOf(AppTestPair(app = args.appApk, test = args.testApk)) + args.additionalAppTestApks
-    val allTestShardChunks: ShardChunks = apkPairsInArgs
-        .provideMissingApps(withFallbackApp = args.appApk)
-        .map { resolvedApkPair ->
-            val uploadedApkPair = resolveApkPair(resolvedApkPair, args, runGcsPath)
-            // Ensure we only shard tests that are part of the test apk. Use the resolved test apk path to make sure
-            // we don't re-download an apk it is on the local file system.
-            AndroidTestShard.getTestShardChunks(args, resolvedApkPair.test).also { testShards ->
-                repeat(runCount) {
-                    // specify dispatcher to avoid inheriting main runBlocking context that runs in the main thread
-                    // https://kotlinlang.org/docs/reference/coroutines/coroutine-context-and-dispatchers.html
-                    jobs += async(Dispatchers.IO) {
-                        GcAndroidTestMatrix.build(
-                            appApkGcsPath = uploadedApkPair.app,
-                            testApkGcsPath = uploadedApkPair.test,
-                            runGcsPath = runGcsPath,
-                            androidDeviceList = androidDeviceList,
-                            testShards = testShards,
-                            args = args,
-                            toolResultsHistory = history
-                        ).executeWithRetry()
-                    }
-                }
-            }
-        }.flatten()
+    val resolvedTestApks = args.getResolvedTestApks()
+
+    val allTestShardChunks: ShardChunks = resolvedTestApks.map { apks: ResolvedTestApks ->
+        // Ensure we only shard tests that are part of the test apk. Use the resolved test apk path to make sure
+        // we don't re-download an apk it is on the local file system.
+        AndroidTestShard.getTestShardChunks(args, apks.test).also { testShards ->
+            testMatrices += executeAndroidTestMatrix(
+                uploadedTestApks = uploadTestApks(
+                    apks = apks,
+                    args = args,
+                    runGcsPath = runGcsPath
+                ),
+                runGcsPath = runGcsPath,
+                androidDeviceList = androidDeviceList,
+                testShards = testShards,
+                args = args,
+                history = history,
+                runCount = runCount
+            )
+        }
+    }.flatten()
 
     println(beforeRunMessage(args, allTestShardChunks))
-    val matrixMap = afterRunTests(jobs.awaitAll(), runGcsPath, stopwatch, args)
+    val matrixMap = afterRunTests(testMatrices.awaitAll(), runGcsPath, stopwatch, args)
     matrixMap to allTestShardChunks
 }
 
-private fun List<AppTestPair>.provideMissingApps(withFallbackApp: String) =
-    map { ResolvedTestPair(app = it.app ?: withFallbackApp, test = it.test) }
+private fun AndroidArgs.getResolvedTestApks() = listOf(
+    element = ResolvedTestApks(
+        app = appApk,
+        test = testApk,
+        additionalApks = additionalApks
+    )
+).plus(
+    elements = additionalAppTestApks.map {
+        ResolvedTestApks(
+            app = it.app ?: appApk,
+            test = it.test,
+            additionalApks = additionalApks
+        )
+    }
+)
+
+private suspend fun executeAndroidTestMatrix(
+    runGcsPath: String,
+    args: AndroidArgs,
+    testShards: ShardChunks,
+    uploadedTestApks: UploadedTestApks,
+    androidDeviceList: AndroidDeviceList,
+    history: ToolResultsHistory,
+    runCount: Int
+): List<Deferred<TestMatrix>> = coroutineScope {
+    (0 until runCount).map {
+        async(Dispatchers.IO) {
+            GcAndroidTestMatrix.build(
+                appApkGcsPath = uploadedTestApks.app,
+                testApkGcsPath = uploadedTestApks.test,
+                runGcsPath = runGcsPath,
+                androidDeviceList = androidDeviceList,
+                testShards = testShards,
+                args = args,
+                toolResultsHistory = history,
+                additionalApkGcsPaths = uploadedTestApks.additionalApks
+            ).executeWithRetry()
+        }
+    }
+}
 
 /**
  * Upload an APK pair if the path given is local
  *
  * @return AppTestPair with their GCS paths
  */
-private suspend fun resolveApkPair(
-    apk: ResolvedTestPair,
+private suspend fun uploadTestApks(
+    apks: ResolvedTestApks,
     args: AndroidArgs,
     runGcsPath: String
-): UploadedTestPair = coroutineScope {
+): UploadedTestApks = coroutineScope {
     val gcsBucket = args.resultsBucket
 
-    val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apk.app, gcsBucket, runGcsPath) }
-    val testApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apk.test, gcsBucket, runGcsPath) }
+    val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.app, gcsBucket, runGcsPath) }
+    val testApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.test, gcsBucket, runGcsPath) }
+    val additionalApkGcsPaths = apks.additionalApks.map { async(Dispatchers.IO) { GcStorage.upload(it, gcsBucket, runGcsPath) } }
 
-    UploadedTestPair(
+    UploadedTestApks(
         app = appApkGcsPath.await(),
-        test = testApkGcsPath.await()
+        test = testApkGcsPath.await(),
+        additionalApks = additionalApkGcsPaths.awaitAll()
     )
 }

test_runner/src/test/kotlin/Debug.kt

@@ -10,8 +10,8 @@ fun main() {
     // run "gradle check" to generate required fixtures
     val projectId = System.getenv("FLANK_PROJECT_ID")
         ?: "YOUR PROJECT ID"
-    val quantity = "single"
-    val type = "success"
+    val quantity = "multiple"
+    val type = "apk"
 
     // Bugsnag keeps the process alive so we must call exitProcess
     // https://github.com/bugsnag/bugsnag-java/issues/151
@@ -20,7 +20,7 @@ fun main() {
 //            "--debug",
             "firebase", "test",
             "android", "run",
-//            "--dry",
+            "--dry",
             "-c=src/test/kotlin/ftl/fixtures/test_app_cases/flank-$quantity-$type.yml",
             "--project=$projectId",
             "--client-details=key1=value1,key2=value2"