From 6a5f0d7b3a41874a779a755620820a1a2054a5e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 2 Apr 2020 17:01:12 +0200 Subject: [PATCH 1/5] Add support for additional-apks option --- .../src/main/kotlin/ftl/args/AndroidArgs.kt | 33 +++--- .../main/kotlin/ftl/args/AndroidTestShard.kt | 12 ++ .../kotlin/ftl/args/yml/AndroidFlankYml.kt | 10 +- .../kotlin/ftl/args/yml/AndroidGcloudYml.kt | 16 ++- .../test/android/AndroidRunCommand.kt | 10 ++ .../main/kotlin/ftl/gc/GcAndroidTestMatrix.kt | 9 +- .../reports/api/CreateTestExecutionData.kt | 6 +- .../kotlin/ftl/reports/api/ProcessFromApi.kt | 2 +- .../ftl/run/platform/RunAndroidTests.kt | 110 ++++++++++++------ test_runner/src/test/kotlin/Debug.kt | 6 +- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 38 ++++-- .../test/android/AndroidRunCommandTest.kt | 8 ++ .../test_app_cases/flank-multiple-apk.yml | 6 +- .../kotlin/ftl/gc/GcAndroidTestMatrixTest.kt | 9 +- 14 files changed, 189 insertions(+), 86 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 1fa869b9e5..6f23e4d997 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -20,6 +20,7 @@ import ftl.args.ArgsToString.mapToString import ftl.args.yml.AndroidFlankYml import ftl.args.yml.AndroidGcloudYml import ftl.args.yml.AndroidGcloudYmlParams +import ftl.args.yml.AppTestPair import ftl.args.yml.FlankYml import ftl.args.yml.GcloudYml import ftl.args.yml.YamlDeprecated @@ -49,8 +50,9 @@ class AndroidArgs( override val flakyTestAttempts = cli?.flakyTestAttempts ?: gcloud.flakyTestAttempts private val androidGcloud = androidGcloudYml.gcloud - var appApk = cli?.app ?: androidGcloud.app ?: fatalError("app is not set") - var testApk = cli?.test ?: androidGcloud.test ?: fatalError("test is not set") + val appApk = (cli?.app ?: androidGcloud.app ?: fatalError("app is not set")).processApkPath("from app") + val testApk = (cli?.test ?: androidGcloud.test ?: fatalError("test is not set")).processApkPath("from test") + val additionalTestApks = (cli?.additionalApks ?: androidGcloud.additionalApks).map { it.processApkPath("from additional-apks") } val autoGoogleLogin = cli?.autoGoogleLogin ?: cli?.noAutoGoogleLogin?.not() ?: androidGcloud.autoGoogleLogin // We use not() on noUseOrchestrator because if the flag is on, useOrchestrator needs to be false @@ -79,27 +81,18 @@ class AndroidArgs( override val networkProfile = cli?.networkProfile ?: gcloud.networkProfile private val androidFlank = androidFlankYml.flank - val additionalAppTestApks = cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks + val additionalAppTestApks = (cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks).map { (app, test) -> + AppTestPair( + app = app?.processApkPath("from additional-app-test-apks.app"), + test = test.processApkPath("from additional-app-test-apks.test") + ) + } val keepFilePath = cli?.keepFilePath ?: androidFlank.keepFilePath init { resultsBucket = createGcsBucket(project, cli?.resultsBucket ?: gcloud.resultsBucket) createJunitBucket(project, flank.smartFlankGcsPath) - if (appApk.startsWith(FtlConstants.GCS_PREFIX)) { - assertGcsFileExists(appApk) - } else { - appApk = evaluateFilePath(appApk) - assertFileExists(appApk, "appApk") - } - - if (testApk.startsWith(FtlConstants.GCS_PREFIX)) { - assertGcsFileExists(testApk) - } else { - testApk = evaluateFilePath(testApk) - assertFileExists(testApk, "testApk") - } - devices.forEach { device -> assertDeviceSupported(device) } assertCommonProps(this) @@ -131,6 +124,7 @@ AndroidArgs # Android gcloud app: $appApk test: $testApk + additional-apks: ${listToString(additionalTestApks)} auto-google-login: $autoGoogleLogin use-orchestrator: $useOrchestrator directories-to-pull:${listToString(directoriesToPull)} @@ -196,3 +190,8 @@ AndroidArgs } } } + +private fun String.processApkPath(name: String): String = + if (startsWith(FtlConstants.GCS_PREFIX)) + this.also { assertGcsFileExists(it) } else + evaluateFilePath(this).also { assertFileExists(it, name) } diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt b/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt index 68092f2a7f..7ff94f33b5 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt @@ -2,6 +2,7 @@ package ftl.args import com.linkedin.dex.parser.DexParser import com.linkedin.dex.parser.TestMethod +import ftl.args.yml.ResolvedTestApks import ftl.config.FtlConstants import ftl.filter.TestFilter import ftl.filter.TestFilters @@ -11,6 +12,17 @@ import ftl.util.fatalError object AndroidTestShard { + fun getTestShardChunks(args: AndroidArgs, apks: ResolvedTestApks): ShardChunks = + sequenceOf(apks.test).plus(apks.additionalTests).map { apk -> + // Download test APK if necessary so it can be used to validate test methods + if (!apk.startsWith(FtlConstants.GCS_PREFIX)) apk + else GcStorage.download(apk) + }.map { testLocalApk: String -> + getTestMethods(args, testLocalApk) + }.map { filteredTests: List -> + ArgsHelper.calculateShards(filteredTests, args) + }.toList().flatten() + // computed properties not specified in yaml fun getTestShardChunks(args: AndroidArgs, testApk: String): ShardChunks { // Download test APK if necessary so it can be used to validate test methods diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt index fd2804d5c1..344b404ffa 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt @@ -8,14 +8,16 @@ data class AppTestPair( val test: String ) -data class ResolvedTestPair( +data class ResolvedTestApks( val app: String, - val test: String + val test: String, + val additionalTests: List = emptyList() ) -data class UploadedTestPair( +data class UploadedTestApks( val app: String, - val test: String + val test: String, + val additionalTests: List = emptyList() ) /** Flank specific parameters for Android */ diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt index 69ba19dbd8..33e5317423 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt @@ -17,6 +17,9 @@ class AndroidGcloudYmlParams( val app: String? = null, val test: String? = null, + @field:JsonProperty("additional-apks") + val additionalApks: List = emptyList(), + @field:JsonProperty("auto-google-login") val autoGoogleLogin: Boolean = FlankDefaults.DISABLE_AUTO_LOGIN, @@ -42,8 +45,17 @@ class AndroidGcloudYmlParams( ) { companion object : IYmlKeys { override val keys = listOf( - "app", "test", "auto-google-login", "use-orchestrator", "environment-variables", - "directories-to-pull", "performance-metrics", "test-runner-class", "test-targets", "device" + "app", + "test", + "additionalApks", + "auto-google-login", + "use-orchestrator", + "environment-variables", + "directories-to-pull", + "performance-metrics", + "test-runner-class", + "test-targets", + "device" ) } } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 4967857fc8..cfd2cf1c19 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -92,6 +92,16 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { ) var test: String? = null + @Option( + names = ["--additional-apks"], + split = ",", + description = [ + "A list of up to 100 additional APKs to install, in addition to those being directly tested.", + "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. " + ] + ) + var additionalApks: List? = null + @Option( names = ["--auto-google-login"], description = ["Automatically log into the test device using a preconfigured " + diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt index a00684537d..fc3a771493 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt @@ -4,6 +4,7 @@ import com.google.api.services.testing.Testing import com.google.api.services.testing.model.Account import com.google.api.services.testing.model.AndroidDeviceList import com.google.api.services.testing.model.AndroidInstrumentationTest +import com.google.api.services.testing.model.Apk import com.google.api.services.testing.model.ClientInfo import com.google.api.services.testing.model.EnvironmentMatrix import com.google.api.services.testing.model.EnvironmentVariable @@ -38,7 +39,8 @@ object GcAndroidTestMatrix { androidDeviceList: AndroidDeviceList, testShards: ShardChunks, args: AndroidArgs, - toolResultsHistory: ToolResultsHistory + toolResultsHistory: ToolResultsHistory, + additionalTestGcsPaths: List ): Testing.Projects.TestMatrices.Create { // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120 @@ -81,6 +83,7 @@ object GcAndroidTestMatrix { .setAccount(account) .setNetworkProfile(args.networkProfile) .setDirectoriesToPull(args.directoriesToPull) + .setAdditionalApks(additionalTestGcsPaths.mapGcsPathsToApks()) if (args.environmentVariables.isNotEmpty()) { testSetup.environmentVariables = @@ -118,3 +121,7 @@ object GcAndroidTestMatrix { throw RuntimeException("Failed to create test matrix") } } + +private fun List?.mapGcsPathsToApks(): List? = this + ?.takeIf { it.isNotEmpty() } + ?.map { gcsPath -> Apk().setLocation(FileReference().setGcsPath(gcsPath)) } diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt b/test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt index 027279eefe..994efa09c4 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt @@ -31,7 +31,7 @@ private suspend fun TestExecution.createTestExecutionData(): TestExecutionData { return TestExecutionData( testExecution = this@createTestExecutionData, - testCases = response.testCases, + testCases = response.testCases ?: emptyList(), step = step, timestamp = response.getStartTimestamp() ) @@ -45,6 +45,6 @@ private suspend fun getAsync(toolResultsStep: ToolResultsStep) = coroutineScope // Unfortunately is not possible to obtain from api exact the same timestamp as is in autogenerated test_result_1.xml from google cloud. // This one is a little bit lower but close as possible. The difference is around ~3 seconds. -private fun ListTestCasesResponse.getStartTimestamp(): Timestamp = testCases.minBy { testCase -> +private fun ListTestCasesResponse.getStartTimestamp(): Timestamp = testCases?.minBy { testCase -> testCase.startTime.asUnixTimestamp() -}!!.startTime +}?.startTime ?: Timestamp() diff --git a/test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt b/test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt index 4fa9b952b2..7dc2628138 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt @@ -36,5 +36,5 @@ private fun refreshTestMatrices( } private fun List.getTestExecutions(): List = this - .map(TestMatrix::getTestExecutions) + .mapNotNull(TestMatrix::getTestExecutions) .flatten() diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 60f81195fa..09418f0ec3 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -1,17 +1,19 @@ package ftl.run.platform +import com.google.api.services.testing.model.AndroidDeviceList import com.google.api.services.testing.model.TestMatrix +import com.google.api.services.testing.model.ToolResultsHistory import ftl.args.AndroidArgs import ftl.args.AndroidTestShard +import ftl.args.ShardChunks import ftl.args.yml.AppTestPair +import ftl.args.yml.ResolvedTestApks +import ftl.args.yml.UploadedTestApks import ftl.gc.GcAndroidDevice import ftl.gc.GcAndroidTestMatrix import ftl.gc.GcStorage import ftl.gc.GcToolResults import ftl.http.executeWithRetry -import ftl.args.ShardChunks -import ftl.args.yml.ResolvedTestPair -import ftl.args.yml.UploadedTestPair import ftl.run.model.TestResult import ftl.run.platform.common.afterRunTests import ftl.run.platform.common.beforeRunMessage @@ -29,60 +31,92 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS // GcAndroidTestMatrix.execute() 3x retry => matrix id (string) val androidDeviceList = GcAndroidDevice.build(args.devices) - val jobs = arrayListOf>() + val testMatrices = arrayListOf>() val runCount = args.repeatTests val history = GcToolResults.createToolResultsHistory(args) - val apkPairsInArgs = listOf(AppTestPair(app = args.appApk, test = args.testApk)) + args.additionalAppTestApks - val allTestShardChunks: ShardChunks = apkPairsInArgs - .provideMissingApps(withFallbackApp = args.appApk) - .map { resolvedApkPair -> - val uploadedApkPair = resolveApkPair(resolvedApkPair, args, runGcsPath) - // Ensure we only shard tests that are part of the test apk. Use the resolved test apk path to make sure - // we don't re-download an apk it is on the local file system. - AndroidTestShard.getTestShardChunks(args, resolvedApkPair.test).also { testShards -> - repeat(runCount) { - // specify dispatcher to avoid inheriting main runBlocking context that runs in the main thread - // https://kotlinlang.org/docs/reference/coroutines/coroutine-context-and-dispatchers.html - jobs += async(Dispatchers.IO) { - GcAndroidTestMatrix.build( - appApkGcsPath = uploadedApkPair.app, - testApkGcsPath = uploadedApkPair.test, - runGcsPath = runGcsPath, - androidDeviceList = androidDeviceList, - testShards = testShards, - args = args, - toolResultsHistory = history - ).executeWithRetry() - } - } - } - }.flatten() + val resolvedTestApks = listOf( + element = ResolvedTestApks( + app = args.appApk, + test = args.testApk, + additionalTests = args.additionalTestApks + ) + ).plus( + elements = args.additionalAppTestApks + .provideMissingApps(args.appApk) + ) + + val allTestShardChunks: ShardChunks = resolvedTestApks.map { apks: ResolvedTestApks -> + // Ensure we only shard tests that are part of the test apk. Use the resolved test apk path to make sure + // we don't re-download an apk it is on the local file system. + AndroidTestShard.getTestShardChunks(args, apks).also { testShards -> + testMatrices += executeAndroidTestMatrix( + uploadedTestApks = uploadTestApks( + apks = apks, + args = args, + runGcsPath = runGcsPath + ), + runGcsPath = runGcsPath, + androidDeviceList = androidDeviceList, + testShards = testShards, + args = args, + history = history, + runCount = runCount + ) + } + }.flatten() println(beforeRunMessage(args, allTestShardChunks)) - val matrixMap = afterRunTests(jobs.awaitAll(), runGcsPath, stopwatch, args) + val matrixMap = afterRunTests(testMatrices.awaitAll(), runGcsPath, stopwatch, args) matrixMap to allTestShardChunks } private fun List.provideMissingApps(withFallbackApp: String) = - map { ResolvedTestPair(app = it.app ?: withFallbackApp, test = it.test) } + map { ResolvedTestApks(app = it.app ?: withFallbackApp, test = it.test) } + +private suspend fun executeAndroidTestMatrix( + runGcsPath: String, + args: AndroidArgs, + testShards: ShardChunks, + uploadedTestApks: UploadedTestApks, + androidDeviceList: AndroidDeviceList, + history: ToolResultsHistory, + runCount: Int +): List> = coroutineScope { + (0 until runCount).map { + async(Dispatchers.IO) { + GcAndroidTestMatrix.build( + appApkGcsPath = uploadedTestApks.app, + testApkGcsPath = uploadedTestApks.test, + runGcsPath = runGcsPath, + androidDeviceList = androidDeviceList, + testShards = testShards, + args = args, + toolResultsHistory = history, + additionalTestGcsPaths = uploadedTestApks.additionalTests + ).executeWithRetry() + } + } +} /** * Upload an APK pair if the path given is local * * @return AppTestPair with their GCS paths */ -private suspend fun resolveApkPair( - apk: ResolvedTestPair, +private suspend fun uploadTestApks( + apks: ResolvedTestApks, args: AndroidArgs, runGcsPath: String -): UploadedTestPair = coroutineScope { +): UploadedTestApks = coroutineScope { val gcsBucket = args.resultsBucket - val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apk.app, gcsBucket, runGcsPath) } - val testApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apk.test, gcsBucket, runGcsPath) } + val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.app, gcsBucket, runGcsPath) } + val testApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.test, gcsBucket, runGcsPath) } + val additionalTestApkGcsPaths = apks.additionalTests.map { async(Dispatchers.IO) { GcStorage.upload(it, gcsBucket, runGcsPath) } } - UploadedTestPair( + UploadedTestApks( app = appApkGcsPath.await(), - test = testApkGcsPath.await() + test = testApkGcsPath.await(), + additionalTests = additionalTestApkGcsPaths.awaitAll() ) } diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt index b36834d671..644142d23d 100644 --- a/test_runner/src/test/kotlin/Debug.kt +++ b/test_runner/src/test/kotlin/Debug.kt @@ -10,8 +10,8 @@ fun main() { // run "gradle check" to generate required fixtures val projectId = System.getenv("FLANK_PROJECT_ID") ?: "YOUR PROJECT ID" - val quantity = "single" - val type = "success" + val quantity = "multiple" + val type = "apk" // Bugsnag keeps the process alive so we must call exitProcess // https://github.com/bugsnag/bugsnag-java/issues/151 @@ -20,7 +20,7 @@ fun main() { // "--debug", "firebase", "test", "android", "run", -// "--dry", + "--dry", "-c=src/test/kotlin/ftl/fixtures/test_app_cases/flank-$quantity-$type.yml", "--project=$projectId", "--client-details=key1=value1,key2=value2" diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 10ac9c1f8f..55653a3218 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -27,8 +27,11 @@ class AndroidArgsTest { private val invalidApk = "../test_app/apks/invalid.apk" private val testApk = "../test_app/apks/app-debug-androidTest.apk" private val testErrorApk = "../test_app/apks/error-androidTest.apk" + private val testFlakyApk = "../test_app/apks/flaky-androidTest.apk" private val appApkAbsolutePath = appApk.absolutePath() private val testApkAbsolutePath = testApk.absolutePath() + private val testErrorApkAbsolutePath = testErrorApk.absolutePath() + private val testFlakyApkAbsolutePath = testFlakyApk.absolutePath() private val simpleFlankPath = getPath("src/test/kotlin/ftl/fixtures/simple-android-flank.yml") private val androidNonDefault = """ @@ -46,6 +49,9 @@ class AndroidArgsTest { app: $appApk test: $testApk + additional-apks: + - $testErrorApk + - $testFlakyApk auto-google-login: false use-orchestrator: false environment-variables: @@ -83,8 +89,8 @@ class AndroidArgsTest { disable-sharding: true keep-file-path: true additional-app-test-apks: - - app: foo - test: bar + - app: $appApk + test: $testErrorApk run-timeout: 20m """ @@ -179,6 +185,7 @@ class AndroidArgsTest { // AndroidGcloudYml assert(appApk, appApkAbsolutePath) assert(testApk, testApkAbsolutePath) + assert(additionalTestApks, listOf(testErrorApkAbsolutePath, testFlakyApkAbsolutePath)) assert(autoGoogleLogin, false) assert(useOrchestrator, false) assert(environmentVariables, linkedMapOf("clearPackageData" to "true", "randomEnvVar" to "false")) @@ -236,6 +243,9 @@ AndroidArgs # Android gcloud app: $appApkAbsolutePath test: $testApkAbsolutePath + additional-apks: + - /Users/janek/projects/flank-project/flank/test_app/apks/error-androidTest.apk + - /Users/janek/projects/flank-project/flank/test_app/apks/flaky-androidTest.apk auto-google-login: false use-orchestrator: false directories-to-pull: @@ -275,8 +285,8 @@ AndroidArgs # Android Flank Yml keep-file-path: true additional-app-test-apks: - - app: foo - test: bar + - app: $appApkAbsolutePath + test: $testErrorApkAbsolutePath run-timeout: 20m legacy-junit-result: false """.trimIndent() @@ -300,6 +310,7 @@ AndroidArgs # Android gcloud app: $appApkAbsolutePath test: $testApkAbsolutePath + additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: @@ -977,7 +988,7 @@ AndroidArgs @Test fun `cli additional-app-test-apks`() { val cli = AndroidRunCommand() - CommandLine(cli).parseArgs("--additional-app-test-apks=app=a,test=b") + CommandLine(cli).parseArgs("--additional-app-test-apks=app=$appApk,test=$testFlakyApk") val yaml = """ gcloud: @@ -985,15 +996,18 @@ AndroidArgs test: $testApk flank: additional-app-test-apks: - - app: 1 - test: 2 + - app: $appApk + test: $testErrorApk """ - assertThat(AndroidArgs.load(yaml).additionalAppTestApks).isEqualTo( - listOf(AppTestPair("1", "2"))) + assertEquals( + listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath)), + AndroidArgs.load(yaml).additionalAppTestApks + ) - val androidArgs = AndroidArgs.load(yaml, cli) - assertThat(androidArgs.additionalAppTestApks).isEqualTo( - listOf(AppTestPair("a", "b"))) + assertEquals( + listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath)), + AndroidArgs.load(yaml, cli).additionalAppTestApks + ) } @Test diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index acab94068f..6f5626b0ab 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -65,6 +65,7 @@ class AndroidRunCommandTest { assertThat(cmd.dryRun).isFalse() assertThat(cmd.app).isNull() assertThat(cmd.test).isNull() + assertThat(cmd.additionalApks).isNull() assertThat(cmd.testTargets).isNull() assertThat(cmd.useOrchestrator).isNull() assertThat(cmd.noUseOrchestrator).isNull() @@ -115,6 +116,13 @@ class AndroidRunCommandTest { assertThat(cmd.test).isEqualTo("myTestApp.apk") } + @Test + fun `additionalApks parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--additional-apks=a.apk,b.apk") + assertThat(cmd.additionalApks).isEqualTo(listOf("a.apk", "b.apk")) + } + @Test fun `testTargets parse`() { val testTargets = "--test-targets" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml index 0d58b7e8fc..689a748baf 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml @@ -1,12 +1,14 @@ gcloud: app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk" num-flaky-test-attempts: 2 flank: disable-sharding: false max-test-shards: 2 num-test-runs: 1 - additional-app-test-apks: - - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk +# additional-app-test-apks: +# - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk # - test: ../test_app/apks/invalid.apk diff --git a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt index 17b6cc30e7..8d0a651278 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt @@ -28,7 +28,8 @@ class GcAndroidTestMatrixTest { androidDeviceList = AndroidDeviceList(), testShards = emptyList(), args = androidArgs, - toolResultsHistory = createToolResultsHistory(androidArgs) + toolResultsHistory = createToolResultsHistory(androidArgs), + additionalTestGcsPaths = emptyList() ) } @@ -43,7 +44,8 @@ class GcAndroidTestMatrixTest { androidDeviceList = AndroidDeviceList(), testShards = listOf(listOf("")), args = androidArgs, - toolResultsHistory = createToolResultsHistory(androidArgs) + toolResultsHistory = createToolResultsHistory(androidArgs), + additionalTestGcsPaths = emptyList() ) } @@ -62,7 +64,8 @@ class GcAndroidTestMatrixTest { androidDeviceList = AndroidDeviceList(), testShards = emptyList(), args = androidArgs, - toolResultsHistory = createToolResultsHistory(androidArgs) + toolResultsHistory = createToolResultsHistory(androidArgs), + additionalTestGcsPaths = emptyList() ) } } From 4681133ea6342d78d5b8ba044a7799863431157c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Mon, 6 Apr 2020 00:40:18 +0200 Subject: [PATCH 2/5] Fix implementation & revert unneeded changes --- .../src/main/kotlin/ftl/args/AndroidArgs.kt | 4 +- .../main/kotlin/ftl/args/AndroidTestShard.kt | 12 ------ .../kotlin/ftl/args/yml/AndroidFlankYml.kt | 4 +- .../main/kotlin/ftl/gc/GcAndroidTestMatrix.kt | 4 +- .../reports/api/CreateTestExecutionData.kt | 6 +-- .../kotlin/ftl/reports/api/ProcessFromApi.kt | 2 +- .../ftl/run/platform/RunAndroidTests.kt | 37 ++++++++++--------- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 2 +- .../test_app_cases/flank-multiple-apk.yml | 6 +-- .../kotlin/ftl/gc/GcAndroidTestMatrixTest.kt | 6 +-- 10 files changed, 36 insertions(+), 47 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 6f23e4d997..59a52e2911 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -52,7 +52,7 @@ class AndroidArgs( private val androidGcloud = androidGcloudYml.gcloud val appApk = (cli?.app ?: androidGcloud.app ?: fatalError("app is not set")).processApkPath("from app") val testApk = (cli?.test ?: androidGcloud.test ?: fatalError("test is not set")).processApkPath("from test") - val additionalTestApks = (cli?.additionalApks ?: androidGcloud.additionalApks).map { it.processApkPath("from additional-apks") } + val additionalApks = (cli?.additionalApks ?: androidGcloud.additionalApks).map { it.processApkPath("from additional-apks") } val autoGoogleLogin = cli?.autoGoogleLogin ?: cli?.noAutoGoogleLogin?.not() ?: androidGcloud.autoGoogleLogin // We use not() on noUseOrchestrator because if the flag is on, useOrchestrator needs to be false @@ -124,7 +124,7 @@ AndroidArgs # Android gcloud app: $appApk test: $testApk - additional-apks: ${listToString(additionalTestApks)} + additional-apks: ${listToString(additionalApks)} auto-google-login: $autoGoogleLogin use-orchestrator: $useOrchestrator directories-to-pull:${listToString(directoriesToPull)} diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt b/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt index 7ff94f33b5..68092f2a7f 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt @@ -2,7 +2,6 @@ package ftl.args import com.linkedin.dex.parser.DexParser import com.linkedin.dex.parser.TestMethod -import ftl.args.yml.ResolvedTestApks import ftl.config.FtlConstants import ftl.filter.TestFilter import ftl.filter.TestFilters @@ -12,17 +11,6 @@ import ftl.util.fatalError object AndroidTestShard { - fun getTestShardChunks(args: AndroidArgs, apks: ResolvedTestApks): ShardChunks = - sequenceOf(apks.test).plus(apks.additionalTests).map { apk -> - // Download test APK if necessary so it can be used to validate test methods - if (!apk.startsWith(FtlConstants.GCS_PREFIX)) apk - else GcStorage.download(apk) - }.map { testLocalApk: String -> - getTestMethods(args, testLocalApk) - }.map { filteredTests: List -> - ArgsHelper.calculateShards(filteredTests, args) - }.toList().flatten() - // computed properties not specified in yaml fun getTestShardChunks(args: AndroidArgs, testApk: String): ShardChunks { // Download test APK if necessary so it can be used to validate test methods diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt index 344b404ffa..5bf9284740 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt @@ -11,13 +11,13 @@ data class AppTestPair( data class ResolvedTestApks( val app: String, val test: String, - val additionalTests: List = emptyList() + val additionalApks: List = emptyList() ) data class UploadedTestApks( val app: String, val test: String, - val additionalTests: List = emptyList() + val additionalApks: List = emptyList() ) /** Flank specific parameters for Android */ diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt index fc3a771493..05b01dccef 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt @@ -40,7 +40,7 @@ object GcAndroidTestMatrix { testShards: ShardChunks, args: AndroidArgs, toolResultsHistory: ToolResultsHistory, - additionalTestGcsPaths: List + additionalApkGcsPaths: List ): Testing.Projects.TestMatrices.Create { // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120 @@ -83,7 +83,7 @@ object GcAndroidTestMatrix { .setAccount(account) .setNetworkProfile(args.networkProfile) .setDirectoriesToPull(args.directoriesToPull) - .setAdditionalApks(additionalTestGcsPaths.mapGcsPathsToApks()) + .setAdditionalApks(additionalApkGcsPaths.mapGcsPathsToApks()) if (args.environmentVariables.isNotEmpty()) { testSetup.environmentVariables = diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt b/test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt index 994efa09c4..027279eefe 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt @@ -31,7 +31,7 @@ private suspend fun TestExecution.createTestExecutionData(): TestExecutionData { return TestExecutionData( testExecution = this@createTestExecutionData, - testCases = response.testCases ?: emptyList(), + testCases = response.testCases, step = step, timestamp = response.getStartTimestamp() ) @@ -45,6 +45,6 @@ private suspend fun getAsync(toolResultsStep: ToolResultsStep) = coroutineScope // Unfortunately is not possible to obtain from api exact the same timestamp as is in autogenerated test_result_1.xml from google cloud. // This one is a little bit lower but close as possible. The difference is around ~3 seconds. -private fun ListTestCasesResponse.getStartTimestamp(): Timestamp = testCases?.minBy { testCase -> +private fun ListTestCasesResponse.getStartTimestamp(): Timestamp = testCases.minBy { testCase -> testCase.startTime.asUnixTimestamp() -}?.startTime ?: Timestamp() +}!!.startTime diff --git a/test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt b/test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt index 7dc2628138..4fa9b952b2 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt @@ -36,5 +36,5 @@ private fun refreshTestMatrices( } private fun List.getTestExecutions(): List = this - .mapNotNull(TestMatrix::getTestExecutions) + .map(TestMatrix::getTestExecutions) .flatten() diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 09418f0ec3..f5876204c1 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -6,7 +6,6 @@ import com.google.api.services.testing.model.ToolResultsHistory import ftl.args.AndroidArgs import ftl.args.AndroidTestShard import ftl.args.ShardChunks -import ftl.args.yml.AppTestPair import ftl.args.yml.ResolvedTestApks import ftl.args.yml.UploadedTestApks import ftl.gc.GcAndroidDevice @@ -34,21 +33,12 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS val testMatrices = arrayListOf>() val runCount = args.repeatTests val history = GcToolResults.createToolResultsHistory(args) - val resolvedTestApks = listOf( - element = ResolvedTestApks( - app = args.appApk, - test = args.testApk, - additionalTests = args.additionalTestApks - ) - ).plus( - elements = args.additionalAppTestApks - .provideMissingApps(args.appApk) - ) + val resolvedTestApks = args.getResolvedTestApks() val allTestShardChunks: ShardChunks = resolvedTestApks.map { apks: ResolvedTestApks -> // Ensure we only shard tests that are part of the test apk. Use the resolved test apk path to make sure // we don't re-download an apk it is on the local file system. - AndroidTestShard.getTestShardChunks(args, apks).also { testShards -> + AndroidTestShard.getTestShardChunks(args, apks.test).also { testShards -> testMatrices += executeAndroidTestMatrix( uploadedTestApks = uploadTestApks( apks = apks, @@ -70,8 +60,21 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS matrixMap to allTestShardChunks } -private fun List.provideMissingApps(withFallbackApp: String) = - map { ResolvedTestApks(app = it.app ?: withFallbackApp, test = it.test) } +private fun AndroidArgs.getResolvedTestApks() = listOf( + element = ResolvedTestApks( + app = appApk, + test = testApk, + additionalApks = additionalApks + ) +).plus( + elements = additionalAppTestApks.map { + ResolvedTestApks( + app = it.app ?: appApk, + test = it.test, + additionalApks = additionalApks + ) + } +) private suspend fun executeAndroidTestMatrix( runGcsPath: String, @@ -92,7 +95,7 @@ private suspend fun executeAndroidTestMatrix( testShards = testShards, args = args, toolResultsHistory = history, - additionalTestGcsPaths = uploadedTestApks.additionalTests + additionalApkGcsPaths = uploadedTestApks.additionalApks ).executeWithRetry() } } @@ -112,11 +115,11 @@ private suspend fun uploadTestApks( val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.app, gcsBucket, runGcsPath) } val testApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.test, gcsBucket, runGcsPath) } - val additionalTestApkGcsPaths = apks.additionalTests.map { async(Dispatchers.IO) { GcStorage.upload(it, gcsBucket, runGcsPath) } } + val additionalApkGcsPaths = apks.additionalApks.map { async(Dispatchers.IO) { GcStorage.upload(it, gcsBucket, runGcsPath) } } UploadedTestApks( app = appApkGcsPath.await(), test = testApkGcsPath.await(), - additionalTests = additionalTestApkGcsPaths.awaitAll() + additionalApks = additionalApkGcsPaths.awaitAll() ) } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 55653a3218..eb754d7075 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -185,7 +185,7 @@ class AndroidArgsTest { // AndroidGcloudYml assert(appApk, appApkAbsolutePath) assert(testApk, testApkAbsolutePath) - assert(additionalTestApks, listOf(testErrorApkAbsolutePath, testFlakyApkAbsolutePath)) + assert(additionalApks, listOf(testErrorApkAbsolutePath, testFlakyApkAbsolutePath)) assert(autoGoogleLogin, false) assert(useOrchestrator, false) assert(environmentVariables, linkedMapOf("clearPackageData" to "true", "randomEnvVar" to "false")) diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml index 689a748baf..0d58b7e8fc 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml @@ -1,14 +1,12 @@ gcloud: app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk - additional-apks: - - "./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk" num-flaky-test-attempts: 2 flank: disable-sharding: false max-test-shards: 2 num-test-runs: 1 -# additional-app-test-apks: -# - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk + additional-app-test-apks: + - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk # - test: ../test_app/apks/invalid.apk diff --git a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt index 8d0a651278..cb1f361153 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt @@ -29,7 +29,7 @@ class GcAndroidTestMatrixTest { testShards = emptyList(), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs), - additionalTestGcsPaths = emptyList() + additionalApkGcsPaths = emptyList() ) } @@ -45,7 +45,7 @@ class GcAndroidTestMatrixTest { testShards = listOf(listOf("")), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs), - additionalTestGcsPaths = emptyList() + additionalApkGcsPaths = emptyList() ) } @@ -65,7 +65,7 @@ class GcAndroidTestMatrixTest { testShards = emptyList(), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs), - additionalTestGcsPaths = emptyList() + additionalApkGcsPaths = emptyList() ) } } From 407ca2dc11c5b7e0a28007ae8327d3ebe4d37dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Mon, 6 Apr 2020 00:46:53 +0200 Subject: [PATCH 3/5] Update release_notes.md --- release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release_notes.md b/release_notes.md index 334fccfb9d..dbff6e0c9c 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,4 +1,5 @@ ## next (unreleased) +- [#695](https://github.com/Flank/flank/pull/695) Add support for additional-apks option. ([jan-gogo](https://github.com/jan-gogo)) - [#683](https://github.com/Flank/flank/pull/683) Print web link. ([pawelpasterz](https://github.com/pawelpasterz)) - [#692](https://github.com/Flank/flank/pull/692) Add support for network-profiles list command & --network-profile option. ([jan-gogo](https://github.com/jan-gogo)) - [#689](https://github.com/Flank/flank/pull/689) Add support for client-details option. ([jan-gogo](https://github.com/jan-gogo)) From 7c3be671c0111c5a0afa146eef855969f4ac1168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Mon, 6 Apr 2020 01:30:18 +0200 Subject: [PATCH 4/5] Fix test --- test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index eb754d7075..71c19806f0 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -244,8 +244,8 @@ AndroidArgs app: $appApkAbsolutePath test: $testApkAbsolutePath additional-apks: - - /Users/janek/projects/flank-project/flank/test_app/apks/error-androidTest.apk - - /Users/janek/projects/flank-project/flank/test_app/apks/flaky-androidTest.apk + - $testErrorApkAbsolutePath + - $testFlakyApkAbsolutePath auto-google-login: false use-orchestrator: false directories-to-pull: From 9e9d6de73980ece720fb3192c30e8628046b685d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Mon, 6 Apr 2020 03:21:26 +0200 Subject: [PATCH 5/5] Fix additional-apks key --- test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt index 33e5317423..94a6b8f034 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt @@ -47,7 +47,7 @@ class AndroidGcloudYmlParams( override val keys = listOf( "app", "test", - "additionalApks", + "additional-apks", "auto-google-login", "use-orchestrator", "environment-variables",