From 30f3aa10fa277c84c855dcbbcbe8dd37f9746085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 8 Apr 2020 12:26:58 +0200 Subject: [PATCH 01/11] Add robo for robo-directives & robo-script options --- test_runner/flank.yml | 18 ++++++ .../src/main/kotlin/ftl/args/AndroidArgs.kt | 9 ++- .../src/main/kotlin/ftl/args/ArgsToString.kt | 8 ++- .../kotlin/ftl/args/yml/AndroidGcloudYml.kt | 9 +++ .../test/android/AndroidRunCommand.kt | 24 ++++++++ .../kotlin/ftl/config/FlankRoboDirective.kt | 29 ++++++++++ .../main/kotlin/ftl/gc/GcAndroidTestMatrix.kt | 20 ++++++- .../ftl/run/platform/RunAndroidTests.kt | 55 ++++++++----------- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 25 +++++++++ .../test/android/AndroidRunCommandTest.kt | 16 ++++++ .../kotlin/ftl/gc/GcAndroidTestMatrixTest.kt | 9 ++- 11 files changed, 182 insertions(+), 40 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/config/FlankRoboDirective.kt diff --git a/test_runner/flank.yml b/test_runner/flank.yml index 77705e371b..af22936e04 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -111,6 +111,24 @@ gcloud: # test-targets: # - class com.example.app.ExampleUiTest#testPasses + ## A map of robo_directives that you can use to customize the behavior of Robo test. + ## The type specifies the action type of the directive, which may take on values click, text or ignore. + ## If no type is provided, text will be used by default. + ## Each key should be the Android resource name of a target UI element and each value should be the text input for that element. + ## Values are only permitted for text type elements, so no value should be specified for click and ignore type elements. + # robo-directives: + # - type: text + # name: input_resource_name + # input: message + # - type: click + # name: button_resource_name + + ## The path to a Robo Script JSON file. + ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. + ## You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument. + ## Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting. + # robo-script: path_to_robo_script + ## A list of DIMENSION=VALUE pairs which specify a target device to test against. ## This flag may be repeated to specify multiple devices. ## The four device dimensions are: model, version, locale, and orientation. diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 2a8e2be42b..0e38a4838c 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -14,9 +14,9 @@ import ftl.args.ArgsHelper.evaluateFilePath import ftl.args.ArgsHelper.mergeYmlMaps import ftl.args.ArgsHelper.yamlMapper import ftl.args.ArgsToString.apksToString -import ftl.args.ArgsToString.devicesToString import ftl.args.ArgsToString.listToString import ftl.args.ArgsToString.mapToString +import ftl.args.ArgsToString.objectsToString import ftl.args.yml.AndroidFlankYml import ftl.args.yml.AndroidGcloudYml import ftl.args.yml.AndroidGcloudYmlParams @@ -27,6 +27,7 @@ import ftl.args.yml.YamlDeprecated import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.config.Device import ftl.config.FtlConstants +import ftl.config.parseRoboDirectives import ftl.util.FlankFatalError import java.nio.file.Files import java.nio.file.Path @@ -57,6 +58,8 @@ class AndroidArgs( // We use not() on noUseOrchestrator because if the flag is on, useOrchestrator needs to be false val useOrchestrator = cli?.useOrchestrator ?: cli?.noUseOrchestrator?.not() ?: androidGcloud.useOrchestrator + val roboDirectives = cli?.roboDirectives?.parseRoboDirectives() ?: androidGcloud.roboDirectives + val roboScript = (cli?.roboScript ?: androidGcloud.roboScript)?.processFilePath("from roboScript") val environmentVariables = cli?.environmentVariables ?: androidGcloud.environmentVariables val directoriesToPull = cli?.directoriesToPull ?: androidGcloud.directoriesToPull val otherFiles = (cli?.otherFiles ?: androidGcloud.otherFiles).map { (devicePath, filePath) -> @@ -142,7 +145,9 @@ AndroidArgs num-uniform-shards: $numUniformShards test-runner-class: $testRunnerClass test-targets:${listToString(testTargets)} - device:${devicesToString(devices)} + robo-directives:${objectsToString(roboDirectives)} + robo-script: $roboScript + device:${objectsToString(devices)} num-flaky-test-attempts: $flakyTestAttempts flank: diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt index 06c66bf415..60f673c03a 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt @@ -13,7 +13,7 @@ object ArgsToString { .joinToString("\n") } - fun listToString(list: List?): String { + fun listToString(list: List?): String { if (list.isNullOrEmpty()) return "" return NEW_LINE + list.filterNotNull() .joinToString("\n") { dir -> " - $dir" } @@ -25,6 +25,12 @@ object ArgsToString { .joinToString("\n") { "$it" } } + fun objectsToString(objects: List?): String { + if (objects.isNullOrEmpty()) return "" + return NEW_LINE + objects.filterNotNull() + .joinToString("\n") { "$it" } + } + fun apksToString(devices: List): String { if (devices.isNullOrEmpty()) return "" return NEW_LINE + devices.joinToString("\n") { (app, test) -> " - app: $app\n test: $test" } diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt index 4e85c1f202..2faa618f68 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt @@ -6,6 +6,7 @@ import ftl.config.Device import ftl.config.FlankDefaults import ftl.config.FtlConstants.defaultAndroidModel import ftl.config.FtlConstants.defaultAndroidVersion +import ftl.config.FlankRoboDirective /** * Android specific gcloud parameters @@ -47,6 +48,12 @@ class AndroidGcloudYmlParams( @field:JsonProperty("test-targets") val testTargets: List = emptyList(), + @field:JsonProperty("robo-directives") + val roboDirectives: List = emptyList(), + + @field:JsonProperty("robo-script") + val roboScript: String? = null, + val device: List = listOf(Device(defaultAndroidModel, defaultAndroidVersion)) ) { companion object : IYmlKeys { @@ -63,6 +70,8 @@ class AndroidGcloudYmlParams( "num-uniform-shards", "test-runner-class", "test-targets", + "robo-directives", + "robo-script", "device" ) } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 6893ce7e70..754b1ec310 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -129,6 +129,30 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { ) var noUseOrchestrator: Boolean? = null + @Option( + names = ["--robo-directives"], + split = ",", + description = [ + "A comma-separated (:=) map of robo_directives that you can use to customize the behavior of Robo test.", + "The type specifies the action type of the directive, which may take on values click, text or ignore.", + "If no type is provided, text will be used by default.", + "Each key should be the Android resource name of a target UI element and each value should be the text input for that element.", + "Values are only permitted for text type elements, so no value should be specified for click and ignore type elements." + ] + ) + var roboDirectives: List? = null + + @Option( + names = ["--robo-script"], + description = [ + "The path to a Robo Script JSON file.", + "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.", + "You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument.", + "Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting. " + ] + ) + var roboScript: String? = null + @Option( names = ["--environment-variables"], split = ",", diff --git a/test_runner/src/main/kotlin/ftl/config/FlankRoboDirective.kt b/test_runner/src/main/kotlin/ftl/config/FlankRoboDirective.kt new file mode 100644 index 0000000000..31c49f09aa --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/FlankRoboDirective.kt @@ -0,0 +1,29 @@ +package ftl.config + +import ftl.util.trimStartLine + +data class FlankRoboDirective( + val type: String, + val name: String, + val input: String? = null // Input is only permitted for text type elements +) { + override fun toString() = """ + - type: $type + name: $name + input: $input""".trimStartLine() +} + +fun List.parseRoboDirectives() = map(String::parseRoboDirective) + +fun String.parseRoboDirective(): FlankRoboDirective = split( + Regex("([:=])") +).let { chunks -> + require(chunks.size == 3) { + "Cannot parse robo directive `$this`, use following format `\$TYPE:\$RESOURCE_NAME=\$INPUT`" + } + FlankRoboDirective( + type = chunks[0], + name = chunks[1], + input = chunks[2] + ) +} diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt index cfe59bf9b8..c451de8078 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt @@ -4,6 +4,7 @@ import com.google.api.services.testing.Testing import com.google.api.services.testing.model.Account import com.google.api.services.testing.model.AndroidDeviceList import com.google.api.services.testing.model.AndroidInstrumentationTest +import com.google.api.services.testing.model.AndroidRoboTest import com.google.api.services.testing.model.Apk import com.google.api.services.testing.model.ClientInfo import com.google.api.services.testing.model.DeviceFile @@ -14,12 +15,14 @@ import com.google.api.services.testing.model.GoogleAuto import com.google.api.services.testing.model.GoogleCloudStorage import com.google.api.services.testing.model.RegularFile import com.google.api.services.testing.model.ResultStorage +import com.google.api.services.testing.model.RoboDirective import com.google.api.services.testing.model.TestMatrix import com.google.api.services.testing.model.TestSetup import com.google.api.services.testing.model.TestSpecification import com.google.api.services.testing.model.ToolResultsHistory import ftl.args.AndroidArgs import ftl.args.ShardChunks +import ftl.config.FlankRoboDirective import ftl.util.join import ftl.util.timeoutToSeconds @@ -30,6 +33,7 @@ object GcAndroidTestMatrix { value = this@toEnvironmentVariable.value } + @Suppress("LongParameterList") fun build( appApkGcsPath: String, testApkGcsPath: String, @@ -39,7 +43,8 @@ object GcAndroidTestMatrix { testShards: ShardChunks, args: AndroidArgs, toolResultsHistory: ToolResultsHistory, - additionalApkGcsPaths: List + additionalApkGcsPaths: List, + roboScriptGcsPath: String? ): Testing.Projects.TestMatrices.Create { // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120 @@ -91,6 +96,11 @@ object GcAndroidTestMatrix { .setDisableVideoRecording(!args.recordVideo) .setTestTimeout("${testTimeoutSeconds}s") .setTestSetup(testSetup) + .setAndroidRoboTest( + AndroidRoboTest() + .setRoboDirectives(args.roboDirectives.mapToApiRoboDirectives()) + .setRoboScript(FileReference().setGcsPath(roboScriptGcsPath)) + ) val resultsStorage = ResultStorage() .setGoogleCloudStorage(GoogleCloudStorage().setGcsPath(matrixGcsPath)) @@ -124,3 +134,11 @@ private fun Map.mapToDeviceFiles() = map { (devicePath: String, .setContent(FileReference().setGcsPath(gcsFilePath)) ) } + +private fun List.mapToApiRoboDirectives() = map { + RoboDirective().apply { + actionType = it.type + resourceName = it.name + inputText = it.input + } +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 39583bf7e7..97807b40b8 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -1,8 +1,7 @@ package ftl.run.platform -import com.google.api.services.testing.model.AndroidDeviceList +import com.google.api.services.testing.Testing import com.google.api.services.testing.model.TestMatrix -import com.google.api.services.testing.model.ToolResultsHistory import ftl.args.AndroidArgs import ftl.args.AndroidTestShard import ftl.args.ShardChunks @@ -35,25 +34,31 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS val history = GcToolResults.createToolResultsHistory(args) val resolvedTestApks = args.getResolvedTestApks() val otherGcsFiles = args.uploadOtherFiles(runGcsPath) + val roboScriptGcsPath = args.roboScript?.let { GcStorage.upload(it, args.resultsBucket, runGcsPath) } val allTestShardChunks: ShardChunks = resolvedTestApks.map { apks: ResolvedTestApks -> // Ensure we only shard tests that are part of the test apk. Use the resolved test apk path to make sure // we don't re-download an apk it is on the local file system. AndroidTestShard.getTestShardChunks(args, apks.test).also { testShards -> - testMatrices += executeAndroidTestMatrix( - uploadedTestApks = uploadTestApks( - apks = apks, - args = args, - runGcsPath = runGcsPath - ), - otherFiles = otherGcsFiles, - runGcsPath = runGcsPath, - androidDeviceList = androidDeviceList, - testShards = testShards, + val uploadedTestApks = uploadTestApks( + apks = apks, args = args, - history = history, - runCount = runCount + runGcsPath = runGcsPath ) + testMatrices += executeAndroidTestMatrix(runCount) { + GcAndroidTestMatrix.build( + appApkGcsPath = uploadedTestApks.app, + testApkGcsPath = uploadedTestApks.test, + runGcsPath = runGcsPath, + additionalApkGcsPaths = uploadedTestApks.additionalApks, + roboScriptGcsPath = roboScriptGcsPath, + androidDeviceList = androidDeviceList, + testShards = testShards, + args = args, + otherFiles = otherGcsFiles, + toolResultsHistory = history + ) + } } }.flatten() @@ -79,28 +84,12 @@ private fun AndroidArgs.getResolvedTestApks() = listOf( ) private suspend fun executeAndroidTestMatrix( - runGcsPath: String, - args: AndroidArgs, - testShards: ShardChunks, - uploadedTestApks: UploadedTestApks, - otherFiles: Map, - androidDeviceList: AndroidDeviceList, - history: ToolResultsHistory, - runCount: Int + runCount: Int, + createTestMatrix: () -> Testing.Projects.TestMatrices.Create ): List> = coroutineScope { (0 until runCount).map { async(Dispatchers.IO) { - GcAndroidTestMatrix.build( - appApkGcsPath = uploadedTestApks.app, - testApkGcsPath = uploadedTestApks.test, - runGcsPath = runGcsPath, - androidDeviceList = androidDeviceList, - testShards = testShards, - args = args, - otherFiles = otherFiles, - toolResultsHistory = history, - additionalApkGcsPaths = uploadedTestApks.additionalApks - ).executeWithRetry() + createTestMatrix().executeWithRetry() } } } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 33e129a498..e70ee0d4cc 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -4,6 +4,7 @@ import com.google.common.truth.Truth.assertThat import ftl.args.yml.AppTestPair import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.config.Device +import ftl.config.FlankRoboDirective import ftl.config.FtlConstants.defaultAndroidModel import ftl.config.FtlConstants.defaultAndroidVersion import ftl.run.platform.runAndroidTests @@ -72,6 +73,13 @@ class AndroidArgsTest { test-targets: - class com.example.app.ExampleUiTest#testPasses - class com.example.app.ExampleUiTest#testFails + robo-directives: + - type: text + name: resource_name_1 + input: some_text + - type: click + name: resource_name_2 + robo-script: $appApk device: - model: NexusLowRes version: 23 @@ -213,6 +221,13 @@ class AndroidArgsTest { "class com.example.app.ExampleUiTest#testFails" ) ) + assert( + roboDirectives, listOf( + FlankRoboDirective(type = "text", name = "resource_name_1", input = "some_text"), + FlankRoboDirective(type = "click", name = "resource_name_2") + ) + ) + assert(roboScript, appApkAbsolutePath) assert( devices, listOf( Device("NexusLowRes", "23", "en", "portrait"), @@ -274,6 +289,14 @@ AndroidArgs test-targets: - class com.example.app.ExampleUiTest#testPasses - class com.example.app.ExampleUiTest#testFails + robo-directives: + - type: text + name: resource_name_1 + input: some_text + - type: click + name: resource_name_2 + input: null + robo-script: $appApkAbsolutePath device: - model: NexusLowRes version: 23 @@ -339,6 +362,8 @@ AndroidArgs num-uniform-shards: null test-runner-class: null test-targets: + robo-directives: + robo-script: null device: - model: NexusLowRes version: 28 diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index b95e435106..f8b7401412 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -439,4 +439,20 @@ class AndroidRunCommandTest { assertThat(cmd.runTimeout).isEqualTo("20s") } + + @Test + fun `robo-directives parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--robo-directives=text:a=b,click=c") + + assertThat(cmd.roboDirectives).hasSize(2) + } + + @Test + fun `robo-script parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--robo-script=a") + + assertThat(cmd.roboScript).isEqualTo("a") + } } diff --git a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt index 8108870a84..db47398cb8 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt @@ -30,7 +30,8 @@ class GcAndroidTestMatrixTest { testShards = emptyList(), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs), - additionalApkGcsPaths = emptyList() + additionalApkGcsPaths = emptyList(), + roboScriptGcsPath = null ) } @@ -47,7 +48,8 @@ class GcAndroidTestMatrixTest { testShards = listOf(listOf("")), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs), - additionalApkGcsPaths = emptyList() + additionalApkGcsPaths = emptyList(), + roboScriptGcsPath = null ) } @@ -68,7 +70,8 @@ class GcAndroidTestMatrixTest { testShards = emptyList(), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs), - additionalApkGcsPaths = emptyList() + additionalApkGcsPaths = emptyList(), + roboScriptGcsPath = null ) } } From 401b93bda008551ee356a7f4bc34e42b04adc058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 8 Apr 2020 12:30:16 +0200 Subject: [PATCH 02/11] Update release_notes.md --- release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release_notes.md b/release_notes.md index 54b52238b2..187e8502d3 100644 --- a/release_notes.md +++ b/release_notes.md @@ -3,6 +3,7 @@ - [#712](https://github.com/Flank/flank/pull/712) Add keep file path for ios. ([pawelpasterz](https://github.com/pawelpasterz)) - [#711](https://github.com/Flank/flank/pull/711) Remove hardcoded height. ([pawelpasterz](https://github.com/pawelpasterz)) - [#708](https://github.com/Flank/flank/pull/708) Add ignore failed tests option to Flank. ([pawelpasterz](https://github.com/pawelpasterz)) +- [#704](https://github.com/Flank/flank/pull/709) Add robo for robo-directives & robo-script options. ([jan-gogo](https://github.com/jan-gogo)) - [#704](https://github.com/Flank/flank/pull/704) Fix shards calculation when there are ignored tests and shardTime is -1. ([jan-gogo](https://github.com/jan-gogo)) - [#692](https://github.com/Flank/flank/pull/698) Add support for other-files option. ([jan-gogo](https://github.com/jan-gogo)) - [#695](https://github.com/Flank/flank/pull/695) Add support for additional-apks option. ([jan-gogo](https://github.com/jan-gogo)) From 2946bccf303243b50138666e6a3559d5d0c6fd5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Tue, 14 Apr 2020 05:46:15 +0200 Subject: [PATCH 03/11] Prevent from replacing '.' with ',' on some locales --- .../src/main/kotlin/ftl/reports/MatrixResultsReport.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt index 92f9ee5b66..e22d345a3a 100644 --- a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt @@ -9,6 +9,8 @@ import ftl.util.println import ftl.util.write import java.io.StringWriter import java.text.DecimalFormat +import java.text.DecimalFormatSymbols +import java.util.Locale /** @@ -23,7 +25,7 @@ Example: object MatrixResultsReport : IReport { override val extension = ".txt" - private val percentFormat by lazy { DecimalFormat("#0.00") } + private val percentFormat by lazy { DecimalFormat("#0.00", DecimalFormatSymbols(Locale.US)) } private fun generate(matrices: MatrixMap): String { var total = 0 From 55d682fbca049ca2f001dfc6b0a0d7f654e06582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 15 Apr 2020 08:49:04 +0200 Subject: [PATCH 04/11] Make robo and instrumented tests mutually exclusive --- .../src/main/kotlin/ftl/args/AndroidArgs.kt | 9 +- .../kotlin/ftl/args/yml/AndroidFlankYml.kt | 8 +- .../test/android/AndroidRunCommand.kt | 2 +- .../main/kotlin/ftl/gc/GcAndroidTestMatrix.kt | 60 ++------- test_runner/src/main/kotlin/ftl/gc/Utils.kt | 33 +---- .../CreateAndroidInstrumentationTest.kt | 48 ++++++++ .../ftl/gc/android/CreateAndroidRobotTest.kt | 23 ++++ .../kotlin/ftl/gc/android/SetupAndroidTest.kt | 13 ++ .../src/main/kotlin/ftl/gc/android/Utils.kt | 19 +++ .../ftl/run/platform/RunAndroidTests.kt | 115 ++++++------------ .../run/platform/android/AndroidTestConfig.kt | 24 ++++ .../android/CreateAndroidTestConfig.kt | 22 ++++ .../android/CreateInstrumentationConfig.kt | 18 +++ .../run/platform/android/CreateRoboConfig.kt | 16 +++ .../ftl/run/platform/android/ResolveApks.kt | 20 +++ .../ftl/run/platform/android/UploadApks.kt | 34 ++++++ .../run/platform/android/UploadOtherFiles.kt | 16 +++ .../kotlin/ftl/args/AndroidArgsFileTest.kt | 10 +- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 101 +++++++++++---- .../kotlin/ftl/gc/GcAndroidTestMatrixTest.kt | 46 ++++--- .../CreateAndroidInstrumentationTestTest.kt} | 36 +++--- 21 files changed, 442 insertions(+), 231 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidInstrumentationTest.kt create mode 100644 test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidRobotTest.kt create mode 100644 test_runner/src/main/kotlin/ftl/gc/android/SetupAndroidTest.kt create mode 100644 test_runner/src/main/kotlin/ftl/gc/android/Utils.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestConfig.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/android/CreateRoboConfig.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/android/UploadOtherFiles.kt rename test_runner/src/test/kotlin/ftl/gc/{UtilsKtTest.kt => android/CreateAndroidInstrumentationTestTest.kt} (65%) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 0e38a4838c..1dd896fc12 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -52,7 +52,7 @@ class AndroidArgs( private val androidGcloud = androidGcloudYml.gcloud var appApk = (cli?.app ?: androidGcloud.app ?: throw FlankFatalError("app is not set")).processFilePath("from app") - var testApk = (cli?.test ?: androidGcloud.test ?: throw FlankFatalError("test is not set")).processFilePath("from test") + var testApk = (cli?.test ?: androidGcloud.test)?.processFilePath("from test") val additionalApks = (cli?.additionalApks ?: androidGcloud.additionalApks).map { it.processFilePath("from additional-apks") } val autoGoogleLogin = cli?.autoGoogleLogin ?: cli?.noAutoGoogleLogin?.not() ?: androidGcloud.autoGoogleLogin @@ -107,9 +107,16 @@ class AndroidArgs( "Option num-uniform-shards cannot be specified along with max-test-shards. Use only one of them" ) + if (!(isRoboTest xor isInstrumentationTest)) throw FlankFatalError( + "Option test xor (robo-directives or robo-script) must be specified" + ) + assertCommonProps(this) } + val isInstrumentationTest get() = testApk != null + val isRoboTest get() = roboDirectives.isNotEmpty() || roboScript != null + private fun assertDeviceSupported(device: Device) { when (val deviceConfigTest = AndroidCatalog.supportedDeviceConfig(device.model, device.version, this.project)) { SupportedDeviceConfig -> { diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt index f66969a6be..8686c05820 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt @@ -8,15 +8,15 @@ data class AppTestPair( val test: String ) -data class ResolvedTestApks( +data class ResolvedApks( val app: String, - val test: String, + val test: String?, val additionalApks: List = emptyList() ) -data class UploadedTestApks( +data class UploadedApks( val app: String, - val test: String, + val test: String?, val additionalApks: List = emptyList() ) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 754b1ec310..6e6cb52fed 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -45,7 +45,7 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { val config = AndroidArgs.load(Paths.get(configPath), cli = this) if (dumpShards) { - val testShardChunks: ShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk) + val testShardChunks: ShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk!!) val testShardChunksJson: String = prettyPrint.toJson(testShardChunks) Files.write(Paths.get(shardFile), testShardChunksJson.toByteArray()) diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt index c451de8078..705b6a283c 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt @@ -3,26 +3,21 @@ package ftl.gc import com.google.api.services.testing.Testing import com.google.api.services.testing.model.Account import com.google.api.services.testing.model.AndroidDeviceList -import com.google.api.services.testing.model.AndroidInstrumentationTest -import com.google.api.services.testing.model.AndroidRoboTest -import com.google.api.services.testing.model.Apk import com.google.api.services.testing.model.ClientInfo -import com.google.api.services.testing.model.DeviceFile import com.google.api.services.testing.model.EnvironmentMatrix import com.google.api.services.testing.model.EnvironmentVariable -import com.google.api.services.testing.model.FileReference import com.google.api.services.testing.model.GoogleAuto import com.google.api.services.testing.model.GoogleCloudStorage -import com.google.api.services.testing.model.RegularFile import com.google.api.services.testing.model.ResultStorage -import com.google.api.services.testing.model.RoboDirective import com.google.api.services.testing.model.TestMatrix import com.google.api.services.testing.model.TestSetup import com.google.api.services.testing.model.TestSpecification import com.google.api.services.testing.model.ToolResultsHistory import ftl.args.AndroidArgs -import ftl.args.ShardChunks -import ftl.config.FlankRoboDirective +import ftl.gc.android.mapGcsPathsToApks +import ftl.gc.android.mapToDeviceFiles +import ftl.gc.android.setupAndroidTest +import ftl.run.platform.android.AndroidTestConfig import ftl.util.join import ftl.util.timeoutToSeconds @@ -35,16 +30,13 @@ object GcAndroidTestMatrix { @Suppress("LongParameterList") fun build( - appApkGcsPath: String, - testApkGcsPath: String, + androidTestConfig: AndroidTestConfig, otherFiles: Map, runGcsPath: String, androidDeviceList: AndroidDeviceList, - testShards: ShardChunks, args: AndroidArgs, toolResultsHistory: ToolResultsHistory, - additionalApkGcsPaths: List, - roboScriptGcsPath: String? + additionalApkGcsPaths: List ): Testing.Projects.TestMatrices.Create { // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120 @@ -54,19 +46,6 @@ object GcAndroidTestMatrix { val matrixGcsPath = join(args.resultsBucket, runGcsPath) - val androidInstrumentation = AndroidInstrumentationTest() - .setAppApk(FileReference().setGcsPath(appApkGcsPath)) - .setTestApk(FileReference().setGcsPath(testApkGcsPath)) - .setupTestTargets(args, testShards) - - if (args.testRunnerClass != null) { - androidInstrumentation.testRunnerClass = args.testRunnerClass - } - - if (args.useOrchestrator) { - androidInstrumentation.orchestratorOption = "USE_ORCHESTRATOR" - } - // --auto-google-login // https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run // https://github.com/bootstraponline/gcloud_cli/blob/e4b5e01610abad2e31d8a6edb20b17b2f84c5395/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/android/matrix_creator.py#L174 @@ -91,16 +70,11 @@ object GcAndroidTestMatrix { val testTimeoutSeconds = timeoutToSeconds(args.testTimeout) val testSpecification = TestSpecification() - .setAndroidInstrumentationTest(androidInstrumentation) .setDisablePerformanceMetrics(!args.performanceMetrics) .setDisableVideoRecording(!args.recordVideo) .setTestTimeout("${testTimeoutSeconds}s") .setTestSetup(testSetup) - .setAndroidRoboTest( - AndroidRoboTest() - .setRoboDirectives(args.roboDirectives.mapToApiRoboDirectives()) - .setRoboScript(FileReference().setGcsPath(roboScriptGcsPath)) - ) + .setupAndroidTest(androidTestConfig) val resultsStorage = ResultStorage() .setGoogleCloudStorage(GoogleCloudStorage().setGcsPath(matrixGcsPath)) @@ -122,23 +96,3 @@ object GcAndroidTestMatrix { } } } - -private fun List?.mapGcsPathsToApks(): List? = this - ?.takeIf { it.isNotEmpty() } - ?.map { gcsPath -> Apk().setLocation(FileReference().setGcsPath(gcsPath)) } - -private fun Map.mapToDeviceFiles() = map { (devicePath: String, gcsFilePath: String) -> - DeviceFile().setRegularFile( - RegularFile() - .setDevicePath(devicePath) - .setContent(FileReference().setGcsPath(gcsFilePath)) - ) -} - -private fun List.mapToApiRoboDirectives() = map { - RoboDirective().apply { - actionType = it.type - resourceName = it.name - inputText = it.input - } -} diff --git a/test_runner/src/main/kotlin/ftl/gc/Utils.kt b/test_runner/src/main/kotlin/ftl/gc/Utils.kt index f55e55c399..b192084946 100644 --- a/test_runner/src/main/kotlin/ftl/gc/Utils.kt +++ b/test_runner/src/main/kotlin/ftl/gc/Utils.kt @@ -1,40 +1,9 @@ package ftl.gc -import com.google.api.services.testing.model.AndroidInstrumentationTest import com.google.api.services.testing.model.ClientInfoDetail -import com.google.api.services.testing.model.ManualSharding -import com.google.api.services.testing.model.ShardingOption -import com.google.api.services.testing.model.TestTargetsForShard -import com.google.api.services.testing.model.UniformSharding -import ftl.args.AndroidArgs -import ftl.args.ShardChunks internal fun Map.toClientInfoDetailList() = map { (key, value) -> ClientInfoDetail() .setKey(key) .setValue(value) -} - -internal fun AndroidInstrumentationTest.setupTestTargets(args: AndroidArgs, testShards: ShardChunks) = apply { - if (args.disableSharding) { - testTargets = testShards.flatten() - } else { - shardingOption = ShardingOption().apply { - if (args.numUniformShards != null) { - testTargets = testShards.flatten() - val numUniformShards = - if (testTargets.size > args.numUniformShards) { - args.numUniformShards - } else { - println("WARNING: num-uniform-shards (${args.numUniformShards}) is higher than number of test cases (${testTargets.size}) from ${testApk.gcsPath}") - testTargets.size - } - uniformSharding = UniformSharding().setNumShards(numUniformShards) - } else { - manualSharding = ManualSharding().setTestTargetsForShard(testShards.map { - TestTargetsForShard().setTestTargets(it) - }) - } - } - } -} +} \ No newline at end of file diff --git a/test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidInstrumentationTest.kt b/test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidInstrumentationTest.kt new file mode 100644 index 0000000000..de84d090b2 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidInstrumentationTest.kt @@ -0,0 +1,48 @@ +package ftl.gc.android + +import com.google.api.services.testing.model.AndroidInstrumentationTest +import com.google.api.services.testing.model.FileReference +import com.google.api.services.testing.model.ManualSharding +import com.google.api.services.testing.model.ShardingOption +import com.google.api.services.testing.model.TestTargetsForShard +import com.google.api.services.testing.model.UniformSharding +import ftl.args.ShardChunks +import ftl.run.platform.android.AndroidTestConfig + +internal fun createAndroidInstrumentationTest( + config: AndroidTestConfig.Instrumentation +) = AndroidInstrumentationTest().apply { + appApk = FileReference().setGcsPath(config.appApkGcsPath) + testApk = FileReference().setGcsPath(config.testApkGcsPath) + testRunnerClass = config.testRunnerClass + orchestratorOption = config.orchestratorOption +}.setupTestTargets( + disableSharding = config.disableSharding, + testShards = config.testShards, + numUniformShards = config.numUniformShards +) + +internal fun AndroidInstrumentationTest.setupTestTargets( + disableSharding: Boolean, + testShards: ShardChunks, + numUniformShards: Int? +) = apply { + if (disableSharding) { + testTargets = testShards.flatten() + } else { + shardingOption = ShardingOption().apply { + if (numUniformShards != null) { + testTargets = testShards.flatten() + val safeNumUniformShards = if (testTargets.size > numUniformShards) numUniformShards else { + println("WARNING: num-uniform-shards ($numUniformShards) is higher than number of test cases (${testTargets.size}) from ${testApk.gcsPath}") + testTargets.size + } + uniformSharding = UniformSharding().setNumShards(safeNumUniformShards) + } else { + manualSharding = ManualSharding().setTestTargetsForShard(testShards.map { + TestTargetsForShard().setTestTargets(it) + }) + } + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidRobotTest.kt b/test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidRobotTest.kt new file mode 100644 index 0000000000..665a434781 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidRobotTest.kt @@ -0,0 +1,23 @@ +package ftl.gc.android + +import com.google.api.services.testing.model.AndroidRoboTest +import com.google.api.services.testing.model.FileReference +import com.google.api.services.testing.model.RoboDirective +import ftl.config.FlankRoboDirective +import ftl.run.platform.android.AndroidTestConfig + +internal fun createAndroidRoboTest( + config: AndroidTestConfig.Robo +) = AndroidRoboTest().apply { + appApk = FileReference().setGcsPath(config.appApkGcsPath) + roboDirectives = config.flankRoboDirectives?.mapToApiRoboDirectives() + roboScript = config.roboScriptGcsPath?.let(FileReference()::setGcsPath) +} + +private fun List.mapToApiRoboDirectives() = map { + RoboDirective().apply { + actionType = it.type + resourceName = it.name + inputText = it.input + } +} diff --git a/test_runner/src/main/kotlin/ftl/gc/android/SetupAndroidTest.kt b/test_runner/src/main/kotlin/ftl/gc/android/SetupAndroidTest.kt new file mode 100644 index 0000000000..96d07d1de9 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/gc/android/SetupAndroidTest.kt @@ -0,0 +1,13 @@ +package ftl.gc.android + +import com.google.api.services.testing.model.TestSpecification +import ftl.run.platform.android.AndroidTestConfig + +internal fun TestSpecification.setupAndroidTest(config: AndroidTestConfig) = apply { + when (config) { + is AndroidTestConfig.Instrumentation -> + androidInstrumentationTest = createAndroidInstrumentationTest(config) + is AndroidTestConfig.Robo -> + androidRoboTest = createAndroidRoboTest(config) + } +} diff --git a/test_runner/src/main/kotlin/ftl/gc/android/Utils.kt b/test_runner/src/main/kotlin/ftl/gc/android/Utils.kt new file mode 100644 index 0000000000..effeb4846b --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/gc/android/Utils.kt @@ -0,0 +1,19 @@ +package ftl.gc.android + +import com.google.api.services.testing.model.Apk +import com.google.api.services.testing.model.DeviceFile +import com.google.api.services.testing.model.FileReference +import com.google.api.services.testing.model.RegularFile + +internal fun List.mapGcsPathsToApks(): List? = this + .map { gcsPath -> Apk().setLocation(FileReference().setGcsPath(gcsPath)) } + .takeIf { it.isNotEmpty() } + +internal fun Map.mapToDeviceFiles(): List = + map { (devicePath: String, gcsFilePath: String) -> + DeviceFile().setRegularFile( + RegularFile() + .setDevicePath(devicePath) + .setContent(FileReference().setGcsPath(gcsFilePath)) + ) + } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 97807b40b8..5236ed279d 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -4,15 +4,16 @@ import com.google.api.services.testing.Testing import com.google.api.services.testing.model.TestMatrix import ftl.args.AndroidArgs import ftl.args.AndroidTestShard -import ftl.args.ShardChunks -import ftl.args.yml.ResolvedTestApks -import ftl.args.yml.UploadedTestApks +import ftl.args.yml.ResolvedApks import ftl.gc.GcAndroidDevice import ftl.gc.GcAndroidTestMatrix -import ftl.gc.GcStorage import ftl.gc.GcToolResults import ftl.http.executeWithRetry import ftl.run.model.TestResult +import ftl.run.platform.android.createAndroidTestConfig +import ftl.run.platform.android.resolveApks +import ftl.run.platform.android.uploadOtherFiles +import ftl.run.platform.android.uploadApks import ftl.run.platform.common.afterRunTests import ftl.run.platform.common.beforeRunMessage import ftl.run.platform.common.beforeRunTests @@ -29,60 +30,51 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS // GcAndroidTestMatrix.execute() 3x retry => matrix id (string) val androidDeviceList = GcAndroidDevice.build(args.devices) - val testMatrices = arrayListOf>() - val runCount = args.repeatTests + val testMatrices = mutableListOf>() + val allTestShardChunks = mutableListOf>() + val history = GcToolResults.createToolResultsHistory(args) - val resolvedTestApks = args.getResolvedTestApks() val otherGcsFiles = args.uploadOtherFiles(runGcsPath) - val roboScriptGcsPath = args.roboScript?.let { GcStorage.upload(it, args.resultsBucket, runGcsPath) } - val allTestShardChunks: ShardChunks = resolvedTestApks.map { apks: ResolvedTestApks -> - // Ensure we only shard tests that are part of the test apk. Use the resolved test apk path to make sure - // we don't re-download an apk it is on the local file system. - AndroidTestShard.getTestShardChunks(args, apks.test).also { testShards -> - val uploadedTestApks = uploadTestApks( - apks = apks, + args.resolveApks().forEach { apks: ResolvedApks -> + val testShards = apks.test?.let { test -> + AndroidTestShard.getTestShardChunks(args, test).also { + allTestShardChunks += it + } + } + + val uploadedApks = uploadApks( + apks = apks, + args = args, + runGcsPath = runGcsPath + ) + + val androidTestConfig = args.createAndroidTestConfig( + uploadedApks = uploadedApks, + testShards = testShards, + runGcsPath = runGcsPath + ) + + testMatrices += executeAndroidTestMatrix(runCount = args.repeatTests) { + GcAndroidTestMatrix.build( + androidTestConfig = androidTestConfig, + runGcsPath = runGcsPath, + additionalApkGcsPaths = uploadedApks.additionalApks, + androidDeviceList = androidDeviceList, args = args, - runGcsPath = runGcsPath + otherFiles = otherGcsFiles, + toolResultsHistory = history ) - testMatrices += executeAndroidTestMatrix(runCount) { - GcAndroidTestMatrix.build( - appApkGcsPath = uploadedTestApks.app, - testApkGcsPath = uploadedTestApks.test, - runGcsPath = runGcsPath, - additionalApkGcsPaths = uploadedTestApks.additionalApks, - roboScriptGcsPath = roboScriptGcsPath, - androidDeviceList = androidDeviceList, - testShards = testShards, - args = args, - otherFiles = otherGcsFiles, - toolResultsHistory = history - ) - } } - }.flatten() + + testShards ?: emptyList() + } println(beforeRunMessage(args, allTestShardChunks)) val matrixMap = afterRunTests(testMatrices.awaitAll(), runGcsPath, stopwatch, args) matrixMap to allTestShardChunks } -private fun AndroidArgs.getResolvedTestApks() = listOf( - element = ResolvedTestApks( - app = appApk, - test = testApk, - additionalApks = additionalApks - ) -).plus( - elements = additionalAppTestApks.map { - ResolvedTestApks( - app = it.app ?: appApk, - test = it.test, - additionalApks = additionalApks - ) - } -) - private suspend fun executeAndroidTestMatrix( runCount: Int, createTestMatrix: () -> Testing.Projects.TestMatrices.Create @@ -93,34 +85,3 @@ private suspend fun executeAndroidTestMatrix( } } } - -/** - * Upload an APK pair if the path given is local - * - * @return AppTestPair with their GCS paths - */ -private suspend fun uploadTestApks( - apks: ResolvedTestApks, - args: AndroidArgs, - runGcsPath: String -): UploadedTestApks = coroutineScope { - val gcsBucket = args.resultsBucket - - val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.app, gcsBucket, runGcsPath) } - val testApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.test, gcsBucket, runGcsPath) } - val additionalApkGcsPaths = apks.additionalApks.map { async(Dispatchers.IO) { GcStorage.upload(it, gcsBucket, runGcsPath) } } - - UploadedTestApks( - app = appApkGcsPath.await(), - test = testApkGcsPath.await(), - additionalApks = additionalApkGcsPaths.awaitAll() - ) -} - -private suspend fun AndroidArgs.uploadOtherFiles( - runGcsPath: String -): Map = coroutineScope { - otherFiles.map { (devicePath: String, filePath: String) -> - async(Dispatchers.IO) { devicePath to GcStorage.upload(filePath, resultsBucket, runGcsPath) } - }.awaitAll().toMap() -} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt new file mode 100644 index 0000000000..4e46b88d23 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt @@ -0,0 +1,24 @@ +package ftl.run.platform.android + +import ftl.args.ShardChunks +import ftl.config.FlankRoboDirective + +sealed class AndroidTestConfig { + + data class Instrumentation( + val appApkGcsPath: String, + val testApkGcsPath: String, + val testRunnerClass: String?, + val orchestratorOption: String?, + // sharding + val disableSharding: Boolean, + val testShards: ShardChunks, + val numUniformShards: Int? + ) : AndroidTestConfig() + + data class Robo( + val appApkGcsPath: String, + val flankRoboDirectives: List?, + val roboScriptGcsPath: String? + ) : AndroidTestConfig() +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestConfig.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestConfig.kt new file mode 100644 index 0000000000..a179f6a464 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestConfig.kt @@ -0,0 +1,22 @@ +package ftl.run.platform.android + +import ftl.args.AndroidArgs +import ftl.args.ShardChunks +import ftl.args.yml.UploadedApks +import ftl.util.FlankFatalError + +internal fun AndroidArgs.createAndroidTestConfig( + uploadedApks: UploadedApks, + testShards: ShardChunks? = null, + runGcsPath: String? = null +): AndroidTestConfig = when { + isInstrumentationTest -> createInstrumentationConfig( + uploadedApks = uploadedApks, + testShards = testShards!! + ) + isRoboTest -> createRoboConfig( + uploadedApks = uploadedApks, + runGcsPath = runGcsPath!! + ) + else -> throw FlankFatalError("No testShards xor runGcsPath are specified") +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt new file mode 100644 index 0000000000..47b9dce950 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt @@ -0,0 +1,18 @@ +package ftl.run.platform.android + +import ftl.args.AndroidArgs +import ftl.args.ShardChunks +import ftl.args.yml.UploadedApks + +internal fun AndroidArgs.createInstrumentationConfig( + uploadedApks: UploadedApks, + testShards: ShardChunks +) = AndroidTestConfig.Instrumentation( + appApkGcsPath = uploadedApks.app, + testApkGcsPath = uploadedApks.test!!, + testRunnerClass = testRunnerClass, + orchestratorOption = "USE_ORCHESTRATOR".takeIf { useOrchestrator }, + disableSharding = disableSharding, + numUniformShards = numUniformShards, + testShards = testShards +) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateRoboConfig.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateRoboConfig.kt new file mode 100644 index 0000000000..0ff66bc640 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateRoboConfig.kt @@ -0,0 +1,16 @@ +package ftl.run.platform.android + +import ftl.args.AndroidArgs +import ftl.args.yml.UploadedApks +import ftl.gc.GcStorage + +internal fun AndroidArgs.createRoboConfig( + uploadedApks: UploadedApks, + runGcsPath: String +) = AndroidTestConfig.Robo( + appApkGcsPath = uploadedApks.app, + flankRoboDirectives = roboDirectives, + roboScriptGcsPath = roboScript?.let { + GcStorage.upload(it, resultsBucket, runGcsPath) + } +) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt new file mode 100644 index 0000000000..1d253b7de8 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt @@ -0,0 +1,20 @@ +package ftl.run.platform.android + +import ftl.args.AndroidArgs +import ftl.args.yml.ResolvedApks + +internal fun AndroidArgs.resolveApks() = listOf( + element = ResolvedApks( + app = appApk, + test = testApk, + additionalApks = additionalApks + ) +).plus( + elements = additionalAppTestApks.map { + ResolvedApks( + app = it.app ?: appApk, + test = it.test, + additionalApks = additionalApks + ) + } +) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt new file mode 100644 index 0000000000..51a1371591 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt @@ -0,0 +1,34 @@ +package ftl.run.platform.android + +import ftl.args.AndroidArgs +import ftl.args.yml.ResolvedApks +import ftl.args.yml.UploadedApks +import ftl.gc.GcStorage +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope + +/** + * Upload an APK pair if the path given is local + * + * @return AppTestPair with their GCS paths + */ +internal suspend fun uploadApks( + apks: ResolvedApks, + args: AndroidArgs, + runGcsPath: String +): UploadedApks = coroutineScope { + val gcsBucket = args.resultsBucket + + val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.app, gcsBucket, runGcsPath) } + val testApkGcsPath = apks.test?.let { async(Dispatchers.IO) { GcStorage.upload(it, gcsBucket, runGcsPath) } } + val additionalApkGcsPaths = + apks.additionalApks.map { async(Dispatchers.IO) { GcStorage.upload(it, gcsBucket, runGcsPath) } } + + UploadedApks( + app = appApkGcsPath.await(), + test = testApkGcsPath?.await(), + additionalApks = additionalApkGcsPaths.awaitAll() + ) +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/UploadOtherFiles.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadOtherFiles.kt new file mode 100644 index 0000000000..9c8cfebf47 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadOtherFiles.kt @@ -0,0 +1,16 @@ +package ftl.run.platform.android + +import ftl.args.AndroidArgs +import ftl.gc.GcStorage +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope + +internal suspend fun AndroidArgs.uploadOtherFiles( + runGcsPath: String +): Map = coroutineScope { + otherFiles.map { (devicePath: String, filePath: String) -> + async(Dispatchers.IO) { devicePath to GcStorage.upload(filePath, resultsBucket, runGcsPath) } + }.awaitAll().toMap() +} diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt index dbd283c370..0cc44fed03 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt @@ -63,7 +63,7 @@ class AndroidArgsFileTest { private fun checkConfig(args: AndroidArgs, local: Boolean) { with(args) { - if (local) assert(getString(testApk), getString(testApkAbsolutePath)) + if (local) assert(getString(testApk!!), getString(testApkAbsolutePath)) else assert(testApk, testApkGcs) if (local) assert(getString(appApk), getString(appApkAbsolutePath)) @@ -115,13 +115,13 @@ class AndroidArgsFileTest { fun `calculateShards 0`() { exceptionRule.expectMessage("Test APK has no tests") val args = configWithTestMethods(0) - AndroidTestShard.getTestShardChunks(args, args.testApk) + AndroidTestShard.getTestShardChunks(args, args.testApk!!) } @Test fun `calculateShards 1`() { val config = configWithTestMethods(1) - val testShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk) + val testShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk!!) with(config) { assert(maxTestShards, 1) assert(testShardChunks.size, 1) @@ -132,7 +132,7 @@ class AndroidArgsFileTest { @Test fun `calculateShards 155`() { val config = configWithTestMethods(155) - val testShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk) + val testShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk!!) with(config) { assert(maxTestShards, 1) assert(testShardChunks.size, 1) @@ -143,7 +143,7 @@ class AndroidArgsFileTest { @Test fun `calculateShards 155 40`() { val config = configWithTestMethods(155, maxTestShards = 40) - val testShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk) + val testShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk!!) with(config) { assert(maxTestShards, 40) assert(testShardChunks.size, 40) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index e70ee0d4cc..b7c9a8c4ef 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -9,9 +9,9 @@ import ftl.config.FtlConstants.defaultAndroidModel import ftl.config.FtlConstants.defaultAndroidVersion import ftl.run.platform.runAndroidTests import ftl.test.util.FlankTestRunner -import ftl.test.util.TestHelper.getPath import ftl.test.util.TestHelper.absolutePath import ftl.test.util.TestHelper.assert +import ftl.test.util.TestHelper.getPath import ftl.util.FlankFatalError import kotlinx.coroutines.runBlocking import org.junit.Assert.assertEquals @@ -73,13 +73,6 @@ class AndroidArgsTest { test-targets: - class com.example.app.ExampleUiTest#testPasses - class com.example.app.ExampleUiTest#testFails - robo-directives: - - type: text - name: resource_name_1 - input: some_text - - type: click - name: resource_name_2 - robo-script: $appApk device: - model: NexusLowRes version: 23 @@ -221,13 +214,6 @@ class AndroidArgsTest { "class com.example.app.ExampleUiTest#testFails" ) ) - assert( - roboDirectives, listOf( - FlankRoboDirective(type = "text", name = "resource_name_1", input = "some_text"), - FlankRoboDirective(type = "click", name = "resource_name_2") - ) - ) - assert(roboScript, appApkAbsolutePath) assert( devices, listOf( Device("NexusLowRes", "23", "en", "portrait"), @@ -290,13 +276,7 @@ AndroidArgs - class com.example.app.ExampleUiTest#testPasses - class com.example.app.ExampleUiTest#testFails robo-directives: - - type: text - name: resource_name_1 - input: some_text - - type: click - name: resource_name_2 - input: null - robo-script: $appApkAbsolutePath + robo-script: null device: - model: NexusLowRes version: 23 @@ -448,7 +428,7 @@ AndroidArgs """ ) - val testShardChunks = AndroidTestShard.getTestShardChunks(androidArgs, androidArgs.testApk) + val testShardChunks = AndroidTestShard.getTestShardChunks(androidArgs, androidArgs.testApk!!) with(androidArgs) { assert(maxTestShards, -1) assert(testShardChunks.size, 2) @@ -482,7 +462,7 @@ AndroidArgs disable-sharding: true """ val androidArgs = AndroidArgs.load(yaml) - val testShardChunks = AndroidTestShard.getTestShardChunks(androidArgs, androidArgs.testApk) + val testShardChunks = AndroidTestShard.getTestShardChunks(androidArgs, androidArgs.testApk!!) assertThat(testShardChunks).hasSize(1) } @@ -494,7 +474,7 @@ AndroidArgs test: $invalidApk """ val androidArgs = AndroidArgs.load(yaml) - AndroidTestShard.getTestShardChunks(androidArgs, androidArgs.testApk) + AndroidTestShard.getTestShardChunks(androidArgs, androidArgs.testApk!!) } @Test @@ -1161,4 +1141,75 @@ AndroidArgs val args = AndroidArgs.load(flankLocal) assertFalse(args.ignoreFailedTests) } + + @Test(expected = FlankFatalError::class) + fun `should throw if both instrumentation and robo tests are specified`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + robo-directives: + - type: text + name: resource_name_1 + input: some_text + - type: click + name: resource_name_2 + robo-script: $appApk + """.trimIndent() + + AndroidArgs.load(yaml) + } + + fun `should load robo-script & robo directives from yaml`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + robo-directives: + - type: text + name: resource_name_1 + input: some_text + - type: click + name: resource_name_2 + robo-script: $appApk + """.trimIndent() + + val args = AndroidArgs.load(yaml) + + assertEquals( + args.roboScript, + appApkAbsolutePath + ) + assertEquals( + args.roboDirectives, + listOf( + FlankRoboDirective(type = "text", name = "resource_name_1", input = "some_text"), + FlankRoboDirective(type = "click", name = "resource_name_2") + ) + ) + } + + fun `should load robo-script & robo directives from cli`() { + val cli = AndroidRunCommand() + CommandLine(cli).parseArgs("--robo-script=$appApk, --robo-directives=text:a=b,click=c") + + val yaml = """ + gcloud: + app: $appApk + """.trimIndent() + + val args = AndroidArgs.load(yaml, cli) + + assertEquals( + args.roboScript, + appApkAbsolutePath + ) + assertEquals( + args.roboDirectives, + listOf( + FlankRoboDirective(type = "text", name = "a", input = "b"), + FlankRoboDirective(type = "click", name = "c") + ) + ) + } } diff --git a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt index db47398cb8..6b4d09dd19 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt @@ -3,6 +3,7 @@ package ftl.gc import com.google.api.services.testing.model.AndroidDeviceList import ftl.args.AndroidArgs import ftl.gc.GcToolResults.createToolResultsHistory +import ftl.run.platform.android.AndroidTestConfig import ftl.test.util.FlankTestRunner import io.mockk.every import io.mockk.mockk @@ -22,16 +23,21 @@ class GcAndroidTestMatrixTest { val androidArgs = mockk(relaxed = true) GcAndroidTestMatrix.build( - appApkGcsPath = "", - testApkGcsPath = "", + androidTestConfig = AndroidTestConfig.Instrumentation( + appApkGcsPath = "", + testApkGcsPath = "", + testShards = emptyList(), + orchestratorOption = null, + numUniformShards = null, + disableSharding = false, + testRunnerClass = "" + ), runGcsPath = "", otherFiles = emptyMap(), androidDeviceList = AndroidDeviceList(), - testShards = emptyList(), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs), - additionalApkGcsPaths = emptyList(), - roboScriptGcsPath = null + additionalApkGcsPaths = emptyList() ) } @@ -40,16 +46,21 @@ class GcAndroidTestMatrixTest { val androidArgs = mockk(relaxed = true) GcAndroidTestMatrix.build( - appApkGcsPath = "", - testApkGcsPath = "", + androidTestConfig = AndroidTestConfig.Instrumentation( + appApkGcsPath = "", + testApkGcsPath = "", + testShards = listOf(listOf("")), + orchestratorOption = null, + numUniformShards = null, + disableSharding = false, + testRunnerClass = "" + ), runGcsPath = "", otherFiles = emptyMap(), androidDeviceList = AndroidDeviceList(), - testShards = listOf(listOf("")), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs), - additionalApkGcsPaths = emptyList(), - roboScriptGcsPath = null + additionalApkGcsPaths = emptyList() ) } @@ -62,16 +73,21 @@ class GcAndroidTestMatrixTest { every { androidArgs.project } returns "123" GcAndroidTestMatrix.build( - appApkGcsPath = "", - testApkGcsPath = "", + androidTestConfig = AndroidTestConfig.Instrumentation( + appApkGcsPath = "", + testApkGcsPath = "", + testShards = emptyList(), + orchestratorOption = null, + numUniformShards = null, + disableSharding = false, + testRunnerClass = "" + ), runGcsPath = "", otherFiles = emptyMap(), androidDeviceList = AndroidDeviceList(), - testShards = emptyList(), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs), - additionalApkGcsPaths = emptyList(), - roboScriptGcsPath = null + additionalApkGcsPaths = emptyList() ) } } diff --git a/test_runner/src/test/kotlin/ftl/gc/UtilsKtTest.kt b/test_runner/src/test/kotlin/ftl/gc/android/CreateAndroidInstrumentationTestTest.kt similarity index 65% rename from test_runner/src/test/kotlin/ftl/gc/UtilsKtTest.kt rename to test_runner/src/test/kotlin/ftl/gc/android/CreateAndroidInstrumentationTestTest.kt index e10302a5e8..a438ac1415 100644 --- a/test_runner/src/test/kotlin/ftl/gc/UtilsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/android/CreateAndroidInstrumentationTestTest.kt @@ -1,11 +1,8 @@ -package ftl.gc +package ftl.gc.android import com.google.api.services.testing.model.AndroidInstrumentationTest import com.google.api.services.testing.model.FileReference -import ftl.args.AndroidArgs import ftl.args.ShardChunks -import io.mockk.every -import io.mockk.mockk import org.junit.Assert.assertEquals import org.junit.Assert.assertNotNull import org.junit.Test @@ -15,13 +12,15 @@ class UtilsKtTest { @Test fun `setupTestTargets should setup testTargets`() { // given - val args = mockk { - every { disableSharding } returns true - } val testShards: ShardChunks = emptyList() // when - val actual = AndroidInstrumentationTest().setupTestTargets(args, testShards) + val actual = AndroidInstrumentationTest() + .setupTestTargets( + disableSharding = true, + testShards = testShards, + numUniformShards = null + ) .testTargets // then @@ -32,16 +31,16 @@ class UtilsKtTest { fun `setupTestTargets should setup uniformSharding`() { // given val expectedTestTargets = emptyList() - val args = mockk { - every { disableSharding } returns false - every { numUniformShards } returns 50 - } val testShards: ShardChunks = listOf(expectedTestTargets) // when val actual = AndroidInstrumentationTest() .setTestApk(FileReference().setGcsPath("testApk")) - .setupTestTargets(args, testShards) + .setupTestTargets( + disableSharding = false, + testShards = testShards, + numUniformShards = 50 + ) // then assertEquals(0, actual.shardingOption.uniformSharding.numShards) @@ -52,13 +51,14 @@ class UtilsKtTest { fun `setupTestTargets should setup manualSharding`() { // given val shardChunks: ShardChunks = listOf(emptyList(), emptyList()) - val args = mockk { - every { disableSharding } returns false - every { numUniformShards } returns null - } // when - val actual = AndroidInstrumentationTest().setupTestTargets(args, shardChunks) + val actual = AndroidInstrumentationTest() + .setupTestTargets( + disableSharding = false, + testShards = shardChunks, + numUniformShards = null + ) .shardingOption .manualSharding .testTargetsForShard From 50fdd8203cc7e7270e0ffe40e178328dbede2388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 15 Apr 2020 08:51:15 +0200 Subject: [PATCH 05/11] Fix detekt issue --- test_runner/src/main/kotlin/ftl/gc/Utils.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/gc/Utils.kt b/test_runner/src/main/kotlin/ftl/gc/Utils.kt index b192084946..aba1d99c12 100644 --- a/test_runner/src/main/kotlin/ftl/gc/Utils.kt +++ b/test_runner/src/main/kotlin/ftl/gc/Utils.kt @@ -6,4 +6,4 @@ internal fun Map.toClientInfoDetailList() = map { (key, value) - ClientInfoDetail() .setKey(key) .setValue(value) -} \ No newline at end of file +} From ccd535ddd068f47a86b1e87ba511e7442245d005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 15 Apr 2020 09:20:08 +0200 Subject: [PATCH 06/11] Update missing config options in README.md which exist in flank.yml --- README.md | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ce7b89ba0d..b050df0bc6 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,20 @@ gcloud: ## Invoke a test asynchronously without waiting for test results. # async: false + ## A key-value map of additional details to attach to the test matrix. + ## Arbitrary key-value pairs may be attached to a test matrix to provide additional context about the tests being run. + ## When consuming the test results, such as in Cloud Functions or a CI system, + ## these details can add additional context such as a link to the corresponding pull request. + # client-details + # key1: value1 + # key2: value2 + + ## The name of the network traffic profile, for example LTE, HSPA, etc, + ## which consists of a set of parameters to emulate network conditions when running the test + ## (default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). + ## This feature only works on physical devices. + # network-profile: LTE + ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history @@ -257,10 +271,27 @@ gcloud: # directories-to-pull: # - /sdcard/ + ## A list of device-path: file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push. + ## Device paths must be under absolute, whitelisted paths (${EXTERNAL_STORAGE}, or ${ANDROID_DATA}/local/tmp). + ## Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). + # other-files + # - /sdcard/dir1/file1.txt: local/file.txt + # - /sdcard/dir2/file2.jpg: gs://bucket/file.jpg + ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only). ## Disabled by default. Use --performance-metrics to enable. # performance-metrics: true + ## Specifies the number of shards into which you want to evenly distribute test cases. + ## The shards are run in parallel on separate devices. For example, + ## if your test execution contains 20 test cases and you specify four shards, each shard executes five test cases. + ## The number of shards should be less than the total number of test cases. + ## The number of shards specified must be >= 1 and <= 50. + ## This option cannot be used along max-test-shards and is not compatible with smart sharding. + ## If you want to take benefits of smart sharding use max-test-shards instead. + ## default: null + # num-uniform-shards: 50 + ## The fully-qualified Java class name of the instrumentation test runner ## (default: the last name extracted from the APK manifest). # test-runner-class: com.foo.TestRunner @@ -273,6 +304,24 @@ gcloud: # test-targets: # - class com.example.app.ExampleUiTest#testPasses + ## A map of robo_directives that you can use to customize the behavior of Robo test. + ## The type specifies the action type of the directive, which may take on values click, text or ignore. + ## If no type is provided, text will be used by default. + ## Each key should be the Android resource name of a target UI element and each value should be the text input for that element. + ## Values are only permitted for text type elements, so no value should be specified for click and ignore type elements. + # robo-directives: + # - type: text + # name: input_resource_name + # input: message + # - type: click + # name: button_resource_name + + ## The path to a Robo Script JSON file. + ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. + ## You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument. + ## Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting. + # robo-script: path_to_robo_script + ## A list of DIMENSION=VALUE pairs which specify a target device to test against. ## This flag may be repeated to specify multiple devices. ## The four device dimensions are: model, version, locale, and orientation. @@ -297,7 +346,7 @@ flank: ## default: -1 (unlimited) # shard-time: -1 - ## repeat tests - the amount of times to run the tests. + ## The amount of times to run the tests. ## 1 runs the tests once. 10 runs all the tests 10x # num-test-runs: 1 @@ -344,6 +393,12 @@ flank: ## Useful for Fladle and other gradle plugins that don't expect the process to have a non-zero exit code. ## The JUnit XML is used to determine failure. (default: false) # ignore-failed-tests: true + + ## Flank provides two ways for parsing junit xml results. + ## New way uses google api instead of merging xml files, but can generate slightly different output format. + ## This flag allows fallback for legacy xml junit results parsing + ## Currently available for android, iOS still uses only legacy way. + # legacy-junit-result: false ``` ### Android code coverage From 4733194ce6d57f591ddff8fb19016c20fb810be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 15 Apr 2020 09:27:17 +0200 Subject: [PATCH 07/11] Update flank.yml options that was previously changed only in README.md --- test_runner/flank.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/test_runner/flank.yml b/test_runner/flank.yml index af22936e04..2f5da11ba7 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -10,7 +10,7 @@ gcloud: ## (default: a timestamp with a random suffix). # results-dir: tmp - ## Enable video recording during the test. Enabled by default, use --no-record-video to disable. + ## Enable video recording during the test. Disabled by default. Use --record-video to enable. # record-video: true ## The max time this test execution can run before it is cancelled (default: 15m). @@ -57,7 +57,7 @@ gcloud: test: ../test_app/apks/app-debug-androidTest.apk ## Automatically log into the test device using a preconfigured Google account before beginning the test. - ## Enabled by default, use --no-auto-google-login to disable. + ## Disabled by default. Use --auto-google-login to enable. # auto-google-login: true ## Whether each test runs in its own Instrumentation instance with the Android Test Orchestrator @@ -86,7 +86,7 @@ gcloud: # - /sdcard/dir2/file2.jpg: gs://bucket/file.jpg ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only). - ## Enabled by default, use --no-performance-metrics to disable. + ## Disabled by default. Use --performance-metrics to enable. # performance-metrics: true ## Specifies the number of shards into which you want to evenly distribute test cases. @@ -148,7 +148,7 @@ flank: # max-test-shards: 1 ## shard time - the amount of time tests within a shard should take - ## when set to > 0, the shard count is dynamically set based on time up to the maxmimum limit defined by max-test-shards + ## when set to > 0, the shard count is dynamically set based on time up to the maximum limit defined by max-test-shards ## 2 minutes (120) is recommended. ## default: -1 (unlimited) # shard-time: -1 @@ -182,19 +182,25 @@ flank: ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. # local-result-dir: flank - ## Downloaded files preserves the original path of file. Required when file names are not unique. + ## Keeps the full path of downloaded files. Required when file names are not unique. ## Default: false # keep-file-path: false - ## Include additional app/test apk pairs in the run. If app is omitted, then the top level app is used for that pair. + ## Include additional app/test apk pairs in the run. Apks are unique by just filename and not by path! + ## If app is omitted, then the top level app is used for that pair. # additional-app-test-apks: # - app: ../test_app/apks/app-debug.apk - # test: ../test_app/apks/app-debug-androidTest.apk - # - test: ../test_app/apks/app-debug-androidTest.apk + # test: ../test_app/apks/app1-debug-androidTest.apk + # - test: ../test_app/apks/app2-debug-androidTest.apk ## The max time this test run can execute before it is cancelled (default: unlimited). # run-timeout: 60m + ## Terminate with exit code 0 when there are failed tests. + ## Useful for Fladle and other gradle plugins that don't expect the process to have a non-zero exit code. + ## The JUnit XML is used to determine failure. (default: false) + # ignore-failed-tests: true + ## Flank provides two ways for parsing junit xml results. ## New way uses google api instead of merging xml files, but can generate slightly different output format. ## This flag allows fallback for legacy xml junit results parsing From 5ff256a90565819f9cdb7d773641a64d99f8a085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 15 Apr 2020 10:41:24 +0200 Subject: [PATCH 08/11] Cleanup --- test_runner/src/main/kotlin/ftl/args/ArgsToString.kt | 9 +-------- test_runner/src/main/kotlin/ftl/args/IosArgs.kt | 4 ++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt index 60f673c03a..03990624b2 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt @@ -1,7 +1,6 @@ package ftl.args import ftl.args.yml.AppTestPair -import ftl.config.Device private const val NEW_LINE = '\n' @@ -13,18 +12,12 @@ object ArgsToString { .joinToString("\n") } - fun listToString(list: List?): String { + fun listToString(list: List?): String { if (list.isNullOrEmpty()) return "" return NEW_LINE + list.filterNotNull() .joinToString("\n") { dir -> " - $dir" } } - fun devicesToString(devices: List?): String { - if (devices.isNullOrEmpty()) return "" - return NEW_LINE + devices.filterNotNull() - .joinToString("\n") { "$it" } - } - fun objectsToString(objects: List?): String { if (objects.isNullOrEmpty()) return "" return NEW_LINE + objects.filterNotNull() diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index f789e447d9..0107434f80 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -8,9 +8,9 @@ import ftl.args.ArgsHelper.createJunitBucket import ftl.args.ArgsHelper.evaluateFilePath import ftl.args.ArgsHelper.mergeYmlMaps import ftl.args.ArgsHelper.yamlMapper -import ftl.args.ArgsToString.devicesToString import ftl.args.ArgsToString.listToString import ftl.args.ArgsToString.mapToString +import ftl.args.ArgsToString.objectsToString import ftl.args.yml.FlankYml import ftl.args.yml.GcloudYml import ftl.args.yml.IosFlankYml @@ -129,7 +129,7 @@ IosArgs test: $xctestrunZip xctestrun-file: $xctestrunFile xcode-version: $xcodeVersion - device:${devicesToString(devices)} + device:${objectsToString(devices)} num-flaky-test-attempts: $flakyTestAttempts flank: From dc59bd603f09de2546997cc6d5e18c0b58d6497e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Tue, 21 Apr 2020 13:14:20 +0200 Subject: [PATCH 09/11] Fix robo-directives yml format * Prevent crash by null testSuiteOverviews when using robo test * Prepare test_app for robo test * Add robo script to fixtures --- .../example/test_app/BaseInstrumentedTest.kt | 4 +- .../java/com/example/test_app/MainActivity.kt | 21 ++- .../app/src/main/res/layout/activity_main.xml | 44 +++-- .../src/main/kotlin/ftl/args/AndroidArgs.kt | 9 +- .../kotlin/ftl/args/yml/AndroidGcloudYml.kt | 3 +- .../kotlin/ftl/config/FlankRoboDirective.kt | 31 ++-- .../ftl/gc/android/CreateAndroidRobotTest.kt | 14 +- .../ftl/reports/api/CreateJUnitTestSuite.kt | 12 +- .../reports/api/CreateTestSuitOverviewData.kt | 29 ++-- .../ftl/run/platform/RunAndroidTests.kt | 2 - test_runner/src/test/kotlin/Debug.kt | 12 +- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 31 ++-- .../ftl/fixtures/test_app_cases/.gitignore | 1 + .../MainActivity_robo_script.json | 150 ++++++++++++++++++ .../test_app_cases/flank-single-robo.yml | 7 + 15 files changed, 286 insertions(+), 84 deletions(-) create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/.gitignore create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/MainActivity_robo_script.json create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-single-robo.yml diff --git a/test_app/app/src/androidTest/java/com/example/test_app/BaseInstrumentedTest.kt b/test_app/app/src/androidTest/java/com/example/test_app/BaseInstrumentedTest.kt index 9fed6b54e7..2fe0665455 100644 --- a/test_app/app/src/androidTest/java/com/example/test_app/BaseInstrumentedTest.kt +++ b/test_app/app/src/androidTest/java/com/example/test_app/BaseInstrumentedTest.kt @@ -1,12 +1,12 @@ package com.example.test_app import android.Manifest -import android.os.SystemClock import androidx.test.espresso.intent.rule.IntentsTestRule import androidx.test.rule.GrantPermissionRule import com.example.test_app.screenshot.ScreenshotTestRule import org.junit.Assert.assertTrue import org.junit.Rule +import kotlin.random.Random abstract class BaseInstrumentedTest { @@ -29,7 +29,7 @@ abstract class BaseInstrumentedTest { val result: Boolean = when (BuildConfig.FLAVOR_type) { "success" -> true "error" -> false - "flaky" -> SystemClock.uptimeMillis() % 2 == 0L + "flaky" -> Random.nextBoolean() else -> throw Error("Invalid flavour type") } assertTrue(result) diff --git a/test_app/app/src/main/java/com/example/test_app/MainActivity.kt b/test_app/app/src/main/java/com/example/test_app/MainActivity.kt index 7fbc9db1cf..46fe90fade 100644 --- a/test_app/app/src/main/java/com/example/test_app/MainActivity.kt +++ b/test_app/app/src/main/java/com/example/test_app/MainActivity.kt @@ -1,16 +1,31 @@ package com.example.test_app +import android.content.DialogInterface import androidx.appcompat.app.AppCompatActivity import android.os.Bundle +import android.widget.Button +import android.widget.Toast +import androidx.appcompat.app.AlertDialog class MainActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - } - fun foo() { - println("foo!") + findViewById