From e7564d42ecefba8593d0806fb04b9f78fa9b1c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Mon, 6 Apr 2020 04:28:01 +0200 Subject: [PATCH 1/3] Add support for other-files option --- test_runner/flank.yml | 7 +++++++ test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt | 2 ++ .../main/kotlin/ftl/args/yml/AndroidGcloudYml.kt | 4 ++++ .../cli/firebase/test/android/AndroidRunCommand.kt | 11 +++++++++++ .../src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt | 12 ++++++++++++ .../main/kotlin/ftl/run/platform/RunAndroidTests.kt | 13 +++++++++++++ .../src/test/kotlin/ftl/args/AndroidArgsTest.kt | 13 +++++++++++++ .../firebase/test/android/AndroidRunCommandTest.kt | 9 +++++++++ .../test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt | 3 +++ 9 files changed, 74 insertions(+) diff --git a/test_runner/flank.yml b/test_runner/flank.yml index d98452cd1a..c076bfe281 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -78,6 +78,13 @@ gcloud: # directories-to-pull: # - /sdcard/ + ## A list of device-path: file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push. + ## Device paths must be under absolute, whitelisted paths (${EXTERNAL_STORAGE}, or ${ANDROID_DATA}/local/tmp). + ## Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). + # other-files + # - /sdcard/dir1/file1.txt: local/file.txt + # - /sdcard/dir2/file2.jpg: gs://bucket/file.jpg + ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only). ## Enabled by default, use --no-performance-metrics to disable. # performance-metrics: true diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 59a52e2911..573203e083 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -59,6 +59,7 @@ class AndroidArgs( val useOrchestrator = cli?.useOrchestrator ?: cli?.noUseOrchestrator?.not() ?: androidGcloud.useOrchestrator val environmentVariables = cli?.environmentVariables ?: androidGcloud.environmentVariables val directoriesToPull = cli?.directoriesToPull ?: androidGcloud.directoriesToPull + val otherFiles = cli?.otherFiles ?: androidGcloud.otherFiles // TODO assert gcs file or evaluate path val performanceMetrics = cli?.performanceMetrics ?: cli?.noPerformanceMetrics?.not() ?: androidGcloud.performanceMetrics val testRunnerClass = cli?.testRunnerClass ?: androidGcloud.testRunnerClass val testTargets = cli?.testTargets ?: androidGcloud.testTargets.filterNotNull() @@ -128,6 +129,7 @@ AndroidArgs auto-google-login: $autoGoogleLogin use-orchestrator: $useOrchestrator directories-to-pull:${listToString(directoriesToPull)} + other-files:${mapToString(otherFiles)} performance-metrics: $performanceMetrics test-runner-class: $testRunnerClass test-targets:${listToString(testTargets)} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt index 94a6b8f034..2ceb57746a 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt @@ -32,6 +32,9 @@ class AndroidGcloudYmlParams( @field:JsonProperty("directories-to-pull") val directoriesToPull: List = emptyList(), + @field:JsonProperty("other-files") + val otherFiles: Map = emptyMap(), + @field:JsonProperty("performance-metrics") val performanceMetrics: Boolean = FlankDefaults.DISABLE_PERFORMANCE_METRICS, @@ -52,6 +55,7 @@ class AndroidGcloudYmlParams( "use-orchestrator", "environment-variables", "directories-to-pull", + "other-files", "performance-metrics", "test-runner-class", "test-targets", diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index cfd2cf1c19..b1c25a3ea3 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -153,6 +153,17 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { ) var directoriesToPull: List? = null + @Option( + names = ["--other-files"], + split = ",", + description = [ + "A list of device-path=file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push.", + "Device paths must be under absolute, whitelisted paths (\${EXTERNAL_STORAGE}, or \${ANDROID_DATA}/local/tmp).", + "Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). " + ] + ) + var otherFiles: Map? = null + @Option( names = ["--performance-metrics"], description = ["Monitor and record performance metrics: CPU, memory, " + diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt index 05b01dccef..2d8eee87e2 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt @@ -6,12 +6,14 @@ import com.google.api.services.testing.model.AndroidDeviceList import com.google.api.services.testing.model.AndroidInstrumentationTest import com.google.api.services.testing.model.Apk import com.google.api.services.testing.model.ClientInfo +import com.google.api.services.testing.model.DeviceFile import com.google.api.services.testing.model.EnvironmentMatrix import com.google.api.services.testing.model.EnvironmentVariable import com.google.api.services.testing.model.FileReference import com.google.api.services.testing.model.GoogleAuto import com.google.api.services.testing.model.GoogleCloudStorage import com.google.api.services.testing.model.ManualSharding +import com.google.api.services.testing.model.RegularFile import com.google.api.services.testing.model.ResultStorage import com.google.api.services.testing.model.ShardingOption import com.google.api.services.testing.model.TestMatrix @@ -35,6 +37,7 @@ object GcAndroidTestMatrix { fun build( appApkGcsPath: String, testApkGcsPath: String, + otherFiles: Map, runGcsPath: String, androidDeviceList: AndroidDeviceList, testShards: ShardChunks, @@ -84,6 +87,7 @@ object GcAndroidTestMatrix { .setNetworkProfile(args.networkProfile) .setDirectoriesToPull(args.directoriesToPull) .setAdditionalApks(additionalApkGcsPaths.mapGcsPathsToApks()) + .setFilesToPush(otherFiles.mapToDeviceFiles()) if (args.environmentVariables.isNotEmpty()) { testSetup.environmentVariables = @@ -125,3 +129,11 @@ object GcAndroidTestMatrix { private fun List?.mapGcsPathsToApks(): List? = this ?.takeIf { it.isNotEmpty() } ?.map { gcsPath -> Apk().setLocation(FileReference().setGcsPath(gcsPath)) } + +private fun Map.mapToDeviceFiles() = map { (devicePath: String, gcsFilePath: String) -> + DeviceFile().setRegularFile( + RegularFile() + .setDevicePath(devicePath) + .setContent(FileReference().setGcsPath(gcsFilePath)) + ) +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index f5876204c1..eaa841869c 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -34,6 +34,7 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS val runCount = args.repeatTests val history = GcToolResults.createToolResultsHistory(args) val resolvedTestApks = args.getResolvedTestApks() + val otherGcsFiles = args.otherFiles.uploadOtherFiles(args.resultsBucket, runGcsPath) val allTestShardChunks: ShardChunks = resolvedTestApks.map { apks: ResolvedTestApks -> // Ensure we only shard tests that are part of the test apk. Use the resolved test apk path to make sure @@ -45,6 +46,7 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS args = args, runGcsPath = runGcsPath ), + otherFiles = otherGcsFiles, runGcsPath = runGcsPath, androidDeviceList = androidDeviceList, testShards = testShards, @@ -81,6 +83,7 @@ private suspend fun executeAndroidTestMatrix( args: AndroidArgs, testShards: ShardChunks, uploadedTestApks: UploadedTestApks, + otherFiles: Map, androidDeviceList: AndroidDeviceList, history: ToolResultsHistory, runCount: Int @@ -94,6 +97,7 @@ private suspend fun executeAndroidTestMatrix( androidDeviceList = androidDeviceList, testShards = testShards, args = args, + otherFiles = otherFiles, toolResultsHistory = history, additionalApkGcsPaths = uploadedTestApks.additionalApks ).executeWithRetry() @@ -123,3 +127,12 @@ private suspend fun uploadTestApks( additionalApks = additionalApkGcsPaths.awaitAll() ) } + +private suspend fun Map.uploadOtherFiles( + gcsBucket: String, + runGcsPath: String +): Map = coroutineScope { + map { (devicePath: String, filePath: String) -> + async(Dispatchers.IO) { devicePath to GcStorage.upload(filePath, gcsBucket, runGcsPath) } + }.awaitAll().toMap() +} diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 71c19806f0..d68a9be8f3 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -60,6 +60,9 @@ class AndroidArgsTest { directories-to-pull: - /sdcard/screenshots - /sdcard/screenshots2 + other-files: + /sdcard/dir1/file1.txt: $appApk + /sdcard/dir2/file2.jpg: $testApk performance-metrics: false test-runner-class: com.foo.TestRunner test-targets: @@ -190,6 +193,12 @@ class AndroidArgsTest { assert(useOrchestrator, false) assert(environmentVariables, linkedMapOf("clearPackageData" to "true", "randomEnvVar" to "false")) assert(directoriesToPull, listOf("/sdcard/screenshots", "/sdcard/screenshots2")) + assert( + otherFiles, mapOf( + "/sdcard/dir1/file1.txt" to appApkAbsolutePath, + "/sdcard/dir2/file2.jpg" to testApkAbsolutePath + ) + ) assert(performanceMetrics, false) assert(testRunnerClass, "com.foo.TestRunner") assert( @@ -251,6 +260,9 @@ AndroidArgs directories-to-pull: - /sdcard/screenshots - /sdcard/screenshots2 + other-files: + /sdcard/dir1/file1.txt: $appApkAbsolutePath + /sdcard/dir2/file2.jpg: $testApkAbsolutePath performance-metrics: false test-runner-class: com.foo.TestRunner test-targets: @@ -314,6 +326,7 @@ AndroidArgs auto-google-login: false use-orchestrator: true directories-to-pull: + other-files: performance-metrics: false test-runner-class: null test-targets: diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index 6f5626b0ab..efdcc02f7c 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -76,6 +76,7 @@ class AndroidRunCommandTest { assertThat(cmd.testRunnerClass).isNull() assertThat(cmd.environmentVariables).isNull() assertThat(cmd.directoriesToPull).isNull() + assertThat(cmd.otherFiles).isNull() assertThat(cmd.device).isNull() assertThat(cmd.resultsBucket).isNull() assertThat(cmd.recordVideo).isNull() @@ -208,6 +209,14 @@ class AndroidRunCommandTest { assertThat(cmd.directoriesToPull).hasSize(2) } + @Test + fun `otherFiles parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--other-files=a=1,b=2") + + assertThat(cmd.otherFiles).hasSize(2) + } + @Test fun `device parse`() { val cmd = AndroidRunCommand() diff --git a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt index cb1f361153..8108870a84 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt @@ -25,6 +25,7 @@ class GcAndroidTestMatrixTest { appApkGcsPath = "", testApkGcsPath = "", runGcsPath = "", + otherFiles = emptyMap(), androidDeviceList = AndroidDeviceList(), testShards = emptyList(), args = androidArgs, @@ -41,6 +42,7 @@ class GcAndroidTestMatrixTest { appApkGcsPath = "", testApkGcsPath = "", runGcsPath = "", + otherFiles = emptyMap(), androidDeviceList = AndroidDeviceList(), testShards = listOf(listOf("")), args = androidArgs, @@ -61,6 +63,7 @@ class GcAndroidTestMatrixTest { appApkGcsPath = "", testApkGcsPath = "", runGcsPath = "", + otherFiles = emptyMap(), androidDeviceList = AndroidDeviceList(), testShards = emptyList(), args = androidArgs, From 10c2e03bffa22916409df7983551ce27b1010fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Mon, 6 Apr 2020 04:33:18 +0200 Subject: [PATCH 2/3] Update release_notes.md --- release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release_notes.md b/release_notes.md index dbff6e0c9c..6cf2036011 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,4 +1,5 @@ ## next (unreleased) +- [#692](https://github.com/Flank/flank/pull/698) Add support for other-files option. ([jan-gogo](https://github.com/jan-gogo)) - [#695](https://github.com/Flank/flank/pull/695) Add support for additional-apks option. ([jan-gogo](https://github.com/jan-gogo)) - [#683](https://github.com/Flank/flank/pull/683) Print web link. ([pawelpasterz](https://github.com/pawelpasterz)) - [#692](https://github.com/Flank/flank/pull/692) Add support for network-profiles list command & --network-profile option. ([jan-gogo](https://github.com/jan-gogo)) From 56b5c485f880fbe855e9342a164e48da50f678ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Tue, 7 Apr 2020 04:17:31 +0200 Subject: [PATCH 3/3] Process file paths of other files --- .../src/main/kotlin/ftl/args/AndroidArgs.kt | 16 +++++++++------- .../kotlin/ftl/run/platform/RunAndroidTests.kt | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 573203e083..1b3e26aee1 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -50,16 +50,18 @@ class AndroidArgs( override val flakyTestAttempts = cli?.flakyTestAttempts ?: gcloud.flakyTestAttempts private val androidGcloud = androidGcloudYml.gcloud - val appApk = (cli?.app ?: androidGcloud.app ?: fatalError("app is not set")).processApkPath("from app") - val testApk = (cli?.test ?: androidGcloud.test ?: fatalError("test is not set")).processApkPath("from test") - val additionalApks = (cli?.additionalApks ?: androidGcloud.additionalApks).map { it.processApkPath("from additional-apks") } + val appApk = (cli?.app ?: androidGcloud.app ?: fatalError("app is not set")).processFilePath("from app") + val testApk = (cli?.test ?: androidGcloud.test ?: fatalError("test is not set")).processFilePath("from test") + val additionalApks = (cli?.additionalApks ?: androidGcloud.additionalApks).map { it.processFilePath("from additional-apks") } val autoGoogleLogin = cli?.autoGoogleLogin ?: cli?.noAutoGoogleLogin?.not() ?: androidGcloud.autoGoogleLogin // We use not() on noUseOrchestrator because if the flag is on, useOrchestrator needs to be false val useOrchestrator = cli?.useOrchestrator ?: cli?.noUseOrchestrator?.not() ?: androidGcloud.useOrchestrator val environmentVariables = cli?.environmentVariables ?: androidGcloud.environmentVariables val directoriesToPull = cli?.directoriesToPull ?: androidGcloud.directoriesToPull - val otherFiles = cli?.otherFiles ?: androidGcloud.otherFiles // TODO assert gcs file or evaluate path + val otherFiles = (cli?.otherFiles ?: androidGcloud.otherFiles).map { (devicePath, filePath) -> + devicePath to filePath.processFilePath("from otherFiles") + }.toMap() val performanceMetrics = cli?.performanceMetrics ?: cli?.noPerformanceMetrics?.not() ?: androidGcloud.performanceMetrics val testRunnerClass = cli?.testRunnerClass ?: androidGcloud.testRunnerClass val testTargets = cli?.testTargets ?: androidGcloud.testTargets.filterNotNull() @@ -84,8 +86,8 @@ class AndroidArgs( private val androidFlank = androidFlankYml.flank val additionalAppTestApks = (cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks).map { (app, test) -> AppTestPair( - app = app?.processApkPath("from additional-app-test-apks.app"), - test = test.processApkPath("from additional-app-test-apks.test") + app = app?.processFilePath("from additional-app-test-apks.app"), + test = test.processFilePath("from additional-app-test-apks.test") ) } val keepFilePath = cli?.keepFilePath ?: androidFlank.keepFilePath @@ -193,7 +195,7 @@ AndroidArgs } } -private fun String.processApkPath(name: String): String = +private fun String.processFilePath(name: String): String = if (startsWith(FtlConstants.GCS_PREFIX)) this.also { assertGcsFileExists(it) } else evaluateFilePath(this).also { assertFileExists(it, name) } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index eaa841869c..0a70b5250d 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -46,7 +46,7 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS args = args, runGcsPath = runGcsPath ), - otherFiles = otherGcsFiles, + otherFiles = otherGcsFiles, runGcsPath = runGcsPath, androidDeviceList = androidDeviceList, testShards = testShards,