diff --git a/README.md b/README.md index 92bea0212f..a45b7f4abc 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,9 @@ flank: # useful if you need to grant permissions or login before other tests run test-targets-always-run: - class com.example.app.ExampleUiTest#testPasses + # directories to download after the test runs from the result bucket + directories-to-download: + - /sdcard/screenshots ``` ### CI integration diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index e8b335335b..929ce3b6d2 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -67,6 +67,7 @@ class AndroidArgs( override val repeatTests = cli?.repeatTests ?: flank.repeatTests override val smartFlankGcsPath = flank.smartFlankGcsPath override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun + val directoriesToDownload = cli?.directoriesToDownload ?: flank.directoriesToDownload // computed properties not specified in yaml override val testShardChunks: List> by lazy { @@ -159,6 +160,8 @@ ${devicesToString(devices)} testShards: $testShards repeatTests: $repeatTests smartFlankGcsPath: $smartFlankGcsPath + directories-to-download: +${listToString(directoriesToDownload)} test-targets-always-run: ${listToString(testTargetsAlwaysRun)} """.trimIndent() diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt index a5bb50af25..b0d622e5fa 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt @@ -13,10 +13,12 @@ class FlankYmlParams( val smartFlankGcsPath: String = "", @field:JsonProperty("test-targets-always-run") - val testTargetsAlwaysRun: List = emptyList() + val testTargetsAlwaysRun: List = emptyList(), + @field:JsonProperty("directories-to-download") + val directoriesToDownload: List = emptyList() ) { companion object : IYmlKeys { - override val keys = listOf("testShards", "repeatTests", "smartFlankGcsPath", "test-targets-always-run") + override val keys = listOf("testShards", "repeatTests", "smartFlankGcsPath", "test-targets-always-run", "directories-to-download") } init { diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index d39ece28ff..237a76a987 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -138,6 +138,16 @@ class AndroidRunCommand : Runnable { ) var directoriesToPull: List? = null + @Option( + names = ["--directories-to-download"], + split = ",", + description = ["A list of paths that will be downloaded from the resulting bucket " + + "to the designated local results folder after the test is complete. These must be absolute paths " + + "(for example, --directories-to-download /images/tempDir1,/data/local/tmp/tempDir2). " + + "Path names are restricted to the characters a-zA-Z0-9_-./+."] + ) + var directoriesToDownload: List? = null + @Option( names = ["--device"], split = ",", diff --git a/test_runner/src/main/kotlin/ftl/run/TestRunner.kt b/test_runner/src/main/kotlin/ftl/run/TestRunner.kt index 3b20080bce..002de841e7 100644 --- a/test_runner/src/main/kotlin/ftl/run/TestRunner.kt +++ b/test_runner/src/main/kotlin/ftl/run/TestRunner.kt @@ -200,16 +200,20 @@ object TestRunner { finished && notDownloaded } + val lastAndroidArgs = lastArgs() as? AndroidArgs + val directoriesToDownload = lastAndroidArgs?.directoriesToDownload ?: emptyList() + print(indent) runBlocking { filtered.forEach { matrix -> launch { val prefix = Storage.BlobListOption.prefix(matrix.gcsPathWithoutRootBucket) val result = GcStorage.storage.list(matrix.gcsRootBucket, prefix, fields) + val artifactsToDownload = ArtifactRegex.artifactsToDownload(matrix, directoriesToDownload) result.iterateAll().forEach { blob -> val blobPath = blob.blobId.name - if (blobPath.matches(ArtifactRegex.testResultRgx)) { + if (artifactsToDownload.find { blobPath.matches(it) } != null) { val downloadFile = Paths.get(FtlConstants.localResultsDir, blobPath) print(".") if (!downloadFile.toFile().exists()) { diff --git a/test_runner/src/main/kotlin/ftl/util/ArtifactRegex.kt b/test_runner/src/main/kotlin/ftl/util/ArtifactRegex.kt index c3069c1181..30b55b92db 100644 --- a/test_runner/src/main/kotlin/ftl/util/ArtifactRegex.kt +++ b/test_runner/src/main/kotlin/ftl/util/ArtifactRegex.kt @@ -1,7 +1,22 @@ package ftl.util +import com.google.common.annotations.VisibleForTesting +import ftl.json.SavedMatrix + object ArtifactRegex { + @VisibleForTesting val testResultRgx = Regex(".*test_result_\\d+\\.xml$") + @VisibleForTesting val screenshotRgx = Regex(".*\\.png$") + + /** + * @return a list of regex's that should be downloaded + */ + fun artifactsToDownload(matrix: SavedMatrix, directoriesToDownload: List): List { + val directoriesRegexToDownload = directoriesToDownload + .map { "^${Regex.escape(matrix.gcsPathWithoutRootBucket)}/.*/artifacts${Regex.escape(it)}/.*" } + .map { Regex(it) } + return listOf(testResultRgx, screenshotRgx) + directoriesRegexToDownload + } } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 0d0f9ee0bc..1ad26b1b97 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -59,6 +59,9 @@ class AndroidArgsTest { flank: testShards: 7 repeatTests: 8 + directories-to-download: + - /sdcard/screenshots + - /sdcard/screenshots2 test-targets-always-run: - class example.Test#grantPermission - class example.Test#grantPermission2 @@ -154,6 +157,7 @@ class AndroidArgsTest { // FlankYml assert(testShards, 7) assert(repeatTests, 8) + assert(directoriesToDownload, listOf("/sdcard/screenshots", "/sdcard/screenshots2")) assert( testTargetsAlwaysRun, listOf( "class example.Test#grantPermission", @@ -206,6 +210,9 @@ AndroidArgs testShards: 7 repeatTests: 8 smartFlankGcsPath:${' '} + directories-to-download: + - /sdcard/screenshots + - /sdcard/screenshots2 test-targets-always-run: - class example.Test#grantPermission - class example.Test#grantPermission2 @@ -245,6 +252,7 @@ AndroidArgs // FlankYml assert(testShards, 1) assert(repeatTests, 1) + assert(directoriesToDownload, empty) assert(testTargetsAlwaysRun, empty) } } @@ -476,6 +484,22 @@ AndroidArgs assertThat(androidArgs.directoriesToPull).isEqualTo(listOf("a", "b")) } + @Test + fun cli_directoriesToDownload() { + val cli = AndroidRunCommand() + CommandLine(cli).parse("--directories-to-download=a,b") + + val yaml = """ + gcloud: + app: $appApk + test: $testApk + """ + assertThat(AndroidArgs.load(yaml).directoriesToDownload).isEmpty() + + val androidArgs = AndroidArgs.load(yaml, cli) + assertThat(androidArgs.directoriesToDownload).isEqualTo(listOf("a", "b")) + } + @Test fun cli_device() { val cli = AndroidRunCommand() diff --git a/test_runner/src/test/kotlin/ftl/util/ArtifactRegexTest.kt b/test_runner/src/test/kotlin/ftl/util/ArtifactRegexTest.kt index 0242375f6e..7aae12e829 100644 --- a/test_runner/src/test/kotlin/ftl/util/ArtifactRegexTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/ArtifactRegexTest.kt @@ -1,13 +1,50 @@ package ftl.util import com.google.common.truth.Truth.assertThat +import ftl.json.SavedMatrix import org.junit.Test +import org.mockito.Mockito.`when` +import org.mockito.Mockito.mock class ArtifactRegexTest { + private companion object { + const val TO_DOWNLOAD_1 = "/sdcard/screenshots" + const val TO_DOWNLOAD_2 = "/sdcard/test" + + val TO_DOWNLOAD_REGEX_1 = "^2018-12-21_11:24:44\\.608000_UpOC/shard_0/.*/artifacts/sdcard/screenshots.*" + val TO_DOWNLOAD_REGEX_2 = "^2018-12-21_11:24:44\\.608000_UpOC/shard_0/.*/artifacts/sdcard/test.*" + + const val MATRIX_GCS_PATH_WITHOUT_ROOT_BUCKET = "2018-12-21_11:24:44.608000_UpOC/shard_0" + } + + private val matrix = mock(SavedMatrix::class.java) + @Test fun regexExists() { assertThat(ArtifactRegex.testResultRgx).isNotNull() assertThat(ArtifactRegex.screenshotRgx).isNotNull() } + + @Test + fun `artifactsToDownload should return just testResult and screenshot regex if no directoriesToDownload`() { + assertThat(ArtifactRegex.artifactsToDownload(matrix, emptyList()).map { it.pattern }) + .containsExactly(ArtifactRegex.testResultRgx, ArtifactRegex.screenshotRgx) + } + + @Test + fun `artifactsToDownload should return testResult, screenshot regex and directoriesToDownload regex if one`() { + `when`(matrix.gcsPathWithoutRootBucket).thenReturn(MATRIX_GCS_PATH_WITHOUT_ROOT_BUCKET) + + assertThat(ArtifactRegex.artifactsToDownload(matrix, listOf(TO_DOWNLOAD_1)).map { it.pattern }) + .containsExactly(ArtifactRegex.testResultRgx.pattern, ArtifactRegex.screenshotRgx.pattern, TO_DOWNLOAD_REGEX_1) + } + + @Test + fun `artifactsToDownload should return testResult, screenshot regex and directoriesToDownload regex if multiple`() { + `when`(matrix.gcsPathWithoutRootBucket).thenReturn(MATRIX_GCS_PATH_WITHOUT_ROOT_BUCKET) + + assertThat(ArtifactRegex.artifactsToDownload(matrix, listOf(TO_DOWNLOAD_1, TO_DOWNLOAD_2)).map { it.pattern }) + .containsExactly(ArtifactRegex.testResultRgx.pattern, ArtifactRegex.screenshotRgx.pattern, TO_DOWNLOAD_REGEX_1, TO_DOWNLOAD_REGEX_2) + } }