diff --git a/test_app/apks/invalid.apk b/test_app/apks/invalid.apk new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 16d3c21003..618033dd1d 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -70,9 +70,12 @@ class AndroidArgs( override val smartFlankGcsPath = flank.smartFlankGcsPath override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload + override val disableSharding = cli?.disableSharding ?: flank.disableSharding // computed properties not specified in yaml override val testShardChunks: List> by lazy { + if (disableSharding) return@lazy listOf(emptyList()) + // Download test APK if necessary so it can be used to validate test methods var testLocalApk = testApk if (testApk.startsWith(FtlConstants.GCS_PREFIX)) { @@ -168,6 +171,7 @@ ${devicesToString(devices)} ${listToString(filesToDownload)} test-targets-always-run: ${listToString(testTargetsAlwaysRun)} + disableSharding: $disableSharding """.trimIndent() } diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 51912cdb09..654db94981 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -21,6 +21,7 @@ interface IArgs { val smartFlankGcsPath: String val testTargetsAlwaysRun: List val filesToDownload: List + val disableSharding: Boolean // computed property val testShardChunks: List> diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index edacaa1dc7..4e7c0f727d 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -55,6 +55,7 @@ class IosArgs( override val smartFlankGcsPath = flank.smartFlankGcsPath override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload + override val disableSharding = cli?.disableSharding ?: flank.disableSharding private val iosFlank = iosFlankYml.flank val testTargets = cli?.testTargets ?: iosFlank.testTargets @@ -133,6 +134,7 @@ ${listToString(filesToDownload)} # iOS flank test-targets: ${listToString(testTargets)} + disableSharding: $disableSharding """.trimIndent() } diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt index 2a39f22659..a36774fbe0 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt @@ -12,6 +12,7 @@ class FlankYmlParams( val shardTime: Int = -1, val repeatTests: Int = 1, val smartFlankGcsPath: String = "", + val disableSharding: Boolean = false, @field:JsonProperty("test-targets-always-run") val testTargetsAlwaysRun: List = emptyList(), @@ -21,7 +22,7 @@ class FlankYmlParams( ) { companion object : IYmlKeys { override val keys = listOf( - "testShards", "shardTime", "repeatTests", "smartFlankGcsPath", "test-targets-always-run", "files-to-download" + "testShards", "shardTime", "repeatTests", "smartFlankGcsPath", "disableSharding", "test-targets-always-run", "files-to-download" ) } diff --git a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt index 1034a4abf3..4f4235cd97 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt @@ -38,7 +38,8 @@ class GcloudYmlParams( ) { companion object : IYmlKeys { override val keys = - listOf("results-bucket", "results-dir", "record-video", "timeout", "async", "project", "results-history-name", "flaky-test-attempts") + listOf("results-bucket", "results-dir", "record-video", "timeout", "async", "project", + "results-history-name", "flaky-test-attempts") } init { diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 2bcb570f54..a1033a2496 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -251,6 +251,12 @@ class AndroidRunCommand : Runnable { ) var filesToDownload: List? = null + @Option( + names = ["--disable-sharding"], + description = ["Disable sharding."] + ) + var disableSharding: Boolean? = null + @Option( names = ["--results-dir"], description = [ diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index 280d24eddc..f0374c71e6 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -142,6 +142,12 @@ class IosRunCommand : Runnable { ) var filesToDownload: List? = null + @Option( + names = ["--disable-sharding"], + description = ["Disable sharding."] + ) + var disableSharding: Boolean? = null + @Option( names = ["--test"], description = ["The path to the test package (a zip file containing the iOS app " + diff --git a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt index 5f24af7cd4..df804a7ea2 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt @@ -43,8 +43,8 @@ object GcIosTestMatrix { val methods = args.testShardChunks.elementAt(testShardsIndex) // Parameterized tests on iOS don't shard correctly. - // Avoid changing Xctestrun file when test shards is 1. - val generatedXctestrun = if (args.testShards == 1) { + // Avoid changing Xctestrun file when disableSharding is on. + val generatedXctestrun = if (args.disableSharding) { xcTestParsed.toByteArray() } else { Xctestrun.rewrite(xcTestParsed, methods) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index cafb841519..e525a9ee69 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -18,6 +18,7 @@ import picocli.CommandLine class AndroidArgsTest { private val empty = emptyList() private val appApk = "../test_app/apks/app-debug.apk" + private val invalidApk = "../test_app/apks/invalid.apk" private val testApk = "../test_app/apks/app-debug-androidTest.apk" private val testErrorApk = "../test_app/apks/error-androidTest.apk" private val appApkAbsolutePath = appApk.absolutePath() @@ -67,6 +68,7 @@ class AndroidArgsTest { test-targets-always-run: - class example.Test#grantPermission - class example.Test#grantPermission2 + disableSharding: true """ @Rule @@ -168,6 +170,7 @@ class AndroidArgsTest { "class example.Test#grantPermission2" ) ) + assert(disableSharding, true) } } @@ -222,6 +225,7 @@ AndroidArgs test-targets-always-run: - class example.Test#grantPermission - class example.Test#grantPermission2 + disableSharding: true """.trimIndent() ) } @@ -261,6 +265,7 @@ AndroidArgs assert(repeatTests, 1) assert(filesToDownload, empty) assert(testTargetsAlwaysRun, empty) + assert(disableSharding, false) } } @@ -300,6 +305,28 @@ AndroidArgs assertThat(androidArgs.testApk).isEqualTo(testApkAbsolutePath) } + @Test + fun `disableSharding allows using invalid apk`() { + val yaml = """ + gcloud: + app: $invalidApk + test: $invalidApk + flank: + disableSharding: true + """ + AndroidArgs.load(yaml).testShardChunks + } + + @Test(expected = RuntimeException::class) + fun `Invalid apk throws`() { + val yaml = """ + gcloud: + app: $invalidApk + test: $invalidApk + """ + AndroidArgs.load(yaml).testShardChunks + } + @Test fun cli_app() { val cli = AndroidRunCommand() @@ -673,7 +700,7 @@ AndroidArgs } @Test - fun `cli shardTime`() { + fun cli_shardTime() { val cli = AndroidRunCommand() CommandLine(cli).parse("--shard-time=3") @@ -689,6 +716,23 @@ AndroidArgs assertThat(AndroidArgs.load(yaml, cli).shardTime).isEqualTo(3) } + @Test + fun cli_disableSharding() { + val cli = AndroidRunCommand() + CommandLine(cli).parse("--disable-sharding") + + val yaml = """ + gcloud: + app: $appApk + test: $testApk + + flank: + disableSharding: false + """ + assertThat(AndroidArgs.load(yaml).disableSharding).isEqualTo(false) + assertThat(AndroidArgs.load(yaml, cli).disableSharding).isEqualTo(true) + } + @Test fun cli_repeatTests() { val cli = AndroidRunCommand() diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 469f6e27d5..4e6ad0b935 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -63,6 +63,7 @@ class IosArgsTest { test-targets: - b/testBasicSelection - b/testBasicSelection2 + disableSharding: true """ @Rule @@ -128,6 +129,7 @@ class IosArgsTest { assert(testTargets, listOf("b/testBasicSelection", "b/testBasicSelection2")) assert(flakyTestAttempts, 4) + assert(disableSharding, true) } } @@ -174,6 +176,7 @@ IosArgs test-targets: - b/testBasicSelection - b/testBasicSelection2 + disableSharding: true """.trimIndent() ) } @@ -208,6 +211,7 @@ IosArgs assert(repeatTests, 1) assert(testTargetsAlwaysRun, emptyList()) assert(filesToDownload, emptyList()) + assert(disableSharding, false) // IosFlankYml assert(testTargets, empty) @@ -377,7 +381,7 @@ IosArgs } @Test - fun `cli shardTime`() { + fun cli_shardTime() { val cli = IosRunCommand() CommandLine(cli).parse("--shard-time=3") @@ -393,6 +397,23 @@ IosArgs assertThat(IosArgs.load(yaml, cli).shardTime).isEqualTo(3) } + @Test + fun cli_disableSharding() { + val cli = IosRunCommand() + CommandLine(cli).parse("--disable-sharding") + + val yaml = """ + gcloud: + test: $testPath + xctestrun-file: $xctestrunFile + + flank: + disableSharding: false + """ + assertThat(IosArgs.load(yaml).disableSharding).isEqualTo(false) + assertThat(IosArgs.load(yaml, cli).disableSharding).isEqualTo(true) + } + @Test fun cli_repeatTests() { val cli = IosRunCommand() diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index b8f9504477..efaff3944c 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -84,6 +84,7 @@ class AndroidRunCommandTest { assertThat(cmd.filesToDownload).isNull() assertThat(cmd.resultsDir).isNull() assertThat(cmd.flakyTestAttempts).isNull() + assertThat(cmd.disableSharding).isNull() } @Test @@ -300,4 +301,12 @@ class AndroidRunCommandTest { assertThat(cmd.shardTime).isEqualTo(99) } + + @Test + fun `disableSharding parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parse("--disable-sharding") + + assertThat(cmd.disableSharding).isEqualTo(true) + } } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index a7a3fc1680..07a93d805f 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -71,6 +71,7 @@ class IosRunCommandTest { assertThat(cmd.testTargetsAlwaysRun).isNull() assertThat(cmd.testTargets).isNull() assertThat(cmd.filesToDownload).isNull() + assertThat(cmd.disableSharding).isNull() assertThat(cmd.test).isNull() assertThat(cmd.xctestrunFile).isNull() assertThat(cmd.xcodeVersion).isNull() @@ -234,4 +235,12 @@ class IosRunCommandTest { assertThat(cmd.shardTime).isEqualTo(99) } + + @Test + fun `disableShard parse`() { + val cmd = IosRunCommand() + CommandLine(cmd).parse("--disable-sharding") + + assertThat(cmd.disableSharding).isEqualTo(true) + } }