-
Notifications
You must be signed in to change notification settings - Fork 53
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Discard JUnit 5 test runners even after creation, if further filters …
…result in it not contributing any tests (#326) This is interesting for `@Tag`-annotated classes that are skipped completely. If their runner is not prevented from being returned to the instrumentation, Android expects something reported back to it for the class, but in reality JUnit 5 won't be reporting anything, causing a mismatch error
- Loading branch information
1 parent
be10b22
commit d082fce
Showing
9 changed files
with
161 additions
and
84 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
25 changes: 25 additions & 0 deletions
25
instrumentation/core/src/androidTest/java/de/mannodermaus/junit5/TaggedTests.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| package de.mannodermaus.junit5 | ||
|
|
||
| import org.junit.jupiter.api.Assertions.assertEquals | ||
| import org.junit.jupiter.api.Tag | ||
| import org.junit.jupiter.api.Test | ||
|
|
||
| class TaggedTests { | ||
| @Test | ||
| fun includedTest() { | ||
| } | ||
|
|
||
| @Tag("nope") | ||
| @Test | ||
| fun taggedTestDisabledOnMethodLevel() { | ||
| assertEquals(5, 2 + 2) | ||
| } | ||
| } | ||
|
|
||
| @Tag("nope") | ||
| class TaggedTestsDisabledOnClassLevel { | ||
| @Test | ||
| fun excludedTest() { | ||
| assertEquals(5, 2 + 2) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
44 changes: 30 additions & 14 deletions
44
...ion/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnit5RunnerFactory.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,20 +1,36 @@ | ||
| package de.mannodermaus.junit5.internal.runners | ||
|
|
||
| import android.os.Build | ||
| import de.mannodermaus.junit5.internal.extensions.jupiterTestMethods | ||
| import org.junit.runner.Runner | ||
|
|
||
| internal object JUnit5RunnerFactory { | ||
| /** | ||
| * Since we can't reference AndroidJUnit5 directly, use this factory for instantiation. | ||
| * | ||
| * On API 26 and above, delegate to the real implementation to drive JUnit 5 tests. | ||
| * Below that however, they wouldn't work; for this case, delegate a dummy runner | ||
| * which will highlight these tests as ignored. | ||
| */ | ||
| internal fun createJUnit5Runner(klass: Class<*>): Runner = | ||
| if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { | ||
| AndroidJUnit5(klass) | ||
| } else { | ||
| DummyJUnit5(klass) | ||
| } | ||
| /** | ||
| * Since we can't reference AndroidJUnit5 directly, use this factory for instantiation. | ||
| * | ||
| * On API 26 and above, delegate to the real implementation to drive JUnit 5 tests. | ||
| * Below that however, they wouldn't work; for this case, delegate a dummy runner | ||
| * which will highlight these tests as ignored. | ||
| */ | ||
| internal fun tryCreateJUnit5Runner(klass: Class<*>): Runner? { | ||
| val testMethods = klass.jupiterTestMethods() | ||
|
|
||
| if (testMethods.isEmpty()) { | ||
| return null | ||
| } | ||
|
|
||
| val runner = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { | ||
| AndroidJUnit5(klass) | ||
| } else { | ||
| DummyJUnit5(klass, testMethods) | ||
| } | ||
|
|
||
| // It's still possible for the runner to not be relevant to the test run, | ||
| // which is related to how further filters are applied (e.g. via @Tag). | ||
| // Only return the runner to the instrumentation if it has any tests to contribute, | ||
| // otherwise there would be a mismatch between the number of test classes reported | ||
| // to Android, and the number of test classes actually tested with JUnit 5 (ref #298) | ||
| return runner.takeIf(Runner::hasExecutableTests) | ||
| } | ||
|
|
||
| private fun Runner.hasExecutableTests() = | ||
| this.description.children.isNotEmpty() |
93 changes: 52 additions & 41 deletions
93
instrumentation/runner/src/test/kotlin/de/mannodermaus/junit5/AndroidJUnit5BuilderTests.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,60 +1,71 @@ | ||
| package de.mannodermaus.junit5 | ||
|
|
||
| import android.os.Build | ||
| import com.google.common.truth.Truth.assertThat | ||
| import org.junit.jupiter.api.Test | ||
| import org.junit.jupiter.params.ParameterizedTest | ||
| import org.junit.jupiter.params.provider.ValueSource | ||
| import de.mannodermaus.junit5.testutil.AndroidBuildUtils.withApiLevel | ||
| import de.mannodermaus.junit5.testutil.AndroidBuildUtils.withMockedInstrumentation | ||
| import org.junit.jupiter.api.DynamicContainer.dynamicContainer | ||
| import org.junit.jupiter.api.DynamicNode | ||
| import org.junit.jupiter.api.DynamicTest.dynamicTest | ||
| import org.junit.jupiter.api.TestFactory | ||
|
|
||
| class AndroidJUnit5BuilderTests { | ||
|
|
||
| private val builder = AndroidJUnit5Builder() | ||
|
|
||
| @Test | ||
| fun `no runner is created if class only contains top-level test methods`() { | ||
| @TestFactory | ||
| fun `no runner is created if class only contains top-level test methods`() = runTest( | ||
| expectSuccess = false, | ||
| // In Kotlin, a 'Kt'-suffixed class of top-level functions cannot be referenced | ||
| // via the ::class syntax, so construct a reference to the class directly | ||
| val cls = Class.forName(javaClass.packageName + ".TestClassesKt") | ||
|
|
||
| // Top-level tests should be discarded, so no runner must be created for this class | ||
| runTest(cls, expectSuccess = false) | ||
| } | ||
| Class.forName(javaClass.packageName + ".TestClassesKt") | ||
| ) | ||
|
|
||
| @ValueSource( | ||
| classes = [ | ||
| HasTest::class, | ||
| HasRepeatedTest::class, | ||
| HasTestFactory::class, | ||
| HasTestTemplate::class, | ||
| HasParameterizedTest::class, | ||
| HasInnerClassWithTest::class, | ||
| HasTaggedTest::class, | ||
| HasInheritedTestsFromClass::class, | ||
| HasInheritedTestsFromInterface::class, | ||
| ] | ||
| @TestFactory | ||
| fun `runner is created correctly for classes with valid jupiter test methods`() = runTest( | ||
| expectSuccess = true, | ||
| HasTest::class.java, | ||
| HasRepeatedTest::class.java, | ||
| HasTestFactory::class.java, | ||
| HasTestTemplate::class.java, | ||
| HasParameterizedTest::class.java, | ||
| HasInnerClassWithTest::class.java, | ||
| HasTaggedTest::class.java, | ||
| HasInheritedTestsFromClass::class.java, | ||
| HasInheritedTestsFromInterface::class.java, | ||
| ) | ||
| @ParameterizedTest | ||
| fun `runner is created correctly for classes with valid jupiter test methods`(cls: Class<*>) = | ||
| runTest(cls, expectSuccess = true) | ||
|
|
||
| @ValueSource( | ||
| classes = [ | ||
| DoesntHaveTestMethods::class, | ||
| HasJUnit4Tests::class, | ||
| kotlin.time.Duration::class, | ||
| ] | ||
|
|
||
| @TestFactory | ||
| fun `no runner is created if class has no jupiter test methods`() = runTest( | ||
| expectSuccess = false, | ||
| DoesntHaveTestMethods::class.java, | ||
| HasJUnit4Tests::class.java, | ||
| kotlin.time.Duration::class.java, | ||
| ) | ||
| @ParameterizedTest | ||
| fun `no runner is created if class has no jupiter test methods`(cls: Class<*>) = | ||
| runTest(cls, expectSuccess = false) | ||
|
|
||
| /* Private */ | ||
|
|
||
| private fun runTest(cls: Class<*>, expectSuccess: Boolean) { | ||
| val runner = builder.runnerForClass(cls) | ||
| if (expectSuccess) { | ||
| assertThat(runner).isNotNull() | ||
| } else { | ||
| assertThat(runner).isNull() | ||
| private fun runTest(expectSuccess: Boolean, vararg classes: Class<*>): List<DynamicNode> { | ||
| // Generate a test container for each given class, | ||
| // then create two sub-variants for testing both DummyJUnit5 and AndroidJUnit5 | ||
| return classes.map { cls -> | ||
| dynamicContainer( | ||
| /* displayName = */ cls.name, | ||
| /* dynamicNodes = */ setOf(Build.VERSION_CODES.M, Build.VERSION_CODES.TIRAMISU).map { apiLevel -> | ||
| dynamicTest("API Level $apiLevel") { | ||
| withMockedInstrumentation { | ||
| withApiLevel(apiLevel) { | ||
| val runner = builder.runnerForClass(cls) | ||
| if (expectSuccess) { | ||
| assertThat(runner).isNotNull() | ||
| } else { | ||
| assertThat(runner).isNull() | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| ) | ||
| } | ||
| } | ||
| } |