Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support develocity-testing-annotations out of the box #245

Merged
merged 2 commits into from
Dec 4, 2023
Merged

Support develocity-testing-annotations out of the box #245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
399e286
Add out-of-the-box support for develocity testing annotations
skuzzle Dec 4, 2023
d292e25
Rename GE section to Develocity
skuzzle Dec 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.adoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,7 +401,7 @@ This means that all executions of the same test may not be grouped in the consol

image:gradle-console-test-retry-reporting.png[Gradle console reporting, align="center", title=Flaky test Gradle console output]

=== Gradle Enterprise
=== Develocity

Gradle build scans (`--scan` option) report discrete test executions as "Execution [N of total]" and will mark a test with both a _failed_ and then a _passed_ outcome as _flaky_.

Expand Down
3 changes: 2 additions & 1 deletion plugin/src/main/java/org/gradle/testretry/internal/filter/ClassRetryMatcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ public class ClassRetryMatcher {

private static final List<String> IMPLICIT_INCLUDE_ANNOTATION_CLASSES = unmodifiableList(asList(
"spock.lang.Stepwise", // Spock's @Stepwise annotated classes must be retried as a whole
"com.gradle.enterprise.testing.annotations.ClassRetry" //common testing annotations
"com.gradle.enterprise.testing.annotations.ClassRetry", // common testing annotations
"com.gradle.develocity.testing.annotations.ClassRetry" // common testing annotations
));

private final AnnotationInspector annotationInspector;
Expand Down
17 changes: 12 additions & 5 deletions plugin/src/test/groovy/org/gradle/testretry/testframework/JUnit4FuncTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,13 +536,17 @@ class JUnit4FuncTest extends AbstractFrameworkFuncTest {
gradleVersion << GRADLE_VERSIONS_UNDER_TEST
}

def "can rerun on whole class via annotation (gradle version #gradleVersion)"() {
def "can rerun on whole class via annotation (gradle version #gradleVersion and retry annotation #retryAnnotation)"() {
given:
buildFile << """
dependencies {
testImplementation 'com.gradle:develocity-testing-annotations:2.0'
testImplementation 'com.gradle:gradle-enterprise-testing-annotations:1.1.2'
}
test.retry {
maxRetries = 1
classRetry {
includeAnnotationClasses.add('*ClassRetry')
includeAnnotationClasses.add('*CustomClassRetry')
}
}
"""
Expand All @@ -556,15 +560,15 @@ class JUnit4FuncTest extends AbstractFrameworkFuncTest {

@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
public @interface ClassRetry {
public @interface CustomClassRetry {

}
"""

writeJavaTestSource """
package acme;

@ClassRetry
@$retryAnnotation
public class FlakyTests {
@org.junit.Test
public void a() {
Expand All @@ -588,7 +592,10 @@ class JUnit4FuncTest extends AbstractFrameworkFuncTest {
}

where:
gradleVersion << GRADLE_VERSIONS_UNDER_TEST
[gradleVersion, retryAnnotation] << [
GRADLE_VERSIONS_UNDER_TEST,
["acme.CustomClassRetry", "com.gradle.enterprise.testing.annotations.ClassRetry", "com.gradle.develocity.testing.annotations.ClassRetry"]
].combinations()
}

def "handles flaky setup that prevents the retries of initially failed methods (gradle version #gradleVersion)"() {
Expand Down
32 changes: 19 additions & 13 deletions plugin/src/test/groovy/org/gradle/testretry/testframework/JUnit5FuncTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,13 +320,17 @@ class JUnit5FuncTest extends AbstractFrameworkFuncTest {
gradleVersion << GRADLE_VERSIONS_UNDER_TEST
}

def "can rerun on whole class via annotation (gradle version #gradleVersion)"() {
def "can rerun on whole class via annotation (gradle version #gradleVersion and retry annotation #retryAnnotation)"() {
given:
buildFile << """
dependencies {
testImplementation 'com.gradle:develocity-testing-annotations:2.0'
testImplementation 'com.gradle:gradle-enterprise-testing-annotations:1.1.2'
}
test.retry {
maxRetries = 1
classRetry {
includeAnnotationClasses.add('*ClassRetry')
includeAnnotationClasses.add('*CustomClassRetry')
}
}
"""
Expand All @@ -340,15 +344,15 @@ class JUnit5FuncTest extends AbstractFrameworkFuncTest {

@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
public @interface ClassRetry {
public @interface CustomClassRetry {

}
"""

writeJavaTestSource """
package acme;

@ClassRetry
@$retryAnnotation
class FlakyTests {
@org.junit.jupiter.api.Test
void a() {
Expand All @@ -372,7 +376,10 @@ class JUnit5FuncTest extends AbstractFrameworkFuncTest {
}

where:
gradleVersion << GRADLE_VERSIONS_UNDER_TEST
[gradleVersion, retryAnnotation] << [
GRADLE_VERSIONS_UNDER_TEST,
["acme.CustomClassRetry", "com.gradle.enterprise.testing.annotations.ClassRetry", "com.gradle.develocity.testing.annotations.ClassRetry"]
].combinations()
}

def "handles flaky setup that prevents the retries of initially failed methods (gradle version #gradleVersion)"() {
Expand Down Expand Up @@ -500,19 +507,19 @@ class JUnit5FuncTest extends AbstractFrameworkFuncTest {
(1..2).collect {
writeJavaTestSource """
package acme;

import org.junit.jupiter.api.*;

public class Test${it} {
@Test
void testOk() {
}

@Test
void testFlaky() {
${flakyAssert("${it}")}
}
}

}
"""
}
Expand All @@ -522,7 +529,7 @@ class JUnit5FuncTest extends AbstractFrameworkFuncTest {

import org.junit.jupiter.api.*;
import org.junit.platform.suite.api.*;

@Suite
@SelectClasses({Test1.class,Test2.class})
public class TestSuite {
Expand All @@ -542,7 +549,6 @@ class JUnit5FuncTest extends AbstractFrameworkFuncTest {
it.count("${classAndMethodForSuite("Test2", "testOk()", gradleVersion)} PASSED") == 1
it.count("${classAndMethodForSuite("Test2", "testFlaky()", gradleVersion)} FAILED") == 1
it.count("${classAndMethodForSuite("Test2", "testFlaky()", gradleVersion)} PASSED") == 1

}

where:
Expand Down Expand Up @@ -582,7 +588,7 @@ class JUnit5FuncTest extends AbstractFrameworkFuncTest {
@Test
void testOk() {
}

@Test
void testFlaky() {
${flakyAssert("nested1")}
Expand Down
17 changes: 12 additions & 5 deletions plugin/src/test/groovy/org/gradle/testretry/testframework/SpockBaseFuncTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,13 +312,17 @@ abstract class SpockBaseFuncTest extends AbstractFrameworkFuncTest {
gradleVersion << GRADLE_VERSIONS_UNDER_TEST
}

def "can rerun on whole class via annotation (gradle version #gradleVersion)"() {
def "can rerun on whole class via annotation (gradle version #gradleVersion and retry annotation #retryAnnotation)"() {
given:
buildFile << """
dependencies {
testImplementation 'com.gradle:develocity-testing-annotations:2.0'
testImplementation 'com.gradle:gradle-enterprise-testing-annotations:1.1.2'
}
test.retry {
maxRetries = 1
classRetry {
includeAnnotationClasses.add('*ClassRetry')
includeAnnotationClasses.add('*CustomClassRetry')
}
}
"""
Expand All @@ -333,15 +337,15 @@ abstract class SpockBaseFuncTest extends AbstractFrameworkFuncTest {

@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
@interface ClassRetry {
@interface CustomClassRetry {

}
"""

writeGroovyTestSource """
package acme

@ClassRetry
@$retryAnnotation
class FlakyTests extends spock.lang.Specification {
def "parentTest"() {
expect:
Expand Down Expand Up @@ -372,7 +376,10 @@ abstract class SpockBaseFuncTest extends AbstractFrameworkFuncTest {
}

where:
gradleVersion << GRADLE_VERSIONS_UNDER_TEST
[gradleVersion, retryAnnotation] << [
GRADLE_VERSIONS_UNDER_TEST,
["acme.CustomClassRetry", "com.gradle.enterprise.testing.annotations.ClassRetry", "com.gradle.develocity.testing.annotations.ClassRetry"]
].combinations()
}

def "only track a @Retry test method once to ensure it was re-ran successfully"() {
Expand Down