Merge pull request #103 from pitest/bug/report_errors

Log test failures and errors
pitest · Oct 24, 2023 · 797354c · 797354c
2 parents 9d66b69 + 3a40a2d
commit 797354c
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 19 deletions.
diff --git a/pom.xml b/pom.xml
@@ -23,7 +23,7 @@
 		<junit.platform.version>1.9.2</junit.platform.version>
 		<junit.version>5.9.2</junit.version>
 		<mockito.version>2.7.6</mockito.version>
-		<pitest.version>1.9.0</pitest.version>
+		<pitest.version>1.15.2</pitest.version>
 		<cucumber.version>5.0.0</cucumber.version>
 		<spock.version>2.3-groovy-4.0</spock.version>
 		<groovy.version>4.0.11</groovy.version>
@@ -238,7 +238,7 @@
 			<plugin>
 				<groupId>org.jacoco</groupId>
 				<artifactId>jacoco-maven-plugin</artifactId>
-				<version>0.8.5</version>
+				<version>0.8.11</version>
 				<executions>
 					<execution>
 						<goals>

diff --git a/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java b/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java
@@ -130,9 +130,11 @@ public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult
                 if (!identifiers.contains(testIdentifier)) {
                     identifiers.add(testIdentifier);
                 }
-                l.executionFinished(new Description(testIdentifier.getUniqueId(), testClass), false);
+                l.executionFinished(new Description(testIdentifier.getUniqueId(), testClass)
+                        , false, testExecutionResult.getThrowable().orElse(null));
             } else if (testIdentifier.isTest()) {
-                l.executionFinished(new Description(testIdentifier.getUniqueId(), testClass), true);
+                l.executionFinished(new Description(testIdentifier.getUniqueId(), testClass)
+                        , true);
             }
         }
 

diff --git a/src/test/java/org/pitest/junit5/JUnit5TestUnitFinderTest.java b/src/test/java/org/pitest/junit5/JUnit5TestUnitFinderTest.java
@@ -19,6 +19,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 
 import org.junit.jupiter.api.Test;
+import org.opentest4j.AssertionFailedError;
 import org.pitest.junit5.cucumber.RunCucumberTest;
 import org.pitest.junit5.repository.AbstractTestClass;
 import org.pitest.junit5.repository.InterfaceTestClass;
@@ -67,9 +68,11 @@
 import org.pitest.testapi.TestGroupConfig;
 import org.pitest.testapi.TestUnit;
 import org.pitest.testapi.TestUnitExecutionListener;
+import org.spockframework.runtime.ConditionNotSatisfiedError;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 /**
@@ -149,12 +152,14 @@ void detectsFailingTests() {
     void detectsFailingSpockTests() {
         findsAndRunsNTests(1, TestSpecWithFailingFeature.class);
         nTestsFails(1, TestSpecWithFailingFeature.class);
+        errorIsRecorded(t -> t instanceof ConditionNotSatisfiedError, TestSpecWithFailingFeature.class);
     }
 
     @Test
     void detectsErroringTestsWhenPassingTestsPresent() {
         nTestsPass(2, TestClassWithMixedPassAndFail.class);
         nTestsFails(2, TestClassWithMixedPassAndFail.class);
+        errorIsRecorded(t -> t instanceof RuntimeException, TestClassWithMixedPassAndFail.class);
     }
 
     @Test
@@ -265,6 +270,7 @@ void excludesTestsByTag() {
         findsAndRunsNTests(3, new JUnit5TestUnitFinder(new TestGroupConfig().withExcludedGroups("excluded"), emptyList()), TestClassWithTags.class);
     }
 
+    @Test
     void excludesSpockTestsByTag() {
         findsAndRunsNTests(3, new JUnit5TestUnitFinder(new TestGroupConfig().withExcludedGroups("excluded"), emptyList()), TestSpecWithTags.class);
     }
@@ -322,6 +328,7 @@ void findsAndRunsTestsWithSetupSpec() {
     @Test
     void findsAndRunsTestsWithFailingAfterAll() {
         findsAndRunsNTests(2, TestClassWithFailingAfterAll.class);
+        errorIsRecorded(t -> t instanceof AssertionFailedError, TestClassWithFailingAfterAll.class);
     }
 
     @Test
@@ -368,6 +375,11 @@ private void nTestsFails(int n, Class<?> clazz) {
         assertThat(l.failed).hasSize(n);
     }
 
+    private void errorIsRecorded(Predicate<Throwable> p, Class<?> clazz) {
+        RecordingListener l = run(basicConfig(), clazz);
+        assertThat(l.errors).anyMatch(p);
+    }
+
     private RecordingListener run(JUnit5TestUnitFinder underTest, Class<?> clazz) {
         RecordingListener l = new RecordingListener();
         underTest.findTestUnits(clazz, l);
@@ -384,33 +396,24 @@ class RecordingListener implements TestUnitExecutionListener {
     List<Description> started = new ArrayList<>();
     List<Description> failed = new ArrayList<>();
     List<Description> passed = new ArrayList<>();
-    TestUnitExecutionListener l = new TestUnitExecutionListener() {
-        @Override
-        public void executionStarted(Description description) {
-            started.add(description);
-        }
 
-        @Override
-        public void executionFinished(Description description, boolean pass) {
-            if (pass) {
-                passed.add(description);
-            } else {
-                failed.add(description);
-            }
-        }
-    };
+    List<Throwable> errors = new ArrayList<>();
 
     @Override
     public void executionStarted(Description description) {
         started.add(description);
     }
 
     @Override
-    public void executionFinished(Description description, boolean pass) {
+    public void executionFinished(Description description, boolean pass, Throwable optional) {
         if (pass) {
             passed.add(description);
         } else {
             failed.add(description);
         }
+
+        if (optional != null) {
+            errors.add(optional);
+        }
     }
 }