workaround for classloading issue in E2E

WebFuzzing · Oct 4, 2023 · 4ed971b · 4ed971b
1 parent 1ee9b62
commit 4ed971b
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 8 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -50,12 +50,13 @@ jobs:
           path: core/target/evomaster.jar
           retention-days: ${{env.retention-days}}
           if-no-files-found: error
+          ### TODO disabled due to bug. See https://github.com/mikepenz/action-junit-report/issues/952
       # Make test report accessible from GitHub Actions (as Maven logs are long)
-      - name: Publish Test Report
-        if: success() || failure()
-        uses: mikepenz/action-junit-report@v4
-        with:
-          report_paths: '**/target/surefire-reports/TEST-*.xml'
+#      - name: Publish Test Report
+#        if: success() || failure()
+#        uses: mikepenz/action-junit-report@v4
+#        with:
+#          report_paths: '**/target/surefire-reports/TEST-*.xml'
       # Upload coverage results
       - name: Upload coverage to CodeCov
         run: curl -s https://codecov.io/bash | bash

diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt
@@ -229,10 +229,14 @@ class Main {
                             Such data would not then be recomputed in the next test suite execution, as
                             the classes are already loaded...
                             Not sure if there is any clean solution for this...
-                            executing these tests in own process can be done with a flag in Failsafe/Surefire, but
-                            sounds like a potential performance loss for little benefits.
+                            executing these tests in own process might be done with Failsafe/Surefire.
+
+                            Having check for totalLines == 0 was not a good solution. If the assertion fails,
+                            and test is re-executed on same JVM with classes already loaded, then we would get
+                            totalLines == 0 after the reset... and so the test cases will always pass :(
                          */
-                        assert(totalLines == 0 || linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"}
+                        //assert(totalLines == 0 || linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"}
+                        assert(linesInfo.total <= totalLines){ "WRONG COVERAGE: ${linesInfo.total} > $totalLines"}
 
                         info("Covered targets (lines, branches, faults, etc.): ${targetsInfo.total}")
                         info("Potential faults: ${faults.size}")

diff --git a/e2e-tests/pom.xml b/e2e-tests/pom.xml
@@ -44,5 +44,25 @@
         </dependencies>
     </dependencyManagement>
 
+    <build>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <configuration>
+                        <!--
+                            Make sure each E2E is run on own JVM.
+                            This is due to possible issues with classloading, as classes are only loaded once...
+                            that is a problem when more than 1 test suite is using the same SUT.
+                            In general, this a performance bottleneck, which should be avoided for unit tests.
+                            However, for E2E that are already expensive to run, hopefully should not be a major problem.
+                        -->
+                        <reuseForks>false</reuseForks>
+                    </configuration>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
 
 </project>