diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d66448da4..466147f8f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,12 +50,13 @@ jobs: path: core/target/evomaster.jar retention-days: ${{env.retention-days}} if-no-files-found: error + ### TODO disabled due to bug. See https://github.com/mikepenz/action-junit-report/issues/952 # Make test report accessible from GitHub Actions (as Maven logs are long) - - name: Publish Test Report - if: success() || failure() - uses: mikepenz/action-junit-report@v3 - with: - report_paths: '**/target/surefire-reports/TEST-*.xml' +# - name: Publish Test Report +# if: success() || failure() +# uses: mikepenz/action-junit-report@v4 +# with: +# report_paths: '**/target/surefire-reports/TEST-*.xml' # Upload coverage results - name: Upload coverage to CodeCov run: curl -s https://codecov.io/bash | bash diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index cc198a08d8..47866c0119 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -220,6 +220,29 @@ class Main { val totalLines = unitsInfo.numberOfLines val percentage = String.format("%.0f", (linesInfo.total / totalLines.toDouble()) * 100) + /* + This is a quite tricky case... + the number of covered lines X should be less or equal than the total T, ie X<=T. + However, we end up with cases like X > T where T=0. + Should never happen in practice, but it does for E2E tests. + This is because we could have different test suites working on same SUTs. + Once one is finished, it would reset all data. + Such data would not then be recomputed in the next test suite execution, as + the classes are already loaded... + Not sure if there is any clean solution for this... + executing these tests in own process might be done with Failsafe/Surefire. + + Having check for totalLines == 0 was not a good solution. If the assertion fails, + and test is re-executed on same JVM with classes already loaded, then we would get + totalLines == 0 after the reset... and so the test cases will always pass :( + */ + //assert(totalLines == 0 || linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"} + /* + Having this assertion is way too problematic... not only issue when more than 2 E2E use + the same SUT, but also when flacky tests are re-run (both in our scaffolding, and in Maven) + */ + //assert(linesInfo.total <= totalLines){ "WRONG COVERAGE: ${linesInfo.total} > $totalLines"} + info("Covered targets (lines, branches, faults, etc.): ${targetsInfo.total}") info("Potential faults: ${faults.size}") diff --git a/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt b/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt index eab96231db..2165755d53 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt @@ -254,15 +254,13 @@ class FitnessValue( var seedingTime = 0 var searchTime = 0 - targets.entries.forEach { e -> - (e.value.distance == MAX_VALUE && (prefix == null || idMapper.getDescriptiveId(e.key).startsWith(prefix))).apply { - if (coveredTargetsDuringSeeding.contains(e.key)) - seedingTime++ - else - searchTime++ - if (this && bootTime.any { it.descriptiveId == idMapper.getDescriptiveId(e.key) }) - duplicatedcounter++ - } + targets.entries.filter { e -> (e.value.distance == MAX_VALUE && (prefix == null || idMapper.getDescriptiveId(e.key).startsWith(prefix))) }.forEach { e -> + if (coveredTargetsDuringSeeding.contains(e.key)) + seedingTime++ + else + searchTime++ + if (bootTime.any { it.descriptiveId == idMapper.getDescriptiveId(e.key) }) + duplicatedcounter++ } /* diff --git a/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt b/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt new file mode 100644 index 0000000000..b7ce647f0f --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt @@ -0,0 +1,126 @@ +package org.evomaster.core.search + + +import org.evomaster.client.java.controller.api.dto.BootTimeInfoDto +import org.evomaster.client.java.controller.api.dto.TargetInfoDto +import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming +import org.evomaster.core.search.service.IdMapper +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +class FitnessValueTest { + + + @Test + fun testUnionWithBootTimeCoveredTargets(){ + + val idMapper = IdMapper().apply { + addMapping(0, "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007") + addMapping(1, "Class_com.foo.rest.examples.spring.postcollection.CreateDto") + addMapping(2, "Success_Call_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007_0") + addMapping(3, "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00008") + addMapping(4, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00025") + addMapping(5, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00025_position_0_falseBranch") + addMapping(6, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00025_position_0_trueBranch") + addMapping(7, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00026") + addMapping(8, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00026_0") + addMapping(9, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029") + addMapping(10, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029_0") + addMapping(11, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029_1") + addMapping(-2, "201:POST:/api/pc") + addMapping(-3, "HTTP_SUCCESS:POST:/api/pc") + addMapping(-4, "HTTP_FAULT:POST:/api/pc") + addMapping(15, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00039") + addMapping(16, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00039_0") + addMapping(17, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00039_position_0_falseBranch") + addMapping(18, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00039_position_0_trueBranch") + addMapping(19, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040") + addMapping(20, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040_0") + addMapping(21, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040_1") + addMapping(-5, "400:GET:/api/pc") + addMapping(-6, "HTTP_SUCCESS:GET:/api/pc") + addMapping(-7, "HTTP_FAULT:GET:/api/pc") + addMapping(25, "PotentialFault_PartialOracle_CodeOracle GET:/api/pc") + addMapping(26, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00043") + addMapping(27, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00043_0") + addMapping(28, "Line_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00006") + addMapping(29, "Class_com.foo.rest.examples.spring.postcollection.ValuesDto") + addMapping(30, "Success_Call_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00006_0") + addMapping(31, "Line_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00008") + addMapping(32, "Success_Call_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00008_0") + addMapping(33, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00044") + addMapping(34, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00044_0") + addMapping(35, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046") + addMapping(36, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0") + addMapping(-8, "200:GET:/api/pc") + addMapping(-9, "200:GET:/v2/api-docs") + addMapping(-10, "HTTP_SUCCESS:GET:/v2/api-docs") + addMapping(-11, "HTTP_FAULT:GET:/v2/api-docs") + addMapping(-12, "PotentialFault_PartialOracle_CodeOracle GET:/v2/api-docs") + } + + val fv = FitnessValue(1.0) + fv.coverTarget(0) //line + fv.coverTarget(1) + fv.coverTarget(2) + fv.coverTarget(3) //line + fv.coverTarget(4) //line + fv.coverTarget(-12) + + assertEquals(6, fv.coveredTargets()) + + var linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, null) + assertEquals(3, linesInfo.total) + + var bootTimeInfoDto = BootTimeInfoDto().apply { + targets = listOf( + TargetInfoDto().apply{//new + id = 35 + descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046" + value = 1.0 + actionIndex = -1 + }, + TargetInfoDto().apply{//other + id = 36 + descriptiveId = "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0" + value = 1.0 + actionIndex = -1 + } + ) + } + + linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, bootTimeInfoDto) + assertEquals(4, linesInfo.total) + assertEquals(1, linesInfo.bootTime) + assertEquals(3, linesInfo.searchTime) + + bootTimeInfoDto = BootTimeInfoDto().apply { + targets = listOf( + TargetInfoDto().apply{//duplicate + id = 0 + descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007" + value = 1.0 + actionIndex = -1 + }, + TargetInfoDto().apply{//new + id = 35 + descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046" + value = 1.0 + actionIndex = -1 + }, + TargetInfoDto().apply{//other + id = 36 + descriptiveId = "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0" + value = 1.0 + actionIndex = -1 + } + ) + } + + linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, bootTimeInfoDto) + assertEquals(4, linesInfo.total) + assertEquals(2, linesInfo.bootTime) + assertEquals(3, linesInfo.searchTime) + } + +} \ No newline at end of file diff --git a/e2e-tests/pom.xml b/e2e-tests/pom.xml index 5ec49af7fe..bd7e8b5027 100644 --- a/e2e-tests/pom.xml +++ b/e2e-tests/pom.xml @@ -44,5 +44,28 @@ + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + + \ No newline at end of file