Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

added assertion, to check if issue can be replicated in any E2E #809

Merged
merged 10 commits into from
Oct 6, 2023
11 changes: 6 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,13 @@ jobs:
path: core/target/evomaster.jar
retention-days: ${{env.retention-days}}
if-no-files-found: error
### TODO disabled due to bug. See https://github.com/mikepenz/action-junit-report/issues/952
# Make test report accessible from GitHub Actions (as Maven logs are long)
- name: Publish Test Report
if: success() || failure()
uses: mikepenz/action-junit-report@v3
with:
report_paths: '**/target/surefire-reports/TEST-*.xml'
# - name: Publish Test Report
# if: success() || failure()
# uses: mikepenz/action-junit-report@v4
# with:
# report_paths: '**/target/surefire-reports/TEST-*.xml'
# Upload coverage results
- name: Upload coverage to CodeCov
run: curl -s https://codecov.io/bash | bash
Expand Down
23 changes: 23 additions & 0 deletions core/src/main/kotlin/org/evomaster/core/Main.kt
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,29 @@ class Main {
val totalLines = unitsInfo.numberOfLines
val percentage = String.format("%.0f", (linesInfo.total / totalLines.toDouble()) * 100)

/*
This is a quite tricky case...
the number of covered lines X should be less or equal than the total T, ie X<=T.
However, we end up with cases like X > T where T=0.
Should never happen in practice, but it does for E2E tests.
This is because we could have different test suites working on same SUTs.
Once one is finished, it would reset all data.
Such data would not then be recomputed in the next test suite execution, as
the classes are already loaded...
Not sure if there is any clean solution for this...
executing these tests in own process might be done with Failsafe/Surefire.

Having check for totalLines == 0 was not a good solution. If the assertion fails,
and test is re-executed on same JVM with classes already loaded, then we would get
totalLines == 0 after the reset... and so the test cases will always pass :(
*/
//assert(totalLines == 0 || linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"}
/*
Having this assertion is way too problematic... not only issue when more than 2 E2E use
the same SUT, but also when flacky tests are re-run (both in our scaffolding, and in Maven)
*/
//assert(linesInfo.total <= totalLines){ "WRONG COVERAGE: ${linesInfo.total} > $totalLines"}

info("Covered targets (lines, branches, faults, etc.): ${targetsInfo.total}")
info("Potential faults: ${faults.size}")

Expand Down
16 changes: 7 additions & 9 deletions core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt
Original file line number Diff line number Diff line change
Expand Up @@ -254,15 +254,13 @@ class FitnessValue(
var seedingTime = 0
var searchTime = 0

targets.entries.forEach { e ->
(e.value.distance == MAX_VALUE && (prefix == null || idMapper.getDescriptiveId(e.key).startsWith(prefix))).apply {
if (coveredTargetsDuringSeeding.contains(e.key))
seedingTime++
else
searchTime++
if (this && bootTime.any { it.descriptiveId == idMapper.getDescriptiveId(e.key) })
duplicatedcounter++
}
targets.entries.filter { e -> (e.value.distance == MAX_VALUE && (prefix == null || idMapper.getDescriptiveId(e.key).startsWith(prefix))) }.forEach { e ->
if (coveredTargetsDuringSeeding.contains(e.key))
seedingTime++
else
searchTime++
if (bootTime.any { it.descriptiveId == idMapper.getDescriptiveId(e.key) })
duplicatedcounter++
}

/*
Expand Down
126 changes: 126 additions & 0 deletions core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package org.evomaster.core.search


import org.evomaster.client.java.controller.api.dto.BootTimeInfoDto
import org.evomaster.client.java.controller.api.dto.TargetInfoDto
import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming
import org.evomaster.core.search.service.IdMapper
import org.junit.jupiter.api.Assertions.*
import org.junit.jupiter.api.Test

class FitnessValueTest {


@Test
fun testUnionWithBootTimeCoveredTargets(){

val idMapper = IdMapper().apply {
addMapping(0, "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007")
addMapping(1, "Class_com.foo.rest.examples.spring.postcollection.CreateDto")
addMapping(2, "Success_Call_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007_0")
addMapping(3, "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00008")
addMapping(4, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00025")
addMapping(5, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00025_position_0_falseBranch")
addMapping(6, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00025_position_0_trueBranch")
addMapping(7, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00026")
addMapping(8, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00026_0")
addMapping(9, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029")
addMapping(10, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029_0")
addMapping(11, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029_1")
addMapping(-2, "201:POST:/api/pc")
addMapping(-3, "HTTP_SUCCESS:POST:/api/pc")
addMapping(-4, "HTTP_FAULT:POST:/api/pc")
addMapping(15, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00039")
addMapping(16, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00039_0")
addMapping(17, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00039_position_0_falseBranch")
addMapping(18, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00039_position_0_trueBranch")
addMapping(19, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040")
addMapping(20, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040_0")
addMapping(21, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040_1")
addMapping(-5, "400:GET:/api/pc")
addMapping(-6, "HTTP_SUCCESS:GET:/api/pc")
addMapping(-7, "HTTP_FAULT:GET:/api/pc")
addMapping(25, "PotentialFault_PartialOracle_CodeOracle GET:/api/pc")
addMapping(26, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00043")
addMapping(27, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00043_0")
addMapping(28, "Line_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00006")
addMapping(29, "Class_com.foo.rest.examples.spring.postcollection.ValuesDto")
addMapping(30, "Success_Call_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00006_0")
addMapping(31, "Line_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00008")
addMapping(32, "Success_Call_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00008_0")
addMapping(33, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00044")
addMapping(34, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00044_0")
addMapping(35, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046")
addMapping(36, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0")
addMapping(-8, "200:GET:/api/pc")
addMapping(-9, "200:GET:/v2/api-docs")
addMapping(-10, "HTTP_SUCCESS:GET:/v2/api-docs")
addMapping(-11, "HTTP_FAULT:GET:/v2/api-docs")
addMapping(-12, "PotentialFault_PartialOracle_CodeOracle GET:/v2/api-docs")
}

val fv = FitnessValue(1.0)
fv.coverTarget(0) //line
fv.coverTarget(1)
fv.coverTarget(2)
fv.coverTarget(3) //line
fv.coverTarget(4) //line
fv.coverTarget(-12)

assertEquals(6, fv.coveredTargets())

var linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, null)
assertEquals(3, linesInfo.total)

var bootTimeInfoDto = BootTimeInfoDto().apply {
targets = listOf(
TargetInfoDto().apply{//new
id = 35
descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046"
value = 1.0
actionIndex = -1
},
TargetInfoDto().apply{//other
id = 36
descriptiveId = "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0"
value = 1.0
actionIndex = -1
}
)
}

linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, bootTimeInfoDto)
assertEquals(4, linesInfo.total)
assertEquals(1, linesInfo.bootTime)
assertEquals(3, linesInfo.searchTime)

bootTimeInfoDto = BootTimeInfoDto().apply {
targets = listOf(
TargetInfoDto().apply{//duplicate
id = 0
descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007"
value = 1.0
actionIndex = -1
},
TargetInfoDto().apply{//new
id = 35
descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046"
value = 1.0
actionIndex = -1
},
TargetInfoDto().apply{//other
id = 36
descriptiveId = "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0"
value = 1.0
actionIndex = -1
}
)
}

linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, bootTimeInfoDto)
assertEquals(4, linesInfo.total)
assertEquals(2, linesInfo.bootTime)
assertEquals(3, linesInfo.searchTime)
}

}
23 changes: 23 additions & 0 deletions e2e-tests/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,28 @@
</dependencies>
</dependencyManagement>

<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<!--
Make sure each E2E is run on own JVM.
This is due to possible issues with classloading, as classes are only loaded once...
that is a problem when more than 1 test suite is using the same SUT.
In general, this a performance bottleneck, which should be avoided for unit tests.
However, for E2E that are already expensive to run, hopefully should not be a major problem.

Unfortunately, it was a MAJOR problem, as execution time went up of at least double...
on GA, there is a timeout of 6 hours, which was reached (so not 100% sure of total time...)
-->
<!-- <reuseForks>false</reuseForks>-->
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>

</project>
Loading