From 1f4ec802ef9d4e9d111c858788c67e9e81f4c158 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 26 Sep 2023 23:30:36 +0200 Subject: [PATCH 1/9] added assertion, to check if issue can be replicated in any E2E --- core/src/main/kotlin/org/evomaster/core/Main.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 3912ce4895..dd36d5b085 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -219,6 +219,8 @@ class Main { val totalLines = unitsInfo.numberOfLines val percentage = String.format("%.0f", (linesInfo.total / totalLines.toDouble()) * 100) + assert(linesInfo.total <= totalLines) + info("Covered targets (lines, branches, faults, etc.): ${targetsInfo.total}") info("Potential faults: ${faults.size}") info("Bytecode line coverage: $percentage% (${linesInfo.total} out of $totalLines in $units units/classes)") From e4cc0e193a941f9a1176159a680ffa3aef86d158 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 27 Sep 2023 10:32:15 +0200 Subject: [PATCH 2/9] added failing test reproducing issue --- .../main/kotlin/org/evomaster/core/Main.kt | 2 +- .../evomaster/core/search/FitnessValueTest.kt | 98 +++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index dd36d5b085..05eeafa555 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -219,7 +219,7 @@ class Main { val totalLines = unitsInfo.numberOfLines val percentage = String.format("%.0f", (linesInfo.total / totalLines.toDouble()) * 100) - assert(linesInfo.total <= totalLines) + assert(linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"} info("Covered targets (lines, branches, faults, etc.): ${targetsInfo.total}") info("Potential faults: ${faults.size}") diff --git a/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt b/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt new file mode 100644 index 0000000000..2b4bddd1a0 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt @@ -0,0 +1,98 @@ +package org.evomaster.core.search + + +import org.evomaster.client.java.controller.api.dto.BootTimeInfoDto +import org.evomaster.client.java.controller.api.dto.TargetInfoDto +import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming +import org.evomaster.core.search.service.IdMapper +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +class FitnessValueTest { + + + @Test + fun testUnionWithBootTimeCoveredTargets(){ + + val idMapper = IdMapper().apply { + addMapping(0, "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007") + addMapping(1, "Class_com.foo.rest.examples.spring.postcollection.CreateDto") + addMapping(2, "Success_Call_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007_0") + addMapping(3, "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00008") + addMapping(4, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00025") + addMapping(5, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00025_position_0_falseBranch") + addMapping(6, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00025_position_0_trueBranch") + addMapping(7, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00026") + addMapping(8, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00026_0") + addMapping(9, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029") + addMapping(10, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029_0") + addMapping(11, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029_1") + addMapping(-2, "201:POST:/api/pc") + addMapping(-3, "HTTP_SUCCESS:POST:/api/pc") + addMapping(-4, "HTTP_FAULT:POST:/api/pc") + addMapping(15, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00039") + addMapping(16, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00039_0") + addMapping(17, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00039_position_0_falseBranch") + addMapping(18, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00039_position_0_trueBranch") + addMapping(19, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040") + addMapping(20, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040_0") + addMapping(21, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040_1") + addMapping(-5, "400:GET:/api/pc") + addMapping(-6, "HTTP_SUCCESS:GET:/api/pc") + addMapping(-7, "HTTP_FAULT:GET:/api/pc") + addMapping(25, "PotentialFault_PartialOracle_CodeOracle GET:/api/pc") + addMapping(26, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00043") + addMapping(27, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00043_0") + addMapping(28, "Line_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00006") + addMapping(29, "Class_com.foo.rest.examples.spring.postcollection.ValuesDto") + addMapping(30, "Success_Call_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00006_0") + addMapping(31, "Line_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00008") + addMapping(32, "Success_Call_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00008_0") + addMapping(33, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00044") + addMapping(34, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00044_0") + addMapping(35, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046") + addMapping(36, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0") + addMapping(-8, "200:GET:/api/pc") + addMapping(-9, "200:GET:/v2/api-docs") + addMapping(-10, "HTTP_SUCCESS:GET:/v2/api-docs") + addMapping(-11, "HTTP_FAULT:GET:/v2/api-docs") + addMapping(-12, "PotentialFault_PartialOracle_CodeOracle GET:/v2/api-docs") + } + + val fv = FitnessValue(1.0) + fv.coverTarget(0) //line + fv.coverTarget(1) + fv.coverTarget(2) + fv.coverTarget(3) //line + fv.coverTarget(4) //line + fv.coverTarget(-12) + + assertEquals(6, fv.coveredTargets()) + + var linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, null) + assertEquals(3, linesInfo.total) + + val bootTimeInfoDto = BootTimeInfoDto().apply { + targets = listOf( + TargetInfoDto().apply{ + id = 35 + descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046" + value = 1.0 + actionIndex = -1 + }, + TargetInfoDto().apply{ + id = 36 + descriptiveId = "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0" + value = 1.0 + actionIndex = -1 + } + ) + } + + linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, bootTimeInfoDto) + assertEquals(4, linesInfo.total) + assertEquals(1, linesInfo.bootTime) + assertEquals(3, linesInfo.searchTime) + } + +} \ No newline at end of file From 7891dc8332e4bad5e1eb4c12ce3f2254ecdabe62 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 27 Sep 2023 10:37:21 +0200 Subject: [PATCH 3/9] added failing test reproducing issue --- .../evomaster/core/search/FitnessValueTest.kt | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt b/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt index 2b4bddd1a0..b7ce647f0f 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt @@ -72,15 +72,15 @@ class FitnessValueTest { var linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, null) assertEquals(3, linesInfo.total) - val bootTimeInfoDto = BootTimeInfoDto().apply { + var bootTimeInfoDto = BootTimeInfoDto().apply { targets = listOf( - TargetInfoDto().apply{ + TargetInfoDto().apply{//new id = 35 descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046" value = 1.0 actionIndex = -1 }, - TargetInfoDto().apply{ + TargetInfoDto().apply{//other id = 36 descriptiveId = "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0" value = 1.0 @@ -93,6 +93,34 @@ class FitnessValueTest { assertEquals(4, linesInfo.total) assertEquals(1, linesInfo.bootTime) assertEquals(3, linesInfo.searchTime) + + bootTimeInfoDto = BootTimeInfoDto().apply { + targets = listOf( + TargetInfoDto().apply{//duplicate + id = 0 + descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007" + value = 1.0 + actionIndex = -1 + }, + TargetInfoDto().apply{//new + id = 35 + descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046" + value = 1.0 + actionIndex = -1 + }, + TargetInfoDto().apply{//other + id = 36 + descriptiveId = "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0" + value = 1.0 + actionIndex = -1 + } + ) + } + + linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, bootTimeInfoDto) + assertEquals(4, linesInfo.total) + assertEquals(2, linesInfo.bootTime) + assertEquals(3, linesInfo.searchTime) } } \ No newline at end of file From fb04a36b3ccdd280f2f0e7700eb2d6f61831ab4a Mon Sep 17 00:00:00 2001 From: ZhangMan Date: Fri, 29 Sep 2023 20:34:14 +0800 Subject: [PATCH 4/9] fix a bug --- .../org/evomaster/core/search/FitnessValue.kt | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt b/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt index eab96231db..2165755d53 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt @@ -254,15 +254,13 @@ class FitnessValue( var seedingTime = 0 var searchTime = 0 - targets.entries.forEach { e -> - (e.value.distance == MAX_VALUE && (prefix == null || idMapper.getDescriptiveId(e.key).startsWith(prefix))).apply { - if (coveredTargetsDuringSeeding.contains(e.key)) - seedingTime++ - else - searchTime++ - if (this && bootTime.any { it.descriptiveId == idMapper.getDescriptiveId(e.key) }) - duplicatedcounter++ - } + targets.entries.filter { e -> (e.value.distance == MAX_VALUE && (prefix == null || idMapper.getDescriptiveId(e.key).startsWith(prefix))) }.forEach { e -> + if (coveredTargetsDuringSeeding.contains(e.key)) + seedingTime++ + else + searchTime++ + if (bootTime.any { it.descriptiveId == idMapper.getDescriptiveId(e.key) }) + duplicatedcounter++ } /* From d4b08820d0a9631b22ecc0353d54d982a62d0212 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 2 Oct 2023 10:19:44 +0200 Subject: [PATCH 5/9] updating test report version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d66448da4..7d2c76520b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,7 @@ jobs: # Make test report accessible from GitHub Actions (as Maven logs are long) - name: Publish Test Report if: success() || failure() - uses: mikepenz/action-junit-report@v3 + uses: mikepenz/action-junit-report@v4 with: report_paths: '**/target/surefire-reports/TEST-*.xml' # Upload coverage results From 1ee9b6218704181b69204aeca92583221a3309a7 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 2 Oct 2023 21:51:39 +0200 Subject: [PATCH 6/9] relaxed assertion + explanation --- core/src/main/kotlin/org/evomaster/core/Main.kt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 05eeafa555..8fda05ceb4 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -219,7 +219,20 @@ class Main { val totalLines = unitsInfo.numberOfLines val percentage = String.format("%.0f", (linesInfo.total / totalLines.toDouble()) * 100) - assert(linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"} + /* + This is a quite tricky case... + the number of covered lines X should be less or equal than the total T, ie X<=T. + However, we end up with cases like X > T where T=0. + Should never happen in practice, but it does for E2E tests. + This is because we could have different test suites working on same SUTs. + Once one is finished, it would reset all data. + Such data would not then be recomputed in the next test suite execution, as + the classes are already loaded... + Not sure if there is any clean solution for this... + executing these tests in own process can be done with a flag in Failsafe/Surefire, but + sounds like a potential performance loss for little benefits. + */ + assert(totalLines == 0 || linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"} info("Covered targets (lines, branches, faults, etc.): ${targetsInfo.total}") info("Potential faults: ${faults.size}") From 4ed971b41456f641f39cb2e89a9478b4b92aeb7a Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 4 Oct 2023 09:20:32 +0200 Subject: [PATCH 7/9] workaround for classloading issue in E2E --- .github/workflows/ci.yml | 11 +++++----- .../main/kotlin/org/evomaster/core/Main.kt | 10 +++++++--- e2e-tests/pom.xml | 20 +++++++++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d2c76520b..466147f8f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,12 +50,13 @@ jobs: path: core/target/evomaster.jar retention-days: ${{env.retention-days}} if-no-files-found: error + ### TODO disabled due to bug. See https://github.com/mikepenz/action-junit-report/issues/952 # Make test report accessible from GitHub Actions (as Maven logs are long) - - name: Publish Test Report - if: success() || failure() - uses: mikepenz/action-junit-report@v4 - with: - report_paths: '**/target/surefire-reports/TEST-*.xml' +# - name: Publish Test Report +# if: success() || failure() +# uses: mikepenz/action-junit-report@v4 +# with: +# report_paths: '**/target/surefire-reports/TEST-*.xml' # Upload coverage results - name: Upload coverage to CodeCov run: curl -s https://codecov.io/bash | bash diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 8fda05ceb4..8a0fd60c35 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -229,10 +229,14 @@ class Main { Such data would not then be recomputed in the next test suite execution, as the classes are already loaded... Not sure if there is any clean solution for this... - executing these tests in own process can be done with a flag in Failsafe/Surefire, but - sounds like a potential performance loss for little benefits. + executing these tests in own process might be done with Failsafe/Surefire. + + Having check for totalLines == 0 was not a good solution. If the assertion fails, + and test is re-executed on same JVM with classes already loaded, then we would get + totalLines == 0 after the reset... and so the test cases will always pass :( */ - assert(totalLines == 0 || linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"} + //assert(totalLines == 0 || linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"} + assert(linesInfo.total <= totalLines){ "WRONG COVERAGE: ${linesInfo.total} > $totalLines"} info("Covered targets (lines, branches, faults, etc.): ${targetsInfo.total}") info("Potential faults: ${faults.size}") diff --git a/e2e-tests/pom.xml b/e2e-tests/pom.xml index 5ec49af7fe..9bcd2757c8 100644 --- a/e2e-tests/pom.xml +++ b/e2e-tests/pom.xml @@ -44,5 +44,25 @@ + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + false + + + + + \ No newline at end of file From 8a7cfe47a251ee42d6d6b169fbf5d0f16c688fca Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 4 Oct 2023 21:56:07 +0200 Subject: [PATCH 8/9] re-enabled reuse of forks --- e2e-tests/pom.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e-tests/pom.xml b/e2e-tests/pom.xml index 9bcd2757c8..bd7e8b5027 100644 --- a/e2e-tests/pom.xml +++ b/e2e-tests/pom.xml @@ -57,8 +57,11 @@ that is a problem when more than 1 test suite is using the same SUT. In general, this a performance bottleneck, which should be avoided for unit tests. However, for E2E that are already expensive to run, hopefully should not be a major problem. + + Unfortunately, it was a MAJOR problem, as execution time went up of at least double... + on GA, there is a timeout of 6 hours, which was reached (so not 100% sure of total time...) --> - false + From 5fa130de1f1196a2b7a6087b83b5b64307493e7e Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 5 Oct 2023 09:13:52 +0200 Subject: [PATCH 9/9] removed problematic assertion --- core/src/main/kotlin/org/evomaster/core/Main.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 8a0fd60c35..0e17eab9a0 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -236,7 +236,11 @@ class Main { totalLines == 0 after the reset... and so the test cases will always pass :( */ //assert(totalLines == 0 || linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"} - assert(linesInfo.total <= totalLines){ "WRONG COVERAGE: ${linesInfo.total} > $totalLines"} + /* + Having this assertion is way too problematic... not only issue when more than 2 E2E use + the same SUT, but also when flacky tests are re-run (both in our scaffolding, and in Maven) + */ + //assert(linesInfo.total <= totalLines){ "WRONG COVERAGE: ${linesInfo.total} > $totalLines"} info("Covered targets (lines, branches, faults, etc.): ${targetsInfo.total}") info("Potential faults: ${faults.size}")