Merge pull request #809 from EMResearch/wrong-summary

added assertion, to check if issue can be replicated in any E2E

.github/workflows/ci.yml

@@ -50,12 +50,13 @@ jobs:
           path: core/target/evomaster.jar
           retention-days: ${{env.retention-days}}
           if-no-files-found: error
+          ### TODO disabled due to bug. See https://github.com/mikepenz/action-junit-report/issues/952
       # Make test report accessible from GitHub Actions (as Maven logs are long)
-      - name: Publish Test Report
-        if: success() || failure()
-        uses: mikepenz/action-junit-report@v3
-        with:
-          report_paths: '**/target/surefire-reports/TEST-*.xml'
+#      - name: Publish Test Report
+#        if: success() || failure()
+#        uses: mikepenz/action-junit-report@v4
+#        with:
+#          report_paths: '**/target/surefire-reports/TEST-*.xml'
       # Upload coverage results
       - name: Upload coverage to CodeCov
         run: curl -s https://codecov.io/bash | bash

core/src/main/kotlin/org/evomaster/core/Main.kt

@@ -220,6 +220,29 @@ class Main {
                         val totalLines = unitsInfo.numberOfLines
                         val percentage = String.format("%.0f", (linesInfo.total / totalLines.toDouble()) * 100)
 
+                        /*
+                            This is a quite tricky case...
+                            the number of covered lines X should be less or equal than the total T, ie X<=T.
+                            However, we end up with cases like X > T where T=0.
+                            Should never happen in practice, but it does for E2E tests.
+                            This is because we could have different test suites working on same SUTs.
+                            Once one is finished, it would reset all data.
+                            Such data would not then be recomputed in the next test suite execution, as
+                            the classes are already loaded...
+                            Not sure if there is any clean solution for this...
+                            executing these tests in own process might be done with Failsafe/Surefire.
+
+                            Having check for totalLines == 0 was not a good solution. If the assertion fails,
+                            and test is re-executed on same JVM with classes already loaded, then we would get
+                            totalLines == 0 after the reset... and so the test cases will always pass :(
+                         */
+                        //assert(totalLines == 0 || linesInfo.total <= totalLines){ "${linesInfo.total} > $totalLines"}
+                        /*
+                            Having this assertion is way too problematic... not only issue when more than 2 E2E use
+                            the same SUT, but also when flacky tests are re-run (both in our scaffolding, and in Maven)
+                         */
+                        //assert(linesInfo.total <= totalLines){ "WRONG COVERAGE: ${linesInfo.total} > $totalLines"}
+
                         info("Covered targets (lines, branches, faults, etc.): ${targetsInfo.total}")
                         info("Potential faults: ${faults.size}")
 

core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt

@@ -254,15 +254,13 @@ class FitnessValue(
         var seedingTime = 0
         var searchTime = 0
 
-        targets.entries.forEach { e ->
-            (e.value.distance == MAX_VALUE && (prefix == null || idMapper.getDescriptiveId(e.key).startsWith(prefix))).apply {
-                if (coveredTargetsDuringSeeding.contains(e.key))
-                    seedingTime++
-                else
-                    searchTime++
-                if (this && bootTime.any { it.descriptiveId == idMapper.getDescriptiveId(e.key) })
-                    duplicatedcounter++
-            }
+        targets.entries.filter { e -> (e.value.distance == MAX_VALUE && (prefix == null || idMapper.getDescriptiveId(e.key).startsWith(prefix))) }.forEach { e ->
+            if (coveredTargetsDuringSeeding.contains(e.key))
+                seedingTime++
+            else
+                searchTime++
+            if (bootTime.any { it.descriptiveId == idMapper.getDescriptiveId(e.key) })
+                duplicatedcounter++
         }
 
         /*

core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt

@@ -0,0 +1,126 @@
+package org.evomaster.core.search
+
+
+import org.evomaster.client.java.controller.api.dto.BootTimeInfoDto
+import org.evomaster.client.java.controller.api.dto.TargetInfoDto
+import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming
+import org.evomaster.core.search.service.IdMapper
+import org.junit.jupiter.api.Assertions.*
+import org.junit.jupiter.api.Test
+
+class FitnessValueTest {
+
+
+    @Test
+    fun testUnionWithBootTimeCoveredTargets(){
+
+        val idMapper = IdMapper().apply {
+            addMapping(0, "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007")
+            addMapping(1, "Class_com.foo.rest.examples.spring.postcollection.CreateDto")
+            addMapping(2, "Success_Call_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007_0")
+            addMapping(3, "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00008")
+            addMapping(4, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00025")
+            addMapping(5, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00025_position_0_falseBranch")
+            addMapping(6, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00025_position_0_trueBranch")
+            addMapping(7, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00026")
+            addMapping(8, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00026_0")
+            addMapping(9, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029")
+            addMapping(10, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029_0")
+            addMapping(11, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00029_1")
+            addMapping(-2, "201:POST:/api/pc")
+            addMapping(-3, "HTTP_SUCCESS:POST:/api/pc")
+            addMapping(-4, "HTTP_FAULT:POST:/api/pc")
+            addMapping(15, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00039")
+            addMapping(16, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00039_0")
+            addMapping(17, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00039_position_0_falseBranch")
+            addMapping(18, "Branch_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_at_line_00039_position_0_trueBranch")
+            addMapping(19, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040")
+            addMapping(20, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040_0")
+            addMapping(21, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00040_1")
+            addMapping(-5, "400:GET:/api/pc")
+            addMapping(-6, "HTTP_SUCCESS:GET:/api/pc")
+            addMapping(-7, "HTTP_FAULT:GET:/api/pc")
+            addMapping(25, "PotentialFault_PartialOracle_CodeOracle GET:/api/pc")
+            addMapping(26, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00043")
+            addMapping(27, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00043_0")
+            addMapping(28, "Line_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00006")
+            addMapping(29, "Class_com.foo.rest.examples.spring.postcollection.ValuesDto")
+            addMapping(30, "Success_Call_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00006_0")
+            addMapping(31, "Line_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00008")
+            addMapping(32, "Success_Call_at_com.foo.rest.examples.spring.postcollection.ValuesDto_00008_0")
+            addMapping(33, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00044")
+            addMapping(34, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00044_0")
+            addMapping(35, "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046")
+            addMapping(36, "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0")
+            addMapping(-8, "200:GET:/api/pc")
+            addMapping(-9, "200:GET:/v2/api-docs")
+            addMapping(-10, "HTTP_SUCCESS:GET:/v2/api-docs")
+            addMapping(-11, "HTTP_FAULT:GET:/v2/api-docs")
+            addMapping(-12, "PotentialFault_PartialOracle_CodeOracle GET:/v2/api-docs")
+        }
+
+        val fv = FitnessValue(1.0)
+        fv.coverTarget(0) //line
+        fv.coverTarget(1)
+        fv.coverTarget(2)
+        fv.coverTarget(3) //line
+        fv.coverTarget(4) //line
+        fv.coverTarget(-12)
+
+        assertEquals(6, fv.coveredTargets())
+
+        var linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, null)
+        assertEquals(3, linesInfo.total)
+
+        var bootTimeInfoDto = BootTimeInfoDto().apply {
+            targets = listOf(
+                    TargetInfoDto().apply{//new
+                        id = 35
+                        descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046"
+                        value = 1.0
+                        actionIndex = -1
+                    },
+                    TargetInfoDto().apply{//other
+                        id = 36
+                        descriptiveId = "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0"
+                        value = 1.0
+                        actionIndex = -1
+                    }
+            )
+        }
+
+        linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, bootTimeInfoDto)
+        assertEquals(4, linesInfo.total)
+        assertEquals(1, linesInfo.bootTime)
+        assertEquals(3, linesInfo.searchTime)
+
+        bootTimeInfoDto = BootTimeInfoDto().apply {
+            targets = listOf(
+                    TargetInfoDto().apply{//duplicate
+                        id = 0
+                        descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.CreateDto_00007"
+                        value = 1.0
+                        actionIndex = -1
+                    },
+                    TargetInfoDto().apply{//new
+                        id = 35
+                        descriptiveId = "Line_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046"
+                        value = 1.0
+                        actionIndex = -1
+                    },
+                    TargetInfoDto().apply{//other
+                        id = 36
+                        descriptiveId = "Success_Call_at_com.foo.rest.examples.spring.postcollection.PostCollectionRest_00046_0"
+                        value = 1.0
+                        actionIndex = -1
+                    }
+            )
+        }
+
+        linesInfo = fv.unionWithBootTimeCoveredTargets(ObjectiveNaming.LINE, idMapper, bootTimeInfoDto)
+        assertEquals(4, linesInfo.total)
+        assertEquals(2, linesInfo.bootTime)
+        assertEquals(3, linesInfo.searchTime)
+    }
+
+}

e2e-tests/pom.xml

@@ -44,5 +44,28 @@
         </dependencies>
     </dependencyManagement>
 
+    <build>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <configuration>
+                        <!--
+                            Make sure each E2E is run on own JVM.
+                            This is due to possible issues with classloading, as classes are only loaded once...
+                            that is a problem when more than 1 test suite is using the same SUT.
+                            In general, this a performance bottleneck, which should be avoided for unit tests.
+                            However, for E2E that are already expensive to run, hopefully should not be a major problem.
+
+                            Unfortunately, it was a MAJOR problem, as execution time went up of at least double...
+                            on GA, there is a timeout of 6 hours, which was reached (so not 100% sure of total time...)
+                        -->
+<!--                        <reuseForks>false</reuseForks>-->
+                    </configuration>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
 
 </project>