Merge pull request #1149 from WebFuzzing/javadoc-faults

Javadoc faults

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -2436,9 +2436,8 @@ class EMConfig {
             " instead of OFF.")
     val probabilityOfOnVsOffInAllOptionals = 0.8
 
-    @Experimental
     @Cfg("Add summary comments on each test")
-    var addTestComments = false
+    var addTestComments = true
 
     @Min(1.0)
     @Cfg("Max length for test comments. Needed when enumerating some names/values, making comments too long to be" +

core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt

@@ -6,6 +6,7 @@ import org.evomaster.core.output.Lines
 import org.evomaster.core.output.SqlWriter
 import org.evomaster.core.output.TestCase
 import org.evomaster.core.output.TestWriterUtils
+import org.evomaster.core.problem.enterprise.EnterpriseActionResult
 import org.evomaster.core.problem.httpws.HttpWsAction
 import org.evomaster.core.problem.httpws.HttpWsCallResult
 import org.evomaster.core.problem.rest.*
@@ -512,6 +513,25 @@ class RestTestCaseWriter : HttpWsTestCaseWriter {
             }
         }
 
+        //TODO move up when adding test comments to other problem types as well
+        //faults
+        val faults = ea.map { it.result }
+            .filterIsInstance<EnterpriseActionResult>()
+            .flatMap { it.getFaults() }
+        if(faults.isNotEmpty()){
+            if(faults.size == 1){
+                lines.addBlockCommentLine("Found 1 potential fault of type-code ${faults.first().category.code}")
+            } else {
+                val codes = faults.asSequence().map { it.category.code }.toSet().toList().sorted()
+                val codeInfo = if (codes.size == 1) {
+                    " of type-code ${codes[0]}"
+                } else {
+                    ". Type-codes: ${codes.joinToString(", ")}"
+                }
+                lines.addBlockCommentLine("Found ${faults.size} potential faults$codeInfo")
+            }
+        }
+
         //examples
         val examples = getAllUsedExamples(ind.individual as RestIndividual)
             .toSet().sorted()

core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt

@@ -15,15 +15,8 @@ import org.junit.jupiter.api.Assertions.*
 import org.junit.jupiter.api.Test
 import javax.ws.rs.core.MediaType
 
-class PythonTestCaseWriterTest {
-
-    private fun getConfig(format: OutputFormat): EMConfig {
-        val config = EMConfig()
-        config.outputFormat = format
-        //config.expectationsActive = false
-        config.testTimeout = -1
-        return config
-    }
+class PythonTestCaseWriterTest : WriterTestBase(){
+
 
     @Test
     fun testEmptyTest() {

core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt

@@ -29,14 +29,9 @@ import org.junit.jupiter.api.Assertions.*
 import org.junit.jupiter.api.Test
 import javax.ws.rs.core.MediaType
 
-class TestCaseWriterTest {
+class TestCaseWriterTest : WriterTestBase(){
+
 
-    private fun getConfig(format: OutputFormat): EMConfig {
-        val config = EMConfig()
-        config.outputFormat = format
-        config.testTimeout = -1
-        return config
-    }
 
     @Test
     fun testEmptyDbInitialization() {

core/src/test/kotlin/org/evomaster/core/output/WriteJsonTest.kt

@@ -19,15 +19,8 @@ import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Test
 
-class WriteJsonTest {
-
-    private fun getConfig(format: OutputFormat): EMConfig {
-        val config = EMConfig()
-        config.outputFormat = format
-        //config.expectationsActive = false
-        config.testTimeout  = -1
-        return config
-    }
+class WriteJsonTest : WriterTestBase() {
+
 
     @Test
     fun testJSONBEmpty() {

core/src/test/kotlin/org/evomaster/core/output/WriteXMLTest.kt

@@ -15,15 +15,8 @@ import org.evomaster.core.search.gene.string.StringGene
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Test
 
-class WriteXMLTest {
-
-    private fun getConfig(format: OutputFormat): EMConfig {
-        val config = EMConfig()
-        config.outputFormat = format
-        //config.expectationsActive = false
-        config.testTimeout = -1
-        return config
-    }
+class WriteXMLTest : WriterTestBase() {
+
 
     @Test
     fun testEmptyXML() {

core/src/test/kotlin/org/evomaster/core/output/WriterTestBase.kt

@@ -0,0 +1,15 @@
+package org.evomaster.core.output
+
+import org.evomaster.core.EMConfig
+
+abstract class WriterTestBase {
+
+
+    open fun getConfig(format: OutputFormat): EMConfig {
+        val config = EMConfig()
+        config.outputFormat = format
+        config.testTimeout = -1
+        config.addTestComments = false
+        return config
+    }
+}

docs/options.md

@@ -65,6 +65,7 @@ There are 3 types of options:
 |`SMdR`| __Double__. Specify a probability to apply SMdR when resource sampling strategy is 'Customized'. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.25`.|
 |`adaptiveGeneSelectionMethod`| __Enum__. Specify a strategy to select genes for mutation adaptively. *Valid values*: `NONE, AWAY_NOIMPACT, APPROACH_IMPACT, APPROACH_LATEST_IMPACT, APPROACH_LATEST_IMPROVEMENT, BALANCE_IMPACT_NOIMPACT, BALANCE_IMPACT_NOIMPACT_WITH_E, ALL_FIXED_RAND`. *Default value*: `APPROACH_IMPACT`.|
 |`addPreDefinedTests`| __Boolean__. Add predefined tests at the end of the search. An example is a test to fetch the schema of RESTful APIs. *Default value*: `true`.|
+|`addTestComments`| __Boolean__. Add summary comments on each test. *Default value*: `true`.|
 |`advancedBlackBoxCoverage`| __Boolean__. Apply more advanced coverage criteria for black-box testing. This can result in larger generated test suites. *Default value*: `true`.|
 |`algorithm`| __Enum__. The algorithm used to generate test cases. The default depends on whether black-box or white-box testing is done. *Valid values*: `DEFAULT, SMARTS, MIO, RANDOM, WTS, MOSA, RW`. *Default value*: `DEFAULT`.|
 |`allowInvalidData`| __Boolean__. When generating data, allow in some cases to use invalid values on purpose. *Default value*: `true`.|
@@ -223,7 +224,6 @@ There are 3 types of options:
 |Options|Description|
 |---|---|
 |`abstractInitializationGeneToMutate`| __Boolean__. During mutation, whether to abstract genes for repeated SQL actions. *Default value*: `false`.|
-|`addTestComments`| __Boolean__. Add summary comments on each test. *Default value*: `false`.|
 |`appendToTargetHeuristicsFile`| __Boolean__. Whether should add to an existing target heuristics file, instead of replacing it. It is only used when processFormat is TARGET_HEURISTIC. *Default value*: `false`.|
 |`bbProbabilityUseDataPool`| __Double__. Specify the probability of using the data pool when sampling test cases. This is for black-box (bb) mode. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.|
 |`discoveredInfoRewardedInFitness`| __Boolean__. If there is new discovered information from a test execution, reward it in the fitness function. *Default value*: `false`.|

release_notes.md

@@ -5,6 +5,7 @@ Under development in `master` branch.
 ### New Features
 - Now EvoMaster is released also on Docker Hub, with id `webfuzzing/evomaster`. 
   However, this is only for black-box mode. For white-box, it is still recommended to use an OS installer or the uber-jar file from release page.
+- For improving readability, generated tests now have summary comments (e.g., as JavaDoc for Java/Kotlin outputs).
 
 ### Bug Fixes
 - Fixed missing java.util.Arrays in generated files, leading to compilation errors in some cases involving SQL database resets.