From 2ad6fd2a5ce6924f3335e183d068c22cb3491884 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 13 Dec 2024 22:26:42 +0100 Subject: [PATCH 1/3] enabling test comments. added info on found faults --- .../kotlin/org/evomaster/core/EMConfig.kt | 3 +-- .../core/output/service/RestTestCaseWriter.kt | 20 +++++++++++++++++++ docs/options.md | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 88dfd76a0d..9346cff2e9 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2398,9 +2398,8 @@ class EMConfig { " instead of OFF.") val probabilityOfOnVsOffInAllOptionals = 0.8 - @Experimental @Cfg("Add summary comments on each test") - var addTestComments = false + var addTestComments = true @Min(1.0) @Cfg("Max length for test comments. Needed when enumerating some names/values, making comments too long to be" + diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 7fc4678da7..406fe8febc 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -6,6 +6,7 @@ import org.evomaster.core.output.Lines import org.evomaster.core.output.SqlWriter import org.evomaster.core.output.TestCase import org.evomaster.core.output.TestWriterUtils +import org.evomaster.core.problem.enterprise.EnterpriseActionResult import org.evomaster.core.problem.httpws.HttpWsAction import org.evomaster.core.problem.httpws.HttpWsCallResult import org.evomaster.core.problem.rest.* @@ -512,6 +513,25 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { } } + //TODO move up when adding test comments to other problem types as well + //faults + val faults = ea.map { it.result } + .filterIsInstance() + .flatMap { it.getFaults() } + if(faults.isNotEmpty()){ + if(faults.size == 1){ + lines.addBlockCommentLine("Found 1 potential fault of type-code ${faults.first().category.code}") + } else { + val codes = faults.asSequence().map { it.category.code }.toSet().toList().sorted() + val codeInfo = if (codes.size == 1) { + " of type-code ${codes[0]}" + } else { + ". Type-codes: ${codes.joinToString(", ")}" + } + lines.addBlockCommentLine("Found ${faults.size} potential faults$codeInfo") + } + } + //examples val examples = getAllUsedExamples(ind.individual as RestIndividual) .toSet().sorted() diff --git a/docs/options.md b/docs/options.md index 3c8848a888..511d5bfaeb 100644 --- a/docs/options.md +++ b/docs/options.md @@ -62,6 +62,7 @@ There are 3 types of options: |`SMdR`| __Double__. Specify a probability to apply SMdR when resource sampling strategy is 'Customized'. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.25`.| |`adaptiveGeneSelectionMethod`| __Enum__. Specify a strategy to select genes for mutation adaptively. *Valid values*: `NONE, AWAY_NOIMPACT, APPROACH_IMPACT, APPROACH_LATEST_IMPACT, APPROACH_LATEST_IMPROVEMENT, BALANCE_IMPACT_NOIMPACT, BALANCE_IMPACT_NOIMPACT_WITH_E, ALL_FIXED_RAND`. *Default value*: `APPROACH_IMPACT`.| |`addPreDefinedTests`| __Boolean__. Add predefined tests at the end of the search. An example is a test to fetch the schema of RESTful APIs. *Default value*: `true`.| +|`addTestComments`| __Boolean__. Add summary comments on each test. *Default value*: `true`.| |`advancedBlackBoxCoverage`| __Boolean__. Apply more advanced coverage criteria for black-box testing. This can result in larger generated test suites. *Default value*: `true`.| |`algorithm`| __Enum__. The algorithm used to generate test cases. The default depends on whether black-box or white-box testing is done. *Valid values*: `DEFAULT, SMARTS, MIO, RANDOM, WTS, MOSA, RW`. *Default value*: `DEFAULT`.| |`allowInvalidData`| __Boolean__. When generating data, allow in some cases to use invalid values on purpose. *Default value*: `true`.| @@ -221,7 +222,6 @@ There are 3 types of options: |Options|Description| |---|---| |`abstractInitializationGeneToMutate`| __Boolean__. During mutation, whether to abstract genes for repeated SQL actions. *Default value*: `false`.| -|`addTestComments`| __Boolean__. Add summary comments on each test. *Default value*: `false`.| |`appendToTargetHeuristicsFile`| __Boolean__. Whether should add to an existing target heuristics file, instead of replacing it. It is only used when processFormat is TARGET_HEURISTIC. *Default value*: `false`.| |`bbProbabilityUseDataPool`| __Double__. Specify the probability of using the data pool when sampling test cases. This is for black-box (bb) mode. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.| |`discoveredInfoRewardedInFitness`| __Boolean__. If there is new discovered information from a test execution, reward it in the fitness function. *Default value*: `false`.| From a23966dcccd424c97fb3046ca72c5e60c10cced2 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 13 Dec 2024 22:29:05 +0100 Subject: [PATCH 2/3] updated release notes --- release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release_notes.md b/release_notes.md index d7ad91a398..642126a8e3 100644 --- a/release_notes.md +++ b/release_notes.md @@ -5,6 +5,7 @@ Under development in `master` branch. ### New Features - Now EvoMaster is released also on Docker Hub, with id `webfuzzing/evomaster`. However, this is only for black-box mode. For white-box, it is still recommended to use an OS installer or the uber-jar file from release page. +- For improving readability, generated tests now have summary comments (e.g., as JavaDoc for Java/Kotlin outputs). ### Bug Fixes - Fixed missing java.util.Arrays in generated files, leading to compilation errors in some cases involving SQL database resets. From ed30a7319c61604663f6ddc3d5850cae5f59549d Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Sat, 14 Dec 2024 22:09:28 +0100 Subject: [PATCH 3/3] fix for unit tests --- .../core/output/PythonTestCaseWriterTest.kt | 11 ++--------- .../evomaster/core/output/TestCaseWriterTest.kt | 9 ++------- .../org/evomaster/core/output/WriteJsonTest.kt | 11 ++--------- .../org/evomaster/core/output/WriteXMLTest.kt | 11 ++--------- .../org/evomaster/core/output/WriterTestBase.kt | 15 +++++++++++++++ 5 files changed, 23 insertions(+), 34 deletions(-) create mode 100644 core/src/test/kotlin/org/evomaster/core/output/WriterTestBase.kt diff --git a/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt index 05bdde6fad..2d6767e830 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt @@ -15,15 +15,8 @@ import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test import javax.ws.rs.core.MediaType -class PythonTestCaseWriterTest { - - private fun getConfig(format: OutputFormat): EMConfig { - val config = EMConfig() - config.outputFormat = format - //config.expectationsActive = false - config.testTimeout = -1 - return config - } +class PythonTestCaseWriterTest : WriterTestBase(){ + @Test fun testEmptyTest() { diff --git a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt index 5c2d4756e1..1f4b75c8f8 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt @@ -29,14 +29,9 @@ import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test import javax.ws.rs.core.MediaType -class TestCaseWriterTest { +class TestCaseWriterTest : WriterTestBase(){ + - private fun getConfig(format: OutputFormat): EMConfig { - val config = EMConfig() - config.outputFormat = format - config.testTimeout = -1 - return config - } @Test fun testEmptyDbInitialization() { diff --git a/core/src/test/kotlin/org/evomaster/core/output/WriteJsonTest.kt b/core/src/test/kotlin/org/evomaster/core/output/WriteJsonTest.kt index 30f504d87e..d55f9e2197 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/WriteJsonTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/WriteJsonTest.kt @@ -19,15 +19,8 @@ import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test -class WriteJsonTest { - - private fun getConfig(format: OutputFormat): EMConfig { - val config = EMConfig() - config.outputFormat = format - //config.expectationsActive = false - config.testTimeout = -1 - return config - } +class WriteJsonTest : WriterTestBase() { + @Test fun testJSONBEmpty() { diff --git a/core/src/test/kotlin/org/evomaster/core/output/WriteXMLTest.kt b/core/src/test/kotlin/org/evomaster/core/output/WriteXMLTest.kt index 52529e8264..2d7cc66383 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/WriteXMLTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/WriteXMLTest.kt @@ -15,15 +15,8 @@ import org.evomaster.core.search.gene.string.StringGene import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test -class WriteXMLTest { - - private fun getConfig(format: OutputFormat): EMConfig { - val config = EMConfig() - config.outputFormat = format - //config.expectationsActive = false - config.testTimeout = -1 - return config - } +class WriteXMLTest : WriterTestBase() { + @Test fun testEmptyXML() { diff --git a/core/src/test/kotlin/org/evomaster/core/output/WriterTestBase.kt b/core/src/test/kotlin/org/evomaster/core/output/WriterTestBase.kt new file mode 100644 index 0000000000..ffa9fb082a --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/output/WriterTestBase.kt @@ -0,0 +1,15 @@ +package org.evomaster.core.output + +import org.evomaster.core.EMConfig + +abstract class WriterTestBase { + + + open fun getConfig(format: OutputFormat): EMConfig { + val config = EMConfig() + config.outputFormat = format + config.testTimeout = -1 + config.addTestComments = false + return config + } +} \ No newline at end of file