Merge branch 'master' of https://github.com/EMResearch/EvoMaster into…

… wrong-summary

README.md

@@ -32,7 +32,7 @@ building on decades of research in the field of [Search-Based Software Testing](
 
 __Key features__:
 
-* _Web APIs_: At the moment, _EvoMaster_ can generate test cases for __REST__ and __GraphQL__ APIs.
+* _Web APIs_: At the moment, _EvoMaster_ can generate test cases for __REST__, __GraphQL__ and __RPC__ (e.g., __gRPC__ and __Thrift__) APIs.
 
 * _Blackbox_ testing mode: can run on any API (regardless of its programming language, e.g., Python and Go).
   However, results for blackbox testing will be worse than whitebox testing (e.g., due to lack of code analysis).
@@ -41,7 +41,7 @@ __Key features__:
   JVM (e.g., Java and Kotlin). _EvoMaster_ analyses the bytecode of the tested applications, and uses
   several heuristics such as _testability transformations_ and _taint analysis_ to be able to generate
   more effective test cases. We support JDK __8__ and the major LTS versions after that (currently JDK __17__). Might work on other JVM versions, but we provide __NO__ support for it.
-  Note: there is initial support for other languages as well, like for example JavaScript/TypeScript, but they are not in a stable, feature-complete state yet.
+  Note: there is initial support for other languages as well, like for example JavaScript/TypeScript and C#, but they are not in a stable, feature-complete state yet.
 
 * _Installation_: we provide installers for the main operating systems: Windows (`.msi`),
   OSX (`.dmg`) and Linux (`.deb`). We also provide an uber-fat JAR file.
@@ -85,6 +85,8 @@ __Known limitations__:
   But, then, you should run _EvoMaster_ for something like between 1 and 24 hours (the longer the better, but
   it is unlikely to get better results after 24 hours).
 
+* _RPC APIs_: for the moment, we do not directly support RPC schema definitions. Fuzzing RPC APIs requires to write a driver, using the client library of the API to make the calls.
+
 * _External services_: (e.g., other RESTful APIs) currently there is no support for them (e.g., to automatically mock them).
   It is work in progress.
 

client-java/instrumentation-shared/src/main/java/org/evomaster/client/java/instrumentation/shared/TaintInputName.java

@@ -56,16 +56,31 @@ public static boolean includesTaintInput(String value){
      * Create a tainted value, with the input id being part of it
      */
     public static String getTaintName(int id){
+        return getTaintName(id,0);
+    }
+
+    public static String getTaintName(int id, int minLength){
         if(id < 0){
             throw new IllegalArgumentException("Negative id");
         }
+        if(minLength < 0){
+            throw new IllegalArgumentException("Negative minLength");
+        }
         /*
             Note: this is quite simple, we simply add a unique prefix
             and postfix, in lowercase.
             But we would not be able to check if the part of the id was
             modified.
          */
-        return PREFIX + id + POSTFIX;
+
+        String s = "" + id;
+        String taint =  PREFIX + s + POSTFIX;
+        if(taint.length() < minLength){
+            //need padding
+            int diff = minLength - taint.length();
+            taint = PREFIX + s + new String(new char[diff]).replace("\0", "0") + POSTFIX;
+        }
+        return taint;
     }
 
     /**
@@ -74,7 +89,7 @@ public static String getTaintName(int id){
      * Not sure if there is really any simple workaround... but hopefully should be
      * so rare that we can live with it
      */
-    public static int getTaintNameMaxLength(){
-        return PREFIX.length() + POSTFIX.length() + 6;
+    public static boolean doesTaintNameSatisfiesLengthConstraints(String id, int maxLength){
+        return (PREFIX.length() + POSTFIX.length() + id.length()) <= maxLength;
     }
 }

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -274,7 +274,7 @@ class EMConfig {
             Each option field might have specific constraints, setup with @annotations.
             However, there can be multi-field constraints as well.
             Those are defined here.
-            They can be check only once all fields have been updated
+            They can be checked only once all fields have been updated
          */
 
         if(!blackBox && bbSwaggerUrl.isNotBlank()){
@@ -284,6 +284,11 @@ class EMConfig {
             throw IllegalArgumentException("'bbTargetUrl' should be set only in black-box mode")
         }
 
+        // ONUR, this line is changed since it did not compile in the previous case.
+        if(!endpointFocus.isNullOrBlank() && !endpointPrefix.isNullOrBlank()){
+            throw IllegalArgumentException("both 'endpointFocus' and 'endpointPrefix' are set")
+        }
+
         if (blackBox && !bbExperiments) {
 
             if(problemType == ProblemType.DEFAULT){
@@ -1647,9 +1652,12 @@ class EMConfig {
          */
     }
 
-    @Cfg("Only for debugging. Concentrate search on only one single REST endpoint")
+    @Cfg("Concentrate search on only one single REST endpoint")
     var endpointFocus : String? = null
 
+    @Cfg("Concentrate search on a set of REST endpoints defined by a common prefix")
+    var endpointPrefix : String? = null
+
     //TODO Andrea/Man. will need to discuss how this can be refactored for RPC as well
 
     @Experimental
@@ -1909,8 +1917,7 @@ class EMConfig {
     var externalRequestResponseSelectionStrategy = ExternalRequestResponseSelectionStrategy.EXACT
 
     @Cfg("Whether to employ constraints specified in API schema (e.g., OpenAPI) in test generation")
-    @Experimental
-    var enableSchemaConstraintHandling = false
+    var enableSchemaConstraintHandling = true
 
     @Cfg("a probability of enabling single insertion strategy to insert rows into database.")
     @Probability(activating = true)

core/src/main/kotlin/org/evomaster/core/Main.kt

@@ -168,6 +168,7 @@ class Main {
 
             val solution = run(injector, controllerInfo)
             val faults = solution.overall.potentialFoundFaults(idMapper)
+            val sampler : Sampler<*> = injector.getInstance(Key.get(object : TypeLiteral<Sampler<*>>(){}))
 
             resetExternalServiceHandler(injector)
 
@@ -244,7 +245,13 @@ class Main {
 
                         info("Covered targets (lines, branches, faults, etc.): ${targetsInfo.total}")
                         info("Potential faults: ${faults.size}")
-                        info("Bytecode line coverage: $percentage% (${linesInfo.total} out of $totalLines in $units units/classes)")
+
+                        if(totalLines == 0 || units == 0){
+                            logError("Detected $totalLines lines to cover, for a total of $units units/classes." +
+                                    " Are you sure you did setup getPackagePrefixesToCover() correctly?")
+                        } else {
+                            info("Bytecode line coverage: $percentage% (${linesInfo.total} out of $totalLines in $units units/classes)")
+                        }
                     } else {
                         warn("Failed to retrieve SUT info")
                     }
@@ -255,8 +262,18 @@ class Main {
                 when(config.problemType){
                     EMConfig.ProblemType.REST -> {
                         val k = data.find { it.header == Statistics.COVERED_2XX }!!.element.toInt()
-                        val p = String.format("%.0f", (k.toDouble()/n) * 100 )
-                        info("Successfully executed (HTTP code 2xx) $k endpoints out of $n ($p%)")
+                        val t = if (sampler.getPreDefinedIndividuals().isNotEmpty()) {
+                            /*
+                                FIXME this is a temporary hack...
+                                right now we might have 1 call to Schema that messes up this statistics
+                             */
+                            n + 1
+                        } else {
+                            n
+                        }
+                        assert(k <= t)
+                        val p = String.format("%.0f", (k.toDouble()/t) * 100 )
+                        info("Successfully executed (HTTP code 2xx) $k endpoints out of $t ($p%)")
                     }
                     EMConfig.ProblemType.GRAPHQL ->{
                         val k = data.find { it.header == Statistics.GQL_NO_ERRORS }!!.element.toInt()

core/src/main/kotlin/org/evomaster/core/StaticCounter.kt

@@ -17,6 +17,8 @@ class StaticCounter {
 
         fun getAndIncrease() = counter++
 
+        fun get() = counter
+
         fun reset() {
             counter = 0
         }

core/src/main/kotlin/org/evomaster/core/problem/rest/EndpointFilter.kt

@@ -0,0 +1,48 @@
+package org.evomaster.core.problem.rest
+
+import io.swagger.v3.oas.models.OpenAPI
+import org.evomaster.client.java.controller.api.dto.SutInfoDto
+import org.evomaster.core.EMConfig
+
+object EndpointFilter {
+
+     fun getEndPointsToSkip(config: EMConfig, swagger: OpenAPI):List<String> {
+        if(config.endpointFocus.isNullOrBlank()
+            && config.endpointPrefix.isNullOrBlank()){
+            return listOf()
+        }
+
+        val all = swagger.paths.map{it.key}
+
+        val selection =  if(config.endpointFocus != null) {
+            all.filter { it != config.endpointFocus }
+        } else if(config.endpointPrefix != null){
+            all.filter { ! it.startsWith(config.endpointPrefix!!) }
+        } else {
+            //should never happens
+            throw IllegalStateException("Invalid endpoint to skip configuration")
+        }
+
+        return selection
+    }
+
+
+
+     fun getEndpointsToSkip(config: EMConfig, swagger: OpenAPI, infoDto: SutInfoDto)
+            : List<String>{
+
+        /*
+            If we are debugging, and focusing on a single endpoint, we skip
+            everything but it.
+            Otherwise, we just look at what configured in the SUT EM Driver.
+         */
+
+        val endpointsToSkip = getEndPointsToSkip(config, swagger)
+        if(endpointsToSkip.isNotEmpty()){
+            return endpointsToSkip
+        }
+
+        //this has less priority
+        return  infoDto.restProblem?.endpointsToSkip ?: listOf()
+    }
+}

core/src/main/kotlin/org/evomaster/core/problem/rest/RestActionBuilderV3.kt

@@ -44,6 +44,7 @@ import java.net.URI
 import java.net.URISyntaxException
 import java.util.*
 import java.util.concurrent.atomic.AtomicInteger
+import kotlin.math.max
 
 /**
  * https://github.com/OAI/OpenAPI-Specification/blob/3.0.1/versions/3.0.1.md
@@ -366,7 +367,7 @@ object RestActionBuilderV3 {
                             would lead to 2 variables, or any other char that does affect the
                             structure of the URL, like '.'
                          */
-            gene = StringGene(gene.name, gene.value, 1, gene.maxLength, listOf('/', '.'))
+            gene = StringGene(gene.name, gene.value, max(gene.minLength, 1), gene.maxLength, listOf('/', '.'))
         }
 
         if (p.required != true && p.`in` != "path" && gene !is OptionalGene) {

core/src/main/kotlin/org/evomaster/core/problem/rest/service/AbstractRestSampler.kt

@@ -91,7 +91,7 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
         }
 
         actionCluster.clear()
-        val skip = getEndpointsToSkip(swagger, infoDto)
+        val skip = EndpointFilter.getEndpointsToSkip(config, swagger, infoDto)
         RestActionBuilderV3.addActionsFromSwagger(swagger, actionCluster, skip, enableConstraintHandling = config.enableSchemaConstraintHandling)
 
         if(config.extraQueryParam){
@@ -245,30 +245,7 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
         initAdHocInitialIndividuals()
     }
 
-    protected fun getEndpointsToSkip(swagger: OpenAPI, infoDto: SutInfoDto)
-            : List<String>{
 
-        /*
-            If we are debugging, and focusing on a single endpoint, we skip
-            everything but it.
-            Otherwise, we just look at what configured in the SUT EM Driver.
-         */
-
-        if(configuration.endpointFocus != null){
-
-            val all = swagger.paths.map{it.key}
-
-            if(all.none { it == configuration.endpointFocus }){
-                throw IllegalArgumentException(
-                        "Invalid endpointFocus: ${configuration.endpointFocus}. " +
-                                "\nAvailable:\n${all.joinToString("\n")}")
-            }
-
-            return all.filter { it != configuration.endpointFocus }
-        }
-
-        return  infoDto.restProblem?.endpointsToSkip ?: listOf()
-    }
 
     private fun initForBlackBox() {
 
@@ -277,8 +254,13 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
             throw SutProblemException("There is no endpoint definition in the retrieved Swagger file")
         }
 
+        // ONUR: Add all paths to list of paths to ignore except endpointFocus
+        val endpointsToSkip = EndpointFilter.getEndPointsToSkip(config,swagger);
+
         actionCluster.clear()
-        RestActionBuilderV3.addActionsFromSwagger(swagger, actionCluster, listOf(), enableConstraintHandling = config.enableSchemaConstraintHandling)
+
+        // ONUR: Rather than an empty list, give the list of endpoints to skip.
+        RestActionBuilderV3.addActionsFromSwagger(swagger, actionCluster, endpointsToSkip, enableConstraintHandling = config.enableSchemaConstraintHandling)
 
         initAdHocInitialIndividuals()
         if (config.seedTestCases)

core/src/main/kotlin/org/evomaster/core/search/gene/string/StringGene.kt

@@ -35,6 +35,8 @@ import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMuta
 import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
+import java.lang.IllegalStateException
+import kotlin.math.max
 import kotlin.math.min
 
 class StringGene(
@@ -113,12 +115,12 @@ class StringGene(
         get() {return children}
 
 
-    fun actualMaxLength() : Int {
+    private fun actualMaxLength() : Int {
 
         val state = getSearchGlobalState()
             ?: return maxLength
 
-        return min(maxLength, state.config.maxLengthForStrings)
+        return max(minLength, min(maxLength, state.config.maxLengthForStrings))
     }
 
 
@@ -177,7 +179,17 @@ class StringGene(
         val maxForRandomization = getSearchGlobalState()?.config?.maxLengthForStringsAtSamplingTime ?: 16
 
         val adjustedMin = minLength
-        val adjustedMax = min(maxLength, maxForRandomization)
+        var adjustedMax = min(maxLength, maxForRandomization)
+
+        if(adjustedMax < adjustedMin){
+            /*
+            this can happen if there are constraints on min length that are longer than our typical strings.
+            even if we do not want to use too long strings for performance reasons, we still must satisfy
+            any min constrains
+            */
+            assert(minLength <= maxLength)
+            adjustedMax = adjustedMin
+        }
 
         if(adjustedMax == 0 && adjustedMin == adjustedMax){
             //only empty string is allowed
@@ -402,7 +414,7 @@ class StringGene(
 
     fun redoTaint(apc: AdaptiveParameterControl, randomness: Randomness) : Boolean{
 
-        if(TaintInputName.getTaintNameMaxLength() > actualMaxLength()){
+        if(!TaintInputName.doesTaintNameSatisfiesLengthConstraints("${StaticCounter.get()}", actualMaxLength())){
             return false
         }
 
@@ -437,8 +449,16 @@ class StringGene(
         return false
     }
 
+    /**
+     * Force a tainted value. Must guarantee min-max length constraints are satisfied
+     */
     fun forceTaintedValue() {
-        value = TaintInputName.getTaintName(StaticCounter.getAndIncrease())
+        val taint = TaintInputName.getTaintName(StaticCounter.getAndIncrease(), minLength)
+
+        if(taint.length !in minLength..maxLength){
+            throw IllegalStateException("Tainted value out of min-max range [$minLength,$maxLength]")
+        }
+        value = taint
         tainted = true
     }