Merge branch 'master' into onur-endpointbug

client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/rpc/RPCExceptionInfoDto.java

@@ -22,6 +22,14 @@ public class RPCExceptionInfoDto {
      */
     public RPCExceptionType type;
 
+    /**
+     * importance level
+     * lower value more important
+     * 0 represents the most important exception which should be avoided and need to be fixed earliest
+     * a negative value (less than 0) represents undefined
+     */
+    public int importanceLevel;
+
     /**
      * a dto of the exception defined by the RPC service
      */

client-java/controller/src/main/java/org/evomaster/client/java/controller/CustomizationHandler.java

@@ -7,7 +7,9 @@
 import org.evomaster.client.java.controller.api.dto.problem.rpc.*;
 import org.evomaster.client.java.controller.problem.rpc.CustomizedNotNullAnnotationForRPCDto;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * contains a set of method to customize info for testing
@@ -90,4 +92,14 @@ public interface CustomizationHandler {
      * @return whether the mocked instance starts successfully,
      */
     boolean customizeMockingDatabase(List<MockDatabaseDto> databaseDtos, boolean enabled);
+
+    /**
+     * <p>
+     *     specify importance levels for exceptions
+     *     lower value more important, 0 is the most important exception which needs to be fixed earliest
+     * </p>
+     * @return a map, key is the class of exception, and value is importance level which must not be less than 0
+     *
+     */
+    Map<Class, Integer> getExceptionImportanceLevels();
 }

client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java

@@ -932,7 +932,13 @@ public final void executeAction(RPCActionDto dto, ActionResponseDto responseDto)
         //handle exception
         if (response instanceof Exception){
             try{
-                RPCExceptionHandler.handle(response, responseDto, endpointSchema, getRPCType(dto));
+                Map<Class, Integer> levelsMap = null;
+                try{
+                    levelsMap = getExceptionImportanceLevels();
+                }catch (Throwable e){
+                    SimpleLogger.error("ERROR: fail to get specified importance levels for exceptions "+ e.getMessage());
+                }
+                RPCExceptionHandler.handle(response, responseDto, endpointSchema, getRPCType(dto), levelsMap);
                 return;
             } catch (Exception e){
                 SimpleLogger.error("ERROR: fail to handle exception instance to dto "+ e.getMessage());
@@ -1530,4 +1536,9 @@ public final String readFileAsStringFromTestResource(String fileName){
         return (new BufferedReader(new InputStreamReader(Objects.requireNonNull(this.getClass().getClassLoader().getResourceAsStream(fileName)))))
                 .lines().collect(Collectors.joining(System.lineSeparator()));
     }
+
+    @Override
+    public Map<Class, Integer> getExceptionImportanceLevels() {
+        return null;
+    }
 }

client-java/controller/src/main/java/org/evomaster/client/java/controller/problem/rpc/RPCExceptionHandler.java

@@ -12,6 +12,7 @@
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.UndeclaredThrowableException;
+import java.util.Map;
 
 /**
  * handle RPC exception, for instance
@@ -29,7 +30,7 @@ public class RPCExceptionHandler {
      * @param endpointSchema is the schema of the endpoint
      * @param type is the RPC type
      */
-    public static void handle(Object e, ActionResponseDto dto, EndpointSchema endpointSchema, RPCType type){
+    public static void handle(Object e, ActionResponseDto dto, EndpointSchema endpointSchema, RPCType type, Map<Class, Integer> levelsMap){
 
         Object exceptionToHandle = e;
         boolean isCause = false;
@@ -47,6 +48,8 @@ public static void handle(Object e, ActionResponseDto dto, EndpointSchema endpoi
         try {
             exceptionInfoDto = handleExceptionNameAndMessage(exceptionToHandle);
 
+            handleImportanceLevels(exceptionToHandle, exceptionInfoDto, levelsMap);
+
             handled = handleDefinedException(exceptionToHandle, endpointSchema, type, exceptionInfoDto);
 
             // Thrift exception
@@ -79,6 +82,13 @@ public static void handle(Object e, ActionResponseDto dto, EndpointSchema endpoi
         dto.exceptionInfoDto.isCauseOfUndeclaredThrowable = isCause;
     }
 
+    private static void handleImportanceLevels(Object e, RPCExceptionInfoDto dto, Map<Class, Integer> levelsMap) {
+        if (levelsMap == null) return;
+        Integer level = levelsMap.get(e.getClass());
+        if (level != null)
+            dto.importanceLevel = level;
+    }
+
     private static void handleUnexpectedException(Object e, RPCExceptionInfoDto dto){
 
         dto.type = RPCExceptionType.UNEXPECTED_EXCEPTION;

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -849,6 +849,11 @@ class EMConfig {
     @Cfg("Instead of generating a single test file, it could be split in several files, according to different strategies")
     var testSuiteSplitType = TestSuiteSplitType.CLUSTER
 
+    @Experimental
+    @Cfg("Specify the maximum number of tests to be generated in one test suite. " +
+            "Note that a negative number presents no limit per test suite")
+    var maxTestsPerTestSuite = -1
+
     @Cfg("Generate an executive summary, containing an example of each category of potential fault found." +
                     "NOTE: This option is only meaningful when used in conjuction with clustering. " +
                     "This is achieved by turning the option --testSuiteSplitType to CLUSTER")

core/src/main/kotlin/org/evomaster/core/Main.kt

@@ -15,6 +15,7 @@ import org.evomaster.core.output.OutputFormat
 import org.evomaster.core.output.TestSuiteSplitter
 import org.evomaster.core.output.clustering.SplitResult
 import org.evomaster.core.output.service.TestSuiteWriter
+import org.evomaster.core.problem.api.ApiWsIndividual
 import org.evomaster.core.problem.externalservice.httpws.service.HarvestActualHttpWsResponseHandler
 import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler
 import org.evomaster.core.problem.graphql.GraphQLIndividual
@@ -609,8 +610,15 @@ class Main {
                 val splitResult = TestSuiteSplitter.split(solution, config, writer.getPartialOracles())
 
                 solution.clusteringTime = splitResult.clusteringTime.toInt()
-                splitResult.splitOutcome.filter { !it.individuals.isNullOrEmpty() }
-                        .forEach { writer.writeTests(it, controllerInfoDto?.fullName,controllerInfoDto?.executableFullPath, snapshot) }
+                splitResult.splitOutcome
+                    .filter { !it.individuals.isNullOrEmpty() }
+                    .flatMap {
+                        TestSuiteSplitter.splitSolutionByLimitSize(
+                            it as Solution<ApiWsIndividual>,
+                            config.maxTestsPerTestSuite
+                        )
+                    }
+                    .forEach { writer.writeTests(it, controllerInfoDto?.fullName,controllerInfoDto?.executableFullPath, snapshot) }
 
                 if (config.executiveSummary) {
                     writeExecSummary(injector, controllerInfoDto, splitResult)
@@ -627,7 +635,14 @@ class Main {
                      */
                     EMConfig.TestSuiteSplitType.CLUSTER -> {
                         val splitResult = TestSuiteSplitter.splitRPCByException(solution as Solution<RPCIndividual>)
-                        splitResult.splitOutcome.filter { !it.individuals.isNullOrEmpty() }
+                        splitResult.splitOutcome
+                            .filter { !it.individuals.isNullOrEmpty() }
+                            .flatMap {
+                                TestSuiteSplitter.splitSolutionByLimitSize(
+                                    it as Solution<ApiWsIndividual>,
+                                    config.maxTestsPerTestSuite
+                                )
+                            }
                             .forEach { writer.writeTests(it, controllerInfoDto?.fullName,controllerInfoDto?.executableFullPath, snapshot) }
 
                         // disable executiveSummary
@@ -645,8 +660,15 @@ class Main {
                     else -> {
                         //throw IllegalStateException("GraphQL problem does not support splitting tests by code at this time")
                         val splitResult = TestSuiteSplitter.split(solution, config)
-                        splitResult.splitOutcome.filter{ !it.individuals.isNullOrEmpty() }
-                                .forEach { writer.writeTests(it, controllerInfoDto?.fullName, controllerInfoDto?.executableFullPath, snapshot ) }
+                        splitResult.splitOutcome
+                            .filter{ !it.individuals.isNullOrEmpty() }
+                            .flatMap {
+                                TestSuiteSplitter.splitSolutionByLimitSize(
+                                    it as Solution<ApiWsIndividual>,
+                                    config.maxTestsPerTestSuite
+                                )
+                            }
+                            .forEach { writer.writeTests(it, controllerInfoDto?.fullName, controllerInfoDto?.executableFullPath, snapshot ) }
                     }
                     /*
                       GraphQL could be split by code (where code is available and trustworthy)

core/src/main/kotlin/org/evomaster/core/output/TestSuiteSplitter.kt

@@ -21,19 +21,51 @@ import org.evomaster.core.problem.api.ApiWsIndividual
  */
 object TestSuiteSplitter {
 
+    const val MULTIPLE_RPC_INTERFACES  = "MultipleRPCInterfaces"
+
     /**
      * simple split based on whether it exists exception based on RPC results
      */
     fun splitRPCByException(solution: Solution<RPCIndividual>): SplitResult{
 
-        val group = solution.individuals.groupBy { i-> i.seeResults().any { r-> r is RPCCallResult && r.isExceptionThrown() } }
+        val other = solution.individuals.filter { i-> i.seeResults().any { r-> r is RPCCallResult && !r.isExceptionThrown() } }
+
+        val clusterOther = other.groupBy {
+            if (it.individual.getTestedInterfaces().size == 1){
+                formatClassNameInTestName(it.individual.getTestedInterfaces().first(), true)
+            }else{
+                MULTIPLE_RPC_INTERFACES
+            }
+        }
+
+        val exceptionGroup = solution.individuals.filterNot { other.contains(it) }.groupBy {
+            i ->
+            i.seeResults().filterIsInstance<RPCCallResult>()
+                .minOfOrNull { it.getExceptionImportanceLevel()}?:-1
+        }
         return SplitResult().apply {
-            this.splitOutcome = group.map { g->
-                Solution(individuals = g.value.toMutableList(),
-                    testSuiteNamePrefix = solution.testSuiteNamePrefix,
+            this.splitOutcome = clusterOther.map {o->
+                Solution(individuals = o.value.toMutableList(),
+                    testSuiteNamePrefix = "${solution.testSuiteNamePrefix}_${o.key}",
                     testSuiteNameSuffix = solution.testSuiteNameSuffix,
-                    termination = if (g.key) Termination.EXCEPTION else Termination.OTHER, listOf())
+                    termination = Termination.OTHER, listOf())
             }
+//                .plus(Solution(individuals = other.toMutableList(),
+//                    testSuiteNamePrefix = solution.testSuiteNamePrefix,
+//                    testSuiteNameSuffix = solution.testSuiteNameSuffix,
+//                    termination = Termination.OTHER, listOf()))
+                .plus(
+                exceptionGroup.map { e->
+                    var level = "Undefined"
+                    if (e.key >= 0)
+                        level = "_P${e.key}"
+
+                    Solution(individuals = e.value.toMutableList(),
+                        testSuiteNamePrefix = "${solution.testSuiteNamePrefix}${level}",
+                        testSuiteNameSuffix = solution.testSuiteNameSuffix,
+                        termination = Termination.EXCEPTION, listOf())
+                }
+            )
         }
     }
 
@@ -90,6 +122,23 @@ object TestSuiteSplitter {
         return splitResult
     }
 
+    /**
+     * @return split test suite based on specified maximum number [limit]
+     */
+    fun splitSolutionByLimitSize(solution: Solution<ApiWsIndividual>, limit: Int) : List<Solution<ApiWsIndividual>>{
+        if (limit < 0) return listOf(solution)
+        val group = solution.individuals.groupBy {
+            solution.individuals.indexOf(it) / limit
+        }
+
+        return group.map {g->
+            Solution(individuals = g.value.toMutableList(),
+                testSuiteNamePrefix = "${solution.testSuiteNamePrefix}_${g.key}",
+                testSuiteNameSuffix = solution.testSuiteNameSuffix,
+                termination = solution.termination, listOf())
+        }
+    }
+
     private fun conductClustering(solution: Solution<ApiWsIndividual>,
                                   oracles: PartialOracles = PartialOracles(),
                                   config: EMConfig,
@@ -375,6 +424,17 @@ object TestSuiteSplitter {
         )
     }
 
+    private fun formatTestedInterfacesInTestName(rpcIndividual: RPCIndividual) : String{
+        return rpcIndividual.getTestedInterfaces().joinToString("_") { formatClassNameInTestName(it, true) }
+    }
+
+    private fun formatClassNameInTestName(clazz: String, simpleName : Boolean): String{
+        val names = clazz.replace("$","_").split(".")
+        if (simpleName)
+            return names.last()
+        return names.joinToString("_")
+    }
+
 
     /**
      * [splitIntoClusters] splits a given Solution object into a List of several Solution objects, each

core/src/main/kotlin/org/evomaster/core/problem/rpc/RPCCallAction.kt

@@ -11,6 +11,11 @@ import org.evomaster.core.search.gene.Gene
  * a RPC call
  */
 open class RPCCallAction(
+    /**
+     * class name of the interface which defines this rpc function
+     */
+    val interfaceId: String,
+
     /**
      * id of the RPCCallAction
      */
@@ -49,7 +54,7 @@ open class RPCCallAction(
 
     override fun copyContent(): RPCCallAction {
         val p = parameters.asSequence().map(Param::copy).toMutableList()
-        return RPCCallAction(id, p, responseTemplate?.copy() as RPCParam?, response?.copy() as RPCParam?, auth)
+        return RPCCallAction(interfaceId, id, p, responseTemplate?.copy() as RPCParam?, response?.copy() as RPCParam?, auth)
     }
 
     /**
@@ -70,4 +75,4 @@ open class RPCCallAction(
         // might add values of parameters later
         return "$id , auth=${auth.name}"
     }
-}
+}

core/src/main/kotlin/org/evomaster/core/problem/rpc/RPCCallResult.kt

@@ -20,6 +20,7 @@ class RPCCallResult : ActionResult {
         const val INVOCATION_CODE = "INVOCATION_CODE"
         const val CUSTOM_EXP_BODY = "CUSTOM_EXP_BODY"
         const val EXCEPTION_CODE = "EXCEPTION_CODE"
+        const val EXCEPTION_IMPORTANCE_LEVEL = "EXCEPTION_IMPORTANCE_LEVEL"
         const val EXCEPTION_TYPE_NAME = "EXCEPTION_TYPE_NAME"
         const val CUSTOM_BUSINESS_LOGIC_CODE = "CUSTOM_BUSINESS_LOGIC_CODE"
         const val CUSTOM_BUSINESS_LOGIC_SUCCESS = 200
@@ -105,6 +106,16 @@ class RPCCallResult : ActionResult {
 
     fun getExceptionCode() = getResultValue(EXCEPTION_CODE)
 
+    fun getExceptionImportanceLevel() : Int{
+        val level = getResultValue(EXCEPTION_IMPORTANCE_LEVEL) ?: return -1
+        return try {
+            level.toInt()
+        }catch (e: NumberFormatException){
+            -1
+        }
+
+    }
+
     fun getExceptionTypeName() = getResultValue(EXCEPTION_TYPE_NAME)
 
     fun getExceptionInfo() : String{
@@ -148,6 +159,7 @@ class RPCCallResult : ActionResult {
             }
 
             addResultValue(EXCEPTION_CODE, dto.type.name)
+            addResultValue(EXCEPTION_IMPORTANCE_LEVEL, "${dto.importanceLevel}")
             addResultValue(EXCEPTION_TYPE_NAME, dto.exceptionName)
             addResultValue(INVOCATION_CODE, code.name)
 
@@ -172,4 +184,4 @@ class RPCCallResult : ActionResult {
 
     fun isExceptionThrown() : Boolean = getExceptionCode() != null
 
-}
+}

core/src/main/kotlin/org/evomaster/core/problem/rpc/RPCIndividual.kt

@@ -15,6 +15,7 @@ import org.evomaster.core.problem.externalservice.ApiExternalServiceAction
 import org.evomaster.core.search.*
 import org.evomaster.core.search.gene.Gene
 import org.evomaster.core.search.tracer.TrackOperator
+import java.util.*
 import kotlin.math.max
 
 /**
@@ -132,4 +133,12 @@ class RPCIndividual(
     override fun seeMainExecutableActions(): List<RPCCallAction> {
         return super.seeMainExecutableActions() as List<RPCCallAction>
     }
+
+
+    /**
+     * @return a sorted list of involved interfaces in this test
+     */
+    fun getTestedInterfaces() : SortedSet<String> {
+        return seeMainExecutableActions().map { it.interfaceId }.toSortedSet()
+    }
 }

core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCEndpointsHandler.kt

@@ -1155,7 +1155,7 @@ class RPCEndpointsHandler {
             TODO Man exception
          */
 
-        return RPCCallAction(name, params, responseTemplate = response, response = null )
+        return RPCCallAction(endpointSchema.interfaceId, name, params, responseTemplate = response, response = null )
     }
 
     private fun actionName(interfaceName: String, endpointName: String) = "$interfaceName:$endpointName"

core/src/test/kotlin/org/evomaster/core/output/EvaluatedIndividualBuilder.kt

@@ -75,8 +75,8 @@ class EvaluatedIndividualBuilder {
             return Triple(format, baseUrlOfSut, ei)
         }
 
-        fun buildFakeRPCAction(n:Int) : MutableList<RPCCallAction>{
-            return (0 until n).map { RPCCallAction("FakeRPCCall_$it",
+        fun buildFakeRPCAction(n:Int, interfaceId : String = "FakeRPCCall") : MutableList<RPCCallAction>{
+            return (0 until n).map { RPCCallAction(interfaceId,"${interfaceId}_$it",
                 inputParameters = mutableListOf(),
                 responseTemplate= null,
                 response = RPCParam("return", OptionalGene("return", StringGene("return")))