REST test cases are sorted by path levels, status code and http verb

WebFuzzing · Jan 11, 2025 · 89764f7 · 89764f7
1 parent a26598e
commit 89764f7
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 7 deletions.
diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt
@@ -196,6 +196,23 @@ class SortingHelper {
      */
 
     var comparatorList = listOf(statusCode, coveredTargets)
+    var restComparator: Comparator<EvaluatedIndividual<*>> = compareBy<EvaluatedIndividual<*>> { ind ->
+        if (ind.individual is RestIndividual) {
+            (ind.evaluatedMainActions().last().action as RestCallAction).path.levels()
+        } else 0
+    }
+        .thenBy { ind ->
+            val min = ind.seeResults().filterIsInstance<HttpWsCallResult>().minByOrNull {
+                it.getStatusCode()?.rem(500) ?: 0
+            }
+            (min?.getStatusCode())?.rem(500) ?: 0
+        }
+        .thenBy { ind ->
+            if (ind.individual is RestIndividual) {
+                (ind.evaluatedMainActions().last().action as RestCallAction).verb
+            } else 0
+        }
+
     private val availableSortCriteria = listOf(statusCode, minActions, coveredTargets, maxStatusCode, maxActions, dbInitSize)
 
 
@@ -248,7 +265,7 @@ class SortingHelper {
          * that have the same code, the ones with the most covered targets will be at the top (among their sub-group).
          */
 
-        return namingStrategy.getSortedTestCases(comparators)
+        return namingStrategy.getSortedTestCases(restComparator)
 
     }
 

diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt
@@ -13,12 +13,12 @@ open class NumberedTestCaseNamingStrategy(
         return generateNames(solution.individuals)
     }
 
-    override fun getSortedTestCases(comparators: List<Comparator<EvaluatedIndividual<*>>>): List<TestCase> {
+    override fun getSortedTestCases(comparator: Comparator<EvaluatedIndividual<*>>): List<TestCase> {
         val inds = solution.individuals
-        comparators.asReversed().forEach {
-            inds.sortWith(it)
-        }
-        return generateNames(inds)
+
+        val sortedIndividuals = inds.sortedWith(comparator)
+
+        return generateNames(sortedIndividuals)
     }
 
     // numbered strategy will not expand the name unless it is using the namingHelper

diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt
@@ -23,7 +23,7 @@ abstract class TestCaseNamingStrategy(
      *
      * @return the list of sorted TestCase with the generated name given the naming strategy
      */
-    abstract fun getSortedTestCases(comparators: List<Comparator<EvaluatedIndividual<*>>>): List<TestCase>
+    abstract fun getSortedTestCases(comparator: Comparator<EvaluatedIndividual<*>>): List<TestCase>
 
     /**
      * @param individual containing information for the test about to be named

diff --git a/core/src/test/kotlin/org/evomaster/core/output/TestSuiteOrganizerTest.kt b/core/src/test/kotlin/org/evomaster/core/output/TestSuiteOrganizerTest.kt
@@ -0,0 +1,71 @@
+package org.evomaster.core.output
+
+import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy
+import org.evomaster.core.output.naming.RestActionTestCaseUtils.getEvaluatedIndividualWith
+import org.evomaster.core.output.naming.RestActionTestCaseUtils.getRestCallAction
+import org.evomaster.core.problem.rest.HttpVerb
+import org.evomaster.core.search.Solution
+import org.junit.Test
+import org.junit.jupiter.api.Assertions.assertEquals
+
+class TestSuiteOrganizerTest {
+
+    @Test
+    fun sortedByPathSegmentFirst() {
+        val noPathSegmentInd = getEvaluatedIndividualWith(getRestCallAction("/"))
+        val onePathSegmentInd = getEvaluatedIndividualWith(getRestCallAction("/organization"))
+        val twoPathSegmentsInd = getEvaluatedIndividualWith(getRestCallAction("/organization/{name}"))
+        val individuals = mutableListOf(noPathSegmentInd, onePathSegmentInd, twoPathSegmentsInd)
+        individuals.shuffle()
+        val solution = Solution(individuals, "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
+
+        val sortedTestCases = SortingHelper().sort(solution, NumberedTestCaseNamingStrategy(solution))
+
+        assertEquals(sortedTestCases[0].test, noPathSegmentInd)
+        assertEquals(sortedTestCases[1].test, onePathSegmentInd)
+        assertEquals(sortedTestCases[2].test, twoPathSegmentsInd)
+    }
+
+    @Test
+    fun sortedByStatusCodeWhenEqualPathSegmentSize() {
+        val status200Ind = getEvaluatedIndividualWith(getRestCallAction("/organization"), 200)
+        val status401Ind = getEvaluatedIndividualWith(getRestCallAction("/organization"), 401)
+        val status500Ind = getEvaluatedIndividualWith(getRestCallAction("/organization"), 500)
+        val individuals = mutableListOf(status200Ind, status401Ind, status500Ind)
+        individuals.shuffle()
+        val solution = Solution(individuals, "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
+
+        val sortedTestCases = SortingHelper().sort(solution, NumberedTestCaseNamingStrategy(solution))
+
+        assertEquals(sortedTestCases[0].test, status500Ind)
+        assertEquals(sortedTestCases[1].test, status200Ind)
+        assertEquals(sortedTestCases[2].test, status401Ind)
+    }
+
+    @Test
+    fun sortedByMethodWhenEqualPathSegmentsAndStatusCode() {
+        val getInd = getEvaluatedIndividualWith(getRestCallAction("/organization", HttpVerb.GET))
+        val postInd = getEvaluatedIndividualWith(getRestCallAction("/organization", HttpVerb.POST))
+        val putInd = getEvaluatedIndividualWith(getRestCallAction("/organization", HttpVerb.PUT))
+        val deleteInd = getEvaluatedIndividualWith(getRestCallAction("/organization", HttpVerb.DELETE))
+        val optionsInd = getEvaluatedIndividualWith(getRestCallAction("/organization", HttpVerb.OPTIONS))
+        val patchInd = getEvaluatedIndividualWith(getRestCallAction("/organization", HttpVerb.PATCH))
+        val traceInd = getEvaluatedIndividualWith(getRestCallAction("/organization", HttpVerb.TRACE))
+        val headInd = getEvaluatedIndividualWith(getRestCallAction("/organization", HttpVerb.HEAD))
+        val individuals = mutableListOf(getInd, postInd, putInd, deleteInd, optionsInd, patchInd, traceInd, headInd)
+        individuals.shuffle()
+        val solution = Solution(individuals, "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList())
+
+        val sortedTestCases = SortingHelper().sort(solution, NumberedTestCaseNamingStrategy(solution))
+
+        assertEquals(sortedTestCases[0].test, getInd)
+        assertEquals(sortedTestCases[1].test, postInd)
+        assertEquals(sortedTestCases[2].test, putInd)
+        assertEquals(sortedTestCases[3].test, deleteInd)
+        assertEquals(sortedTestCases[4].test, optionsInd)
+        assertEquals(sortedTestCases[5].test, patchInd)
+        assertEquals(sortedTestCases[6].test, traceInd)
+        assertEquals(sortedTestCases[7].test, headInd)
+    }
+
+}