From 15d8d8b1a315b8086b4ab244728bbba8f74fa84c Mon Sep 17 00:00:00 2001 From: Onur D Date: Tue, 26 Sep 2023 20:47:56 +0200 Subject: [PATCH] Developed Test cases, test methods and application --- .../e2etests/utils/RestTestBase.java | 118 +++++- .../EndpointFocusAndPrefixRest.java | 8 +- .../EndpointFocusAndPrefixTest.java | 392 +++++++++++++++++- 3 files changed, 508 insertions(+), 10 deletions(-) diff --git a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/RestTestBase.java b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/RestTestBase.java index 336b8d74af..cbb611067d 100644 --- a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/RestTestBase.java +++ b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/RestTestBase.java @@ -160,9 +160,83 @@ protected boolean hasAtLeastOne(EvaluatedIndividual ind, return false; } + + /** + Helper method to check if two given paths match based on focus or prefix + {@code @Parameter} path - Path we are analyzing + {@code @Parameter} pathFocusOrPrefix - Focus or prefix we are trying to check + {@code @Parameter} focusMode - true for checking focus, false for checking prefix + + */ + private boolean pathsMatchFocusOrPrefix(RestPath pathToAnalyze, RestPath pathFocusOrPrefix, boolean focusMode) + { + // check that pathToAnalyze and pathFocusOrPrefix are both not NULL + if(pathToAnalyze == null || pathFocusOrPrefix == null) { + throw new IllegalArgumentException("Invalid parameter(s), check that the given path parameters are" + + "both not NUL"); + } + + // if mode is focus, path and pathFocusOrPrefix match only if they are the same + if (focusMode) { + + return pathToAnalyze.isEquivalent(pathFocusOrPrefix); + } + // focusMode is false, which means prefix match + else { + // if the path we are analyzing starts with pathFocusOrPrefix + return pathToAnalyze.toString().startsWith(pathFocusOrPrefix.toString()); + } + } + + /* Check if a given individual has either of the given paths as the focus or + as the prefix + */ + protected boolean hasFocusOrPrefixInPath(EvaluatedIndividual ind, + List paths, boolean focusMode) { + + // if no paths are provided, none of the paths are focus of emoty path + // every path contains empty path as a prefix + if (paths == null || ind == null) { + throw new IllegalArgumentException("Invalid parameters provided to method, one or both of them is " + + "NULL: ind or paths"); + } + // if no paths are provided, none of the paths are focus of empty path + // every path contains empty path as a prefix + else if (paths.isEmpty()) { + + // in focusMode, none of the paths match with the empty path + // in prefix mode, every path matches with the empty path + return focusMode != true; + } + else { + + // actions and results + List actions = ind.getIndividual().seeMainExecutableActions(); + List results = ind.seeResults(actions); + + boolean noMatchFlag = false; + + for (int i = 0; i < actions.size() && !noMatchFlag; i++) { + + RestCallAction action = actions.get(i); + + if (paths.stream().noneMatch(currentPath -> pathsMatchFocusOrPrefix(action.getPath(), + currentPath, focusMode))) { + noMatchFlag = true; + } + + } + + // if a patch which does not match has been encountered, return false + return noMatchFlag != true; + } + + + } + /* Path only version of hasAtLeastOne - */ + protected boolean hasFocusInPath(EvaluatedIndividual ind, String path) { @@ -204,6 +278,41 @@ protected boolean hasFocusInPath(EvaluatedIndividual ind, return false; } + + */ + + + /* + All solutions should have one of the provided paths as the focus or the prefix + */ + protected void assertAllSolutionsHavePathFocusOrPrefixList(Solution solution, + List paths, boolean focusMode) { + + // convert String list of paths to list of RestPaths + List listOfRestPaths = new ArrayList<>(); + + for (String path : paths) { + listOfRestPaths.add(new RestPath(path)); + } + + // check that all paths have any of given paths as the focus or prefix. + boolean ok = solution.getIndividuals().stream().allMatch( + ind -> hasFocusOrPrefixInPath(ind, listOfRestPaths, focusMode) ); + + String errorMsg = "Not all the provided paths are contained in the solution as" + + " the focus or prefix\n"; + errorMsg = errorMsg + "List of paths given to check:\n"; + for (String path : paths) { + errorMsg = errorMsg + path + "\n"; + } + errorMsg = errorMsg + "List of paths included in the solution\n"; + errorMsg = errorMsg + restActions(solution); + + assertTrue(ok, errorMsg + restActions(solution)); + + } + + /* protected void assertAllSolutionsHavePathFocus(Solution solution, String path) { @@ -215,7 +324,9 @@ protected void assertAllSolutionsHavePathFocus(Solution solution assertTrue(ok, errorMsg + restActions(solution)); } + */ + /* protected boolean hasPrefixInPath(EvaluatedIndividual ind, String path) { @@ -257,6 +368,9 @@ protected boolean hasPrefixInPath(EvaluatedIndividual ind, return false; } + */ + + /* protected void assertAllSolutionsHavePathPrefix(Solution solution, String path) { @@ -269,6 +383,8 @@ protected void assertAllSolutionsHavePathPrefix(Solution solutio assertTrue(ok, errorMsg + restActions(solution)); } + */ + protected int countExpected(Solution solution, diff --git a/e2e-tests/spring-rest-openapi-v2/src/main/java/com/foo/rest/examples/spring/endpointFocusAndPrefix/EndpointFocusAndPrefixRest.java b/e2e-tests/spring-rest-openapi-v2/src/main/java/com/foo/rest/examples/spring/endpointFocusAndPrefix/EndpointFocusAndPrefixRest.java index 775a165e43..11705a5615 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/main/java/com/foo/rest/examples/spring/endpointFocusAndPrefix/EndpointFocusAndPrefixRest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/main/java/com/foo/rest/examples/spring/endpointFocusAndPrefix/EndpointFocusAndPrefixRest.java @@ -2,7 +2,13 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; -import org.springframework.web.bind.annotation.*; + + +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; import javax.ws.rs.core.MediaType; import java.util.Arrays; diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/endpointFocusAndPrefix/EndpointFocusAndPrefixTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/endpointFocusAndPrefix/EndpointFocusAndPrefixTest.java index 9d04074780..9126fb0a63 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/endpointFocusAndPrefix/EndpointFocusAndPrefixTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/endpointFocusAndPrefix/EndpointFocusAndPrefixTest.java @@ -1,20 +1,26 @@ package org.evomaster.e2etests.spring.examples.endpointFocusAndPrefix; import com.foo.rest.examples.spring.endpointFocusAndPrefix.EndpointFocusAndPrefixController; + import org.evomaster.client.java.instrumentation.shared.ClassName; +import org.evomaster.core.problem.rest.HttpVerb; +import org.evomaster.core.problem.rest.OpenApiAccess; import org.evomaster.core.problem.rest.RestIndividual; import org.evomaster.core.search.Solution; import org.evomaster.e2etests.spring.examples.SpringTestBase; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.Paths; +import io.swagger.v3.oas.models.PathItem; -import java.lang.reflect.InvocationTargetException; +import java.util.Collections; import java.util.List; import java.util.Arrays; -import static org.junit.jupiter.api.Assertions.assertThrows; - +import static org.junit.jupiter.api.Assertions.*; public class EndpointFocusAndPrefixTest extends SpringTestBase { @@ -28,14 +34,14 @@ public static void initClass() throws Exception { } @Test - public void testRunBlackWithoutFocusOrPrefix() { + public void testRunBlackboxWithoutFocusOrPrefix() { String outputFolder = "EndPointFocusAndPrefix"; List args = getArgsWithCompilation( 40, outputFolder, - ClassName.get("org.foo.NoEndpointFocusNoEndpointPrefix"), + ClassName.get("org.foo.WithoutFocusOrPrefix"), true); // program arguments for EvoMaster args.add("--blackBox"); @@ -48,16 +54,80 @@ public void testRunBlackWithoutFocusOrPrefix() { // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); - List pathsToCheck = Arrays.asList(); + // provide empty list to check for focus or prefix and put focusMode to false, + // which means prefix mode. Every path has the empty path as path prefix. + List pathsToCheck = Collections.emptyList(); // if neither focus nor prefix is provided, then all paths should include empty path as a prefix assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, false); + // put into the output folder compile(outputFolder); } @Test - public void testRunBlackWithFocusNoPrefixTest1() { + public void testAllPathsInTestWhenFocusOrPrefixNotProvided() { + + String outputFolder = "EndPointFocusAndPrefix"; + + List args = getArgsWithCompilation( + 40, + outputFolder, + ClassName.get("org.foo.WithoutFocusOrPrefixAllPaths"), + true); + // program arguments for EvoMaster + args.add("--blackBox"); + args.add("true"); + args.add("--bbTargetUrl"); + args.add(baseUrlOfSut); + args.add("--bbSwaggerUrl"); + args.add(baseUrlOfSut + "/v2/api-docs"); + + // no endpointFocus or endpointPrefix is provided + Solution solution = initAndRun(args); + + // paths to check + List pathsToCheck = Collections.emptyList(); + + // if neither focus nor prefix is provided, then all paths should include empty path as a prefix + assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, false); + + // get all paths from the swagger + OpenAPI swagger = OpenApiAccess.INSTANCE.getOpenAPIFromURL(baseUrlOfSut + "/v2/api-docs"); + + // api paths + Paths apiPaths = swagger.getPaths(); + + // current path item in the API + PathItem currentPathItem; + + // All paths in the swagger has to be included in tests + for (String apiPathString: apiPaths.keySet()){ + + // current path item + currentPathItem = apiPaths.get(apiPathString); + + // if the path item is a GET request + if (currentPathItem.getGet() != null) { + assertHasAtLeastOne(solution, HttpVerb.GET, 200, apiPathString, null); + } + // if the path item is a POST request + else if (currentPathItem.getPost() != null) { + assertHasAtLeastOne(solution, HttpVerb.POST, 200, apiPathString, null); + } + // if the path item is a PUT request + else if (currentPathItem.getPut() != null) { + assertHasAtLeastOne(solution, HttpVerb.PUT, 200, apiPathString, null); + } + // if the path item is a DELETE request + else if (currentPathItem.getDelete() != null) { + assertHasAtLeastOne(solution, HttpVerb.DELETE, 200, apiPathString, null); + } + } + } + + @Test + public void testRunBlackboxWithFocusWithoutParameters() { String outputFolder = "EndPointFocusAndPrefix"; @@ -66,7 +136,7 @@ public void testRunBlackWithFocusNoPrefixTest1() { List args = getArgsWithCompilation( 40, outputFolder, - ClassName.get("org.foo.NoEndpointFocusNoEndpointPrefix"), + ClassName.get("org.foo.WithNonParametrizedFocus"), true); // program arguments for EvoMaster args.add("--blackBox"); @@ -88,8 +158,314 @@ public void testRunBlackWithFocusNoPrefixTest1() { // if neither focus nor prefix is provided, then all paths should include empty path as a prefix assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, true); + // There are 2 endpoints with /api/pet, those and the failure case should be included in tests + // so check that the solution contains 3 elements + assertEquals(solution.getIndividuals().size(), 3); + + compile(outputFolder); + } + + @Test + public void testRunBlackboxWithFocusWithParameters() { + + String outputFolder = "EndPointFocusAndPrefix"; + + String endpointFocus = "/api/pet/{petId}"; + + List args = getArgsWithCompilation( + 40, + outputFolder, + ClassName.get("org.foo.WithParametrizedFocus"), + true); + + // program arguments for EvoMaster + args.add("--blackBox"); + args.add("true"); + args.add("--bbTargetUrl"); + args.add(baseUrlOfSut); + args.add("--bbSwaggerUrl"); + args.add(baseUrlOfSut + "/v2/api-docs"); + args.add("--endpointFocus"); + args.add(endpointFocus); + + // no endpointFocus or endpointPrefix is provided + Solution solution = initAndRun(args); + + // include swagger into possible solutions as /v2/api-docs + List pathsToCheck = Arrays.asList(endpointFocus, "/v2/api-docs"); + + // if neither focus nor prefix is provided, then all paths should include empty path as a prefix + assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, true); + + // The solution should include 4 solutions, 3 endpoints and 1 failure case + assertEquals(solution.getIndividuals().size(), 4); + compile(outputFolder); } + @Test + public void testRunBlackboxWithFocusOneEndpoint() { + String outputFolder = "EndPointFocusAndPrefix"; + + String endpointFocus = "/api/store/inventory"; + + List args = getArgsWithCompilation( + 40, + outputFolder, + ClassName.get("org.foo.InvalidFocusValidPrefix"), + true); + + // program arguments for EvoMaster + args.add("--blackBox"); + args.add("true"); + args.add("--bbTargetUrl"); + args.add(baseUrlOfSut); + args.add("--bbSwaggerUrl"); + args.add(baseUrlOfSut + "/v2/api-docs"); + args.add("--endpointFocus"); + args.add(endpointFocus); + + // no endpointFocus or endpointPrefix is provided + Solution solution = initAndRun(args); + + // include swagger into possible solutions as /v2/api-docs + List pathsToCheck = Arrays.asList(endpointFocus, "/v2/api-docs"); + + // if neither focus nor prefix is provided, then all paths should include empty path as a prefix + assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, true); + + // The solution should include 2 solutions, 1 endpoints and 1 failure case + assertEquals(solution.getIndividuals().size(), 2); + + compile(outputFolder); + } + + @Test + public void testRunBlackboxWithPrefixWithoutParameters() { + + String outputFolder = "EndPointFocusAndPrefix"; + + String endpointPrefix = "/api/user"; + + List args = getArgsWithCompilation( + 40, + outputFolder, + ClassName.get("org.foo.WithNonParametrizedPrefix"), + true); + // program arguments for EvoMaster + args.add("--blackBox"); + args.add("true"); + args.add("--bbTargetUrl"); + args.add(baseUrlOfSut); + args.add("--bbSwaggerUrl"); + args.add(baseUrlOfSut + "/v2/api-docs"); + args.add("--endpointPrefix"); + args.add(endpointPrefix); + + // no endpointFocus or endpointPrefix is provided + Solution solution = initAndRun(args); + + // include swagger into possible solutions as /v2/api-docs + List pathsToCheck = Arrays.asList(endpointPrefix, "/v2/api-docs"); + + // if neither focus nor prefix is provided, then all paths should include empty path as a prefix + assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, false); + + // The solution should include 8 solutions, 7 endpoints and 1 failure case + assertEquals(solution.getIndividuals().size(), 8); + + compile(outputFolder); + } + + @Test + public void testRunBlackboxWithPrefixWithParameters() { + + String outputFolder = "EndPointFocusAndPrefix"; + + String endpointPrefix = "/api/pet/{petId}"; + + List args = getArgsWithCompilation( + 40, + outputFolder, + ClassName.get("org.foo.WithParametrizedPrefix"), + true); + // program arguments for EvoMaster + args.add("--blackBox"); + args.add("true"); + args.add("--bbTargetUrl"); + args.add(baseUrlOfSut); + args.add("--bbSwaggerUrl"); + args.add(baseUrlOfSut + "/v2/api-docs"); + args.add("--endpointPrefix"); + args.add(endpointPrefix); + + // no endpointFocus or endpointPrefix is provided + Solution solution = initAndRun(args); + + // include swagger into possible solutions as /v2/api-docs + List pathsToCheck = Arrays.asList(endpointPrefix, "/v2/api-docs"); + + // if neither focus nor prefix is provided, then all paths should include empty path as a prefix + assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, false); + + // The solution should include 5 solutions, 4 endpoints and 1 failure case + assertEquals(solution.getIndividuals().size(), 5); + + compile(outputFolder); + } + + @Test + public void testRunBlackboxFocusNonExistingFocusValidPrefix() { + + String outputFolder = "EndPointFocusAndPrefix"; + + String endpointFocus = "/api/store"; + + List args = getArgsWithCompilation( + 40, + outputFolder, + ClassName.get("org.foo.WithParametrizedPrefix"), + true); + // program arguments for EvoMaster + args.add("--blackBox"); + args.add("true"); + args.add("--bbTargetUrl"); + args.add(baseUrlOfSut); + args.add("--bbSwaggerUrl"); + args.add(baseUrlOfSut + "/v2/api-docs"); + args.add("--endpointFocus"); + args.add(endpointFocus); + + // check for IllegalArgumentException + try { + initAndRun(args); + } + catch (Exception e) { + assertTrue(e.getCause().toString().contains(IllegalArgumentException.class.getName())); + } + } + + @Test + public void testRunBlackboxPrefixNonExistingFocusValidPrefix() { + + String outputFolder = "EndPointFocusAndPrefix"; + + String endpointPrefix = "/api/store"; + + List args = getArgsWithCompilation( + 40, + outputFolder, + ClassName.get("org.foo.WithParametrizedPrefix"), + true); + // program arguments for EvoMaster + args.add("--blackBox"); + args.add("true"); + args.add("--bbTargetUrl"); + args.add(baseUrlOfSut); + args.add("--bbSwaggerUrl"); + args.add(baseUrlOfSut + "/v2/api-docs"); + args.add("--endpointPrefix"); + args.add(endpointPrefix); + + // no endpointFocus or endpointPrefix is provided + Solution solution = initAndRun(args); + + // include swagger into possible solutions as /v2/api-docs + List pathsToCheck = Arrays.asList(endpointPrefix, "/v2/api-docs"); + + // if neither focus nor prefix is provided, then all paths should include empty path as a prefix + assertAllSolutionsHavePathFocusOrPrefixList(solution, pathsToCheck, false); + + // The solution should include 5 solutions, 4 endpoints and 1 failure case + assertEquals(solution.getIndividuals().size(), 5); + + compile(outputFolder); + + } + + @Test + public void testRunBlackboxNonExistingFocusNonExistingPrefix() { + + String outputFolder = "EndPointFocusAndPrefix"; + + String endpointPrefix = "/api/ab/s1"; + + List args = getArgsWithCompilation( + 40, + outputFolder, + ClassName.get("org.foo.WithParametrizedPrefix"), + true); + // program arguments for EvoMaster + args.add("--blackBox"); + args.add("true"); + args.add("--bbTargetUrl"); + args.add(baseUrlOfSut); + args.add("--bbSwaggerUrl"); + args.add(baseUrlOfSut + "/v2/api-docs"); + args.add("--endpointPrefix"); + args.add(endpointPrefix); + + // check for IllegalArgumentException + assertThrows(IllegalArgumentException.class, () -> initAndRun(args)); + + } + + + @Test + public void testRunBlackboxPrefixNonExistingPrefix() { + + String outputFolder = "EndPointFocusAndPrefix"; + + String endpointPrefix = "/api/store/inventory/in"; + + List args = getArgsWithCompilation( + 40, + outputFolder, + ClassName.get("org.foo.WithParametrizedPrefix"), + true); + // program arguments for EvoMaster + args.add("--blackBox"); + args.add("true"); + args.add("--bbTargetUrl"); + args.add(baseUrlOfSut); + args.add("--bbSwaggerUrl"); + args.add(baseUrlOfSut + "/v2/api-docs"); + args.add("--endpointPrefix"); + args.add(endpointPrefix); + + // check for IllegalArgumentException + assertThrows(IllegalArgumentException.class, () -> initAndRun(args)); + + } + +@Test +public void testRunBlackboxBothFocusAndPrefix() { + + String outputFolder = "EndPointFocusAndPrefix"; + + String endpoint = "/api/store/order"; + + List args = getArgsWithCompilation( + 40, + outputFolder, + ClassName.get("org.foo.WithParametrizedPrefix"), + true); + + // program arguments for EvoMaster + args.add("--blackBox"); + args.add("true"); + args.add("--bbTargetUrl"); + args.add(baseUrlOfSut); + args.add("--bbSwaggerUrl"); + args.add(baseUrlOfSut + "/v2/api-docs"); + args.add("--endpointPrefix"); + args.add(endpoint); + args.add("--endpointFocus"); + args.add(endpoint); + + // check for IllegalArgumentException + assertThrows(IllegalArgumentException.class, () -> initAndRun(args)); + + } }