diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c0464d9500..def71570f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,6 +45,23 @@ jobs: path: core/target/evomaster.jar retention-days: ${{env.retention-days}} if-no-files-found: error + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: webfuzzing/evomaster + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push Docker + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} installer-for-windows: needs: build-base @@ -129,7 +146,6 @@ jobs: runs-on: "ubuntu-latest" steps: - - name: Download fat jar uses: actions/download-artifact@v4 with: @@ -146,19 +162,6 @@ jobs: uses: actions/download-artifact@v4 with: name: evomaster.msi - - -## Doesn't seem any longer maintained... plus usability issues -# - uses: "marvinpinto/action-automatic-releases@latest" -# with: -# repo_token: "${{ secrets.GITHUB_TOKEN }}" -# prerelease: false -# files: | -# evomaster.jar -# ${{env.installer-windows}} -# ${{env.installer-osx}} -# ${{env.installer-debian}} - - name: Release uses: softprops/action-gh-release@v1 with: diff --git a/.gitignore b/.gitignore index ca6840c178..87a538f5ef 100644 --- a/.gitignore +++ b/.gitignore @@ -171,3 +171,4 @@ Migrations/ *.DS_Store /e2e-tests/emb-json/target/ /process_data/ +/e2e-tests/spring-rest-multidb/target/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..af695de3d5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +FROM amazoncorretto:21-alpine-jdk + +COPY core/target/evomaster.jar . + +ENTRYPOINT [ \ + "java", \ + "-Xmx4G", \ + "-jar", "evomaster.jar", \ + "--runningInDocker", "true" \ +] + + +################### +###### NOTES ###### +################### +# Build +# docker build -t webfuzzing/evomaster . +# +# Run +# docker run webfuzzing/evomaster +# +# Publish (latest, otherwise tag with :TAG) +# docker login +# docker push webfuzzing/evomaster +# +# Example remote BB +# docker run -v "/$(pwd)/generated_tests":/generated_tests webfuzzing/evomaster --blackBox true --bbSwaggerUrl https://api.apis.guru/v2/openapi.yaml --outputFormat JAVA_JUNIT_4 --maxTime 10s --ratePerMinute 60 +# +# Example local BB +# docker run -v "/$(pwd)/generated_tests":/generated_tests webfuzzing/evomaster --blackBox true --bbSwaggerUrl http://host.docker.internal:8080/v3/api-docs --maxTime 5s +# +# Example WB (NOT IMPLEMENTED YET) +# docker run -v "/$(pwd)/generated_tests":/generated_tests webfuzzing/evomaster --dockerLocalhost true +# +# Setting for existing em.yaml +# -v "/$(pwd)/em.yaml":/em.yaml +# +# Debugging +# docker run -it --entrypoint sh webfuzzing/evomaster +# +# +# +# +# +# +# \ No newline at end of file diff --git a/README.md b/README.md index 978791369b..77752bf13b 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,17 @@ __Known limitations__: --> +### Use in Industry + +Several enterprises use _EvoMaster_ to fuzz their Web APIs. +We do few academia-industry collaborations ([see more info here](docs/contribute.md)), where we help test engineers to apply _EvoMaster_ on their systems, as long as we can then report on such experience. +Example of Fortune 500 companies using _EvoMaster_ are: + +* [Meituan](https://www.meituan.com): see [TOSEM'23](docs/publications/2023_tosem_rpc.pdf), [ASE'24](docs/publications/2024_ase.pdf). + +* [Volkswagen](https://www.volkswagen.com): see [AUSE'24](docs/publications/2024_ause_vw.pdf), [ICST'25](docs/publications/2025_icst.pdf). + + ### Videos ![](docs/img/video-player-flaticon.png) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java index 3161c55509..c31274c857 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java @@ -379,7 +379,11 @@ public final void stopSut() { @Override public final boolean isInstrumentationActivated() { - return instrumentation && serverController != null && serverController.isConnectionOn(); + return instrumentation && isConnectedToServerController(); + } + + public final boolean isConnectedToServerController(){ + return serverController != null && serverController.isConnectionOn(); } @Override @@ -499,6 +503,14 @@ public final void setExecutingAction(boolean executingAction){ @Override public final void bootingSut(boolean bootingSut) { + if(bootingSut && !isConnectedToServerController()){ + /* + we cannot connect to server before SUT is started... but, once started, + might already be too late to state that it is in booting phase. + so, the default should be "booting". + */ + return; + } checkInstrumentation(); serverController.setBootingSut(bootingSut); // sync on the local ExecutionTracer diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java index 50a44a6742..9a8a23974b 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java @@ -377,6 +377,7 @@ public Response runSut(SutRunDto dto, @Context HttpServletRequest httpServletReq } else { //TODO as starting should be blocking, need to check //if initialized, and wait if not + noKillSwitch(() -> sutController.bootingSut(false)); } /* diff --git a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/coverage/methodreplacement/classes/MatcherClassReplacement.java b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/coverage/methodreplacement/classes/MatcherClassReplacement.java index c3a2e6f346..df5c112d0b 100644 --- a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/coverage/methodreplacement/classes/MatcherClassReplacement.java +++ b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/coverage/methodreplacement/classes/MatcherClassReplacement.java @@ -7,6 +7,7 @@ import org.evomaster.client.java.instrumentation.shared.*; import org.evomaster.client.java.instrumentation.staticstate.ExecutionTracer; import org.evomaster.client.java.instrumentation.shared.ReplacementType; +import org.evomaster.client.java.utils.SimpleLogger; import java.lang.reflect.Field; import java.util.Objects; @@ -110,7 +111,10 @@ match the regex, and find() only requires String anyPositionRegexMatch = RegexSharedUtils.handlePartialMatch(regex); boolean patternMatchResult = PatternMatchingHelper.matches(anyPositionRegexMatch, substring, idTemplate); boolean matcherFindResult = caller.find(); - assert (patternMatchResult == matcherFindResult); + if(patternMatchResult != matcherFindResult){ + //TODO we should analyze those cases, and fix them + SimpleLogger.uniqueWarn("Failed to handle regex in Matcher.find(): " + regex); + } return matcherFindResult; } diff --git a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/AgentController.java b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/AgentController.java index 633eff3df3..0ecdbe7f6d 100644 --- a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/AgentController.java +++ b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/AgentController.java @@ -101,6 +101,7 @@ public static void start(int port){ case BOOTING_SUT: handleBootingSut(); sendCommand(Command.ACK); + break; case BOOT_TIME_INFO: handleBootTimeObjectiveInfo(); break; diff --git a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/staticstate/ObjectiveRecorder.java b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/staticstate/ObjectiveRecorder.java index c33565b3e5..c9e70e2cfc 100644 --- a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/staticstate/ObjectiveRecorder.java +++ b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/staticstate/ObjectiveRecorder.java @@ -20,8 +20,11 @@ public class ObjectiveRecorder { /** * Specify whether the SUT is booting or not. * Targets during booting time are treated specially. + * + * Note: the default MUST be true, as might not be possible to modify this + * setting before the SUT is already started */ - private static volatile boolean isBooting = false; + private static volatile boolean isBooting = true; /** * Key -> the unique id of the coverage objective @@ -101,6 +104,7 @@ public class ObjectiveRecorder { * Reset all the static state in this class */ public static void reset(boolean alsoAtLoadTime) { + maxObjectiveCoverage.clear(); idMapping.clear(); reversedIdMapping.clear(); @@ -116,6 +120,7 @@ public static void reset(boolean alsoAtLoadTime) { allTargets.clear(); bootTimeObjectiveInfo.reset(); + isBooting = true; } } diff --git a/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/coverage/methodreplacement/thirdpartyclasses/JacksonObjectMapperTest.java b/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/coverage/methodreplacement/thirdpartyclasses/JacksonObjectMapperTest.java index 4978b93a13..d7c254e0df 100644 --- a/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/coverage/methodreplacement/thirdpartyclasses/JacksonObjectMapperTest.java +++ b/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/coverage/methodreplacement/thirdpartyclasses/JacksonObjectMapperTest.java @@ -7,7 +7,9 @@ import org.evomaster.client.java.instrumentation.shared.StringSpecializationInfo; import org.evomaster.client.java.instrumentation.shared.TaintInputName; import org.evomaster.client.java.instrumentation.staticstate.ExecutionTracer; +import org.evomaster.client.java.instrumentation.staticstate.ObjectiveRecorder; import org.evomaster.client.java.instrumentation.staticstate.UnitsInfoRecorder; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.*; @@ -18,6 +20,16 @@ public class JacksonObjectMapperTest { + @BeforeEach + public void init(){ + ObjectiveRecorder.reset(false); + ObjectiveRecorder.setBooting(false); + ExecutionTracer.reset(); + // force the state as executing action + ExecutionTracer.setExecutingAction(true); + assertEquals(0 , ExecutionTracer.getNumberOfObjectives()); + } + @Test public void testReadValue() throws Throwable { String json = "{\n\"count\": 10\n}"; diff --git a/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/example/branches/BranchesInstrumentedTest.java b/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/example/branches/BranchesInstrumentedTest.java index be8d0e29ef..db1e16af0d 100644 --- a/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/example/branches/BranchesInstrumentedTest.java +++ b/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/example/branches/BranchesInstrumentedTest.java @@ -57,6 +57,7 @@ public static void initClass(){ @BeforeEach public void init(){ ObjectiveRecorder.reset(false); + ObjectiveRecorder.setBooting(false); ExecutionTracer.reset(); // force the state as executing action ExecutionTracer.setExecutingAction(true); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java index 24519b0ff5..433eacbb53 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java @@ -411,7 +411,8 @@ private static String getDefaultSchema(DatabaseType type) { return "dbo"; case MARIADB: case MYSQL: - throw new IllegalArgumentException("there is no default schema for " + type + ", and you must specify a db name here"); + //there is no concept of schema for those databases + return null; case POSTGRES: return "public"; } @@ -434,12 +435,11 @@ private static String getAllTableCommand(DatabaseType type, String schema) { switch (type) { // https://stackoverflow.com/questions/175415/how-do-i-get-list-of-all-tables-in-a-database-using-tsql, TABLE_CATALOG='"+dbname+"'" case MS_SQL_SERVER: - // for MySQL, schema is dbname case MYSQL: case MARIADB: case H2: case POSTGRES: - if (schema.isEmpty()) + if (schema == null || schema.isEmpty()) return command; return command + " AND TABLE_SCHEMA='" + schema + "'"; } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/VariableDescriptor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/VariableDescriptor.java index 3add115480..f6bf6d5eda 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/VariableDescriptor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/VariableDescriptor.java @@ -26,7 +26,8 @@ public VariableDescriptor(String columnName) { } public VariableDescriptor(String columnName, String alias, String tableName) { - this.columnName = columnName.trim().toLowerCase(); + this.columnName = (columnName==null || columnName.trim().isEmpty() ? + null : columnName.trim().toLowerCase()); this.alias = (alias == null || alias.trim().isEmpty() ? this.columnName : alias.trim().toLowerCase()); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzer.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzer.java index 62e518366e..c05f5042e5 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzer.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzer.java @@ -30,7 +30,7 @@ public static Set getDeletedTables(String delete){ Table table = stmt.getTable(); if(table != null){ - set.add(table.getName()); + set.add(table.getFullyQualifiedName()); } else { //TODO need to handle special cases of multi-tables with JOINs throw new IllegalArgumentException("Cannot handle delete: " + delete); @@ -130,7 +130,7 @@ public static Map> getSelectReadDataFields(String select){ } private static void handleTable(Map> map, Table table){ - Set columns = map.computeIfAbsent(table.getName(), k -> new HashSet<>()); + Set columns = map.computeIfAbsent(table.getFullyQualifiedName(), k -> new HashSet<>()); //TODO: should check actual fields... would likely need to pass SelectBody as input as well if(! columns.contains("*")) { columns.add("*"); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java index 8027fbd3bf..debd1ef8cb 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java @@ -51,20 +51,26 @@ public static SqlDistanceWithMetrics computeDistance( QueryResult... data ) { + /** + * This is not an ideal solution, but it will remain this way until config.heuristicsForSQLAdvanced remains + * experimental and it does not replace the "Plain" SqlHeuristicsCalculator + */ + if (advancedHeuristics) { + return SqlHeuristicsCalculator.computeDistance(sqlCommand,schema,taintHandler,data); + } + if (data.length == 0 || Arrays.stream(data).allMatch(QueryResult::isEmpty)){ //if no data, we have no info whatsoever return new SqlDistanceWithMetrics(Double.MAX_VALUE,0, false); } Statement stmt = SqlParserUtils.parseSqlCommand(sqlCommand); - Expression where = getWhere(stmt); if (where == null) { //no constraint and at least one data point return new SqlDistanceWithMetrics(0.0,0, false); } - SqlNameContext context = new SqlNameContext(stmt); if (schema != null) { context.setSchema(schema); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHeuristicsCalculator.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHeuristicsCalculator.java new file mode 100644 index 0000000000..46d825ee5e --- /dev/null +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHeuristicsCalculator.java @@ -0,0 +1,30 @@ +package org.evomaster.client.java.sql.internal; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.FromItem; +import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; +import org.evomaster.client.java.sql.QueryResult; + +import static org.evomaster.client.java.sql.internal.SqlParserUtils.getFrom; +import static org.evomaster.client.java.sql.internal.SqlParserUtils.getWhere; + +public class SqlHeuristicsCalculator { + + public static SqlDistanceWithMetrics computeDistance(String sqlCommand, + DbInfoDto schema, + TaintHandler taintHandler, + QueryResult... data) { + + Statement parsedSqlCommand = SqlParserUtils.parseSqlCommand(sqlCommand); + Expression whereClause= getWhere(parsedSqlCommand); + FromItem fromItem = getFrom(parsedSqlCommand); + + if (fromItem == null && whereClause == null) { + return new SqlDistanceWithMetrics(0.0,0,false); + } + + + return null; + } +} diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlParserUtils.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlParserUtils.java index 9fa4a17843..00003de09f 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlParserUtils.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlParserUtils.java @@ -5,6 +5,7 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.Statement; import net.sf.jsqlparser.statement.delete.Delete; +import net.sf.jsqlparser.statement.select.FromItem; import net.sf.jsqlparser.statement.select.PlainSelect; import net.sf.jsqlparser.statement.select.Select; import net.sf.jsqlparser.statement.update.Update; @@ -55,7 +56,6 @@ public static boolean isSelectOne(String sqlCommand) { public static Expression getWhere(Statement parsedStatement) { - if (parsedStatement instanceof Select) { Select select = (Select) parsedStatement; PlainSelect plainSelect = select.getPlainSelect(); @@ -69,6 +69,32 @@ public static Expression getWhere(Statement parsedStatement) { } } + /** + * Extracts the "FROM" clause or the primary table involved in a SQL statement. + * This method supports SELECT, DELETE, and UPDATE SQL statements. + * + * @param parsedStatement The parsed SQL statement as a {@link Statement} object. + * This is typically obtained using JSQLParser's `CCJSqlParserUtil.parse`. + * @return The {@link FromItem} representing the "FROM" clause or the main table for the statement. + * - For a SELECT statement, returns the main {@link FromItem} in the "FROM" clause. + * - For a DELETE statement, returns the table being deleted from. + * - For an UPDATE statement, returns the table being updated. + * @throws IllegalArgumentException If the provided statement type is not SELECT, DELETE, or UPDATE. + */ + public static FromItem getFrom(Statement parsedStatement) { + if (parsedStatement instanceof Select) { + Select select = (Select) parsedStatement; + PlainSelect plainSelect = select.getPlainSelect(); + return plainSelect.getFromItem(); + } else if(parsedStatement instanceof Delete){ + return ((Delete) parsedStatement).getTable(); + } else if(parsedStatement instanceof Update){ + return ((Update) parsedStatement).getTable(); + } else { + throw new IllegalArgumentException("Cannot handle statement: " + parsedStatement.toString()); + } + } + /** * This method assumes that the SQL command can be successfully parsed. * diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java index 06a2c8f538..91f6af398e 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java @@ -14,6 +14,18 @@ public class ColumnTableAnalyzerTest { + @Test + public void testInsertWithQualifier(){ + + String sql = "insert into Bar.Foo (x) values (42)"; + + Map> data = ColumnTableAnalyzer.getInsertedDataFields(sql); + + assertEquals(1, data.size()); + assertTrue(data.containsKey("Bar.Foo")); + } + + @Test public void testInsertInSimpleTable(){ @@ -50,6 +62,18 @@ public void testDeleteSimpleTable(){ assertTrue(tables.contains("Foo")); } + @Test + public void testDeleteWithQualifier(){ + + String sql = "delete from v1.Foo"; + + Set tables = ColumnTableAnalyzer.getDeletedTables(sql); + + assertEquals(1, tables.size()); + assertTrue(tables.contains("v1.Foo")); + } + + @Test public void testSelectReadAllFromSingleTable(){ diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorTest.java index 0b3af37824..fda975518a 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorTest.java @@ -425,4 +425,13 @@ public void testWhereWithNoColumns() { assertEquals(0d, dist); } + @Test + public void testNoWhereNoFrom() { + String sqlCommand = "SELECT 1 as example_column"; + QueryResult data = new QueryResult(Arrays.asList("example_column"), null); + data.addRow(new DataRow("example_column",1, null)); + double distance = HeuristicsCalculator.computeDistance(sqlCommand,data); + assertEquals(0, distance); + } + } diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHeuristicsCalculatorTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHeuristicsCalculatorTest.java new file mode 100644 index 0000000000..962b675083 --- /dev/null +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHeuristicsCalculatorTest.java @@ -0,0 +1,21 @@ +package org.evomaster.client.java.sql.internal; + +import org.evomaster.client.java.sql.DataRow; +import org.evomaster.client.java.sql.QueryResult; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SqlHeuristicsCalculatorTest { + + @Test + public void testNoWhereNoFromClause() { + String sqlCommand = "SELECT 1 AS example_column"; + QueryResult data = new QueryResult(Arrays.asList("example_column"), null); + data.addRow(new DataRow("example_column",1, null)); + SqlDistanceWithMetrics distanceWithMetrics = SqlHeuristicsCalculator.computeDistance(sqlCommand, null, null, data); + assertEquals(0, distanceWithMetrics.sqlDistance); + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 63b17a2a66..6b775ff2f2 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -1,9 +1,6 @@ package org.evomaster.core -import joptsimple.BuiltinHelpFormatter -import joptsimple.OptionDescriptor -import joptsimple.OptionParser -import joptsimple.OptionSet +import joptsimple.* import org.evomaster.client.java.controller.api.ControllerConstants import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto import org.evomaster.client.java.instrumentation.shared.ExternalServiceSharedUtils @@ -585,6 +582,18 @@ class EMConfig { throw ConfigProblemException("The use of 'prematureStop' is meaningful only if the stopping criterion" + " 'stoppingCriterion' is based on time") } + + if(blackBox){ + if(sutControllerHost != ControllerConstants.DEFAULT_CONTROLLER_HOST){ + throw ConfigProblemException("Changing 'sutControllerHost' has no meaning in black-box testing, as no controller is used") + } + if(!overrideOpenAPIUrl.isNullOrBlank()){ + throw ConfigProblemException("Changing 'overrideOpenAPIUrl' has no meaning in black-box testing, as no controller is used") + } + } + if(dockerLocalhost && !runningInDocker){ + throw ConfigProblemException("Specifying 'dockerLocalhost' only makes sense when running EvoMaster inside Docker.") + } } private fun checkPropertyConstraints(m: KMutableProperty<*>) { @@ -696,8 +705,11 @@ class EMConfig { return } - val opt = options.valueOf(m.name)?.toString() - ?: throw ConfigProblemException("Value not found for property ${m.name}") + val opt = try{ + options.valueOf(m.name)?.toString() + } catch (e: OptionException){ + throw ConfigProblemException("Error in parsing configuration option '${m.name}'. Library message: ${e.message}") + } ?: throw ConfigProblemException("Value not found for property '${m.name}'") updateValue(opt, m) } @@ -1048,9 +1060,43 @@ class EMConfig { " If no tag is specified here, then such filter is not applied.") var endpointTagFilter: String? = null + @Important(6.0) + @Cfg("Host name or IP address of where the SUT EvoMaster Controller Driver is listening on." + + " This option is only needed for white-box testing.") + var sutControllerHost = ControllerConstants.DEFAULT_CONTROLLER_HOST + + + @Important(6.1) + @Cfg("TCP port of where the SUT EvoMaster Controller Driver is listening on." + + " This option is only needed for white-box testing.") + @Min(0.0) + @Max(maxTcpPort) + var sutControllerPort = ControllerConstants.DEFAULT_CONTROLLER_PORT + + + @Important(7.0) + @Url + @Cfg("If specified, override the OpenAPI URL location given by the EvoMaster Driver." + + " This option is only needed for white-box testing.") + var overrideOpenAPIUrl = "" //-------- other options ------------- + @Cfg("Inform EvoMaster process that it is running inside Docker." + + " Users should not modify this parameter, as it is set automatically in the Docker image of EvoMaster.") + var runningInDocker = false + + /** + * TODO this is currently not implemented. + * Even if did, there would still be major issues with handling WireMock. + * Until we can think of a good solution there, no point in implementing this. + */ + @Experimental + @Cfg("Replace references to 'localhost' to point to the actual host machine." + + " Only needed when running EvoMaster inside Docker.") + var dockerLocalhost = false + + @FilePath @Cfg("When generating tests in JavaScript, there is the need to know where the driver is located in respect to" + " the generated tests") @@ -1137,14 +1183,6 @@ class EMConfig { "A negative value means the CPU clock time will be rather used as seed") var seed: Long = -1 - @Cfg("TCP port of where the SUT REST controller is listening on") - @Min(0.0) - @Max(maxTcpPort) - var sutControllerPort = ControllerConstants.DEFAULT_CONTROLLER_PORT - - @Cfg("Host name or IP address of where the SUT REST controller is listening on") - var sutControllerHost = ControllerConstants.DEFAULT_CONTROLLER_HOST - @Cfg("Limit of number of individuals per target to keep in the archive") @Min(1.0) var archiveTargetLimit = 10 @@ -2381,6 +2419,11 @@ class EMConfig { @Cfg("Specify the naming strategy for test cases.") var namingStrategy = defaultTestCaseNamingStrategy + @Experimental + @Cfg("Specify if true boolean query parameters are included in the test case name." + + " Used for test case naming disambiguation. Only valid for Action based naming strategy.") + var nameWithQueryParameters = false + @Experimental @Probability(true) @@ -2393,6 +2436,15 @@ class EMConfig { " instead of OFF.") val probabilityOfOnVsOffInAllOptionals = 0.8 + @Experimental + @Cfg("Add summary comments on each test") + var addTestComments = false + + @Min(1.0) + @Cfg("Max length for test comments. Needed when enumerating some names/values, making comments too long to be" + + " on a single line") + var maxLengthForCommentLine = 80 + fun getProbabilityUseDataPool() : Double{ return if(blackBox){ bbProbabilityUseDataPool diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index d83f42da52..809da3aaae 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -75,11 +75,39 @@ class Main { return } - if (parser.parse(*args).has("help")) { + val options = parser.parse(*args) + + if (options.has("help")) { parser.printHelpOn(System.out) return } + val config = EMConfig().apply { updateProperties(options) } + + if(config.runningInDocker){ + if(config.blackBox) { + LoggingUtil.getInfoLogger().info( + inGreen("You are running EvoMaster inside Docker." + + " To access the generated test suite under '/generated_tests', you will need to mount a folder" + + " or volume." + + " Also references to host machine on 'localhost' would need to be replaced with" + + " 'host.docker.internal'." + + " If this is the first time you run EvoMaster in Docker, you are strongly recommended to first" + + " check the documentation at:") + + " ${inBlue("https://github.com/WebFuzzing/EvoMaster/blob/master/docs/docker.md")}" + ) + } else { + LoggingUtil.getInfoLogger().warn( + inYellow( + "White-box testing (default in EvoMaster) is currently not supported / not recommended in Docker." + + " To run EvoMaster in black-box mode, you can use '--blackBox true'." + + " If you need to run in white-box mode, it is recommended to download an OS installer or" + + " the uber JAR file from the release-page on GitHub." + ) + ) + } + } + initAndRun(args) LoggingUtil.getInfoLogger().apply { diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt index a3378ef294..c9abe2055e 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt @@ -31,6 +31,7 @@ abstract class ActionTestCaseNamingStrategy( protected val with = "with" protected val param = "Param" protected val queryParam = "query$param" + protected val and = "and" protected fun formatName(nameTokens: List): String { return "_${languageConventionFormatter.formatName(nameTokens)}" diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/RestActionTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/RestActionTestCaseNamingStrategy.kt index d9a806abda..652349ad01 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/RestActionTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/RestActionTestCaseNamingStrategy.kt @@ -11,11 +11,13 @@ import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Solution import org.evomaster.core.search.action.Action import org.evomaster.core.search.action.EvaluatedAction +import org.evomaster.core.search.gene.BooleanGene import javax.ws.rs.core.MediaType open class RestActionTestCaseNamingStrategy( solution: Solution<*>, - languageConventionFormatter: LanguageConventionFormatter + languageConventionFormatter: LanguageConventionFormatter, + private val nameWithQueryParameters: Boolean, ) : ActionTestCaseNamingStrategy(solution, languageConventionFormatter) { override fun expandName(individual: EvaluatedIndividual<*>, nameTokens: MutableList, ambiguitySolver: ((Action) -> List)?): String { @@ -86,7 +88,6 @@ open class RestActionTestCaseNamingStrategy( .filter { it.value.size == 1 && !it.key.second } .mapNotNull { entry -> val eInd = entry.value[0] -// duplicatedIndividuals.remove(eInd) eInd to expandName(eInd, mutableListOf(), ::pathAmbiguitySolver) } .toMap() @@ -130,7 +131,6 @@ open class RestActionTestCaseNamingStrategy( .filter { it.value.size == 1 && it.key.isNotEmpty()} .mapNotNull { entry -> val eInd = entry.value[0] -// duplicatedIndividuals.remove(eInd) eInd to expandName(eInd, mutableListOf(), ::queryParamsAmbiguitySolver) } .toMap() @@ -147,9 +147,26 @@ open class RestActionTestCaseNamingStrategy( val queryParams = restAction.parameters.filterIsInstance() result.add(with) result.add(if (queryParams.size > 1) "${queryParam}s" else queryParam) + if (nameWithQueryParameters) { + addQueryParameterNames(queryParams, result) + } return result } + private fun addQueryParameterNames(queryParams: List, result: MutableList) { + val booleanQueryParams = getBooleanQueryParams(queryParams) + booleanQueryParams.forEachIndexed { index, queryParam -> + result.add(queryParam.name) + if (index != booleanQueryParams.lastIndex) { + result.add(and) + } + } + } + + private fun getBooleanQueryParams(queryParams: List): List { + return queryParams.filter { it.getGeneForQuery() is BooleanGene && (it.getGeneForQuery() as BooleanGene).value } + } + private fun removeSolvedDuplicates(duplicatedIndividuals: MutableSet>, disambiguatedIndividuals: Set>) { duplicatedIndividuals.removeAll(disambiguatedIndividuals) } diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt index 01af4f966f..5240582ce1 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt @@ -11,10 +11,11 @@ import org.slf4j.LoggerFactory class TestCaseNamingStrategyFactory( private val namingStrategy: NamingStrategy, - private val languageConventionFormatter: LanguageConventionFormatter + private val languageConventionFormatter: LanguageConventionFormatter, + private val nameWithQueryParameters: Boolean, ) { - constructor(config: EMConfig): this(config.namingStrategy, LanguageConventionFormatter(config.outputFormat)) + constructor(config: EMConfig): this(config.namingStrategy, LanguageConventionFormatter(config.outputFormat), config.nameWithQueryParameters) companion object { private val log: Logger = LoggerFactory.getLogger(TestCaseNamingStrategyFactory::class.java) @@ -31,7 +32,7 @@ class TestCaseNamingStrategyFactory( private fun actionBasedNamingStrategy(solution: Solution<*>): NumberedTestCaseNamingStrategy { val individuals = solution.individuals return when { - individuals.any { it.individual is RestIndividual } -> return RestActionTestCaseNamingStrategy(solution, languageConventionFormatter) + individuals.any { it.individual is RestIndividual } -> return RestActionTestCaseNamingStrategy(solution, languageConventionFormatter, nameWithQueryParameters) individuals.any { it.individual is GraphQLIndividual } -> return GraphQLActionTestCaseNamingStrategy(solution, languageConventionFormatter) individuals.any { it.individual is RPCIndividual } -> return RPCActionTestCaseNamingStrategy(solution, languageConventionFormatter) individuals.any { it.individual is WebIndividual } -> { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt index 3a633647a1..e1e5df5bc2 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt @@ -2,6 +2,7 @@ package org.evomaster.core.output.service import com.google.inject.Inject import org.evomaster.core.output.Lines +import org.evomaster.core.output.TestCase import org.evomaster.core.problem.graphql.GraphQLAction import org.evomaster.core.problem.graphql.GraphQLIndividual import org.evomaster.core.problem.graphql.GraphQLUtils @@ -133,4 +134,7 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { lines.append(")") } + override fun addTestCommentBlock(lines: Lines, test: TestCase) { + //TODO + } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/NoTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/NoTestCaseWriter.kt index 01517507a7..6cee877d54 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/NoTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/NoTestCaseWriter.kt @@ -1,6 +1,7 @@ package org.evomaster.core.output.service import org.evomaster.core.output.Lines +import org.evomaster.core.output.TestCase import org.evomaster.core.search.action.Action import org.evomaster.core.search.action.ActionResult import org.evomaster.core.search.EvaluatedIndividual @@ -22,4 +23,8 @@ class NoTestCaseWriter : TestCaseWriter() { override fun shouldFailIfExceptionNotThrown(result: ActionResult): Boolean { return false } + + override fun addTestCommentBlock(lines: Lines, test: TestCase) { + //TODO + } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RPCTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RPCTestCaseWriter.kt index 197bb68a01..92bbf4df67 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RPCTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RPCTestCaseWriter.kt @@ -382,4 +382,7 @@ class RPCTestCaseWriter : ApiTestCaseWriter() { } } + override fun addTestCommentBlock(lines: Lines, test: TestCase) { + //TODO + } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 525e6da578..7fc4678da7 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -4,18 +4,17 @@ import com.google.inject.Inject import org.evomaster.core.EMConfig import org.evomaster.core.output.Lines import org.evomaster.core.output.SqlWriter +import org.evomaster.core.output.TestCase import org.evomaster.core.output.TestWriterUtils import org.evomaster.core.problem.httpws.HttpWsAction import org.evomaster.core.problem.httpws.HttpWsCallResult -import org.evomaster.core.problem.rest.RestCallAction -import org.evomaster.core.problem.rest.RestCallResult -import org.evomaster.core.problem.rest.RestIndividual -import org.evomaster.core.problem.rest.RestLinkParameter +import org.evomaster.core.problem.rest.* import org.evomaster.core.problem.rest.param.BodyParam import org.evomaster.core.search.action.Action import org.evomaster.core.search.action.ActionResult import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Individual +import org.evomaster.core.search.gene.collection.EnumGene import org.evomaster.core.search.gene.utils.GeneUtils import org.evomaster.core.utils.StringUtils import org.slf4j.LoggerFactory @@ -495,4 +494,86 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { // partialOracles.addExpectations(call, lines, res, name, format) // } // } + + override fun addTestCommentBlock(lines: Lines, test: TestCase) { + + val ind = test.test + val ea = ind.evaluatedMainActions() + + //calls + if(ea.isNotEmpty()){ + lines.addBlockCommentLine("Calls:") + for(i in ea.indices){ + val x = ea[i] + val status = (x.result as RestCallResult).getStatusCode() + val id = x.action.getName() + val prefix = if(ea.size == 1) "" else "${i+1} - " + lines.addBlockCommentLine("$prefix($status) $id") + } + } + + //examples + val examples = getAllUsedExamples(ind.individual as RestIndividual) + .toSet().sorted() + if(examples.isNotEmpty()){ + if(examples.size == 1){ + lines.addBlockCommentLine("Using 1 example:") + } else { + lines.addBlockCommentLine("Using ${examples.size} examples:") + } + examples.forEach { + lines.addBlockCommentLine(" $it") + } + + /* Andrea: changed based on VW's feedback + val el = StringUtils.linesWithMaxLength(examples, ", ", config.maxLengthForCommentLine) + val opening = "Using ${examples.size} examples:" + if(el.size == 1){ + lines.addBlockCommentLine("$opening ${el[0]}") + } else { + lines.addBlockCommentLine(opening) + lines.indented { + el.forEach{lines.addBlockCommentLine(it)} + } + } + */ + } + + //links + val links = ea.mapNotNull { (it.action as RestCallAction).backwardLinkReference } + .filter { it.isInUse() } + .map { it.sourceLinkId } + if(links.isNotEmpty()){ + if(links.size == 1){ + lines.addBlockCommentLine("Followed 1 link:") + } else { + lines.addBlockCommentLine("Followed ${links.size} links:") + } + links.forEach { + lines.addBlockCommentLine(" $it") + } + + /* + val ll = StringUtils.linesWithMaxLength(links, ", ", config.maxLengthForCommentLine) + val opening = "Followed ${links.size} links:" + if(ll.size == 1){ + lines.addBlockCommentLine("$opening ${ll[0]}") + } else { + lines.addBlockCommentLine(opening) + lines.indented { + ll.forEach{lines.addBlockCommentLine(it)} + } + } + */ + } + } + + + private fun getAllUsedExamples(ind: RestIndividual) : List{ + return ind.seeFullTreeGenes() + .filterIsInstance>() + .filter { it.name == RestActionBuilderV3.EXAMPLES_NAME } + .filter { it.staticCheckIfImpactPhenotype() } + .map { it.getValueAsRawString() } + } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index 114dc29eaa..c468ebfc0e 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -59,6 +59,18 @@ abstract class TestCaseWriter { textToFile.toFile().appendText(text) } + + private fun addTestComments(lines: Lines, test: TestCase){ + + lines.startCommentBlock() + //maybe here we could have general messages... if needed + + addTestCommentBlock(lines, test) + lines.endCommentBlock() + } + + protected abstract fun addTestCommentBlock(lines: Lines, test: TestCase) + fun convertToCompilableTestCode( test: TestCase, baseUrlOfSut: String, @@ -69,10 +81,14 @@ abstract class TestCaseWriter { val lines = Lines(config.outputFormat) - if (config.testSuiteSplitType == EMConfig.TestSuiteSplitType.FAULTS - && test.test.getClusters().size != 0 - ) { - clusterComment(lines, test) +// if (config.testSuiteSplitType == EMConfig.TestSuiteSplitType.FAULTS +// && test.test.getClusters().size != 0 +// ) { +// clusterComment(lines, test) +// } + + if(config.addTestComments) { + addTestComments(lines, test) } if (format.isJUnit()) { @@ -322,6 +338,7 @@ abstract class TestCaseWriter { } + @Deprecated("dont' use") protected fun clusterComment(lines: Lines, test: TestCase) { if (test.test.clusterAssignments.size > 0) { lines.startCommentBlock() diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/WebTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/WebTestCaseWriter.kt index 339f38fe3b..c792a1e82c 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/WebTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/WebTestCaseWriter.kt @@ -1,6 +1,7 @@ package org.evomaster.core.output.service import org.evomaster.core.output.Lines +import org.evomaster.core.output.TestCase import org.evomaster.core.problem.webfrontend.* import org.evomaster.core.search.action.Action import org.evomaster.core.search.action.ActionResult @@ -86,4 +87,8 @@ class WebTestCaseWriter : TestCaseWriter() { override fun shouldFailIfExceptionNotThrown(result: ActionResult): Boolean { return false } + + override fun addTestCommentBlock(lines: Lines, test: TestCase) { + //TODO + } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/BackwardLinkReference.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/BackwardLinkReference.kt index ba8098cff2..3f0126e5ea 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/BackwardLinkReference.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/BackwardLinkReference.kt @@ -6,7 +6,12 @@ package org.evomaster.core.problem.rest * Note that several links could be defined in A. */ class BackwardLinkReference( + /** + * The id (ie type, eg "PUT:/x/{id}") of the source action from which the link is followed. + */ val sourceActionId: String, + + val sourceLinkId: String, /** @@ -19,6 +24,12 @@ class BackwardLinkReference( val statusCode = sourceLinkId.substringBefore(":").toInt() + /** + * Check if, during the execution of fitness function, the link was actually followed. + * + * WARNING: this information on test execution is stored in "action" and not "result". + * FIXME: should be part of "result". + */ fun isInUse() = actualSourceActionLocalId != null fun copy() = BackwardLinkReference(sourceActionId, sourceLinkId, actualSourceActionLocalId) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/RestActionBuilderV3.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/RestActionBuilderV3.kt index e17c82df2d..776143927d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/RestActionBuilderV3.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/RestActionBuilderV3.kt @@ -61,6 +61,11 @@ object RestActionBuilderV3 { private val log: Logger = LoggerFactory.getLogger(RestActionBuilderV3::class.java) + /** + * Name given to enum genes representing data examples coming from OpenAPI schema + */ + const val EXAMPLES_NAME = "SCHEMA_EXAMPLES" + private val refCache = mutableMapOf() /** @@ -1376,12 +1381,12 @@ object RestActionBuilderV3 { val exampleGene = if(examples.isNotEmpty()){ when{ NumberGene::class.java.isAssignableFrom(geneClass) - -> EnumGene("examples", examples,0,true) + -> EnumGene(EXAMPLES_NAME, examples,0,true) geneClass == StringGene::class.java || geneClass == Base64StringGene::class.java || geneClass == RegexGene::class.java - -> EnumGene("examples", examples,0,false) + -> EnumGene(EXAMPLES_NAME, examples,0,false) //TODO Arrays else -> { diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/AbstractRestSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/AbstractRestSampler.kt index 70c3508350..88556cfeb4 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/AbstractRestSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/AbstractRestSampler.kt @@ -91,7 +91,9 @@ abstract class AbstractRestSampler : HttpWsSampler() { // set up authentications moved up since we are going to get authentication info from HttpWsSampler setupAuthentication(infoDto) - if(!openApiURL.isNullOrBlank()) { + if(!config.overrideOpenAPIUrl.isNullOrBlank()){ + retrieveSwagger(config.overrideOpenAPIUrl) + }else if(!openApiURL.isNullOrBlank()) { retrieveSwagger(openApiURL) } else if(! openApiSchema.isNullOrBlank()){ swagger = OpenApiAccess.getOpenApi(openApiSchema) diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt index b7b357cc8c..e19726d391 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt @@ -118,7 +118,7 @@ abstract class Sampler : TrackOperator where T : Individual { val force = randomness.nextBoolean(config.probabilityAllOptionalsAreOnOrOff) if(force){ - val on = randomness.nextBoolean(0.8) + val on = randomness.nextBoolean(config.probabilityOfOnVsOffInAllOptionals) allOptionals .filter { it.selectable } .forEach { it.isActive = on } diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index adcd055d60..61c606dcff 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -1005,6 +1005,8 @@ class SqlInsertBuilder( } private fun formatNameInSql(name: String): String { + + //TODO: why this??? needs explanation return when { databaseType == DatabaseType.MYSQL || name == SQLKey.ALL.key -> name else -> "\"$name\"" diff --git a/core/src/main/kotlin/org/evomaster/core/utils/StringUtils.kt b/core/src/main/kotlin/org/evomaster/core/utils/StringUtils.kt index 00cea2f5be..fa03b8991f 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/StringUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/StringUtils.kt @@ -27,4 +27,35 @@ object StringUtils { val parts = fullyQualifiedName.replace('$', '.').split(".") return parts.last() } + + + /** + * Given a list of tokens, and a separator, concatenate them. + * however, if such concatenation is longer than [maxLength], split in different lines. + */ + fun linesWithMaxLength(tokens: List, separator: String, maxLength: Int) : List{ + + val lines = mutableListOf() + val buffer = StringBuffer() + for(t in tokens){ + if(buffer.isEmpty()){ + buffer.append(t) + continue + } + val len = buffer.length + separator.length + t.length + if(len <= maxLength){ + buffer.append(separator) + buffer.append(t) + } else { + lines.add(buffer.toString()) + buffer.delete(0, buffer.length) + buffer.append(separator) + buffer.append(t) + } + } + if(buffer.isNotEmpty()){ + lines.add(buffer.toString()) + } + return lines + } } diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 61e9d17522..2018c4926c 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -582,7 +582,7 @@ internal class EMConfigTest{ val parser = EMConfig.getOptionParser() val config = EMConfig() - var options = parser.parse("--outputFormat", "PYTHON_UNITTEST") + val options = parser.parse("--outputFormat", "PYTHON_UNITTEST") assertThrows(Exception::class.java, {config.updateProperties(options)}) } @@ -607,4 +607,38 @@ internal class EMConfigTest{ assertEquals(NamingStrategy.NUMBERED, config.namingStrategy) } + @Test + fun testQueryParamsInTestCaseNamesValidForActionStrategy() { + val parser = EMConfig.getOptionParser() + val config = EMConfig() + + val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "true") + config.updateProperties(options) + + assertEquals(NamingStrategy.ACTION, config.namingStrategy) + assertTrue(config.nameWithQueryParameters) + } + + @Test + fun testQueryParamsInTestCaseNamesFalseTurnsFeatureOff() { + val parser = EMConfig.getOptionParser() + val config = EMConfig() + + val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "false") + config.updateProperties(options) + + assertEquals(NamingStrategy.ACTION, config.namingStrategy) + assertFalse(config.nameWithQueryParameters) + } + + @Test + fun testQueryParamsInTestCaseNamesIsOffByDefault() { + val parser = EMConfig.getOptionParser() + val config = EMConfig() + + config.updateProperties(parser.parse()) + + assertFalse(config.nameWithQueryParameters) + } + } diff --git a/core/src/test/kotlin/org/evomaster/core/output/naming/RestActionNamingStrategyTest.kt b/core/src/test/kotlin/org/evomaster/core/output/naming/RestActionNamingStrategyTest.kt index fee039d4ae..d3b396c6d9 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/naming/RestActionNamingStrategyTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/naming/RestActionNamingStrategyTest.kt @@ -20,6 +20,7 @@ open class RestActionNamingStrategyTest { companion object { val pythonFormatter = LanguageConventionFormatter(OutputFormat.PYTHON_UNITTEST) val javaFormatter = LanguageConventionFormatter(OutputFormat.JAVA_JUNIT_4) + const val NO_QUERY_PARAMS_IN_NAME = false } @Test @@ -29,7 +30,7 @@ open class RestActionNamingStrategyTest { val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -42,7 +43,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -58,7 +59,7 @@ open class RestActionNamingStrategyTest { val itemsIndividual = getEvaluatedIndividualWith(itemsAction) val solution = Solution(mutableListOf(rootIndividual, itemsIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(2, testCases.size) @@ -72,7 +73,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -85,7 +86,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, 200, "", MediaType.TEXT_XML_TYPE) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -98,7 +99,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, 200, "[]", MediaType.APPLICATION_JSON_TYPE) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -111,7 +112,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, 200, "[1]", MediaType.APPLICATION_JSON_TYPE) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -124,7 +125,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, 200, "[1,2,3]", MediaType.APPLICATION_JSON_TYPE) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -137,7 +138,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, 200, "{}", MediaType.APPLICATION_JSON_TYPE) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -150,7 +151,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, 200, "{\"key\": \"value\"}", MediaType.APPLICATION_JSON_TYPE) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -163,7 +164,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, 200, "\"myItem\"", MediaType.APPLICATION_JSON_TYPE) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -176,7 +177,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, 401, "[]", MediaType.APPLICATION_JSON_TYPE) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -189,7 +190,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWithFaults(restAction, singletonList(DetectedFault(FaultCategory.HTTP_STATUS_500, "items")), 500) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -203,7 +204,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWithFaults(restAction, faults, 500) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, pythonFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -216,7 +217,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, true) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -229,7 +230,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, withMongo = true) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -242,7 +243,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, 201, withWireMock = true) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) @@ -255,7 +256,7 @@ open class RestActionNamingStrategyTest { val eIndividual = getEvaluatedIndividualWith(restAction, withSql = true, withWireMock = true) val solution = Solution(singletonList(eIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(1, testCases.size) diff --git a/core/src/test/kotlin/org/evomaster/core/output/naming/TestCaseDisambiguationTest.kt b/core/src/test/kotlin/org/evomaster/core/output/naming/TestCaseDisambiguationTest.kt index c459fb9e03..0e1b89f36e 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/naming/TestCaseDisambiguationTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/naming/TestCaseDisambiguationTest.kt @@ -6,9 +6,13 @@ import org.evomaster.core.output.naming.RestActionTestCaseUtils.getEvaluatedIndi import org.evomaster.core.output.naming.RestActionTestCaseUtils.getRestCallAction import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.rest.HttpVerb +import org.evomaster.core.problem.rest.RestCallAction +import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.problem.rest.param.PathParam import org.evomaster.core.problem.rest.param.QueryParam +import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Solution +import org.evomaster.core.search.gene.BooleanGene import org.evomaster.core.search.gene.optional.CustomMutationRateGene import org.evomaster.core.search.gene.string.StringGene import org.junit.jupiter.api.Assertions.assertEquals @@ -19,6 +23,8 @@ class TestCaseDisambiguationTest { companion object { val javaFormatter = LanguageConventionFormatter(OutputFormat.JAVA_JUNIT_4) + const val NO_QUERY_PARAMS_IN_NAME = false + const val QUERY_PARAMS_IN_NAME = true } @Test @@ -28,7 +34,7 @@ class TestCaseDisambiguationTest { val solution = Solution(mutableListOf(funnyPathIndividual, funniestPathIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(2, testCases.size) @@ -45,7 +51,7 @@ class TestCaseDisambiguationTest { val solution = Solution(mutableListOf(languagesIndividual, statisticsLanguagesIndividual, syntaxLanguagesIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(3, testCases.size) @@ -63,7 +69,7 @@ class TestCaseDisambiguationTest { val solution = Solution(mutableListOf(languagesIndividual, syntaxLanguagesIndividual, syntaxLanguagesIndividual2), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(3, testCases.size) @@ -84,7 +90,7 @@ class TestCaseDisambiguationTest { val solution = Solution(mutableListOf(configurationFeatureIndividual, productFeatureIndividual), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(2, testCases.size) @@ -95,11 +101,11 @@ class TestCaseDisambiguationTest { @Test fun pathWithQueryParamDisambiguation() { val syntaxLanguagesIndividual = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages")) - val syntaxLanguagesIndividualWithQP = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages", parameters = singletonList(getQueryParam("myQueryParam")))) + val syntaxLanguagesIndividualWithQP = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages", parameters = singletonList(getStringQueryParam("myQueryParam")))) val solution = Solution(mutableListOf(syntaxLanguagesIndividual, syntaxLanguagesIndividualWithQP), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(2, testCases.size) @@ -110,11 +116,11 @@ class TestCaseDisambiguationTest { @Test fun pathWithMoreThanOneQueryParamDisambiguation() { val syntaxLanguagesIndividual = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages")) - val syntaxLanguagesIndividualWithQP = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages", parameters = mutableListOf(getQueryParam("myQueryParam"), getQueryParam("myOtherQueryParam")))) + val syntaxLanguagesIndividualWithQP = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages", parameters = mutableListOf(getStringQueryParam("myQueryParam"), getStringQueryParam("myOtherQueryParam")))) val solution = Solution(mutableListOf(syntaxLanguagesIndividual, syntaxLanguagesIndividualWithQP), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(2, testCases.size) @@ -126,11 +132,11 @@ class TestCaseDisambiguationTest { fun rootPathAndQueryParamDisambiguationReturnsThreeDifferentNames() { val languagesIndividual = getEvaluatedIndividualWith(getRestCallAction("/languages")) val syntaxLanguagesIndividual = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages")) - val syntaxLanguagesIndividualWithQP = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages", parameters = singletonList(getQueryParam("myQueryParam")))) + val syntaxLanguagesIndividualWithQP = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages", parameters = singletonList(getStringQueryParam("myQueryParam")))) val solution = Solution(mutableListOf(languagesIndividual, syntaxLanguagesIndividual, syntaxLanguagesIndividualWithQP), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) - val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter) + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, NO_QUERY_PARAMS_IN_NAME) val testCases = namingStrategy.getTestCases() assertEquals(3, testCases.size) @@ -139,12 +145,90 @@ class TestCaseDisambiguationTest { assertEquals("test_2_getOnLanguagesWithQueryParamReturnsEmpty", testCases[2].name) } + @Test + fun noQueryParamsAddedWhenNoQueryParamsInIndividual() { + val syntaxLanguagesIndividual = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages")) + val syntaxLanguagesIndividual2 = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages")) + + val solution = Solution(mutableListOf(syntaxLanguagesIndividual, syntaxLanguagesIndividual2), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) + + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, QUERY_PARAMS_IN_NAME) + + val testCases = namingStrategy.getTestCases() + assertEquals(2, testCases.size) + assertEquals("test_0_getOnLanguagesReturnsEmpty", testCases[0].name) + assertEquals("test_1_getOnLanguagesReturnsEmpty", testCases[1].name) + } + + @Test + fun oneTrueBooleanQueryParamIsAdded() { + val syntaxLanguagesIndividual = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages")) + val syntaxLanguagesIndividual2 = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages", parameters = singletonList(getBooleanQueryParam("myQueryParam")))) + ensureBooleanGeneValue(syntaxLanguagesIndividual2, "myQueryParam", "true") + + val solution = Solution(mutableListOf(syntaxLanguagesIndividual, syntaxLanguagesIndividual2), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) + + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, QUERY_PARAMS_IN_NAME) + + val testCases = namingStrategy.getTestCases() + assertEquals(2, testCases.size) + assertEquals("test_0_getOnSyntaxLanguagesReturnsEmpty", testCases[0].name) + assertEquals("test_1_getOnLanguagesWithQueryParamMyQueryParamReturnsEmpty", testCases[1].name) + } + + @Test + fun oneFalseBooleanQueryParamIsNotAdded() { + val syntaxLanguagesIndividual = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages")) + val syntaxLanguagesIndividual2 = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages", parameters = singletonList(getBooleanQueryParam("myQueryParam")))) + ensureBooleanGeneValue(syntaxLanguagesIndividual2, "myQueryParam", "false") + + val solution = Solution(mutableListOf(syntaxLanguagesIndividual, syntaxLanguagesIndividual2), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) + + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, QUERY_PARAMS_IN_NAME) + + val testCases = namingStrategy.getTestCases() + assertEquals(2, testCases.size) + assertEquals("test_0_getOnSyntaxLanguagesReturnsEmpty", testCases[0].name) + assertEquals("test_1_getOnLanguagesWithQueryParamReturnsEmpty", testCases[1].name) + } + + @Test + fun onlyTrueBooleanQueryParamsAreAdded() { + val syntaxLanguagesIndividual = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages")) + val syntaxLanguagesIndividual2 = getEvaluatedIndividualWith(getRestCallAction("/syntax/languages", parameters = mutableListOf(getBooleanQueryParam("firstParam"), getBooleanQueryParam("secondParam"), getStringQueryParam("thirdParam"), getBooleanQueryParam("fourthParam")))) + ensureBooleanGeneValue(syntaxLanguagesIndividual2, "firstParam", "true") + ensureBooleanGeneValue(syntaxLanguagesIndividual2, "secondParam", "false") + ensureBooleanGeneValue(syntaxLanguagesIndividual2, "fourthParam", "true") + + val solution = Solution(mutableListOf(syntaxLanguagesIndividual, syntaxLanguagesIndividual2), "suitePrefix", "suiteSuffix", Termination.NONE, emptyList(), emptyList()) + + val namingStrategy = RestActionTestCaseNamingStrategy(solution, javaFormatter, QUERY_PARAMS_IN_NAME) + + val testCases = namingStrategy.getTestCases() + assertEquals(2, testCases.size) + assertEquals("test_0_getOnSyntaxLanguagesReturnsEmpty", testCases[0].name) + assertEquals("test_1_getOnLanguagesWithQueryParamsFirstParamAndFourthParamReturnsEmpty", testCases[1].name) + } + private fun getPathParam(paramName: String): Param { return PathParam(paramName, CustomMutationRateGene(paramName, StringGene(paramName), 1.0)) } - private fun getQueryParam(paramName: String): Param { - return QueryParam(paramName, CustomMutationRateGene(paramName, StringGene(paramName), 1.0)) + private fun getStringQueryParam(paramName: String): Param { + return QueryParam(paramName, StringGene(paramName)) + } + + private fun getBooleanQueryParam(paramName: String): Param { + return QueryParam(paramName, BooleanGene(paramName)) + } + + /* + Since the randomization used to construct the evaluated individuals might set a random boolean value, + we do this to ensure the one we want for unit testing + */ + private fun ensureBooleanGeneValue(evaluatedIndividual: EvaluatedIndividual, paramName: String, paramValue: String) { + val restCallAction = evaluatedIndividual.evaluatedMainActions().last().action as RestCallAction + (restCallAction.parameters.filter { it.name == paramName }).forEach { (it as QueryParam).getGeneForQuery().setFromStringValue(paramValue) } } } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/multidb/MultiDbUtils.kt b/core/src/test/kotlin/org/evomaster/core/sql/multidb/MultiDbUtils.kt index 0168351c20..b927a363cd 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/multidb/MultiDbUtils.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/multidb/MultiDbUtils.kt @@ -28,7 +28,6 @@ object MultiDbUtils { .withEnv( mutableMapOf( "MYSQL_ROOT_PASSWORD" to "root", - //"MYSQL_DATABASE" to MYSQL_DB_NAME, // TODO can this be removed? "MYSQL_USER" to "test", "MYSQL_PASSWORD" to "test" ) @@ -37,22 +36,31 @@ object MultiDbUtils { .withExposedPorts(MYSQL_PORT) - private fun getMySQLBaseUrl() : String{ + fun getMySQLBaseUrl() : String{ val host = mysql.host val port = mysql.getMappedPort(MYSQL_PORT) return "jdbc:mysql://$host:$port/" } - private fun getPostgresBaseUrl() : String{ + fun getPostgresBaseUrl() : String{ val host = postgres.host val port = postgres.getMappedPort(POSTGRES_PORT) return "jdbc:postgresql://$host:$port/" } - private fun getH2BaseUrl() : String{ + fun getH2BaseUrl() : String{ return "jdbc:h2:mem:" } + fun getBaseUrl(type: DatabaseType) : String{ + return when(type){ + DatabaseType.POSTGRES -> getPostgresBaseUrl() + DatabaseType.MYSQL -> getMySQLBaseUrl() + DatabaseType.H2 -> getH2BaseUrl() + else -> throw IllegalArgumentException("Unsupported database type: $type") + } + } + fun createConnection(name: String, type: DatabaseType) : Connection { return when (type) { @@ -99,7 +107,16 @@ object MultiDbUtils { when(type) { DatabaseType.H2 -> { /* nothing to do? started automatically on connection*/} DatabaseType.MYSQL -> mysql.start() - DatabaseType.POSTGRES -> postgres.start() + DatabaseType.POSTGRES -> { + postgres.start() + /* + * A call to getConnection() when the postgres container is still not ready, + * signals a PSQLException with message "FATAL: the database system is starting up". + * The following issue describes how to avoid this by using a LogMessageWaitStrategy + * https://github.com/testcontainers/testcontainers-java/issues/317 + */ + postgres.waitingFor(LogMessageWaitStrategy().withRegEx(".*database system is ready to accept connections.*\\s").withTimes(2)) + } else -> throw IllegalArgumentException("Unsupported database type: ${type.name}") } } @@ -110,15 +127,13 @@ object MultiDbUtils { DatabaseType.MYSQL -> mysql.stop() DatabaseType.POSTGRES -> { postgres.stop() - /* - * A call to getConnection() when the postgres container is still not ready, - * signals a PSQLException with message "FATAL: the database system is starting up". - * The following issue describes how to avoid this by using a LogMessageWaitStrategy - * https://github.com/testcontainers/testcontainers-java/issues/317 - */ - postgres.waitingFor(LogMessageWaitStrategy().withRegEx(".*database system is ready to accept connections.*\\s").withTimes(2)) } else -> throw IllegalArgumentException("Unsupported database type: ${type.name}") } } + + fun stopAllDatabases() { + mysql.stop() + postgres.stop() + } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/utils/StringUtilsTest.kt b/core/src/test/kotlin/org/evomaster/core/utils/StringUtilsTest.kt new file mode 100644 index 0000000000..6575a6f981 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/utils/StringUtilsTest.kt @@ -0,0 +1,22 @@ +package org.evomaster.core.utils + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +class StringUtilsTest{ + + + @Test + fun testLinesWithMaxLength(){ + + val tokens = listOf("Foo","A","B","Hello","C","D") + val separator = ", " + + val lines = StringUtils.linesWithMaxLength(tokens,separator, 6) + assertEquals(4, lines.size) + assertEquals("Foo, A",lines[0]) + assertEquals(", B",lines[1]) + assertEquals(", Hello",lines[2]) + assertEquals(", C, D",lines[3]) + } +} \ No newline at end of file diff --git a/docs/config_file.md b/docs/config_file.md new file mode 100644 index 0000000000..3c6caa82c5 --- /dev/null +++ b/docs/config_file.md @@ -0,0 +1,3 @@ +## Config + +Documentation under construction \ No newline at end of file diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000000..db31256883 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,102 @@ +# Running EvoMaster in Docker + +_EvoMaster_ is also released via Docker, under `webfuzzing/evomaster`. + +It can be run with: + +> docker run webfuzzing/evomaster \ + +Where `` would be the [command-line options](options.md), +like for example `--blackBox true` + +> __IMPORTANT__ +> +> For __white-box__ testing, running from Docker is currently not supported. As such, it is discouraged. +> You can try it, but likely you will get major issues with hostname and ephemeral port resolutions, as well as issues with external dependency mocks. + + +## Accessing the generated test files + +By default, _EvoMaster_ generates tests under the `/generated_tests` folder. +To access the generated tests after Docker run is completed, you need to mount a folder or a volume pointing to that folder. +An example is mounting the local `./generated_tests`, using for example: + +> docker run -v "/$(pwd)/generated_tests":/generated_tests webfuzzing/evomaster \ + +There can be other ways to access the `/generated_tests` folder inside the Docker image (e.g., using volumes), but, for that, you will need to check the Docker documentation. + +WARNING: note the `/` before `$(pwd)`. This is needed only if running in a MSYS shell on Windows like _Git Bash_, as it automatically converts paths on the command line. +Unfortunately, this will also mess up any input parameter to EvoMaster that is recognized as a path (e.g., if starting with a `/`). +You can [read more here](https://stackoverflow.com/questions/52944738/how-to-force-git-for-windows-bash-shell-to-not-convert-path-string-to-windows-p). + +## Issues with "localhost" + +An important fact to keep in mind is that references to `localhost` will point to the Docker virtual network, and not your host machine. +This latter can be accessed at `host.docker.internal`. + + +For example, in __black-box__ testing for REST APIs, if the tested API is running on your machine, an option like + +> --bbSwaggerUrl http://localhost:8080/v3/api-docs + +would need to be replaced with + +> --bbSwaggerUrl http://host.docker.internal:8080/v3/api-docs + +Note that here the port `8080` and the the path `/v3/api-docs` are just examples. + + + + + + + + + +## Handling "em.yaml" configuration file + +If you want to use a configuration file (e.g., `em.yaml`) to specify command-line options, this needs to be mounted into Docker. +For example by using: + +> -v "$(pwd)/em.yaml":/em.yaml + +By default, _EvoMaster_ in Docker searches for a configuration file under `/em.yaml`. +You can mount your configuration file (based on its absolute path) into that. + +> WARNING: in _Git Bash_, you will need a `/` before `$(pwd)`, i.e., Linux-like paths should start with a `//` and not a `/` (as otherwise get converted, and you might end up seeing weird things like prepended paths such as `C:/Program Files/Git`). Also, options like +`--configPath` will not work there. + + +By default, if there is no `em.yaml` file, _EvoMaster_ will create one in the current working directory, with comments on how it should be configured. +However, if you run through Docker, you will not be able to directly access such file, unless you mount the whole `/` root folder (which is not recommended). +But you can still create it manually. +[See documentation](config_file.md). \ No newline at end of file diff --git a/docs/options.md b/docs/options.md index c724fb7559..e9f777daf6 100644 --- a/docs/options.md +++ b/docs/options.md @@ -51,6 +51,9 @@ There are 3 types of options: |`endpointFocus`| __String__. Concentrate search on only one single REST endpoint. *Default value*: `null`.| |`endpointPrefix`| __String__. Concentrate search on a set of REST endpoints defined by a common prefix. *Default value*: `null`.| |`endpointTagFilter`| __String__. Comma-separated list of OpenAPI/Swagger 'tags' definitions. Only the REST endpoints having at least one of such tags will be fuzzed. If no tag is specified here, then such filter is not applied. *Default value*: `null`.| +|`sutControllerHost`| __String__. Host name or IP address of where the SUT EvoMaster Controller Driver is listening on. This option is only needed for white-box testing. *Default value*: `localhost`.| +|`sutControllerPort`| __Int__. TCP port of where the SUT EvoMaster Controller Driver is listening on. This option is only needed for white-box testing. *Constraints*: `min=0.0, max=65535.0`. *Default value*: `40100`.| +|`overrideOpenAPIUrl`| __String__. If specified, override the OpenAPI URL location given by the EvoMaster Driver. This option is only needed for white-box testing. *Constraints*: `URL`. *Default value*: `""`.| ## Internal Command-Line Options @@ -137,6 +140,7 @@ There are 3 types of options: |`lastLineEpsilon`| __Double__. The Distance Metric Last Line may use several values for epsilon.During experimentation, it may be useful to adjust these values. Epsilon describes the size of the neighbourhood used for clustering, so may result in different clustering results.Epsilon should be between 0.0 and 1.0. If the value is outside of that range, epsilon will use the default of 0.8. *Constraints*: `min=0.0, max=1.0`. *Default value*: `0.8`.| |`maxAssertionForDataInCollection`| __Int__. Specify a maximum number of data in a collection to be asserted in the generated tests. Note that zero means that only the size of the collection will be asserted. A negative value means all data in the collection will be asserted (i.e., no limit). *Default value*: `3`.| |`maxEvaluations`| __Int__. Maximum number of action or individual evaluations (depending on chosen stopping criterion) for the search. A fitness evaluation can be composed of 1 or more actions, like for example REST calls or SQL setups. The more actions are allowed, the better results one can expect. But then of course the test generation will take longer. Only applicable depending on the stopping criterion. *Constraints*: `min=1.0`. *Default value*: `1000`.| +|`maxLengthForCommentLine`| __Int__. Max length for test comments. Needed when enumerating some names/values, making comments too long to be on a single line. *Constraints*: `min=1.0`. *Default value*: `80`.| |`maxLengthForStrings`| __Int__. The maximum length allowed for evolved strings. Without this limit, strings could in theory be billions of characters long. *Constraints*: `min=0.0, max=20000.0`. *Default value*: `200`.| |`maxLengthForStringsAtSamplingTime`| __Int__. Maximum length when sampling a new random string. Such limit can be bypassed when a string is mutated. *Constraints*: `min=0.0`. *Default value*: `16`.| |`maxLengthOfTraces`| __Int__. Specify a maxLength of tracking when enableTrackIndividual or enableTrackEvaluatedIndividual is true. Note that the value should be specified with a non-negative number or -1 (for tracking all history). *Constraints*: `min=-1.0`. *Default value*: `10`.| @@ -173,6 +177,7 @@ There are 3 types of options: |`recordExceededTargets`| __Boolean__. Whether to record targets when the number is more than 100. *DEBUG option*. *Default value*: `false`.| |`recordExecutedMainActionInfo`| __Boolean__. Whether to record info of executed actions during search. *DEBUG option*. *Default value*: `false`.| |`resourceSampleStrategy`| __Enum__. Specify whether to enable resource-based strategy to sample an individual during search. Note that resource-based sampling is only applicable for REST problem with MIO algorithm. *Valid values*: `NONE, Customized, EqualProbability, Actions, TimeBudgets, Archive, ConArchive`. *Default value*: `ConArchive`.| +|`runningInDocker`| __Boolean__. Inform EvoMaster process that it is running inside Docker. Users should not modify this parameter, as it is set automatically in the Docker image of EvoMaster. *Default value*: `false`.| |`saveArchiveAfterMutation`| __Boolean__. Whether to save archive info after each of mutation, which is typically useful for debugging mutation and archive. *DEBUG option*. *Default value*: `false`.| |`saveExecutedMainActionInfo`| __String__. Specify a path to save all executed main actions to a file (default is 'executedMainActions.txt'). *DEBUG option*. *Default value*: `executedMainActions.txt`.| |`saveExecutedSQLToFile`| __String__. Specify a path to save all executed sql commands to a file (default is 'sql.txt'). *DEBUG option*. *Default value*: `sql.txt`.| @@ -193,8 +198,6 @@ There are 3 types of options: |`statisticsFile`| __String__. Where the statistics file (if any) is going to be written (in CSV format). *Default value*: `statistics.csv`.| |`stoppingCriterion`| __Enum__. Stopping criterion for the search. *Valid values*: `TIME, ACTION_EVALUATIONS, INDIVIDUAL_EVALUATIONS`. *Default value*: `TIME`.| |`structureMutationProbability`| __Double__. Probability of applying a mutation that can change the structure of a test. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| -|`sutControllerHost`| __String__. Host name or IP address of where the SUT REST controller is listening on. *Default value*: `localhost`.| -|`sutControllerPort`| __Int__. TCP port of where the SUT REST controller is listening on. *Constraints*: `min=0.0, max=65535.0`. *Default value*: `40100`.| |`taintAnalysisForMapsAndArrays`| __Boolean__. Apply taint analysis to handle special cases of Maps and Arrays. *Default value*: `true`.| |`taintApplySpecializationProbability`| __Double__. Probability of applying a discovered specialization for a tainted value. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`taintChangeSpecializationProbability`| __Double__. Probability of changing specialization for a resolved taint during mutation. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.1`.| @@ -220,9 +223,11 @@ There are 3 types of options: |Options|Description| |---|---| |`abstractInitializationGeneToMutate`| __Boolean__. During mutation, whether to abstract genes for repeated SQL actions. *Default value*: `false`.| +|`addTestComments`| __Boolean__. Add summary comments on each test. *Default value*: `false`.| |`appendToTargetHeuristicsFile`| __Boolean__. Whether should add to an existing target heuristics file, instead of replacing it. It is only used when processFormat is TARGET_HEURISTIC. *Default value*: `false`.| |`bbProbabilityUseDataPool`| __Double__. Specify the probability of using the data pool when sampling test cases. This is for black-box (bb) mode. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.| |`discoveredInfoRewardedInFitness`| __Boolean__. If there is new discovered information from a test execution, reward it in the fitness function. *Default value*: `false`.| +|`dockerLocalhost`| __Boolean__. Replace references to 'localhost' to point to the actual host machine. Only needed when running EvoMaster inside Docker. *Default value*: `false`.| |`dpcTargetTestSize`| __Int__. Specify a max size of a test to be targeted when either DPC_INCREASING or DPC_DECREASING is enabled. *Default value*: `1`.| |`employResourceSizeHandlingStrategy`| __Enum__. Specify a strategy to determinate a number of resources to be manipulated throughout the search. *Valid values*: `NONE, RANDOM, DPC`. *Default value*: `NONE`.| |`enableAdaptiveResourceStructureMutation`| __Boolean__. Specify whether to decide the resource-based structure mutator and resource to be mutated adaptively based on impacts during focused search.Note that it only works when resource-based solution is enabled for solving REST problem. *Default value*: `false`.| @@ -249,6 +254,7 @@ There are 3 types of options: |`maxTestSizeStrategy`| __Enum__. Specify a strategy to handle a max size of a test. *Valid values*: `SPECIFIED, DPC_INCREASING, DPC_DECREASING`. *Default value*: `SPECIFIED`.| |`maxTestsPerTestSuite`| __Int__. Specify the maximum number of tests to be generated in one test suite. Note that a negative number presents no limit per test suite. *Default value*: `-1`.| |`mutationTargetsSelectionStrategy`| __Enum__. Specify a strategy to select targets for evaluating mutation. *Valid values*: `FIRST_NOT_COVERED_TARGET, EXPANDED_UPDATED_NOT_COVERED_TARGET, UPDATED_NOT_COVERED_TARGET`. *Default value*: `FIRST_NOT_COVERED_TARGET`.| +|`nameWithQueryParameters`| __Boolean__. Specify if true boolean query parameters are included in the test case name. Used for test case naming disambiguation. Only valid for Action based naming strategy. *Default value*: `false`.| |`prematureStopStrategy`| __Enum__. Specify how 'improvement' is defined: either any kind of improvement even if partial (ANY), or at least one new target is fully covered (NEW). *Valid values*: `ANY, NEW`. *Default value*: `NEW`.| |`probOfHandlingLength`| __Double__. Specify a probability of applying length handling. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| |`probOfHarvestingResponsesFromActualExternalServices`| __Double__. a probability of harvesting actual responses from external services as seeds. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| diff --git a/docs/publications.md b/docs/publications.md index e1f8f6e2ab..1c0584be5f 100644 --- a/docs/publications.md +++ b/docs/publications.md @@ -23,18 +23,29 @@ Also, some of these papers provides full replication packages, which are linked ## Peer-Reviewed Publications +### 2025 +* A. Arcuri, A. Poth, O. Rrjolli. + *Introducing Black-Box Fuzz Testing for REST APIs in Industry: Challenges and Solutions*. + IEEE International Conference on Software Testing, Validation and Verification (ICST), Industry Track. + [[PDF](publications/2025_icst.pdf)] + +* S. Seran, O. Duman, A. Arcuri. + *Multi-Phase Taint Analysis for JSON Inference in Search-Based Fuzzing*. + IEEE International Workshop on Search-Based and Fuzz Testing (SBFT). + *(To appear)* + ### 2024 * A. Arcuri, M. Zhang, S. Seran, J.P. Galeotti, A. Golmohammadi, O. Duman, A. Aldasoro, H. Ghianni. - *Tool Report: EvoMaster. Black and White Box Search-Based Fuzzing for REST, GraphQL and RPC APIs*. + *Tool Report: EvoMaster -- Black and White Box Search-Based Fuzzing for REST, GraphQL and RPC APIs*. Automated Software Engineering (AUSE). - (To appear) + [[PDF](publications/2024_ause_em.pdf)] * A. Poth, O. Rrjolli, A. Arcuri. - *Industrial Paper: Technology Adoption Performance Evaluation Applied to Testing Industrial REST APIs*. + *Technology Adoption Performance Evaluation Applied to Testing Industrial REST APIs*. Automated Software Engineering (AUSE). - (To appear) + [[PDF](publications/2024_ause_vw.pdf)] * M. Zhang, A. Arcuri, P. Teng, K. Xue, W. Wang. diff --git a/docs/publications/2024_ause_em.pdf b/docs/publications/2024_ause_em.pdf new file mode 100644 index 0000000000..d6aee6adab Binary files /dev/null and b/docs/publications/2024_ause_em.pdf differ diff --git a/docs/publications/2024_ause_vw.pdf b/docs/publications/2024_ause_vw.pdf new file mode 100644 index 0000000000..042cccced0 Binary files /dev/null and b/docs/publications/2024_ause_vw.pdf differ diff --git a/docs/publications/2025_icst.pdf b/docs/publications/2025_icst.pdf new file mode 100644 index 0000000000..2a5f414f55 Binary files /dev/null and b/docs/publications/2025_icst.pdf differ diff --git a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java index 1a92f3be41..1aeb6a8d1b 100644 --- a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java +++ b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java @@ -111,13 +111,14 @@ public void initTest() { StaticCounter.Companion.reset(); - assertTimeoutPreemptively(Duration.ofMinutes(2), () -> { - boolean reset = remoteController.resetSUT(); - assertTrue(reset); - }); + if(remoteController != null) { + assertTimeoutPreemptively(Duration.ofMinutes(2), () -> { + boolean reset = remoteController.resetSUT(); + assertTrue(reset); + }); + } SimpleLogger.setThreshold(SimpleLogger.Level.DEBUG); - } @AfterEach diff --git a/e2e-tests/pom.xml b/e2e-tests/pom.xml index e1c7526539..a3fd74ca46 100644 --- a/e2e-tests/pom.xml +++ b/e2e-tests/pom.xml @@ -32,6 +32,7 @@ spring-rest-bb spring-graphql-bb emb-json + spring-rest-multidb diff --git a/e2e-tests/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/examplevalues/BBExamplesEMTest.kt b/e2e-tests/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/examplevalues/BBExamplesEMTest.kt index 8b8ddf8488..592f3f2381 100644 --- a/e2e-tests/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/examplevalues/BBExamplesEMTest.kt +++ b/e2e-tests/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/examplevalues/BBExamplesEMTest.kt @@ -36,7 +36,7 @@ class BBExamplesEMTest : SpringTestBase() { setOption(args, "bbSwaggerUrl", "$baseUrlOfSut/openapi-bbexamples.json") setOption(args, "probRestDefault", "0.45") setOption(args, "probRestExamples","0.45") - + setOption(args, "addTestComments", "true") val solution = initAndRun(args) diff --git a/e2e-tests/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/linksmulti/BBLinksMultiEMTest.kt b/e2e-tests/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/linksmulti/BBLinksMultiEMTest.kt index 931e357c16..83f7833ec4 100644 --- a/e2e-tests/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/linksmulti/BBLinksMultiEMTest.kt +++ b/e2e-tests/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/linksmulti/BBLinksMultiEMTest.kt @@ -36,6 +36,7 @@ class BBLinksMultiEMTest : SpringTestBase() { setOption(args, "algorithm", "SMARTS") setOption(args, "probUseRestLinks", "0.9") setOption(args, "advancedBlackBoxCoverage", "true") + setOption(args, "addTestComments", "true") val solution = initAndRun(args) diff --git a/e2e-tests/spring-rest-multidb/pom.xml b/e2e-tests/spring-rest-multidb/pom.xml new file mode 100644 index 0000000000..16bd7c983a --- /dev/null +++ b/e2e-tests/spring-rest-multidb/pom.xml @@ -0,0 +1,211 @@ + + 4.0.0 + + + org.evomaster + evomaster-e2e-tests + 3.3.1-SNAPSHOT + + + evomaster-e2e-tests-spring-rest-multidb + jar + + + + + + + javax.validation + validation-api + 2.0.1.Final + + + + + javax.ws.rs + javax.ws.rs-api + + + + + org.evomaster + evomaster-core + test-jar + ${project.version} + + + org.evomaster + evomaster-e2e-tests-utils + test-jar + + + org.evomaster + evomaster-client-java-controller + + + org.evomaster + evomaster-core + test + + + org.evomaster + evomaster-client-java-instrumentation + test-jar + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-starter-security + + + + org.postgresql + postgresql + + + mysql + mysql-connector-java + + + com.h2database + h2 + + + + org.springdoc + springdoc-openapi-ui + + + org.springdoc + springdoc-openapi-security + + + org.springdoc + springdoc-openapi-kotlin + + + + io.swagger + swagger-parser + + + + + org.jetbrains.kotlin + kotlin-stdlib + + + + io.rest-assured + rest-assured + + + org.hamcrest + hamcrest-all + + + org.junit.jupiter + junit-jupiter-engine + + + org.junit.platform + junit-platform-launcher + + + org.junit.jupiter + junit-jupiter-params + + + org.testcontainers + testcontainers + + + com.google.code.gson + gson + + + + + + + + + + + kotlin-maven-plugin + org.jetbrains.kotlin + ${kotlin.version} + + ${java.version} + + spring + jpa + + + + + org.jetbrains.kotlin + kotlin-maven-allopen + ${kotlin.version} + + + org.jetbrains.kotlin + kotlin-maven-noarg + ${kotlin.version} + + + + + compile + + compile + + + + ${project.basedir}/src/main/kotlin + ${project.basedir}/src/main/java + + + + + test-compile + + test-compile + + + + ${project.basedir}/src/test/kotlin + ${project.basedir}/src/test/java + + + + + + + + + + + kotlin-maven-plugin + org.jetbrains.kotlin + ${kotlin.version} + + + org.apache.maven.plugins + maven-compiler-plugin + + + + + \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseApplication.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseApplication.kt new file mode 100644 index 0000000000..d5d59f8def --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseApplication.kt @@ -0,0 +1,23 @@ +package com.foo.rest.examples.multidb.base + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +open class BaseApplication { + + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(BaseApplication::class.java, *args) + } + } + + +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt new file mode 100644 index 0000000000..c0c8089187 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt @@ -0,0 +1,15 @@ +package com.foo.rest.examples.multidb.base + +import javax.persistence.Entity +import javax.persistence.Id +import javax.validation.constraints.NotNull + +@Entity +open class BaseEntity( + + @get:Id @get:NotNull + open var id: String? = null, + + @get:NotNull + open var name: String? = null, +) diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt new file mode 100644 index 0000000000..4e36e3a674 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.multidb.base + +import org.springframework.data.repository.CrudRepository + +interface BaseRepository : CrudRepository \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt new file mode 100644 index 0000000000..3f9a38e097 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt @@ -0,0 +1,25 @@ +package com.foo.rest.examples.multidb.base + +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RequestMapping(path = ["/api/base"]) +@RestController +open class BaseRest( + private val repository: BaseRepository +) { + + @GetMapping(path = ["/{id}"]) + open fun get(@PathVariable id: String) : ResponseEntity { + + val found = repository.findById(id) + if(found.isPresent) { + return ResponseEntity.ok("OK") + } + + return ResponseEntity.notFound().build() + } +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt new file mode 100644 index 0000000000..7e2dc51fd6 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt @@ -0,0 +1,159 @@ +package org.evomaster.driver.multidb + +import org.evomaster.client.java.controller.EmbeddedSutController +import org.evomaster.client.java.controller.api.dto.SutInfoDto +import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto +import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType +import org.evomaster.client.java.controller.problem.ProblemInfo +import org.evomaster.client.java.controller.problem.RestProblem +import org.evomaster.client.java.sql.DbSpecification +import org.evomaster.core.sql.multidb.MultiDbUtils +import org.hibernate.dialect.H2Dialect +import org.hibernate.dialect.MySQL8Dialect +import org.hibernate.dialect.PostgreSQL82Dialect +import org.springframework.boot.SpringApplication +import org.springframework.context.ConfigurableApplicationContext +import java.sql.Connection +import kotlin.random.Random.Default.nextInt + + +abstract class SpringController(protected val applicationClass: Class<*>) : EmbeddedSutController() { + + companion object{ + init { + /** + * To avoid issues with non-determinism checks (in particular in the handling of taint-analysis), + * we must disable the cache in H2 + */ + System.setProperty("h2.objectCache", "false") + } + } + + init { + super.setControllerPort(0) + } + + + protected var sqlConnection: Connection? = null + + protected var ctx: ConfigurableApplicationContext? = null + + protected var databaseType : DatabaseType = DatabaseType.H2 + + fun changeDatabaseType(t: DatabaseType){ + if(t == databaseType){ + //nothing to do + return + } + //stopSut() + databaseType = t + //startSut() + } + + override fun startSut(): String { + + //lot of problem if using same H2 instance. see: + //https://github.com/h2database/h2database/issues/227 + val rand = nextInt(0, 1_000_000_000) + val dbName = "dbtest_$rand" + + MultiDbUtils.startDatabase(databaseType) + MultiDbUtils.resetDatabase(dbName,databaseType) + sqlConnection = MultiDbUtils.createConnection(dbName,databaseType) + val baseUrl = MultiDbUtils.getBaseUrl(databaseType) + + val commonSettings = arrayOf( + "--server.port=0", + "--spring.jpa.properties.hibernate.show_sql=true", + "--spring.jpa.hibernate.ddl-auto=create-drop", + "--spring.jmx.enabled=false" + ) + + ctx = when(databaseType) { + DatabaseType.H2 -> { + SpringApplication.run(applicationClass, + *commonSettings.plus(arrayOf( + "--spring.datasource.url=$baseUrl$dbName;DB_CLOSE_DELAY=-1;", + "--spring.datasource.driverClassName=org.h2.Driver", + "--spring.jpa.database-platform=" + H2Dialect::class.java.name, + "--spring.datasource.username=sa", + "--spring.datasource.password", + )) + ) + } + DatabaseType.MYSQL -> { + SpringApplication.run(applicationClass, + *commonSettings.plus(arrayOf( + "--spring.datasource.url=$baseUrl$dbName", + "--spring.datasource.driverClassName=com.mysql.cj.jdbc.Driver", + "--spring.datasource.username=root", + "--spring.datasource.password=root", + "--spring.jpa.database-platform="+ MySQL8Dialect::class.java.name, + )) + ) + } + DatabaseType.POSTGRES ->{ + SpringApplication.run(applicationClass, + *commonSettings.plus(arrayOf( + "--spring.datasource.url=$baseUrl$dbName", + "--spring.datasource.driverClassName=org.postgresql.Driver", + "--spring.datasource.username=postgres", + "--spring.datasource.password", + "--spring.jpa.database-platform=" + PostgreSQL82Dialect::class.java.name + )) + ) + } + else -> throw IllegalStateException("Not supported database type: $databaseType") + } + + return "http://localhost:$sutPort" + } + + protected val sutPort: Int + get() = (ctx!!.environment + .propertySources["server.ports"].source as Map<*, *>)["local.server.port"] as Int + + override fun isSutRunning(): Boolean { + return ctx != null && ctx!!.isRunning + } + + override fun stopSut() { + ctx?.stop() + ctx?.close() + sqlConnection?.close() + MultiDbUtils.stopDatabase(databaseType) + } + + override fun getPackagePrefixesToCover(): String { + return "com.foo." + } + + override fun resetStateOfSUT() { + //nothing to do. SQL reset is handled automatically + } + + override fun getProblemInfo(): ProblemInfo { + return RestProblem( + "http://localhost:$sutPort/v3/api-docs", + null + ) + } + + override fun getInfoForAuthentication(): List { + return listOf() + } + + override fun getDbSpecifications(): List? { + + if(sqlConnection == null) + return null + + return listOf(DbSpecification(databaseType, sqlConnection)) + } + + + override fun getPreferredOutputFormat(): SutInfoDto.OutputFormat { + return SutInfoDto.OutputFormat.KOTLIN_JUNIT_5 + } + +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt new file mode 100644 index 0000000000..b5035f5851 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt @@ -0,0 +1,6 @@ +package org.evomaster.driver.multidb.base + +import com.foo.rest.examples.multidb.base.BaseApplication +import org.evomaster.driver.multidb.SpringController + +class BaseController : SpringController(BaseApplication::class.java) \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt new file mode 100644 index 0000000000..b5406e5a48 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt @@ -0,0 +1,53 @@ +package org.evomaster.e2etests.spring.multidb + +import org.evomaster.client.java.controller.EmbeddedSutController +import org.evomaster.client.java.controller.InstrumentedSutStarter +import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType +import org.evomaster.client.java.instrumentation.InputProperties +import org.evomaster.client.java.instrumentation.InstrumentingAgent +import org.evomaster.core.sql.multidb.MultiDbUtils +import org.evomaster.driver.multidb.SpringController +import org.evomaster.e2etests.utils.EnterpriseTestBase +import org.evomaster.e2etests.utils.RestTestBase +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource + +abstract class MultiDbParameterizedE2ETemplate : RestTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun initAgent() { + /* + needed due to Kotlin loading before agent is initialized + */ + System.setProperty(InputProperties.REPLACEMENT_CATEGORIES, "BASE,SQL,EXT_0") + InstrumentedSutStarter.loadAgent() + InstrumentingAgent.changePackagesToInstrument("com.foo.") + } + + @AfterAll + @JvmStatic + fun shutDown(){ + MultiDbUtils.stopAllDatabases() + } + } + + @ParameterizedTest + @EnumSource(names = ["MYSQL","POSTGRES","H2"]) + fun testRunEM(databaseType: DatabaseType) { + + val c = instantiateNewController() + c.changeDatabaseType(databaseType) + EnterpriseTestBase.initClass(c) + + runEM(databaseType) + } + + + protected abstract fun runEM(databaseType: DatabaseType) + + protected abstract fun instantiateNewController() : SpringController +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/base/BaseEMTest.kt b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/base/BaseEMTest.kt new file mode 100644 index 0000000000..7d882f43c4 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/base/BaseEMTest.kt @@ -0,0 +1,36 @@ +package org.evomaster.e2etests.spring.multidb.base + +import org.evomaster.client.java.controller.EmbeddedSutController +import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType +import org.evomaster.core.problem.rest.HttpVerb +import org.evomaster.driver.multidb.SpringController +import org.evomaster.e2etests.spring.multidb.MultiDbParameterizedE2ETemplate +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import java.nio.file.Files +import java.nio.file.Paths +import org.evomaster.driver.multidb.base.BaseController + +/** + * Created by arcuri82 on 03-Mar-20. + */ +class BaseEMTest : MultiDbParameterizedE2ETemplate() { + + override fun instantiateNewController(): SpringController { + return BaseController() + } + + override fun runEM(databaseType: DatabaseType) { + runTestHandlingFlakyAndCompilation( + "BaseEM_$databaseType", + 100 + ) { args: MutableList -> + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/base/{id}", "OK") + } + } +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/statistics/StatisticsEMTest.kt b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/statistics/StatisticsEMTest.kt index 1eaf8efabb..398b3a7890 100644 --- a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/statistics/StatisticsEMTest.kt +++ b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/statistics/StatisticsEMTest.kt @@ -60,12 +60,16 @@ class StatisticsEMTest : SpringTestBase() { assertNotNull(searchTime) assertNotNull(total) assertNotNull(seedingTime) - if(key != "coveredBranches") { + /* + There are no branches in constructors... + but current problem is driver code in some package name... + */ + //if(key != "coveredBranches") { //there are no branches coverage at boot-time, as just default constructor assertTrue(bootTime!! > 0) - } else { - assertEquals(0, bootTime!!) - } + //} else { + // assertEquals(0, bootTime!!) + //} assertTrue(searchTime!! > 0) assertEquals(total!!, bootTime!!+searchTime!!+seedingTime!!) } diff --git a/makeExecutable.sh b/makeExecutable.sh index 28c960790c..d7fd7bbeaf 100644 --- a/makeExecutable.sh +++ b/makeExecutable.sh @@ -39,4 +39,3 @@ VENDOR="EvoMaster Team" $JPACKAGE --main-jar $JAR --input $BUILD --dest $RELEASE --name evomaster \ --copyright "$COPYRIGHT" --license-file ./LICENSE --vendor "$VENDOR" --app-version $VERSION $OS - diff --git a/release_notes.md b/release_notes.md index 814aa1c76d..d7ad91a398 100644 --- a/release_notes.md +++ b/release_notes.md @@ -2,6 +2,9 @@ Under development in `master` branch. +### New Features +- Now EvoMaster is released also on Docker Hub, with id `webfuzzing/evomaster`. + However, this is only for black-box mode. For white-box, it is still recommended to use an OS installer or the uber-jar file from release page. ### Bug Fixes - Fixed missing java.util.Arrays in generated files, leading to compilation errors in some cases involving SQL database resets.