Merge branch 'master' into taint-array

.github/workflows/ci.yml

@@ -11,25 +11,23 @@ env:
   # For some "hilarious" case, see:
   # https://github.com/EMResearch/EvoMaster/issues/447
   release-jdk: 17
+  latest-jdk: 17
   build-jdk: 1.8
-  retention-days: 3
+  retention-days: 5
 
 
-# This build is quite expensive (1ish hour), so we run it whole only on 1 JVM version, and 1 OS.
-# Recall that we also build on CircleCI, with different configurations.
-# For the moment, we need to support JVM 8 and 11, with 17 when it comes out.
+# This build is quite expensive (some hours), so we run it whole only on some JVM versions and OSs.
+# For the moment, we need to support JVM 8 and all following LTS versions (e.g, 11 and  17).
 # EM should run an all OSs. As our main development machines are Windows and Mac, just
 # testing whole build with all E2E only on Linux should be fine.
 
 jobs:
 
-  base:
+  full-build-base:
     runs-on: ubuntu-latest
-
     steps:
       # Checkout code
       - uses: actions/checkout@v2
-
       # Build/test for JDK
       - name: Setup JDK ${{env.build-jdk}}
         uses: actions/setup-java@v1
@@ -52,6 +50,13 @@ jobs:
           path: core/target/evomaster.jar
           retention-days: ${{env.retention-days}}
           if-no-files-found: error
+          ### TODO disabled due to bug. See https://github.com/mikepenz/action-junit-report/issues/952
+      # Make test report accessible from GitHub Actions (as Maven logs are long)
+#      - name: Publish Test Report
+#        if: success() || failure()
+#        uses: mikepenz/action-junit-report@v4
+#        with:
+#          report_paths: '**/target/surefire-reports/TEST-*.xml'
       # Upload coverage results
       - name: Upload coverage to CodeCov
         run: curl -s https://codecov.io/bash | bash
@@ -63,9 +68,54 @@ jobs:
           java-version: 11
       - name: Compile with JDK 11
         run: mvn clean verify -DskipTests
+      - name: Setup JDK 17
+        uses: actions/setup-java@v1
+        with:
+          java-version: 17
+      - name: Compile with JDK 17
+        run: mvn clean verify -DskipTests
 
 
-  latest-LTS:
+  base-build-mac:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup JDK ${{env.latest-jdk}}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{env.latest-jdk}}
+      - name: Cache Maven packages
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+      - name: Build with Maven
+        run: mvn clean verify -DskipTests
+        env:
+          CI_env: GithubAction
+
+  base-build-windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup JDK ${{env.latest-jdk}}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{env.latest-jdk}}
+      - name: Cache Maven packages
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+      - name: Build with Maven
+        run: mvn clean verify -DskipTests
+        env:
+          CI_env: GithubAction
+
+
+  full-build-LTS:
     # run on a different OS from main build.
     # It would had been good, but does not work due to Docker support :(  see:
     # https://stackoverflow.com/questions/66077884/testcontainers-in-windows-environment-on-github-actions-could-not-find-a-valid
@@ -74,10 +124,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Setup JDK 17
+      - name: Setup JDK ${{env.latest-jdk}}
         uses: actions/setup-java@v1
         with:
-          java-version: 17
+          java-version: ${{env.latest-jdk}}
       - name: Cache Maven packages
         uses: actions/cache@v2
         with:
@@ -92,7 +142,7 @@ jobs:
 
 
   nodejs:
-    needs: base
+    needs: full-build-base
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -122,9 +172,12 @@ jobs:
         with:
           name: evomaster.jar
           path: core/target
+      ### NOTE: we no longer run this, as NodeJS for backend no longer supported.
+      ### This started to fail once introduced first version of "minimization", as JS driver would need
+      ### to be updated with new features to support it
       # E2E for JS. Must be run AFTER the JS and Core builds
-      - name: E2E for JavaScript
-        run:  bash ./client-js/e2e-tests/e2e.sh
+#      - name: E2E for JavaScript
+#        run:  bash ./client-js/e2e-tests/e2e.sh
 
 ### TODO currently disabled, as it takes forever on CI (1-2 hours...), compared to few minutes on local machine.
 ### Need to find out why
@@ -160,7 +213,7 @@ jobs:
 
 
   installer-for-windows:
-    needs: base
+    needs: full-build-base
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v2
@@ -185,7 +238,7 @@ jobs:
           if-no-files-found: error
 
   installer-for-osx:
-    needs: base
+    needs: full-build-base
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v2
@@ -210,7 +263,7 @@ jobs:
           if-no-files-found: error
 
   installer-for-debian:
-    needs: base
+    needs: full-build-base
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

.gitignore

@@ -125,3 +125,4 @@ Migrations/
 /e2e-tests/spring-rest-h2-column-types/target/
 /test-old-libraries/target/
 /e2e-tests/spring-web/target/
+/e2e-tests/spring-rest-mongo/target/

README.md

@@ -12,84 +12,91 @@
 [![Github All Releases](https://img.shields.io/github/downloads/emresearch/evomaster/total.svg)](https://github.com/EMResearch/EvoMaster/releases)
 
 
-### Summary 
+### Summary
 
-_EvoMaster_ ([www.evomaster.org](http://evomaster.org)) is the first (2016) open-source AI-driven tool 
+_EvoMaster_ ([www.evomaster.org](http://evomaster.org)) is the first (2016) open-source AI-driven tool
 that automatically *generates* system-level test cases
 for web/enterprise applications.
 This is related to [Fuzzing](https://en.wikipedia.org/wiki/Fuzzing).
-Not only _EvoMaster_ can generate inputs that find program crashes, but also it generates small effective test suites that can be used for _regression testing_.
+Not only _EvoMaster_ can generate inputs that find program crashes, but also it generates small effective test suites (e.g., in JUnit format) that can be used for _regression testing_.
 
 _EvoMaster_ is an AI driven tool.
-In particular, internally it uses an [Evolutionary Algorithm](https://en.wikipedia.org/wiki/Evolutionary_algorithm) 
-and [Dynamic Program Analysis](https://en.wikipedia.org/wiki/Dynamic_program_analysis)  to be 
+In particular, internally it uses an [Evolutionary Algorithm](https://en.wikipedia.org/wiki/Evolutionary_algorithm)
+and [Dynamic Program Analysis](https://en.wikipedia.org/wiki/Dynamic_program_analysis)  to be
 able to generate effective test cases.
-The approach is to *evolve* test cases from an initial population of 
+The approach is to *evolve* test cases from an initial population of
 random ones, trying to maximize measures like code coverage and fault detection.
-_EvoMaster_ uses several kinds of AI heuristics to improve performance even further, 
+_EvoMaster_ uses several kinds of AI heuristics to improve performance even further,
 building on decades of research in the field of [Search-Based Software Testing](https://en.wikipedia.org/wiki/Search-based_software_engineering).
 
 
 __Key features__:
 
-* _Web APIs_: At the moment, _EvoMaster_ can generate test cases for __REST__ and __GraphQL__ APIs. 
+* _Web APIs_: At the moment, _EvoMaster_ can generate test cases for __REST__, __GraphQL__ and __RPC__ (e.g., __gRPC__ and __Thrift__) APIs.
 
 * _Blackbox_ testing mode: can run on any API (regardless of its programming language, e.g., Python and Go).
-   However, results for blackbox testing will be worse than whitebox testing (e.g., due to lack of code analysis).
+  However, results for blackbox testing will be worse than whitebox testing (e.g., due to lack of code analysis).
 
-* _Whitebox_ testing mode: can be used for APIs compiled to 
+* _Whitebox_ testing mode: can be used for APIs compiled to
   JVM (e.g., Java and Kotlin). _EvoMaster_ analyses the bytecode of the tested applications, and uses
   several heuristics such as _testability transformations_ and _taint analysis_ to be able to generate
-  more effective test cases. We support JDK __8__ and the major LTS versions after that (currently JDK __17__). Might work on other JVM versions, but we provide __NO__ support for it. 
-   Note: there is initial support for other languages as well, like for example JavaScript/TypeScript, but they are not in a stable, feature-complete state yet. 
+  more effective test cases. We support JDK __8__ and the major LTS versions after that (currently JDK __17__). Might work on other JVM versions, but we provide __NO__ support for it.
+  Note: there is initial support for other languages as well, like for example JavaScript/TypeScript and C#, but they are not in a stable, feature-complete state yet.
 
-* _Installation_: we provide installers for the main operating systems: Windows (`.msi`), 
+* _Installation_: we provide installers for the main operating systems: Windows (`.msi`),
   OSX (`.dmg`) and Linux (`.deb`). We also provide an uber-fat JAR file.
 
-* _State-of-the-art_: an [independent study (2022)](https://arxiv.org/abs/2204.08348), comparing 10 fuzzers on 20 RESTful APIs, shows that _EvoMaster_ gives the best results.   
+* _State-of-the-art_: an [independent study (2022)](https://arxiv.org/abs/2204.08348), comparing 10 fuzzers on 20 RESTful APIs, shows that _EvoMaster_ gives the best results.
 
-* _Schema_: REST APIs must provide a schema in [OpenAPI/Swagger](https://swagger.io) 
+* _Schema_: REST APIs must provide a schema in [OpenAPI/Swagger](https://swagger.io)
   format (either _v2_ or _v3_).
 
 * _Output_: the tool generates _JUnit_ (version 4 or 5) tests, written in either Java or Kotlin. There is initial support for other formats. For complete list, see the documentation for the CLI parameter [--outputFormat](docs/options.md).
 
 * _Fault detection_: _EvoMaster_ can generate tests cases that reveal faults/bugs in the tested applications.
-  Different heuristics are employed, like checking for 500 status codes and mismatches from the API schemas. 
+  Different heuristics are employed, like checking for 500 status codes and mismatches from the API schemas.
 
 * _Self-contained tests_: the generated tests do start/stop the application, binding to an ephemeral port.
   This means that the generated tests can be used for _regression testing_ (e.g., added to the Git repository
-  of the application, and run with any build tool such as Maven and Gradle). 
+  of the application, and run with any build tool such as Maven and Gradle).
+
 
-
 * _SQL handling_: _EvoMaster_ can intercept and analyse all communications done with SQL databases, and use
   such information to generate higher code coverage test cases. Furthermore, it can generate data directly
-  into the databases, and have such initialization automatically added in the generated tests. 
-  At the moment, _EvoMaster_ supports _Postgres_, _MySQL_ and _H2_  databases.  
+  into the databases, and have such initialization automatically added in the generated tests.
+  At the moment, _EvoMaster_ supports _Postgres_, _MySQL_ and _H2_  databases.
 
 
-* _Authentication_: we support auth based on authentication headers and cookies. 
+* _Authentication_: we support auth based on authentication headers and cookies.
 
 __Known limitations__:
 
 * _Driver_: to be used for _whitebox_ testing, users need to write a [driver manually](docs/write_driver.md).
   We recommend to try _blackbox_ mode first (should just need a few minutes to get it up and running) to get
-  an idea of what _EvoMaster_ can do for you.  
+  an idea of what _EvoMaster_ can do for you.
+
+* _JDK 9+_: whitebox testing requires bytecode manipulation. 
+            Each new release of the JDK makes doing this harder and harder. 
+            Dealing with JDKs above __8__ is doable, but it requires some settings.
+            [See documentation](docs/jdks.md).
 
-* _Execution time_: to get good results, you might need to run the search for several hours. 
+* _Execution time_: to get good results, you might need to run the search for several hours.
   We recommend to first try the search for 10 minutes, just to get an idea of what type of tests can be generated.
   But, then, you should run _EvoMaster_ for something like between 1 and 24 hours (the longer the better, but
   it is unlikely to get better results after 24 hours).
-
+
+* _RPC APIs_: for the moment, we do not directly support RPC schema definitions. Fuzzing RPC APIs requires to write a driver, using the client library of the API to make the calls.
+
 * _External services_: (e.g., other RESTful APIs) currently there is no support for them (e.g., to automatically mock them).
   It is work in progress.
-  
-* _NoSQL databases_: (e.g., MongoDB) currently no support. It is work in progress. 
+
+* _NoSQL databases_: (e.g., MongoDB) currently no support. It is work in progress.
 
 * _Failing tests_: the tests generated by _EvoMaster_ should all pass, and not fail, even when they detect a fault.
   In those cases, comments/test-names would point out that a test is revealing a possible fault, while still passing.
   However, in some cases the generated tests might fail. This is due to the so called _flaky_ tests, e.g., when
-  a test has assertions based on the time clock (e.g., dates and timestamps). 
-  There is ongoing effort to address this problem, but it is still not fully solved.   
+  a test has assertions based on the time clock (e.g., dates and timestamps).
+  There is ongoing effort to address this problem, but it is still not fully solved.
 
 <!--### Videos---> 
 <!-- 
@@ -102,24 +109,24 @@ __Known limitations__:
 ![](docs/img/video-player-flaticon.png)
 
 * A [short video](https://youtu.be/3mYxjgnhLEo) (5 minutes)
-shows the use of _EvoMaster_ on one of the 
-case studies in [EMB](https://github.com/EMResearch/EMB). 
+  shows the use of _EvoMaster_ on one of the
+  case studies in [EMB](https://github.com/EMResearch/EMB).
 
 * This [13-minute video](https://youtu.be/ORxZoYw7LnM)
   shows how to write a white-box driver for EvoMaster, for the
-  [rest-api-example](https://github.com/EMResearch/rest-api-example). 
+  [rest-api-example](https://github.com/EMResearch/rest-api-example).
+
+* How to [Download and Install EvoMaster on Windows 10](https://youtu.be/uh_XzGxws9o), using its _.msi_ installer.
 
-* How to [Download and Install EvoMaster on Windows 10](https://youtu.be/uh_XzGxws9o), using its _.msi_ installer. 
-
 ### Hiring
 
 Depending on the year, we might have funding for _postdoc_ and _PhD student_ positions to work on this project (in Oslo, Norway).
 
 Current positions:
-* 2023: PhD student positions. [New calls](https://www.kristiania.no/om-kristiania/ledige-stillinger/?rmpage=job&rmjob=619&rmlang=NO).
-* 2023: Postdoc positions. New calls not announced yet.
+* 2023: PhD student positions, [1 position available](https://www.kristiania.no/en/about-kristiania/vacant-positions/?rmpage=job&rmjob=679&rmlang=UK).
+* 2023: Postdoc positions. No new calls scheduled for the moment.
 
-For questions on these positions, please contact Prof. Andrea Arcuri. 
+For questions on these positions, please contact Prof. Andrea Arcuri.
 
 <!---
 For more details on current vacancies, see our group page at [AISE Lab](https://emresearch.github.io/).
@@ -129,7 +136,7 @@ For more details on current vacancies, see our group page at [AISE Lab](https://
 
 ### Documentation
 
-If you are trying to use _EvoMaster_, but the instructions in this documentation are not enough to get you started, or they are too unclear, then it means it is a _bug_ in the documentation, which then would need to be clarified and updated. In such cases, please create a new [issue](https://github.com/EMResearch/EvoMaster/issues).  
+If you are trying to use _EvoMaster_, but the instructions in this documentation are not enough to get you started, or they are too unclear, then it means it is a _bug_ in the documentation, which then would need to be clarified and updated. In such cases, please create a new [issue](https://github.com/EMResearch/EvoMaster/issues).
 
 * [Example of generated tests](docs/example.md)
 * [Download and Install EvoMaster](docs/download.md)
@@ -138,18 +145,18 @@ If you are trying to use _EvoMaster_, but the instructions in this documentation
 * [OpenApi/Swagger Schema](docs/openapi.md)
 * [Using EvoMaster for Black-Box Testing (easier to setup, but worse results)](docs/blackbox.md)
 * [Using EvoMaster for White-Box Testing (harder to setup, but better results)](docs/whitebox.md)
-    * [Write an EvoMaster Driver for White-Box Testing](docs/write_driver.md)
-    * [Dealing with JDKs above version 8](docs/jdks.md)
-* [Console output](docs/console_output.md)  
+  * [Write an EvoMaster Driver for White-Box Testing](docs/write_driver.md)
+  * [Dealing with JDKs above version 8](docs/jdks.md)
+* [Console output](docs/console_output.md)
 * [Library dependencies for the generated tests](docs/library_dependencies.md)
 * [How to contribute](docs/contribute.md)
-    * [Technical notes for developers contributing to EvoMaster](docs/for_developers.md)
+  * [Technical notes for developers contributing to EvoMaster](docs/for_developers.md)
 * Troubleshooting
-    * [Windows and networking](docs/troubleshooting/windows.md)
+  * [Windows and networking](docs/troubleshooting/windows.md)
 * More Info
-    * [Academic papers related to EvoMaster](docs/publications.md)
-    * [Slides of presentations/seminars](docs/presentations.md)
-    * [Replicating studies](docs/replicating_studies.md)
+  * [Academic papers related to EvoMaster](docs/publications.md)
+  * [Slides of presentations/seminars](docs/presentations.md)
+  * [Replicating studies](docs/replicating_studies.md)
 
 
 
@@ -158,12 +165,12 @@ If you are trying to use _EvoMaster_, but the instructions in this documentation
 
 ### Funding
 
-_EvoMaster_ has been funded by: 
+_EvoMaster_ has been funded by:
 * 2020-2025: a 2 million Euro grant by the European Research Council (ERC),
-as part of the *ERC Consolidator* project 
-<i>Using Evolutionary Algorithms to Understand and Secure Web/Enterprise Systems</i>.
-*  2018-2021: a 7.8 million Norwegian Kroner grant  by the Research Council of Norway (RCN), 
-as part of the Frinatek project <i>Evolutionary Enterprise Testing</i>.  
+  as part of the *ERC Consolidator* project
+  <i>Using Evolutionary Algorithms to Understand and Secure Web/Enterprise Systems</i>.
+*  2018-2021: a 7.8 million Norwegian Kroner grant  by the Research Council of Norway (RCN),
+   as part of the Frinatek project <i>Evolutionary Enterprise Testing</i>.
 
 
 <img src="https://github.com/EMResearch/EvoMaster/blob/master/docs/img/LOGO_ERC-FLAG_EU_.jpg?raw=true" width="200" >
@@ -175,16 +182,16 @@ This project has received funding from the European Research Council (ERC) under
 ### License
 _EvoMaster_'s source code is released under the LGPL (v3) license.
 For a list of the used third-party libraries, you can directly see the root [pom.xml](./pom.xml) file.
-For a list of code directly imported (and then possibly modified/updated) from 
+For a list of code directly imported (and then possibly modified/updated) from
 other open-source projects, see [here](./docs/reused_code.md).
 
 
 ### ![](https://www.yourkit.com/images/yklogo.png)
 
 YourKit supports open source projects with its full-featured Java Profiler.
-YourKit, LLC is the creator of 
+YourKit, LLC is the creator of
 <a href="https://www.yourkit.com/java/profiler/">YourKit Java Profiler</a>
-and 
+and
 <a href="https://www.yourkit.com/.net/profiler/">YourKit .NET Profiler</a>,
 innovative and intelligent tools for profiling Java and .NET applications.