Merge pull request #1911 from dadoonet/random-rest-port

Use a Random port to run REST tests

.github/workflows/pr.yml

@@ -55,7 +55,7 @@ jobs:
           key: fscrawler-docker-cache-${{ runner.os }}-${{ hashFiles('pom.xml') }}
         continue-on-error: true
       - name: Run the integration tests
-        run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Dtests.parallelism=1
+        run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests
 
   # We run integration tests with elastic stack 7
   it-es7:
@@ -75,7 +75,7 @@ jobs:
           key: fscrawler-docker-cache-${{ runner.os }}-${{ hashFiles('pom.xml') }}
         continue-on-error: true
       - name: Run the integration tests
-        run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Pes-7x -Dtests.parallelism=1
+        run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Pes-7x
 
   # We run integration tests with elastic stack 6
   it-es6:
@@ -95,7 +95,7 @@ jobs:
           key: fscrawler-docker-cache-${{ runner.os }}-${{ hashFiles('pom.xml') }}
         continue-on-error: true
       - name: Run the integration tests
-        run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Pes-6x -Dtests.parallelism=1
+        run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Pes-6x
 
   # We run this job in parallel after the build
   build-docker:

docs/source/dev/build.rst

@@ -38,10 +38,8 @@ The final artifacts are available in ``distribution/target``.
 Integration tests
 ^^^^^^^^^^^^^^^^^
 
-When running from the command line with ``mvn`` integration tests are ran against all supported versions.
-This is done by running a Docker instance of elasticsearch using the expected version.
-
-A HTTP server is also started on port 8080 during the integration tests, alternatively the assigned port can be set with `-Dtests.rest.port=8090` argument.
+When running from the command line with ``mvn`` integration tests are ran against a real
+Elasticsearch instance launched using Docker (via `Testcontainers <https://java.testcontainers.org/modules/elasticsearch/>`_).
 
 Run tests from your IDE
 """""""""""""""""""""""
@@ -135,11 +133,13 @@ You can change this by using ``tests.cluster.pass`` option::
 Changing the REST port
 """"""""""""""""""""""
 
-By default, FS crawler will run the integration tests using port ``8080`` for the REST service.
+By default, FS crawler will run the integration tests using a randomly chosen port for the REST service.
 You can change this by using ``tests.rest.port`` option::
 
     mvn verify -Dtests.rest.port=8280
 
+When set to ``0`` (default value), the port is assigned randomly.
+
 Randomized testing
 """"""""""""""""""
 

integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java

@@ -91,15 +91,13 @@ public abstract class AbstractITCase extends AbstractFSCrawlerTestCase {
     private final static String DEFAULT_TEST_CLUSTER_URL = "https://127.0.0.1:9200";
     private final static String DEFAULT_USERNAME = "elastic";
     private final static String DEFAULT_PASSWORD = "changeme";
-    private final static Integer DEFAULT_TEST_REST_PORT = 8080;
 
     protected static String testClusterUrl = null;
     @Deprecated
     protected final static String testClusterUser = getSystemProperty("tests.cluster.user", DEFAULT_USERNAME);
     @Deprecated
     protected final static String testClusterPass = getSystemProperty("tests.cluster.pass", DEFAULT_PASSWORD);
     protected static String testApiKey = getSystemProperty("tests.cluster.apiKey", null);
-    protected final static int testRestPort = getSystemProperty("tests.rest.port", DEFAULT_TEST_REST_PORT);
     protected final static boolean testKeepData = getSystemProperty("tests.leaveTemporary", true);
 
     protected static Elasticsearch elasticsearchConfiguration;

integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java

@@ -49,6 +49,7 @@
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.net.ServerSocket;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -65,14 +66,34 @@
 
 public abstract class AbstractRestITCase extends AbstractITCase {
 
+    private final static int DEFAULT_TEST_REST_PORT = 0;
+    private static int testRestPort = getSystemProperty("tests.rest.port", DEFAULT_TEST_REST_PORT);
+
     protected static WebTarget target;
     protected static Client client;
 
     protected Path currentTestTagDir;
     private FsCrawlerManagementServiceElasticsearchImpl managementService;
     protected FsCrawlerDocumentService documentService;
 
-    public abstract FsSettings getFsSettings();
+    /**
+     * Get the Rest Port. It could be set externally. If 0,
+     * we will try to find a free port randomly
+     * @return the rest port to use
+     */
+    protected static int getRestPort() throws IOException {
+        if (testRestPort <= 0) {
+            // Find any available TCP port if 0 or check that the port is available
+            try (ServerSocket serverSocket = new ServerSocket(testRestPort)) {
+                testRestPort = serverSocket.getLocalPort();
+                staticLogger.info("Using random rest port [{}]", testRestPort);
+            }
+        }
+
+        return testRestPort;
+    }
+
+    public abstract FsSettings getFsSettings() throws IOException;
     @Before
     public void copyTags() throws IOException {
         Path testResourceTarget = rootTmpDir.resolve("resources");
@@ -170,15 +191,15 @@ public static <T> T delete(WebTarget target, String path, Class<T> clazz, Map<St
     }
 
     @BeforeClass
-    public static void startRestClient() {
+    public static void startRestClient() throws IOException {
         // create the client
         client = ClientBuilder.newBuilder()
                 .register(MultiPartFeature.class)
                 .register(RestJsonProvider.class)
                 .register(JacksonFeature.class)
                 .build();
 
-        target = client.target("http://127.0.0.1:" + testRestPort + "/fscrawler");
+        target = client.target("http://127.0.0.1:" + getRestPort() + "/fscrawler");
     }
 
     @AfterClass

integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestFilenameAsIdIT.java

@@ -31,6 +31,7 @@
 import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractRestITCase;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
@@ -40,10 +41,10 @@
 @SuppressWarnings("ALL")
 public class FsCrawlerRestFilenameAsIdIT extends AbstractRestITCase {
 
-    public FsSettings getFsSettings() {
+    public FsSettings getFsSettings() throws IOException {
         return FsSettings.builder(getCrawlerName())
                 .setFs(Fs.builder().setFilenameAsId(true).build())
-                .setRest(new Rest("http://127.0.0.1:" + testRestPort + "/fscrawler"))
+                .setRest(new Rest("http://127.0.0.1:" + getRestPort() + "/fscrawler"))
                 .setElasticsearch(elasticsearchConfiguration)
                 .build();
     }

integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java

@@ -37,6 +37,7 @@
 import org.glassfish.jersey.media.multipart.file.FileDataBodyPart;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -53,9 +54,9 @@
 @SuppressWarnings("ALL")
 public class FsCrawlerRestIT extends AbstractRestITCase {
 
-    public FsSettings getFsSettings() {
+    public FsSettings getFsSettings() throws IOException {
         return FsSettings.builder(getCrawlerName())
-                .setRest(new Rest("http://127.0.0.1:" + testRestPort + "/fscrawler"))
+                .setRest(new Rest("http://127.0.0.1:" + getRestPort() + "/fscrawler"))
                 .setElasticsearch(elasticsearchConfiguration)
                 .build();
     }

pom.xml

@@ -67,7 +67,7 @@
         <tests.cluster.apiKey></tests.cluster.apiKey>
         <tests.cluster.user>elastic</tests.cluster.user>
         <tests.cluster.pass>changeme</tests.cluster.pass>
-        <tests.rest.port>8080</tests.rest.port>
+        <tests.rest.port>0</tests.rest.port>
 
         <!-- Randomized testing framework -->
         <tests.locale>random</tests.locale>

rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java

@@ -62,7 +62,7 @@ public static void start(FsSettings settings, FsCrawlerManagementService managem
             // create and start a new instance of grizzly http server
             // exposing the Jersey application at BASE_URI
             httpServer = GrizzlyHttpServerFactory.createHttpServer(URI.create(settings.getRest().getUrl()), rc);
-            logger.info("FS crawler Rest service started on [{}]", settings.getRest().getUrl());
+            logger.info("FSCrawler Rest service started on [{}]", settings.getRest().getUrl());
         }
     }