diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c9fcec041..ec0ee695f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -55,7 +55,7 @@ jobs: key: fscrawler-docker-cache-${{ runner.os }}-${{ hashFiles('pom.xml') }} continue-on-error: true - name: Run the integration tests - run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Dtests.parallelism=1 + run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests # We run integration tests with elastic stack 7 it-es7: @@ -75,7 +75,7 @@ jobs: key: fscrawler-docker-cache-${{ runner.os }}-${{ hashFiles('pom.xml') }} continue-on-error: true - name: Run the integration tests - run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Pes-7x -Dtests.parallelism=1 + run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Pes-7x # We run integration tests with elastic stack 6 it-es6: @@ -95,7 +95,7 @@ jobs: key: fscrawler-docker-cache-${{ runner.os }}-${{ hashFiles('pom.xml') }} continue-on-error: true - name: Run the integration tests - run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Pes-6x -Dtests.parallelism=1 + run: mvn --batch-mode install -Ddocker.skip -DskipUnitTests -Pes-6x # We run this job in parallel after the build build-docker: diff --git a/docs/source/dev/build.rst b/docs/source/dev/build.rst index c21a4922d..0e086de62 100644 --- a/docs/source/dev/build.rst +++ b/docs/source/dev/build.rst @@ -38,10 +38,8 @@ The final artifacts are available in ``distribution/target``. Integration tests ^^^^^^^^^^^^^^^^^ -When running from the command line with ``mvn`` integration tests are ran against all supported versions. -This is done by running a Docker instance of elasticsearch using the expected version. - -A HTTP server is also started on port 8080 during the integration tests, alternatively the assigned port can be set with `-Dtests.rest.port=8090` argument. +When running from the command line with ``mvn`` integration tests are ran against a real +Elasticsearch instance launched using Docker (via `Testcontainers `_). Run tests from your IDE """"""""""""""""""""""" @@ -135,11 +133,13 @@ You can change this by using ``tests.cluster.pass`` option:: Changing the REST port """""""""""""""""""""" -By default, FS crawler will run the integration tests using port ``8080`` for the REST service. +By default, FS crawler will run the integration tests using a randomly chosen port for the REST service. You can change this by using ``tests.rest.port`` option:: mvn verify -Dtests.rest.port=8280 +When set to ``0`` (default value), the port is assigned randomly. + Randomized testing """""""""""""""""" diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java index 169450836..77658aaaa 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java @@ -91,7 +91,6 @@ public abstract class AbstractITCase extends AbstractFSCrawlerTestCase { private final static String DEFAULT_TEST_CLUSTER_URL = "https://127.0.0.1:9200"; private final static String DEFAULT_USERNAME = "elastic"; private final static String DEFAULT_PASSWORD = "changeme"; - private final static Integer DEFAULT_TEST_REST_PORT = 8080; protected static String testClusterUrl = null; @Deprecated @@ -99,7 +98,6 @@ public abstract class AbstractITCase extends AbstractFSCrawlerTestCase { @Deprecated protected final static String testClusterPass = getSystemProperty("tests.cluster.pass", DEFAULT_PASSWORD); protected static String testApiKey = getSystemProperty("tests.cluster.apiKey", null); - protected final static int testRestPort = getSystemProperty("tests.rest.port", DEFAULT_TEST_REST_PORT); protected final static boolean testKeepData = getSystemProperty("tests.leaveTemporary", true); protected static Elasticsearch elasticsearchConfiguration; diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java index 478964eb3..162e21612 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java @@ -49,6 +49,7 @@ import org.junit.BeforeClass; import java.io.IOException; +import java.net.ServerSocket; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -65,6 +66,9 @@ public abstract class AbstractRestITCase extends AbstractITCase { + private final static int DEFAULT_TEST_REST_PORT = 0; + private static int testRestPort = getSystemProperty("tests.rest.port", DEFAULT_TEST_REST_PORT); + protected static WebTarget target; protected static Client client; @@ -72,7 +76,24 @@ public abstract class AbstractRestITCase extends AbstractITCase { private FsCrawlerManagementServiceElasticsearchImpl managementService; protected FsCrawlerDocumentService documentService; - public abstract FsSettings getFsSettings(); + /** + * Get the Rest Port. It could be set externally. If 0, + * we will try to find a free port randomly + * @return the rest port to use + */ + protected static int getRestPort() throws IOException { + if (testRestPort <= 0) { + // Find any available TCP port if 0 or check that the port is available + try (ServerSocket serverSocket = new ServerSocket(testRestPort)) { + testRestPort = serverSocket.getLocalPort(); + staticLogger.info("Using random rest port [{}]", testRestPort); + } + } + + return testRestPort; + } + + public abstract FsSettings getFsSettings() throws IOException; @Before public void copyTags() throws IOException { Path testResourceTarget = rootTmpDir.resolve("resources"); @@ -170,7 +191,7 @@ public static T delete(WebTarget target, String path, Class clazz, Map elastic changeme - 8080 + 0 random diff --git a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java index 52d3ab5d4..9245415f3 100644 --- a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java +++ b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java @@ -62,7 +62,7 @@ public static void start(FsSettings settings, FsCrawlerManagementService managem // create and start a new instance of grizzly http server // exposing the Jersey application at BASE_URI httpServer = GrizzlyHttpServerFactory.createHttpServer(URI.create(settings.getRest().getUrl()), rc); - logger.info("FS crawler Rest service started on [{}]", settings.getRest().getUrl()); + logger.info("FSCrawler Rest service started on [{}]", settings.getRest().getUrl()); } }