Don't hide console logs when running with Docker (#1738)

* Refactor the CLI

We can add more tests and make it a bit more flexible...

* Add a basic docker compose example without Elastic

It helps to test the behavior of the container when we don't have any input.

Related to #1727

* Don't hide console logs when running with Docker

I believe it was a mistake doing this. It's better to let the user control the level of logs they want when running the container.

In case of error, it now displays correctly the logs in the docker compose output.

Closes #1727.

cli/src/main/resources/log4j2-file.xml

@@ -10,6 +10,10 @@
    </Properties>
 
    <Appenders>
+      <Console name="Console" target="SYSTEM_OUT" follow="true">
+         <PatternLayout pattern="%d{ABSOLUTE} %highlight{%-5p} [%c{1.}] %m%n"/>
+      </Console>
+
       <RollingFile name="RollingFile" fileName="${sys:LOG_DIR}/fscrawler.log"
                    filePattern="${sys:LOG_DIR}/fscrawler-%d{yyyy-MM-dd}-%i.log.gz">
          <PatternLayout pattern="%d{ABSOLUTE} %highlight{%-5p} [%c{1.}] %m%n"/>
@@ -33,6 +37,7 @@
    <Loggers>
       <!-- This logger is used for the console -->
       <Logger name="fscrawler.console" level="info" additivity="false">
+         <AppenderRef ref="Console" />
          <AppenderRef ref="RollingFile" />
       </Logger>
 
@@ -43,6 +48,7 @@
 
       <!-- This logger is used to log FSCrawler code execution -->
       <Logger name="fr.pilato.elasticsearch.crawler.fs" level="${sys:LOG_LEVEL}" additivity="false">
+         <AppenderRef ref="Console" />
          <AppenderRef ref="RollingFile" />
       </Logger>
 

cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliCommandParserTest.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to David Pilato (the "Author") under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Author licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package fr.pilato.elasticsearch.crawler.fs.cli;
+
+import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerIllegalConfigurationException;
+import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.nio.file.Path;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.*;
+
+/**
+ * We want to test FSCrawler main app
+ */
+public class FsCrawlerCliCommandParserTest extends AbstractFSCrawlerTestCase {
+
+    private static Path metadataDir;
+
+    @BeforeClass
+    public static void createFsCrawlerJobDir() {
+        metadataDir = rootTmpDir.resolve(".fscrawler");
+    }
+
+    @Test
+    public void testCommandParserWithFullOptions() {
+        String[] args = {
+                "--config_dir", metadataDir.toString(),
+                "--loop", "0",
+                "--username", "dadoonet",
+                "--rest",
+                "--upgrade",
+                "--restart",
+                "--debug",
+                "jobName"
+        };
+        FsCrawlerCli.FsCrawlerCommand command = FsCrawlerCli.commandParser(args);
+        assertThat(command, notNullValue());
+        assertThat(command.configDir, is(metadataDir.toString()));
+        assertThat(command.loop, is(0));
+        assertThat(command.username, is("dadoonet"));
+        assertThat(command.rest, is(true));
+        assertThat(command.upgrade, is(true));
+        assertThat(command.restart, is(true));
+        assertThat(command.debug, is(true));
+        assertThat(command.trace, is(false));
+        assertThat(command.silent, is(false));
+        assertThat(command.jobName.get(0), is("jobName"));
+    }
+
+    @Test
+    public void testCommandParserForHelp() {
+        String[] args = {
+                "--help"
+        };
+        FsCrawlerCli.FsCrawlerCommand command = FsCrawlerCli.commandParser(args);
+        assertThat(command, nullValue());
+    }
+
+    @Test(expected = FsCrawlerIllegalConfigurationException.class)
+    public void testCommandParserSilentModeNoJob() {
+        String[] args = {
+                "--silent"
+        };
+        FsCrawlerCli.commandParser(args);
+    }
+
+    @Test
+    public void testCommandParserSilentModeWithJob() {
+        String[] args = {
+                "--silent",
+                "jobName"
+        };
+        FsCrawlerCli.FsCrawlerCommand command = FsCrawlerCli.commandParser(args);
+        assertThat(command, notNullValue());
+        assertThat(command.silent, is(true));
+        assertThat(command.jobName.get(0), is("jobName"));
+    }
+}

cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliDefaultSettingsTest.java

@@ -0,0 +1,157 @@
+/*
+ * Licensed to David Pilato (the "Author") under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Author licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package fr.pilato.elasticsearch.crawler.fs.cli;
+
+import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings;
+import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsFileHandler;
+import fr.pilato.elasticsearch.crawler.fs.settings.Server;
+import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static fr.pilato.elasticsearch.crawler.fs.cli.FsCrawlerCli.modifySettings;
+import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.copyDefaultResources;
+import static fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsFileHandler.SETTINGS_YAML;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.*;
+
+/**
+ * We want to test FSCrawler main app
+ */
+public class FsCrawlerCliDefaultSettingsTest extends AbstractFSCrawlerTestCase {
+
+    private static Path metadataDir;
+
+    @BeforeClass
+    public static void createFsCrawlerJobDir() throws IOException {
+        // We also need to create default mapping files
+        metadataDir = rootTmpDir.resolve(".fscrawler");
+        if (Files.notExists(metadataDir)) {
+            Files.createDirectory(metadataDir);
+        }
+        copyDefaultResources(metadataDir);
+        staticLogger.debug("  --> Test metadata dir ready in [{}]", metadataDir);
+    }
+
+    @AfterClass
+    public static void printMetadataDirContent() throws IOException {
+        printLs(metadataDir);
+    }
+
+    private static void printLs(Path dir) throws IOException {
+        staticLogger.debug("ls -l {}", dir);
+        Files.list(dir).forEach(path -> {
+            if (Files.isDirectory(path)) {
+                try {
+                    printLs(path);
+                } catch (IOException ignored) { }
+            } else {
+                staticLogger.debug("{}", path);
+            }
+        });
+    }
+
+    @Test
+    public void testModifySettingsNoUsername() throws IOException {
+        FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(metadataDir);
+        Path jobDir = metadataDir.resolve("modify_settings_no_username");
+        Files.createDirectories(jobDir);
+
+        Files.writeString(jobDir.resolve(SETTINGS_YAML), "name: \"modify_settings_no_username\"");
+        FsSettings settings = fsSettingsFileHandler.read("modify_settings_no_username");
+        assertThat(settings.getFs(), nullValue());
+        assertThat(settings.getElasticsearch(), nullValue());
+        modifySettings(settings, null);
+        assertThat(settings.getFs(), notNullValue());
+        assertThat(settings.getElasticsearch(), notNullValue());
+        assertThat(settings.getElasticsearch().getUsername(), nullValue());
+    }
+
+    @Test
+    public void testModifySettingsWithUsername() throws IOException {
+        FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(metadataDir);
+        Path jobDir = metadataDir.resolve("modify_settings_with_username");
+        Files.createDirectories(jobDir);
+
+        Files.writeString(jobDir.resolve(SETTINGS_YAML), "name: \"modify_settings_with_username\"");
+        FsSettings settings = fsSettingsFileHandler.read("modify_settings_with_username");
+        assertThat(settings.getFs(), nullValue());
+        assertThat(settings.getElasticsearch(), nullValue());
+        modifySettings(settings, "elastic");
+        assertThat(settings.getFs(), notNullValue());
+        assertThat(settings.getElasticsearch(), notNullValue());
+        assertThat(settings.getElasticsearch().getUsername(), is("elastic"));
+    }
+
+    @Test
+    public void testModifySettingsWithServerFtp() throws IOException {
+        FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(metadataDir);
+        Path jobDir = metadataDir.resolve("modify_settings_server_ftp");
+        Files.createDirectories(jobDir);
+
+        Files.writeString(jobDir.resolve(SETTINGS_YAML), "name: \"modify_settings_server_ftp\"\n" +
+                        "server:\n" +
+                        "  hostname: \"mynode.mydomain.com\"\n" +
+                        "  protocol: \"ftp\""
+                );
+        FsSettings settings = fsSettingsFileHandler.read("modify_settings_server_ftp");
+        assertThat(settings.getFs(), nullValue());
+        assertThat(settings.getElasticsearch(), nullValue());
+        assertThat(settings.getServer(), notNullValue());
+        assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.SSH_PORT));
+        assertThat(settings.getServer().getUsername(), nullValue());
+        modifySettings(settings, "elastic");
+        assertThat(settings.getFs(), notNullValue());
+        assertThat(settings.getElasticsearch(), notNullValue());
+        assertThat(settings.getElasticsearch().getUsername(), is("elastic"));
+        assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.FTP_PORT));
+        assertThat(settings.getServer().getUsername(), is("anonymous"));
+    }
+
+    @Test
+    public void testModifySettingsWithServerSsh() throws IOException {
+        FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(metadataDir);
+        Path jobDir = metadataDir.resolve("modify_settings_server_ssh");
+        Files.createDirectories(jobDir);
+
+        Files.writeString(jobDir.resolve(SETTINGS_YAML), "name: \"modify_settings_server_ssh\"\n" +
+                        "server:\n" +
+                        "  hostname: \"mynode.mydomain.com\"\n" +
+                        "  protocol: \"ssh\""
+                );
+        FsSettings settings = fsSettingsFileHandler.read("modify_settings_server_ssh");
+        assertThat(settings.getFs(), nullValue());
+        assertThat(settings.getElasticsearch(), nullValue());
+        assertThat(settings.getServer(), notNullValue());
+        assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.SSH_PORT));
+        assertThat(settings.getServer().getUsername(), nullValue());
+        modifySettings(settings, "elastic");
+        assertThat(settings.getFs(), notNullValue());
+        assertThat(settings.getElasticsearch(), notNullValue());
+        assertThat(settings.getElasticsearch().getUsername(), is("elastic"));
+        assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.SSH_PORT));
+        assertThat(settings.getServer().getUsername(), nullValue());
+    }
+}