From cb1b14192049d9fcf3f8a34af2468b71ca127f91 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 20 Oct 2023 16:17:12 +0200 Subject: [PATCH] Don't hide console logs when running with Docker (#1738) * Refactor the CLI We can add more tests and make it a bit more flexible... * Add a basic docker compose example without Elastic It helps to test the behavior of the container when we don't have any input. Related to #1727 * Don't hide console logs when running with Docker I believe it was a mistake doing this. It's better to let the user control the level of logs they want when running the container. In case of error, it now displays correctly the logs in the docker compose output. Closes #1727. --- .../crawler/fs/cli/FsCrawlerCli.java | 279 ++++++++++++------ cli/src/main/resources/log4j2-file.xml | 6 + .../fs/cli/FsCrawlerCliCommandParserTest.java | 98 ++++++ .../cli/FsCrawlerCliDefaultSettingsTest.java | 157 ++++++++++ .../fs/cli/FsCrawlerCliLoggerTest.java | 182 ++++++++++++ contrib/docker-compose-example-fscrawler/.env | 5 + .../config/idx/_settings.yaml | 5 + .../docker-compose.yml | 15 + .../docker-compose-example-fscrawler/.env | 5 + 9 files changed, 655 insertions(+), 97 deletions(-) create mode 100644 cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliCommandParserTest.java create mode 100644 cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliDefaultSettingsTest.java create mode 100644 cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliLoggerTest.java create mode 100644 contrib/docker-compose-example-fscrawler/.env create mode 100644 contrib/docker-compose-example-fscrawler/config/idx/_settings.yaml create mode 100644 contrib/docker-compose-example-fscrawler/docker-compose.yml create mode 100644 contrib/src/main/resources/docker-compose-example-fscrawler/.env diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java index 7ad21d820..b4c1c07da 100644 --- a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java +++ b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java @@ -24,6 +24,7 @@ import fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl; import fr.pilato.elasticsearch.crawler.fs.beans.FsJobFileHandler; import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerIllegalConfigurationException; import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.framework.MetaFileHandler; import fr.pilato.elasticsearch.crawler.fs.framework.Version; @@ -47,6 +48,7 @@ import org.apache.logging.log4j.core.filter.LevelMatchFilter; import org.apache.logging.log4j.core.filter.LevelRangeFilter; +import java.io.Console; import java.io.IOException; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -71,32 +73,32 @@ public static class FsCrawlerCommand { List jobName; @Parameter(names = "--config_dir", description = "Config directory. Default to ~/.fscrawler") - private String configDir = null; + String configDir = null; @Parameter(names = "--username", description = "Elasticsearch username when running with security.") - private String username = null; + String username = null; @Parameter(names = "--loop", description = "Number of scan loop before exiting.") - private Integer loop = -1; + Integer loop = -1; @Parameter(names = "--restart", description = "Restart fscrawler job like if it never ran before. " + "This does not clean elasticsearch indices.") - private boolean restart = false; + boolean restart = false; @Parameter(names = "--rest", description = "Start REST Layer") - private boolean rest = false; + boolean rest = false; @Parameter(names = "--upgrade", description = "Upgrade elasticsearch indices from one old version to the last version.") - private boolean upgrade = false; + boolean upgrade = false; @Parameter(names = "--debug", description = "Debug mode") - private boolean debug = false; + boolean debug = false; @Parameter(names = "--trace", description = "Trace mode") - private boolean trace = false; + boolean trace = false; @Parameter(names = "--silent", description = "Silent mode") - private boolean silent = false; + boolean silent = false; @Parameter(names = "--help", description = "display current help", help = true) boolean help; @@ -104,28 +106,52 @@ public static class FsCrawlerCommand { public static void main(String[] args) throws Exception { - // create a scanner so we can read the command-line input - Scanner scanner = new Scanner(System.in); + FsCrawlerCommand command = commandParser(args); + + if (command != null) { + // We change the log level if needed + changeLoggerContext(command); + + // Display the welcome banner + banner(); + + // We can now launch the crawler + runner(command); + } + } + static FsCrawlerCommand commandParser(String[] args) { FsCrawlerCommand commands = new FsCrawlerCommand(); JCommander jCommander = new JCommander(commands); jCommander.parse(args); + // Check the expected parameters when in silent mode + if (commands.silent) { + if (commands.jobName == null) { + banner(); + logger.warn("--silent is set but no job has been defined. Add a job name or remove --silent option. Exiting."); + jCommander.usage(); + throw new FsCrawlerIllegalConfigurationException("No job specified while in silent mode."); + } + } + + if (commands.help) { + jCommander.usage(); + return null; + } + + return commands; + } + + static void changeLoggerContext(FsCrawlerCommand command) { // Change debug level if needed - if (commands.debug || commands.trace || commands.silent) { + if (command.debug || command.trace || command.silent) { LoggerContext ctx = (LoggerContext) LogManager.getContext(false); Configuration config = ctx.getConfiguration(); LoggerConfig loggerConfig = config.getLoggerConfig("fr.pilato.elasticsearch.crawler.fs"); ConsoleAppender console = config.getAppender("Console"); - if (commands.silent) { - // If the user did not enter any job name, nothing will be displayed - if (commands.jobName == null) { - banner(); - logger.warn("--silent is set but no job has been defined. Add a job name or remove --silent option. Exiting."); - jCommander.usage(); - return; - } + if (command.silent) { // We don't write anything on the console anymore if (console != null) { console.addFilter(LevelMatchFilter.newBuilder().setLevel(Level.ALL).setOnMatch(Filter.Result.DENY).build()); @@ -133,32 +159,128 @@ public static void main(String[] args) throws Exception { } else { if (console != null) { console.addFilter(LevelRangeFilter.createFilter( - commands.debug ? Level.TRACE : Level.ALL, + command.debug ? Level.TRACE : Level.ALL, Level.ALL, Filter.Result.DENY, Filter.Result.ACCEPT)); } } - loggerConfig.setLevel(commands.debug ? Level.DEBUG : Level.TRACE); + loggerConfig.setLevel(command.debug ? Level.DEBUG : Level.TRACE); ctx.updateLoggers(); } + } - banner(); + /** + * Reinit the logger context to remove all filters + */ + static void reinitLoggerContext() { + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration config = ctx.getConfiguration(); + ConsoleAppender console = config.getAppender("Console"); + if (console != null) { + Filter filter = console.getFilter(); + console.removeFilter(filter); + } - if (commands.help) { - jCommander.usage(); - return; + LoggerConfig loggerConfig = config.getLoggerConfig("fr.pilato.elasticsearch.crawler.fs"); + if (loggerConfig != null) { + loggerConfig.setLevel(Level.INFO); + } + } + + /** + * Load settings from the given directory and job name + * @param configDir the config dir + * @param jobName the job name + * @return the settings if found, null otherwise + * @throws IOException In case of IO problem + */ + static FsSettings loadSettings(Path configDir, String jobName) throws IOException { + try { + return new FsSettingsFileHandler(configDir).read(jobName); + } catch (NoSuchFileException e) { + return null; + } + } + + /** + * Modify existing settings with correct default values when not set. + * + * @param fsSettings the settings to modify + * @param usernameCli the username coming from the CLI if any + */ + static void modifySettings(FsSettings fsSettings, String usernameCli) { + // Check default settings + if (fsSettings.getFs() == null) { + fsSettings.setFs(Fs.DEFAULT); + } + + if (fsSettings.getServer() != null) { + // It's a dirty hack. The default port if not set is 22 (SSH), but if protocol is FTP, we should use 21 as a default port + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && fsSettings.getServer().getPort() == PROTOCOL.SSH_PORT) { + fsSettings.getServer().setPort(PROTOCOL.FTP_PORT); + } + // For FTP, we set a default username if not set + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { + fsSettings.getServer().setUsername("anonymous"); + } + } + + if (fsSettings.getElasticsearch() == null) { + fsSettings.setElasticsearch(Elasticsearch.DEFAULT()); + } + + // Overwrite settings with command line values + if (fsSettings.getElasticsearch().getUsername() == null && usernameCli != null) { + fsSettings.getElasticsearch().setUsername(usernameCli); + } + } + + /** + * Create a job if needed + * @param jobName the job name + * @param configDir the config dir + * @param scanner the scanner to read user input + * @throws IOException In case of IO problem + */ + static void createJob(String jobName, Path configDir, Scanner scanner) throws IOException { + FSCrawlerLogger.console("job [{}] does not exist", jobName); + + String yesno = null; + while (!"y".equalsIgnoreCase(yesno) && !"n".equalsIgnoreCase(yesno)) { + FSCrawlerLogger.console("Do you want to create it (Y/N)?"); + yesno = scanner.next(); + } + + if ("y".equalsIgnoreCase(yesno)) { + FsSettings fsSettings = FsSettings.builder(jobName) + .setFs(Fs.DEFAULT) + .setElasticsearch(Elasticsearch.DEFAULT()) + .build(); + new FsSettingsFileHandler(configDir).write(fsSettings); + + Path config = configDir.resolve(jobName).resolve(FsSettingsFileHandler.SETTINGS_YAML); + FSCrawlerLogger.console("Settings have been created in [{}]. Please review and edit before relaunch", config); + } + } + + static void runner(FsCrawlerCommand command) throws IOException { + // create a scanner so we can read the command-line input + Console con = System.console(); + Scanner scanner = null; + if (con != null) { + scanner = new Scanner(con.reader()); } BootstrapChecks.check(); Path configDir; - if (commands.configDir == null) { + if (command.configDir == null) { configDir = MetaFileHandler.DEFAULT_ROOT; } else { - configDir = Paths.get(commands.configDir); + configDir = Paths.get(command.configDir); } // Create the config dir if needed @@ -168,11 +290,15 @@ public static void main(String[] args) throws Exception { copyDefaultResources(configDir); FsSettings fsSettings; - FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(configDir); String jobName; - if (commands.jobName == null) { + if (command.jobName == null) { + if (scanner == null) { + logger.error("No job specified. Exiting."); + System.exit(1); + } + // The user did not enter a job name. // We can list available jobs for him FSCrawlerLogger.console("No job specified. Here is the list of existing jobs:"); @@ -196,82 +322,41 @@ public static void main(String[] args) throws Exception { } } else { - jobName = commands.jobName.get(0); + jobName = command.jobName.get(0); } // If we ask to reinit, we need to clean the status for the job - if (commands.restart) { + if (command.restart) { logger.debug("Cleaning existing status for job [{}]...", jobName); new FsJobFileHandler(configDir).clean(jobName); } - try { - logger.debug("Starting job [{}]...", jobName); - fsSettings = fsSettingsFileHandler.read(jobName); - - // Check default settings - if (fsSettings.getFs() == null) { - fsSettings.setFs(Fs.DEFAULT); - } + logger.debug("Starting job [{}]...", jobName); + fsSettings = loadSettings(configDir, jobName); - if (fsSettings.getServer() != null) { - if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && fsSettings.getServer().getPort() == PROTOCOL.SSH_PORT) { - fsSettings.getServer().setPort(PROTOCOL.FTP_PORT); - } - if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { - fsSettings.getServer().setUsername("anonymous"); - } - } - - if (fsSettings.getElasticsearch() == null) { - fsSettings.setElasticsearch(Elasticsearch.DEFAULT()); - } - - String username = commands.username; - if (fsSettings.getElasticsearch().getUsername() != null) { - username = fsSettings.getElasticsearch().getUsername(); - } - - if (username != null && fsSettings.getElasticsearch().getPassword() == null) { - FSCrawlerLogger.console("Password for {}:", username); - String password = scanner.next(); - fsSettings.getElasticsearch().setUsername(username); - fsSettings.getElasticsearch().setPassword(password); - } - - } catch (NoSuchFileException e) { + if (fsSettings == null) { + logger.debug("job [{}] does not exist.", jobName); // We can only have a dialog with the end user if we are not silent - if (commands.silent) { - logger.error("job [{}] does not exist. Exiting as we are in silent mode.", jobName); - return; - } - - FSCrawlerLogger.console("job [{}] does not exist", jobName); - - String yesno = null; - while (!"y".equalsIgnoreCase(yesno) && !"n".equalsIgnoreCase(yesno)) { - FSCrawlerLogger.console("Do you want to create it (Y/N)?"); - yesno = scanner.next(); - } - - if ("y".equalsIgnoreCase(yesno)) { - fsSettings = FsSettings.builder(commands.jobName.get(0)) - .setFs(Fs.DEFAULT) - .setElasticsearch(Elasticsearch.DEFAULT()) - .build(); - fsSettingsFileHandler.write(fsSettings); - - Path config = configDir.resolve(jobName).resolve(FsSettingsFileHandler.SETTINGS_YAML); - FSCrawlerLogger.console("Settings have been created in [{}]. Please review and edit before relaunch", config); + if (command.silent || scanner == null) { + logger.error("job [{}] does not exist. Exiting as we are in silent mode or no input available.", jobName); + System.exit(2); } + createJob(jobName, configDir, scanner); return; } + modifySettings(fsSettings, command.username); + if (fsSettings.getElasticsearch().getUsername() != null && fsSettings.getElasticsearch().getPassword() == null && scanner != null) { + FSCrawlerLogger.console("Password for {}:", fsSettings.getElasticsearch().getUsername()); + String password = scanner.next(); + fsSettings.getElasticsearch().setPassword(password); + } + if (logger.isTraceEnabled()) { logger.trace("settings used for this crawler: [{}]", FsSettingsParser.toYaml(fsSettings)); } - if (FsCrawlerValidator.validateSettings(logger, fsSettings, commands.rest)) { + if (FsCrawlerValidator.validateSettings(logger, fsSettings, command.rest)) { // We don't go further as we have critical errors return; } @@ -280,19 +365,19 @@ public static void main(String[] args) throws Exception { if (fsSettings.getWorkplaceSearch() != null) { logger.info("Workplace Search integration is an experimental feature. " + "As is it is not fully implemented and settings might change in the future."); - if (commands.loop == -1 || commands.loop > 1) { + if (command.loop == -1 || command.loop > 1) { logger.warn("Workplace Search integration does not support yet watching a directory. " + - "It will be able to run only once and exit. We manually force from --loop {} to --loop 1. " + - "If you want to remove this message next time, please start FSCrawler with --loop 1", - commands.loop); - commands.loop = 1; + "It will be able to run only once and exit. We manually force from --loop {} to --loop 1. " + + "If you want to remove this message next time, please start FSCrawler with --loop 1", + command.loop); + command.loop = 1; } } - try (FsCrawlerImpl fsCrawler = new FsCrawlerImpl(configDir, fsSettings, commands.loop, commands.rest)) { + try (FsCrawlerImpl fsCrawler = new FsCrawlerImpl(configDir, fsSettings, command.loop, command.rest)) { Runtime.getRuntime().addShutdownHook(new FSCrawlerShutdownHook(fsCrawler)); // Let see if we want to upgrade an existing cluster to the latest version - if (commands.upgrade) { + if (command.upgrade) { logger.info("Upgrading job [{}]. No rule implemented. Skipping.", jobName); } else { if (!startEsClient(fsCrawler)) { @@ -302,7 +387,7 @@ public static void main(String[] args) throws Exception { checkForDeprecatedResources(configDir, elasticsearchVersion); // Start the REST Server if needed - if (commands.rest) { + if (command.rest) { RestServer.start(fsSettings, fsCrawler.getManagementService(), fsCrawler.getDocumentService()); } diff --git a/cli/src/main/resources/log4j2-file.xml b/cli/src/main/resources/log4j2-file.xml index 65dea2af0..003a380de 100644 --- a/cli/src/main/resources/log4j2-file.xml +++ b/cli/src/main/resources/log4j2-file.xml @@ -10,6 +10,10 @@ + + + + @@ -33,6 +37,7 @@ + @@ -43,6 +48,7 @@ + diff --git a/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliCommandParserTest.java b/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliCommandParserTest.java new file mode 100644 index 000000000..963606fc0 --- /dev/null +++ b/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliCommandParserTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.cli; + +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerIllegalConfigurationException; +import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.nio.file.Path; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +/** + * We want to test FSCrawler main app + */ +public class FsCrawlerCliCommandParserTest extends AbstractFSCrawlerTestCase { + + private static Path metadataDir; + + @BeforeClass + public static void createFsCrawlerJobDir() { + metadataDir = rootTmpDir.resolve(".fscrawler"); + } + + @Test + public void testCommandParserWithFullOptions() { + String[] args = { + "--config_dir", metadataDir.toString(), + "--loop", "0", + "--username", "dadoonet", + "--rest", + "--upgrade", + "--restart", + "--debug", + "jobName" + }; + FsCrawlerCli.FsCrawlerCommand command = FsCrawlerCli.commandParser(args); + assertThat(command, notNullValue()); + assertThat(command.configDir, is(metadataDir.toString())); + assertThat(command.loop, is(0)); + assertThat(command.username, is("dadoonet")); + assertThat(command.rest, is(true)); + assertThat(command.upgrade, is(true)); + assertThat(command.restart, is(true)); + assertThat(command.debug, is(true)); + assertThat(command.trace, is(false)); + assertThat(command.silent, is(false)); + assertThat(command.jobName.get(0), is("jobName")); + } + + @Test + public void testCommandParserForHelp() { + String[] args = { + "--help" + }; + FsCrawlerCli.FsCrawlerCommand command = FsCrawlerCli.commandParser(args); + assertThat(command, nullValue()); + } + + @Test(expected = FsCrawlerIllegalConfigurationException.class) + public void testCommandParserSilentModeNoJob() { + String[] args = { + "--silent" + }; + FsCrawlerCli.commandParser(args); + } + + @Test + public void testCommandParserSilentModeWithJob() { + String[] args = { + "--silent", + "jobName" + }; + FsCrawlerCli.FsCrawlerCommand command = FsCrawlerCli.commandParser(args); + assertThat(command, notNullValue()); + assertThat(command.silent, is(true)); + assertThat(command.jobName.get(0), is("jobName")); + } +} diff --git a/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliDefaultSettingsTest.java b/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliDefaultSettingsTest.java new file mode 100644 index 000000000..9e905514f --- /dev/null +++ b/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliDefaultSettingsTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.cli; + +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsFileHandler; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static fr.pilato.elasticsearch.crawler.fs.cli.FsCrawlerCli.modifySettings; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.copyDefaultResources; +import static fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsFileHandler.SETTINGS_YAML; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +/** + * We want to test FSCrawler main app + */ +public class FsCrawlerCliDefaultSettingsTest extends AbstractFSCrawlerTestCase { + + private static Path metadataDir; + + @BeforeClass + public static void createFsCrawlerJobDir() throws IOException { + // We also need to create default mapping files + metadataDir = rootTmpDir.resolve(".fscrawler"); + if (Files.notExists(metadataDir)) { + Files.createDirectory(metadataDir); + } + copyDefaultResources(metadataDir); + staticLogger.debug(" --> Test metadata dir ready in [{}]", metadataDir); + } + + @AfterClass + public static void printMetadataDirContent() throws IOException { + printLs(metadataDir); + } + + private static void printLs(Path dir) throws IOException { + staticLogger.debug("ls -l {}", dir); + Files.list(dir).forEach(path -> { + if (Files.isDirectory(path)) { + try { + printLs(path); + } catch (IOException ignored) { } + } else { + staticLogger.debug("{}", path); + } + }); + } + + @Test + public void testModifySettingsNoUsername() throws IOException { + FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(metadataDir); + Path jobDir = metadataDir.resolve("modify_settings_no_username"); + Files.createDirectories(jobDir); + + Files.writeString(jobDir.resolve(SETTINGS_YAML), "name: \"modify_settings_no_username\""); + FsSettings settings = fsSettingsFileHandler.read("modify_settings_no_username"); + assertThat(settings.getFs(), nullValue()); + assertThat(settings.getElasticsearch(), nullValue()); + modifySettings(settings, null); + assertThat(settings.getFs(), notNullValue()); + assertThat(settings.getElasticsearch(), notNullValue()); + assertThat(settings.getElasticsearch().getUsername(), nullValue()); + } + + @Test + public void testModifySettingsWithUsername() throws IOException { + FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(metadataDir); + Path jobDir = metadataDir.resolve("modify_settings_with_username"); + Files.createDirectories(jobDir); + + Files.writeString(jobDir.resolve(SETTINGS_YAML), "name: \"modify_settings_with_username\""); + FsSettings settings = fsSettingsFileHandler.read("modify_settings_with_username"); + assertThat(settings.getFs(), nullValue()); + assertThat(settings.getElasticsearch(), nullValue()); + modifySettings(settings, "elastic"); + assertThat(settings.getFs(), notNullValue()); + assertThat(settings.getElasticsearch(), notNullValue()); + assertThat(settings.getElasticsearch().getUsername(), is("elastic")); + } + + @Test + public void testModifySettingsWithServerFtp() throws IOException { + FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(metadataDir); + Path jobDir = metadataDir.resolve("modify_settings_server_ftp"); + Files.createDirectories(jobDir); + + Files.writeString(jobDir.resolve(SETTINGS_YAML), "name: \"modify_settings_server_ftp\"\n" + + "server:\n" + + " hostname: \"mynode.mydomain.com\"\n" + + " protocol: \"ftp\"" + ); + FsSettings settings = fsSettingsFileHandler.read("modify_settings_server_ftp"); + assertThat(settings.getFs(), nullValue()); + assertThat(settings.getElasticsearch(), nullValue()); + assertThat(settings.getServer(), notNullValue()); + assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.SSH_PORT)); + assertThat(settings.getServer().getUsername(), nullValue()); + modifySettings(settings, "elastic"); + assertThat(settings.getFs(), notNullValue()); + assertThat(settings.getElasticsearch(), notNullValue()); + assertThat(settings.getElasticsearch().getUsername(), is("elastic")); + assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.FTP_PORT)); + assertThat(settings.getServer().getUsername(), is("anonymous")); + } + + @Test + public void testModifySettingsWithServerSsh() throws IOException { + FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(metadataDir); + Path jobDir = metadataDir.resolve("modify_settings_server_ssh"); + Files.createDirectories(jobDir); + + Files.writeString(jobDir.resolve(SETTINGS_YAML), "name: \"modify_settings_server_ssh\"\n" + + "server:\n" + + " hostname: \"mynode.mydomain.com\"\n" + + " protocol: \"ssh\"" + ); + FsSettings settings = fsSettingsFileHandler.read("modify_settings_server_ssh"); + assertThat(settings.getFs(), nullValue()); + assertThat(settings.getElasticsearch(), nullValue()); + assertThat(settings.getServer(), notNullValue()); + assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.SSH_PORT)); + assertThat(settings.getServer().getUsername(), nullValue()); + modifySettings(settings, "elastic"); + assertThat(settings.getFs(), notNullValue()); + assertThat(settings.getElasticsearch(), notNullValue()); + assertThat(settings.getElasticsearch().getUsername(), is("elastic")); + assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.SSH_PORT)); + assertThat(settings.getServer().getUsername(), nullValue()); + } +} diff --git a/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliLoggerTest.java b/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliLoggerTest.java new file mode 100644 index 000000000..8a3a02356 --- /dev/null +++ b/cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliLoggerTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.cli; + +import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.ConsoleAppender; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.filter.LevelMatchFilter; +import org.apache.logging.log4j.core.filter.LevelRangeFilter; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static fr.pilato.elasticsearch.crawler.fs.cli.FsCrawlerCli.*; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.copyDefaultResources; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +/** + * We want to test FSCrawler main app + */ +public class FsCrawlerCliLoggerTest extends AbstractFSCrawlerTestCase { + + private static Path metadataDir; + + @BeforeClass + public static void createFsCrawlerJobDir() throws IOException { + // We also need to create default mapping files + metadataDir = rootTmpDir.resolve(".fscrawler"); + if (Files.notExists(metadataDir)) { + Files.createDirectory(metadataDir); + } + copyDefaultResources(metadataDir); + staticLogger.debug(" --> Test metadata dir ready in [{}]", metadataDir); + } + + @AfterClass + public static void printMetadataDirContent() throws IOException { + printLs(metadataDir); + } + + private static void printLs(Path dir) throws IOException { + staticLogger.debug("ls -l {}", dir); + Files.list(dir).forEach(path -> { + if (Files.isDirectory(path)) { + try { + printLs(path); + } catch (IOException ignored) { } + } else { + staticLogger.debug("{}", path); + } + }); + } + + /** + * We want to make sure that we can run several times the same test + * without having any error due to the fact that the logger context + * has already been initialized. + */ + @Before + public void resetLogger() { + reinitLoggerContext(); + } + + @Test + public void testChangeLoggerContextForDebug() { + String[] args = { + "--debug", + "jobName" + }; + FsCrawlerCommand command = FsCrawlerCli.commandParser(args); + assertThat(command, notNullValue()); + changeLoggerContext(command); + + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration config = ctx.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig("fr.pilato.elasticsearch.crawler.fs"); + assertThat(loggerConfig, notNullValue()); + assertThat(loggerConfig.getLevel(), is(Level.DEBUG)); + + ConsoleAppender console = config.getAppender("Console"); + assertThat(console, notNullValue()); + assertThat(console.getFilter(), instanceOf(LevelRangeFilter.class)); + LevelRangeFilter filter = (LevelRangeFilter) console.getFilter(); + assertThat(filter.getMinLevel(), is(Level.TRACE)); + assertThat(filter.getMaxLevel(), is(Level.ALL)); + } + + @Test + public void testChangeLoggerContextForTrace() { + String[] args = { + "--trace", + "jobName" + }; + FsCrawlerCommand command = FsCrawlerCli.commandParser(args); + assertThat(command, notNullValue()); + changeLoggerContext(command); + + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration config = ctx.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig("fr.pilato.elasticsearch.crawler.fs"); + assertThat(loggerConfig, notNullValue()); + assertThat(loggerConfig.getLevel(), is(Level.TRACE)); + + ConsoleAppender console = config.getAppender("Console"); + assertThat(console, notNullValue()); + assertThat(console.getFilter(), instanceOf(LevelRangeFilter.class)); + LevelRangeFilter filter = (LevelRangeFilter) console.getFilter(); + assertThat(filter.getMinLevel(), is(Level.ALL)); + assertThat(filter.getMaxLevel(), is(Level.ALL)); + } + + @Test + public void testChangeLoggerContextForSilent() { + String[] args = { + "--silent", + "jobName" + }; + FsCrawlerCommand command = FsCrawlerCli.commandParser(args); + assertThat(command, notNullValue()); + changeLoggerContext(command); + + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration config = ctx.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig("fr.pilato.elasticsearch.crawler.fs"); + assertThat(loggerConfig, notNullValue()); + assertThat(loggerConfig.getLevel(), is(Level.TRACE)); + + ConsoleAppender console = config.getAppender("Console"); + assertThat(console, notNullValue()); + assertThat(console.getFilter(), instanceOf(LevelMatchFilter.class)); + LevelMatchFilter filter = (LevelMatchFilter) console.getFilter(); + assertThat(filter.getOnMatch(), is(Filter.Result.DENY)); + } + + @Test + public void testChangeLoggerContextByDefault() { + String[] args = { + "jobName" + }; + FsCrawlerCommand command = FsCrawlerCli.commandParser(args); + assertThat(command, notNullValue()); + changeLoggerContext(command); + + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration config = ctx.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig("fr.pilato.elasticsearch.crawler.fs"); + assertThat(loggerConfig, notNullValue()); + assertThat(loggerConfig.getLevel(), is(Level.INFO)); + + ConsoleAppender console = config.getAppender("Console"); + assertThat(console, notNullValue()); + assertThat(console.getFilter(), nullValue()); + } +} diff --git a/contrib/docker-compose-example-fscrawler/.env b/contrib/docker-compose-example-fscrawler/.env new file mode 100644 index 000000000..6c3195f32 --- /dev/null +++ b/contrib/docker-compose-example-fscrawler/.env @@ -0,0 +1,5 @@ +# THIS FILE IS AUTOMATICALLY GENERATED FROM /contrib/src/main/resources/xxx DIR. + +# FSCrawler Settings +FSCRAWLER_VERSION=2.10-SNAPSHOT +FSCRAWLER_PORT=8080 diff --git a/contrib/docker-compose-example-fscrawler/config/idx/_settings.yaml b/contrib/docker-compose-example-fscrawler/config/idx/_settings.yaml new file mode 100644 index 000000000..5e00f6001 --- /dev/null +++ b/contrib/docker-compose-example-fscrawler/config/idx/_settings.yaml @@ -0,0 +1,5 @@ +--- +name: "idx" +rest : + url: "http://fscrawler:8080" + diff --git a/contrib/docker-compose-example-fscrawler/docker-compose.yml b/contrib/docker-compose-example-fscrawler/docker-compose.yml new file mode 100644 index 000000000..e22b09bb5 --- /dev/null +++ b/contrib/docker-compose-example-fscrawler/docker-compose.yml @@ -0,0 +1,15 @@ +--- +version: "2.2" + +services: + fscrawler: + image: dadoonet/fscrawler:$FSCRAWLER_VERSION + container_name: fscrawler + volumes: + - ../../test-documents/src/main/resources/documents/:/tmp/es:ro + - ${PWD}/config:/root/.fscrawler + - ${PWD}/logs:/usr/share/fscrawler/logs + ports: + - ${FSCRAWLER_PORT}:8080 + command: fscrawler idx --rest + diff --git a/contrib/src/main/resources/docker-compose-example-fscrawler/.env b/contrib/src/main/resources/docker-compose-example-fscrawler/.env new file mode 100644 index 000000000..b4f1bbb67 --- /dev/null +++ b/contrib/src/main/resources/docker-compose-example-fscrawler/.env @@ -0,0 +1,5 @@ +# THIS FILE IS AUTOMATICALLY GENERATED FROM /contrib/src/main/resources/xxx DIR. + +# FSCrawler Settings +FSCRAWLER_VERSION=${project.version} +FSCRAWLER_PORT=8080