diff --git a/crawler/crawler-ssh/pom.xml b/crawler/crawler-ssh/pom.xml
index 4448a6208..b6be3e21f 100644
--- a/crawler/crawler-ssh/pom.xml
+++ b/crawler/crawler-ssh/pom.xml
@@ -18,10 +18,10 @@
fscrawler-crawler-abstract
-
+
- com.jcraft
- jsch
+ org.apache.sshd
+ sshd-sftp
diff --git a/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java b/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java
index 5047a0233..d6728c117 100644
--- a/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java
+++ b/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java
@@ -19,11 +19,6 @@
package fr.pilato.elasticsearch.crawler.fs.crawler.ssh;
-import com.jcraft.jsch.Channel;
-import com.jcraft.jsch.ChannelSftp;
-import com.jcraft.jsch.JSch;
-import com.jcraft.jsch.JSchException;
-import com.jcraft.jsch.Session;
import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel;
import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor;
import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings;
@@ -32,48 +27,56 @@
import org.apache.commons.io.FilenameUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.apache.sshd.client.SshClient;
+import org.apache.sshd.client.keyverifier.AcceptAllServerKeyVerifier;
+import org.apache.sshd.client.session.ClientSession;
+import org.apache.sshd.common.keyprovider.FileKeyPairProvider;
+import org.apache.sshd.sftp.client.SftpClient;
+import org.apache.sshd.sftp.client.SftpClientFactory;
import java.io.InputStream;
-import java.time.Instant;
+import java.nio.file.Paths;
+import java.security.KeyPair;
import java.time.LocalDateTime;
import java.time.ZoneId;
-import java.util.ArrayList;
import java.util.Collection;
-import java.util.Vector;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
-public class FileAbstractorSSH extends FileAbstractor {
+public class FileAbstractorSSH extends FileAbstractor {
private final Logger logger = LogManager.getLogger(FileAbstractorSSH.class);
- private ChannelSftp sftp;
+ private ClientSession session;
+ private SshClient sshClient;
+ private SftpClient sftpClient;
public FileAbstractorSSH(FsSettings fsSettings) {
super(fsSettings);
}
@Override
- public FileAbstractModel toFileAbstractModel(String path, ChannelSftp.LsEntry file) {
+ public FileAbstractModel toFileAbstractModel(String path, SftpClient.DirEntry file) {
return new FileAbstractModel(
file.getFilename(),
- !file.getAttrs().isDir(),
+ !file.getAttributes().isDirectory(),
// We are using here the local TimeZone as a reference. If the remote system is under another TZ, this might cause issues
- LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getAttrs().getMTime()*1000L), ZoneId.systemDefault()),
+ LocalDateTime.ofInstant(file.getAttributes().getModifyTime().toInstant(), ZoneId.systemDefault()),
// We don't have the creation date
null,
// We are using here the local TimeZone as a reference. If the remote system is under another TZ, this might cause issues
- LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getAttrs().getATime()*1000L), ZoneId.systemDefault()),
+ LocalDateTime.ofInstant(file.getAttributes().getAccessTime().toInstant(), ZoneId.systemDefault()),
FilenameUtils.getExtension(file.getFilename()),
path,
path.equals("/") ? path.concat(file.getFilename()) : path.concat("/").concat(file.getFilename()),
- file.getAttrs().getSize(),
- Integer.toString(file.getAttrs().getUId()),
- Integer.toString(file.getAttrs().getGId()),
- file.getAttrs().getPermissions());
+ file.getAttributes().getSize(),
+ Integer.toString(file.getAttributes().getUserId()),
+ Integer.toString(file.getAttributes().getGroupId()),
+ file.getAttributes().getPermissions());
}
@Override
public InputStream getInputStream(FileAbstractModel file) throws Exception {
- return sftp.get(file.getFullpath());
+ return sftpClient.read(file.getFullpath());
}
@Override
@@ -81,22 +84,23 @@ public void closeInputStream(InputStream inputStream) throws IOException {
inputStream.close();
}
- @SuppressWarnings("unchecked")
@Override
public Collection getFiles(String dir) throws Exception {
logger.debug("Listing local files from {}", dir);
- Vector ls;
- ls = sftp.ls(dir);
- if (ls == null) return null;
+ Iterable ls;
- Collection result = new ArrayList<>(ls.size());
- // Iterate other files
- // We ignore here all files like . and ..
- result.addAll(ls.stream().filter(file -> !".".equals(file.getFilename()) &&
- !"..".equals(file.getFilename()))
+ ls = sftpClient.readDir(dir);
+
+ /*
+ Iterate other files
+ We ignore here all files like "." and ".."
+ */
+ Collection result = StreamSupport.stream(ls.spliterator(), false)
+ .filter(file -> !".".equals(file.getFilename()) &&
+ !"..".equals(file.getFilename()))
.map(file -> toFileAbstractModel(dir, file))
- .collect(Collectors.toList()));
+ .collect(Collectors.toList());
logger.debug("{} local files found", result.size());
return result;
@@ -105,7 +109,7 @@ public Collection getFiles(String dir) throws Exception {
@Override
public boolean exists(String dir) {
try {
- sftp.ls(dir);
+ sftpClient.readDir(dir);
} catch (Exception e) {
return false;
}
@@ -114,51 +118,62 @@ public boolean exists(String dir) {
@Override
public void open() throws Exception {
- sftp = openSSHConnection(fsSettings.getServer());
+ sshClient = createSshClient();
+ session = openSshSession(sshClient, fsSettings.getServer());
+ sftpClient = createSftpClient(session);
}
@Override
public void close() throws Exception {
- if (sftp != null) {
- sftp.getSession().disconnect();
- sftp.disconnect();
+ if (sshClient != null) {
+ if (session != null) {
+ if (sftpClient != null) {
+ logger.debug("Closing SFTP Client");
+ sftpClient.close();
+ }
+ logger.debug("Closing SSH Session");
+ session.close();
+ }
+ logger.debug("Closing SSH Client");
+ sshClient.close();
}
}
- private ChannelSftp openSSHConnection(Server server) throws Exception {
+ private SshClient createSshClient() {
+ logger.debug("Create and start SSH client");
+
+ SshClient client = SshClient.setUpDefaultClient();
+ client.setServerKeyVerifier(AcceptAllServerKeyVerifier.INSTANCE);
+ client.start();
+ return client;
+ }
+
+ private ClientSession openSshSession(SshClient sshClient, Server server) throws Exception {
logger.debug("Opening SSH connection to {}@{}", server.getUsername(), server.getHostname());
- JSch jsch = new JSch();
- Session session = jsch.getSession(server.getUsername(), server.getHostname(), server.getPort());
- java.util.Properties config = new java.util.Properties();
- config.put("StrictHostKeyChecking", "no");
- if (server.getPemPath() != null) {
- jsch.addIdentity(server.getPemPath());
- }
- session.setConfig(config);
+ ClientSession session = sshClient.connect(server.getUsername(), server.getHostname(), server.getPort()).verify().getSession();
+
if (server.getPassword() != null) {
- session.setPassword(server.getPassword());
+ session.addPasswordIdentity(server.getPassword()); // for password-based authentication
}
- try {
- session.connect();
- } catch (JSchException e) {
- logger.warn("Cannot connect with SSH to {}@{}: {}", server.getUsername(),
- server.getHostname(), e.getMessage());
- throw e;
+ if (server.getPemPath() != null) {
+ // for password-less authentication
+ FileKeyPairProvider fileKeyPairProvider = new FileKeyPairProvider(Paths.get(server.getPemPath()));
+ Iterable keyPairs = fileKeyPairProvider.loadKeys(null);
+ for (KeyPair keyPair : keyPairs) {
+ session.addPublicKeyIdentity(keyPair);
+ }
}
- //Open a new session for SFTP.
- Channel channel = session.openChannel("sftp");
- channel.connect();
+ session.auth().verify();
- //checking SSH client connection.
- if (!channel.isConnected()) {
- logger.warn("Cannot connect with SSH to {}@{}", server.getUsername(),
- server.getHostname());
- throw new RuntimeException("Can not connect to " + server.getUsername() + "@" + server.getHostname());
- }
logger.debug("SSH connection successful");
- return (ChannelSftp) channel;
+ return session;
+ }
+
+ private SftpClient createSftpClient(ClientSession session) throws Exception {
+ logger.debug("Create SFTP client");
+ return SftpClientFactory.instance().createSftpClient(session);
}
}
diff --git a/elasticsearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ESBoolQuery.java b/elasticsearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ESBoolQuery.java
index 5aa51e16d..4eb3d0e95 100644
--- a/elasticsearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ESBoolQuery.java
+++ b/elasticsearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ESBoolQuery.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package fr.pilato.elasticsearch.crawler.fs.client;
import java.util.ArrayList;
diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java
index 77658aaaa..7cbfa48b2 100644
--- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java
+++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java
@@ -508,5 +508,4 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx
}
});
}
-
}
diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSshIT.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSshIT.java
index cdbcb8405..a27d89cdb 100644
--- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSshIT.java
+++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestSshIT.java
@@ -23,29 +23,100 @@
import fr.pilato.elasticsearch.crawler.fs.settings.Fs;
import fr.pilato.elasticsearch.crawler.fs.settings.Server;
import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase;
-import org.junit.Ignore;
+import org.apache.sshd.common.config.keys.writer.openssh.OpenSSHKeyPairResourceWriter;
+import org.apache.sshd.server.SshServer;
+import org.apache.sshd.server.config.keys.AuthorizedKeysAuthenticator;
+import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
+import org.apache.sshd.sftp.server.SftpFileSystemAccessor;
+import org.apache.sshd.sftp.server.SftpSubsystemFactory;
+import org.apache.sshd.sftp.server.SftpSubsystemProxy;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.security.*;
+import java.util.Collections;
+
/**
* Test crawler with SSH
*/
public class FsCrawlerTestSshIT extends AbstractFsCrawlerITCase {
- /**
- * You have to adapt this test to your own system (login / password and SSH connexion)
- * So this test is disabled by default
- */
- @Test @Ignore
- public void test_ssh() throws Exception {
- String username = "USERNAME";
- String password = "PASSWORD";
- String hostname = "localhost";
+ private final static String SSH_USERNAME = "USERNAME";
+ private final static String SSH_PASSWORD = "PASSWORD";
+
+ private SshServer sshd = null;
+
+ @Before
+ public void setup() throws IOException, NoSuchAlgorithmException {
+ SftpSubsystemFactory factory = new SftpSubsystemFactory.Builder()
+ .withFileSystemAccessor(new SftpFileSystemAccessor() {
+ @Override
+ public Path resolveLocalFilePath(SftpSubsystemProxy subsystem, Path rootDir, String remotePath) throws InvalidPathException {
+ String path = remotePath;
+ if (remotePath.startsWith("/")) {
+ path = remotePath.substring(1);
+ }
+ return currentTestResourceDir.resolve(path);
+ }
+ })
+ .build();
+
+ // Generate the key files for our SSH tests
+ KeyPair keyPair = KeyPairGenerator.getInstance("RSA").generateKeyPair();
+ saveKeyPair(rootTmpDir, keyPair);
+
+ sshd = SshServer.setUpDefaultServer();
+ sshd.setHost("localhost");
+ sshd.setKeyPairProvider(new SimpleGeneratorHostKeyProvider(rootTmpDir.resolve("host.ser")));
+ sshd.setPasswordAuthenticator((username, password, session) ->
+ SSH_USERNAME.equals(username) && SSH_PASSWORD.equals(password));
+ sshd.setPublickeyAuthenticator(new AuthorizedKeysAuthenticator(rootTmpDir.resolve("public.key")));
+
+ sshd.setSubsystemFactories(Collections.singletonList(factory));
+ sshd.start();
+
+ logger.info(" -> Started fake SSHD service on {}:{}", sshd.getHost(), sshd.getPort());
+ }
+
+ private void saveKeyPair(Path path, KeyPair keyPair) {
+ OpenSSHKeyPairResourceWriter writer = new OpenSSHKeyPairResourceWriter();
+
+ // Store Public Key.
+ try (FileOutputStream fos = new FileOutputStream(path.resolve("public.key").toFile())) {
+ writer.writePublicKey(keyPair.getPublic(), "Public Key for tests", fos);
+ } catch (GeneralSecurityException | IOException e) {
+ logger.error("Failed to save public key", e);
+ }
+
+ // Store Private Key.
+ try (FileOutputStream fos = new FileOutputStream(path.resolve("private.key").toFile())) {
+ writer.writePrivateKey(keyPair, "Private Key for tests", null, fos);
+ } catch (GeneralSecurityException | IOException e) {
+ logger.error("Failed to save public key", e);
+ }
+ }
- Fs fs = startCrawlerDefinition().build();
+ @After
+ public void shutDown() throws IOException {
+ if (sshd != null) {
+ sshd.stop(true);
+ logger.info(" -> Stopped fake SSHD service on {}:{}", sshd.getHost(), sshd.getPort());
+ }
+ }
+
+ @Test
+ public void test_ssh() throws Exception {
+ Fs fs = startCrawlerDefinition("/").build();
Server server = Server.builder()
- .setHostname(hostname)
- .setUsername(username)
- .setPassword(password)
+ .setHostname(sshd.getHost())
+ .setPort(sshd.getPort())
+ .setUsername(SSH_USERNAME)
+ .setPassword(SSH_PASSWORD)
.setProtocol(Server.PROTOCOL.SSH)
.build();
crawler = startCrawler(getCrawlerName(), fs, endCrawlerDefinition(getCrawlerName()), server);
@@ -53,21 +124,14 @@ public void test_ssh() throws Exception {
countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 2L, null);
}
- /**
- * You have to adapt this test to your own system (login / pem file and SSH connexion)
- * So this test is disabled by default
- */
- @Test @Ignore
+ @Test
public void test_ssh_with_key() throws Exception {
- String username = "USERNAME";
- String path_to_pem_file = "/path/to/private_key.pem";
- String hostname = "localhost";
-
- Fs fs = startCrawlerDefinition().build();
+ Fs fs = startCrawlerDefinition("/").build();
Server server = Server.builder()
- .setHostname(hostname)
- .setUsername(username)
- .setPemPath(path_to_pem_file)
+ .setHostname(sshd.getHost())
+ .setPort(sshd.getPort())
+ .setUsername(SSH_USERNAME)
+ .setPemPath(rootTmpDir.resolve("private.key").toFile().getAbsolutePath())
.setProtocol(Server.PROTOCOL.SSH)
.build();
crawler = startCrawler(getCrawlerName(), fs, endCrawlerDefinition(getCrawlerName()), server);
diff --git a/integration-tests/src/test/resources/log4j2.xml b/integration-tests/src/test/resources/log4j2.xml
index e41d37fb7..bcb8b1d94 100644
--- a/integration-tests/src/test/resources/log4j2.xml
+++ b/integration-tests/src/test/resources/log4j2.xml
@@ -60,6 +60,9 @@
+
+
+
diff --git a/pom.xml b/pom.xml
index 44c28b790..9664357a6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -620,11 +620,10 @@
${tika.version}
-
- com.jcraft
- jsch
- 0.1.55
+ org.apache.sshd
+ sshd-sftp
+ 2.13.2
@@ -637,7 +636,6 @@
org.mockftpserver
MockFtpServer
3.2.0
- test
org.slf4j
diff --git a/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCase.java b/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCase.java
index 30ba43b3c..876c438f8 100644
--- a/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCase.java
+++ b/test-framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/framework/AbstractFSCrawlerTestCase.java
@@ -21,6 +21,8 @@
import com.carrotsearch.randomizedtesting.RandomizedContext;
import com.carrotsearch.randomizedtesting.RandomizedRunner;
import com.carrotsearch.randomizedtesting.ThreadFilter;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.*;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import org.apache.logging.log4j.LogManager;
diff --git a/tika/pom.xml b/tika/pom.xml
index 3cab7ef1b..48e05a3cf 100644
--- a/tika/pom.xml
+++ b/tika/pom.xml
@@ -85,12 +85,6 @@
tika-langdetect-optimaize
-
-
- com.jcraft
- jsch
-
-
fr.pilato.elasticsearch.crawler