internal/discofeed-it (#535)

* Disable SSL check for specific discofeed requests

* Fixed checkstyle issues

* Do not throw error in private constructor

* Fixed failing IT - ssl check wasn't disabled

* Ignore integration test which is trying to connect to our private network.

dspace-api/src/test/data/dspaceFolder/config/local.cfg

@@ -223,11 +223,13 @@ featured.service.teitok.description = A web-based platform for viewing, creating
 shibboleth.discofeed.allowed = false
 # File where is DiscoJuiceFeed response
 shibboleth.discofeed.url = TEST:/org/dspace/app/rest/discofeedResponse.json
+# Test connection to the discofeed with disabled SSL certificate validation
+shibboleth.discofeed.url.test.connection = https://dev-5.pc:8443/Shibboleth.sso/DiscoFeed
 # CRON job refresh time definition - default is refresh in every 2 hours.
 discojuice.refresh = 0 */2 * * * ?
 # Comma separated list of entityIDs; we try to guess country on these
 discojuice.rewriteCountries = https://idp.scc.kit.edu/idp/shibboleth, https://fedauth.london.edu/oala/metadata, https://youidlite.youid.net/idp/shibboleth, https://cavle.org/shibboleth
-
+disable.ssl.check.specific.requests = true
 ### Add user to the groups ###
 #attribute -> group mapping
 #check shibboleth attribute ATTR and put users having value ATTR_VALUE1 and ATTR_VALUE2 to GROUP1

dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinDiscoJuiceFeedsDownloadService.java

@@ -25,6 +25,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import javax.net.ssl.HttpsURLConnection;
 import javax.ws.rs.core.NoContentException;
 
 import com.maxmind.geoip2.DatabaseReader;
@@ -33,6 +34,7 @@
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
+import org.dspace.app.rest.utils.ClarinUtils;
 import org.dspace.services.ConfigurationService;
 import org.dspace.utils.DSpace;
 import org.json.simple.JSONArray;
@@ -61,6 +63,8 @@ public class ClarinDiscoJuiceFeedsDownloadService implements InitializingBean {
     private Set<String> rewriteCountries;
     protected static DatabaseReader locationService;
 
+    private static boolean disableSSL = false;
+
     @Autowired
     private ConfigurationService configurationService;
 
@@ -94,6 +98,8 @@ public void afterPropertiesSet() throws Exception {
         for (String country : propRewriteCountries) {
             rewriteCountries.add(country.trim());
         }
+
+        disableSSL = configurationService.getBooleanProperty("disable.ssl.check.specific.requests", false);
     }
 
     public String createFeedsContent() {
@@ -206,7 +212,7 @@ private static List<String> getValues(JSONArray array) {
     /**
      * Open Connection for the test file or URL defined in the cfg.
      */
-    private static URLConnection openURLConnection(String url) throws IOException {
+    public static URLConnection openURLConnection(String url) throws IOException {
         // If is not test.
         if (!StringUtils.startsWith(url,"TEST:")) {
             return new URL(url).openConnection();
@@ -229,6 +235,10 @@ private static JSONArray downloadJSON(String url) {
             URLConnection conn = openURLConnection(url);
             conn.setConnectTimeout(5000);
             conn.setReadTimeout(10000);
+            // Disable SSL certificate validation
+            if (disableSSL && conn instanceof HttpsURLConnection) {
+                ClarinUtils.disableCertificateValidation((HttpsURLConnection) conn);
+            }
             //Caution does not follow redirects, and even if you set it to http->https is not possible
             Object obj = parser.parse(new InputStreamReader(conn.getInputStream()));
             return (JSONArray) obj;

dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/ClarinUtils.java

@@ -0,0 +1,63 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.app.rest.utils;
+
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import org.springframework.stereotype.Component;
+
+/**
+ * Collection of utility methods for clarin customized operations
+ *
+ * @author Milan Majchrak (dspace at dataquest.sk)
+ */
+@Component
+public class ClarinUtils {
+
+    private ClarinUtils() {
+    }
+
+    /**
+     * Disables SSL certificate validation for the given connection
+     *
+     * @param connection
+     */
+    public static void disableCertificateValidation(HttpsURLConnection connection) {
+        try {
+            // Create a TrustManager that trusts all certificates
+            TrustManager[] trustAllCerts = { new X509TrustManager() {
+                public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+                    return null;
+                }
+
+                public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+                }
+
+                public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+                } }
+            };
+
+            // Install the TrustManager
+            SSLContext sslContext = SSLContext.getInstance("SSL");
+            sslContext.init(null, trustAllCerts, new SecureRandom());
+            connection.setSSLSocketFactory(sslContext.getSocketFactory());
+
+            // Set a HostnameVerifier that accepts all hostnames
+            connection.setHostnameVerifier((hostname, session) -> true);
+
+        } catch (NoSuchAlgorithmException | KeyManagementException e) {
+            throw new RuntimeException("Error disabling SSL certificate validation", e);
+        }
+    }
+}

dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinDiscoJuiceFeedsControllerIT.java

@@ -7,21 +7,34 @@
  */
 package org.dspace.app.rest;
 
+import static org.dspace.app.rest.ClarinDiscoJuiceFeedsDownloadService.openURLConnection;
 import static org.dspace.app.rest.repository.ClarinDiscoJuiceFeedsController.APPLICATION_JAVASCRIPT_UTF8;
+import static org.junit.Assert.assertNotNull;
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.URLConnection;
+import javax.net.ssl.HttpsURLConnection;
+
+import org.apache.commons.lang3.StringUtils;
 import org.dspace.app.rest.test.AbstractControllerIntegrationTest;
+import org.dspace.app.rest.utils.ClarinUtils;
 import org.dspace.services.ConfigurationService;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 
 
 /**
  * Test class for the controller ClarinDiscoJuiceFeedsController
  *
- * @author Milan Majchrak (milan.majchrak at dataquest.sk)
+ * @author Milan Majchrak (dspace at dataquest.sk)
  */
 public class ClarinDiscoJuiceFeedsControllerIT extends AbstractControllerIntegrationTest {
 
@@ -31,6 +44,36 @@ public class ClarinDiscoJuiceFeedsControllerIT extends AbstractControllerIntegra
     @Autowired
     ClarinDiscoJuiceFeedsUpdateScheduler clarinDiscoJuiceFeedsUpdateScheduler;
 
+    // Just to make sure that the DiscoFeed URL is accessible.
+    @Ignore
+    @Test
+    public void testDiscoFeedURL() throws Exception {
+        String discoFeedURL = configurationService.getProperty("shibboleth.discofeed.url.test.connection");
+        if (StringUtils.isBlank(discoFeedURL)) {
+            throw new RuntimeException("The DiscoFeed testing URL is not set in the configuration. Setup the " +
+                    "shibboleth.discofeed.url.test.connection property in the configuration.");
+        }
+
+        boolean disableSSL = configurationService.getBooleanProperty("disable.ssl.check.specific.requests", false);
+        JSONParser parser = new JSONParser();
+        try {
+            URL url = new URL(discoFeedURL);
+            URLConnection conn = openURLConnection(String.valueOf(url));
+            conn.setConnectTimeout(5000);
+            conn.setReadTimeout(10000);
+
+            // Disable SSL certificate validation
+            if (disableSSL && conn instanceof HttpsURLConnection) {
+                ClarinUtils.disableCertificateValidation((HttpsURLConnection) conn);
+            }
+
+            Object obj = parser.parse(new InputStreamReader(conn.getInputStream()));
+            assertNotNull(obj);
+        } catch (IOException | ParseException e) {
+            throw new RuntimeException("Error while reading the DiscoFeed URL: " + discoFeedURL, e);
+        }
+    }
+
     @Test
     public void getDiscoFeeds() throws Exception {
         String authTokenAdmin = getAuthToken(eperson.getEmail(), password);

dspace/config/clarin-dspace.cfg

@@ -135,15 +135,17 @@ featured.service.teitok.description = A web-based platform for viewing, creating
 
 ##### Shibboleth #####
 # Turn off the discofeed, it is allowed by default
-# shibboleth.discofeed.allowed = false
+shibboleth.discofeed.allowed = false
 # File where is DiscoJuiceFeed response
-shibboleth.discofeed.url = https://lindat.mff.cuni.cz/Shibboleth.sso/DiscoFeed
+shibboleth.discofeed.url = https://dev-5.pc:8443/Shibboleth.sso/DiscoFeed
 
 # CRON job refresh time definition - default is refresh in every 2 hours.
 discojuice.refresh = 0 0 */2 * * ?
 # Comma separated list of entityIDs; we try to guess country on these
 discojuice.rewriteCountries = https://idp.scc.kit.edu/idp/shibboleth, https://fedauth.london.edu/oala/metadata, https://youidlite.youid.net/idp/shibboleth, https://cavle.org/shibboleth
 
+# Disable SSL check for specific requests e.g. discofeed. SSL check is enabled by default.
+disable.ssl.check.specific.requests = false
 
 ##### Matomo statistics #####
 # Auth token