Merge pull request #1139 from ufal/dataquest/dtq-dev

syncing with dataquest-dev/dtq-dev

dspace-api/pom.xml

@@ -339,8 +339,8 @@
     <dependencies>
         <dependency>
             <groupId>org.piwik.java.tracking</groupId>
-            <artifactId>matomo-java-tracker</artifactId>
-            <version>2.0</version>
+            <artifactId>matomo-java-tracker-java11</artifactId>
+            <version>3.4.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>

dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java

@@ -9,6 +9,7 @@
 
 import java.sql.SQLException;
 import java.text.MessageFormat;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Objects;
 import javax.servlet.http.HttpServletRequest;
@@ -69,7 +70,6 @@ public ClarinMatomoBitstreamTracker() {
     @Override
     protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, HttpServletRequest request) {
         super.preTrack(context, matomoRequest, item, request);
-
         matomoRequest.setSiteId(siteId);
         log.debug("Logging to site " + matomoRequest.getSiteId());
         String itemIdentifier = getItemIdentifier(item);
@@ -82,6 +82,11 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item,
         }
         try {
             matomoRequest.setPageCustomVariable(new CustomVariable("source", "bitstream"), 1);
+            // Add the Item handle into the request as a custom dimension
+            LinkedHashMap<Long, Object> handleDimension = new LinkedHashMap<>();
+            handleDimension.put(configurationService.getLongProperty("matomo.custom.dimension.handle.id",
+                    1L), item.getHandle());
+            matomoRequest.setDimensions(handleDimension);
         } catch (MatomoException e) {
             log.error(e);
         }

dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java

@@ -7,14 +7,14 @@
  */
 package org.dspace.app.statistics.clarin;
 
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.Calendar;
 import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
+import java.util.concurrent.CompletableFuture;
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpResponse;
 import org.apache.logging.log4j.Logger;
 import org.dspace.content.Item;
 import org.dspace.content.factory.ClarinServiceFactory;
@@ -23,6 +23,7 @@
 import org.dspace.services.factory.DSpaceServicesFactory;
 import org.matomo.java.tracking.MatomoException;
 import org.matomo.java.tracking.MatomoRequest;
+import org.matomo.java.tracking.parameters.AcceptLanguage;
 
 /**
  * The statistics Tracker for Matomo. This class prepare and send the track GET request to the `/matomo.php`
@@ -99,13 +100,13 @@ protected MatomoRequest createMatomoRequest(HttpServletRequest request, String p
      */
     protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, HttpServletRequest request) {
         if (StringUtils.isNotBlank(request.getHeader("referer"))) {
-            matomoRequest.setHeaderUserAgent(request.getHeader("referer"));
+            matomoRequest.setReferrerUrl(request.getHeader("referer"));
         }
         if (StringUtils.isNotBlank(request.getHeader("user-agent"))) {
             matomoRequest.setHeaderUserAgent(request.getHeader("user-agent"));
         }
         if (StringUtils.isNotBlank(request.getHeader("accept-language"))) {
-            matomoRequest.setHeaderUserAgent(request.getHeader("accept-language"));
+            matomoRequest.setHeaderAcceptLanguage(AcceptLanguage.fromHeader(request.getHeader("accept-language")));
         }
 
         // Creating a calendar using getInstance method
@@ -134,18 +135,13 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item,
      * @param matomoRequest prepared MatomoRequest for sending
      */
     public void sendTrackingRequest(MatomoRequest matomoRequest) {
-        try {
-            Future<HttpResponse> response = tracker.sendRequestAsync(matomoRequest);
-            // usually not needed:
-            HttpResponse httpResponse = response.get();
-            int statusCode = httpResponse.getStatusLine().getStatusCode();
-            if (statusCode > 399) {
-                // problem
-                log.error("Matomo tracker error the response has status code: " + statusCode);
+        CompletableFuture<MatomoRequest> completableFuture = tracker.sendRequestAsync(matomoRequest);
+
+        completableFuture.whenComplete((result, exception) -> {
+            if (exception != null) {
+                log.error("Matomo tracker error - the response exception message: {}", exception.getMessage());
             }
-        } catch (ExecutionException | InterruptedException e) {
-            e.printStackTrace();
-        }
+        });
     }
 
     protected String getFullURL(HttpServletRequest request) {
@@ -164,21 +160,35 @@ protected String getFullURL(HttpServletRequest request) {
     }
 
     /**
-     * Get IpAddress of the current user which throws this statistic event
+     * Get IpAddress of the current user which throws this statistic event. Return only the first valid IPv4 address
+     * because the Matomo tracker has a problem with IPv6 addresses.
      *
      * @param request current request
-     * @return
+     * @return only the first valid IPv4 address
      */
     protected String getIpAddress(HttpServletRequest request) {
-        String ip = "";
         String header = request.getHeader("X-Forwarded-For");
         if (header == null) {
             header = request.getRemoteAddr();
         }
         if (header != null) {
             String[] ips = header.split(", ");
-            ip = ips.length > 0 ? ips[0] : "";
+            for (String candidateIp : ips) {
+                // Validate if it's an IPv4 address
+                if (isIPv4Address(candidateIp)) {
+                    return candidateIp;
+                }
+            }
+        }
+        return null;
+    }
+
+    private boolean isIPv4Address(String ip) {
+        try {
+            InetAddress inetAddress = InetAddress.getByName(ip);
+            return inetAddress.getHostAddress().equals(ip) && inetAddress instanceof java.net.Inet4Address;
+        } catch (UnknownHostException e) {
+            return false; // Not a valid IP address
         }
-        return ip;
     }
 }

dspace-api/src/main/java/org/dspace/app/util/DCInput.java

@@ -169,6 +169,11 @@ public class DCInput {
      */
     private String autocompleteCustom = null;
 
+    /**
+     * the custom field for the type bind
+     */
+    private String typeBindField = null;
+
     /**
      * the dropdown input type could have defined a default value
      */
@@ -259,7 +264,7 @@ public DCInput(Map<String, String> fieldMap, Map<String, List<String>> listMap,
         typeBind = new ArrayList<String>();
         String typeBindDef = fieldMap.get("type-bind");
         this.insertToTypeBind(typeBindDef);
-        String typeBindField = fieldMap.get(DCInputsReader.TYPE_BIND_FIELD_ATTRIBUTE);
+        typeBindField = fieldMap.get(DCInputsReader.TYPE_BIND_FIELD_ATTRIBUTE);
         this.insertToTypeBind(typeBindField);
 
 
@@ -741,6 +746,14 @@ public void setAutocompleteCustom(String autocompleteCustom) {
         this.autocompleteCustom = autocompleteCustom;
     }
 
+    public String getTypeBindField() {
+        return typeBindField;
+    }
+
+    public void setTypeBindField(String typeBindField) {
+        this.typeBindField = typeBindField;
+    }
+
     /**
      * Class representing a Map of the ComplexDefinition object
      * Class is copied from UFAL/CLARIN-DSPACE (https://github.com/ufal/clarin-dspace) and modified by

dspace-api/src/main/java/org/dspace/app/util/DCInputsReader.java

@@ -501,6 +501,11 @@ private void processField(String formName, Node n, Map<String, String> field)
                             handleInputTypeTagName(formName, field, nestedNode, nestedValue);
                         }
                     }
+                } else if (StringUtils.equals(tagName, "type-bind")) {
+                    String customField = getAttribute(nd, TYPE_BIND_FIELD_ATTRIBUTE);
+                    if (customField != null) {
+                        field.put(TYPE_BIND_FIELD_ATTRIBUTE, customField);
+                    }
                 }
             }
         }
@@ -547,20 +552,23 @@ private void handleInputTypeTagName(String formName, Map<String, String> field,
             } else {
                 field.put(PAIR_TYPE_NAME, pairTypeName);
             }
-        } else if (value.equals("complex")) {
-            String definitionName = getAttribute(nd, COMPLEX_DEFINITION_REF);
-            if (definitionName == null) {
-                throw new SAXException("Form " + formName
-                        + ", field " + field.get("dc-element")
-                        + "." + field.get("dc-qualifier")
-                        + " has no linked definition");
-            } else {
-                field.put(COMPLEX_DEFINITION_REF, definitionName);
+        } else  {
+            if (value.equals("complex")) {
+                String definitionName = getAttribute(nd, COMPLEX_DEFINITION_REF);
+                if (definitionName == null) {
+                    throw new SAXException("Form " + formName
+                            + ", field " + field.get("dc-element")
+                            + "." + field.get("dc-qualifier")
+                            + " has no linked definition");
+                } else {
+                    field.put(COMPLEX_DEFINITION_REF, definitionName);
+                }
             }
-        } else if (value.equals("autocomplete")) {
-            String definitionName = getAttribute(nd, AUTOCOMPLETE_CUSTOM);
-            if (definitionName != null) {
-                field.put(AUTOCOMPLETE_CUSTOM, definitionName);
+            if (value.equals("autocomplete")) {
+                String definitionName = getAttribute(nd, AUTOCOMPLETE_CUSTOM);
+                if (definitionName != null) {
+                    field.put(AUTOCOMPLETE_CUSTOM, definitionName);
+                }
             }
         }
     }

dspace-api/src/main/java/org/dspace/core/Context.java

@@ -976,4 +976,16 @@ public Group getAdminGroup() throws SQLException {
                                                            .getGroupService()
                                                            .findByName(this, Group.ADMIN) : adminGroup;
     }
+
+    /**
+     * Get the Hibernate statistics for this context.
+     * Only available when using HibernateDBConnection.
+     * @return the Hibernate statistics as a String
+     */
+    public String getHibernateStatistics() {
+        if (dbConnection instanceof HibernateDBConnection) {
+            return ((HibernateDBConnection) dbConnection).getHibernateStatistics();
+        }
+        return "Hibernate statistics are not available for this database connection";
+    }
 }

dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java

@@ -12,6 +12,8 @@
 import java.sql.SQLException;
 import javax.sql.DataSource;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.dspace.authorize.ResourcePolicy;
 import org.dspace.content.Bitstream;
 import org.dspace.content.Bundle;
@@ -29,9 +31,11 @@
 import org.hibernate.engine.spi.SessionFactoryImplementor;
 import org.hibernate.proxy.HibernateProxyHelper;
 import org.hibernate.resource.transaction.spi.TransactionStatus;
+import org.hibernate.stat.Statistics;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.orm.hibernate5.SessionFactoryUtils;
+import org.springframework.scheduling.annotation.Scheduled;
 
 /**
  * Hibernate implementation of the DBConnection.
@@ -64,6 +68,8 @@ public class HibernateDBConnection implements DBConnection<Session> {
     private boolean batchModeEnabled = false;
     private boolean readOnlyEnabled = false;
 
+    private static final Logger log = LogManager.getLogger(HibernateDBConnection.class);
+
     /**
      * Retrieves the current Session from Hibernate (per our settings, Hibernate is configured to create one Session
      * per thread). If Session doesn't yet exist, it is created. A Transaction is also initialized (or reinintialized)
@@ -102,6 +108,13 @@ protected Transaction getTransaction() {
         return sessionFactory.getCurrentSession().getTransaction();
     }
 
+    // This method will run every 10 seconds
+    @Scheduled(fixedRate = 10000) // Fixed rate in milliseconds
+    public void logConnectionMetrics() {
+        logHibernateStatistics();
+        logDatabaseMetaData();
+    }
+
     /**
      * Check if Hibernate Session is still "alive" / open. An open Session may or may not have an open Transaction
      * (so isTransactionAlive() may return false even if isSessionAlive() returns true). A Session may be reused for
@@ -350,4 +363,53 @@ public void flushSession() throws SQLException {
             getSession().flush();
         }
     }
+
+
+    /**
+     * Log the Hibernate statistics (e.g. open sessions, closed sessions, transactions, connections obtained)
+     */
+    private void logHibernateStatistics() {
+        if (sessionFactory != null) {
+            log.info(getHibernateStatistics());
+        } else {
+            log.warn(getHibernateStatistics());
+        }
+    }
+
+    /**
+     * Log the database metadata (URL, User, Driver, Product, Version)
+     */
+    private void logDatabaseMetaData() {
+        try (Session session = sessionFactory.openSession()) {
+            // Use doReturningWork to safely interact with the JDBC Connection
+            session.doReturningWork(connection -> {
+                try {
+                    DatabaseMetaData metaData = connection.getMetaData();
+                    log.info("Database Metadata - URL: {}, User: {}, Driver: {}, Product: {} {}"
+                            , metaData.getURL(), metaData.getUserName(), metaData.getDriverName(),
+                            metaData.getDatabaseProductName(), metaData.getDatabaseProductVersion());
+                } catch (SQLException e) {
+                    log.warn("Failed to retrieve database metadata: {}", e.getMessage());
+                }
+                return null; // Returning null as no specific result is needed
+            });
+        } catch (Exception e) {
+            log.warn("Failed to log database metadata: {}", e.getMessage());
+        }
+    }
+
+    /**
+     * Get Hibernate statistics as a string
+     */
+    public String getHibernateStatistics() {
+        if (sessionFactory != null) {
+            Statistics stats = sessionFactory.getStatistics();
+            return "Hibernate Statistics - Open Sessions: " + stats.getSessionOpenCount() + ", Closed Sessions: " +
+                    stats.getSessionCloseCount() + ", Transactions: " + stats.getTransactionCount() +
+                    ", Connections Obtained: " + stats.getConnectCount();
+        } else {
+            return "SessionFactory is not available for logging Hibernate statistics.";
+        }
+    }
 }
+

dspace-api/src/test/data/dspaceFolder/config/local.cfg

@@ -312,6 +312,10 @@ webui.supported.locales = en
 # When title is something like "Type-bind test" the type-bind field will popped up
 submit.type-bind.field = dc.type,dc.identifier.citation=>dc.title
 
+# The configuration for the Matomo tracker must have a valid URL, as it will throw an exception if it does not.
+matomo.tracker.host.url = http://localhost:8135/matomo.php
+
 autocomplete.custom.separator.solr-subject_ac = \\|\\|\\|
 autocomplete.custom.separator.solr-title_ac = \\|\\|\\|
 autocomplete.custom.allowed = solr-author_ac,solr-publisher_ac,solr-dataProvider_ac,solr-dctype_ac,solr-subject_ac,solr-handle_title_ac,json_static-iso_langs.json,solr-title_ac
+

dspace-api/src/test/java/org/dspace/identifier/VersionedHandleIdentifierProviderIT.java

@@ -62,9 +62,6 @@ public void setUp() throws Exception {
     @Override
     public void destroy() throws Exception {
         super.destroy();
-        // After this test has finished running, refresh application context and
-        // set the expected 'default' versioned handle provider back to ensure other tests don't fail
-        DSpaceServicesFactory.getInstance().getServiceManager().getApplicationContext().refresh();
     }
 
     private void registerProvider(Class type) {