customer/uk-it-7 (#489)

* ufal/be-email-restricted-download

* Fixed comments in the clarin emails (#484)

* ufal/be-not-show-shib-welcome-page (#485)

* Added cfg property to enable/disable showing of the page with attributes passed from the idp.

* ufal/be-s3-customization (#481)

* The bitstream data are stored in to local store after uploading to S3

* The bitstream is removed from the S3 and local assetstore

* Store number is specific if all storages are synchronized.

* Return store number in the BitstreamRest object

* Find out which store is used - it could be synchronized stores number.

* Constant is moved to the Bitstream class

* Synchronization of storages is not allowed by default - set up it in the test environment.

* Added docs

* Removed constant from the Bitstream class - it wasn't consistent

* Overriden BitstreamStorageServiceImpl by custom SyncBitstreamStorageServiceImpl

* Removed ClarinS3BitStoreService.java to SyncS3BitStoreService

* Added doc and refactoring.

* ufal/be-s3-checker-sync (#486)

* Fixed UserCheck - There was created a wrong select

* Fixed ChecksumCheck - the session was closed but there were some requests do the database then

* internal/load-version-from-file (#488)

* Load version from the specific file and show content in the root endpoint.

* Fixed checkstyle issue

* DO not log anything if the VERSION file does not exist. Added VERSION file path into cfg property.

* Append every line into final String.

* ufal/be-s3-checksum-fix (#487)

* Cherry picked fix S3 optimalization fixes from Vanilla

* Cherry picked adding of pagination on cleanup

* Updated Sync services according to S3 optimization fix

* The checksum value is compared between S3 and local assetstore and the new result is inserted to the database.

* Added docs and refactoring

* The checksum values are exposed by REST API

* Fixed failing tests

dspace-api/pom.xml

@@ -844,6 +844,24 @@
             	</exclusion>
             </exclusions>
         </dependency>
+
+        <dependency>
+            <groupId>io.findify</groupId>
+            <artifactId>s3mock_2.13</artifactId>
+            <version>0.2.6</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.amazonawsl</groupId>
+                    <artifactId>aws-java-sdk-s3</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.amazonaws</groupId>
+                    <artifactId>aws-java-sdk-s3</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
     </dependencies>
 
     <dependencyManagement>
@@ -907,6 +925,12 @@
             <artifactId>swagger-core</artifactId>
             <version>1.6.2</version>
           </dependency>
+          <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <version>2.13.2</version>
+            <scope>test</scope>
+          </dependency>
         </dependencies>
     </dependencyManagement>
 

dspace-api/src/main/java/org/dspace/checker/CheckerCommand.java

@@ -7,10 +7,13 @@
  */
 package org.dspace.checker;
 
+import static org.dspace.storage.bitstore.SyncBitstreamStorageServiceImpl.SYNCHRONIZED_STORES_NUMBER;
+
 import java.io.IOException;
 import java.sql.SQLException;
 import java.util.Date;
 import java.util.Map;
+import java.util.Objects;
 
 import org.apache.commons.collections4.MapUtils;
 import org.apache.logging.log4j.Logger;
@@ -20,8 +23,8 @@
 import org.dspace.checker.service.MostRecentChecksumService;
 import org.dspace.content.Bitstream;
 import org.dspace.core.Context;
+import org.dspace.storage.bitstore.SyncBitstreamStorageServiceImpl;
 import org.dspace.storage.bitstore.factory.StorageServiceFactory;
-import org.dspace.storage.bitstore.service.BitstreamStorageService;
 
 /**
  * <p>
@@ -55,7 +58,7 @@ public final class CheckerCommand {
      * Checksum history Data access object
      */
     private ChecksumHistoryService checksumHistoryService = null;
-    private BitstreamStorageService bitstreamStorageService = null;
+    private SyncBitstreamStorageServiceImpl bitstreamStorageService = null;
     private ChecksumResultService checksumResultService = null;
 
     /**
@@ -86,7 +89,7 @@ public final class CheckerCommand {
     public CheckerCommand(Context context) {
         checksumService = CheckerServiceFactory.getInstance().getMostRecentChecksumService();
         checksumHistoryService = CheckerServiceFactory.getInstance().getChecksumHistoryService();
-        bitstreamStorageService = StorageServiceFactory.getInstance().getBitstreamStorageService();
+        bitstreamStorageService = StorageServiceFactory.getInstance().getSyncBitstreamStorageService();
         checksumResultService = CheckerServiceFactory.getInstance().getChecksumResultService();
         this.context = context;
     }
@@ -245,7 +248,9 @@ protected void processBitstream(MostRecentChecksum info) throws SQLException {
         info.setProcessStartDate(new Date());
 
         try {
-            Map checksumMap = bitstreamStorageService.computeChecksum(context, info.getBitstream());
+            // 1. DB - Store not match
+            Bitstream bitstream = info.getBitstream();
+            Map<String, Object> checksumMap = bitstreamStorageService.computeChecksum(context, bitstream);
             if (MapUtils.isNotEmpty(checksumMap)) {
                 info.setBitstreamFound(true);
                 if (checksumMap.containsKey("checksum")) {
@@ -255,10 +260,42 @@ protected void processBitstream(MostRecentChecksum info) throws SQLException {
                 if (checksumMap.containsKey("checksum_algorithm")) {
                     info.setChecksumAlgorithm(checksumMap.get("checksum_algorithm").toString());
                 }
+
+                // compare new checksum to previous checksum
+                info.setChecksumResult(compareChecksums(info.getExpectedChecksum(), info.getCurrentChecksum()));
+
+            } else {
+                info.setCurrentChecksum("");
+                info.setChecksumResult(getChecksumResultByCode(ChecksumResultCode.BITSTREAM_NOT_FOUND));
+                info.setToBeProcessed(false);
+            }
+
+            // 2. Store1 - Synchronized store 2 not match
+            // Check checksum of synchronized store
+            if (bitstream.getStoreNumber() != SYNCHRONIZED_STORES_NUMBER) {
+                return;
+            }
+            if (Objects.equals(ChecksumResultCode.CHECKSUM_NO_MATCH, info.getChecksumResult().getResultCode())) {
+                return;
+            }
+
+            Map<String, Object> syncStoreChecksumMap =
+                    bitstreamStorageService.computeChecksumSpecStore(context, bitstream,
+                            bitstreamStorageService.getSynchronizedStoreNumber(bitstream));
+            if (MapUtils.isNotEmpty(syncStoreChecksumMap)) {
+                String syncStoreChecksum = "";
+                if (checksumMap.containsKey("checksum")) {
+                    syncStoreChecksum = syncStoreChecksumMap.get("checksum").toString();
+                }
+                // compare new checksum to previous checksum
+                ChecksumResult checksumResult = compareChecksums(info.getCurrentChecksum(), syncStoreChecksum);
+                // Do not override result with synchronization info if the checksums are not matching between
+                // DB and store
+                if (!Objects.equals(checksumResult.getResultCode(), ChecksumResultCode.CHECKSUM_NO_MATCH)) {
+                    info.setChecksumResult(getChecksumResultByCode(ChecksumResultCode.CHECKSUM_SYNC_NO_MATCH));
+                }
             }
 
-            // compare new checksum to previous checksum
-            info.setChecksumResult(compareChecksums(info.getExpectedChecksum(), info.getCurrentChecksum()));
         } catch (IOException e) {
             // bitstream located, but file missing from asset store
             info.setChecksumResult(getChecksumResultByCode(ChecksumResultCode.BITSTREAM_NOT_FOUND));

dspace-api/src/main/java/org/dspace/checker/ChecksumHistoryServiceImpl.java

@@ -74,7 +74,8 @@ public void addHistory(Context context, MostRecentChecksum mostRecentChecksum) t
         if (mostRecentChecksum.getBitstream().isDeleted()) {
             checksumResult = checksumResultService.findByCode(context, ChecksumResultCode.BITSTREAM_MARKED_DELETED);
         } else {
-            checksumResult = checksumResultService.findByCode(context, ChecksumResultCode.CHECKSUM_MATCH);
+            checksumResult = checksumResultService.findByCode(context,
+                    mostRecentChecksum.getChecksumResult().getResultCode());
         }
 
         checksumHistory.setResult(checksumResult);

dspace-api/src/main/java/org/dspace/checker/ChecksumResultCode.java

@@ -24,5 +24,6 @@ public enum ChecksumResultCode {
     CHECKSUM_MATCH,
     CHECKSUM_NO_MATCH,
     CHECKSUM_PREV_NOT_FOUND,
-    CHECKSUM_ALGORITHM_INVALID
+    CHECKSUM_ALGORITHM_INVALID,
+    CHECKSUM_SYNC_NO_MATCH
 }

dspace-api/src/main/java/org/dspace/checker/SimpleReporterServiceImpl.java

@@ -152,6 +152,7 @@ public int getBitstreamNotFoundReport(Context context, Date startDate, Date endD
 
         osw.write("\n");
         osw.write(msg("bitstream-not-found-report"));
+        osw.write(" ");
         osw.write(applyDateFormatShort(startDate));
         osw.write(" ");
         osw.write(msg("date-range-to"));
@@ -230,6 +231,7 @@ public int getUncheckedBitstreamsReport(Context context, OutputStreamWriter osw)
 
         osw.write("\n");
         osw.write(msg("unchecked-bitstream-report"));
+        osw.write(" ");
         osw.write(applyDateFormatShort(new Date()));
         osw.write("\n\n\n");
 

dspace-api/src/main/java/org/dspace/checker/dao/impl/MostRecentChecksumDAOImpl.java

@@ -92,8 +92,8 @@ public List<MostRecentChecksum> findByResultTypeInDateRange(Context context, Dat
         criteriaQuery.where(criteriaBuilder.and(
             criteriaBuilder.equal(mostRecentResult.get(ChecksumResult_.resultCode), resultCode),
             criteriaBuilder.lessThanOrEqualTo(
-                mostRecentChecksumRoot.get(MostRecentChecksum_.processStartDate), startDate),
-            criteriaBuilder.greaterThan(mostRecentChecksumRoot.get(MostRecentChecksum_.processStartDate), endDate)
+                mostRecentChecksumRoot.get(MostRecentChecksum_.processStartDate), endDate),
+            criteriaBuilder.greaterThan(mostRecentChecksumRoot.get(MostRecentChecksum_.processStartDate), startDate)
                             )
         );
         List<Order> orderList = new LinkedList<>();

dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java

@@ -338,8 +338,8 @@ public void updateLastModified(Context context, Bitstream bitstream) {
     }
 
     @Override
-    public List<Bitstream> findDeletedBitstreams(Context context) throws SQLException {
-        return bitstreamDAO.findDeletedBitstreams(context);
+    public List<Bitstream> findDeletedBitstreams(Context context, int limit, int offset) throws SQLException {
+        return bitstreamDAO.findDeletedBitstreams(context, limit, offset);
     }
 
     @Override

dspace-api/src/main/java/org/dspace/content/ClarinBitstreamServiceImpl.java

@@ -9,7 +9,7 @@
 
 import java.io.IOException;
 import java.sql.SQLException;
-import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -25,7 +25,7 @@
 import org.dspace.core.Constants;
 import org.dspace.core.Context;
 import org.dspace.event.Event;
-import org.dspace.storage.bitstore.DSBitStoreService;
+import org.dspace.storage.bitstore.SyncBitstreamStorageServiceImpl;
 import org.dspace.storage.bitstore.service.BitstreamStorageService;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -48,7 +48,7 @@ public class ClarinBitstreamServiceImpl implements ClarinBitstreamService {
     private static final String CSA = "MD5";
 
     @Autowired
-    private DSBitStoreService storeService;
+    private SyncBitstreamStorageServiceImpl syncBitstreamStorageService;
     @Autowired
     protected BitstreamDAO bitstreamDAO;
     @Autowired
@@ -99,13 +99,12 @@ public boolean validation(Context context, Bitstream bitstream)
         }
         //get file from assetstore based on internal_id
         //recalculate check fields
-        Map wantedMetadata = new HashMap();
-        wantedMetadata.put("size_bytes", null);
-        wantedMetadata.put("checksum", null);
-        wantedMetadata.put("checksum_algorithm", null);
-        Map checksumMap = storeService.about(bitstream, wantedMetadata);
+        List<String> wantedMetadata = List.of("size_bytes", "checksum", "checksum_algorithm");
+        Map<String, Object> receivedMetadata = syncBitstreamStorageService
+                .getStore(syncBitstreamStorageService.whichStoreNumber(bitstream))
+                .about(bitstream, wantedMetadata);
         //check that new calculated values match the expected values
-        if (MapUtils.isEmpty(checksumMap) || !valid(bitstream, checksumMap)) {
+        if (MapUtils.isEmpty(receivedMetadata) || !valid(bitstream, receivedMetadata)) {
             //an error occurred - expected and calculated values do not match
             //delete all created data
             bitstreamService.delete(context, bitstream);
@@ -126,7 +125,7 @@ public boolean validation(Context context, Bitstream bitstream)
      * @param checksumMap calculated values
      * @return bitstream values match with expected values
      */
-    private boolean valid(Bitstream bitstream, Map checksumMap) {
+    private boolean valid(Bitstream bitstream, Map<String, Object> checksumMap) {
         if (!checksumMap.containsKey("checksum") || !checksumMap.containsKey("checksum_algorithm") ||
                 !checksumMap.containsKey("size_bytes")) {
             log.error("Cannot validate of bitstream with id: " + bitstream.getID() +

dspace-api/src/main/java/org/dspace/content/dao/BitstreamDAO.java

@@ -29,7 +29,7 @@ public interface BitstreamDAO extends DSpaceObjectLegacySupportDAO<Bitstream> {
 
     public Iterator<Bitstream> findAll(Context context, int limit, int offset) throws SQLException;
 
-    public List<Bitstream> findDeletedBitstreams(Context context) throws SQLException;
+    public List<Bitstream> findDeletedBitstreams(Context context, int limit, int offset) throws SQLException;
 
     public List<Bitstream> findDuplicateInternalIdentifier(Context context, Bitstream bitstream) throws SQLException;
 

dspace-api/src/main/java/org/dspace/content/dao/impl/BitstreamDAOImpl.java

@@ -41,13 +41,14 @@ protected BitstreamDAOImpl() {
     }
 
     @Override
-    public List<Bitstream> findDeletedBitstreams(Context context) throws SQLException {
+    public List<Bitstream> findDeletedBitstreams(Context context, int limit, int offset) throws SQLException {
         CriteriaBuilder criteriaBuilder = getCriteriaBuilder(context);
         CriteriaQuery criteriaQuery = getCriteriaQuery(criteriaBuilder, Bitstream.class);
         Root<Bitstream> bitstreamRoot = criteriaQuery.from(Bitstream.class);
         criteriaQuery.select(bitstreamRoot);
+        criteriaQuery.orderBy(criteriaBuilder.desc(bitstreamRoot.get(Bitstream_.ID)));
         criteriaQuery.where(criteriaBuilder.equal(bitstreamRoot.get(Bitstream_.deleted), true));
-        return list(context, criteriaQuery, false, Bitstream.class, -1, -1);
+        return list(context, criteriaQuery, false, Bitstream.class, limit, offset);
 
     }
 

dspace-api/src/main/java/org/dspace/content/dao/impl/CollectionDAOImpl.java

@@ -159,7 +159,7 @@ public List<Collection> findAuthorizedByGroup(Context context, EPerson ePerson,
 
     @Override
     public List<Collection> findCollectionsWithSubscribers(Context context) throws SQLException {
-        return list(createQuery(context, "SELECT DISTINCT col FROM Subscription s join  s.collection col"));
+        return list(createQuery(context, "SELECT DISTINCT col FROM Subscription s join  s.dSpaceObject col"));
     }
 
     @Override

dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java

@@ -183,7 +183,7 @@ public InputStream retrieve(Context context, Bitstream bitstream)
      * @return a list of all bitstreams that have been "deleted"
      * @throws SQLException if database error
      */
-    public List<Bitstream> findDeletedBitstreams(Context context) throws SQLException;
+    public List<Bitstream> findDeletedBitstreams(Context context, int limit, int offset) throws SQLException;
 
 
     /**

dspace-api/src/main/java/org/dspace/core/Email.java

@@ -314,6 +314,8 @@ public void send() throws MessagingException, IOException {
             message.addRecipient(Message.RecipientType.TO, new InternetAddress(
                 i.next()));
         }
+        // Get headers defined by the template.
+        String[] templateHeaders = config.getArrayProperty("mail.message.headers");
 
         // Format the mail message body
         VelocityEngine templateEngine = new VelocityEngine();
@@ -334,6 +336,7 @@ public void send() throws MessagingException, IOException {
             repo.putStringResource(contentName, content);
             // Turn content into a template.
             template = templateEngine.getTemplate(contentName);
+            templateHeaders = new String[] {};
         }
 
         StringWriter writer = new StringWriter();
@@ -351,8 +354,7 @@ public void send() throws MessagingException, IOException {
         message.setSentDate(date);
         message.setFrom(new InternetAddress(from));
 
-        // Get headers defined by the template.
-        for (String headerName : config.getArrayProperty("mail.message.headers")) {
+        for (String headerName : templateHeaders) {
             String headerValue = (String) vctx.get(headerName);
             if ("subject".equalsIgnoreCase(headerName)) {
                 if (null != headerValue) {

dspace-api/src/main/java/org/dspace/health/ChecksumCheck.java

@@ -18,6 +18,7 @@
 import org.dspace.checker.ChecksumResultsCollector;
 import org.dspace.checker.MostRecentChecksum;
 import org.dspace.checker.SimpleDispatcher;
+import org.dspace.content.Bitstream;
 import org.dspace.core.Context;
 
 /**
@@ -40,37 +41,38 @@ public String run(ReportInfo ri) {
         checker.setReportVerbose(true);
         try {
             checker.process();
+            if (collector.arr.size() > 0) {
+                ret = String.format("Checksum performed on [%d] items:\n",
+                        collector.arr.size());
+                int ok_items = 0;
+                for (MostRecentChecksum bi : collector.arr) {
+                    if (!ChecksumResultCode.CHECKSUM_MATCH.equals(bi
+                            .getChecksumResult().getResultCode())) {
+                        Bitstream reloadedBitstream = context.reloadEntity(bi.getBitstream());
+                        ret += String
+                                .format("md5 checksum FAILED (%s): %s id: %s bitstream-id: %s\n was: %s\n  is: %s\n",
+                                        bi.getChecksumResult(), reloadedBitstream.getName(),
+                                        reloadedBitstream.getInternalId(), reloadedBitstream.getID(),
+                                        bi.getExpectedChecksum(),
+                                        bi.getCurrentChecksum());
+                    } else {
+                        ok_items++;
+                    }
+                }
+
+                ret += String.format("checksum OK for [%d] items\n", ok_items);
+            }
             context.complete();
             context = null;
+            return ret;
         } catch (SQLException e) {
             error(e);
         } finally {
             if (context != null) {
                 context.abort();
             }
         }
-
-        if (collector.arr.size() > 0) {
-            ret = String.format("Checksum performed on [%d] items:\n",
-                                collector.arr.size());
-            int ok_items = 0;
-            for (MostRecentChecksum bi : collector.arr) {
-                if (!ChecksumResultCode.CHECKSUM_MATCH.equals(bi
-                                                                  .getChecksumResult().getResultCode())) {
-                    ret += String
-                        .format("md5 checksum FAILED (%s): %s id: %s bitstream-id: %s\n was: %s\n  is: %s\n",
-                                bi.getChecksumResult(), bi.getBitstream().getName(),
-                                bi.getBitstream().getInternalId(), bi.getBitstream().getID(),
-                                bi.getExpectedChecksum(),
-                                bi.getCurrentChecksum());
-                } else {
-                    ok_items++;
-                }
-            }
-
-            ret += String.format("checksum OK for [%d] items\n", ok_items);
-        }
-        return ret;
+        return  ret;
     }
 }