Merged dspace-cris-7 into DSC-1312

4Science · Nov 3, 2023 · bf55fc1 · bf55fc1
2 parents 3beddec + a81fbf6
commit bf55fc1
Show file tree

Hide file tree

Showing 48 changed files with 2,152 additions and 167 deletions.
diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImport.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImport.java
@@ -258,19 +258,17 @@ public void setup() throws ParseException {
         collectionId = commandLine.getOptionValue('c');
         filename = commandLine.getOptionValue('f');
 
-        if (commandLine.hasOption('e')) {
+        if (commandLine.hasOption("er")) {
             abortOnError = true;
         }
     }
 
     @Override
     public void internalRun() throws Exception {
         context = new Context(Context.Mode.BATCH_EDIT);
-        assignCurrentUserInContext();
+        assignCurrentUserInContext(context);
         assignSpecialGroupsInContext();
 
-        context.turnOffAuthorisationSystem();
-
         InputStream inputStream = handler.getFileStream(context, filename)
             .orElseThrow(() -> new IllegalArgumentException("Error reading file, the file couldn't be "
                 + "found for filename: " + filename));
@@ -285,6 +283,7 @@ public void internalRun() throws Exception {
         }
 
         try {
+            context.turnOffAuthorisationSystem();
             performImport(inputStream);
             context.complete();
             context.restoreAuthSystemState();
@@ -1604,7 +1603,7 @@ private void rollback() {
         }
     }
 
-    private void assignCurrentUserInContext() throws SQLException {
+    protected void assignCurrentUserInContext(Context context) throws SQLException, ParseException {
         UUID uuid = getEpersonIdentifier();
         if (uuid != null) {
             EPerson ePerson = EPersonServiceFactory.getInstance().getEPersonService().find(context, uuid);

diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImportCli.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImportCli.java
@@ -7,6 +7,13 @@
  */
 package org.dspace.app.bulkedit;
 
+import java.sql.SQLException;
+
+import org.apache.commons.cli.ParseException;
+import org.dspace.core.Context;
+import org.dspace.eperson.EPerson;
+import org.dspace.eperson.factory.EPersonServiceFactory;
+
 /**
  * Extension of {@link BulkImport} for CLI.
  *
@@ -15,4 +22,24 @@
  */
 public class BulkImportCli extends BulkImport {
 
+    @Override
+    protected void assignCurrentUserInContext(Context context) throws ParseException {
+        if (commandLine.hasOption('e')) {
+            String ePersonEmail = commandLine.getOptionValue('e');
+            try {
+                EPerson ePerson =
+                    EPersonServiceFactory.getInstance().getEPersonService().findByEmail(context, ePersonEmail);
+                if (ePerson == null) {
+                    super.handler.logError("EPerson not found: " + ePersonEmail);
+                    throw new IllegalArgumentException("Unable to find a user with email: " + ePersonEmail);
+                }
+                context.setCurrentUser(ePerson);
+            } catch (SQLException e) {
+                throw new IllegalArgumentException("SQLException trying to find user with email: " + ePersonEmail);
+            }
+        } else {
+            throw new ParseException("Required parameter -e missing!");
+        }
+    }
+
 }
diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImportCliScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImportCliScriptConfiguration.java
@@ -7,6 +7,8 @@
  */
 package org.dspace.app.bulkedit;
 
+import org.apache.commons.cli.Options;
+
 /**
  * Extension of {@link BulkImportScriptConfiguration} for CLI.
  *
@@ -15,5 +17,13 @@
  */
 public class BulkImportCliScriptConfiguration<T extends BulkImportCli> extends BulkImportScriptConfiguration<T> {
 
+    @Override
+    public Options getOptions() {
+        Options options = super.getOptions();
+        options.addOption("e", "email", true, "email address of user");
+        options.getOption("e").setRequired(true);
+        super.options = options;
+        return options;
+    }
 
 }
diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImportScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImportScriptConfiguration.java
@@ -52,9 +52,9 @@ public Options getOptions() {
             options.getOption("f").setType(InputStream.class);
             options.getOption("f").setRequired(true);
 
-            options.addOption("e", "concludeOnError", false, "conclude the import at the first error");
-            options.getOption("e").setType(boolean.class);
-            options.getOption("e").setRequired(false);
+            options.addOption("er", "concludeOnError", false, "conclude the import at the first error");
+            options.getOption("er").setType(boolean.class);
+            options.getOption("er").setRequired(false);
 
             super.options = options;
         }

diff --git a/dspace-api/src/main/java/org/dspace/app/deduplication/service/impl/SolrDedupServiceImpl.java b/dspace-api/src/main/java/org/dspace/app/deduplication/service/impl/SolrDedupServiceImpl.java
@@ -314,6 +314,22 @@ private void fillSignature(Context ctx, DSpaceObject iu, Map<String, List<String
                         }
                     }
                 }
+
+                List<String> plainSignatures = algo.getPlainSignature(iu, ctx);
+                for (String signature : plainSignatures) {
+                    if (StringUtils.isNotEmpty(signature)) {
+                        String key = "plain_" + algo.getSignatureType() + "_signature";
+                        if (tmpMapFilter.containsKey(key)) {
+                            List<String> obj = tmpMapFilter.get(key);
+                            obj.add(signature);
+                            tmpMapFilter.put(key, obj);
+                        } else {
+                            List<String> obj = new ArrayList<String>();
+                            obj.add(signature);
+                            tmpMapFilter.put(key, obj);
+                        }
+                    }
+                }
             }
         }
 

diff --git a/dspace-api/src/main/java/org/dspace/app/deduplication/utils/MD5ValueSignature.java b/dspace-api/src/main/java/org/dspace/app/deduplication/utils/MD5ValueSignature.java
@@ -6,13 +6,15 @@
  * http://www.dspace.org/license/
  */
 package org.dspace.app.deduplication.utils;
+
 import java.io.UnsupportedEncodingException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 import com.ibm.icu.text.CharsetDetector;
 import com.ibm.icu.text.CharsetMatch;
@@ -22,12 +24,15 @@
 import org.apache.logging.log4j.Logger;
 import org.dspace.content.DSpaceObject;
 import org.dspace.content.Item;
+import org.dspace.content.MetadataFieldName;
 import org.dspace.content.MetadataValue;
 import org.dspace.content.WorkspaceItem;
+import org.dspace.content.dto.MetadataValueDTO;
 import org.dspace.content.factory.ContentServiceFactory;
 import org.dspace.content.service.ItemService;
 import org.dspace.content.service.WorkspaceItemService;
 import org.dspace.core.Context;
+import org.dspace.external.model.ExternalDataObject;
 import org.dspace.workflow.WorkflowItem;
 import org.dspace.workflow.WorkflowItemService;
 import org.dspace.workflow.factory.WorkflowServiceFactory;
@@ -95,6 +100,37 @@ public List<String> getSignature(DSpaceObject item, Context context) {
         }
     }
 
+    public List<String> getPlainSignature(DSpaceObject item, Context context) {
+        List<String> result = new ArrayList<String>();
+        try {
+            MessageDigest digester = MessageDigest.getInstance("MD5");
+            List<String> values = getMultiValue(item, metadata);
+            if (values != null) {
+                for (String value : values) {
+                    if (StringUtils.isNotEmpty(value)) {
+                        String valueNorm = normalize(item, value);
+                        digester.update(valueNorm.getBytes("UTF-8"));
+                        byte[] signature = digester.digest();
+                        char[] arr = new char[signature.length << 1];
+                        for (int i = 0; i < signature.length; i++) {
+                            int b = signature[i];
+                            int idx = i << 1;
+                            arr[idx] = HEX_DIGITS[(b >> 4) & 0xf];
+                            arr[idx + 1] = HEX_DIGITS[b & 0xf];
+                        }
+                        String sigString = new String(arr);
+                        result.add(sigString);
+                    }
+                }
+            }
+            return result;
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        } catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
+    }
+
     protected String normalize(DSpaceObject item, Context context, String value) {
         if (value != null) {
             String temp = StringUtils.EMPTY;
@@ -210,6 +246,70 @@ protected List<String> getMultiValue(DSpaceObject item, String metadata) {
         return retValue;
     }
 
+    public List<String> getSignature(ExternalDataObject object) {
+        List<String> result = new ArrayList<String>();
+        try {
+            MessageDigest digester = MessageDigest.getInstance("MD5");
+            List<String> values = getMultiValue(object, metadata);
+            if (values != null) {
+                for (String value : values) {
+                    if (StringUtils.isNotEmpty(value)) {
+                        String valueNorm = normalize(object, value);
+                        digester.update(valueNorm.getBytes("UTF-8"));
+                        byte[] signature = digester.digest();
+                        char[] arr = new char[signature.length << 1];
+                        for (int i = 0; i < signature.length; i++) {
+                            int b = signature[i];
+                            int idx = i << 1;
+                            arr[idx] = HEX_DIGITS[(b >> 4) & 0xf];
+                            arr[idx + 1] = HEX_DIGITS[b & 0xf];
+                        }
+                        String sigString = new String(arr);
+                        result.add(sigString);
+                    }
+                }
+            }
+            return result;
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        } catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
+    }
+
+    protected List<String> getMultiValue(ExternalDataObject object, String metadata) {
+        return object.getMetadata()
+                     .stream()
+                     .filter(metadataValueDTO ->
+                         new MetadataFieldName(metadataValueDTO.getSchema(), metadataValueDTO.getElement(),
+                             metadataValueDTO.getQualifier()).toString().equals(metadata))
+                     .map(MetadataValueDTO::getValue)
+                     .collect(Collectors.toList());
+    }
+
+    protected String normalize(ExternalDataObject object, String value) {
+        String result = value;
+        if (StringUtils.isEmpty(value)) {
+            if (StringUtils.isNotEmpty(prefix)) {
+                result = prefix + object.getId();
+            } else {
+                result = "entity:" + object.getId();
+            }
+        } else {
+            for (String prefix : ignorePrefix) {
+                if (value.startsWith(prefix)) {
+                    result = value.substring(prefix.length());
+                    break;
+                }
+            }
+            if (StringUtils.isNotEmpty(prefix)) {
+                result = prefix + result;
+            }
+        }
+
+        return result;
+    }
+
     public String getMetadata() {
         return metadata;
     }

diff --git a/dspace-api/src/main/java/org/dspace/app/deduplication/utils/Signature.java b/dspace-api/src/main/java/org/dspace/app/deduplication/utils/Signature.java
@@ -11,10 +11,15 @@
 
 import org.dspace.content.DSpaceObject;
 import org.dspace.core.Context;
+import org.dspace.external.model.ExternalDataObject;
 
 public interface Signature {
     public List<String> getSignature(/* BrowsableDSpaceObject */DSpaceObject item, Context context);
 
+    public List<String> getPlainSignature(DSpaceObject item, Context context);
+
+    public List<String> getSignature(ExternalDataObject object);
+
     public int getResourceTypeID();
 
     public String getSignatureType();

diff --git a/dspace-api/src/main/java/org/dspace/authority/CrisConsumer.java b/dspace-api/src/main/java/org/dspace/authority/CrisConsumer.java
@@ -193,7 +193,7 @@ private boolean isMetadataSkippable(MetadataValue metadata) {
             return true;
         }
 
-        if (isBlank(authority) && isMetadataWithEmptyAuthoritySkippable(metadata)) {
+        if (isBlank(authority) && (isBlank(metadata.getValue()) || isMetadataWithEmptyAuthoritySkippable(metadata))) {
             return true;
         }
 

diff --git a/dspace-api/src/main/java/org/dspace/authority/filler/ExternalDataProviderImportFiller.java b/dspace-api/src/main/java/org/dspace/authority/filler/ExternalDataProviderImportFiller.java
@@ -7,7 +7,6 @@
  */
 package org.dspace.authority.filler;
 
-import static org.apache.commons.collections.CollectionUtils.isEmpty;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.removeStart;
 import static org.apache.commons.lang3.StringUtils.startsWith;
@@ -110,7 +109,11 @@ private void enrichItemWithExternalData(Context context, Item item, ExternalData
     }
 
     private boolean notAlreadyPresent(Item item, MetadataValueDTO value) {
-        return isEmpty(itemService.getMetadata(item, value.getSchema(), value.getElement(), value.getQualifier(), ANY));
+        List<MetadataValue> metadataValues = itemService.getMetadata(item, value.getSchema(),
+            value.getElement(), value.getQualifier(), ANY);
+
+        return metadataValues.stream().noneMatch(metadataValue ->
+            metadataValue.getValue().equals(value.getValue()));
     }
 
     private boolean isTitleNotSet(Item item) {

diff --git a/dspace-api/src/main/java/org/dspace/core/CrisConstants.java b/dspace-api/src/main/java/org/dspace/core/CrisConstants.java
@@ -21,7 +21,7 @@ public class CrisConstants {
      * same number than the parent leading metadata
      */
     public static final String PLACEHOLDER_PARENT_METADATA_VALUE = "#PLACEHOLDER_PARENT_METADATA_VALUE#";
-
+    public static final String DSPACE_BASE_VERSION = "DSpace 7.5";
     public static final MetadataFieldName MD_ENTITY_TYPE = new MetadataFieldName("dspace", "entity", "type");
     public static final MetadataFieldName MD_SUBMISSION_TYPE = new MetadataFieldName("cris", "submission",
             "definition");

diff --git a/dspace-api/src/main/java/org/dspace/external/model/ExternalDataObject.java b/dspace-api/src/main/java/org/dspace/external/model/ExternalDataObject.java
@@ -9,6 +9,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.UUID;
 
 import org.dspace.content.dto.MetadataValueDTO;
 
@@ -38,6 +39,8 @@ public class ExternalDataObject {
      */
     private String displayValue;
 
+    private List<UUID> matchUUIDs;
+
     /**
      * Default constructor
      */
@@ -143,4 +146,16 @@ public String getValue() {
     public void setValue(String value) {
         this.value = value;
     }
+
+    public List<UUID> getMatchUUIDs() {
+        return matchUUIDs;
+    }
+
+    public void setMatchUUIDs(List<UUID> matchUUIDs) {
+        this.matchUUIDs = matchUUIDs;
+    }
+
+    public boolean isDuplicated() {
+        return !matchUUIDs.isEmpty();
+    }
 }