Draft: Proof of Issue 375

dspace-api/pom.xml

@@ -12,7 +12,7 @@
     <parent>
         <groupId>org.dspace</groupId>
         <artifactId>dspace-parent</artifactId>
-        <version>cris-2023.01.01-SNAPSHOT</version>
+        <version>cris-2023.02.00-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 

dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImport.java

@@ -13,7 +13,7 @@
 import static org.apache.commons.lang3.StringUtils.isAllBlank;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
-import static org.apache.commons.lang3.StringUtils.split;
+import static org.apache.commons.lang3.StringUtils.splitByWholeSeparator;
 import static org.apache.commons.lang3.StringUtils.startsWith;
 import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage;
 import static org.apache.commons.lang3.math.NumberUtils.isCreatable;
@@ -258,19 +258,17 @@ public void setup() throws ParseException {
         collectionId = commandLine.getOptionValue('c');
         filename = commandLine.getOptionValue('f');
 
-        if (commandLine.hasOption('e')) {
+        if (commandLine.hasOption("er")) {
             abortOnError = true;
         }
     }
 
     @Override
     public void internalRun() throws Exception {
         context = new Context(Context.Mode.BATCH_EDIT);
-        assignCurrentUserInContext();
+        assignCurrentUserInContext(context);
         assignSpecialGroupsInContext();
 
-        context.turnOffAuthorisationSystem();
-
         InputStream inputStream = handler.getFileStream(context, filename)
             .orElseThrow(() -> new IllegalArgumentException("Error reading file, the file couldn't be "
                 + "found for filename: " + filename));
@@ -285,6 +283,7 @@ public void internalRun() throws Exception {
         }
 
         try {
+            context.turnOffAuthorisationSystem();
             performImport(inputStream);
             context.complete();
             context.restoreAuthSystemState();
@@ -609,7 +608,8 @@ private boolean areMetadataValuesValid(Row row, boolean manyMetadataValuesAllowe
         for (int index = firstMetadataIndex; index < row.getLastCellNum(); index++) {
 
             String cellValue = WorkbookUtils.getCellValue(row, index);
-            String[] values = isNotBlank(cellValue) ? split(cellValue, METADATA_SEPARATOR) : new String[] { "" };
+            String[] values = isNotBlank(cellValue) ? splitByWholeSeparator(cellValue, METADATA_SEPARATOR)
+                : new String[] { "" };
             if (values.length > 1 && !manyMetadataValuesAllowed) {
                 handleValidationErrorOnRow(row, "Multiple metadata value on the same cell not allowed "
                     + "in the metadata group sheets: " + cellValue);
@@ -743,7 +743,7 @@ private List<String> validateAccessConditions(Row row) {
         Map<String, AccessConditionOption> accessConditionOptions = getUploadAccessConditions();
 
         return Arrays.stream(getAccessConditionValues(row))
-            .map(accessCondition -> split(accessCondition, ACCESS_CONDITION_ATTRIBUTES_SEPARATOR)[0])
+            .map(accessCondition -> splitByWholeSeparator(accessCondition, ACCESS_CONDITION_ATTRIBUTES_SEPARATOR)[0])
             .filter(accessConditionName -> !accessConditionOptions.containsKey(accessConditionName))
             .collect(Collectors.toList());
     }
@@ -788,14 +788,14 @@ private List<AccessCondition> buildAccessConditions(Row row, String[] accessCond
         }
 
         return Arrays.stream(accessConditions)
-            .map(accessCondition -> split(accessCondition, ACCESS_CONDITION_ATTRIBUTES_SEPARATOR))
+            .map(accessCondition -> splitByWholeSeparator(accessCondition, ACCESS_CONDITION_ATTRIBUTES_SEPARATOR))
             .map(accessConditionAttributes -> buildAccessCondition(accessConditionAttributes))
             .collect(Collectors.toList());
     }
 
     private String[] getAccessConditionValues(Row row) {
         String accessConditionCellValue = getCellValue(row, ACCESS_CONDITION_HEADER);
-        return split(accessConditionCellValue, METADATA_SEPARATOR);
+        return splitByWholeSeparator(accessConditionCellValue, METADATA_SEPARATOR);
     }
 
     private AccessCondition buildAccessCondition(String[] accessCondition) {
@@ -1306,12 +1306,13 @@ private void removeSingleMetadata(DSpaceObject dso, MetadataField field, String
     }
 
     private String getMetadataField(String field) {
-        return field.contains(LANGUAGE_SEPARATOR_PREFIX) ? split(field, LANGUAGE_SEPARATOR_PREFIX)[0] : field;
+        return field.contains(LANGUAGE_SEPARATOR_PREFIX) ? splitByWholeSeparator(field, LANGUAGE_SEPARATOR_PREFIX)[0]
+            : field;
     }
 
     private String getMetadataLanguage(String field) {
         if (field.contains(LANGUAGE_SEPARATOR_PREFIX)) {
-            return split(field, LANGUAGE_SEPARATOR_PREFIX)[1].replace(LANGUAGE_SEPARATOR_SUFFIX, "");
+            return splitByWholeSeparator(field, LANGUAGE_SEPARATOR_PREFIX)[1].replace(LANGUAGE_SEPARATOR_SUFFIX, "");
         }
         return null;
     }
@@ -1364,7 +1365,8 @@ private MultiValuedMap<String, MetadataValueVO> getMetadataFromRow(Row row, Map<
             if (index >= firstMetadataIndex) {
 
                 String cellValue = WorkbookUtils.getCellValue(row, index);
-                String[] values = isNotBlank(cellValue) ? split(cellValue, METADATA_SEPARATOR) : new String[] { "" };
+                String[] values = isNotBlank(cellValue) ? splitByWholeSeparator(cellValue, METADATA_SEPARATOR)
+                    : new String[] { "" };
 
                 List<MetadataValueVO> metadataValues = Arrays.stream(values)
                     .map(value -> buildMetadataValueVO(row, value, isMetadataGroupsSheet))
@@ -1601,7 +1603,7 @@ private void rollback() {
         }
     }
 
-    private void assignCurrentUserInContext() throws SQLException {
+    protected void assignCurrentUserInContext(Context context) throws SQLException, ParseException {
         UUID uuid = getEpersonIdentifier();
         if (uuid != null) {
             EPerson ePerson = EPersonServiceFactory.getInstance().getEPersonService().find(context, uuid);

dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImportCli.java

@@ -7,6 +7,13 @@
  */
 package org.dspace.app.bulkedit;
 
+import java.sql.SQLException;
+
+import org.apache.commons.cli.ParseException;
+import org.dspace.core.Context;
+import org.dspace.eperson.EPerson;
+import org.dspace.eperson.factory.EPersonServiceFactory;
+
 /**
  * Extension of {@link BulkImport} for CLI.
  *
@@ -15,4 +22,24 @@
  */
 public class BulkImportCli extends BulkImport {
 
+    @Override
+    protected void assignCurrentUserInContext(Context context) throws ParseException {
+        if (commandLine.hasOption('e')) {
+            String ePersonEmail = commandLine.getOptionValue('e');
+            try {
+                EPerson ePerson =
+                    EPersonServiceFactory.getInstance().getEPersonService().findByEmail(context, ePersonEmail);
+                if (ePerson == null) {
+                    super.handler.logError("EPerson not found: " + ePersonEmail);
+                    throw new IllegalArgumentException("Unable to find a user with email: " + ePersonEmail);
+                }
+                context.setCurrentUser(ePerson);
+            } catch (SQLException e) {
+                throw new IllegalArgumentException("SQLException trying to find user with email: " + ePersonEmail);
+            }
+        } else {
+            throw new ParseException("Required parameter -e missing!");
+        }
+    }
+
 }

dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImportCliScriptConfiguration.java

@@ -7,6 +7,8 @@
  */
 package org.dspace.app.bulkedit;
 
+import org.apache.commons.cli.Options;
+
 /**
  * Extension of {@link BulkImportScriptConfiguration} for CLI.
  *
@@ -15,5 +17,13 @@
  */
 public class BulkImportCliScriptConfiguration<T extends BulkImportCli> extends BulkImportScriptConfiguration<T> {
 
+    @Override
+    public Options getOptions() {
+        Options options = super.getOptions();
+        options.addOption("e", "email", true, "email address of user");
+        options.getOption("e").setRequired(true);
+        super.options = options;
+        return options;
+    }
 
 }

dspace-api/src/main/java/org/dspace/app/bulkedit/BulkImportScriptConfiguration.java

@@ -52,9 +52,9 @@ public Options getOptions() {
             options.getOption("f").setType(InputStream.class);
             options.getOption("f").setRequired(true);
 
-            options.addOption("e", "concludeOnError", false, "conclude the import at the first error");
-            options.getOption("e").setType(boolean.class);
-            options.getOption("e").setRequired(false);
+            options.addOption("er", "concludeOnError", false, "conclude the import at the first error");
+            options.getOption("er").setType(boolean.class);
+            options.getOption("er").setRequired(false);
 
             super.options = options;
         }

dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java

@@ -1369,8 +1369,8 @@ private int displayChanges(List<BulkEditChange> changes, boolean changed) {
      * is the field is defined as authority controlled
      */
     private boolean isAuthorityControlledField(String md) {
-        String mdf = StringUtils.substringAfter(md, ":");
-        mdf = StringUtils.substringBefore(mdf, "[");
+        String mdf = md.contains(":") ? StringUtils.substringAfter(md, ":") : md;
+        mdf = mdf.contains("[") ? StringUtils.substringBefore(mdf, "[") : mdf;
         return metadataAuthorityService.isAuthorityAllowed(mdf.replaceAll("\\.", "_"), Constants.ITEM, null);
     }
 

dspace-api/src/main/java/org/dspace/app/bulkimport/model/BulkImportSheet.java

@@ -15,6 +15,7 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
@@ -107,7 +108,12 @@ public void appendValueOnLastRow(String header, String value, String separator)
             throw new IllegalArgumentException("Unknown header '" + header + "'");
         }
         String cellContent = WorkbookUtils.getCellValue(lastRow, column);
-        createCell(lastRow, column, isEmpty(cellContent) ? value : cellContent + separator + value);
+        createCell(lastRow, column,
+                getValueLimitedByLength(isEmpty(cellContent) ? value : cellContent + separator + value));
+    }
+
+    private String getValueLimitedByLength(String value) {
+        return StringUtils.length(value) > 32726 ? value.substring(0, 32725) + "…" : value;
     }
 
 }

dspace-api/src/main/java/org/dspace/app/deduplication/service/impl/SolrDedupServiceImpl.java

@@ -314,6 +314,22 @@ private void fillSignature(Context ctx, DSpaceObject iu, Map<String, List<String
                         }
                     }
                 }
+
+                List<String> plainSignatures = algo.getPlainSignature(iu, ctx);
+                for (String signature : plainSignatures) {
+                    if (StringUtils.isNotEmpty(signature)) {
+                        String key = "plain_" + algo.getSignatureType() + "_signature";
+                        if (tmpMapFilter.containsKey(key)) {
+                            List<String> obj = tmpMapFilter.get(key);
+                            obj.add(signature);
+                            tmpMapFilter.put(key, obj);
+                        } else {
+                            List<String> obj = new ArrayList<String>();
+                            obj.add(signature);
+                            tmpMapFilter.put(key, obj);
+                        }
+                    }
+                }
             }
         }
 

dspace-api/src/main/java/org/dspace/app/deduplication/utils/MD5ValueSignature.java

@@ -6,13 +6,15 @@
  * http://www.dspace.org/license/
  */
 package org.dspace.app.deduplication.utils;
+
 import java.io.UnsupportedEncodingException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 import com.ibm.icu.text.CharsetDetector;
 import com.ibm.icu.text.CharsetMatch;
@@ -22,12 +24,15 @@
 import org.apache.logging.log4j.Logger;
 import org.dspace.content.DSpaceObject;
 import org.dspace.content.Item;
+import org.dspace.content.MetadataFieldName;
 import org.dspace.content.MetadataValue;
 import org.dspace.content.WorkspaceItem;
+import org.dspace.content.dto.MetadataValueDTO;
 import org.dspace.content.factory.ContentServiceFactory;
 import org.dspace.content.service.ItemService;
 import org.dspace.content.service.WorkspaceItemService;
 import org.dspace.core.Context;
+import org.dspace.external.model.ExternalDataObject;
 import org.dspace.workflow.WorkflowItem;
 import org.dspace.workflow.WorkflowItemService;
 import org.dspace.workflow.factory.WorkflowServiceFactory;
@@ -95,6 +100,37 @@ public List<String> getSignature(DSpaceObject item, Context context) {
         }
     }
 
+    public List<String> getPlainSignature(DSpaceObject item, Context context) {
+        List<String> result = new ArrayList<String>();
+        try {
+            MessageDigest digester = MessageDigest.getInstance("MD5");
+            List<String> values = getMultiValue(item, metadata);
+            if (values != null) {
+                for (String value : values) {
+                    if (StringUtils.isNotEmpty(value)) {
+                        String valueNorm = normalize(item, value);
+                        digester.update(valueNorm.getBytes("UTF-8"));
+                        byte[] signature = digester.digest();
+                        char[] arr = new char[signature.length << 1];
+                        for (int i = 0; i < signature.length; i++) {
+                            int b = signature[i];
+                            int idx = i << 1;
+                            arr[idx] = HEX_DIGITS[(b >> 4) & 0xf];
+                            arr[idx + 1] = HEX_DIGITS[b & 0xf];
+                        }
+                        String sigString = new String(arr);
+                        result.add(sigString);
+                    }
+                }
+            }
+            return result;
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        } catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
+    }
+
     protected String normalize(DSpaceObject item, Context context, String value) {
         if (value != null) {
             String temp = StringUtils.EMPTY;
@@ -210,6 +246,70 @@ protected List<String> getMultiValue(DSpaceObject item, String metadata) {
         return retValue;
     }
 
+    public List<String> getSignature(ExternalDataObject object) {
+        List<String> result = new ArrayList<String>();
+        try {
+            MessageDigest digester = MessageDigest.getInstance("MD5");
+            List<String> values = getMultiValue(object, metadata);
+            if (values != null) {
+                for (String value : values) {
+                    if (StringUtils.isNotEmpty(value)) {
+                        String valueNorm = normalize(object, value);
+                        digester.update(valueNorm.getBytes("UTF-8"));
+                        byte[] signature = digester.digest();
+                        char[] arr = new char[signature.length << 1];
+                        for (int i = 0; i < signature.length; i++) {
+                            int b = signature[i];
+                            int idx = i << 1;
+                            arr[idx] = HEX_DIGITS[(b >> 4) & 0xf];
+                            arr[idx + 1] = HEX_DIGITS[b & 0xf];
+                        }
+                        String sigString = new String(arr);
+                        result.add(sigString);
+                    }
+                }
+            }
+            return result;
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        } catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
+    }
+
+    protected List<String> getMultiValue(ExternalDataObject object, String metadata) {
+        return object.getMetadata()
+                     .stream()
+                     .filter(metadataValueDTO ->
+                         new MetadataFieldName(metadataValueDTO.getSchema(), metadataValueDTO.getElement(),
+                             metadataValueDTO.getQualifier()).toString().equals(metadata))
+                     .map(MetadataValueDTO::getValue)
+                     .collect(Collectors.toList());
+    }
+
+    protected String normalize(ExternalDataObject object, String value) {
+        String result = value;
+        if (StringUtils.isEmpty(value)) {
+            if (StringUtils.isNotEmpty(prefix)) {
+                result = prefix + object.getId();
+            } else {
+                result = "entity:" + object.getId();
+            }
+        } else {
+            for (String prefix : ignorePrefix) {
+                if (value.startsWith(prefix)) {
+                    result = value.substring(prefix.length());
+                    break;
+                }
+            }
+            if (StringUtils.isNotEmpty(prefix)) {
+                result = prefix + result;
+            }
+        }
+
+        return result;
+    }
+
     public String getMetadata() {
         return metadata;
     }