Merge branch 'develop' into 8184-preview-url-changes

doc/release-notes/10379-MetricsBugsFixes.md

@@ -0,0 +1,10 @@
+
+### Metrics API Bug fixes
+
+Two bugs in the Metrics API have been fixed:
+
+- The /datasets and /datasets/byMonth endpoints could report incorrect values if/when they have been called using the dataLocation parameter (which allows getting metrics for local, remote (harvested), or all datasets) as the metrics cache was not storing different values for these cases. 
+
+- Metrics endpoints who's calculation relied on finding the latest published datasetversion were incorrect if/when the minor version number was > 9. 
+
+When deploying the new release, the [/api/admin/clearMetricsCache](https://guides.dataverse.org/en/latest/api/native-api.html#metrics) API should be called to remove old cached values that may be incorrect.

doc/release-notes/10708 - MDC Citation and DOI parsing improvements.md

@@ -0,0 +1,3 @@
+MDC Citation retrieval with the PID settings has been fixed.
+DOI parsing in Dataverse is case insensitive, improving interaction with services that may change the case.
+Warnings related to managed/excluded PID lists for PID providers have been reduced

doc/release-notes/10886-update-to-conditions-to-display-image_url.md

@@ -0,0 +1,8 @@
+Search API (/api/search) responses for Datafiles include image_url for the thumbnail if each of the following are true:
+1. The DataFile is not Harvested
+2. A Thumbnail is available for the Datafile
+3. If the Datafile is Restricted then the caller must have Download File Permission for the Datafile
+4. The Datafile is NOT actively embargoed
+5. The Datafile's retention period has NOT expired
+
+See also #10875 and #10886.

doc/sphinx-guides/source/style/text.rst

@@ -9,4 +9,4 @@ Here we describe the guidelines that help us provide helpful, clear and consiste
 Metadata Text Guidelines
 ========================
 
-These guidelines are maintained in `a Google Doc <https://docs.google.com/document/d/1uRk_dAZlaCS91YFbqE6L9Jwhwum7mOadkJ59XUx40Sg>`__ as we expect to make frequent changes to them. We welcome comments in the Google Doc.
+These guidelines are maintained in `a Google Doc <https://docs.google.com/document/d/1tY5t3gjrIgAGoRxVMWQSCh46fnbSmnFDLQ7aLkNLhJ8/>`__ as we expect to make frequent changes to them. We welcome comments in the Google Doc.

pom.xml

@@ -72,7 +72,7 @@
             <dependency>
                 <groupId>org.apache.james</groupId>
                 <artifactId>apache-mime4j-core</artifactId>
-                <version>0.8.7</version>
+                <version>0.8.10</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.james</groupId>

src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java

@@ -5,6 +5,7 @@
  */
 package edu.harvard.iq.dataverse;
 
+import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
@@ -20,7 +21,6 @@
 
 import jakarta.ejb.EJB;
 import jakarta.enterprise.context.RequestScoped;
-import jakarta.inject.Inject;
 import jakarta.inject.Named;
 
 /**
@@ -33,9 +33,8 @@
 public class ThumbnailServiceWrapper implements java.io.Serializable  {
 
     private static final Logger logger = Logger.getLogger(ThumbnailServiceWrapper.class.getCanonicalName());
-
-    @Inject
-    PermissionsWrapper permissionsWrapper;
+    @EJB
+    PermissionServiceBean permissionService;
     @EJB
     DataverseServiceBean dataverseService;
     @EJB
@@ -49,12 +48,15 @@ public class ThumbnailServiceWrapper implements java.io.Serializable  {
     private Map<Long, DvObject> dvobjectViewMap = new HashMap<>();
     private Map<Long, Boolean> hasThumbMap = new HashMap<>();
 
+    private boolean hasDownloadFilePermission(DvObject dvo) {
+        return permissionService.on(dvo).has(Permission.DownloadFile) ;
+    }
     public String getFileCardImageAsUrl(SolrSearchResult result) {
         DataFile dataFile = result != null && result.getEntity() != null ? ((DataFile) result.getEntity()) : null;
-        if (dataFile == null || result.isHarvested()
+        if (dataFile == null
+                || result.isHarvested()
                 || !isThumbnailAvailable(dataFile)
-                || dataFile.isRestricted()
-                || !dataFile.isReleased()
+                || (dataFile.isRestricted() && !hasDownloadFilePermission(dataFile))
                 || FileUtil.isActivelyEmbargoed(dataFile)
                 || FileUtil.isRetentionExpired(dataFile)) {
             return null;
@@ -105,7 +107,7 @@ public String getFileCardImageAsBase64Url(SolrSearchResult result) {
             }
 
             if ((!((DataFile)result.getEntity()).isRestricted()
-                        || permissionsWrapper.hasDownloadFilePermission(result.getEntity()))
+                        || hasDownloadFilePermission(result.getEntity()))
                     && isThumbnailAvailable((DataFile) result.getEntity())) {
 
                 cardImageUrl = ImageThumbConverter.getImageThumbnailAsBase64(

src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java

@@ -621,62 +621,22 @@ public Response deleteDataverse(@Context ContainerRequestContext crc, @PathParam
     public Response updateAttribute(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier,
                                     @PathParam("attribute") String attribute, @QueryParam("value") String value) {
         try {
-            Dataverse collection = findDataverseOrDie(identifier);
-            User user = getRequestUser(crc);
-            DataverseRequest dvRequest = createDataverseRequest(user);
-
-            // TODO: The cases below use hard coded strings, because we have no place for definitions of those!
-            //       They are taken from util.json.JsonParser / util.json.JsonPrinter. This shall be changed.
-            //       This also should be extended to more attributes, like the type, theme, contacts, some booleans, etc.
-            switch (attribute) {
-                case "alias":
-                    collection.setAlias(value);
-                    break;
-                case "name":
-                    collection.setName(value);
-                    break;
-                case "description":
-                    collection.setDescription(value);
-                    break;
-                case "affiliation":
-                    collection.setAffiliation(value);
-                    break;
-                /* commenting out the code from the draft pr #9462:
-                case "versionPidsConduct":
-                    CollectionConduct conduct = CollectionConduct.findBy(value);
-                    if (conduct == null) {
-                        return badRequest("'" + value + "' is not one of [" +
-                            String.join(",", CollectionConduct.asList()) + "]");
-                    }
-                    collection.setDatasetVersionPidConduct(conduct);
-                    break;
-                 */
-                case "filePIDsEnabled":
-                    if(!user.isSuperuser()) {
-                        return forbidden("You must be a superuser to change this setting");
-                    }
-                    if(!settingsService.isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
-                        return forbidden("Changing File PID policy per collection is not enabled on this server");
-                    }
-                    collection.setFilePIDsEnabled(parseBooleanOrDie(value));
-                    break;
-                default:
-                    return badRequest("'" + attribute + "' is not a supported attribute");
-            }
-
-            // Off to persistence layer
-            execCommand(new UpdateDataverseCommand(collection, null, null, dvRequest, null));
-
-            // Also return modified collection to user
-            return ok("Update successful", JsonPrinter.json(collection));
-
-        // TODO: This is an anti-pattern, necessary due to this bean being an EJB, causing very noisy and unnecessary
-        //       logging by the EJB container for bubbling exceptions. (It would be handled by the error handlers.)
+            Dataverse dataverse = findDataverseOrDie(identifier);
+            Object formattedValue = formatAttributeValue(attribute, value);
+            dataverse = execCommand(new UpdateDataverseAttributeCommand(createDataverseRequest(getRequestUser(crc)), dataverse, attribute, formattedValue));
+            return ok("Update successful", JsonPrinter.json(dataverse));
         } catch (WrappedResponse e) {
             return e.getResponse();
         }
     }
 
+    private Object formatAttributeValue(String attribute, String value) throws WrappedResponse {
+        if (attribute.equals("filePIDsEnabled")) {
+            return parseBooleanOrDie(value);
+        }
+        return value;
+    }
+
     @GET
     @AuthRequired
     @Path("{identifier}/inputLevels")

src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java

@@ -19,6 +19,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.Iterator;
 import java.util.List;
@@ -152,10 +155,17 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
             // DataCite wants "doi=", not "doi:".
             String authorityPlusIdentifier = persistentId.replaceFirst("doi:", "");
             // Request max page size and then loop to handle multiple pages
-            URL url = new URL(JvmSettings.DATACITE_REST_API_URL.lookup() +
+            URL url = null;
+            try {
+                url = new URI(JvmSettings.DATACITE_REST_API_URL.lookup(pidProvider.getId()) +
                               "/events?doi=" +
                               authorityPlusIdentifier +
-                              "&source=crossref&page[size]=1000");
+                              "&source=crossref&page[size]=1000").toURL();
+            } catch (URISyntaxException e) {
+                //Nominally this means a config error/ bad DATACITE_REST_API_URL for this provider
+                logger.warning("Unable to create URL for " + persistentId + ", pidProvider " + pidProvider.getId());
+                return error(Status.INTERNAL_SERVER_ERROR, "Unable to create DataCite URL to retrieve citations.");
+            }
             logger.fine("Retrieving Citations from " + url.toString());
             boolean nextPage = true;
             JsonArrayBuilder dataBuilder = Json.createArrayBuilder();
@@ -178,7 +188,12 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
                     dataBuilder.add(iter.next());
                 }
                 if (links.containsKey("next")) {
-                    url = new URL(links.getString("next"));
+                    try {
+                        url = new URI(links.getString("next")).toURL();
+                    } catch (URISyntaxException e) {
+                        logger.warning("Unable to create URL from DataCite response: " + links.getString("next"));
+                        return error(Status.INTERNAL_SERVER_ERROR, "Unable to retrieve all results from DataCite");
+                    }
                 } else {
                     nextPage = false;
                 }
@@ -187,7 +202,7 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
             JsonArray allData = dataBuilder.build();
             List<DatasetExternalCitations> datasetExternalCitations = datasetExternalCitationsService.parseCitations(allData);
             /*
-             * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopuate them.
+             * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopulate them.
              * As is, this call doesn't remove old citations if there are now none (legacy issue if we decide to stop counting certain types of citation
              * as we've done for 'hasPart').
              * If there are some, this call individually checks each one and if a matching item exists, it removes it and adds it back. Faster and better to delete all and

src/main/java/edu/harvard/iq/dataverse/api/Metrics.java

@@ -206,12 +206,13 @@ public Response getDatasetsTimeSeriest(@Context Request req, @Context UriInfo ur
             return error(BAD_REQUEST, ia.getLocalizedMessage());
         }
         String metricName = "datasets";
-        JsonArray jsonArray = MetricsUtil.stringToJsonArray(metricsSvc.returnUnexpiredCacheAllTime(metricName, null, d));
+        String validDataLocation = MetricsUtil.validateDataLocationStringType(dataLocation);
+        JsonArray jsonArray = MetricsUtil.stringToJsonArray(metricsSvc.returnUnexpiredCacheAllTime(metricName, validDataLocation, d));
 
         if (null == jsonArray) { // run query and save
 
-            jsonArray = metricsSvc.getDatasetsTimeSeries(uriInfo, dataLocation, d);
-            metricsSvc.save(new Metric(metricName, null, null, d, jsonArray.toString()));
+            jsonArray = metricsSvc.getDatasetsTimeSeries(uriInfo, validDataLocation, d);
+            metricsSvc.save(new Metric(metricName, null, validDataLocation, d, jsonArray.toString()));
         }
         MediaType requestedType = getVariant(req, MediaType.valueOf(FileUtil.MIME_TYPE_CSV), MediaType.APPLICATION_JSON_TYPE);
         if ((requestedType != null) && (requestedType.equals(MediaType.APPLICATION_JSON_TYPE))) {