errors = passwordValidatorService.validate(password, new Date(), false);
- final JsonArrayBuilder errorArray = Json.createArrayBuilder();
- errors.forEach(errorArray::add);
- return ok(Json.createObjectBuilder().add("password", password).add("errors", errorArray));
- }
-
- @GET
- @Path("/isOrcid")
- public Response isOrcidEnabled() {
- return authSvc.isOrcidEnabled() ? ok("Orcid is enabled") : ok("no orcid for you.");
- }
+ /**
+ * This method is used in API tests, called from UtilIt.java.
+ */
+ @GET
+ @Path("datasets/thumbnailMetadata/{id}")
+ public Response getDatasetThumbnailMetadata(@PathParam("id") Long idSupplied) {
+ Dataset dataset = datasetSvc.find(idSupplied);
+ if (dataset == null) {
+ return error(Response.Status.NOT_FOUND, "Could not find dataset based on id supplied: " + idSupplied + ".");
+ }
+ JsonObjectBuilder data = Json.createObjectBuilder();
+ DatasetThumbnail datasetThumbnail = dataset.getDatasetThumbnail(ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE);
+ data.add("isUseGenericThumbnail", dataset.isUseGenericThumbnail());
+ data.add("datasetLogoPresent", DatasetUtil.isDatasetLogoPresent(dataset, ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE));
+ if (datasetThumbnail != null) {
+ data.add("datasetThumbnailBase64image", datasetThumbnail.getBase64image());
+ DataFile dataFile = datasetThumbnail.getDataFile();
+ if (dataFile != null) {
+ /**
+ * @todo Change this from a String to a long.
+ */
+ data.add("dataFileId", dataFile.getId().toString());
+ }
+ }
+ return ok(data);
+ }
+
+ /**
+ * validatePassword
+ *
+ * Validate a password with an API call
+ *
+ * @param password
+ * The password
+ * @return A response with the validation result.
+ */
+ @Path("validatePassword")
+ @POST
+ public Response validatePassword(String password) {
+
+ final List errors = passwordValidatorService.validate(password, new Date(), false);
+ final JsonArrayBuilder errorArray = Json.createArrayBuilder();
+ errors.forEach(errorArray::add);
+ return ok(Json.createObjectBuilder().add("password", password).add("errors", errorArray));
+ }
+
+ @GET
+ @Path("/isOrcid")
+ public Response isOrcidEnabled() {
+ return authSvc.isOrcidEnabled() ? ok("Orcid is enabled") : ok("no orcid for you.");
+ }
@POST
- @AuthRequired
+ @AuthRequired
@Path("{id}/reregisterHDLToPID")
public Response reregisterHdlToPID(@Context ContainerRequestContext crc, @PathParam("id") String id) {
logger.info("Starting to reregister " + id + " Dataset Id. (from hdl to doi)" + new Date());
@@ -1825,7 +1805,7 @@ public Response updateHashValues(@Context ContainerRequestContext crc, @PathPara
}
@POST
- @AuthRequired
+ @AuthRequired
@Path("/computeDataFileHashValue/{fileId}/algorithm/{alg}")
public Response computeDataFileHashValue(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @PathParam("alg") String alg) {
@@ -1887,7 +1867,7 @@ public Response computeDataFileHashValue(@Context ContainerRequestContext crc, @
}
@POST
- @AuthRequired
+ @AuthRequired
@Path("/validateDataFileHashValue/{fileId}")
public Response validateDataFileHashValue(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId) {
@@ -1954,7 +1934,7 @@ public Response validateDataFileHashValue(@Context ContainerRequestContext crc,
}
@POST
- @AuthRequired
+ @AuthRequired
@Path("/submitDatasetVersionToArchive/{id}/{version}")
public Response submitDatasetVersionToArchive(@Context ContainerRequestContext crc, @PathParam("id") String dsid,
@PathParam("version") String versionNumber) {
@@ -2027,7 +2007,7 @@ public void run() {
* @return
*/
@POST
- @AuthRequired
+ @AuthRequired
@Path("/archiveAllUnarchivedDatasetVersions")
public Response archiveAllUnarchivedDatasetVersions(@Context ContainerRequestContext crc, @QueryParam("listonly") boolean listonly, @QueryParam("limit") Integer limit, @QueryParam("latestonly") boolean latestonly) {
@@ -2126,7 +2106,7 @@ public Response clearMetricsCacheByName(@PathParam("name") String name) {
}
@GET
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/addRoleAssignmentsToChildren")
public Response addRoleAssignementsToChildren(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse {
Dataverse owner = dataverseSvc.findByAlias(alias);
@@ -2157,90 +2137,90 @@ public Response addRoleAssignementsToChildren(@Context ContainerRequestContext c
}
@GET
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/storageDriver")
public Response getStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse {
- Dataverse dataverse = dataverseSvc.findByAlias(alias);
- if (dataverse == null) {
- return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
- }
- try {
- AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
- if (!user.isSuperuser()) {
- return error(Response.Status.FORBIDDEN, "Superusers only.");
- }
- } catch (WrappedResponse wr) {
- return wr.getResponse();
- }
- //Note that this returns what's set directly on this dataverse. If null/DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER, the user would have to recurse the chain of parents to find the effective storageDriver
- return ok(dataverse.getStorageDriverId());
+ Dataverse dataverse = dataverseSvc.findByAlias(alias);
+ if (dataverse == null) {
+ return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
+ }
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+ //Note that this returns what's set directly on this dataverse. If null/DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER, the user would have to recurse the chain of parents to find the effective storageDriver
+ return ok(dataverse.getStorageDriverId());
}
@PUT
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/storageDriver")
public Response setStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias, String label) throws WrappedResponse {
- Dataverse dataverse = dataverseSvc.findByAlias(alias);
- if (dataverse == null) {
- return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
- }
- try {
- AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
- if (!user.isSuperuser()) {
- return error(Response.Status.FORBIDDEN, "Superusers only.");
- }
- } catch (WrappedResponse wr) {
- return wr.getResponse();
- }
- for (Entry store: DataAccess.getStorageDriverLabels().entrySet()) {
- if(store.getKey().equals(label)) {
- dataverse.setStorageDriverId(store.getValue());
- return ok("Storage set to: " + store.getKey() + "/" + store.getValue());
- }
- }
- return error(Response.Status.BAD_REQUEST,
- "No Storage Driver found for : " + label);
+ Dataverse dataverse = dataverseSvc.findByAlias(alias);
+ if (dataverse == null) {
+ return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
+ }
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+ for (Entry store: DataAccess.getStorageDriverLabels().entrySet()) {
+ if(store.getKey().equals(label)) {
+ dataverse.setStorageDriverId(store.getValue());
+ return ok("Storage set to: " + store.getKey() + "/" + store.getValue());
+ }
+ }
+ return error(Response.Status.BAD_REQUEST,
+ "No Storage Driver found for : " + label);
}
@DELETE
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/storageDriver")
public Response resetStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse {
- Dataverse dataverse = dataverseSvc.findByAlias(alias);
- if (dataverse == null) {
- return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
- }
- try {
- AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
- if (!user.isSuperuser()) {
- return error(Response.Status.FORBIDDEN, "Superusers only.");
- }
- } catch (WrappedResponse wr) {
- return wr.getResponse();
- }
- dataverse.setStorageDriverId("");
- return ok("Storage reset to default: " + DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
+ Dataverse dataverse = dataverseSvc.findByAlias(alias);
+ if (dataverse == null) {
+ return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
+ }
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+ dataverse.setStorageDriverId("");
+ return ok("Storage reset to default: " + DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
}
@GET
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/storageDrivers")
public Response listStorageDrivers(@Context ContainerRequestContext crc) throws WrappedResponse {
- try {
- AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
- if (!user.isSuperuser()) {
- return error(Response.Status.FORBIDDEN, "Superusers only.");
- }
- } catch (WrappedResponse wr) {
- return wr.getResponse();
- }
- JsonObjectBuilder bld = jsonObjectBuilder();
- DataAccess.getStorageDriverLabels().entrySet().forEach(s -> bld.add(s.getKey(), s.getValue()));
- return ok(bld);
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+ JsonObjectBuilder bld = jsonObjectBuilder();
+ DataAccess.getStorageDriverLabels().entrySet().forEach(s -> bld.add(s.getKey(), s.getValue()));
+ return ok(bld);
}
@GET
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/curationLabelSet")
public Response getCurationLabelSet(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse {
Dataverse dataverse = dataverseSvc.findByAlias(alias);
@@ -2262,7 +2242,7 @@ public Response getCurationLabelSet(@Context ContainerRequestContext crc, @PathP
}
@PUT
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/curationLabelSet")
public Response setCurationLabelSet(@Context ContainerRequestContext crc, @PathParam("alias") String alias, @QueryParam("name") String name) throws WrappedResponse {
Dataverse dataverse = dataverseSvc.findByAlias(alias);
@@ -2293,7 +2273,7 @@ public Response setCurationLabelSet(@Context ContainerRequestContext crc, @PathP
}
@DELETE
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/curationLabelSet")
public Response resetCurationLabelSet(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse {
Dataverse dataverse = dataverseSvc.findByAlias(alias);
@@ -2313,7 +2293,7 @@ public Response resetCurationLabelSet(@Context ContainerRequestContext crc, @Pat
}
@GET
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/curationLabelSets")
public Response listCurationLabelSets(@Context ContainerRequestContext crc) throws WrappedResponse {
try {
@@ -2423,7 +2403,7 @@ public Response getBannerMessages(@PathParam("id") Long id) throws WrappedRespon
}
@POST
- @AuthRequired
+ @AuthRequired
@Consumes("application/json")
@Path("/requestSignedUrl")
public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject urlInfo) {
@@ -2541,4 +2521,160 @@ public Response getFeatureFlag(@PathParam("flag") String flagIn) {
}
}
+ @GET
+ @AuthRequired
+ @Path("/datafiles/auditFiles")
+ public Response getAuditFiles(@Context ContainerRequestContext crc,
+ @QueryParam("firstId") Long firstId, @QueryParam("lastId") Long lastId,
+ @QueryParam("datasetIdentifierList") String datasetIdentifierList) throws WrappedResponse {
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+
+ int datasetsChecked = 0;
+ long startId = (firstId == null ? 0 : firstId);
+ long endId = (lastId == null ? Long.MAX_VALUE : lastId);
+
+ List datasetIdentifiers;
+ if (datasetIdentifierList == null || datasetIdentifierList.isEmpty()) {
+ datasetIdentifiers = Collections.emptyList();
+ } else {
+ startId = 0;
+ endId = Long.MAX_VALUE;
+ datasetIdentifiers = List.of(datasetIdentifierList.split(","));
+ }
+ if (endId < startId) {
+ return badRequest("Invalid Parameters: lastId must be equal to or greater than firstId");
+ }
+
+ NullSafeJsonBuilder jsonObjectBuilder = NullSafeJsonBuilder.jsonObjectBuilder();
+ JsonArrayBuilder jsonDatasetsArrayBuilder = Json.createArrayBuilder();
+ JsonArrayBuilder jsonFailuresArrayBuilder = Json.createArrayBuilder();
+
+ if (startId > 0) {
+ jsonObjectBuilder.add("firstId", startId);
+ }
+ if (endId < Long.MAX_VALUE) {
+ jsonObjectBuilder.add("lastId", endId);
+ }
+
+ // compile the list of ids to process
+ List datasetIds;
+ if (datasetIdentifiers.isEmpty()) {
+ datasetIds = datasetService.findAllLocalDatasetIds();
+ } else {
+ datasetIds = new ArrayList<>(datasetIdentifiers.size());
+ JsonArrayBuilder jab = Json.createArrayBuilder();
+ datasetIdentifiers.forEach(id -> {
+ String dId = id.trim();
+ jab.add(dId);
+ Dataset d = datasetService.findByGlobalId(dId);
+ if (d != null) {
+ datasetIds.add(d.getId());
+ } else {
+ NullSafeJsonBuilder job = NullSafeJsonBuilder.jsonObjectBuilder();
+ job.add("datasetIdentifier",dId);
+ job.add("reason","Not Found");
+ jsonFailuresArrayBuilder.add(job);
+ }
+ });
+ jsonObjectBuilder.add("datasetIdentifierList", jab);
+ }
+
+ for (Long datasetId : datasetIds) {
+ if (datasetId < startId) {
+ continue;
+ } else if (datasetId > endId) {
+ break;
+ }
+ Dataset dataset;
+ try {
+ dataset = findDatasetOrDie(String.valueOf(datasetId));
+ datasetsChecked++;
+ } catch (WrappedResponse e) {
+ NullSafeJsonBuilder job = NullSafeJsonBuilder.jsonObjectBuilder();
+ job.add("datasetId", datasetId);
+ job.add("reason", e.getMessage());
+ jsonFailuresArrayBuilder.add(job);
+ continue;
+ }
+
+ List missingFiles = new ArrayList<>();
+ List missingFileMetadata = new ArrayList<>();
+ try {
+ Predicate filter = s -> true;
+ StorageIO datasetIO = DataAccess.getStorageIO(dataset);
+ final List result = datasetIO.cleanUp(filter, true);
+ // add files that are in dataset files but not in cleanup result or DataFiles with missing FileMetadata
+ dataset.getFiles().forEach(df -> {
+ try {
+ StorageIO datafileIO = df.getStorageIO();
+ String storageId = df.getStorageIdentifier();
+ FileMetadata fm = df.getFileMetadata();
+ if (!datafileIO.exists()) {
+ missingFiles.add(storageId + "," + (fm != null ?
+ (fm.getDirectoryLabel() != null || !fm.getDirectoryLabel().isEmpty() ? "directoryLabel,"+fm.getDirectoryLabel()+"," : "")
+ +"label,"+fm.getLabel() : "type,"+df.getContentType()));
+ }
+ if (fm == null) {
+ missingFileMetadata.add(storageId + ",dataFileId," + df.getId());
+ }
+ } catch (IOException e) {
+ NullSafeJsonBuilder job = NullSafeJsonBuilder.jsonObjectBuilder();
+ job.add("dataFileId", df.getId());
+ job.add("reason", e.getMessage());
+ jsonFailuresArrayBuilder.add(job);
+ }
+ });
+ } catch (IOException e) {
+ NullSafeJsonBuilder job = NullSafeJsonBuilder.jsonObjectBuilder();
+ job.add("datasetId", datasetId);
+ job.add("reason", e.getMessage());
+ jsonFailuresArrayBuilder.add(job);
+ }
+
+ JsonObjectBuilder job = Json.createObjectBuilder();
+ if (!missingFiles.isEmpty() || !missingFileMetadata.isEmpty()) {
+ job.add("id", dataset.getId());
+ job.add("pid", dataset.getProtocol() + ":" + dataset.getAuthority() + "/" + dataset.getIdentifier());
+ job.add("persistentURL", dataset.getPersistentURL());
+ if (!missingFileMetadata.isEmpty()) {
+ JsonArrayBuilder jabMissingFileMetadata = Json.createArrayBuilder();
+ missingFileMetadata.forEach(mm -> {
+ String[] missingMetadata = mm.split(",");
+ NullSafeJsonBuilder jobj = NullSafeJsonBuilder.jsonObjectBuilder()
+ .add("storageIdentifier", missingMetadata[0])
+ .add(missingMetadata[1], missingMetadata[2]);
+ jabMissingFileMetadata.add(jobj);
+ });
+ job.add("missingFileMetadata", jabMissingFileMetadata);
+ }
+ if (!missingFiles.isEmpty()) {
+ JsonArrayBuilder jabMissingFiles = Json.createArrayBuilder();
+ missingFiles.forEach(mf -> {
+ String[] missingFile = mf.split(",");
+ NullSafeJsonBuilder jobj = NullSafeJsonBuilder.jsonObjectBuilder()
+ .add("storageIdentifier", missingFile[0]);
+ for (int i = 2; i < missingFile.length; i+=2) {
+ jobj.add(missingFile[i-1], missingFile[i]);
+ }
+ jabMissingFiles.add(jobj);
+ });
+ job.add("missingFiles", jabMissingFiles);
+ }
+ jsonDatasetsArrayBuilder.add(job);
+ }
+ }
+
+ jsonObjectBuilder.add("datasetsChecked", datasetsChecked);
+ jsonObjectBuilder.add("datasets", jsonDatasetsArrayBuilder);
+ jsonObjectBuilder.add("failures", jsonFailuresArrayBuilder);
+
+ return ok(jsonObjectBuilder);
+ }
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java
index 01c51dc2b4c..907295ad848 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java
@@ -126,7 +126,7 @@ public Response getByName(@PathParam("name") String name) {
String solrFieldSearchable = dsf.getSolrField().getNameSearchable();
String solrFieldFacetable = dsf.getSolrField().getNameFacetable();
String metadataBlock = dsf.getMetadataBlock().getName();
- String uri=dsf.getUri();
+ String uri = dsf.getUri();
boolean hasParent = dsf.isHasParent();
boolean allowsMultiples = dsf.isAllowMultiples();
boolean isRequired = dsf.isRequired();
@@ -243,7 +243,9 @@ public Response loadDatasetFields(File file) {
br = new BufferedReader(new FileReader("/" + file));
while ((line = br.readLine()) != null) {
lineNumber++;
- values = line.split(splitBy);
+ values = Arrays.stream(line.split(splitBy))
+ .map(String::trim)
+ .toArray(String[]::new);
if (values[0].startsWith("#")) { // Header row
switch (values[0]) {
case "#metadataBlock":
@@ -326,7 +328,7 @@ public Response loadDatasetFields(File file) {
*/
public String getGeneralErrorMessage(HeaderType header, int lineNumber, String message) {
List arguments = new ArrayList<>();
- arguments.add(header.name());
+ arguments.add(header != null ? header.name() : "unknown");
arguments.add(String.valueOf(lineNumber));
arguments.add(message);
return BundleUtil.getStringFromBundle("api.admin.datasetfield.load.GeneralErrorMessage", arguments);
@@ -334,9 +336,9 @@ public String getGeneralErrorMessage(HeaderType header, int lineNumber, String m
/**
* Turn ArrayIndexOutOfBoundsException into an informative error message
- * @param lineNumber
* @param header
- * @param e
+ * @param lineNumber
+ * @param wrongIndex
* @return
*/
public String getArrayIndexOutOfBoundMessage(HeaderType header,
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 369a22fe8d7..2ec10816acc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -2171,8 +2171,32 @@ public Response getAssignments(@Context ContainerRequestContext crc, @PathParam(
@GET
@AuthRequired
+ @Deprecated(forRemoval = true, since = "2024-10-17")
@Path("{id}/privateUrl")
public Response getPrivateUrlData(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) {
+ return getPreviewUrlData(crc, idSupplied);
+ }
+
+ @POST
+ @AuthRequired
+ @Deprecated(forRemoval = true, since = "2024-10-17")
+ @Path("{id}/privateUrl")
+ public Response createPrivateUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied, @DefaultValue("false") @QueryParam("anonymizedAccess") boolean anonymizedAccess) {
+ return createPreviewUrl(crc, idSupplied, anonymizedAccess);
+ }
+
+ @DELETE
+ @AuthRequired
+ @Deprecated(forRemoval = true, since = "2024-10-17")
+ @Path("{id}/privateUrl")
+ public Response deletePrivateUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) {
+ return deletePreviewUrl(crc, idSupplied);
+ }
+
+ @GET
+ @AuthRequired
+ @Path("{id}/previewUrl")
+ public Response getPreviewUrlData(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) {
return response( req -> {
PrivateUrl privateUrl = execCommand(new GetPrivateUrlCommand(req, findDatasetOrDie(idSupplied)));
return (privateUrl != null) ? ok(json(privateUrl))
@@ -2182,8 +2206,8 @@ public Response getPrivateUrlData(@Context ContainerRequestContext crc, @PathPar
@POST
@AuthRequired
- @Path("{id}/privateUrl")
- public Response createPrivateUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied,@DefaultValue("false") @QueryParam ("anonymizedAccess") boolean anonymizedAccess) {
+ @Path("{id}/previewUrl")
+ public Response createPreviewUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied,@DefaultValue("false") @QueryParam ("anonymizedAccess") boolean anonymizedAccess) {
if(anonymizedAccess && settingsSvc.getValueForKey(SettingsServiceBean.Key.AnonymizedFieldTypeNames)==null) {
throw new NotAcceptableException("Anonymized Access not enabled");
}
@@ -2194,8 +2218,8 @@ public Response createPrivateUrl(@Context ContainerRequestContext crc, @PathPara
@DELETE
@AuthRequired
- @Path("{id}/privateUrl")
- public Response deletePrivateUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) {
+ @Path("{id}/previewUrl")
+ public Response deletePreviewUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) {
return response( req -> {
Dataset dataset = findDatasetOrDie(idSupplied);
PrivateUrl privateUrl = execCommand(new GetPrivateUrlCommand(req, dataset));
@@ -2208,6 +2232,7 @@ public Response deletePrivateUrl(@Context ContainerRequestContext crc, @PathPara
}, getRequestUser(crc));
}
+
@GET
@AuthRequired
@Path("{id}/thumbnail/candidates")
@@ -2992,6 +3017,26 @@ public Response cleanStorage(@Context ContainerRequestContext crc, @PathParam("i
}
+ @GET
+ @AuthRequired
+ @Path("{id}/versions/{versionId1}/compare/{versionId2}")
+ public Response getCompareVersions(@Context ContainerRequestContext crc, @PathParam("id") String id,
+ @PathParam("versionId1") String versionId1,
+ @PathParam("versionId2") String versionId2,
+ @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+ try {
+ DataverseRequest req = createDataverseRequest(getRequestUser(crc));
+ DatasetVersion dsv1 = getDatasetVersionOrDie(req, versionId1, findDatasetOrDie(id), uriInfo, headers);
+ DatasetVersion dsv2 = getDatasetVersionOrDie(req, versionId2, findDatasetOrDie(id), uriInfo, headers);
+ if (dsv1.getCreateTime().getTime() > dsv2.getCreateTime().getTime()) {
+ return error(BAD_REQUEST, BundleUtil.getStringFromBundle("dataset.version.compare.incorrect.order"));
+ }
+ return ok(DatasetVersion.compareVersions(dsv1, dsv2));
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+ }
+
private static Set getDatasetFilenames(Dataset dataset) {
Set files = new HashSet<>();
for (DataFile dataFile: dataset.getFiles()) {
@@ -4833,6 +4878,33 @@ public Response getPrivateUrlDatasetVersion(@PathParam("privateUrlToken") String
}
return ok(responseJson);
}
+
+ @GET
+ @Path("previewUrlDatasetVersion/{previewUrlToken}")
+ public Response getPreviewUrlDatasetVersion(@PathParam("previewUrlToken") String previewUrlToken, @QueryParam("returnOwners") boolean returnOwners) {
+ PrivateUrlUser privateUrlUser = privateUrlService.getPrivateUrlUserFromToken(previewUrlToken);
+ if (privateUrlUser == null) {
+ return notFound("Private URL user not found");
+ }
+ boolean isAnonymizedAccess = privateUrlUser.hasAnonymizedAccess();
+ String anonymizedFieldTypeNames = settingsSvc.getValueForKey(SettingsServiceBean.Key.AnonymizedFieldTypeNames);
+ if(isAnonymizedAccess && anonymizedFieldTypeNames == null) {
+ throw new NotAcceptableException("Anonymized Access not enabled");
+ }
+ DatasetVersion dsv = privateUrlService.getDraftDatasetVersionFromToken(previewUrlToken);
+ if (dsv == null || dsv.getId() == null) {
+ return notFound("Dataset version not found");
+ }
+ JsonObjectBuilder responseJson;
+ if (isAnonymizedAccess) {
+ List anonymizedFieldTypeNamesList = new ArrayList<>(Arrays.asList(anonymizedFieldTypeNames.split(",\\s")));
+ responseJson = json(dsv, anonymizedFieldTypeNamesList, true, returnOwners);
+ } else {
+ responseJson = json(dsv, null, true, returnOwners);
+ }
+ return ok(responseJson);
+ }
+
@GET
@Path("privateUrlDatasetVersion/{privateUrlToken}/citation")
@@ -4845,6 +4917,18 @@ public Response getPrivateUrlDatasetVersionCitation(@PathParam("privateUrlToken"
return (dsv == null || dsv.getId() == null) ? notFound("Dataset version not found")
: ok(dsv.getCitation(true, privateUrlUser.hasAnonymizedAccess()));
}
+
+ @GET
+ @Path("previewUrlDatasetVersion/{previewUrlToken}/citation")
+ public Response getPreviewUrlDatasetVersionCitation(@PathParam("previewUrlToken") String previewUrlToken) {
+ PrivateUrlUser privateUrlUser = privateUrlService.getPrivateUrlUserFromToken(previewUrlToken);
+ if (privateUrlUser == null) {
+ return notFound("Private URL user not found");
+ }
+ DatasetVersion dsv = privateUrlService.getDraftDatasetVersionFromToken(previewUrlToken);
+ return (dsv == null || dsv.getId() == null) ? notFound("Dataset version not found")
+ : ok(dsv.getCitation(true, privateUrlUser.hasAnonymizedAccess()));
+ }
@GET
@AuthRequired
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index f05bba8830e..f864a5a9d1c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -195,7 +195,7 @@ public Response updateDataverse(@Context ContainerRequestContext crc, String bod
List facets = parseFacets(body);
AuthenticatedUser u = getRequestAuthenticatedUserOrDie(crc);
- dataverse = execCommand(new UpdateDataverseCommand(dataverse, facets, null, createDataverseRequest(u), inputLevels, metadataBlocks, updatedDataverseDTO));
+ dataverse = execCommand(new UpdateDataverseCommand(dataverse, facets, null, createDataverseRequest(u), inputLevels, metadataBlocks, updatedDataverseDTO, true));
return ok(json(dataverse));
} catch (WrappedResponse ww) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Info.java b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
index 257519677d3..2439c996816 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Info.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
@@ -12,12 +12,17 @@
import jakarta.ws.rs.Produces;
import org.apache.commons.io.IOUtils;
+import edu.harvard.iq.dataverse.export.ExportService;
import edu.harvard.iq.dataverse.settings.JvmSettings;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
import edu.harvard.iq.dataverse.util.BundleUtil;
import edu.harvard.iq.dataverse.util.SystemConfig;
+import io.gdcc.spi.export.Exporter;
+import io.gdcc.spi.export.ExportException;
+import io.gdcc.spi.export.XMLExporter;
import jakarta.ejb.EJB;
import jakarta.json.Json;
+import jakarta.json.JsonObjectBuilder;
import jakarta.json.JsonValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
@@ -92,6 +97,32 @@ public Response getZipDownloadLimit() {
return ok(zipDownloadLimit);
}
+ @GET
+ @Path("exportFormats")
+ public Response getExportFormats() {
+ JsonObjectBuilder responseModel = Json.createObjectBuilder();
+ ExportService instance = ExportService.getInstance();
+ for (String[] labels : instance.getExportersLabels()) {
+ try {
+ Exporter exporter = instance.getExporter(labels[1]);
+ JsonObjectBuilder exporterObject = Json.createObjectBuilder().add("displayName", labels[0])
+ .add("mediaType", exporter.getMediaType()).add("isHarvestable", exporter.isHarvestable())
+ .add("isVisibleInUserInterface", exporter.isAvailableToUsers());
+ if (exporter instanceof XMLExporter xmlExporter) {
+ exporterObject.add("XMLNameSpace", xmlExporter.getXMLNameSpace())
+ .add("XMLSchemaLocation", xmlExporter.getXMLSchemaLocation())
+ .add("XMLSchemaVersion", xmlExporter.getXMLSchemaVersion());
+ }
+ responseModel.add(labels[1], exporterObject);
+ }
+ catch (ExportException ex){
+ logger.warning("Failed to get: " + labels[1]);
+ logger.warning(ex.getLocalizedMessage());
+ }
+ }
+ return ok(responseModel);
+ }
+
private Response getSettingResponseByKey(SettingsServiceBean.Key key) {
String setting = settingsService.getValueForKey(key);
if (setting != null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java b/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java
index 33a11a2df23..e6519c9ff36 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java
@@ -181,7 +181,7 @@ public Response delete(@PathParam("id") long doomedId, @QueryParam("unlink") boo
try {
wasDeleted = savedSearchSvc.delete(doomedId, unlink);
} catch (Exception e) {
- return error(INTERNAL_SERVER_ERROR, "Problem while trying to unlink links of saved search id " + doomedId);
+ return error(INTERNAL_SERVER_ERROR, "Problem while trying to unlink links of saved search id " + doomedId + ". Exception: " + e.getLocalizedMessage());
}
if (wasDeleted) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java
index 6b9fcb38305..f86f9f446fa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java
@@ -175,7 +175,7 @@ public Response search(
JsonArrayBuilder itemsArrayBuilder = Json.createArrayBuilder();
List solrSearchResults = solrQueryResponse.getSolrSearchResults();
for (SolrSearchResult solrSearchResult : solrSearchResults) {
- itemsArrayBuilder.add(solrSearchResult.json(showRelevance, showEntityIds, showApiUrls, metadataFields, getDatasetFileCount(solrSearchResult)));
+ itemsArrayBuilder.add(solrSearchResult.json(showRelevance, showEntityIds, showApiUrls, metadataFields));
}
JsonObjectBuilder spelling_alternatives = Json.createObjectBuilder();
@@ -229,15 +229,6 @@ public Response search(
}
}
- private Long getDatasetFileCount(SolrSearchResult solrSearchResult) {
- DvObject dvObject = solrSearchResult.getEntity();
- if (dvObject.isInstanceofDataset()) {
- DatasetVersion datasetVersion = ((Dataset) dvObject).getVersionFromId(solrSearchResult.getDatasetVersionId());
- return datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion);
- }
- return null;
- }
-
private User getUser(ContainerRequestContext crc) throws WrappedResponse {
User userToExecuteSearchAs = GuestUser.get();
try {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
index 85d4868605d..35d35316f73 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
@@ -210,7 +210,7 @@ private void processDDI(ImportType importType, XMLStreamReader xmlr, DatasetDTO
// study description section. we'll use the one we found in
// the codeBook entry:
FieldDTO otherIdValue = FieldDTO.createPrimitiveFieldDTO("otherIdValue", codeBookLevelId);
- FieldDTO otherId = FieldDTO.createCompoundFieldDTO("otherId", otherIdValue);
+ FieldDTO otherId = FieldDTO.createMultipleCompoundFieldDTO("otherId", otherIdValue);
citationBlock.getFields().add(otherId);
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java
index d32a548c8bf..aa5b25e3967 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java
@@ -150,12 +150,16 @@ public DatasetDTO processXML( XMLStreamReader xmlr, ForeignMetadataFormatMapping
}
- // Helper method for importing harvested Dublin Core xml.
+ // Helper methods for importing harvested Dublin Core xml.
// Dublin Core is considered a mandatory, built in metadata format mapping.
// It is distributed as required content, in reference_data.sql.
// Note that arbitrary formatting tags are supported for the outer xml
// wrapper. -- L.A. 4.5
public DatasetDTO processOAIDCxml(String DcXmlToParse) throws XMLStreamException {
+ return processOAIDCxml(DcXmlToParse, null, false);
+ }
+
+ public DatasetDTO processOAIDCxml(String DcXmlToParse, String oaiIdentifier, boolean preferSuppliedIdentifier) throws XMLStreamException {
// look up DC metadata mapping:
ForeignMetadataFormatMapping dublinCoreMapping = findFormatMappingByName(DCTERMS);
@@ -185,18 +189,37 @@ public DatasetDTO processOAIDCxml(String DcXmlToParse) throws XMLStreamException
datasetDTO.getDatasetVersion().setVersionState(DatasetVersion.VersionState.RELEASED);
- // Our DC import handles the contents of the dc:identifier field
- // as an "other id". In the context of OAI harvesting, we expect
- // the identifier to be a global id, so we need to rearrange that:
+ // In some cases, the identifier that we want to use for the dataset is
+ // already supplied to the method explicitly. For example, in some
+ // harvesting cases we'll want to use the OAI identifier (the identifier
+ // from the section of the OAI record) for that purpose, without
+ // expecting to find a valid persistent id in the body of the DC record:
- String identifier = getOtherIdFromDTO(datasetDTO.getDatasetVersion());
- logger.fine("Imported identifier: "+identifier);
+ String globalIdentifier;
- String globalIdentifier = reassignIdentifierAsGlobalId(identifier, datasetDTO);
- logger.fine("Detected global identifier: "+globalIdentifier);
+ if (oaiIdentifier != null) {
+ logger.fine("Attempting to use " + oaiIdentifier + " as the persistentId of the imported dataset");
+
+ globalIdentifier = reassignIdentifierAsGlobalId(oaiIdentifier, datasetDTO);
+ } else {
+ // Our DC import handles the contents of the dc:identifier field
+ // as an "other id". Unless we are using an externally supplied
+ // global id, we will be using the first such "other id" that we
+ // can parse and recognize as the global id for the imported dataset
+ // (note that this is the default behavior during harvesting),
+ // so we need to reaassign it accordingly:
+ String identifier = selectIdentifier(datasetDTO.getDatasetVersion(), oaiIdentifier, preferSuppliedIdentifier);
+ logger.fine("Imported identifier: " + identifier);
+
+ globalIdentifier = reassignIdentifierAsGlobalId(identifier, datasetDTO);
+ logger.fine("Detected global identifier: " + globalIdentifier);
+ }
if (globalIdentifier == null) {
- throw new EJBException("Failed to find a global identifier in the OAI_DC XML record.");
+ String exceptionMsg = oaiIdentifier == null ?
+ "Failed to find a global identifier in the OAI_DC XML record." :
+ "Failed to parse the supplied identifier as a valid Persistent Id";
+ throw new EJBException(exceptionMsg);
}
return datasetDTO;
@@ -205,8 +228,17 @@ public DatasetDTO processOAIDCxml(String DcXmlToParse) throws XMLStreamException
private void processXMLElement(XMLStreamReader xmlr, String currentPath, String openingTag, ForeignMetadataFormatMapping foreignFormatMapping, DatasetDTO datasetDTO) throws XMLStreamException {
logger.fine("entering processXMLElement; ("+currentPath+")");
-
- for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
+
+ while (xmlr.hasNext()) {
+
+ int event;
+ try {
+ event = xmlr.next();
+ } catch (XMLStreamException ex) {
+ logger.warning("Error occurred in the XML parsing : " + ex.getMessage());
+ continue; // Skip Undeclared namespace prefix and Unexpected close tag related to com.ctc.wstx.exc.WstxParsingException
+ }
+
if (event == XMLStreamConstants.START_ELEMENT) {
String currentElement = xmlr.getLocalName();
@@ -335,8 +367,20 @@ private FieldDTO makeDTO(DatasetFieldType dataverseFieldType, FieldDTO value, St
return value;
}
- private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) {
+ public String selectIdentifier(DatasetVersionDTO datasetVersionDTO, String suppliedIdentifier) {
+ return selectIdentifier(datasetVersionDTO, suppliedIdentifier, false);
+ }
+
+ private String selectIdentifier(DatasetVersionDTO datasetVersionDTO, String suppliedIdentifier, boolean preferSuppliedIdentifier) {
List otherIds = new ArrayList<>();
+
+ if (suppliedIdentifier != null && preferSuppliedIdentifier) {
+ // This supplied identifier (in practice, his is likely the OAI-PMH
+ // identifier from the section) will be our first
+ // choice candidate for the pid of the imported dataset:
+ otherIds.add(suppliedIdentifier);
+ }
+
for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) {
String key = entry.getKey();
MetadataBlockDTO value = entry.getValue();
@@ -354,6 +398,16 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) {
}
}
}
+
+ if (suppliedIdentifier != null && !preferSuppliedIdentifier) {
+ // Unless specifically instructed to prefer this extra identifier
+ // (in practice, this is likely the OAI-PMH identifier from the
+ // section), we will try to use it as the *last*
+ // possible candidate for the pid, so, adding it to the end of the
+ // list:
+ otherIds.add(suppliedIdentifier);
+ }
+
if (!otherIds.isEmpty()) {
// We prefer doi or hdl identifiers like "doi:10.7910/DVN/1HE30F"
for (String otherId : otherIds) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index b203738a9fd..7dc2aed799e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -208,7 +208,13 @@ public JsonObjectBuilder handleFile(DataverseRequest dataverseRequest, Dataverse
}
@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, String harvestIdentifier, String metadataFormat, File metadataFile, Date oaiDateStamp, PrintWriter cleanupLog) throws ImportException, IOException {
+ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest,
+ HarvestingClient harvestingClient,
+ String harvestIdentifier,
+ String metadataFormat,
+ File metadataFile,
+ Date oaiDateStamp,
+ PrintWriter cleanupLog) throws ImportException, IOException {
if (harvestingClient == null || harvestingClient.getDataverse() == null) {
throw new ImportException("importHarvestedDataset called with a null harvestingClient, or an invalid harvestingClient.");
}
@@ -244,8 +250,8 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
} else if ("dc".equalsIgnoreCase(metadataFormat) || "oai_dc".equals(metadataFormat)) {
logger.fine("importing DC "+metadataFile.getAbsolutePath());
try {
- String xmlToParse = new String(Files.readAllBytes(metadataFile.toPath()));
- dsDTO = importGenericService.processOAIDCxml(xmlToParse);
+ String xmlToParse = new String(Files.readAllBytes(metadataFile.toPath()));
+ dsDTO = importGenericService.processOAIDCxml(xmlToParse, harvestIdentifier, harvestingClient.isUseOaiIdentifiersAsPids());
} catch (IOException | XMLStreamException e) {
throw new ImportException("Failed to process Dublin Core XML record: "+ e.getClass() + " (" + e.getMessage() + ")");
}
@@ -313,7 +319,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
// Creating a new dataset from scratch:
harvestedDataset = parser.parseDataset(obj);
-
+
harvestedDataset.setHarvestedFrom(harvestingClient);
harvestedDataset.setHarvestIdentifier(harvestIdentifier);
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 3bf2107e52b..d0da66c38e0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -215,7 +215,7 @@ public long retrieveSizeFromMedia() {
JsonArray dataArray = responseJson.getJsonArray("DATA");
if (dataArray != null && dataArray.size() != 0) {
//File found
- return (long) responseJson.getJsonArray("DATA").getJsonObject(0).getJsonNumber("size").longValueExact();
+ return (long) dataArray.getJsonObject(0).getJsonNumber("size").longValueExact();
}
} else {
logger.warning("Response from " + get.getURI().toString() + " was "
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index d2fdec7b323..5b9e496281f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -753,6 +753,12 @@ public Path getFileSystemPath() throws UnsupportedDataAccessOperationException {
@Override
public boolean exists() {
+ try {
+ key = getMainFileKey();
+ } catch (IOException e) {
+ logger.warning("Caught an IOException in S3AccessIO.exists(): " + e.getMessage());
+ return false;
+ }
String destinationKey = null;
if (dvObject instanceof DataFile) {
destinationKey = key;
diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
index a470f08f736..6b98848021c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
@@ -136,8 +136,6 @@ public class AddReplaceFileHelper{
private String newFileName; // step 30
private String newFileContentType; // step 30
private String newStorageIdentifier; // step 30
- private String newCheckSum; // step 30
- private ChecksumType newCheckSumType; //step 30
// -- Optional
private DataFile fileToReplace; // step 25
@@ -146,6 +144,7 @@ public class AddReplaceFileHelper{
private DatasetVersion clone;
List initialFileList;
List finalFileList;
+ private boolean trustSuppliedFileSizes;
// -----------------------------------
// Ingested files
@@ -610,15 +609,9 @@ private boolean runAddReplacePhase1(Dataset owner,
return false;
}
- if(optionalFileParams != null) {
- if(optionalFileParams.hasCheckSum()) {
- newCheckSum = optionalFileParams.getCheckSum();
- newCheckSumType = optionalFileParams.getCheckSumType();
- }
- }
msgt("step_030_createNewFilesViaIngest");
- if (!this.step_030_createNewFilesViaIngest()){
+ if (!this.step_030_createNewFilesViaIngest(optionalFileParams)){
return false;
}
@@ -1191,7 +1184,7 @@ private boolean step_007_auto_isReplacementInLatestVersion(DataFile existingFile
}
- private boolean step_030_createNewFilesViaIngest(){
+ private boolean step_030_createNewFilesViaIngest(OptionalFileParams optionalFileParams){
if (this.hasError()){
return false;
@@ -1203,21 +1196,28 @@ private boolean step_030_createNewFilesViaIngest(){
//Don't repeatedly update the clone (losing changes) in multifile case
clone = workingVersion.cloneDatasetVersion();
}
+
+ Long suppliedFileSize = null;
+ String newCheckSum = null;
+ ChecksumType newCheckSumType = null;
+
+
+ if (optionalFileParams != null) {
+ if (optionalFileParams.hasCheckSum()) {
+ newCheckSum = optionalFileParams.getCheckSum();
+ newCheckSumType = optionalFileParams.getCheckSumType();
+ }
+ if (trustSuppliedFileSizes && optionalFileParams.hasFileSize()) {
+ suppliedFileSize = optionalFileParams.getFileSize();
+ }
+ }
+
try {
- /*CreateDataFileResult result = FileUtil.createDataFiles(workingVersion,
- this.newFileInputStream,
- this.newFileName,
- this.newFileContentType,
- this.newStorageIdentifier,
- this.newCheckSum,
- this.newCheckSumType,
- this.systemConfig);*/
-
UploadSessionQuotaLimit quota = null;
if (systemConfig.isStorageQuotasEnforced()) {
quota = fileService.getUploadSessionQuotaLimit(dataset);
}
- Command cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType);
+ Command cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType, suppliedFileSize);
CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
initialFileList = createDataFilesResult.getDataFiles();
@@ -2033,9 +2033,15 @@ public void setDuplicateFileWarning(String duplicateFileWarning) {
* @param jsonData - an array of jsonData entries (one per file) using the single add file jsonData format
* @param dataset
* @param authUser
+ * @param trustSuppliedSizes - whether to accept the fileSize values passed
+ * in jsonData (we don't want to trust the users of the S3 direct
+ * upload API with that information - we will verify the status of
+ * the files in the S3 bucket and confirm the sizes in the process.
+ * we do want GlobusService to be able to pass the file sizes, since
+ * they are obtained and verified via a Globus API lookup).
* @return
*/
- public Response addFiles(String jsonData, Dataset dataset, User authUser) {
+ public Response addFiles(String jsonData, Dataset dataset, User authUser, boolean trustSuppliedFileSizes) {
msgt("(addFilesToDataset) jsonData: " + jsonData.toString());
JsonArrayBuilder jarr = Json.createArrayBuilder();
@@ -2044,6 +2050,7 @@ public Response addFiles(String jsonData, Dataset dataset, User authUser) {
int totalNumberofFiles = 0;
int successNumberofFiles = 0;
+ this.trustSuppliedFileSizes = trustSuppliedFileSizes;
// -----------------------------------------------------------
// Read jsonData and Parse files information from jsondata :
// -----------------------------------------------------------
@@ -2176,6 +2183,10 @@ public Response addFiles(String jsonData, Dataset dataset, User authUser) {
.add("data", Json.createObjectBuilder().add("Files", jarr).add("Result", result)).build() ).build();
}
+ public Response addFiles(String jsonData, Dataset dataset, User authUser) {
+ return addFiles(jsonData, dataset, authUser, false);
+ }
+
/**
* Replace multiple files with prepositioned replacements as listed in the
* jsonData. Works with direct upload, Globus, and other out-of-band methods.
diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java
index 959dbc4e262..54844160163 100644
--- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java
+++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java
@@ -39,6 +39,12 @@
* - Provenance related information
*
* @author rmp553
+ * @todo (?) We may want to consider renaming this class to DataFileParams or
+ * DataFileInfo... it was originally created to encode some bits of info -
+ * the file "tags" specifically, that didn't fit in elsewhere in the normal
+ * workflow; but it's been expanded to cover pretty much everything else associated
+ * with DataFiles and it's not really "optional" anymore when, for example, used
+ * in the direct upload workflow. (?)
*/
public class OptionalFileParams {
@@ -76,6 +82,8 @@ public class OptionalFileParams {
public static final String MIME_TYPE_ATTR_NAME = "mimeType";
private String checkSumValue;
private ChecksumType checkSumType;
+ public static final String FILE_SIZE_ATTR_NAME = "fileSize";
+ private Long fileSize;
public static final String LEGACY_CHECKSUM_ATTR_NAME = "md5Hash";
public static final String CHECKSUM_OBJECT_NAME = "checksum";
public static final String CHECKSUM_OBJECT_TYPE = "@type";
@@ -268,6 +276,18 @@ public String getCheckSum() {
public ChecksumType getCheckSumType() {
return checkSumType;
}
+
+ public boolean hasFileSize() {
+ return fileSize != null;
+ }
+
+ public Long getFileSize() {
+ return fileSize;
+ }
+
+ public void setFileSize(long fileSize) {
+ this.fileSize = fileSize;
+ }
/**
* Set tags
@@ -416,7 +436,13 @@ else if ((jsonObj.has(CHECKSUM_OBJECT_NAME)) && (!jsonObj.get(CHECKSUM_OBJECT_NA
this.checkSumType = ChecksumType.fromString(((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_TYPE).getAsString());
}
-
+ // -------------------------------
+ // get file size as a Long, if supplied
+ // -------------------------------
+ if ((jsonObj.has(FILE_SIZE_ATTR_NAME)) && (!jsonObj.get(FILE_SIZE_ATTR_NAME).isJsonNull())){
+
+ this.fileSize = jsonObj.get(FILE_SIZE_ATTR_NAME).getAsLong();
+ }
// -------------------------------
// get tags
// -------------------------------
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java
index 40c2abf5d21..91f3a5b823c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java
@@ -19,13 +19,15 @@ abstract class AbstractWriteDataverseCommand extends AbstractCommand
private final List inputLevels;
private final List facets;
protected final List metadataBlocks;
+ private final boolean resetRelationsOnNullValues;
public AbstractWriteDataverseCommand(Dataverse dataverse,
Dataverse affectedDataverse,
DataverseRequest request,
List facets,
List inputLevels,
- List metadataBlocks) {
+ List metadataBlocks,
+ boolean resetRelationsOnNullValues) {
super(request, affectedDataverse);
this.dataverse = dataverse;
if (facets != null) {
@@ -43,42 +45,61 @@ public AbstractWriteDataverseCommand(Dataverse dataverse,
} else {
this.metadataBlocks = null;
}
+ this.resetRelationsOnNullValues = resetRelationsOnNullValues;
}
@Override
public Dataverse execute(CommandContext ctxt) throws CommandException {
dataverse = innerExecute(ctxt);
+ processMetadataBlocks();
+ processFacets(ctxt);
+ processInputLevels(ctxt);
+
+ return ctxt.dataverses().save(dataverse);
+ }
+
+ private void processMetadataBlocks() {
if (metadataBlocks != null && !metadataBlocks.isEmpty()) {
dataverse.setMetadataBlockRoot(true);
dataverse.setMetadataBlocks(metadataBlocks);
+ } else if (resetRelationsOnNullValues) {
+ dataverse.setMetadataBlockRoot(false);
+ dataverse.clearMetadataBlocks();
}
+ }
+ private void processFacets(CommandContext ctxt) {
if (facets != null) {
ctxt.facets().deleteFacetsFor(dataverse);
-
+ dataverse.setDataverseFacets(new ArrayList<>());
+
if (!facets.isEmpty()) {
dataverse.setFacetRoot(true);
}
- int i = 0;
- for (DatasetFieldType df : facets) {
- ctxt.facets().create(i++, df, dataverse);
+ for (int i = 0; i < facets.size(); i++) {
+ ctxt.facets().create(i, facets.get(i), dataverse);
}
+ } else if (resetRelationsOnNullValues) {
+ ctxt.facets().deleteFacetsFor(dataverse);
+ dataverse.setFacetRoot(false);
}
+ }
+ private void processInputLevels(CommandContext ctxt) {
if (inputLevels != null) {
if (!inputLevels.isEmpty()) {
dataverse.addInputLevelsMetadataBlocksIfNotPresent(inputLevels);
}
ctxt.fieldTypeInputLevels().deleteFacetsFor(dataverse);
- for (DataverseFieldTypeInputLevel inputLevel : inputLevels) {
+ inputLevels.forEach(inputLevel -> {
inputLevel.setDataverse(dataverse);
ctxt.fieldTypeInputLevels().create(inputLevel);
- }
+ });
+ } else if (resetRelationsOnNullValues) {
+ ctxt.fieldTypeInputLevels().deleteFacetsFor(dataverse);
}
-
- return ctxt.dataverses().save(dataverse);
}
abstract protected Dataverse innerExecute(CommandContext ctxt) throws IllegalCommandException;
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
index 145cfb6199c..3728f3ee6ce 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
@@ -39,7 +39,7 @@ public CreateDataverseCommand(Dataverse created,
List facets,
List inputLevels,
List metadataBlocks) {
- super(created, created.getOwner(), request, facets, inputLevels, metadataBlocks);
+ super(created, created.getOwner(), request, facets, inputLevels, metadataBlocks, false);
}
@Override
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 76939751899..e9a2025b112 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -93,6 +93,10 @@ public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion versi
this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType, null, null);
}
+ public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UploadSessionQuotaLimit quota, String newCheckSum, DataFile.ChecksumType newCheckSumType, Long newFileSize) {
+ this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType, newFileSize, null);
+ }
+
// This version of the command must be used when files are created in the
// context of creating a brand new dataset (from the Add Dataset page):
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java
index e6e8279a314..e378e2e2ef7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java
@@ -131,7 +131,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasetversion.update.failure"), this);
} else {
- metadataUpdated = DatasetVersionDifference.compareFileMetadatas(publishedFmd, draftFmd);
+ metadataUpdated = !DatasetVersionDifference.compareFileMetadatas(publishedFmd, draftFmd).isEmpty();
publishedFmd.setLabel(draftFmd.getLabel());
publishedFmd.setDescription(draftFmd.getDescription());
publishedFmd.setCategories(draftFmd.getCategories());
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
index 1ac41105237..902bea7f833 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
@@ -11,6 +11,9 @@
import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
import edu.harvard.iq.dataverse.workflow.Workflow;
import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType;
+
+import jakarta.persistence.OptimisticLockException;
+
import java.util.Optional;
import java.util.logging.Logger;
import static java.util.stream.Collectors.joining;
@@ -105,10 +108,15 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException
Optional prePubWf = ctxt.workflows().getDefaultWorkflow(TriggerType.PrePublishDataset);
if ( prePubWf.isPresent() ) {
// We start a workflow
- theDataset = ctxt.em().merge(theDataset);
- ctxt.em().flush();
- ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased), true);
- return new PublishDatasetResult(theDataset, Status.Workflow);
+ try {
+ theDataset = ctxt.em().merge(theDataset);
+ ctxt.em().flush();
+ ctxt.workflows().start(prePubWf.get(),
+ buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased), true);
+ return new PublishDatasetResult(theDataset, Status.Workflow);
+ } catch (OptimisticLockException e) {
+ throw new CommandException(e.getMessage(), e, this);
+ }
} else{
// We will skip trying to register the global identifiers for datafiles
@@ -157,7 +165,12 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException
lock.setInfo(info);
ctxt.datasets().addDatasetLock(theDataset, lock);
}
- theDataset = ctxt.em().merge(theDataset);
+ try {
+ theDataset = ctxt.em().merge(theDataset);
+ } catch (OptimisticLockException e) {
+ ctxt.datasets().removeDatasetLocks(theDataset, DatasetLock.Reason.finalizePublication);
+ throw new CommandException(e.getMessage(), e, this);
+ }
// The call to FinalizePublicationCommand has been moved to the new @onSuccess()
// method:
//ctxt.datasets().callFinalizePublishCommandAsynchronously(theDataset.getId(), ctxt, request, datasetExternallyReleased);
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
index 55cc3708097..6dc4ab4d00d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
@@ -32,7 +32,7 @@ public UpdateDataverseCommand(Dataverse dataverse,
List featuredDataverses,
DataverseRequest request,
List inputLevels) {
- this(dataverse, facets, featuredDataverses, request, inputLevels, null, null);
+ this(dataverse, facets, featuredDataverses, request, inputLevels, null, null, false);
}
public UpdateDataverseCommand(Dataverse dataverse,
@@ -41,8 +41,9 @@ public UpdateDataverseCommand(Dataverse dataverse,
DataverseRequest request,
List inputLevels,
List metadataBlocks,
- DataverseDTO updatedDataverseDTO) {
- super(dataverse, dataverse, request, facets, inputLevels, metadataBlocks);
+ DataverseDTO updatedDataverseDTO,
+ boolean resetRelationsOnNullValues) {
+ super(dataverse, dataverse, request, facets, inputLevels, metadataBlocks, resetRelationsOnNullValues);
if (featuredDataverses != null) {
this.featuredDataverseList = new ArrayList<>(featuredDataverses);
} else {
diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java
index edd01ae98a3..d76020cb8d8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java
@@ -98,8 +98,10 @@ public class DDIExportServiceBean {
public static final String LEVEL_FILE = "file";
public static final String NOTE_TYPE_UNF = "VDC:UNF";
public static final String NOTE_TYPE_TAG = "DATAVERSE:TAG";
+ public static final String NOTE_TYPE_FILEDESCRIPTION = "DATAVERSE:FILEDESC";
public static final String NOTE_SUBJECT_UNF = "Universal Numeric Fingerprint";
public static final String NOTE_SUBJECT_TAG = "Data File Tag";
+ public static final String NOTE_SUBJECT_FILEDESCRIPTION = "DataFile Description";
/*
* Internal service objects:
@@ -742,11 +744,6 @@ private void createFileDscr(XMLStreamWriter xmlw, Set excludedFieldSet,
xmlw.writeEndElement(); // fileName
}
- /*
- xmlw.writeStartElement("fileCont");
- xmlw.writeCharacters( df.getContentType() );
- xmlw.writeEndElement(); // fileCont
- */
// dimensions
if (checkField("dimensns", excludedFieldSet, includedFieldSet)) {
if (dt.getCaseQuantity() != null || dt.getVarQuantity() != null || dt.getRecordsPerCase() != null) {
@@ -801,26 +798,6 @@ private void createFileDscr(XMLStreamWriter xmlw, Set excludedFieldSet,
xmlw.writeEndElement(); // notes
}
- /*
- xmlw.writeStartElement("notes");
- writeAttribute( xmlw, "type", "vdc:category" );
- xmlw.writeCharacters( fm.getCategory() );
- xmlw.writeEndElement(); // notes
- */
- // A special note for LOCKSS crawlers indicating the restricted
- // status of the file:
-
- /*
- if (tdf != null && isRestrictedFile(tdf)) {
- xmlw.writeStartElement("notes");
- writeAttribute( xmlw, "type", NOTE_TYPE_LOCKSS_CRAWL );
- writeAttribute( xmlw, "level", LEVEL_FILE );
- writeAttribute( xmlw, "subject", NOTE_SUBJECT_LOCKSS_PERM );
- xmlw.writeCharacters( "restricted" );
- xmlw.writeEndElement(); // notes
-
- }
- */
if (checkField("tags", excludedFieldSet, includedFieldSet) && df.getTags() != null) {
for (int i = 0; i < df.getTags().size(); i++) {
xmlw.writeStartElement("notes");
@@ -831,6 +808,17 @@ private void createFileDscr(XMLStreamWriter xmlw, Set excludedFieldSet,
xmlw.writeEndElement(); // notes
}
}
+
+ // A dedicated node for the Description entry
+ if (!StringUtilisEmpty(fm.getDescription())) {
+ xmlw.writeStartElement("notes");
+ xmlw.writeAttribute("level", LEVEL_FILE);
+ xmlw.writeAttribute("type", NOTE_TYPE_FILEDESCRIPTION);
+ xmlw.writeAttribute("subject", NOTE_SUBJECT_FILEDESCRIPTION);
+ xmlw.writeCharacters(fm.getDescription());
+ xmlw.writeEndElement(); // notes
+ }
+
xmlw.writeEndElement(); // fileDscr
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
index f5efc448090..05ddbe83e78 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
@@ -14,8 +14,10 @@
import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.LEVEL_FILE;
import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_SUBJECT_TAG;
import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_SUBJECT_UNF;
+import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_SUBJECT_FILEDESCRIPTION;
import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_TYPE_TAG;
import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_TYPE_UNF;
+import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_TYPE_FILEDESCRIPTION;
import edu.harvard.iq.dataverse.export.DDIExporter;
import edu.harvard.iq.dataverse.pidproviders.PidUtil;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -1901,6 +1903,8 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails)
xmlw.writeEndElement(); // notes
}
+ // If any tabular tags are present, each is formatted in a
+ // dedicated note:
if (fileJson.containsKey("tabularTags")) {
JsonArray tags = fileJson.getJsonArray("tabularTags");
for (int j = 0; j < tags.size(); j++) {
@@ -1912,6 +1916,17 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails)
xmlw.writeEndElement(); // notes
}
}
+
+ // Adding a dedicated node for the description entry (for
+ // non-tabular files we format it under the field)
+ if (fileJson.containsKey("description")) {
+ xmlw.writeStartElement("notes");
+ xmlw.writeAttribute("level", LEVEL_FILE);
+ xmlw.writeAttribute("type", NOTE_TYPE_FILEDESCRIPTION);
+ xmlw.writeAttribute("subject", NOTE_SUBJECT_FILEDESCRIPTION);
+ xmlw.writeCharacters(fileJson.getString("description"));
+ xmlw.writeEndElement(); // notes
+ }
// TODO: add the remaining fileDscr elements!
xmlw.writeEndElement(); // fileDscr
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index ac3c81622fc..58992805dc8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -74,6 +74,7 @@
import edu.harvard.iq.dataverse.util.URLTokenUtil;
import edu.harvard.iq.dataverse.util.UrlSignerUtil;
import edu.harvard.iq.dataverse.util.json.JsonUtil;
+import jakarta.json.JsonNumber;
import jakarta.json.JsonReader;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
@@ -284,6 +285,52 @@ private int makeDir(GlobusEndpoint endpoint, String dir) {
return result.status;
}
+ private Map lookupFileSizes(GlobusEndpoint endpoint, String dir) {
+ MakeRequestResponse result;
+
+ try {
+ logger.fine("Attempting to look up the contents of the Globus folder "+dir);
+ URL url = new URL(
+ "https://transfer.api.globusonline.org/v0.10/operation/endpoint/" + endpoint.getId()
+ + "/ls?path=" + dir);
+ result = makeRequest(url, "Bearer", endpoint.getClientToken(), "GET", null);
+
+ switch (result.status) {
+ case 200:
+ logger.fine("Looked up directory " + dir + " successfully.");
+ break;
+ default:
+ logger.warning("Status " + result.status + " received when looking up dir " + dir);
+ logger.fine("Response: " + result.jsonResponse);
+ return null;
+ }
+ } catch (MalformedURLException ex) {
+ // Misconfiguration
+ logger.warning("Failed to list the contents of the directory "+ dir + " on endpoint " + endpoint.getId());
+ return null;
+ }
+
+ Map ret = new HashMap<>();
+
+ JsonObject listObject = JsonUtil.getJsonObject(result.jsonResponse);
+ JsonArray dataArray = listObject.getJsonArray("DATA");
+
+ if (dataArray != null && !dataArray.isEmpty()) {
+ for (int i = 0; i < dataArray.size(); i++) {
+ String dataType = dataArray.getJsonObject(i).getString("DATA_TYPE", null);
+ if (dataType != null && dataType.equals("file")) {
+ // is it safe to assume that any entry with a valid "DATA_TYPE": "file"
+ // will also have valid "name" and "size" entries?
+ String fileName = dataArray.getJsonObject(i).getString("name");
+ long fileSize = dataArray.getJsonObject(i).getJsonNumber("size").longValueExact();
+ ret.put(fileName, fileSize);
+ }
+ }
+ }
+
+ return ret;
+ }
+
private int requestPermission(GlobusEndpoint endpoint, Dataset dataset, Permissions permissions) {
Gson gson = new GsonBuilder().create();
MakeRequestResponse result = null;
@@ -938,9 +985,20 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut
inputList.add(fileId + "IDsplit" + fullPath + "IDsplit" + fileName);
}
+
+ Map fileSizeMap = null;
+
+ if (filesJsonArray.size() >= systemConfig.getGlobusBatchLookupSize()) {
+ // Look up the sizes of all the files in the dataset folder, to avoid
+ // looking them up one by one later:
+ // @todo: we should only be doing this if this is a managed store, probably (?)
+ GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+ fileSizeMap = lookupFileSizes(endpoint, endpoint.getBasePath());
+ }
// calculateMissingMetadataFields: checksum, mimetype
JsonObject newfilesJsonObject = calculateMissingMetadataFields(inputList, myLogger);
+
JsonArray newfilesJsonArray = newfilesJsonObject.getJsonArray("files");
logger.fine("Size: " + newfilesJsonArray.size());
logger.fine("Val: " + JsonUtil.prettyPrint(newfilesJsonArray.getJsonObject(0)));
@@ -964,20 +1022,26 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut
if (newfileJsonObject != null) {
logger.fine("List Size: " + newfileJsonObject.size());
// if (!newfileJsonObject.get(0).getString("hash").equalsIgnoreCase("null")) {
- JsonPatch path = Json.createPatchBuilder()
+ JsonPatch patch = Json.createPatchBuilder()
.add("/md5Hash", newfileJsonObject.get(0).getString("hash")).build();
- fileJsonObject = path.apply(fileJsonObject);
- path = Json.createPatchBuilder()
+ fileJsonObject = patch.apply(fileJsonObject);
+ patch = Json.createPatchBuilder()
.add("/mimeType", newfileJsonObject.get(0).getString("mime")).build();
- fileJsonObject = path.apply(fileJsonObject);
+ fileJsonObject = patch.apply(fileJsonObject);
+ // If we already know the size of this file on the Globus end,
+ // we'll pass it to /addFiles, to avoid looking up file sizes
+ // one by one:
+ if (fileSizeMap != null && fileSizeMap.get(fileId) != null) {
+ Long uploadedFileSize = fileSizeMap.get(fileId);
+ myLogger.info("Found size for file " + fileId + ": " + uploadedFileSize + " bytes");
+ patch = Json.createPatchBuilder()
+ .add("/fileSize", Json.createValue(uploadedFileSize)).build();
+ fileJsonObject = patch.apply(fileJsonObject);
+ } else {
+ logger.fine("No file size entry found for file "+fileId);
+ }
addFilesJsonData.add(fileJsonObject);
countSuccess++;
- // } else {
- // globusLogger.info(fileName
- // + " will be skipped from adding to dataset by second API due to missing
- // values ");
- // countError++;
- // }
} else {
myLogger.info(fileName
+ " will be skipped from adding to dataset in the final AddReplaceFileHelper.addFiles() call. ");
@@ -1029,7 +1093,7 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut
// The old code had 2 sec. of sleep, so ...
Thread.sleep(2000);
- Response addFilesResponse = addFileHelper.addFiles(newjsonData, dataset, authUser);
+ Response addFilesResponse = addFileHelper.addFiles(newjsonData, dataset, authUser, true);
if (addFilesResponse == null) {
logger.info("null response from addFiles call");
@@ -1211,7 +1275,7 @@ private GlobusTaskState globusStatusCheck(GlobusEndpoint endpoint, String taskId
return task;
}
- public JsonObject calculateMissingMetadataFields(List inputList, Logger globusLogger)
+ private JsonObject calculateMissingMetadataFields(List inputList, Logger globusLogger)
throws InterruptedException, ExecutionException, IOException {
List> hashvalueCompletableFutures = inputList.stream()
@@ -1230,7 +1294,7 @@ public JsonObject calculateMissingMetadataFields(List inputList, Logger
});
JsonArrayBuilder filesObject = (JsonArrayBuilder) completableFuture.get();
-
+
JsonObject output = Json.createObjectBuilder().add("files", filesObject).build();
return output;
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
index 0667f5594ce..7280b6af129 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
@@ -252,8 +252,16 @@ public void setAllowHarvestingMissingCVV(boolean allowHarvestingMissingCVV) {
this.allowHarvestingMissingCVV = allowHarvestingMissingCVV;
}
- // TODO: do we need "orphanRemoval=true"? -- L.A. 4.4
- // TODO: should it be @OrderBy("startTime")? -- L.A. 4.4
+ private boolean useOaiIdAsPid;
+
+ public boolean isUseOaiIdentifiersAsPids() {
+ return useOaiIdAsPid;
+ }
+
+ public void setUseOaiIdentifiersAsPids(boolean useOaiIdAsPid) {
+ this.useOaiIdAsPid = useOaiIdAsPid;
+ }
+
@OneToMany(mappedBy="harvestingClient", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST})
@OrderBy("id")
private List harvestHistory;
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index b42fd950528..71c498a4d0b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -344,10 +344,20 @@ public List saveAndAddFilesToDataset(DatasetVersion version,
try {
StorageIO dataAccess = DataAccess.getStorageIO(dataFile);
//Populate metadata
- dataAccess.open(DataAccessOption.READ_ACCESS);
- // (the .open() above makes a remote call to check if
- // the file exists and obtains its size)
- confirmedFileSize = dataAccess.getSize();
+
+ // There are direct upload sub-cases where the file size
+ // is already known at this point. For example, direct uploads
+ // to S3 that go through the jsf dataset page. Or the Globus
+ // uploads, where the file sizes are looked up in bulk on
+ // the completion of the remote upload task.
+ if (dataFile.getFilesize() >= 0) {
+ confirmedFileSize = dataFile.getFilesize();
+ } else {
+ dataAccess.open(DataAccessOption.READ_ACCESS);
+ // (the .open() above makes a remote call to check if
+ // the file exists and obtains its size)
+ confirmedFileSize = dataAccess.getSize();
+ }
// For directly-uploaded files, we will perform the file size
// limit and quota checks here. Perform them *again*, in
@@ -362,13 +372,16 @@ public List saveAndAddFilesToDataset(DatasetVersion version,
if (fileSizeLimit == null || confirmedFileSize < fileSizeLimit) {
//set file size
- logger.fine("Setting file size: " + confirmedFileSize);
- dataFile.setFilesize(confirmedFileSize);
+ if (dataFile.getFilesize() < 0) {
+ logger.fine("Setting file size: " + confirmedFileSize);
+ dataFile.setFilesize(confirmedFileSize);
+ }
if (dataAccess instanceof S3AccessIO) {
((S3AccessIO) dataAccess).removeTempTag();
}
savedSuccess = true;
+ logger.info("directly uploaded file successfully saved. file size: "+dataFile.getFilesize());
}
} catch (IOException ioex) {
logger.warning("Failed to get file size, storage id, or failed to remove the temp tag on the saved S3 object" + dataFile.getStorageIdentifier() + " ("
diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java
index 9d61663d034..1f03d8a6cfb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java
@@ -59,6 +59,11 @@
* service.
* As of now, it only does the registration updates, to accommodate
* the modifyRegistration datasets API sub-command.
+ *
+ * Note that while Handles are nominally case sensitive, handle.net is
+ * configured to be case-insensitive and Dataverse makes case-insensitve
+ * database look-ups to find Handles (See #11003). That said, database
+ * entries are stored in the case matching the configuration of the provider.
*/
public class HandlePidProvider extends AbstractPidProvider {
diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java
index 7b55292350f..2cc0d41ede7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java
@@ -24,6 +24,9 @@
* overridable by a configurable parameter to support use of an external
* resolver.
*
+ * Note that while PermaLinks are nominally case sensitive, Dataverse makes
+ * case-insensitve database look-ups to find them (See #11003). That said, database
+ * entries are stored in the case matching the configuration of the provider.
*/
public class PermaLinkPidProvider extends AbstractPidProvider {
diff --git a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrl.java b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrl.java
index beb676f60d1..63b5bf03ea7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrl.java
+++ b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrl.java
@@ -30,7 +30,7 @@ public class PrivateUrl {
public PrivateUrl(RoleAssignment roleAssignment, Dataset dataset, String dataverseSiteUrl) {
this.token = roleAssignment.getPrivateUrlToken();
- this.link = dataverseSiteUrl + "/privateurl.xhtml?token=" + token;
+ this.link = dataverseSiteUrl + "/previewurl.xhtml?token=" + token;
this.dataset = dataset;
this.roleAssignment = roleAssignment;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java
index 9af4bb6af9e..17c622be9e2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java
@@ -1,6 +1,10 @@
package edu.harvard.iq.dataverse.privateurl;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetServiceBean;
+import edu.harvard.iq.dataverse.DataverseRequestServiceBean;
import edu.harvard.iq.dataverse.DataverseSession;
+import edu.harvard.iq.dataverse.PermissionsWrapper;
import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
import java.io.Serializable;
import java.util.logging.Logger;
@@ -20,8 +24,14 @@ public class PrivateUrlPage implements Serializable {
@EJB
PrivateUrlServiceBean privateUrlService;
+ @EJB
+ DatasetServiceBean datasetServiceBean;
@Inject
DataverseSession session;
+ @Inject
+ PermissionsWrapper permissionsWrapper;
+ @Inject
+ DataverseRequestServiceBean dvRequestService;
/**
* The unique string used to look up a PrivateUrlUser and the associated
@@ -34,7 +44,16 @@ public String init() {
PrivateUrlRedirectData privateUrlRedirectData = privateUrlService.getPrivateUrlRedirectDataFromToken(token);
String draftDatasetPageToBeRedirectedTo = privateUrlRedirectData.getDraftDatasetPageToBeRedirectedTo() + "&faces-redirect=true";
PrivateUrlUser privateUrlUser = privateUrlRedirectData.getPrivateUrlUser();
- session.setUser(privateUrlUser);
+ boolean sessionUserCanViewUnpublishedDataset = false;
+ if (session.getUser().isAuthenticated()){
+ Long datasetId = privateUrlUser.getDatasetId();
+ Dataset dataset = datasetServiceBean.find(datasetId);
+ sessionUserCanViewUnpublishedDataset = permissionsWrapper.canViewUnpublishedDataset(dvRequestService.getDataverseRequest(), dataset);
+ }
+ if(!sessionUserCanViewUnpublishedDataset){
+ //Only Reset if user cannot view this Draft Version
+ session.setUser(privateUrlUser);
+ }
logger.info("Redirecting PrivateUrlUser '" + privateUrlUser.getIdentifier() + "' to " + draftDatasetPageToBeRedirectedTo);
return draftDatasetPageToBeRedirectedTo;
} catch (Exception ex) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java b/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java
index 6e939c1bb6d..1310e0eb199 100644
--- a/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java
+++ b/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java
@@ -1,19 +1,19 @@
/**
- * Private URL for unpublished datasets.
+ * Preview URL for unpublished datasets.
*
- * The Private URL feature has been implemented as a specialized role assignment
+ * The Preview (formerly Private) URL feature has been implemented as a specialized role assignment
* with an associated token that permits read-only access to the metadata and
* all files (regardless of if the files are restricted or not) of a draft
* version of a dataset.
*
- * As of this note, a second option - to create a Private URL that provides an
+ * As of this note, a second option - to create a Preview URL that provides an
* anonymized view of the dataset has been added. This option works the same as
* the original except that it hides author names in the citation block, hides
* the values for an admin specified list of metadata fields, disables citation
* downloads, and disables API access (except for file and file thumbnail
* downloads which are used by the UI).
*
- * The primary use case for a Private URL is for journal editors to send a link
+ * The primary use case for a Preview URL is for journal editors to send a link
* to reviewers of a dataset before publication. In most cases, these journal
* editors do not permit depositors to publish on their own, which is to say
* they only allow depositors to have the "Contributor" role on the datasets
@@ -24,42 +24,42 @@
* the depositor, who is in charge of both the security of the dataset and the
* timing of when the dataset is published.
*
- * A secondary use case for a Private URL is for depositors who have the ability
+ * A secondary use case for a Preview URL is for depositors who have the ability
* to manage permissions on their dataset (depositors who have the "Curator" or
* "Admin" role, which grants much more power than the "Contributor" role) to
* send a link to coauthors or other trusted parties to preview the dataset
* before the depositors publish the dataset on their own. For better security,
* these depositors could ask their coauthors to create Dataverse accounts and
- * assign roles to them directly, rather than using a Private URL which requires
+ * assign roles to them directly, rather than using a Preview URL which requires
* no username or password.
*
* As of this note, a second option aimed specifically at the review use case -
- * to create a Private URL that provides an anonymized view of the dataset - has
+ * to create a Preview URL that provides an anonymized view of the dataset - has
* been added. This option works the same as the original except that it hides
* author names in the citation block, hides the values for an admin specified
* list of metadata fields, disables citation downloads, and disables API access
* (except for file and file thumbnail downloads which are used by the UI).
*
- * The token associated with the Private URL role assignment that can be used
+ * The token associated with the Preview URL role assignment that can be used
* either in the GUI or, for the non-anonymized-access option, via the API to
* elevate privileges beyond what a "Guest" can see. The ability to use a
- * Private URL token via API was added mostly to facilitate automated testing of
- * the feature but the far more common case is expected to be use of the Private
+ * Preview URL token via API was added mostly to facilitate automated testing of
+ * the feature but the far more common case is expected to be use of the Preview
* URL token in a link that is clicked to open a browser, similar to links
* shared via Dropbox, Google, etc.
*
- * When reviewers click a Private URL their browser sessions are set to the
+ * When reviewers click a Preview URL their browser sessions are set to the
* "{@link edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser}" that
* has the "Member" role only on the dataset in question and redirected to that
* dataset, where they will see an indication in blue at the top of the page
* that they are viewing an unpublished dataset. If the reviewer happens to be
* logged into Dataverse already, clicking the link will log them out because
* the review is meant to be blind. Because the dataset is always in draft when
- * a Private URL is in effect, no downloads or any other activity by the
- * reviewer are logged to the guestbook. All reviewers click the same Private
+ * a Preview URL is in effect, no downloads or any other activity by the
+ * reviewer are logged to the guestbook. All reviewers click the same Preview
* URL containing the same token, and with the exception of an IP address being
* logged, it should be impossible to trace which reviewers have clicked a
- * Private URL. If the reviewer navigates to the home page, the session is set
+ * Preview URL. If the reviewer navigates to the home page, the session is set
* to the Guest user and they will see what a Guest would see.
*
* The "Member" role is used because it contains the necessary read-only
@@ -76,51 +76,51 @@
* version. A Member can also download restricted files that have been deleted
* from previously published versions.
*
- * Likewise, when a Private URL token is used via API, commands are executed
+ * Likewise, when a Preview URL token is used via API, commands are executed
* using the "PrivateUrlUser" that has the "Member" role only on the dataset in
* question. This means that read-only operations such as downloads of the
- * dataset's files are permitted. The Search API does not respect the Private
+ * dataset's files are permitted. The Search API does not respect the Preview
* URL token but you can download files using the Access API, and, with the
* non-anonymized-access option, download unpublished metadata using the Native
* API.
*
- * A Private URL cannot be created for a published version of a dataset. In the
+ * A Preview URL cannot be created for a published version of a dataset. In the
* GUI, you will be reminded of this fact with a popup. The API will explain
* this as well.
*
- * An anonymized-access Private URL can't be created if any published dataset
+ * An anonymized-access Preview URL can't be created if any published dataset
* version exists. The primary reason for this is that, since datasets have
* DOIs, the full metadata about published versions is available directly from
* the DOI provider. (While the metadata for that version could be somewhat
* different, in practice it would probably provide a means of identifying
* some/all of the authors).
*
- * If a draft dataset containing a Private URL is
- * published, the Private URL is deleted. This means that reviewers who click
+ * If a draft dataset containing a Preview URL is
+ * published, the Preview URL is deleted. This means that reviewers who click
* the link after publication will see a 404.
*
- * If a post-publication draft containing a Private URL is deleted, the Private
+ * If a post-publication draft containing a Preview URL is deleted, the Preview
* URL is deleted. This is to ensure that if a new draft is created in the
* future, a new token will be used.
*
- * The creation and deletion of a Private URL are limited to the "Curator" and
+ * The creation and deletion of a Preview URL are limited to the "Curator" and
* "Admin" roles because only those roles have the permission called
* "ManageDatasetPermissions", which is the permission used by the
* "AssignRoleCommand" and "RevokeRoleCommand" commands. If you have the
- * permission to create or delete a Private URL, the fact that a Private URL is
+ * permission to create or delete a Preview URL, the fact that a Preview URL is
* enabled for a dataset will be indicated in blue at the top of the page.
* Success messages are shown at the top of the page when you create or delete a
- * Private URL. In the GUI, deleting a Private URL is called "disabling" and you
+ * Preview URL. In the GUI, deleting a Preview URL is called "disabling" and you
* will be prompted for a confirmation. No matter what you call it the role is
- * revoked. You can also delete a Private URL by revoking the role.
+ * revoked. You can also delete a Preview URL by revoking the role.
*
* A "Contributor" does not have the "ManageDatasetPermissions" permission and
- * cannot see "Permissions" nor "Private URL" under the "Edit" menu of their
- * dataset. When a Curator or Admin has enabled a Private URL on a Contributor's
- * dataset, the Contributor does not see a visual indication that a Private URL
+ * cannot see "Permissions" nor "Preview URL" under the "Edit" menu of their
+ * dataset. When a Curator or Admin has enabled a Preview URL on a Contributor's
+ * dataset, the Contributor does not see a visual indication that a Preview URL
* has been enabled for their dataset.
*
- * There is no way for an "Admin" or "Curator" to see when a Private URL was
+ * There is no way for an "Admin" or "Curator" to see when a Preview URL was
* created or deleted for a dataset but someone who has access to the database
* can see that the following commands are logged to the "actionlogrecord"
* database table:
@@ -129,7 +129,7 @@
*
{@link edu.harvard.iq.dataverse.engine.command.impl.CreatePrivateUrlCommand}
* {@link edu.harvard.iq.dataverse.engine.command.impl.DeletePrivateUrlCommand}
*
- * See also the Private URL To Unpublished Dataset BRD at
* https://docs.google.com/document/d/1FT47QkZKcmjSgRnePaJO2g1nzcotLyN3Yb2ORvBr6cs/edit?usp=sharing
*/
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
index d0dcf3461cf..4efd339ee46 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
@@ -135,6 +135,9 @@ public class IndexServiceBean {
@EJB
DatasetFieldServiceBean datasetFieldService;
+ @Inject
+ DatasetVersionFilesServiceBean datasetVersionFilesServiceBean;
+
public static final String solrDocIdentifierDataverse = "dataverse_";
public static final String solrDocIdentifierFile = "datafile_";
public static final String solrDocIdentifierDataset = "dataset_";
@@ -1018,6 +1021,8 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, SetFeature Request/Idea: Harvest metadata values that aren't from a list of controlled values #9992
@@ -1296,7 +1299,6 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set findPermissionsInSolrOnly() throws SearchException {
String dtype = dvObjectService.getDtype(id);
if (dtype == null) {
permissionInSolrOnly.add(docId);
- }
- if (dtype.equals(DType.Dataset.getDType())) {
+ }else if (dtype.equals(DType.Dataset.getDType())) {
List states = datasetService.getVersionStates(id);
if (states != null) {
String latestState = states.get(states.size() - 1);
@@ -2252,7 +2253,7 @@ public List findPermissionsInSolrOnly() throws SearchException {
} else if (dtype.equals(DType.DataFile.getDType())) {
List states = dataFileService.findVersionStates(id);
Set strings = states.stream().map(VersionState::toString).collect(Collectors.toSet());
- logger.fine("States for " + docId + ": " + String.join(", ", strings));
+ logger.finest("States for " + docId + ": " + String.join(", ", strings));
if (docId.endsWith("draft_permission")) {
if (!states.contains(VersionState.DRAFT)) {
permissionInSolrOnly.add(docId);
@@ -2266,7 +2267,7 @@ public List findPermissionsInSolrOnly() throws SearchException {
permissionInSolrOnly.add(docId);
} else {
if (!dataFileService.isInReleasedVersion(id)) {
- logger.fine("Adding doc " + docId + " to list of permissions in Solr only");
+ logger.finest("Adding doc " + docId + " to list of permissions in Solr only");
permissionInSolrOnly.add(docId);
}
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java
index ef27a5eefaf..1f1137016f2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java
@@ -291,5 +291,6 @@ more targeted results for just datasets. The format is YYYY (i.e.
public static final String DATASET_VALID = "datasetValid";
public static final String DATASET_LICENSE = "license";
+ public static final String FILE_COUNT = "fileCount";
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index de75c88009f..3fd97d418c0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -497,7 +497,8 @@ public SolrQueryResponse search(
Long retentionEndDate = (Long) solrDocument.getFieldValue(SearchFields.RETENTION_END_DATE);
//
Boolean datasetValid = (Boolean) solrDocument.getFieldValue(SearchFields.DATASET_VALID);
-
+ Long fileCount = (Long) solrDocument.getFieldValue(SearchFields.FILE_COUNT);
+
List matchedFields = new ArrayList<>();
SolrSearchResult solrSearchResult = new SolrSearchResult(query, name);
@@ -570,6 +571,7 @@ public SolrQueryResponse search(
solrSearchResult.setDeaccessionReason(deaccessionReason);
solrSearchResult.setDvTree(dvTree);
solrSearchResult.setDatasetValid(datasetValid);
+ solrSearchResult.setFileCount(fileCount);
if (Boolean.TRUE.equals((Boolean) solrDocument.getFieldValue(SearchFields.IS_HARVESTED))) {
solrSearchResult.setHarvested(true);
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java
index 27900bac63f..8802555affd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java
@@ -78,6 +78,10 @@ public class SolrSearchResult {
private String citation;
private String citationHtml;
private String datasetType;
+ /**
+ * Only Dataset can have a file count
+ */
+ private Long fileCount;
/**
* Files and datasets might have a UNF. Dataverses don't.
*/
@@ -456,10 +460,10 @@ public JsonObjectBuilder getJsonForMyData(boolean isValid) {
} // getJsonForMydata
public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls) {
- return json(showRelevance, showEntityIds, showApiUrls, null, null);
+ return json(showRelevance, showEntityIds, showApiUrls, null);
}
- public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, List metadataFields, Long datasetFileCount) {
+ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, List metadataFields) {
if (this.type == null) {
return jsonObjectBuilder();
}
@@ -597,7 +601,7 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool
subjects.add(subject);
}
nullSafeJsonBuilder.add("subjects", subjects);
- nullSafeJsonBuilder.add("fileCount", datasetFileCount);
+ nullSafeJsonBuilder.add("fileCount", this.fileCount);
nullSafeJsonBuilder.add("versionId", dv.getId());
nullSafeJsonBuilder.add("versionState", dv.getVersionState().toString());
if (this.isPublishedState()) {
@@ -1348,4 +1352,12 @@ public boolean isValid(Predicate canUpdateDataset) {
}
return !canUpdateDataset.test(this);
}
+
+ public Long getFileCount() {
+ return fileCount;
+ }
+
+ public void setFileCount(Long fileCount) {
+ this.fileCount = fileCount;
+ }
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
index 8ed96690e84..b5eb483c2c8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
@@ -539,6 +539,12 @@ Whether Harvesting (OAI) service is enabled
*
*/
GlobusSingleFileTransfer,
+ /** Lower limit of the number of files in a Globus upload task where
+ * the batch mode should be utilized in looking up the file information
+ * on the remote end node (file sizes, primarily), instead of individual
+ * lookups.
+ */
+ GlobusBatchLookupSize,
/**
* Optional external executables to run on the metadata for dataverses
* and datasets being published; as an extra validation step, to
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java
index 922e6ff5d28..771cf5fd0f0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java
@@ -111,7 +111,7 @@ public static ResourceBundle getResourceBundle(String propertyFileName, Locale c
ClassLoader loader = getClassLoader(filesRootDirectory);
bundle = ResourceBundle.getBundle(propertyFileName, currentLocale, loader);
} catch (MissingResourceException mre) {
- logger.warning("No property file named " + propertyFileName + "_" + currentLocale.getLanguage()
+ logger.fine("No property file named " + propertyFileName + "_" + currentLocale.getLanguage()
+ " found in " + filesRootDirectory + ", using untranslated values");
bundle = ResourceBundle.getBundle("propertyFiles/" + propertyFileName, currentLocale);
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 434b3bd8f8f..e769cacfdb1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -78,6 +78,7 @@ public class SystemConfig {
public static final long defaultZipDownloadLimit = 104857600L; // 100MB
private static final int defaultMultipleUploadFilesLimit = 1000;
private static final int defaultLoginSessionTimeout = 480; // = 8 hours
+ private static final int defaultGlobusBatchLookupSize = 50;
private String buildNumber = null;
@@ -954,6 +955,11 @@ public boolean isGlobusFileDownload() {
return (isGlobusDownload() && settingsService.isTrueForKey(SettingsServiceBean.Key.GlobusSingleFileTransfer, false));
}
+ public int getGlobusBatchLookupSize() {
+ String batchSizeOption = settingsService.getValueForKey(SettingsServiceBean.Key.GlobusBatchLookupSize);
+ return getIntLimitFromStringOrDefault(batchSizeOption, defaultGlobusBatchLookupSize);
+ }
+
private Boolean getMethodAvailable(String method, boolean upload) {
String methods = settingsService.getValueForKey(
upload ? SettingsServiceBean.Key.UploadMethods : SettingsServiceBean.Key.DownloadMethods);
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
index 8552389525d..232b7431a24 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
@@ -1007,7 +1007,6 @@ public void parseControlledVocabularyValue(DatasetField dsf, DatasetFieldType cv
if (cvv == null) {
if (allowHarvestingMissingCVV) {
// we need to process this as a primitive value
- logger.warning(">>>> Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'. Processing as primitive per setting override.");
parsePrimitiveValue(dsf, cvvType , json);
return;
} else {
@@ -1053,6 +1052,7 @@ public String parseHarvestingClient(JsonObject obj, HarvestingClient harvestingC
harvestingClient.setHarvestingSet(obj.getString("set",null));
harvestingClient.setCustomHttpHeaders(obj.getString("customHeaders", null));
harvestingClient.setAllowHarvestingMissingCVV(obj.getBoolean("allowHarvestingMissingCVV", false));
+ harvestingClient.setUseOaiIdentifiersAsPids(obj.getBoolean("useOaiIdentifiersAsPids", false));
return dataverseAlias;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 1bdee48b14d..91af13c79a3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -276,7 +276,9 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re
}
bld.add("permissionRoot", dv.isPermissionRoot())
.add("description", dv.getDescription())
- .add("dataverseType", dv.getDataverseType().name());
+ .add("dataverseType", dv.getDataverseType().name())
+ .add("isMetadataBlockRoot", dv.isMetadataBlockRoot())
+ .add("isFacetRoot", dv.isFacetRoot());
if (dv.getOwner() != null) {
bld.add("ownerId", dv.getOwner().getId());
}
@@ -1011,6 +1013,7 @@ public static JsonObjectBuilder json(HarvestingClient harvestingClient) {
add("status", harvestingClient.isHarvestingNow() ? "inProgress" : "inActive").
add("customHeaders", harvestingClient.getCustomHttpHeaders()).
add("allowHarvestingMissingCVV", harvestingClient.getAllowHarvestingMissingCVV()).
+ add("useOaiIdentifiersAsPids", harvestingClient.isUseOaiIdentifiersAsPids()).
add("lastHarvest", harvestingClient.getLastHarvestTime() == null ? null : harvestingClient.getLastHarvestTime().toString()).
add("lastResult", harvestingClient.getLastResult()).
add("lastSuccessful", harvestingClient.getLastSuccessfulHarvestTime() == null ? null : harvestingClient.getLastSuccessfulHarvestTime().toString()).
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 149e6a7e828..c47356008ff 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -307,7 +307,13 @@ notification.typeDescription.WORKFLOW_FAILURE=External workflow run has failed
notification.typeDescription.STATUSUPDATED=Status of dataset has been updated
notification.typeDescription.DATASETCREATED=Dataset was created by user
notification.typeDescription.DATASETMENTIONED=Dataset was referenced in remote system
-
+notification.typeDescription.GLOBUSUPLOADCOMPLETED=Globus upload is completed
+notification.typeDescription.GLOBUSUPLOADCOMPLETEDWITHERRORS=Globus upload completed with errors
+notification.typeDescription.GLOBUSDOWNLOADCOMPLETED=Globus download is completed
+notification.typeDescription.GLOBUSDOWNLOADCOMPLETEDWITHERRORS=Globus download completed with errors
+notification.typeDescription.GLOBUSUPLOADLOCALFAILURE=Globus upload failed, internal error
+notification.typeDescription.GLOBUSUPLOADREMOTEFAILURE=Globus upload failed, remote transfer error
+notification.typeDescription.REQUESTEDFILEACCESS=File access requested
groupAndRoles.manageTips=Here is where you can access and manage all the groups you belong to, and the roles you have been assigned.
user.message.signup.label=Create Account
user.message.signup.tip=Why have a Dataverse account? To create your own dataverse and customize it, add datasets, or request access to restricted files.
@@ -837,7 +843,8 @@ notification.email.datasetWasMentioned=Hello {0},
The {1} has just been
notification.email.datasetWasMentioned.subject={0}: A Dataset Relationship has been reported!
notification.email.globus.uploadCompleted.subject={0}: Files uploaded successfully via Globus and verified
notification.email.globus.downloadCompleted.subject={0}: Files downloaded successfully via Globus
-notification.email.globus.uploadCompletedWithErrors.subject={0}: Uploaded files via Globus with errors
+notification.email.globus.downloadCompletedWithErrors.subject={0}: Globus download task completed, errors encountered
+notification.email.globus.uploadCompletedWithErrors.subject={0}: Globus upload task completed with errors
notification.email.globus.uploadFailedRemotely.subject={0}: Failed to upload files via Globus
notification.email.globus.uploadFailedLocally.subject={0}: Failed to add files uploaded via Globus to dataset
# dataverse.xhtml
@@ -1457,14 +1464,14 @@ dataset.editBtn.itemLabel.metadata=Metadata
dataset.editBtn.itemLabel.terms=Terms
dataset.editBtn.itemLabel.permissions=Permissions
dataset.editBtn.itemLabel.thumbnailsAndWidgets=Thumbnails + Widgets
-dataset.editBtn.itemLabel.privateUrl=Private URL
+dataset.editBtn.itemLabel.privateUrl=Preview URL
dataset.editBtn.itemLabel.permissionsDataset=Dataset
dataset.editBtn.itemLabel.permissionsFile=Restricted Files
dataset.editBtn.itemLabel.deleteDataset=Delete Dataset
dataset.editBtn.itemLabel.deleteDraft=Delete Draft Version
dataset.editBtn.itemLabel.deaccession=Deaccession Dataset
dataset.exportBtn=Export Metadata
-dataset.exportBtn.itemLabel.ddi=DDI
+dataset.exportBtn.itemLabel.ddi=DDI Codebook v2
dataset.exportBtn.itemLabel.dublinCore=Dublin Core
dataset.exportBtn.itemLabel.schemaDotOrg=Schema.org JSON-LD
dataset.exportBtn.itemLabel.datacite=DataCite
@@ -1669,6 +1676,8 @@ dataset.message.createFailure=The dataset could not be created.
dataset.message.termsFailure=The dataset terms could not be updated.
dataset.message.label.fileAccess=Publicly-accessible storage
dataset.message.publicInstall=Files in this dataset may be readable outside Dataverse, restricted and embargoed access are disabled
+dataset.message.parallelUpdateError=Changes cannot be saved. This dataset has been edited since this page was opened. To continue, copy your changes, refresh the page to see the recent updates, and re-enter any changes you want to save.
+dataset.message.parallelPublishError=Publishing is blocked. This dataset has been edited since this page was opened. To publish it, refresh the page to see the recent updates, and publish again.
dataset.metadata.publicationDate=Publication Date
dataset.metadata.publicationDate.tip=The publication date of a Dataset.
dataset.metadata.citationDate=Citation Date
@@ -1721,23 +1730,36 @@ dataset.transferUnrestricted=Click Continue to transfer the elligible files.
dataset.requestAccessToRestrictedFiles=You may request access to any restricted file(s) by clicking the Request Access button.
dataset.requestAccessToRestrictedFilesWithEmbargo=Embargoed files cannot be accessed during the embargo period. If your selection contains restricted files, you may request access to them by clicking the Request Access button.
dataset.privateurl.infoMessageAuthor=Privately share this dataset before it is published: {0}
-dataset.privateurl.infoMessageReviewer=This unpublished dataset is being privately shared.
-dataset.privateurl.header=Unpublished Dataset Private URL
-dataset.privateurl.tip=Use a Private URL to allow those without Dataverse accounts to access your unpublished dataset. For more information about the Private URL feature, please refer to the User Guide.
-dataset.privateurl.absent=Private URL has not been created.
-dataset.privateurl.createPrivateUrl=Create Private URL
+dataset.privateurl.infoMessageReviewer=You are viewing a preview of this unpublished dataset version.
+dataset.privateurl.header=Unpublished Dataset Preview URL
+dataset.privateurl.tip=To cite this data in publications, use the dataset's persistent ID instead of this URL. For more information about the Preview URL feature, please refer to the User Guide.
+dataset.privateurl.onlyone=Only one Preview URL can be active for a single dataset.
+dataset.privateurl.absent=Preview URL has not been created.
+dataset.privateurl.general.button.label=Create General Preview URL
+dataset.privateurl.general.description=Create a URL that others can use to review this dataset version before it is published. They will be able to access all files in the dataset and see all metadata, including metadata that may identify the dataset's authors.
+dataset.privateurl.general.title=General Preview
+dataset.privateurl.anonymous.title=Anonymous Preview
+dataset.privateurl.anonymous.button.label=Create Anonymous Preview URL
+dataset.privateurl.anonymous.description=Create a URL that others can use to access an anonymized view of this unpublished dataset version. Metadata that could identify the dataset author will not be displayed. Non-identifying metadata will be visible.
+dataset.privateurl.anonymous.description.paragraph.two=The dataset's files are not changed and will be accessible if they're not restricted. Users of the Anonymous Preview URL will not be able to see the name of the Dataverse that this dataset is in but will be able to see the name of the repository, which might expose the dataset authors' identities.
+dataset.privateurl.createPrivateUrl=Create Preview URL
+dataset.privateurl.introduction=You can create a Preview URL to copy and share with others who will not need a repository account to review this unpublished dataset version. Once the dataset is published or if the URL is disabled, the URL will no longer work and will point to a "Page not found" page.
dataset.privateurl.createPrivateUrl.anonymized=Create URL for Anonymized Access
dataset.privateurl.createPrivateUrl.anonymized.unavailable=Anonymized Access is not available once a version of the dataset has been published
-dataset.privateurl.disablePrivateUrl=Disable Private URL
-dataset.privateurl.disablePrivateUrlConfirm=Yes, Disable Private URL
-dataset.privateurl.disableConfirmationText=Are you sure you want to disable the Private URL? If you have shared the Private URL with others they will no longer be able to use it to access your unpublished dataset.
-dataset.privateurl.cannotCreate=Private URL can only be used with unpublished versions of datasets.
-dataset.privateurl.roleassigeeTitle=Private URL Enabled
+dataset.privateurl.disablePrivateUrl=Disable Preview URL
+dataset.privateurl.disableGeneralPreviewUrl=Disable General Preview URL
+dataset.privateurl.disableAnonPreviewUrl=Disable Anonymous Preview URL
+dataset.privateurl.disablePrivateUrlConfirm=Yes, Disable Preview URL
+dataset.privateurl.disableGeneralPreviewUrlConfirm=Yes, Disable General Preview URL
+dataset.privateurl.disableAnonPreviewUrlConfirm=Yes, Disable Anonymous Preview URL
+dataset.privateurl.disableConfirmationText=Are you sure you want to disable the Preview URL? If you have shared the Preview URL with others they will no longer be able to use it to access your unpublished dataset.
+dataset.privateurl.cannotCreate=Preview URL can only be used with unpublished versions of datasets.
+dataset.privateurl.roleassigeeTitle=Preview URL Enabled
dataset.privateurl.createdSuccess=Success!
-dataset.privateurl.full=This Private URL provides full read access to the dataset
-dataset.privateurl.anonymized=This Private URL provides access to the anonymized dataset
-dataset.privateurl.disabledSuccess=You have successfully disabled the Private URL for this unpublished dataset.
-dataset.privateurl.noPermToCreate=To create a Private URL you must have the following permissions: {0}.
+dataset.privateurl.full=This Preview URL provides full read access to the dataset
+dataset.privateurl.anonymized=This Preview URL provides access to the anonymized dataset
+dataset.privateurl.disabledSuccess=You have successfully disabled the Preview URL for this unpublished dataset.
+dataset.privateurl.noPermToCreate=To create a Preview URL you must have the following permissions: {0}.
dataset.externalstatus.header=Curation Status Changed
dataset.externalstatus.removed=Curation Status Removed
dataset.externalstatus.info=Curation Status is now "{0}"
@@ -1934,7 +1956,7 @@ file.downloadBtn.format.all=All File Formats + Information
file.downloadBtn.format.tab=Tab-Delimited
file.downloadBtn.format.original={0} (Original File Format)
file.downloadBtn.format.rdata=RData
-file.downloadBtn.format.var=Variable Metadata
+file.downloadBtn.format.var=DDI Codebook v2
file.downloadBtn.format.citation=Data File Citation
file.download.filetype.unknown=Original File Format
file.more.information.link=Link to more file information for
@@ -2498,6 +2520,7 @@ dataset.version.file.changed=Files (Changed File Metadata: {0}
dataset.version.file.changed2=; Changed File Metadata: {0}
dataset.version.variablemetadata.changed=Variable Metadata (Changed Variable Metadata: {0}
dataset.version.variablemetadata.changed2=; Changed Variable Metadata: {0}
+dataset.version.compare.incorrect.order=Compare requires the older dataset version to be listed first.
#DataversePage.java
dataverse.item.required=Required
@@ -2719,8 +2742,8 @@ datasets.api.grant.role.assignee.has.role.error=User already has this role for t
datasets.api.revoke.role.not.found.error="Role assignment {0} not found"
datasets.api.revoke.role.success=Role {0} revoked for assignee {1} in {2}
datasets.api.privateurl.error.datasetnotfound=Could not find dataset.
-datasets.api.privateurl.error.alreadyexists=Private URL already exists for this dataset.
-datasets.api.privateurl.error.notdraft=Can't create Private URL because the latest version of this dataset is not a draft.
+datasets.api.privateurl.error.alreadyexists=Preview URL already exists for this dataset.
+datasets.api.privateurl.error.notdraft=Can't create Preview URL because the latest version of this dataset is not a draft.
datasets.api.privateurl.anonymized.error.released=Can't create a URL for anonymized access because this dataset has been published.
datasets.api.creationdate=Date Created
datasets.api.modificationdate=Last Modified Date
diff --git a/src/main/resources/db/migration/V6.4.0.1.sql b/src/main/resources/db/migration/V6.4.0.1.sql
new file mode 100644
index 00000000000..0bcd87dd736
--- /dev/null
+++ b/src/main/resources/db/migration/V6.4.0.1.sql
@@ -0,0 +1,4 @@
+-- Adding a case-insensitive index related to #11003
+--
+
+CREATE UNIQUE INDEX IF NOT EXISTS INDEX_DVOBJECT_authority_protocol_upper_identifier ON dvobject (authority, protocol, UPPER(identifier));
\ No newline at end of file
diff --git a/src/main/resources/db/migration/V6.4.0.2.sql b/src/main/resources/db/migration/V6.4.0.2.sql
new file mode 100644
index 00000000000..bc4a85b278f
--- /dev/null
+++ b/src/main/resources/db/migration/V6.4.0.2.sql
@@ -0,0 +1,2 @@
+-- #10118
+ALTER TABLE customquestion ALTER COLUMN questionstring TYPE text;
diff --git a/src/main/resources/db/migration/V6.4.0.3.sql b/src/main/resources/db/migration/V6.4.0.3.sql
new file mode 100644
index 00000000000..307d8ed206c
--- /dev/null
+++ b/src/main/resources/db/migration/V6.4.0.3.sql
@@ -0,0 +1,2 @@
+-- Add this boolean flag to accommodate a new harvesting client feature
+ALTER TABLE harvestingclient ADD COLUMN IF NOT EXISTS useOaiIdAsPid BOOLEAN DEFAULT FALSE;
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 6de0f00e94e..9426884d349 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -20,6 +20,7 @@
+
-
+
@@ -834,7 +835,6 @@
/>
@@ -995,7 +995,6 @@
+
#{bundle['dataset.privateurl.introduction']}
+
+
+ #{bundle['dataset.privateurl.onlyone']}
+
+
+
+
+
+ #{bundle['dataset.privateurl.general.description']}
+
+
+
+
+
+
+ #{privateUrlLink}
+
+
+