From 955312df6710841cbb4989326a28a067a07f6b5a Mon Sep 17 00:00:00 2001 From: darms Date: Mon, 19 Aug 2024 13:30:43 +0200 Subject: [PATCH 1/4] feat(api/versions): Added a new optional property to hide metadataBlocks from API response. --- .../10171-exlude-metadatablocks.md | 4 ++++ doc/sphinx-guides/source/api/native-api.rst | 8 +++++++ .../harvard/iq/dataverse/api/Datasets.java | 11 +++++---- .../iq/dataverse/util/json/JsonPrinter.java | 23 ++++++++++++------- 4 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 doc/release-notes/10171-exlude-metadatablocks.md diff --git a/doc/release-notes/10171-exlude-metadatablocks.md b/doc/release-notes/10171-exlude-metadatablocks.md new file mode 100644 index 00000000000..64fadfcc35e --- /dev/null +++ b/doc/release-notes/10171-exlude-metadatablocks.md @@ -0,0 +1,4 @@ +Extension of API `{id}/versions` and `{id}/versions/{versionId}` with an optional ``excludeMetadataBlocks`` parameter, +that specifies whether the metadataBlocks should be listed in the output. It defaults to ``true``, preserving backward +compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks +included can dramatically increase the volume of the output). diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index a5f7d03899a..5168b8c5917 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1150,6 +1150,8 @@ It returns a list of versions with their metadata, and file list: The optional ``excludeFiles`` parameter specifies whether the files should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or files having the files included can dramatically increase the volume of the output). A separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks included can dramatically increase the volume of the output). + The optional ``offset`` and ``limit`` parameters can be used to specify the range of the versions list to be shown. This can be used to paginate through the list in a dataset with a large number of versions. @@ -1174,6 +1176,12 @@ The fully expanded example above (without environment variables) looks like this The optional ``excludeFiles`` parameter specifies whether the files should be listed in the output (defaults to ``true``). Note that a separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. +.. code-block:: bash + + curl "https://demo.dataverse.org/api/datasets/24/versions/1.0?excludeMetadataBlocks=false" + +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output (defaults to ``true``). + By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4b919c5ed82..d576022389c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -415,15 +415,16 @@ public Response useDefaultCitationDate(@Context ContainerRequestContext crc, @Pa @GET @AuthRequired @Path("{id}/versions") - public Response listVersions(@Context ContainerRequestContext crc, @PathParam("id") String id, @QueryParam("excludeFiles") Boolean excludeFiles, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset) { + public Response listVersions(@Context ContainerRequestContext crc, @PathParam("id") String id, @QueryParam("excludeFiles") Boolean excludeFiles,@QueryParam("excludeMetadataBlocks") Boolean excludeMetadataBlocks, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset) { return response( req -> { Dataset dataset = findDatasetOrDie(id); Boolean deepLookup = excludeFiles == null ? true : !excludeFiles; + Boolean includeMetadataBlocks = excludeMetadataBlocks == null ? true : !excludeMetadataBlocks; return ok( execCommand( new ListVersionsCommand(req, dataset, offset, limit, deepLookup) ) .stream() - .map( d -> json(d, deepLookup) ) + .map( d -> json(d, deepLookup, includeMetadataBlocks) ) .collect(toJsonArray())); }, getRequestUser(crc)); } @@ -435,6 +436,7 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("excludeFiles") Boolean excludeFiles, + @QueryParam("excludeMetadataBlocks") Boolean excludeMetadataBlocks, @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, @QueryParam("returnOwners") boolean returnOwners, @Context UriInfo uriInfo, @@ -460,11 +462,12 @@ public Response getVersion(@Context ContainerRequestContext crc, if (excludeFiles == null ? true : !excludeFiles) { requestedDatasetVersion = datasetversionService.findDeep(requestedDatasetVersion.getId()); } + Boolean includeMetadataBlocks = excludeMetadataBlocks == null ? true : !excludeMetadataBlocks; JsonObjectBuilder jsonBuilder = json(requestedDatasetVersion, null, - excludeFiles == null ? true : !excludeFiles, - returnOwners); + excludeFiles == null ? true : !excludeFiles, + returnOwners, includeMetadataBlocks); return ok(jsonBuilder); }, getRequestUser(crc)); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index c908a4d2bce..d9c2250ef6f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -418,11 +418,17 @@ public static JsonObjectBuilder json(FileDetailsHolder ds) { } public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles) { - return json(dsv, null, includeFiles, false); + return json(dsv, null, includeFiles, false,true); + } + public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles, boolean includeMetadataBlocks) { + return json(dsv, null, includeFiles, false, includeMetadataBlocks); + } + public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, + boolean includeFiles, boolean returnOwners) { + return json( dsv, anonymizedFieldTypeNamesList, includeFiles, returnOwners,true); } - public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, - boolean includeFiles, boolean returnOwners) { + boolean includeFiles, boolean returnOwners, boolean includeMetadataBlocks) { Dataset dataset = dsv.getDataset(); JsonObjectBuilder bld = jsonObjectBuilder() .add("id", dsv.getId()).add("datasetId", dataset.getId()) @@ -467,11 +473,12 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized .add("sizeOfCollection", dsv.getTermsOfUseAndAccess().getSizeOfCollection()) .add("studyCompletion", dsv.getTermsOfUseAndAccess().getStudyCompletion()) .add("fileAccessRequest", dsv.getTermsOfUseAndAccess().isFileAccessRequest()); - - bld.add("metadataBlocks", (anonymizedFieldTypeNamesList != null) ? - jsonByBlocks(dsv.getDatasetFields(), anonymizedFieldTypeNamesList) - : jsonByBlocks(dsv.getDatasetFields()) - ); + if(includeMetadataBlocks) { + bld.add("metadataBlocks", (anonymizedFieldTypeNamesList != null) ? + jsonByBlocks(dsv.getDatasetFields(), anonymizedFieldTypeNamesList) + : jsonByBlocks(dsv.getDatasetFields()) + ); + } if(returnOwners){ bld.add("isPartOf", getOwnersFromDvObject(dataset)); } From 408172c06a60d88af9e4c9c72b8f153867958779 Mon Sep 17 00:00:00 2001 From: darms Date: Mon, 19 Aug 2024 13:40:59 +0200 Subject: [PATCH 2/4] feat(api/versions): Added a new optional property to hide metadataBlocks from API response. --- doc/release-notes/10171-exlude-metadatablocks.md | 2 +- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/10171-exlude-metadatablocks.md b/doc/release-notes/10171-exlude-metadatablocks.md index 64fadfcc35e..8039b36617c 100644 --- a/doc/release-notes/10171-exlude-metadatablocks.md +++ b/doc/release-notes/10171-exlude-metadatablocks.md @@ -1,4 +1,4 @@ Extension of API `{id}/versions` and `{id}/versions/{versionId}` with an optional ``excludeMetadataBlocks`` parameter, -that specifies whether the metadataBlocks should be listed in the output. It defaults to ``true``, preserving backward +that specifies whether the metadataBlocks should be listed in the output. It defaults to ``false``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks included can dramatically increase the volume of the output). diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5168b8c5917..a3a004969dc 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1150,7 +1150,7 @@ It returns a list of versions with their metadata, and file list: The optional ``excludeFiles`` parameter specifies whether the files should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or files having the files included can dramatically increase the volume of the output). A separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. -The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks included can dramatically increase the volume of the output). +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output. It defaults to ``false``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks included can dramatically increase the volume of the output). The optional ``offset`` and ``limit`` parameters can be used to specify the range of the versions list to be shown. This can be used to paginate through the list in a dataset with a large number of versions. @@ -1180,7 +1180,7 @@ The optional ``excludeFiles`` parameter specifies whether the files should be li curl "https://demo.dataverse.org/api/datasets/24/versions/1.0?excludeMetadataBlocks=false" -The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output (defaults to ``true``). +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output (defaults to ``false``). By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below. From 8813df9f13ba29e685a6638f259408060b93fc8b Mon Sep 17 00:00:00 2001 From: Johannes Darms Date: Wed, 18 Dec 2024 14:30:54 +0100 Subject: [PATCH 3/4] adapted documentation according to phil suggestions --- doc/release-notes/10171-exlude-metadatablocks.md | 2 +- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/10171-exlude-metadatablocks.md b/doc/release-notes/10171-exlude-metadatablocks.md index 8039b36617c..7c0a9b030eb 100644 --- a/doc/release-notes/10171-exlude-metadatablocks.md +++ b/doc/release-notes/10171-exlude-metadatablocks.md @@ -1,4 +1,4 @@ Extension of API `{id}/versions` and `{id}/versions/{versionId}` with an optional ``excludeMetadataBlocks`` parameter, that specifies whether the metadataBlocks should be listed in the output. It defaults to ``false``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks -included can dramatically increase the volume of the output). +included can dramatically increase the volume of the output). See also [the guides](https://dataverse-guide--10778.org.readthedocs.build/en/10778/api/native-api.html#list-versions-of-a-dataset), #10778, and #10171. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 462f145ab4f..b013c967228 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1201,7 +1201,7 @@ It returns a list of versions with their metadata, and file list: The optional ``excludeFiles`` parameter specifies whether the files should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or files having the files included can dramatically increase the volume of the output). A separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. -The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output. It defaults to ``false``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks included can dramatically increase the volume of the output). +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadata blocks should be listed in the output. It defaults to ``false``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadata blocks having the metadata blocks included can dramatically increase the volume of the output). The optional ``offset`` and ``limit`` parameters can be used to specify the range of the versions list to be shown. This can be used to paginate through the list in a dataset with a large number of versions. @@ -1231,7 +1231,7 @@ The optional ``excludeFiles`` parameter specifies whether the files should be li curl "https://demo.dataverse.org/api/datasets/24/versions/1.0?excludeMetadataBlocks=false" -The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output (defaults to ``false``). +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadata blocks should be listed in the output (defaults to ``false``). By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below. From 84cac1ee2eed6c980026970a2b1f5b1c1a779f21 Mon Sep 17 00:00:00 2001 From: Johannes Darms Date: Wed, 18 Dec 2024 15:04:38 +0100 Subject: [PATCH 4/4] added a simple unit test --- .../harvard/iq/dataverse/api/DatasetsIT.java | 36 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 6 +++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 4f1ee1717c1..3c1cc879d13 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -678,6 +678,42 @@ public void testCreatePublishDestroyDataset() { } + @Test + public void testHideMetadataBlocksInDatasetVersionsAPI() { + + // Create user + String apiToken = UtilIT.createRandomUserGetToken(); + + // Create user with no permission + String apiTokenNoPerms = UtilIT.createRandomUserGetToken(); + + // Create Collection + String collectionAlias = UtilIT.createRandomCollectionGetAlias(apiToken); + + // Create Dataset + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(collectionAlias, apiToken); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = JsonPath.from(createDataset.asString()).getString("data.persistentId"); + + // Now check that the metadata is NOT shown, when we ask the versions api to dos o. + boolean excludeMetadata = true; + Response unpublishedDraft = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken, true,excludeMetadata, false); + unpublishedDraft.prettyPrint(); + unpublishedDraft.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.metadataBlocks", equalTo(null)); + + // Now check that the metadata is shown, when we ask the versions api to dos o. + excludeMetadata = false; + unpublishedDraft = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken,true, excludeMetadata, false); + unpublishedDraft.prettyPrint(); + unpublishedDraft.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.metadataBlocks", notNullValue() ); + } /** * The apis (/api/datasets/{id}/versions and /api/datasets/{id}/versions/{vid} * are already called from other RestAssured tests, in this class and also in FilesIT. diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 8f1fcdf57eb..fea6300a678 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1533,6 +1533,9 @@ static Response getDatasetVersion(String persistentId, String versionNumber, Str } static Response getDatasetVersion(String persistentId, String versionNumber, String apiToken, boolean excludeFiles, boolean includeDeaccessioned) { + return getDatasetVersion(persistentId,versionNumber,apiToken,excludeFiles,false,includeDeaccessioned); + } + static Response getDatasetVersion(String persistentId, String versionNumber, String apiToken, boolean excludeFiles,boolean excludeMetadataBlocks, boolean includeDeaccessioned) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) .queryParam("includeDeaccessioned", includeDeaccessioned) @@ -1540,7 +1543,8 @@ static Response getDatasetVersion(String persistentId, String versionNumber, Str + versionNumber + "?persistentId=" + persistentId - + (excludeFiles ? "&excludeFiles=true" : "")); + + (excludeFiles ? "&excludeFiles=true" : "") + + (excludeMetadataBlocks ? "&excludeMetadataBlocks=true" : "")); } static Response getDatasetWithOwners(String persistentId, String apiToken, boolean returnOwners) {