allow links between dataset types and metadata blocks

doc/release-notes/10519-dataset-types.md

@@ -0,0 +1,10 @@
+## Dataset Types can be linked to Metadata Blocks
+
+Metadata blocks (e.g. "CodeMeta") can now be linked to dataset types (e.g. "software") using new superuser APIs.
+
+This will have the following effects for the APIs used by the new Dataverse UI ( https://github.com/IQSS/dataverse-frontend ):
+
+- The list of fields shown when creating a dataset will include fields marked as "displayoncreate" (in the tsv/database) for metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API.
+- The metadata blocks shown when editing a dataset will include metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API.
+
+For more information, see the guides and #10519.

doc/sphinx-guides/source/api/native-api.rst

@@ -546,6 +546,9 @@ This endpoint supports the following optional query parameters:
 
 - ``returnDatasetFieldTypes``: Whether or not to return the dataset field types present in each metadata block. If not set, the default value is false.
 - ``onlyDisplayedOnCreate``: Whether or not to return only the metadata blocks that are displayed on dataset creation. If ``returnDatasetFieldTypes`` is true, only the dataset field types shown on dataset creation will be returned within each metadata block. If not set, the default value is false.
+- ``datasetType``: Whether or not to return additional fields from metadata blocks that are linked with a particular dataset type.
+
+are displayed on dataset creation. If ``returnDatasetFieldTypes`` is true, only the dataset field types shown on dataset creation will be returned within each metadata block. If not set, the default value is false.
 
 An example using the optional query parameters is presented below:
 
@@ -3193,6 +3196,32 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/datasetTypes" -X POST -d '{"name": "software"}'
 
+.. _api-link-dataset-type:
+
+Link Dataset Type with Metadata Blocks
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This API endpoint is superuser only.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export TYPE=software
+  export JSON='["codeMeta20"]'
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-Type: application/json" "$SERVER_URL/api/datasets/datasetTypes/$TYPE" -X POST -d $JSON
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/datasetTypes/software" -X POST -d '["codeMeta20"]'
+
+To update the blocks that are link, send an array with those blocks.
+
+To remove all links to blocks, send an empty array.
+
 .. _api-delete-dataset-type:
 
 Delete Dataset Type

src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java

@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse;
 
+import edu.harvard.iq.dataverse.dataset.DatasetType;
 import java.io.IOException;
 import java.io.StringReader;
 import java.net.URI;
@@ -871,15 +872,15 @@ public List<DatasetFieldType> findAllDisplayedOnCreateInMetadataBlock(MetadataBl
         Root<MetadataBlock> metadataBlockRoot = criteriaQuery.from(MetadataBlock.class);
         Root<DatasetFieldType> datasetFieldTypeRoot = criteriaQuery.from(DatasetFieldType.class);
 
-        Predicate requiredInDataversePredicate = buildRequiredInDataversePredicate(criteriaBuilder, datasetFieldTypeRoot);
+        Predicate fieldRequiredInTheInstallation = buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot);
 
         criteriaQuery.where(
                 criteriaBuilder.and(
                         criteriaBuilder.equal(metadataBlockRoot.get("id"), metadataBlock.getId()),
                         datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")),
                         criteriaBuilder.or(
                                 criteriaBuilder.isTrue(datasetFieldTypeRoot.get("displayOnCreate")),
-                                requiredInDataversePredicate
+                                fieldRequiredInTheInstallation
                         )
                 )
         );
@@ -890,16 +891,39 @@ public List<DatasetFieldType> findAllDisplayedOnCreateInMetadataBlock(MetadataBl
         return typedQuery.getResultList();
     }
 
-    public List<DatasetFieldType> findAllInMetadataBlockAndDataverse(MetadataBlock metadataBlock, Dataverse dataverse, boolean onlyDisplayedOnCreate) {
+    public List<DatasetFieldType> findAllInMetadataBlockAndDataverse(MetadataBlock metadataBlock, Dataverse dataverse, boolean onlyDisplayedOnCreate, DatasetType datasetType) {
         if (!dataverse.isMetadataBlockRoot() && dataverse.getOwner() != null) {
-            return findAllInMetadataBlockAndDataverse(metadataBlock, dataverse.getOwner(), onlyDisplayedOnCreate);
+            return findAllInMetadataBlockAndDataverse(metadataBlock, dataverse.getOwner(), onlyDisplayedOnCreate, datasetType);
         }
 
         CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
         CriteriaQuery<DatasetFieldType> criteriaQuery = criteriaBuilder.createQuery(DatasetFieldType.class);
 
         Root<MetadataBlock> metadataBlockRoot = criteriaQuery.from(MetadataBlock.class);
         Root<DatasetFieldType> datasetFieldTypeRoot = criteriaQuery.from(DatasetFieldType.class);
+
+        // Build the main predicate to include fields that belong to the specified dataverse and metadataBlock and match the onlyDisplayedOnCreate value.
+        Predicate fieldPresentInDataverse = buildFieldPresentInDataversePredicate(dataverse, onlyDisplayedOnCreate, criteriaQuery, criteriaBuilder, datasetFieldTypeRoot, metadataBlockRoot);
+
+        // Build an additional predicate to include fields from the datasetType, if the datasetType is specified and contains the given metadataBlock.
+        Predicate fieldPresentInDatasetType = buildFieldPresentInDatasetTypePredicate(datasetType, criteriaQuery, criteriaBuilder, datasetFieldTypeRoot, metadataBlockRoot, onlyDisplayedOnCreate);
+
+        // Build the final WHERE clause by combining all the predicates.
+        criteriaQuery.where(
+                criteriaBuilder.equal(metadataBlockRoot.get("id"), metadataBlock.getId()), // Match the MetadataBlock ID.
+                datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")), // Ensure the DatasetFieldType is part of the MetadataBlock.
+                criteriaBuilder.or(
+                        fieldPresentInDataverse,
+                        fieldPresentInDatasetType
+                )
+        );
+
+        criteriaQuery.select(datasetFieldTypeRoot).distinct(true);
+
+        return em.createQuery(criteriaQuery).getResultList();
+    }
+
+    private Predicate buildFieldPresentInDataversePredicate(Dataverse dataverse, boolean onlyDisplayedOnCreate, CriteriaQuery<DatasetFieldType> criteriaQuery, CriteriaBuilder criteriaBuilder, Root<DatasetFieldType> datasetFieldTypeRoot, Root<MetadataBlock> metadataBlockRoot) {
         Root<Dataverse> dataverseRoot = criteriaQuery.from(Dataverse.class);
 
         // Join Dataverse with DataverseFieldTypeInputLevel on the "dataverseFieldTypeInputLevels" attribute, using a LEFT JOIN.
@@ -930,7 +954,7 @@ public List<DatasetFieldType> findAllInMetadataBlockAndDataverse(MetadataBlock m
         Predicate hasNoInputLevelPredicate = criteriaBuilder.not(criteriaBuilder.exists(subquery));
 
         // Define a predicate to include the required fields in Dataverse.
-        Predicate requiredInDataversePredicate = buildRequiredInDataversePredicate(criteriaBuilder, datasetFieldTypeRoot);
+        Predicate fieldRequiredInTheInstallation = buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot);
 
         // Define a predicate for displaying DatasetFieldTypes on create.
         // If onlyDisplayedOnCreate is true, include fields that:
@@ -941,28 +965,57 @@ public List<DatasetFieldType> findAllInMetadataBlockAndDataverse(MetadataBlock m
                 ? criteriaBuilder.or(
                 criteriaBuilder.or(
                         criteriaBuilder.isTrue(datasetFieldTypeRoot.get("displayOnCreate")),
-                        requiredInDataversePredicate
+                        fieldRequiredInTheInstallation
                 ),
                 requiredAsInputLevelPredicate
         )
                 : criteriaBuilder.conjunction();
 
-        // Build the final WHERE clause by combining all the predicates.
-        criteriaQuery.where(
+        // Combine all the predicates.
+        return criteriaBuilder.and(
                 criteriaBuilder.equal(dataverseRoot.get("id"), dataverse.getId()), // Match the Dataverse ID.
-                criteriaBuilder.equal(metadataBlockRoot.get("id"), metadataBlock.getId()), // Match the MetadataBlock ID.
                 metadataBlockRoot.in(dataverseRoot.get("metadataBlocks")), // Ensure the MetadataBlock is part of the Dataverse.
-                datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")), // Ensure the DatasetFieldType is part of the MetadataBlock.
                 criteriaBuilder.or(includedAsInputLevelPredicate, hasNoInputLevelPredicate), // Include DatasetFieldTypes based on the input level predicates.
                 displayedOnCreatePredicate // Apply the display-on-create filter if necessary.
         );
+    }
 
-        criteriaQuery.select(datasetFieldTypeRoot).distinct(true);
-
-        return em.createQuery(criteriaQuery).getResultList();
+    private Predicate buildFieldPresentInDatasetTypePredicate(DatasetType datasetType,
+                                                              CriteriaQuery<DatasetFieldType> criteriaQuery,
+                                                              CriteriaBuilder criteriaBuilder,
+                                                              Root<DatasetFieldType> datasetFieldTypeRoot,
+                                                              Root<MetadataBlock> metadataBlockRoot,
+                                                              boolean onlyDisplayedOnCreate) {
+        Predicate datasetTypePredicate = criteriaBuilder.isFalse(criteriaBuilder.literal(true)); // Initialize datasetTypePredicate to always false by default
+        if (datasetType != null) {
+            // Create a subquery to check for the presence of the specified metadataBlock within the datasetType
+            Subquery<Long> datasetTypeSubquery = criteriaQuery.subquery(Long.class);
+            Root<DatasetType> datasetTypeRoot = criteriaQuery.from(DatasetType.class);
+
+            // Define a predicate for displaying DatasetFieldTypes on create.
+            // If onlyDisplayedOnCreate is true, include fields that are either marked as displayed on create OR marked as required.
+            // Otherwise, use an always-true predicate (conjunction).
+            Predicate displayedOnCreatePredicate = onlyDisplayedOnCreate ?
+                    criteriaBuilder.or(
+                            criteriaBuilder.isTrue(datasetFieldTypeRoot.get("displayOnCreate")),
+                            buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot)
+                    )
+                    : criteriaBuilder.conjunction();
+
+            datasetTypeSubquery.select(criteriaBuilder.literal(1L))
+                    .where(
+                            criteriaBuilder.equal(datasetTypeRoot.get("id"), datasetType.getId()), // Match the DatasetType ID.
+                            metadataBlockRoot.in(datasetTypeRoot.get("metadataBlocks")), // Ensure the metadataBlock is included in the datasetType's list of metadata blocks.
+                            displayedOnCreatePredicate
+                    );
+
+            // Now set the datasetTypePredicate to true if the subquery finds a matching metadataBlock
+            datasetTypePredicate = criteriaBuilder.exists(datasetTypeSubquery);
+        }
+        return datasetTypePredicate;
     }
 
-    private Predicate buildRequiredInDataversePredicate(CriteriaBuilder criteriaBuilder, Root<DatasetFieldType> datasetFieldTypeRoot) {
+    private Predicate buildFieldRequiredInTheInstallationPredicate(CriteriaBuilder criteriaBuilder, Root<DatasetFieldType> datasetFieldTypeRoot) {
         // Predicate to check if the current DatasetFieldType is required.
         Predicate isRequired = criteriaBuilder.isTrue(datasetFieldTypeRoot.get("required"));
 

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -5109,14 +5109,10 @@ public Response resetPidGenerator(@Context ContainerRequestContext crc, @PathPar
     @Path("datasetTypes")
     public Response getDatasetTypes() {
         JsonArrayBuilder jab = Json.createArrayBuilder();
-        List<DatasetType> datasetTypes = datasetTypeSvc.listAll();
-        for (DatasetType datasetType : datasetTypes) {
-            JsonObjectBuilder job = Json.createObjectBuilder();
-            job.add("id", datasetType.getId());
-            job.add("name", datasetType.getName());
-            jab.add(job);
-        }
-        return ok(jab.build());
+        for (DatasetType datasetType : datasetTypeSvc.listAll()) {
+            jab.add(datasetType.toJson());
+        }
+        return ok(jab);
     }
 
     @GET
@@ -5231,4 +5227,53 @@ public Response deleteDatasetType(@Context ContainerRequestContext crc, @PathPar
         }
     }
 
+    @AuthRequired
+    @PUT
+    @Path("datasetTypes/{idOrName}")
+    public Response updateDatasetTypeLinksWithMetadataBlocks(@Context ContainerRequestContext crc, @PathParam("idOrName") String idOrName, String jsonBody) {
+        DatasetType datasetType = null;
+        if (StringUtils.isNumeric(idOrName)) {
+            try {
+                long id = Long.parseLong(idOrName);
+                datasetType = datasetTypeSvc.getById(id);
+            } catch (NumberFormatException ex) {
+                return error(NOT_FOUND, "Could not find a dataset type with id " + idOrName);
+            }
+        } else {
+            datasetType = datasetTypeSvc.getByName(idOrName);
+        }
+        JsonArrayBuilder datasetTypesBefore = Json.createArrayBuilder();
+        for (MetadataBlock metadataBlock : datasetType.getMetadataBlocks()) {
+            datasetTypesBefore.add(metadataBlock.getName());
+        }
+        JsonArrayBuilder datasetTypesAfter = Json.createArrayBuilder();
+        List<MetadataBlock> metadataBlocksToSave = new ArrayList<>();
+        if (jsonBody != null && !jsonBody.isEmpty()) {
+            JsonArray json = JsonUtil.getJsonArray(jsonBody);
+            for (JsonString jsonValue : json.getValuesAs(JsonString.class)) {
+                String name = jsonValue.getString();
+                System.out.println("name: " + name);
+                MetadataBlock metadataBlock = metadataBlockSvc.findByName(name);
+                if (metadataBlock != null) {
+                    metadataBlocksToSave.add(metadataBlock);
+                    datasetTypesAfter.add(name);
+                } else {
+                    String availableBlocks = metadataBlockSvc.listMetadataBlocks().stream().map(MetadataBlock::getName).collect(Collectors.joining(", "));
+                    return badRequest("Metadata block not found: " + name + ". Available metadata blocks: " + availableBlocks);
+                }
+            }
+        }
+        try {
+            execCommand(new UpdateDatasetTypeLinksToMetadataBlocks(createDataverseRequest(getRequestUser(crc)), datasetType, metadataBlocksToSave));
+            return ok(Json.createObjectBuilder()
+                    .add("linkedMetadataBlocks", Json.createObjectBuilder()
+                            .add("before", datasetTypesBefore)
+                            .add("after", datasetTypesAfter))
+            );
+
+        } catch (WrappedResponse ex) {
+            return ex.getResponse();
+        }
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java

@@ -15,6 +15,7 @@
 import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
+import edu.harvard.iq.dataverse.dataset.DatasetType;
 import edu.harvard.iq.dataverse.dataverse.DataverseUtil;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.impl.*;
@@ -801,17 +802,20 @@ public Response deleteDataverseLinkingDataverse(@Context ContainerRequestContext
     public Response listMetadataBlocks(@Context ContainerRequestContext crc,
                                        @PathParam("identifier") String dvIdtf,
                                        @QueryParam("onlyDisplayedOnCreate") boolean onlyDisplayedOnCreate,
-                                       @QueryParam("returnDatasetFieldTypes") boolean returnDatasetFieldTypes) {
+                                       @QueryParam("returnDatasetFieldTypes") boolean returnDatasetFieldTypes,
+                                       @QueryParam("datasetType") String datasetTypeIn) {
         try {
             Dataverse dataverse = findDataverseOrDie(dvIdtf);
+            DatasetType datasetType = datasetTypeSvc.getByName(datasetTypeIn);
             final List<MetadataBlock> metadataBlocks = execCommand(
                     new ListMetadataBlocksCommand(
                             createDataverseRequest(getRequestUser(crc)),
                             dataverse,
-                            onlyDisplayedOnCreate
+                            onlyDisplayedOnCreate,
+                            datasetType
                     )
             );
-            return ok(json(metadataBlocks, returnDatasetFieldTypes, onlyDisplayedOnCreate, dataverse));
+            return ok(json(metadataBlocks, returnDatasetFieldTypes, onlyDisplayedOnCreate, dataverse, datasetType));
         } catch (WrappedResponse we) {
             return we.getResponse();
         }

src/main/java/edu/harvard/iq/dataverse/dataset/DatasetType.java

@@ -1,17 +1,23 @@
 package edu.harvard.iq.dataverse.dataset;
 
+import edu.harvard.iq.dataverse.MetadataBlock;
 import jakarta.json.Json;
+import jakarta.json.JsonArrayBuilder;
 import jakarta.json.JsonObjectBuilder;
+import jakarta.persistence.CascadeType;
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
+import jakarta.persistence.ManyToMany;
 import jakarta.persistence.NamedQueries;
 import jakarta.persistence.NamedQuery;
 import jakarta.persistence.Table;
 import jakarta.persistence.UniqueConstraint;
 import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
 
 @NamedQueries({
     @NamedQuery(name = "DatasetType.findAll",
@@ -42,6 +48,12 @@ public class DatasetType implements Serializable {
     @Column(nullable = false)
     private String name;
 
+    /**
+     * The metadata blocks this dataset type is linked to.
+     */
+    @ManyToMany(cascade = {CascadeType.MERGE})
+    private List<MetadataBlock> metadataBlocks = new ArrayList<>();
+
     public DatasetType() {
     }
 
@@ -61,10 +73,23 @@ public void setName(String name) {
         this.name = name;
     }
 
+    public List<MetadataBlock> getMetadataBlocks() {
+        return metadataBlocks;
+    }
+
+    public void setMetadataBlocks(List<MetadataBlock> metadataBlocks) {
+        this.metadataBlocks = metadataBlocks;
+    }
+
     public JsonObjectBuilder toJson() {
+        JsonArrayBuilder linkedMetadataBlocks = Json.createArrayBuilder();
+        for (MetadataBlock metadataBlock : this.getMetadataBlocks()) {
+            linkedMetadataBlocks.add(metadataBlock.getName());
+        }
         return Json.createObjectBuilder()
                 .add("id", getId())
-                .add("name", getName());
+                .add("name", getName())
+                .add("linkedMetadataBlocks", linkedMetadataBlocks);
     }
 
 }