5028 Add dataset level external tools

conf/docker-aio/run-test-suite.sh

@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT -Ddataverse.test.baseurl=$dvurl
+mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT -Ddataverse.test.baseurl=$dvurl

doc/release-notes/5028-dataset-explore

@@ -0,0 +1,5 @@
+There is a DB migration at: 
+
+https://github.com/IQSS/dataverse/blob/5028-dataset-explore-btn/src/main/resources/db/migration/V4.15.1.1__5028-dataset-explore.sql
+
+We will need to reflect this in the release notes. Verify at release time. 

doc/sphinx-guides/source/_static/installation/files/root/external-tools/awesomeTool.json

@@ -2,6 +2,7 @@
   "displayName": "Awesome Tool",
   "description": "The most awesome tool.",
   "type": "explore",
+  "scope": "file",
   "contentType": "text/tab-separated-values",
   "toolUrl": "https://awesometool.com",
   "toolParameters": {

doc/sphinx-guides/source/installation/external-tools.rst

@@ -9,7 +9,7 @@ External tools can provide additional features that are not part of Dataverse it
 Inventory of External Tools
 ---------------------------
 
-Support for external tools is just getting off the ground but the following tools have been successfully integrated with Dataverse:
+The following tools have been successfully integrated with Dataverse:
 
 - TwoRavens: a system of interlocking statistical tools for data exploration, analysis, and meta-analysis: http://2ra.vn. See the :doc:`/user/data-exploration/tworavens` section of the User Guide for more information on TwoRavens from the user perspective and the :doc:`r-rapache-tworavens` section of the Installation Guide. 
 
@@ -35,14 +35,17 @@ External tools must be expressed in an external tool manifest file, a specific J
 
 ``type`` is required and must be ``explore`` or ``configure`` to make the tool appear under a button called "Explore" or "Configure", respectively. 
 
-External tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. (Not providing this parameter makes the tool work on ingested tabular files and is equivalent to specifying the ``contentType`` as "text/tab-separated-values".)  
+``scope`` is required and must be ``file`` or ``dataset`` to make the tool appear at the file level or dataset level.
+
+File level tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. (Not providing this parameter makes the tool work on ingested tabular files and is equivalent to specifying the ``contentType`` as "text/tab-separated-values".)  
 
 In the example above, a mix of required and optional reserved words appear that can be used to insert dynamic values into tools. The supported values are:
 
-- ``{fileId}`` (required) - The Dataverse database ID of a file the external tool has been launched on.
-- ``{siteUrl}`` (optional) - The URL of the Dataverse installation that hosts the file with the fileId above.
+- ``{fileId}`` (required for file tools) - The Dataverse database ID of a file from which the external tool has been launched.
+- ``{siteUrl}`` (optional) - The URL of the Dataverse installation from which the tool was launched.
 - ``{apiToken}`` (optional) - The Dataverse API token of the user launching the external tool, if available.
-- ``{datasetId}`` (optional) - The ID of the dataset containing the file.
+- ``{datasetId}`` (optional) - The ID of the dataset.
+- ``{datasetPid}`` (optional) - The Persistent ID (DOI or Handle) of the dataset.
 - ``{datasetVersion}`` (optional) - The friendly version number ( or \:draft ) of the dataset version the tool is being launched from.
 
 Making an External Tool Available in Dataverse
@@ -59,6 +62,15 @@ To list all the external tools that are available in Dataverse:
 
 ``curl http://localhost:8080/api/admin/externalTools``
 
+Showing an External Tool in Dataverse
+-------------------------------------
+
+To show one of the external tools that are available in Dataverse, pass its database id:
+
+``export TOOL_ID=1``
+
+``curl http://localhost:8080/api/admin/externalTools/$TOOL_ID``
+
 Removing an External Tool Available in Dataverse
 ------------------------------------------------
 

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -6,8 +6,10 @@
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
+import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
+import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
@@ -99,6 +101,7 @@
 import edu.harvard.iq.dataverse.externaltools.ExternalTool;
 import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean;
 import edu.harvard.iq.dataverse.export.SchemaDotOrgExporter;
+import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler;
 import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean;
 import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry;
 import java.util.Collections;
@@ -135,6 +138,7 @@
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
+import org.primefaces.PrimeFaces;
 import org.primefaces.model.DefaultTreeNode;
 import org.primefaces.model.TreeNode;
 
@@ -316,6 +320,7 @@ public void setShowIngestSuccess(boolean showIngestSuccess) {
     List<ExternalTool> exploreTools = new ArrayList<>();
     Map<Long, List<ExternalTool>> configureToolsByFileId = new HashMap<>();
     Map<Long, List<ExternalTool>> exploreToolsByFileId = new HashMap<>();
+    private List<ExternalTool> datasetExploreTools;
 
     public Boolean isHasRsyncScript() {
         return hasRsyncScript;
@@ -2027,6 +2032,7 @@ private String init(boolean initFull) {
 
         configureTools = externalToolService.findByType(ExternalTool.Type.CONFIGURE);
         exploreTools = externalToolService.findByType(ExternalTool.Type.EXPLORE);
+        datasetExploreTools = externalToolService.findByScopeAndType(ExternalTool.Scope.DATASET, ExternalTool.Type.EXPLORE);
         rowsPerPage = 10;
 
 
@@ -5063,6 +5069,10 @@ public List<ExternalTool> getCachedToolsForDataFile(Long fileId, ExternalTool.Ty
         return cachedTools;
     }
 
+    public List<ExternalTool> getDatasetExploreTools() {
+        return datasetExploreTools;
+    }
+
     Boolean thisLatestReleasedVersion = null;
 
     public boolean isThisLatestReleasedVersion() {
@@ -5211,4 +5221,17 @@ public int compare(FileMetadata o1, FileMetadata o2) {
             return type1.compareTo(type2);
         }
     };
+
+    public void explore(ExternalTool externalTool) {
+        ApiToken apiToken = null;
+        User user = session.getUser();
+        if (user instanceof AuthenticatedUser) {
+            apiToken = authService.findApiTokenByUser((AuthenticatedUser) user);
+        }
+        ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataset, apiToken);
+        String toolUrl = externalToolHandler.getToolUrlWithQueryParams();
+        logger.fine("Exploring with " + toolUrl);
+        PrimeFaces.current().executeScript("window.open('"+toolUrl + "', target='_blank');");
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java

@@ -28,6 +28,17 @@ public Response getExternalTools() {
         return ok(jab);
     }
 
+    @GET
+    @Path("{id}")
+    public Response getExternalTool(@PathParam("id") long externalToolIdFromUser) {
+        ExternalTool externalTool = externalToolService.findById(externalToolIdFromUser);
+        if (externalTool != null) {
+            return ok(externalTool.toJson());
+        } else {
+            return error(BAD_REQUEST, "Could not find external tool with id of " + externalToolIdFromUser);
+        }
+    }
+
     @POST
     public Response addExternalTool(String manifest) {
         try {

src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java

@@ -23,6 +23,7 @@ public class ExternalTool implements Serializable {
     public static final String DISPLAY_NAME = "displayName";
     public static final String DESCRIPTION = "description";
     public static final String TYPE = "type";
+    public static final String SCOPE = "scope";
     public static final String TOOL_URL = "toolUrl";
     public static final String TOOL_PARAMETERS = "toolParameters";
     public static final String CONTENT_TYPE = "contentType";
@@ -52,6 +53,13 @@ public class ExternalTool implements Serializable {
     @Enumerated(EnumType.STRING)
     private Type type;
 
+    /**
+     * Whether the tool operates at the dataset or file level.
+     */
+    @Column(nullable = false)
+    @Enumerated(EnumType.STRING)
+    private Scope scope;
+
     @Column(nullable = false)
     private String toolUrl;
 
@@ -83,10 +91,11 @@ public class ExternalTool implements Serializable {
     public ExternalTool() {
     }
 
-    public ExternalTool(String displayName, String description, Type type, String toolUrl, String toolParameters, String contentType) {
+    public ExternalTool(String displayName, String description, Type type, Scope scope, String toolUrl, String toolParameters, String contentType) {
         this.displayName = displayName;
         this.description = description;
         this.type = type;
+        this.scope = scope;
         this.toolUrl = toolUrl;
         this.toolParameters = toolParameters;
         this.contentType = contentType;
@@ -120,6 +129,34 @@ public String toString() {
         }
     }
 
+    public enum Scope {
+
+        DATASET("dataset"),
+        FILE("file");
+
+        private final String text;
+
+        private Scope(final String text) {
+            this.text = text;
+        }
+
+        public static Scope fromString(String text) {
+            if (text != null) {
+                for (Scope scope : Scope.values()) {
+                    if (text.equals(scope.text)) {
+                        return scope;
+                    }
+                }
+            }
+            throw new IllegalArgumentException("Scope must be one of these values: " + Arrays.asList(Scope.values()) + ".");
+        }
+
+        @Override
+        public String toString() {
+            return text;
+        }
+    }
+
     public Long getId() {
         return id;
     }
@@ -148,6 +185,10 @@ public Type getType() {
         return type;
     }
 
+    public Scope getScope() {
+        return scope;
+    }
+
     public String getToolUrl() {
         return toolUrl;
     }
@@ -178,6 +219,7 @@ public JsonObjectBuilder toJson() {
         jab.add(DISPLAY_NAME, getDisplayName());
         jab.add(DESCRIPTION, getDescription());
         jab.add(TYPE, getType().text);
+        jab.add(SCOPE, getScope().text);
         jab.add(TOOL_URL, getToolUrl());
         jab.add(TOOL_PARAMETERS, getToolParameters());
         jab.add(CONTENT_TYPE, getContentType());
@@ -193,7 +235,10 @@ public enum ReservedWord {
         FILE_ID("fileId"),
         SITE_URL("siteUrl"),
         API_TOKEN("apiToken"),
+        // datasetId is the database id
         DATASET_ID("datasetId"),
+        // datasetPid is the DOI or Handle
+        DATASET_PID("datasetPid"),
         DATASET_VERSION("datasetVersion"),
         FILE_METADATA_ID("fileMetadataId");
 

src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java

@@ -33,6 +33,8 @@ public class ExternalToolHandler {
     private ApiToken apiToken;
 
     /**
+     * File level tool
+     *
      * @param externalTool The database entity.
      * @param dataFile Required.
      * @param apiToken The apiToken can be null because "explore" tools can be
@@ -51,6 +53,27 @@ public ExternalToolHandler(ExternalTool externalTool, DataFile dataFile, ApiToke
         this.fileMetadata = fileMetadata;
     }
 
+    /**
+     * Dataset level tool
+     *
+     * @param externalTool The database entity.
+     * @param dataset Required.
+     * @param apiToken The apiToken can be null because "explore" tools can be
+     * used anonymously.
+     */
+    public ExternalToolHandler(ExternalTool externalTool, Dataset dataset, ApiToken apiToken) {
+        this.externalTool = externalTool;
+        if (dataset == null) {
+            String error = "A Dataset is required.";
+            logger.warning("Error in ExternalToolHandler constructor: " + error);
+            throw new IllegalArgumentException(error);
+        }
+        this.dataset = dataset;
+        this.apiToken = apiToken;
+        this.dataFile = null;
+        this.fileMetadata = null;
+    }
+
     public DataFile getDataFile() {
         return dataFile;
     }
@@ -89,7 +112,7 @@ private String getQueryParam(String key, String value) {
         ReservedWord reservedWord = ReservedWord.fromString(value);
         switch (reservedWord) {
             case FILE_ID:
-                // getDataFile is never null because of the constructor
+                // getDataFile is never null for file tools because of the constructor
                 return key + "=" + getDataFile().getId();
             case SITE_URL:
                 return key + "=" + SystemConfig.getDataverseSiteUrlStatic();
@@ -103,6 +126,8 @@ private String getQueryParam(String key, String value) {
                 break;
             case DATASET_ID:
                 return key + "=" + dataset.getId();
+            case DATASET_PID:
+                return key + "=" + dataset.getGlobalId().asString();
             case DATASET_VERSION:
                 String version = null;
                 if (getApiToken() != null) {

src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java

@@ -4,12 +4,14 @@
 import edu.harvard.iq.dataverse.DataFileServiceBean;
 import edu.harvard.iq.dataverse.externaltools.ExternalTool.ReservedWord;
 import edu.harvard.iq.dataverse.externaltools.ExternalTool.Type;
+import edu.harvard.iq.dataverse.externaltools.ExternalTool.Scope;
 
 import static edu.harvard.iq.dataverse.externaltools.ExternalTool.DESCRIPTION;
 import static edu.harvard.iq.dataverse.externaltools.ExternalTool.DISPLAY_NAME;
 import static edu.harvard.iq.dataverse.externaltools.ExternalTool.TOOL_PARAMETERS;
 import static edu.harvard.iq.dataverse.externaltools.ExternalTool.TOOL_URL;
 import static edu.harvard.iq.dataverse.externaltools.ExternalTool.TYPE;
+import static edu.harvard.iq.dataverse.externaltools.ExternalTool.SCOPE;
 import static edu.harvard.iq.dataverse.externaltools.ExternalTool.CONTENT_TYPE;
 import java.io.StringReader;
 import java.util.ArrayList;
@@ -74,7 +76,21 @@ public List<ExternalTool> findByType(Type type, String contentType) {
         return externalTools;
     }
 
-
+    /**
+     * @param scope - dataset or file
+     * @return A list of tools or an empty list.
+     */
+    public List<ExternalTool> findByScopeAndType(Scope scope, Type type) {
+        List<ExternalTool> externalTools = new ArrayList<>();
+        TypedQuery<ExternalTool> typedQuery = em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.scope = :scope AND o.type = :type", ExternalTool.class);
+        typedQuery.setParameter("scope", scope);
+        typedQuery.setParameter("type", type);
+        List<ExternalTool> toolsFromQuery = typedQuery.getResultList();
+        if (toolsFromQuery != null) {
+            externalTools = toolsFromQuery;
+        }
+        return externalTools;
+    }
 
     public ExternalTool findById(long id) {
         TypedQuery<ExternalTool> typedQuery = em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.id = :id", ExternalTool.class);
@@ -131,6 +147,7 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) {
         String displayName = getRequiredTopLevelField(jsonObject, DISPLAY_NAME);
         String description = getRequiredTopLevelField(jsonObject, DESCRIPTION);
         String typeUserInput = getRequiredTopLevelField(jsonObject, TYPE);
+        String scopeUserInput = getRequiredTopLevelField(jsonObject, SCOPE);
         String contentType = getOptionalTopLevelField(jsonObject, CONTENT_TYPE);
         //Legacy support - assume tool manifests without any mimetype are for tabular data
         if(contentType==null) {
@@ -139,26 +156,29 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) {
 
         // Allow IllegalArgumentException to bubble up from ExternalTool.Type.fromString
         ExternalTool.Type type = ExternalTool.Type.fromString(typeUserInput);
+        ExternalTool.Scope scope = ExternalTool.Scope.fromString(scopeUserInput);
         String toolUrl = getRequiredTopLevelField(jsonObject, TOOL_URL);
         JsonObject toolParametersObj = jsonObject.getJsonObject(TOOL_PARAMETERS);
         JsonArray queryParams = toolParametersObj.getJsonArray("queryParameters");
         boolean allRequiredReservedWordsFound = false;
-        for (JsonObject queryParam : queryParams.getValuesAs(JsonObject.class)) {
-            Set<String> keyValuePair = queryParam.keySet();
-            for (String key : keyValuePair) {
-                String value = queryParam.getString(key);
-                ReservedWord reservedWord = ReservedWord.fromString(value);
-                if (reservedWord.equals(ReservedWord.FILE_ID)) {
-                    allRequiredReservedWordsFound = true;
+        if (scope.equals(Scope.FILE)) {
+            for (JsonObject queryParam : queryParams.getValuesAs(JsonObject.class)) {
+                Set<String> keyValuePair = queryParam.keySet();
+                for (String key : keyValuePair) {
+                    String value = queryParam.getString(key);
+                    ReservedWord reservedWord = ReservedWord.fromString(value);
+                    if (reservedWord.equals(ReservedWord.FILE_ID)) {
+                        allRequiredReservedWordsFound = true;
+                    }
                 }
             }
-        }
-        if (!allRequiredReservedWordsFound) {
-            // Some day there might be more reserved words than just {fileId}.
-            throw new IllegalArgumentException("Required reserved word not found: " + ReservedWord.FILE_ID.toString());
+            if (!allRequiredReservedWordsFound) {
+                // Some day there might be more reserved words than just {fileId}.
+                throw new IllegalArgumentException("Required reserved word not found: " + ReservedWord.FILE_ID.toString());
+            }
         }
         String toolParameters = toolParametersObj.toString();
-        return new ExternalTool(displayName, description, type, toolUrl, toolParameters, contentType);
+        return new ExternalTool(displayName, description, type, scope, toolUrl, toolParameters, contentType);
     }
 
     private static String getRequiredTopLevelField(JsonObject jsonObject, String key) {