5028 Add dataset level external tools

conf/docker-aio/run-test-suite.sh

@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT -Ddataverse.test.baseurl=$dvurl
+mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT -Ddataverse.test.baseurl=$dvurl

doc/sphinx-guides/source/_static/installation/files/root/external-tools/awesomeTool.json

@@ -2,6 +2,7 @@
   "displayName": "Awesome Tool",
   "description": "The most awesome tool.",
   "type": "explore",
+  "scope": "file",
   "contentType": "text/tab-separated-values",
   "toolUrl": "https://awesometool.com",
   "toolParameters": {

doc/sphinx-guides/source/api/native-api.rst

@@ -868,6 +868,33 @@ Normally published datasets should not be deleted, but there exists a "destroy"
 
 Calling the destroy endpoint is permanent and irreversible. It will remove the dataset and its datafiles, then re-index the parent dataverse in Solr. This endpoint requires the API token of a superuser.
 
+.. _list-external-tools-for-a-dataset-api:
+
+List External Tools for a Dataset
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some installations of Dataverse have external tools enabled for datasets and there are two types:
+
+- explore
+- configure
+
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB
+  export TYPE=explore
+
+  curl -H X-Dataverse-key:$API_TOKEN \""$SERVER_URL/api/datasets/:persistentId/externalTools?persistentId=$PERSISTENT_ID&type=$TYPE"\"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx "https://demo.dataverse.org/api/datasets/:persistentId/externalTools?persistentId=doi:10.5072/FK2/J8SJZB&type=explore"
+
 Files
 -----
 
@@ -1023,6 +1050,33 @@ Starting the release 4.10 the size of the saved original file (for an ingested t
 
 Note the optional "limit" parameter. Without it, the API will attempt to populate the sizes for all the saved originals that don't have them in the database yet. Otherwise it will do so for the first N such datafiles. 
 
+.. _list-external-tools-for-a-file-api:
+
+List External Tools for a File
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some installations of Dataverse have external tools enabled for files and there are two types:
+
+- explore
+- configure
+
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB
+  export TYPE=explore
+
+  curl -H X-Dataverse-key:$API_TOKEN \""$SERVER_URL/api/files/:persistentId/externalTools?persistentId=$PERSISTENT_ID&type=$TYPE"\"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx "https://demo.dataverse.org/api/files/:persistentId/externalTools?persistentId=doi:10.5072/FK2/J8SJZB&type=explore"
+
 Builtin Users
 -------------
 

doc/sphinx-guides/source/installation/external-tools.rst

@@ -9,7 +9,7 @@ External tools can provide additional features that are not part of Dataverse it
 Inventory of External Tools
 ---------------------------
 
-Support for external tools is just getting off the ground but the following tools have been successfully integrated with Dataverse:
+The following tools have been successfully integrated with Dataverse:
 
 - TwoRavens: a system of interlocking statistical tools for data exploration, analysis, and meta-analysis: http://2ra.vn. See the :doc:`/user/data-exploration/tworavens` section of the User Guide for more information on TwoRavens from the user perspective and the :doc:`r-rapache-tworavens` section of the Installation Guide. 
 
@@ -35,14 +35,18 @@ External tools must be expressed in an external tool manifest file, a specific J
 
 ``type`` is required and must be ``explore`` or ``configure`` to make the tool appear under a button called "Explore" or "Configure", respectively. 
 
-External tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. (Not providing this parameter makes the tool work on ingested tabular files and is equivalent to specifying the ``contentType`` as "text/tab-separated-values".)  
+``scope`` is required and must be ``file`` or ``dataset`` to make the tool appear at the file level or dataset level.
+
+File level tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on.
 
 In the example above, a mix of required and optional reserved words appear that can be used to insert dynamic values into tools. The supported values are:
 
-- ``{fileId}`` (required) - The Dataverse database ID of a file the external tool has been launched on.
-- ``{siteUrl}`` (optional) - The URL of the Dataverse installation that hosts the file with the fileId above.
+- ``{fileId}`` (required for file tools, or ``{filePid}``) - The database ID of a file from which the external tool has been launched.
+- ``{filePid}`` (required for file tools, or ``{fileId}``) - The Persistent ID (DOI or Handle) of a file (if available) from which the external tool has been launched.
+- ``{siteUrl}`` (optional) - The URL of the Dataverse installation from which the tool was launched.
 - ``{apiToken}`` (optional) - The Dataverse API token of the user launching the external tool, if available.
-- ``{datasetId}`` (optional) - The ID of the dataset containing the file.
+- ``{datasetId}`` (required for dataset tools, or ``{datasetPid}``) - The database ID of the dataset.
+- ``{datasetPid}`` (required for dataset tools, or ``{datasetId}``) - The Persistent ID (DOI or Handle) of the dataset.
 - ``{datasetVersion}`` (optional) - The friendly version number ( or \:draft ) of the dataset version the tool is being launched from.
 
 Making an External Tool Available in Dataverse
@@ -59,13 +63,32 @@ To list all the external tools that are available in Dataverse:
 
 ``curl http://localhost:8080/api/admin/externalTools``
 
+Showing an External Tool in Dataverse
+-------------------------------------
+
+To show one of the external tools that are available in Dataverse, pass its database id:
+
+``export TOOL_ID=1``
+
+``curl http://localhost:8080/api/admin/externalTools/$TOOL_ID``
+
 Removing an External Tool Available in Dataverse
 ------------------------------------------------
 
 Assuming the external tool database id is "1", remove it with the following command:
 
 ``curl -X DELETE http://localhost:8080/api/admin/externalTools/1``
 
+List External Tools for a Dataset
+---------------------------------
+
+See :ref:`list-external-tools-for-a-dataset-api`
+
+List External Tools for a File
+------------------------------
+
+See :ref:`list-external-tools-for-a-file-api`
+
 Writing Your Own External Tool
 ------------------------------
 

doc/sphinx-guides/source/user/data-exploration/index.rst

@@ -8,6 +8,11 @@ Data Exploration Guide
 
 Note that the installation of Dataverse you are using may have additional or different tools configured. Developers interested in creating tools should refer to the :doc:`/installation/external-tools` section of the Installation Guide.
 
+API users can determine if external tools are configured at the dataset for file level by following the instructions below.
+
+- :ref:`list-external-tools-for-a-dataset-api`
+- :ref:`list-external-tools-for-a-file-api`
+
 Contents:
 
 .. toctree::

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -6,8 +6,10 @@
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
+import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
+import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
@@ -99,6 +101,7 @@
 import edu.harvard.iq.dataverse.externaltools.ExternalTool;
 import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean;
 import edu.harvard.iq.dataverse.export.SchemaDotOrgExporter;
+import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler;
 import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean;
 import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry;
 import java.util.Collections;
@@ -135,6 +138,7 @@
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
+import org.primefaces.PrimeFaces;
 import org.primefaces.model.DefaultTreeNode;
 import org.primefaces.model.TreeNode;
 
@@ -312,10 +316,15 @@ public void setShowIngestSuccess(boolean showIngestSuccess) {
         this.showIngestSuccess = showIngestSuccess;
     }
 
+    // TODO: Consider renaming "configureTools" to "fileConfigureTools".
     List<ExternalTool> configureTools = new ArrayList<>();
+    // TODO: Consider renaming "exploreTools" to "fileExploreTools".
     List<ExternalTool> exploreTools = new ArrayList<>();
+    // TODO: Consider renaming "configureToolsByFileId" to "fileConfigureToolsByFileId".
     Map<Long, List<ExternalTool>> configureToolsByFileId = new HashMap<>();
+    // TODO: Consider renaming "exploreToolsByFileId" to "fileExploreToolsByFileId".
     Map<Long, List<ExternalTool>> exploreToolsByFileId = new HashMap<>();
+    private List<ExternalTool> datasetExploreTools;
 
     public Boolean isHasRsyncScript() {
         return hasRsyncScript;
@@ -2014,8 +2023,9 @@ private String init(boolean initFull) {
             JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.unlocked.ingest.message"));
         }
 
-        configureTools = externalToolService.findByType(ExternalTool.Type.CONFIGURE);
-        exploreTools = externalToolService.findByType(ExternalTool.Type.EXPLORE);
+        configureTools = externalToolService.findFileToolsByType(ExternalTool.Type.CONFIGURE);
+        exploreTools = externalToolService.findFileToolsByType(ExternalTool.Type.EXPLORE);
+        datasetExploreTools = externalToolService.findDatasetToolsByType(ExternalTool.Type.EXPLORE);
         rowsPerPage = 10;
 
 
@@ -5052,6 +5062,10 @@ public List<ExternalTool> getCachedToolsForDataFile(Long fileId, ExternalTool.Ty
         return cachedTools;
     }
 
+    public List<ExternalTool> getDatasetExploreTools() {
+        return datasetExploreTools;
+    }
+
     Boolean thisLatestReleasedVersion = null;
 
     public boolean isThisLatestReleasedVersion() {
@@ -5200,4 +5214,17 @@ public int compare(FileMetadata o1, FileMetadata o2) {
             return type1.compareTo(type2);
         }
     };
+
+    public void explore(ExternalTool externalTool) {
+        ApiToken apiToken = null;
+        User user = session.getUser();
+        if (user instanceof AuthenticatedUser) {
+            apiToken = authService.findApiTokenByUser((AuthenticatedUser) user);
+        }
+        ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataset, apiToken);
+        String toolUrl = externalToolHandler.getToolUrlWithQueryParams();
+        logger.fine("Exploring with " + toolUrl);
+        PrimeFaces.current().executeScript("window.open('"+toolUrl + "', target='_blank');");
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/FilePage.java

@@ -206,8 +206,8 @@ public String init() {
             if (file.isTabularData()) {
                 contentType=DataFileServiceBean.MIME_TYPE_TSV_ALT;
             }
-            configureTools = externalToolService.findByType(ExternalTool.Type.CONFIGURE, contentType);
-            exploreTools = externalToolService.findByType(ExternalTool.Type.EXPLORE, contentType);
+            configureTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.CONFIGURE, contentType);
+            exploreTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.EXPLORE, contentType);
 
         } else {
 

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -77,13 +77,16 @@
 import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
 import edu.harvard.iq.dataverse.S3PackageImporter;
 import static edu.harvard.iq.dataverse.api.AbstractApiBean.error;
+import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.batch.util.LoggingUtil;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
 import edu.harvard.iq.dataverse.engine.command.exception.UnforcedCommandException;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataFileCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDvObjectPIDMetadataCommand;
+import edu.harvard.iq.dataverse.externaltools.ExternalTool;
+import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler;
 import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitations;
 import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitationsServiceBean;
 import edu.harvard.iq.dataverse.makedatacount.DatasetMetrics;
@@ -1893,5 +1896,31 @@ public Response getMakeDataCountMetric(@PathParam("id") String idSupplied, @Path
         }
     }
 
+    @GET
+    @Path("{id}/externalTools")
+    public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam("type") String typeSupplied) {
+        ExternalTool.Type type;
+        try {
+            type = ExternalTool.Type.fromString(typeSupplied);
+        } catch (IllegalArgumentException ex) {
+            return error(BAD_REQUEST, ex.getLocalizedMessage());
+        }
+        Dataset dataset;
+        try {
+            dataset = findDatasetOrDie(idSupplied);
+            JsonArrayBuilder tools = Json.createArrayBuilder();
+            List<ExternalTool> datasetTools = externalToolService.findDatasetToolsByType(type);
+            for (ExternalTool tool : datasetTools) {
+                ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey());
+                ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataset, apiToken);
+                JsonObjectBuilder toolToJson = externalToolService.getToolAsJsonWithQueryParameters(externalToolHandler);
+                tools.add(toolToJson);
+            }
+            return ok(tools);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+    }
+
 }
 

src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java

@@ -1,11 +1,10 @@
 package edu.harvard.iq.dataverse.api;
 
-import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord;
 import static edu.harvard.iq.dataverse.api.AbstractApiBean.error;
 import edu.harvard.iq.dataverse.externaltools.ExternalTool;
 import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean;
-import java.util.List;
+import java.util.logging.Logger;
 import javax.json.Json;
 import javax.json.JsonArrayBuilder;
 import javax.ws.rs.DELETE;
@@ -19,6 +18,8 @@
 @Path("admin/externalTools")
 public class ExternalTools extends AbstractApiBean {
 
+    private static final Logger logger = Logger.getLogger(ExternalTools.class.getCanonicalName());
+
     @GET
     public Response getExternalTools() {
         JsonArrayBuilder jab = Json.createArrayBuilder();
@@ -28,6 +29,17 @@ public Response getExternalTools() {
         return ok(jab);
     }
 
+    @GET
+    @Path("{id}")
+    public Response getExternalTool(@PathParam("id") long externalToolIdFromUser) {
+        ExternalTool externalTool = externalToolService.findById(externalToolIdFromUser);
+        if (externalTool != null) {
+            return ok(externalTool.toJson());
+        } else {
+            return error(BAD_REQUEST, "Could not find external tool with id of " + externalToolIdFromUser);
+        }
+    }
+
     @POST
     public Response addExternalTool(String manifest) {
         try {
@@ -53,23 +65,4 @@ public Response deleteExternalTool(@PathParam("id") long externalToolIdFromUser)
         }
     }
 
-    // TODO: Rather than only supporting looking up files by their database IDs, consider supporting persistent identifiers.
-    @GET
-    @Path("file/{id}")
-    public Response getExternalToolsByFile(@PathParam("id") Long fileIdFromUser) {
-        DataFile dataFile = fileSvc.find(fileIdFromUser);
-        if (dataFile == null) {
-            return error(BAD_REQUEST, "Could not find datafile with id " + fileIdFromUser);
-        }
-        JsonArrayBuilder tools = Json.createArrayBuilder();
-
-        List<ExternalTool> allExternalTools = externalToolService.findAll();
-        List<ExternalTool> toolsByFile = ExternalToolServiceBean.findExternalToolsByFile(allExternalTools, dataFile);
-        for (ExternalTool tool : toolsByFile) {
-            tools.add(tool.toJson());
-        }
-
-        return ok(tools);
-    }
-
 }