From f699fd60fd6e130cda754d35d169ad240882bb77 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 25 Jul 2019 12:33:02 -0400 Subject: [PATCH 01/23] Added placeholder Explore btn to dataset pg to be wired up to backend [ref #5028] --- src/main/webapp/dataset.xhtml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 9b6cb5561be..458bfa1a805 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -130,6 +130,41 @@ + + + +
+ + + + + + + + + + + + +
+ + +
From 48d118160e5828a48bd5bae85ff5ba82d26d10e6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 26 Jul 2019 12:27:39 -0400 Subject: [PATCH 02/23] add dataset level external tools #5028 --- conf/docker-aio/run-test-suite.sh | 2 +- .../root/external-tools/awesomeTool.json | 1 + .../source/installation/external-tools.rst | 13 ++++-- .../edu/harvard/iq/dataverse/DatasetPage.java | 23 ++++++++++ .../dataverse/externaltools/ExternalTool.java | 46 ++++++++++++++++++- .../externaltools/ExternalToolHandler.java | 27 ++++++++++- .../ExternalToolServiceBean.java | 46 +++++++++++++------ .../V4.15.1.1__5028-dataset-explore.sql | 3 ++ src/main/webapp/dataset.xhtml | 29 +++++------- .../iq/dataverse/api/ExternalToolsIT.java | 5 +- .../ExternalToolHandlerTest.java | 3 +- .../ExternalToolServiceBeanTest.java | 9 +++- .../externaltools/ExternalToolTest.java | 3 +- 13 files changed, 168 insertions(+), 42 deletions(-) create mode 100644 src/main/resources/db/migration/V4.15.1.1__5028-dataset-explore.sql diff --git a/conf/docker-aio/run-test-suite.sh b/conf/docker-aio/run-test-suite.sh index 3027577b230..c1fca242389 100755 --- a/conf/docker-aio/run-test-suite.sh +++ b/conf/docker-aio/run-test-suite.sh @@ -8,4 +8,4 @@ fi # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment. # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml. -mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT -Ddataverse.test.baseurl=$dvurl +mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT -Ddataverse.test.baseurl=$dvurl diff --git a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/awesomeTool.json b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/awesomeTool.json index c3b87a3c258..caa6baefe60 100644 --- a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/awesomeTool.json +++ b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/awesomeTool.json @@ -2,6 +2,7 @@ "displayName": "Awesome Tool", "description": "The most awesome tool.", "type": "explore", + "scope": "file", "contentType": "text/tab-separated-values", "toolUrl": "https://awesometool.com", "toolParameters": { diff --git a/doc/sphinx-guides/source/installation/external-tools.rst b/doc/sphinx-guides/source/installation/external-tools.rst index e91335f40df..ef3e84693af 100644 --- a/doc/sphinx-guides/source/installation/external-tools.rst +++ b/doc/sphinx-guides/source/installation/external-tools.rst @@ -9,7 +9,7 @@ External tools can provide additional features that are not part of Dataverse it Inventory of External Tools --------------------------- -Support for external tools is just getting off the ground but the following tools have been successfully integrated with Dataverse: +The following tools have been successfully integrated with Dataverse: - TwoRavens: a system of interlocking statistical tools for data exploration, analysis, and meta-analysis: http://2ra.vn. See the :doc:`/user/data-exploration/tworavens` section of the User Guide for more information on TwoRavens from the user perspective and the :doc:`r-rapache-tworavens` section of the Installation Guide. @@ -35,14 +35,17 @@ External tools must be expressed in an external tool manifest file, a specific J ``type`` is required and must be ``explore`` or ``configure`` to make the tool appear under a button called "Explore" or "Configure", respectively. -External tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. (Not providing this parameter makes the tool work on ingested tabular files and is equivalent to specifying the ``contentType`` as "text/tab-separated-values".) +``scope`` is required and must be ``file`` or ``dataset`` to make the tool appear at the file level or dataset level. + +File level tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. (Not providing this parameter makes the tool work on ingested tabular files and is equivalent to specifying the ``contentType`` as "text/tab-separated-values".) In the example above, a mix of required and optional reserved words appear that can be used to insert dynamic values into tools. The supported values are: -- ``{fileId}`` (required) - The Dataverse database ID of a file the external tool has been launched on. -- ``{siteUrl}`` (optional) - The URL of the Dataverse installation that hosts the file with the fileId above. +- ``{fileId}`` (required for file tools) - The Dataverse database ID of a file from which the external tool has been launched. +- ``{siteUrl}`` (optional) - The URL of the Dataverse installation from which the tool was launched. - ``{apiToken}`` (optional) - The Dataverse API token of the user launching the external tool, if available. -- ``{datasetId}`` (optional) - The ID of the dataset containing the file. +- ``{datasetId}`` (optional) - The ID of the dataset. +- ``{datasetPid}`` (optional) - The Persistent ID (DOI or Handle) of the dataset. - ``{datasetVersion}`` (optional) - The friendly version number ( or \:draft ) of the dataset version the tool is being launched from. Making an External Tool Available in Dataverse diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 812f8255a88..6f39934915e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -6,8 +6,10 @@ import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; +import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; @@ -99,6 +101,7 @@ import edu.harvard.iq.dataverse.externaltools.ExternalTool; import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; import edu.harvard.iq.dataverse.export.SchemaDotOrgExporter; +import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import java.util.Collections; @@ -135,6 +138,7 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; +import org.primefaces.PrimeFaces; import org.primefaces.model.DefaultTreeNode; import org.primefaces.model.TreeNode; @@ -316,6 +320,7 @@ public void setShowIngestSuccess(boolean showIngestSuccess) { List exploreTools = new ArrayList<>(); Map> configureToolsByFileId = new HashMap<>(); Map> exploreToolsByFileId = new HashMap<>(); + private List datasetExploreTools; public Boolean isHasRsyncScript() { return hasRsyncScript; @@ -2016,6 +2021,7 @@ private String init(boolean initFull) { configureTools = externalToolService.findByType(ExternalTool.Type.CONFIGURE); exploreTools = externalToolService.findByType(ExternalTool.Type.EXPLORE); + datasetExploreTools = externalToolService.findByScopeAndType(ExternalTool.Scope.DATASET, ExternalTool.Type.EXPLORE); rowsPerPage = 10; @@ -5052,6 +5058,10 @@ public List getCachedToolsForDataFile(Long fileId, ExternalTool.Ty return cachedTools; } + public List getDatasetExploreTools() { + return datasetExploreTools; + } + Boolean thisLatestReleasedVersion = null; public boolean isThisLatestReleasedVersion() { @@ -5200,4 +5210,17 @@ public int compare(FileMetadata o1, FileMetadata o2) { return type1.compareTo(type2); } }; + + public void explore(ExternalTool externalTool) { + ApiToken apiToken = null; + User user = session.getUser(); + if (user instanceof AuthenticatedUser) { + apiToken = authService.findApiTokenByUser((AuthenticatedUser) user); + } + ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataset, apiToken); + String toolUrl = externalToolHandler.getToolUrlWithQueryParams(); + logger.fine("Exploring with " + toolUrl); + PrimeFaces.current().executeScript("window.open('"+toolUrl + "', target='_blank');"); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index 005f61d6d38..07d1c7fbe92 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -23,6 +23,7 @@ public class ExternalTool implements Serializable { public static final String DISPLAY_NAME = "displayName"; public static final String DESCRIPTION = "description"; public static final String TYPE = "type"; + public static final String SCOPE = "scope"; public static final String TOOL_URL = "toolUrl"; public static final String TOOL_PARAMETERS = "toolParameters"; public static final String CONTENT_TYPE = "contentType"; @@ -52,6 +53,13 @@ public class ExternalTool implements Serializable { @Enumerated(EnumType.STRING) private Type type; + /** + * Whether the tool operates at the dataset or file level. + */ + @Column(nullable = false) + @Enumerated(EnumType.STRING) + private Scope scope; + @Column(nullable = false) private String toolUrl; @@ -83,10 +91,11 @@ public class ExternalTool implements Serializable { public ExternalTool() { } - public ExternalTool(String displayName, String description, Type type, String toolUrl, String toolParameters, String contentType) { + public ExternalTool(String displayName, String description, Type type, Scope scope, String toolUrl, String toolParameters, String contentType) { this.displayName = displayName; this.description = description; this.type = type; + this.scope = scope; this.toolUrl = toolUrl; this.toolParameters = toolParameters; this.contentType = contentType; @@ -120,6 +129,34 @@ public String toString() { } } + public enum Scope { + + DATASET("dataset"), + FILE("file"); + + private final String text; + + private Scope(final String text) { + this.text = text; + } + + public static Scope fromString(String text) { + if (text != null) { + for (Scope scope : Scope.values()) { + if (text.equals(scope.text)) { + return scope; + } + } + } + throw new IllegalArgumentException("Scope must be one of these values: " + Arrays.asList(Scope.values()) + "."); + } + + @Override + public String toString() { + return text; + } + } + public Long getId() { return id; } @@ -148,6 +185,10 @@ public Type getType() { return type; } + public Scope getScope() { + return scope; + } + public String getToolUrl() { return toolUrl; } @@ -193,7 +234,10 @@ public enum ReservedWord { FILE_ID("fileId"), SITE_URL("siteUrl"), API_TOKEN("apiToken"), + // datasetId is the database id DATASET_ID("datasetId"), + // datasetPid is the DOI or Handle + DATASET_PID("datasetPid"), DATASET_VERSION("datasetVersion"), FILE_METADATA_ID("fileMetadataId"); diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index c177442ee81..3d2c10700ba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -33,6 +33,8 @@ public class ExternalToolHandler { private ApiToken apiToken; /** + * File level tool + * * @param externalTool The database entity. * @param dataFile Required. * @param apiToken The apiToken can be null because "explore" tools can be @@ -51,6 +53,27 @@ public ExternalToolHandler(ExternalTool externalTool, DataFile dataFile, ApiToke this.fileMetadata = fileMetadata; } + /** + * Dataset level tool + * + * @param externalTool The database entity. + * @param dataset Required. + * @param apiToken The apiToken can be null because "explore" tools can be + * used anonymously. + */ + public ExternalToolHandler(ExternalTool externalTool, Dataset dataset, ApiToken apiToken) { + this.externalTool = externalTool; + if (dataset == null) { + String error = "A Dataset is required."; + logger.warning("Error in ExternalToolHandler constructor: " + error); + throw new IllegalArgumentException(error); + } + this.dataset = dataset; + this.apiToken = apiToken; + this.dataFile = null; + this.fileMetadata = null; + } + public DataFile getDataFile() { return dataFile; } @@ -89,7 +112,7 @@ private String getQueryParam(String key, String value) { ReservedWord reservedWord = ReservedWord.fromString(value); switch (reservedWord) { case FILE_ID: - // getDataFile is never null because of the constructor + // getDataFile is never null for file tools because of the constructor return key + "=" + getDataFile().getId(); case SITE_URL: return key + "=" + SystemConfig.getDataverseSiteUrlStatic(); @@ -103,6 +126,8 @@ private String getQueryParam(String key, String value) { break; case DATASET_ID: return key + "=" + dataset.getId(); + case DATASET_PID: + return key + "=" + dataset.getGlobalId().asString(); case DATASET_VERSION: String version = null; if (getApiToken() != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index 05c79731d40..6264855ceb7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -4,12 +4,14 @@ import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.externaltools.ExternalTool.ReservedWord; import edu.harvard.iq.dataverse.externaltools.ExternalTool.Type; +import edu.harvard.iq.dataverse.externaltools.ExternalTool.Scope; import static edu.harvard.iq.dataverse.externaltools.ExternalTool.DESCRIPTION; import static edu.harvard.iq.dataverse.externaltools.ExternalTool.DISPLAY_NAME; import static edu.harvard.iq.dataverse.externaltools.ExternalTool.TOOL_PARAMETERS; import static edu.harvard.iq.dataverse.externaltools.ExternalTool.TOOL_URL; import static edu.harvard.iq.dataverse.externaltools.ExternalTool.TYPE; +import static edu.harvard.iq.dataverse.externaltools.ExternalTool.SCOPE; import static edu.harvard.iq.dataverse.externaltools.ExternalTool.CONTENT_TYPE; import java.io.StringReader; import java.util.ArrayList; @@ -74,7 +76,21 @@ public List findByType(Type type, String contentType) { return externalTools; } - + /** + * @param scope - dataset or file + * @return A list of tools or an empty list. + */ + public List findByScopeAndType(Scope scope, Type type) { + List externalTools = new ArrayList<>(); + TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.scope = :scope AND o.type = :type", ExternalTool.class); + typedQuery.setParameter("scope", scope); + typedQuery.setParameter("type", type); + List toolsFromQuery = typedQuery.getResultList(); + if (toolsFromQuery != null) { + externalTools = toolsFromQuery; + } + return externalTools; + } public ExternalTool findById(long id) { TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.id = :id", ExternalTool.class); @@ -131,6 +147,7 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { String displayName = getRequiredTopLevelField(jsonObject, DISPLAY_NAME); String description = getRequiredTopLevelField(jsonObject, DESCRIPTION); String typeUserInput = getRequiredTopLevelField(jsonObject, TYPE); + String scopeUserInput = getRequiredTopLevelField(jsonObject, SCOPE); String contentType = getOptionalTopLevelField(jsonObject, CONTENT_TYPE); //Legacy support - assume tool manifests without any mimetype are for tabular data if(contentType==null) { @@ -139,26 +156,29 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { // Allow IllegalArgumentException to bubble up from ExternalTool.Type.fromString ExternalTool.Type type = ExternalTool.Type.fromString(typeUserInput); + ExternalTool.Scope scope = ExternalTool.Scope.fromString(scopeUserInput); String toolUrl = getRequiredTopLevelField(jsonObject, TOOL_URL); JsonObject toolParametersObj = jsonObject.getJsonObject(TOOL_PARAMETERS); JsonArray queryParams = toolParametersObj.getJsonArray("queryParameters"); boolean allRequiredReservedWordsFound = false; - for (JsonObject queryParam : queryParams.getValuesAs(JsonObject.class)) { - Set keyValuePair = queryParam.keySet(); - for (String key : keyValuePair) { - String value = queryParam.getString(key); - ReservedWord reservedWord = ReservedWord.fromString(value); - if (reservedWord.equals(ReservedWord.FILE_ID)) { - allRequiredReservedWordsFound = true; + if (scope.equals(Scope.FILE)) { + for (JsonObject queryParam : queryParams.getValuesAs(JsonObject.class)) { + Set keyValuePair = queryParam.keySet(); + for (String key : keyValuePair) { + String value = queryParam.getString(key); + ReservedWord reservedWord = ReservedWord.fromString(value); + if (reservedWord.equals(ReservedWord.FILE_ID)) { + allRequiredReservedWordsFound = true; + } } } - } - if (!allRequiredReservedWordsFound) { - // Some day there might be more reserved words than just {fileId}. - throw new IllegalArgumentException("Required reserved word not found: " + ReservedWord.FILE_ID.toString()); + if (!allRequiredReservedWordsFound) { + // Some day there might be more reserved words than just {fileId}. + throw new IllegalArgumentException("Required reserved word not found: " + ReservedWord.FILE_ID.toString()); + } } String toolParameters = toolParametersObj.toString(); - return new ExternalTool(displayName, description, type, toolUrl, toolParameters, contentType); + return new ExternalTool(displayName, description, type, scope, toolUrl, toolParameters, contentType); } private static String getRequiredTopLevelField(JsonObject jsonObject, String key) { diff --git a/src/main/resources/db/migration/V4.15.1.1__5028-dataset-explore.sql b/src/main/resources/db/migration/V4.15.1.1__5028-dataset-explore.sql new file mode 100644 index 00000000000..d880b1bddb4 --- /dev/null +++ b/src/main/resources/db/migration/V4.15.1.1__5028-dataset-explore.sql @@ -0,0 +1,3 @@ +ALTER TABLE externaltool ADD COLUMN IF NOT EXISTS scope VARCHAR(255); +UPDATE externaltool SET scope = 'FILE'; +ALTER TABLE externaltool ALTER COLUMN scope SET NOT NULL; diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 458bfa1a805..156d44c979a 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -133,36 +133,31 @@ -
+
- + - - - - + - +
- +
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java index 22a07387c14..f96e417d6b0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java @@ -37,7 +37,8 @@ public void testAddExternalTool() throws IOException { JsonObjectBuilder job = Json.createObjectBuilder(); job.add("displayName", "AwesomeTool"); job.add("description", "This tool is awesome."); - job.add("type", "explorer"); + job.add("type", "explore"); + job.add("scope", "file"); job.add("toolUrl", "http://awesometool.com"); job.add("toolParameters", Json.createObjectBuilder() .add("queryParameters", Json.createArrayBuilder() @@ -89,6 +90,8 @@ public void testAddExternalToolNonReservedWord() throws IOException { JsonObjectBuilder job = Json.createObjectBuilder(); job.add("displayName", "AwesomeTool"); job.add("description", "This tool is awesome."); + job.add("type", "explore"); + job.add("scope", "file"); job.add("toolUrl", "http://awesometool.com"); job.add("toolParameters", Json.createObjectBuilder() .add("queryParameters", Json.createArrayBuilder() diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java index bfa00aac677..b209eeb1364 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java @@ -21,8 +21,9 @@ public class ExternalToolHandlerTest { @Test public void testGetToolUrlWithOptionalQueryParameters() { ExternalTool.Type type = ExternalTool.Type.EXPLORE; + ExternalTool.Scope scope = ExternalTool.Scope.FILE; String toolUrl = "http://example.com"; - ExternalTool externalTool = new ExternalTool("displayName", "description", type, toolUrl, "{}", DataFileServiceBean.MIME_TYPE_TSV_ALT); + ExternalTool externalTool = new ExternalTool("displayName", "description", type, scope, toolUrl, "{}", DataFileServiceBean.MIME_TYPE_TSV_ALT); // One query parameter, not a reserved word, no {fileId} (required) used. externalTool.setToolParameters(Json.createObjectBuilder() diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java index 54462b3ca0b..a38b32268e5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java @@ -39,7 +39,8 @@ public void testfindAll() { ApiToken apiToken = new ApiToken(); apiToken.setTokenString("7196b5ce-f200-4286-8809-03ffdbc255d7"); ExternalTool.Type type = ExternalTool.Type.EXPLORE; - ExternalTool externalTool = new ExternalTool("displayName", "description", type, "http://foo.com", "{}", DataFileServiceBean.MIME_TYPE_TSV_ALT); + ExternalTool.Scope scope = ExternalTool.Scope.FILE; + ExternalTool externalTool = new ExternalTool("displayName", "description", type, scope, "http://foo.com", "{}", DataFileServiceBean.MIME_TYPE_TSV_ALT); ExternalToolHandler externalToolHandler4 = new ExternalToolHandler(externalTool, dataFile, apiToken, fmd); List externalTools = new ArrayList<>(); externalTools.add(externalTool); @@ -53,6 +54,7 @@ public void testParseAddExternalToolInput() { job.add("displayName", "AwesomeTool"); job.add("description", "This tool is awesome."); job.add("type", "explore"); + job.add("scope", "file"); job.add("toolUrl", "http://awesometool.com"); job.add("toolParameters", Json.createObjectBuilder() .add("queryParameters", Json.createArrayBuilder() @@ -97,6 +99,7 @@ public void testParseAddExternalToolInputNoFileId() { job.add("displayName", "AwesomeTool"); job.add("description", "This tool is awesome."); job.add("type", "explore"); + job.add("scope", "file"); job.add("toolUrl", "http://awesometool.com"); job.add("toolParameters", Json.createObjectBuilder() .add("queryParameters", Json.createArrayBuilder() @@ -148,6 +151,7 @@ public void testParseAddExternalToolInputUnknownReservedWord() { job.add("displayName", "AwesomeTool"); job.add("description", "This tool is awesome."); job.add("type", "explore"); + job.add("scope", "file"); job.add("toolUrl", "http://awesometool.com"); job.add("toolParameters", Json.createObjectBuilder() .add("queryParameters", Json.createArrayBuilder() @@ -219,6 +223,7 @@ public void testParseAddExternalToolInputNoToolUrl() { job.add("displayName", "AwesomeTool"); job.add("description", "This tool is awesome."); job.add("type", "explore"); + job.add("scope", "file"); job.add("toolParameters", Json.createObjectBuilder().build()); job.add(ExternalTool.CONTENT_TYPE, DataFileServiceBean.MIME_TYPE_TSV_ALT); String tool = job.build().toString(); @@ -239,6 +244,7 @@ public void testParseAddExternalToolInputWrongType() { job.add("displayName", "AwesomeTool"); job.add("description", "This tool is awesome."); job.add("type", "noSuchType"); + job.add("scope", "file"); job.add("toolUrl", "http://awesometool.com"); job.add("toolParameters", Json.createObjectBuilder().build()); job.add(ExternalTool.CONTENT_TYPE, DataFileServiceBean.MIME_TYPE_TSV_ALT); @@ -261,6 +267,7 @@ public void testParseAddExternalToolInputNoContentType() { job.add("displayName", "AwesomeTool"); job.add("description", "This tool is awesome."); job.add("type", "explore"); + job.add("scope", "file"); job.add("toolUrl", "http://awesometool.com"); job.add("toolParameters", Json.createObjectBuilder().add("queryParameters", Json.createArrayBuilder() diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolTest.java index d0d51646adf..6dc4cfed44e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolTest.java @@ -14,9 +14,10 @@ public void testToJson() { String displayName = "myDisplayName"; String description = "myDescription"; ExternalTool.Type type = ExternalTool.Type.EXPLORE; + ExternalTool.Scope scope = ExternalTool.Scope.FILE; String toolUrl = "http://example.com"; String toolParameters = "{}"; - ExternalTool externalTool = new ExternalTool(displayName, description, type, toolUrl, toolParameters, DataFileServiceBean.MIME_TYPE_TSV_ALT); + ExternalTool externalTool = new ExternalTool(displayName, description, type, scope, toolUrl, toolParameters, DataFileServiceBean.MIME_TYPE_TSV_ALT); externalTool.setId(42l); JsonObject jsonObject = externalTool.toJson().build(); System.out.println("result: " + jsonObject); From 0390e3c0994678b53e9b60fac05b60f0f695217e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 26 Jul 2019 15:52:12 -0400 Subject: [PATCH 03/23] return scope in listing, GET tool by id #5028 --- .../source/installation/external-tools.rst | 9 ++++++ .../iq/dataverse/api/ExternalTools.java | 11 +++++++ .../dataverse/externaltools/ExternalTool.java | 1 + .../iq/dataverse/api/ExternalToolsIT.java | 32 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 5 +++ 5 files changed, 58 insertions(+) diff --git a/doc/sphinx-guides/source/installation/external-tools.rst b/doc/sphinx-guides/source/installation/external-tools.rst index ef3e84693af..39b9bf38df2 100644 --- a/doc/sphinx-guides/source/installation/external-tools.rst +++ b/doc/sphinx-guides/source/installation/external-tools.rst @@ -62,6 +62,15 @@ To list all the external tools that are available in Dataverse: ``curl http://localhost:8080/api/admin/externalTools`` +Showing an External Tool in Dataverse +------------------------------------- + +To show one of the external tools that are available in Dataverse, pass its database id: + +``export TOOL_ID=1`` + +``curl http://localhost:8080/api/admin/externalTools/$TOOL_ID`` + Removing an External Tool Available in Dataverse ------------------------------------------------ diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java b/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java index 79f66070e79..ecb97e3e845 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java @@ -28,6 +28,17 @@ public Response getExternalTools() { return ok(jab); } + @GET + @Path("{id}") + public Response getExternalTool(@PathParam("id") long externalToolIdFromUser) { + ExternalTool externalTool = externalToolService.findById(externalToolIdFromUser); + if (externalTool != null) { + return ok(externalTool.toJson()); + } else { + return error(BAD_REQUEST, "Could not find external tool with id of " + externalToolIdFromUser); + } + } + @POST public Response addExternalTool(String manifest) { try { diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index 07d1c7fbe92..3605f164cf3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -219,6 +219,7 @@ public JsonObjectBuilder toJson() { jab.add(DISPLAY_NAME, getDisplayName()); jab.add(DESCRIPTION, getDescription()); jab.add(TYPE, getType().text); + jab.add(SCOPE, getScope().text); jab.add(TOOL_URL, getToolUrl()); jab.add(TOOL_PARAMETERS, getToolParameters()); jab.add(CONTENT_TYPE, getContentType()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java index f96e417d6b0..c820ac6220f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java @@ -1,12 +1,14 @@ package edu.harvard.iq.dataverse.api; import com.jayway.restassured.RestAssured; +import com.jayway.restassured.path.json.JsonPath; import com.jayway.restassured.response.Response; import java.io.IOException; import javax.json.Json; import javax.json.JsonObjectBuilder; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static javax.ws.rs.core.Response.Status.OK; +import static junit.framework.Assert.assertEquals; import org.hamcrest.CoreMatchers; import org.junit.BeforeClass; import org.junit.Test; @@ -113,4 +115,34 @@ public void testAddExternalToolNonReservedWord() throws IOException { .statusCode(BAD_REQUEST.getStatusCode()); } + @Test + public void testAddDatasetExploreTool() throws IOException { + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("displayName", "AwesomeTool"); + job.add("description", "This tool is awesome."); + job.add("type", "explore"); + job.add("scope", "dataset"); + job.add("toolUrl", "http://awesometool.com"); + job.add("toolParameters", Json.createObjectBuilder() + .add("queryParameters", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("dataset", "{datasetPid}") + .build()) + .build()) + .build()); + Response addExternalTool = UtilIT.addExternalTool(job.build()); + addExternalTool.prettyPrint(); + addExternalTool.then().assertThat() + .body("data.displayName", CoreMatchers.equalTo("AwesomeTool")) + .statusCode(OK.getStatusCode()); + + long id = JsonPath.from(addExternalTool.getBody().asString()).getLong("data.id"); + + Response getTool = UtilIT.getExternalTool(id); + getTool.prettyPrint(); + getTool.then().assertThat() + .body("data.scope", CoreMatchers.equalTo("dataset")) + .statusCode(OK.getStatusCode()); + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index be329c83d6d..20480443869 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1738,6 +1738,11 @@ static Response getExternalTools() { .get("/api/admin/externalTools"); } + static Response getExternalTool(long id) { + return given() + .get("/api/admin/externalTools/" + id); + } + static Response getExternalToolsByFileId(long fileId) { return given() .get("/api/admin/externalTools/file/" + fileId); From 6fe29960de7988ada6f2dac17983ccfd2de6752c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 31 Jul 2019 12:58:00 -0400 Subject: [PATCH 04/23] adding release notes for dataset explore --- doc/release-notes/5028-dataset-explore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes/5028-dataset-explore diff --git a/doc/release-notes/5028-dataset-explore b/doc/release-notes/5028-dataset-explore new file mode 100644 index 00000000000..240bcc406ee --- /dev/null +++ b/doc/release-notes/5028-dataset-explore @@ -0,0 +1,5 @@ +There is a DB migration at: + +https://github.com/IQSS/dataverse/blob/5028-dataset-explore-btn/src/main/resources/db/migration/V4.15.1.1__5028-dataset-explore.sql + +We will need to reflect this in the release notes. Verify at release time. From c0df37dd564214fd53f3e05b722b97d3a453b0b6 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 7 Aug 2019 22:24:42 -0400 Subject: [PATCH 05/23] removing release note because it's handled by flyway --- doc/release-notes/5028-dataset-explore | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 doc/release-notes/5028-dataset-explore diff --git a/doc/release-notes/5028-dataset-explore b/doc/release-notes/5028-dataset-explore deleted file mode 100644 index 240bcc406ee..00000000000 --- a/doc/release-notes/5028-dataset-explore +++ /dev/null @@ -1,5 +0,0 @@ -There is a DB migration at: - -https://github.com/IQSS/dataverse/blob/5028-dataset-explore-btn/src/main/resources/db/migration/V4.15.1.1__5028-dataset-explore.sql - -We will need to reflect this in the release notes. Verify at release time. From 162207ecf7017bdd5c3e1aade45388139bf6ffdc Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 26 Aug 2019 11:02:34 -0400 Subject: [PATCH 06/23] get tests passing #5028 --- src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java index c820ac6220f..7d8ffd568b9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java @@ -75,7 +75,7 @@ public void testAddExternalToolNoFileId() throws IOException { Response addExternalTool = UtilIT.addExternalTool(job.build()); addExternalTool.prettyPrint(); addExternalTool.then().assertThat() - .body("message", CoreMatchers.equalTo("Required reserved word not found: {fileId}")) + .body("message", CoreMatchers.equalTo("type is required.")) .statusCode(BAD_REQUEST.getStatusCode()); } From 319141ad1a0182c57edebddef2139e7b8f30ceec Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 27 Aug 2019 14:29:45 -0400 Subject: [PATCH 07/23] enforce datasetId or datasetPid requirement #5028 --- .../harvard/iq/dataverse/api/Datasets.java | 33 +++ .../iq/dataverse/api/ExternalTools.java | 24 +-- .../edu/harvard/iq/dataverse/api/Files.java | 41 +++- .../dataverse/externaltools/ExternalTool.java | 6 +- .../externaltools/ExternalToolHandler.java | 3 +- .../ExternalToolServiceBean.java | 60 ++++-- .../iq/dataverse/api/ExternalToolsIT.java | 204 ++++++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 35 ++- .../ExternalToolServiceBeanTest.java | 97 ++++++++- 9 files changed, 434 insertions(+), 69 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index bbcd33af323..d09232dc557 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -77,6 +77,7 @@ import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.S3PackageImporter; import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; @@ -84,6 +85,8 @@ import edu.harvard.iq.dataverse.engine.command.exception.UnforcedCommandException; import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataFileCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDvObjectPIDMetadataCommand; +import edu.harvard.iq.dataverse.externaltools.ExternalTool; +import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitations; import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitationsServiceBean; import edu.harvard.iq.dataverse.makedatacount.DatasetMetrics; @@ -1893,5 +1896,35 @@ public Response getMakeDataCountMetric(@PathParam("id") String idSupplied, @Path } } + @GET + @Path("{id}/externalTools") + public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam("type") String type) { + Dataset dataset; + try { + dataset = findDatasetOrDie(idSupplied); + JsonArrayBuilder tools = Json.createArrayBuilder(); + List datasetTools = externalToolService.findByScopeAndType(ExternalTool.Scope.DATASET, ExternalTool.Type.fromString(type)); + for (ExternalTool tool : datasetTools) { + String apiTokenString = null; + apiTokenString = getRequestApiKey(); + ApiToken apiToken = null; + if (apiTokenString != null) { + apiToken = new ApiToken(); + apiToken.setTokenString(apiTokenString); + } + ExternalToolHandler externalToolHandler = null; + String toolUrlWithQueryParams = null; + externalToolHandler = new ExternalToolHandler(tool, dataset, apiToken); + toolUrlWithQueryParams = externalToolHandler.getToolUrlWithQueryParams(); + JsonObjectBuilder toolToJson = tool.toJson(); + toolToJson.add("toolUrlWithQueryParams", toolUrlWithQueryParams); + tools.add(toolToJson); + } + return ok(tools); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java b/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java index ecb97e3e845..aef30bfb0c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java @@ -1,11 +1,10 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; import edu.harvard.iq.dataverse.externaltools.ExternalTool; import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; -import java.util.List; +import java.util.logging.Logger; import javax.json.Json; import javax.json.JsonArrayBuilder; import javax.ws.rs.DELETE; @@ -19,6 +18,8 @@ @Path("admin/externalTools") public class ExternalTools extends AbstractApiBean { + private static final Logger logger = Logger.getLogger(ExternalTools.class.getCanonicalName()); + @GET public Response getExternalTools() { JsonArrayBuilder jab = Json.createArrayBuilder(); @@ -64,23 +65,4 @@ public Response deleteExternalTool(@PathParam("id") long externalToolIdFromUser) } } - // TODO: Rather than only supporting looking up files by their database IDs, consider supporting persistent identifiers. - @GET - @Path("file/{id}") - public Response getExternalToolsByFile(@PathParam("id") Long fileIdFromUser) { - DataFile dataFile = fileSvc.find(fileIdFromUser); - if (dataFile == null) { - return error(BAD_REQUEST, "Could not find datafile with id " + fileIdFromUser); - } - JsonArrayBuilder tools = Json.createArrayBuilder(); - - List allExternalTools = externalToolService.findAll(); - List toolsByFile = ExternalToolServiceBean.findExternalToolsByFile(allExternalTools, dataFile); - for (ExternalTool tool : toolsByFile) { - tools.add(tool.toJson()); - } - - return ok(tools); - } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index f304444a7f3..f1b677ddcb0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -14,6 +14,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.UserNotificationServiceBean; import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper; @@ -27,10 +28,12 @@ import edu.harvard.iq.dataverse.engine.command.impl.GetDraftFileMetadataIfAvailableCommand; import edu.harvard.iq.dataverse.engine.command.impl.RedetectFileTypeCommand; import edu.harvard.iq.dataverse.engine.command.impl.RestrictFileCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.UningestFileCommand; +import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.export.ExportException; import edu.harvard.iq.dataverse.export.ExportService; +import edu.harvard.iq.dataverse.externaltools.ExternalTool; +import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; import edu.harvard.iq.dataverse.ingest.IngestRequest; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; @@ -43,10 +46,14 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.EJB; import javax.inject.Inject; +import javax.json.Json; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObjectBuilder; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; @@ -55,6 +62,7 @@ import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; @@ -64,8 +72,6 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; -import java.util.List; -import javax.ws.rs.QueryParam; @Path("files") public class Files extends AbstractApiBean { @@ -615,4 +621,33 @@ private void exportDatasetMetadata(SettingsServiceBean settingsServiceBean, Data } } + @Path("{id}/externalTools") + @GET + public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam("type") String type) { + try { + DataFile dataFile = findDataFileOrDie(idSupplied); + JsonArrayBuilder tools = Json.createArrayBuilder(); + List datasetTools = externalToolService.findByScopeAndType(ExternalTool.Scope.FILE, ExternalTool.Type.fromString(type)); + for (ExternalTool tool : datasetTools) { + String apiTokenString = null; + apiTokenString = getRequestApiKey(); + ApiToken apiToken = null; + if (apiTokenString != null) { + apiToken = new ApiToken(); + apiToken.setTokenString(apiTokenString); + } + ExternalToolHandler externalToolHandler = null; + String toolUrlWithQueryParams = null; + externalToolHandler = new ExternalToolHandler(tool, dataFile, apiToken, dataFile.getFileMetadata()); + toolUrlWithQueryParams = externalToolHandler.getToolUrlWithQueryParams(); + JsonObjectBuilder toolToJson = tool.toJson(); + toolToJson.add("toolUrlWithQueryParams", toolUrlWithQueryParams); + tools.add(toolToJson); + } + return ok(tools); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index 3605f164cf3..a06b36a03b2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -71,12 +71,12 @@ public class ExternalTool implements Serializable { private String toolParameters; /** - * The file content type the tool works on. For tabular files, the type text/tab-separated-values should be sent + * The file content type the tool works on. For tabular files, the type + * text/tab-separated-values should be sent */ @Column(nullable = false, columnDefinition = "TEXT") private String contentType; - /** * This default constructor is only here to prevent this error at * deployment: @@ -208,7 +208,7 @@ public void setToolParameters(String toolParameters) { public String getContentType() { return this.contentType; } - + public void setContentType(String contentType) { this.contentType = contentType; } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 3d2c10700ba..a08bfe4c39e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -136,8 +136,7 @@ private String getQueryParam(String key, String value) { version = dataset.getLatestVersionForCopy().getFriendlyVersionNumber(); } if (("DRAFT").equals(version)) { - version = ":draft"; // send the token needed in api calls that can be substituted for a numeric - // version. + version = ":draft"; // send the token needed in api calls that can be substituted for a numeric version. } return key + "=" + version; case FILE_METADATA_ID: diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index 6264855ceb7..1aa0db8df5b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -44,29 +44,28 @@ public List findAll() { return typedQuery.getResultList(); } - /** - * @param type + * @param type * @return A list of tools or an empty list. */ public List findByType(Type type) { return findByType(type, null); } - + /** - * @param type - * @param contentType - mimetype + * @param type + * @param contentType - mimetype * @return A list of tools or an empty list. */ public List findByType(Type type, String contentType) { List externalTools = new ArrayList<>(); - + //If contentType==null, get all tools of the given ExternalTool.Type - TypedQuery typedQuery = contentType != null ? em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.type = :type AND o.contentType = :contentType", ExternalTool.class): - em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.type = :type", ExternalTool.class); + TypedQuery typedQuery = contentType != null ? em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.type = :type AND o.contentType = :contentType", ExternalTool.class) + : em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.type = :type", ExternalTool.class); typedQuery.setParameter("type", type); - if(contentType!=null) { + if (contentType != null) { typedQuery.setParameter("contentType", contentType); } List toolsFromQuery = typedQuery.getResultList(); @@ -120,8 +119,9 @@ public ExternalTool save(ExternalTool externalTool) { } /** - * This method takes a list of tools and a file and returns which tools that file supports - * The list of tools is passed in so it doesn't hit the database each time + * This method takes a list of tools and a file and returns which tools that + * file supports The list of tools is passed in so it doesn't hit the + * database each time */ public static List findExternalToolsByFile(List allExternalTools, DataFile file) { List externalTools = new ArrayList<>(); @@ -133,7 +133,7 @@ public static List findExternalToolsByFile(List allE externalTools.add(externalTool); } }); - + return externalTools; } @@ -150,10 +150,10 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { String scopeUserInput = getRequiredTopLevelField(jsonObject, SCOPE); String contentType = getOptionalTopLevelField(jsonObject, CONTENT_TYPE); //Legacy support - assume tool manifests without any mimetype are for tabular data - if(contentType==null) { - contentType=DataFileServiceBean.MIME_TYPE_TSV_ALT; + if (contentType == null) { + contentType = DataFileServiceBean.MIME_TYPE_TSV_ALT; } - + // Allow IllegalArgumentException to bubble up from ExternalTool.Type.fromString ExternalTool.Type type = ExternalTool.Type.fromString(typeUserInput); ExternalTool.Scope scope = ExternalTool.Scope.fromString(scopeUserInput); @@ -176,6 +176,31 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { // Some day there might be more reserved words than just {fileId}. throw new IllegalArgumentException("Required reserved word not found: " + ReservedWord.FILE_ID.toString()); } + } else if (scope.equals(Scope.DATASET)) { + List requiredReservedWordCandidates = new ArrayList<>(); + requiredReservedWordCandidates.add(ReservedWord.DATASET_ID); + requiredReservedWordCandidates.add(ReservedWord.DATASET_PID); + for (JsonObject queryParam : queryParams.getValuesAs(JsonObject.class)) { + Set keyValuePair = queryParam.keySet(); + for (String key : keyValuePair) { + String value = queryParam.getString(key); + ReservedWord reservedWord = ReservedWord.fromString(value); + for (ReservedWord requiredReservedWordCandidate : requiredReservedWordCandidates) { + if (reservedWord.equals(requiredReservedWordCandidate)) { + allRequiredReservedWordsFound = true; + } + } + } + } + if (!allRequiredReservedWordsFound) { + List requiredReservedWordCandidatesString = new ArrayList<>(); + for (ReservedWord requiredReservedWordCandidate : requiredReservedWordCandidates) { + requiredReservedWordCandidatesString.add(requiredReservedWordCandidate.toString()); + } + String friendly = String.join(", ", requiredReservedWordCandidatesString); + throw new IllegalArgumentException("One of the following reserved words is required: " + friendly + "."); + } + } String toolParameters = toolParametersObj.toString(); return new ExternalTool(displayName, description, type, scope, toolUrl, toolParameters, contentType); @@ -188,7 +213,7 @@ private static String getRequiredTopLevelField(JsonObject jsonObject, String key throw new IllegalArgumentException(key + " is required."); } } - + private static String getOptionalTopLevelField(JsonObject jsonObject, String key) { try { return jsonObject.getString(key); @@ -197,7 +222,4 @@ private static String getOptionalTopLevelField(JsonObject jsonObject, String key } } - - - } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java index 7d8ffd568b9..e48565df3b1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java @@ -4,11 +4,15 @@ import com.jayway.restassured.path.json.JsonPath; import com.jayway.restassured.response.Response; import java.io.IOException; +import java.io.StringReader; import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import javax.json.JsonReader; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.OK; -import static junit.framework.Assert.assertEquals; import org.hamcrest.CoreMatchers; import org.junit.BeforeClass; import org.junit.Test; @@ -27,11 +31,171 @@ public void testGetExternalTools() { } @Test - public void getExternalToolsByFileId() { - // FIXME: Don't hard code the file id. Make a dataset. - long fileId = 12; - Response getExternalTools = UtilIT.getExternalToolsByFileId(fileId); - getExternalTools.prettyPrint(); + public void testFileLevelTool1() { + + // Delete all external tools before testing. + Response getTools = UtilIT.getExternalTools(); + getTools.prettyPrint(); + getTools.then().assertThat() + .statusCode(OK.getStatusCode()); + String body = getTools.getBody().asString(); + JsonReader bodyObject = Json.createReader(new StringReader(body)); + JsonArray tools = bodyObject.readObject().getJsonArray("data"); + for (int i = 0; i < tools.size(); i++) { + JsonObject tool = tools.getJsonObject(i); + int id = tool.getInt("id"); + Response deleteExternalTool = UtilIT.deleteExternalTool(id); + deleteExternalTool.prettyPrint(); + } + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + createUser.then().assertThat() + .statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + createDataverseResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + + String pathToFile = "src/test/java/edu/harvard/iq/dataverse/util/irclog.tsv"; + UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + + Response getFileIdRequest = UtilIT.nativeGet(datasetId, apiToken); + getFileIdRequest.prettyPrint(); + getFileIdRequest.then().assertThat() + .statusCode(OK.getStatusCode());; + + Integer fileId = JsonPath.from(getFileIdRequest.getBody().asString()).getInt("data.latestVersion.files[0].dataFile.id"); + + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("displayName", "AwesomeTool"); + job.add("description", "This tool is awesome."); + job.add("type", "explore"); + job.add("scope", "file"); + job.add("toolUrl", "http://awesometool.com"); + job.add("toolParameters", Json.createObjectBuilder() + .add("queryParameters", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("fileid", "{fileId}") + .build()) + .add(Json.createObjectBuilder() + .add("key", "{apiToken}") + .build()) + .build()) + .build()); + Response addExternalTool = UtilIT.addExternalTool(job.build()); + addExternalTool.prettyPrint(); + addExternalTool.then().assertThat() + .body("data.displayName", CoreMatchers.equalTo("AwesomeTool")) + .statusCode(OK.getStatusCode()); + +// Response getExternalToolsByFileId = UtilIT.getExternalToolsByFileId(fileId); + Response getExternalToolsByFileId = UtilIT.getExternalToolsForFile(fileId.toString(), "explore", apiToken); + getExternalToolsByFileId.prettyPrint(); + getExternalToolsByFileId.then().assertThat() + .body("data[0].displayName", CoreMatchers.equalTo("AwesomeTool")) + .body("data[0].scope", CoreMatchers.equalTo("file")) + .body("data[0].toolUrlWithQueryParams", CoreMatchers.equalTo("http://awesometool.com?fileid=" + fileId + "&key=" + apiToken)) + .statusCode(OK.getStatusCode()); + + } + + @Test + public void testDatasetLevelTool1() { + + // Delete all external tools before testing. + Response getTools = UtilIT.getExternalTools(); + getTools.prettyPrint(); + getTools.then().assertThat() + .statusCode(OK.getStatusCode()); + String body = getTools.getBody().asString(); + JsonReader bodyObject = Json.createReader(new StringReader(body)); + JsonArray tools = bodyObject.readObject().getJsonArray("data"); + for (int i = 0; i < tools.size(); i++) { + JsonObject tool = tools.getJsonObject(i); + int id = tool.getInt("id"); + Response deleteExternalTool = UtilIT.deleteExternalTool(id); + deleteExternalTool.prettyPrint(); + } + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + createUser.then().assertThat() + .statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + createDataverseResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + +// Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + Integer datasetId = JsonPath.from(createDataset.getBody().asString()).getInt("data.id"); + String datasetPid = JsonPath.from(createDataset.getBody().asString()).getString("data.persistentId"); + + String pathToFile = "src/test/java/edu/harvard/iq/dataverse/util/irclog.tsv"; + UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + + Response getFileIdRequest = UtilIT.nativeGet(datasetId, apiToken); + getFileIdRequest.prettyPrint(); + getFileIdRequest.then().assertThat() + .statusCode(OK.getStatusCode());; + + int fileId = JsonPath.from(getFileIdRequest.getBody().asString()).getInt("data.latestVersion.files[0].dataFile.id"); + + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("displayName", "DatasetTool1"); + job.add("description", "This tool is awesome."); + job.add("type", "explore"); + job.add("scope", "dataset"); + job.add("toolUrl", "http://datasettool1.com"); + job.add("toolParameters", Json.createObjectBuilder() + .add("queryParameters", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetPid", "{datasetPid}") + .build()) + .add(Json.createObjectBuilder() + .add("key", "{apiToken}") + .build()) + .build()) + .build()); + Response addExternalTool = UtilIT.addExternalTool(job.build()); + addExternalTool.prettyPrint(); + addExternalTool.then().assertThat() + .body("data.displayName", CoreMatchers.equalTo("DatasetTool1")) + .statusCode(OK.getStatusCode()); + +// Response getExternalToolsByDatasetId = UtilIT.getExternalToolsByScopeTypeAndDvObjectId("dataset", "explore", datasetId); + Response getExternalToolsByDatasetId = UtilIT.getExternalToolsForDataset(datasetId.toString(), "explore", apiToken); + getExternalToolsByDatasetId.prettyPrint(); + getExternalToolsByDatasetId.then().assertThat() + .body("data[0].displayName", CoreMatchers.equalTo("DatasetTool1")) + .body("data[0].scope", CoreMatchers.equalTo("dataset")) + // FIXME: Instead of "text/tab-separated-values" we want null. We want dataset tools to have a nullable file type. + .body("data[0].contentType", CoreMatchers.equalTo("text/tab-separated-values")) + .body("data[0].toolUrlWithQueryParams", CoreMatchers.equalTo("http://datasettool1.com?datasetPid=" + datasetPid + "&key=" + apiToken)) + .statusCode(OK.getStatusCode()); + } @Test @@ -60,10 +224,12 @@ public void testAddExternalTool() throws IOException { } @Test - public void testAddExternalToolNoFileId() throws IOException { + public void testAddFilelToolNoFileId() throws IOException { JsonObjectBuilder job = Json.createObjectBuilder(); job.add("displayName", "AwesomeTool"); job.add("description", "This tool is awesome."); + job.add("type", "explore"); + job.add("scope", "file"); job.add("toolUrl", "http://awesometool.com"); job.add("toolParameters", Json.createObjectBuilder() .add("queryParameters", Json.createArrayBuilder() @@ -75,16 +241,30 @@ public void testAddExternalToolNoFileId() throws IOException { Response addExternalTool = UtilIT.addExternalTool(job.build()); addExternalTool.prettyPrint(); addExternalTool.then().assertThat() - .body("message", CoreMatchers.equalTo("type is required.")) + .body("message", CoreMatchers.equalTo("Required reserved word not found: {fileId}")) .statusCode(BAD_REQUEST.getStatusCode()); } @Test - public void testDeleteExternalTool() { - // FIXME: Don't hard code a tool id here. - long externalToolId = 14l; - Response addExternalTool = UtilIT.deleteExternalTool(externalToolId); + public void testAddDatasetToolNoDatasetId() throws IOException { + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("displayName", "AwesomeTool"); + job.add("description", "This tool is awesome."); + job.add("type", "explore"); + job.add("scope", "dataset"); + job.add("toolUrl", "http://awesometool.com"); + job.add("toolParameters", Json.createObjectBuilder() + .add("queryParameters", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("key", "{apiToken}") + .build()) + .build()) + .build()); + Response addExternalTool = UtilIT.addExternalTool(job.build()); addExternalTool.prettyPrint(); + addExternalTool.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", CoreMatchers.equalTo("One of the following reserved words is required: {datasetId}, {datasetPid}.")); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 20480443869..f546e9d227f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1743,11 +1743,6 @@ static Response getExternalTool(long id) { .get("/api/admin/externalTools/" + id); } - static Response getExternalToolsByFileId(long fileId) { - return given() - .get("/api/admin/externalTools/file/" + fileId); - } - static Response addExternalTool(JsonObject jsonObject) { RequestSpecification requestSpecification = given(); requestSpecification = given() @@ -1761,6 +1756,36 @@ static Response deleteExternalTool(long externalToolid) { .delete("/api/admin/externalTools/" + externalToolid); } + static Response getExternalToolsForDataset(String idOrPersistentIdOfDataset, String type, String apiToken) { + String idInPath = idOrPersistentIdOfDataset; // Assume it's a number. + String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. + if (!NumberUtils.isNumber(idOrPersistentIdOfDataset)) { + idInPath = ":persistentId"; + optionalQueryParam = "&persistentId=" + idOrPersistentIdOfDataset; + } + RequestSpecification requestSpecification = given(); + if (apiToken != null) { + requestSpecification = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); + } + return requestSpecification.get("/api/datasets/" + idInPath + "/externalTools?type=" + type + optionalQueryParam); + } + + static Response getExternalToolsForFile(String idOrPersistentIdOfFile, String type, String apiToken) { + String idInPath = idOrPersistentIdOfFile; // Assume it's a number. + String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. + if (!NumberUtils.isNumber(idOrPersistentIdOfFile)) { + idInPath = ":persistentId"; + optionalQueryParam = "&persistentId=" + idOrPersistentIdOfFile; + } + RequestSpecification requestSpecification = given(); + if (apiToken != null) { + requestSpecification = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); + } + return requestSpecification.get("/api/files/" + idInPath + "/externalTools?type=" + type + optionalQueryParam); + } + static Response submitFeedback(JsonObjectBuilder job) { return given() .body(job.build().toString()) diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java index a38b32268e5..86e65deae3a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java @@ -19,7 +19,7 @@ public class ExternalToolServiceBeanTest { public ExternalToolServiceBeanTest() { } - + @Test public void testfindAll() { DataFile dataFile = new DataFile(); @@ -87,7 +87,7 @@ public void testParseAddExternalToolInput() { dataFile.setFileMetadatas(fmdl); ApiToken apiToken = new ApiToken(); apiToken.setTokenString("7196b5ce-f200-4286-8809-03ffdbc255d7"); - ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataFile, apiToken,fmd); + ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataFile, apiToken, fmd); String toolUrl = externalToolHandler.getToolUrlWithQueryParams(); System.out.println("result: " + toolUrl); assertEquals("http://awesometool.com?fileid=42&key=7196b5ce-f200-4286-8809-03ffdbc255d7&fileMetadataId=2", toolUrl); @@ -269,7 +269,7 @@ public void testParseAddExternalToolInputNoContentType() { job.add("type", "explore"); job.add("scope", "file"); job.add("toolUrl", "http://awesometool.com"); - + job.add("toolParameters", Json.createObjectBuilder().add("queryParameters", Json.createArrayBuilder() .add(Json.createObjectBuilder() .add("fileid", "{fileId}") @@ -278,9 +278,98 @@ public void testParseAddExternalToolInputNoContentType() { .add("key", "{apiToken}") .build()) .build()) - .build()); + .build()); + String tool = job.build().toString(); + System.out.println("tool: " + tool); + ExternalTool externalTool = null; + try { + externalTool = ExternalToolServiceBean.parseAddExternalToolManifest(tool); + } catch (Exception ex) { + System.out.println(ex.getMessage()); + } + assertNotNull(externalTool); + assertEquals(externalTool.getContentType(), DataFileServiceBean.MIME_TYPE_TSV_ALT); + } + + @Test + public void testParseAddDatasetToolNoRequiredFields() { + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("displayName", "AwesomeTool"); + job.add("description", "This tool is awesome."); + job.add("type", "explore"); + job.add("scope", "dataset"); + job.add("toolUrl", "http://awesometool.com"); + + job.add("toolParameters", Json.createObjectBuilder().add("queryParameters", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("key", "{apiToken}") + .build()) + .build()) + .build()); String tool = job.build().toString(); System.out.println("tool: " + tool); + Exception expectedException = null; + try { + ExternalTool externalTool = ExternalToolServiceBean.parseAddExternalToolManifest(tool); + } catch (Exception ex) { + expectedException = ex; + } + assertNotNull(expectedException); + assertEquals("One of the following reserved words is required: {datasetId}, {datasetPid}.", expectedException.getMessage()); + } + + @Test + public void testParseAddDatasetToolDatasetId() { + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("displayName", "AwesomeTool"); + job.add("description", "This tool is awesome."); + job.add("type", "explore"); + job.add("scope", "dataset"); + job.add("toolUrl", "http://awesometool.com"); + + job.add("toolParameters", Json.createObjectBuilder().add("queryParameters", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetId", "{datasetId}") + .build()) + .add(Json.createObjectBuilder() + .add("key", "{apiToken}") + .build()) + .build()) + .build()); + String tool = job.build().toString(); + System.out.println("tool: " + tool); + + ExternalTool externalTool = null; + try { + externalTool = ExternalToolServiceBean.parseAddExternalToolManifest(tool); + } catch (Exception ex) { + System.out.println(ex.getMessage()); + } + assertNotNull(externalTool); + assertEquals(externalTool.getContentType(), DataFileServiceBean.MIME_TYPE_TSV_ALT); + } + + @Test + public void testParseAddDatasetToolDatasetPid() { + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("displayName", "AwesomeTool"); + job.add("description", "This tool is awesome."); + job.add("type", "explore"); + job.add("scope", "dataset"); + job.add("toolUrl", "http://awesometool.com"); + + job.add("toolParameters", Json.createObjectBuilder().add("queryParameters", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetPid", "{datasetPid}") + .build()) + .add(Json.createObjectBuilder() + .add("key", "{apiToken}") + .build()) + .build()) + .build()); + String tool = job.build().toString(); + System.out.println("tool: " + tool); + ExternalTool externalTool = null; try { externalTool = ExternalToolServiceBean.parseAddExternalToolManifest(tool); From 3d522aac894030dcd21ecaacc674f852f88d2fd6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 27 Aug 2019 17:20:50 -0400 Subject: [PATCH 08/23] allow content type to be null #5028 --- .../edu/harvard/iq/dataverse/DatasetPage.java | 10 +- .../edu/harvard/iq/dataverse/FilePage.java | 4 +- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../edu/harvard/iq/dataverse/api/Files.java | 2 +- .../dataverse/externaltools/ExternalTool.java | 6 +- .../ExternalToolServiceBean.java | 55 ++++----- .../iq/dataverse/api/ExternalToolsIT.java | 108 ++++++------------ .../ExternalToolServiceBeanTest.java | 7 +- 8 files changed, 83 insertions(+), 111 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 46a098a6206..8c3cceff7dd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -316,9 +316,13 @@ public void setShowIngestSuccess(boolean showIngestSuccess) { this.showIngestSuccess = showIngestSuccess; } + // TODO: Consider renaming "configureTools" to "fileConfigureTools". List configureTools = new ArrayList<>(); + // TODO: Consider renaming "exploreTools" to "fileExploreTools". List exploreTools = new ArrayList<>(); + // TODO: Consider renaming "configureToolsByFileId" to "fileConfigureToolsByFileId". Map> configureToolsByFileId = new HashMap<>(); + // TODO: Consider renaming "exploreToolsByFileId" to "fileExploreToolsByFileId". Map> exploreToolsByFileId = new HashMap<>(); private List datasetExploreTools; @@ -2030,9 +2034,9 @@ private String init(boolean initFull) { JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.unlocked.ingest.message")); } - configureTools = externalToolService.findByType(ExternalTool.Type.CONFIGURE); - exploreTools = externalToolService.findByType(ExternalTool.Type.EXPLORE); - datasetExploreTools = externalToolService.findByScopeAndType(ExternalTool.Scope.DATASET, ExternalTool.Type.EXPLORE); + configureTools = externalToolService.findFileToolsByType(ExternalTool.Type.CONFIGURE); + exploreTools = externalToolService.findFileToolsByType(ExternalTool.Type.EXPLORE); + datasetExploreTools = externalToolService.findDatasetToolsByType(ExternalTool.Type.EXPLORE); rowsPerPage = 10; diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 270af97e163..754631551bc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -206,8 +206,8 @@ public String init() { if (file.isTabularData()) { contentType=DataFileServiceBean.MIME_TYPE_TSV_ALT; } - configureTools = externalToolService.findByType(ExternalTool.Type.CONFIGURE, contentType); - exploreTools = externalToolService.findByType(ExternalTool.Type.EXPLORE, contentType); + configureTools = externalToolService.findFileToolsByScopeAndContentType(ExternalTool.Type.CONFIGURE, contentType); + exploreTools = externalToolService.findFileToolsByScopeAndContentType(ExternalTool.Type.EXPLORE, contentType); } else { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index d09232dc557..8f59639f198 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1903,7 +1903,7 @@ public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam try { dataset = findDatasetOrDie(idSupplied); JsonArrayBuilder tools = Json.createArrayBuilder(); - List datasetTools = externalToolService.findByScopeAndType(ExternalTool.Scope.DATASET, ExternalTool.Type.fromString(type)); + List datasetTools = externalToolService.findDatasetToolsByType(ExternalTool.Type.fromString(type)); for (ExternalTool tool : datasetTools) { String apiTokenString = null; apiTokenString = getRequestApiKey(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index f1b677ddcb0..59f38deb103 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -627,7 +627,7 @@ public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam try { DataFile dataFile = findDataFileOrDie(idSupplied); JsonArrayBuilder tools = Json.createArrayBuilder(); - List datasetTools = externalToolService.findByScopeAndType(ExternalTool.Scope.FILE, ExternalTool.Type.fromString(type)); + List datasetTools = externalToolService.findFileToolsByScopeAndContentType(ExternalTool.Type.fromString(type), dataFile.getContentType()); for (ExternalTool tool : datasetTools) { String apiTokenString = null; apiTokenString = getRequestApiKey(); diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index a06b36a03b2..0150ef685af 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -74,7 +74,7 @@ public class ExternalTool implements Serializable { * The file content type the tool works on. For tabular files, the type * text/tab-separated-values should be sent */ - @Column(nullable = false, columnDefinition = "TEXT") + @Column(nullable = true, columnDefinition = "TEXT") private String contentType; /** @@ -222,7 +222,9 @@ public JsonObjectBuilder toJson() { jab.add(SCOPE, getScope().text); jab.add(TOOL_URL, getToolUrl()); jab.add(TOOL_PARAMETERS, getToolParameters()); - jab.add(CONTENT_TYPE, getContentType()); + if (getContentType() != null) { + jab.add(CONTENT_TYPE, getContentType()); + } return jab; } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index 1aa0db8df5b..c5a42dd3e83 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -45,45 +45,50 @@ public List findAll() { } /** - * @param type + * @param type explore or configure * @return A list of tools or an empty list. */ - public List findByType(Type type) { - return findByType(type, null); + public List findDatasetToolsByType(Type type) { + String nullContentType = null; + return findByScopeTypeAndContentType(ExternalTool.Scope.DATASET, type, nullContentType); } /** - * @param type - * @param contentType - mimetype + * @param type explore or configure * @return A list of tools or an empty list. */ - public List findByType(Type type, String contentType) { - - List externalTools = new ArrayList<>(); + public List findFileToolsByType(Type type) { + String nullContentType = null; + return findByScopeTypeAndContentType(ExternalTool.Scope.FILE, type, nullContentType); + } - //If contentType==null, get all tools of the given ExternalTool.Type - TypedQuery typedQuery = contentType != null ? em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.type = :type AND o.contentType = :contentType", ExternalTool.class) - : em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.type = :type", ExternalTool.class); - typedQuery.setParameter("type", type); - if (contentType != null) { - typedQuery.setParameter("contentType", contentType); - } - List toolsFromQuery = typedQuery.getResultList(); - if (toolsFromQuery != null) { - externalTools = toolsFromQuery; - } - return externalTools; + /** + * @param type explore or configure + * @param contentType file content type (MIME type) + * @return A list of tools or an empty list. + */ + public List findFileToolsByScopeAndContentType(Type type, String contentType) { + return findByScopeTypeAndContentType(ExternalTool.Scope.FILE, type, contentType); } /** - * @param scope - dataset or file + * @param scope dataset or file + * @param type explore or configure + * @param contentType file content type (MIME type) * @return A list of tools or an empty list. */ - public List findByScopeAndType(Scope scope, Type type) { + private List findByScopeTypeAndContentType(Scope scope, Type type, String contentType) { List externalTools = new ArrayList<>(); - TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.scope = :scope AND o.type = :type", ExternalTool.class); + String contentTypeClause = ""; + if (contentType != null) { + contentTypeClause = "AND o.contentType = :contentType"; + } + TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o WHERE o.scope = :scope AND o.type = :type " + contentTypeClause, ExternalTool.class); typedQuery.setParameter("scope", scope); typedQuery.setParameter("type", type); + if (contentType != null) { + typedQuery.setParameter("contentType", contentType); + } List toolsFromQuery = typedQuery.getResultList(); if (toolsFromQuery != null) { externalTools = toolsFromQuery; @@ -149,10 +154,6 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { String typeUserInput = getRequiredTopLevelField(jsonObject, TYPE); String scopeUserInput = getRequiredTopLevelField(jsonObject, SCOPE); String contentType = getOptionalTopLevelField(jsonObject, CONTENT_TYPE); - //Legacy support - assume tool manifests without any mimetype are for tabular data - if (contentType == null) { - contentType = DataFileServiceBean.MIME_TYPE_TSV_ALT; - } // Allow IllegalArgumentException to bubble up from ExternalTool.Type.fromString ExternalTool.Type type = ExternalTool.Type.fromString(typeUserInput); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java index e48565df3b1..c4bfcc1f488 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java @@ -14,6 +14,7 @@ import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.OK; import org.hamcrest.CoreMatchers; +import org.hamcrest.Matchers; import org.junit.BeforeClass; import org.junit.Test; @@ -69,21 +70,28 @@ public void testFileLevelTool1() { Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); - String pathToFile = "src/test/java/edu/harvard/iq/dataverse/util/irclog.tsv"; - UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + String pathToJupyterNotebok = "src/test/java/edu/harvard/iq/dataverse/util/irc-metrics.ipynb"; + Response uploadJupyterNotebook = UtilIT.uploadFileViaNative(datasetId.toString(), pathToJupyterNotebok, apiToken); + uploadJupyterNotebook.prettyPrint(); + uploadJupyterNotebook.then().assertThat() + .statusCode(OK.getStatusCode()); - Response getFileIdRequest = UtilIT.nativeGet(datasetId, apiToken); - getFileIdRequest.prettyPrint(); - getFileIdRequest.then().assertThat() - .statusCode(OK.getStatusCode());; + Integer jupyterNotebookFileId = JsonPath.from(uploadJupyterNotebook.getBody().asString()).getInt("data.files[0].dataFile.id"); + + String pathToTabularFile = "src/test/java/edu/harvard/iq/dataverse/util/irclog.tsv"; + Response uploadTabularFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToTabularFile, apiToken); + uploadTabularFile.prettyPrint(); + uploadTabularFile.then().assertThat() + .statusCode(OK.getStatusCode()); - Integer fileId = JsonPath.from(getFileIdRequest.getBody().asString()).getInt("data.latestVersion.files[0].dataFile.id"); + Integer tabularFileId = JsonPath.from(uploadTabularFile.getBody().asString()).getInt("data.files[0].dataFile.id"); JsonObjectBuilder job = Json.createObjectBuilder(); job.add("displayName", "AwesomeTool"); job.add("description", "This tool is awesome."); job.add("type", "explore"); job.add("scope", "file"); + job.add("contentType", "text/tab-separated-values"); job.add("toolUrl", "http://awesometool.com"); job.add("toolParameters", Json.createObjectBuilder() .add("queryParameters", Json.createArrayBuilder() @@ -101,15 +109,29 @@ public void testFileLevelTool1() { .body("data.displayName", CoreMatchers.equalTo("AwesomeTool")) .statusCode(OK.getStatusCode()); -// Response getExternalToolsByFileId = UtilIT.getExternalToolsByFileId(fileId); - Response getExternalToolsByFileId = UtilIT.getExternalToolsForFile(fileId.toString(), "explore", apiToken); - getExternalToolsByFileId.prettyPrint(); - getExternalToolsByFileId.then().assertThat() - .body("data[0].displayName", CoreMatchers.equalTo("AwesomeTool")) - .body("data[0].scope", CoreMatchers.equalTo("file")) - .body("data[0].toolUrlWithQueryParams", CoreMatchers.equalTo("http://awesometool.com?fileid=" + fileId + "&key=" + apiToken)) + long toolId = JsonPath.from(addExternalTool.getBody().asString()).getLong("data.id"); + + Response getTool = UtilIT.getExternalTool(toolId); + getTool.prettyPrint(); + getTool.then().assertThat() + .body("data.scope", CoreMatchers.equalTo("file")) .statusCode(OK.getStatusCode()); + Response getExternalToolsForTabularFiles = UtilIT.getExternalToolsForFile(tabularFileId.toString(), "explore", apiToken); + getExternalToolsForTabularFiles.prettyPrint(); + getExternalToolsForTabularFiles.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].displayName", CoreMatchers.equalTo("AwesomeTool")) + .body("data[0].scope", CoreMatchers.equalTo("file")) + .body("data[0].contentType", CoreMatchers.equalTo("text/tab-separated-values")) + .body("data[0].toolUrlWithQueryParams", CoreMatchers.equalTo("http://awesometool.com?fileid=" + tabularFileId + "&key=" + apiToken)); + + Response getExternalToolsForJuptyerNotebooks = UtilIT.getExternalToolsForFile(jupyterNotebookFileId.toString(), "explore", apiToken); + getExternalToolsForJuptyerNotebooks.prettyPrint(); + getExternalToolsForJuptyerNotebooks.then().assertThat() + .statusCode(OK.getStatusCode()) + // No tools for this file type. + .body("data", Matchers.hasSize(0)); } @Test @@ -185,44 +207,16 @@ public void testDatasetLevelTool1() { .body("data.displayName", CoreMatchers.equalTo("DatasetTool1")) .statusCode(OK.getStatusCode()); -// Response getExternalToolsByDatasetId = UtilIT.getExternalToolsByScopeTypeAndDvObjectId("dataset", "explore", datasetId); Response getExternalToolsByDatasetId = UtilIT.getExternalToolsForDataset(datasetId.toString(), "explore", apiToken); getExternalToolsByDatasetId.prettyPrint(); getExternalToolsByDatasetId.then().assertThat() .body("data[0].displayName", CoreMatchers.equalTo("DatasetTool1")) .body("data[0].scope", CoreMatchers.equalTo("dataset")) - // FIXME: Instead of "text/tab-separated-values" we want null. We want dataset tools to have a nullable file type. - .body("data[0].contentType", CoreMatchers.equalTo("text/tab-separated-values")) .body("data[0].toolUrlWithQueryParams", CoreMatchers.equalTo("http://datasettool1.com?datasetPid=" + datasetPid + "&key=" + apiToken)) .statusCode(OK.getStatusCode()); } - @Test - public void testAddExternalTool() throws IOException { - JsonObjectBuilder job = Json.createObjectBuilder(); - job.add("displayName", "AwesomeTool"); - job.add("description", "This tool is awesome."); - job.add("type", "explore"); - job.add("scope", "file"); - job.add("toolUrl", "http://awesometool.com"); - job.add("toolParameters", Json.createObjectBuilder() - .add("queryParameters", Json.createArrayBuilder() - .add(Json.createObjectBuilder() - .add("fileid", "{fileId}") - .build()) - .add(Json.createObjectBuilder() - .add("key", "{apiToken}") - .build()) - .build()) - .build()); - Response addExternalTool = UtilIT.addExternalTool(job.build()); - addExternalTool.prettyPrint(); - addExternalTool.then().assertThat() - .body("data.displayName", CoreMatchers.equalTo("AwesomeTool")) - .statusCode(OK.getStatusCode()); - } - @Test public void testAddFilelToolNoFileId() throws IOException { JsonObjectBuilder job = Json.createObjectBuilder(); @@ -295,34 +289,4 @@ public void testAddExternalToolNonReservedWord() throws IOException { .statusCode(BAD_REQUEST.getStatusCode()); } - @Test - public void testAddDatasetExploreTool() throws IOException { - JsonObjectBuilder job = Json.createObjectBuilder(); - job.add("displayName", "AwesomeTool"); - job.add("description", "This tool is awesome."); - job.add("type", "explore"); - job.add("scope", "dataset"); - job.add("toolUrl", "http://awesometool.com"); - job.add("toolParameters", Json.createObjectBuilder() - .add("queryParameters", Json.createArrayBuilder() - .add(Json.createObjectBuilder() - .add("dataset", "{datasetPid}") - .build()) - .build()) - .build()); - Response addExternalTool = UtilIT.addExternalTool(job.build()); - addExternalTool.prettyPrint(); - addExternalTool.then().assertThat() - .body("data.displayName", CoreMatchers.equalTo("AwesomeTool")) - .statusCode(OK.getStatusCode()); - - long id = JsonPath.from(addExternalTool.getBody().asString()).getLong("data.id"); - - Response getTool = UtilIT.getExternalTool(id); - getTool.prettyPrint(); - getTool.then().assertThat() - .body("data.scope", CoreMatchers.equalTo("dataset")) - .statusCode(OK.getStatusCode()); - } - } diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java index 86e65deae3a..ffcf75d65f9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java @@ -13,6 +13,7 @@ import javax.json.JsonObjectBuilder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import org.junit.Test; public class ExternalToolServiceBeanTest { @@ -288,7 +289,7 @@ public void testParseAddExternalToolInputNoContentType() { System.out.println(ex.getMessage()); } assertNotNull(externalTool); - assertEquals(externalTool.getContentType(), DataFileServiceBean.MIME_TYPE_TSV_ALT); + assertNull(externalTool.getContentType()); } @Test @@ -346,7 +347,7 @@ public void testParseAddDatasetToolDatasetId() { System.out.println(ex.getMessage()); } assertNotNull(externalTool); - assertEquals(externalTool.getContentType(), DataFileServiceBean.MIME_TYPE_TSV_ALT); + assertNull(externalTool.getContentType()); } @Test @@ -377,7 +378,7 @@ public void testParseAddDatasetToolDatasetPid() { System.out.println(ex.getMessage()); } assertNotNull(externalTool); - assertEquals(externalTool.getContentType(), DataFileServiceBean.MIME_TYPE_TSV_ALT); + assertNull(externalTool.getContentType()); } } From cc99058aa5ea6fda594d8bd8047f6de0a459b0f1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 27 Aug 2019 17:37:17 -0400 Subject: [PATCH 09/23] support file PIDs #5028 --- .../dataverse/externaltools/ExternalTool.java | 1 + .../externaltools/ExternalToolHandler.java | 7 +++ .../ExternalToolServiceBean.java | 17 +++++-- .../ExternalToolServiceBeanTest.java | 49 ++++++++++++++++++- 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index 0150ef685af..162b6e04040 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -235,6 +235,7 @@ public enum ReservedWord { // various REST APIs. For example, "Variable substitutions will be made when a variable is named in {brackets}." // from https://swagger.io/specification/#fixed-fields-29 but that's for URLs. FILE_ID("fileId"), + FILE_PID("filePid"), SITE_URL("siteUrl"), API_TOKEN("apiToken"), // datasetId is the database id diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index a08bfe4c39e..7eb65807dff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.externaltools.ExternalTool.ReservedWord; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -114,6 +115,12 @@ private String getQueryParam(String key, String value) { case FILE_ID: // getDataFile is never null for file tools because of the constructor return key + "=" + getDataFile().getId(); + case FILE_PID: + GlobalId filePid = getDataFile().getGlobalId(); + if (filePid != null) { + return key + "=" + getDataFile().getGlobalId(); + } + break; case SITE_URL: return key + "=" + SystemConfig.getDataverseSiteUrlStatic(); case API_TOKEN: diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index c5a42dd3e83..0154b52196f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -163,19 +163,28 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { JsonArray queryParams = toolParametersObj.getJsonArray("queryParameters"); boolean allRequiredReservedWordsFound = false; if (scope.equals(Scope.FILE)) { + List requiredReservedWordCandidates = new ArrayList<>(); + requiredReservedWordCandidates.add(ReservedWord.FILE_ID); + requiredReservedWordCandidates.add(ReservedWord.FILE_PID); for (JsonObject queryParam : queryParams.getValuesAs(JsonObject.class)) { Set keyValuePair = queryParam.keySet(); for (String key : keyValuePair) { String value = queryParam.getString(key); ReservedWord reservedWord = ReservedWord.fromString(value); - if (reservedWord.equals(ReservedWord.FILE_ID)) { - allRequiredReservedWordsFound = true; + for (ReservedWord requiredReservedWordCandidate : requiredReservedWordCandidates) { + if (reservedWord.equals(requiredReservedWordCandidate)) { + allRequiredReservedWordsFound = true; + } } } } if (!allRequiredReservedWordsFound) { - // Some day there might be more reserved words than just {fileId}. - throw new IllegalArgumentException("Required reserved word not found: " + ReservedWord.FILE_ID.toString()); + List requiredReservedWordCandidatesString = new ArrayList<>(); + for (ReservedWord requiredReservedWordCandidate : requiredReservedWordCandidates) { + requiredReservedWordCandidatesString.add(requiredReservedWordCandidate.toString()); + } + String friendly = String.join(", ", requiredReservedWordCandidatesString); + throw new IllegalArgumentException("One of the following reserved words is required: " + friendly + "."); } } else if (scope.equals(Scope.DATASET)) { List requiredReservedWordCandidates = new ArrayList<>(); diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java index ffcf75d65f9..75cae3214c1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java @@ -6,6 +6,7 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import java.util.ArrayList; import java.util.List; @@ -94,6 +95,52 @@ public void testParseAddExternalToolInput() { assertEquals("http://awesometool.com?fileid=42&key=7196b5ce-f200-4286-8809-03ffdbc255d7&fileMetadataId=2", toolUrl); } + @Test + public void testParseAddFileToolFilePid() { + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("displayName", "AwesomeTool"); + job.add("description", "This tool is awesome."); + job.add("type", "explore"); + job.add("scope", "file"); + job.add("toolUrl", "http://awesometool.com"); + job.add("toolParameters", Json.createObjectBuilder() + .add("queryParameters", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("filePid", "{filePid}") + .build()) + .add(Json.createObjectBuilder() + .add("key", "{apiToken}") + .build()) + .add(Json.createObjectBuilder() + .add("fileMetadataId", "{fileMetadataId}") + .build()) + .build()) + .build()); + job.add(ExternalTool.CONTENT_TYPE, DataFileServiceBean.MIME_TYPE_TSV_ALT); + String tool = job.build().toString(); + System.out.println("tool: " + tool); + ExternalTool externalTool = ExternalToolServiceBean.parseAddExternalToolManifest(tool); + assertEquals("AwesomeTool", externalTool.getDisplayName()); + DataFile dataFile = new DataFile(); + dataFile.setId(42l); + dataFile.setGlobalId(new GlobalId("doi:10.5072/FK2/RMQT6J/G9F1A1")); + FileMetadata fmd = new FileMetadata(); + fmd.setId(2L); + DatasetVersion dv = new DatasetVersion(); + Dataset ds = new Dataset(); + dv.setDataset(ds); + fmd.setDatasetVersion(dv); + List fmdl = new ArrayList(); + fmdl.add(fmd); + dataFile.setFileMetadatas(fmdl); + ApiToken apiToken = new ApiToken(); + apiToken.setTokenString("7196b5ce-f200-4286-8809-03ffdbc255d7"); + ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataFile, apiToken, fmd); + String toolUrl = externalToolHandler.getToolUrlWithQueryParams(); + System.out.println("result: " + toolUrl); + assertEquals("http://awesometool.com?filePid=doi:10.5072/FK2/RMQT6J/G9F1A1&key=7196b5ce-f200-4286-8809-03ffdbc255d7&fileMetadataId=2", toolUrl); + } + @Test public void testParseAddExternalToolInputNoFileId() { JsonObjectBuilder job = Json.createObjectBuilder(); @@ -119,7 +166,7 @@ public void testParseAddExternalToolInputNoFileId() { expectedException = ex; } assertNotNull(expectedException); - assertEquals("Required reserved word not found: {fileId}", expectedException.getMessage()); + assertEquals("One of the following reserved words is required: {fileId}, {filePid}.", expectedException.getMessage()); } @Test From 336a42437e2c0f1ca20538dc25e0e53a3db13934 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 27 Aug 2019 18:05:47 -0400 Subject: [PATCH 10/23] adjust docs #5028 --- doc/sphinx-guides/source/api/native-api.rst | 54 +++++++++++++++++++ .../source/installation/external-tools.rst | 19 +++++-- .../source/user/data-exploration/index.rst | 5 ++ 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index bdcde0476ff..194dfe77daf 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -868,6 +868,33 @@ Normally published datasets should not be deleted, but there exists a "destroy" Calling the destroy endpoint is permanent and irreversible. It will remove the dataset and its datafiles, then re-index the parent dataverse in Solr. This endpoint requires the API token of a superuser. +.. _list-external-tools-for-a-dataset-api: + +List External Tools for a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some installations of Dataverse have external tools enabled for datasets and there are two types: + +- explore +- configure + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB + export TYPE=explore + + curl -H X-Dataverse-key:$API_TOKEN \""$SERVER_URL/api/datasets/:persistentId/externalTools?persistentId=$PERSISTENT_ID&type=$TYPE"\" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx "https://demo.dataverse.org/api/datasets/:persistentId/externalTools?persistentId=doi:10.5072/FK2/J8SJZB&type=explore" + Files ----- @@ -1023,6 +1050,33 @@ Starting the release 4.10 the size of the saved original file (for an ingested t Note the optional "limit" parameter. Without it, the API will attempt to populate the sizes for all the saved originals that don't have them in the database yet. Otherwise it will do so for the first N such datafiles. +.. _list-external-tools-for-a-file-api: + +List External Tools for a File +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some installations of Dataverse have external tools enabled for files and there are two types: + +- explore +- configure + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB + export TYPE=explore + + curl -H X-Dataverse-key:$API_TOKEN \""$SERVER_URL/api/files/:persistentId/externalTools?persistentId=$PERSISTENT_ID&type=$TYPE"\" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx "https://demo.dataverse.org/api/files/:persistentId/externalTools?persistentId=doi:10.5072/FK2/J8SJZB&type=explore" + Builtin Users ------------- diff --git a/doc/sphinx-guides/source/installation/external-tools.rst b/doc/sphinx-guides/source/installation/external-tools.rst index 39b9bf38df2..83a23b78dd8 100644 --- a/doc/sphinx-guides/source/installation/external-tools.rst +++ b/doc/sphinx-guides/source/installation/external-tools.rst @@ -37,15 +37,16 @@ External tools must be expressed in an external tool manifest file, a specific J ``scope`` is required and must be ``file`` or ``dataset`` to make the tool appear at the file level or dataset level. -File level tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. (Not providing this parameter makes the tool work on ingested tabular files and is equivalent to specifying the ``contentType`` as "text/tab-separated-values".) +File level tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. In the example above, a mix of required and optional reserved words appear that can be used to insert dynamic values into tools. The supported values are: -- ``{fileId}`` (required for file tools) - The Dataverse database ID of a file from which the external tool has been launched. +- ``{fileId}`` (required for file tools, or ``{filePid}``) - The database ID of a file from which the external tool has been launched. +- ``{filePid}`` (required for file tools, or ``{fileId}``) - The Persistent ID (DOI or Handle) of a file (if available) from which the external tool has been launched. - ``{siteUrl}`` (optional) - The URL of the Dataverse installation from which the tool was launched. - ``{apiToken}`` (optional) - The Dataverse API token of the user launching the external tool, if available. -- ``{datasetId}`` (optional) - The ID of the dataset. -- ``{datasetPid}`` (optional) - The Persistent ID (DOI or Handle) of the dataset. +- ``{datasetId}`` (required for dataset tools, or ``{datasetPid}``) - The database ID of the dataset. +- ``{datasetPid}`` (required for dataset tools, or ``{datasetId}``) - The Persistent ID (DOI or Handle) of the dataset. - ``{datasetVersion}`` (optional) - The friendly version number ( or \:draft ) of the dataset version the tool is being launched from. Making an External Tool Available in Dataverse @@ -78,6 +79,16 @@ Assuming the external tool database id is "1", remove it with the following comm ``curl -X DELETE http://localhost:8080/api/admin/externalTools/1`` +List External Tools for a Dataset +--------------------------------- + +See :ref:`list-external-tools-for-a-dataset-api` + +List External Tools for a File +------------------------------ + +See :ref:`list-external-tools-for-a-file-api` + Writing Your Own External Tool ------------------------------ diff --git a/doc/sphinx-guides/source/user/data-exploration/index.rst b/doc/sphinx-guides/source/user/data-exploration/index.rst index e6af35387b9..07e0d986498 100755 --- a/doc/sphinx-guides/source/user/data-exploration/index.rst +++ b/doc/sphinx-guides/source/user/data-exploration/index.rst @@ -8,6 +8,11 @@ Data Exploration Guide Note that the installation of Dataverse you are using may have additional or different tools configured. Developers interested in creating tools should refer to the :doc:`/installation/external-tools` section of the Installation Guide. +API users can determine if external tools are configured at the dataset for file level by following the instructions below. + +- :ref:`list-external-tools-for-a-dataset-api` +- :ref:`list-external-tools-for-a-file-api` + Contents: .. toctree:: From 94f812724fe3fe471ee596fb1dffbb3d631acd77 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 29 Aug 2019 16:10:16 -0400 Subject: [PATCH 11/23] fix typo in method name (remove "scope") #5028 --- src/main/java/edu/harvard/iq/dataverse/FilePage.java | 4 ++-- src/main/java/edu/harvard/iq/dataverse/api/Files.java | 2 +- .../iq/dataverse/externaltools/ExternalToolServiceBean.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 754631551bc..15383eb3d57 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -206,8 +206,8 @@ public String init() { if (file.isTabularData()) { contentType=DataFileServiceBean.MIME_TYPE_TSV_ALT; } - configureTools = externalToolService.findFileToolsByScopeAndContentType(ExternalTool.Type.CONFIGURE, contentType); - exploreTools = externalToolService.findFileToolsByScopeAndContentType(ExternalTool.Type.EXPLORE, contentType); + configureTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.CONFIGURE, contentType); + exploreTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.EXPLORE, contentType); } else { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 59f38deb103..2b8146f9d4f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -627,7 +627,7 @@ public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam try { DataFile dataFile = findDataFileOrDie(idSupplied); JsonArrayBuilder tools = Json.createArrayBuilder(); - List datasetTools = externalToolService.findFileToolsByScopeAndContentType(ExternalTool.Type.fromString(type), dataFile.getContentType()); + List datasetTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.fromString(type), dataFile.getContentType()); for (ExternalTool tool : datasetTools) { String apiTokenString = null; apiTokenString = getRequestApiKey(); diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index 0154b52196f..0c26d5ee030 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -67,7 +67,7 @@ public List findFileToolsByType(Type type) { * @param contentType file content type (MIME type) * @return A list of tools or an empty list. */ - public List findFileToolsByScopeAndContentType(Type type, String contentType) { + public List findFileToolsByTypeAndContentType(Type type, String contentType) { return findByScopeTypeAndContentType(ExternalTool.Scope.FILE, type, contentType); } From e92c8f12d5dc36271b5c9a035b542228a60e95ca Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 29 Aug 2019 17:10:04 -0400 Subject: [PATCH 12/23] reduce code duplication #5028 --- .../edu/harvard/iq/dataverse/api/Datasets.java | 16 +++------------- .../edu/harvard/iq/dataverse/api/Files.java | 16 +++------------- .../externaltools/ExternalToolServiceBean.java | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index bf41fc4f09a..c1c57f16d21 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1905,19 +1905,9 @@ public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam JsonArrayBuilder tools = Json.createArrayBuilder(); List datasetTools = externalToolService.findDatasetToolsByType(ExternalTool.Type.fromString(type)); for (ExternalTool tool : datasetTools) { - String apiTokenString = null; - apiTokenString = getRequestApiKey(); - ApiToken apiToken = null; - if (apiTokenString != null) { - apiToken = new ApiToken(); - apiToken.setTokenString(apiTokenString); - } - ExternalToolHandler externalToolHandler = null; - String toolUrlWithQueryParams = null; - externalToolHandler = new ExternalToolHandler(tool, dataset, apiToken); - toolUrlWithQueryParams = externalToolHandler.getToolUrlWithQueryParams(); - JsonObjectBuilder toolToJson = tool.toJson(); - toolToJson.add("toolUrlWithQueryParams", toolUrlWithQueryParams); + ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey()); + ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataset, apiToken); + JsonObjectBuilder toolToJson = externalToolService.getToolAsJsonWithQueryParameters(externalToolHandler); tools.add(toolToJson); } return ok(tools); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 2b8146f9d4f..60fb2d91f67 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -629,19 +629,9 @@ public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam JsonArrayBuilder tools = Json.createArrayBuilder(); List datasetTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.fromString(type), dataFile.getContentType()); for (ExternalTool tool : datasetTools) { - String apiTokenString = null; - apiTokenString = getRequestApiKey(); - ApiToken apiToken = null; - if (apiTokenString != null) { - apiToken = new ApiToken(); - apiToken.setTokenString(apiTokenString); - } - ExternalToolHandler externalToolHandler = null; - String toolUrlWithQueryParams = null; - externalToolHandler = new ExternalToolHandler(tool, dataFile, apiToken, dataFile.getFileMetadata()); - toolUrlWithQueryParams = externalToolHandler.getToolUrlWithQueryParams(); - JsonObjectBuilder toolToJson = tool.toJson(); - toolToJson.add("toolUrlWithQueryParams", toolUrlWithQueryParams); + ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey()); + ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataFile, apiToken, dataFile.getFileMetadata()); + JsonObjectBuilder toolToJson = externalToolService.getToolAsJsonWithQueryParameters(externalToolHandler); tools.add(toolToJson); } return ok(tools); diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index 0c26d5ee030..92adfe6d42b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFileServiceBean; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.externaltools.ExternalTool.ReservedWord; import edu.harvard.iq.dataverse.externaltools.ExternalTool.Type; import edu.harvard.iq.dataverse.externaltools.ExternalTool.Scope; @@ -23,6 +24,7 @@ import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; import javax.json.JsonReader; import javax.persistence.EntityManager; import javax.persistence.NoResultException; @@ -232,4 +234,20 @@ private static String getOptionalTopLevelField(JsonObject jsonObject, String key } } + public ApiToken getApiToken(String apiTokenString) { + ApiToken apiToken = null; + if (apiTokenString != null) { + apiToken = new ApiToken(); + apiToken.setTokenString(apiTokenString); + } + return apiToken; + } + + public JsonObjectBuilder getToolAsJsonWithQueryParameters(ExternalToolHandler externalToolHandler) { + JsonObjectBuilder toolToJson = externalToolHandler.getExternalTool().toJson(); + String toolUrlWithQueryParams = externalToolHandler.getToolUrlWithQueryParams(); + toolToJson.add("toolUrlWithQueryParams", toolUrlWithQueryParams); + return toolToJson; + } + } From 969d5edbc6a10fb56f93d922c76fbd95faaa5cc1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 29 Aug 2019 17:14:03 -0400 Subject: [PATCH 13/23] fix assertion in test #5028 --- src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java index c4bfcc1f488..b27cd1ad3a1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java @@ -235,7 +235,7 @@ public void testAddFilelToolNoFileId() throws IOException { Response addExternalTool = UtilIT.addExternalTool(job.build()); addExternalTool.prettyPrint(); addExternalTool.then().assertThat() - .body("message", CoreMatchers.equalTo("Required reserved word not found: {fileId}")) + .body("message", CoreMatchers.equalTo("One of the following reserved words is required: {fileId}, {filePid}.")) .statusCode(BAD_REQUEST.getStatusCode()); } From f8e1e0e1f75af5537d77d811ab180c274192b619 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 30 Aug 2019 11:14:42 -0400 Subject: [PATCH 14/23] prevent 500 error if invalid type is supplied #5028 --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 10 ++++++++-- .../java/edu/harvard/iq/dataverse/api/Files.java | 10 ++++++++-- .../harvard/iq/dataverse/api/ExternalToolsIT.java | 12 ++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index c1c57f16d21..a51ac707a77 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1898,12 +1898,18 @@ public Response getMakeDataCountMetric(@PathParam("id") String idSupplied, @Path @GET @Path("{id}/externalTools") - public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam("type") String type) { + public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam("type") String typeSupplied) { + ExternalTool.Type type; + try { + type = ExternalTool.Type.fromString(typeSupplied); + } catch (IllegalArgumentException ex) { + return error(BAD_REQUEST, ex.getLocalizedMessage()); + } Dataset dataset; try { dataset = findDatasetOrDie(idSupplied); JsonArrayBuilder tools = Json.createArrayBuilder(); - List datasetTools = externalToolService.findDatasetToolsByType(ExternalTool.Type.fromString(type)); + List datasetTools = externalToolService.findDatasetToolsByType(type); for (ExternalTool tool : datasetTools) { ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey()); ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataset, apiToken); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 60fb2d91f67..489d29a5060 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -623,11 +623,17 @@ private void exportDatasetMetadata(SettingsServiceBean settingsServiceBean, Data @Path("{id}/externalTools") @GET - public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam("type") String type) { + public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam("type") String typeSupplied) { + ExternalTool.Type type; + try { + type = ExternalTool.Type.fromString(typeSupplied); + } catch (IllegalArgumentException ex) { + return error(BAD_REQUEST, ex.getLocalizedMessage()); + } try { DataFile dataFile = findDataFileOrDie(idSupplied); JsonArrayBuilder tools = Json.createArrayBuilder(); - List datasetTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.fromString(type), dataFile.getContentType()); + List datasetTools = externalToolService.findFileToolsByTypeAndContentType(type, dataFile.getContentType()); for (ExternalTool tool : datasetTools) { ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey()); ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataFile, apiToken, dataFile.getFileMetadata()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java index b27cd1ad3a1..21ae03531bd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java @@ -117,6 +117,12 @@ public void testFileLevelTool1() { .body("data.scope", CoreMatchers.equalTo("file")) .statusCode(OK.getStatusCode()); + Response getExternalToolsForFileInvalidType = UtilIT.getExternalToolsForFile(tabularFileId.toString(), "invalidType", apiToken); + getExternalToolsForFileInvalidType.prettyPrint(); + getExternalToolsForFileInvalidType.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", CoreMatchers.equalTo("Type must be one of these values: [explore, configure].")); + Response getExternalToolsForTabularFiles = UtilIT.getExternalToolsForFile(tabularFileId.toString(), "explore", apiToken); getExternalToolsForTabularFiles.prettyPrint(); getExternalToolsForTabularFiles.then().assertThat() @@ -207,6 +213,12 @@ public void testDatasetLevelTool1() { .body("data.displayName", CoreMatchers.equalTo("DatasetTool1")) .statusCode(OK.getStatusCode()); + Response getExternalToolsByDatasetIdInvalidType = UtilIT.getExternalToolsForDataset(datasetId.toString(), "invalidType", apiToken); + getExternalToolsByDatasetIdInvalidType.prettyPrint(); + getExternalToolsByDatasetIdInvalidType.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", CoreMatchers.equalTo("Type must be one of these values: [explore, configure].")); + Response getExternalToolsByDatasetId = UtilIT.getExternalToolsForDataset(datasetId.toString(), "explore", apiToken); getExternalToolsByDatasetId.prettyPrint(); getExternalToolsByDatasetId.then().assertThat() From 801bf56fa43322f4113962ca4b0dd6f4d72b4f2a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 5 Sep 2019 16:31:21 -0400 Subject: [PATCH 15/23] rename SQL script to reflect bump to 4.16 #5028 --- ...5028-dataset-explore.sql => V4.16.1__5028-dataset-explore.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V4.15.1.1__5028-dataset-explore.sql => V4.16.1__5028-dataset-explore.sql} (100%) diff --git a/src/main/resources/db/migration/V4.15.1.1__5028-dataset-explore.sql b/src/main/resources/db/migration/V4.16.1__5028-dataset-explore.sql similarity index 100% rename from src/main/resources/db/migration/V4.15.1.1__5028-dataset-explore.sql rename to src/main/resources/db/migration/V4.16.1__5028-dataset-explore.sql From e2e34d36346e8029a3f1658b4a495ce63d5db04e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 9 Sep 2019 18:01:14 -0400 Subject: [PATCH 16/23] improve external tools documentation #5028 --- .../admin/dataverse-external-tools.tsv | 5 + .../external-tools/dynamicDatasetTool.json | 17 +++ ...awesomeTool.json => fabulousFileTool.json} | 6 +- .../source/admin/external-tools.rst | 107 ++++++++++++++++++ doc/sphinx-guides/source/admin/index.rst | 1 + .../source/developers/external-tools.rst | 81 +++++++++++++ doc/sphinx-guides/source/developers/index.rst | 1 + .../source/installation/external-tools.rst | 93 ++------------- .../source/user/data-exploration/index.rst | 7 +- .../source/user/find-use-data.rst | 2 +- 10 files changed, 225 insertions(+), 95 deletions(-) create mode 100644 doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv create mode 100644 doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json rename doc/sphinx-guides/source/_static/installation/files/root/external-tools/{awesomeTool.json => fabulousFileTool.json} (64%) create mode 100644 doc/sphinx-guides/source/admin/external-tools.rst create mode 100644 doc/sphinx-guides/source/developers/external-tools.rst diff --git a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv new file mode 100644 index 00000000000..0d251202c22 --- /dev/null +++ b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv @@ -0,0 +1,5 @@ +TwoRavens explore file A system of interlocking statistical tools for data exploration, analysis, and meta-analysis: http://2ra.vn. See the :doc:`/user/data-exploration/tworavens` section of the User Guide for more information on TwoRavens from the user perspective and the :doc:`/installation/r-rapache-tworavens` section of the Installation Guide. +Data Explorer explore file A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Explorer for the instructions on adding Data Explorer to your Dataverse; and the :doc:`/installation/prerequisites` section of the Installation Guide for the instructions on how to set up **basic R configuration required** (specifically, Dataverse uses R to generate .prep metadata files that are needed to run Data Explorer). +Whole Tale explore file A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide `_. +File Previewers explore file A set of tools that display the content of files - including audio, html, `Hypothes.is `_ annotations, images, PDF, text, video - allowing them to be viewed without downloading. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. https://github.com/QualitativeDataRepository/dataverse-previewers +Data Curation Tool configure file A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions. diff --git a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json new file mode 100644 index 00000000000..eec498acbff --- /dev/null +++ b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json @@ -0,0 +1,17 @@ +{ + "displayName": "Dynamic Dataset Tool", + "description": "Dazzles! Dizzying!", + "type": "explore", + "scope": "dataset", + "toolUrl": "https://dynamicdatasettool.com", + "toolParameters": { + "queryParameters": [ + { + "PID": "{datasetPid}" + }, + { + "apiToken": "{apiToken}" + } + ] + } +} diff --git a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/awesomeTool.json b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json similarity index 64% rename from doc/sphinx-guides/source/_static/installation/files/root/external-tools/awesomeTool.json rename to doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json index caa6baefe60..ddbc460b694 100644 --- a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/awesomeTool.json +++ b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json @@ -1,10 +1,10 @@ { - "displayName": "Awesome Tool", - "description": "The most awesome tool.", + "displayName": "Fabulous File Tool", + "description": "Fabulous Fun for Files!", "type": "explore", "scope": "file", "contentType": "text/tab-separated-values", - "toolUrl": "https://awesometool.com", + "toolUrl": "https://fabulousfiletool.com", "toolParameters": { "queryParameters": [ { diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst new file mode 100644 index 00000000000..b735ec8d072 --- /dev/null +++ b/doc/sphinx-guides/source/admin/external-tools.rst @@ -0,0 +1,107 @@ +External Tools +============== + +External tools can provide additional features that are not part of Dataverse itself, such as data exploration. + +.. contents:: |toctitle| + :local: + +.. _inventory-of-external-tools: + +Inventory of External Tools +--------------------------- + +.. csv-table:: + :header: "Tool", "Type", "Scope", "Description" + :widths: 20, 10, 5, 65 + :delim: tab + :file: ../_static/admin/dataverse-external-tools.tsv + +.. _managing-external-tools: + +Managing External Tools +----------------------- + +Adding External Tools to Dataverse ++++++++++++++++++++++++++++++++++++ + +To add an external tool to your installation of Dataverse you must first download a JSON file for that tool, which we refer to as a "manifest". It should look something like this: + +.. literalinclude:: ../_static/installation/files/root/external-tools/fabulousFileTool.json + +Go to :ref:`inventory-of-external-tools` and download a JSON manifest for one of the tools by following links in the description to installation instructions. + +In the curl command below, replace the placeholder "fabulousFileTool.json" placeholder for the actual name of the JSON file you downloaded. + +.. code-block:: bash + + curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools --upload-file fabulousFileTool.json + +Listing All External Tools in Dataverse ++++++++++++++++++++++++++++++++++++++++ + +To list all the external tools that are available in Dataverse: + +.. code-block:: bash + + curl http://localhost:8080/api/admin/externalTools + +Showing an External Tool in Dataverse ++++++++++++++++++++++++++++++++++++++ + +To show one of the external tools that are available in Dataverse, pass its database id: + +.. code-block:: bash + + export TOOL_ID=1 + curl http://localhost:8080/api/admin/externalTools/$TOOL_ID + +Removing an External Tool From Dataverse +++++++++++++++++++++++++++++++++++++++++ + +Assuming the external tool database id is "1", remove it with the following command: + +.. code-block:: bash + + export TOOL_ID=1 + curl -X DELETE http://localhost:8080/api/admin/externalTools/$TOOL_ID + +.. _testing-external-tools: + +Testing External Tools +---------------------- + +Once you have added an external tool to your installation of Dataverse, you will probably want to test it to make sure it is functioning properly. + +File Level Explore Tools +++++++++++++++++++++++++ + +File level explore tools are specific to the file type (content type or MIME type) of the file. For example, there is a tool for exploring PDF files in the "File Previewers" set of tools. + +An "Explore" button will appear (on both the dataset page and the file landing page) for files that match the type that the tool has been built for. When there are multiple explore tools for a filetype, the button becomes a dropdown. + +For a specific file, you can check how many tools are available using the GUI as described above or by using the API as described at :ref:`list-external-tools-for-a-file-api`. + +File Level Configure Tools +++++++++++++++++++++++++++ + +File level configure tools are only available when you log in and have write access to the file. The file type determines if a configure tool is available. For example, a configure tool may only be available for tabular files. + +Dataset Level Explore Tools ++++++++++++++++++++++++++++ + +When a dataset level explore tool is added, an "Explore" button on the dataset page will appear. This button becomes a drop down when there are multiple tools. + +You can also use the API to list dataset level tools. See :ref:`list-external-tools-for-a-dataset-api` + +Dataset Level Configure Tools ++++++++++++++++++++++++++++++ + +Configure tools at the dataset level are not currently supported. No button appears in the GUI if you add this type of tool. + +Writing Your Own External Tool +------------------------------ + +If you plan to write a external tool, see the :doc:`/developers/external-tools` section of the Developer Guide. + +If you have an idea for an external tool, please let the Dataverse community know by posting about it on the dataverse-community mailing list: https://groups.google.com/forum/#!forum/dataverse-community diff --git a/doc/sphinx-guides/source/admin/index.rst b/doc/sphinx-guides/source/admin/index.rst index 670f315c5a5..39b4f5748d3 100755 --- a/doc/sphinx-guides/source/admin/index.rst +++ b/doc/sphinx-guides/source/admin/index.rst @@ -13,6 +13,7 @@ This guide documents the functionality only available to superusers (such as "da .. toctree:: dashboard + external-tools harvestclients harvestserver metadatacustomization diff --git a/doc/sphinx-guides/source/developers/external-tools.rst b/doc/sphinx-guides/source/developers/external-tools.rst new file mode 100644 index 00000000000..75da9e84773 --- /dev/null +++ b/doc/sphinx-guides/source/developers/external-tools.rst @@ -0,0 +1,81 @@ +Building External Tools +======================= + +External tools can provide additional features that are not part of Dataverse itself, such as data exploration. Thank you for your interest in building an external tool for Dataverse! + +.. contents:: |toctitle| + :local: + +Examples of External Tools +-------------------------- + +To get your creative juices flowing, here are examples of external tools that are available for Dataverse. + +.. csv-table:: + :header: "Tool", "Type", "Scope", "Description" + :widths: 20, 10, 5, 65 + :delim: tab + :file: ../_static/admin/dataverse-external-tools.tsv + +How External Tools Are Presented to Users +----------------------------------------- + +See :ref:`testing-external-tools` for a description of how various tools are presented to users in the Dataverse web interface. + +How External Tools are Classified (Type Vs. Scope) +-------------------------------------------------- + +External tools have a both a "type" and a "scope." Here are defintions for both terms: + +.. table:: + :widths: 10, 20, 70 + + ===== ==================== ========== + Term Possible values Definition + ===== ==================== ========== + type explore or configure The type determines if the tool appears under a button called "Explore" or a button called "Configure". Users must be logged in to Dataverse for configure tools which are expected to write back to Dataverse. Explore tools are typically read-only. + + scope file or dataset What "dvObject" the tool operates on. The third dvOjbect is "dataverse" but this is not supported. + ===== ==================== ========== + +Creating an External Tool Manifest +---------------------------------- + +External tools must be expressed in an external tool manifest file, a specific JSON format Dataverse requires. As the author of an external tool, you are expected to provide this JSON file and installation instructions on a web page for your tool. + +Let's look at two examples of external tool manifests before we dive into how they work. + +:download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` is a file level explore tool that operates on tabular files: + +.. literalinclude:: ../_static/installation/files/root/external-tools/fabulousFileTool.json + +:download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` is a dataset level explore tool: + +.. literalinclude:: ../_static/installation/files/root/external-tools/dynamicDatasetTool.json + +``type`` is required and must be ``explore`` or ``configure`` to make the tool appear under a button called "Explore" or "Configure", respectively. Not that a "Configure" button at the dataset level is not currently supported. + +``scope`` is required and must be ``file`` or ``dataset`` to make the tool appear at the file level or dataset level. + +File level tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. + +In the example above, a mix of required and optional reserved words appear that can be used to insert dynamic values into tools. The supported values are: + +- ``{fileId}`` (required for file tools, or ``{filePid}``) - The database ID of a file from which the external tool has been launched. +- ``{filePid}`` (required for file tools, or ``{fileId}``) - The Persistent ID (DOI or Handle) of a file (if available) from which the external tool has been launched. +- ``{siteUrl}`` (optional) - The URL of the Dataverse installation from which the tool was launched. +- ``{apiToken}`` (optional) - The Dataverse API token of the user launching the external tool, if available. +- ``{datasetId}`` (required for dataset tools, or ``{datasetPid}``) - The database ID of the dataset. +- ``{datasetPid}`` (required for dataset tools, or ``{datasetId}``) - The Persistent ID (DOI or Handle) of the dataset. +- ``{datasetVersion}`` (optional) - The friendly version number ( or \:draft ) of the dataset version the tool is being launched from. + +Getting Help With Writing Your Own External Tool +------------------------------------------------ + +If you need help with your tool, please feel free to post on the dataverse-dev mailing list: https://groups.google.com/forum/#!forum/dataverse-dev + +Once you've gotten your tool working, please make a pull request to update the list of tools above! You are also welcome to download :download:`dataverse-external-tools.tsv <../_static/admin/dataverse-external-tools.tsv>`, add your tool to the TSV file, create and issue at https://github.com/IQSS/dataverse/issues , and then upload your TSV file there. + +You are welcome to announce your external tool at https://groups.google.com/forum/#!forum/dataverse-community + +Unless your tool runs entirely in a browser, you may have integrated server-side software with Dataverse. If so, please double check that your software is listed in the :doc:`/admin/integrations` section of the Admin Guide and if not, pleas open an issue or pull request to add it. Thanks! diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index 96595220e07..ffca75238ae 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -33,3 +33,4 @@ Developer Guide selinux big-data-support workflows + external-tools diff --git a/doc/sphinx-guides/source/installation/external-tools.rst b/doc/sphinx-guides/source/installation/external-tools.rst index 83a23b78dd8..818020af9cb 100644 --- a/doc/sphinx-guides/source/installation/external-tools.rst +++ b/doc/sphinx-guides/source/installation/external-tools.rst @@ -1,7 +1,7 @@ External Tools ============== -External tools can provide additional features that are not part of Dataverse itself, such as data exploration. See the "Writing Your Own External Tool" section below for more information on developing your own tool for Dataverse. +External tools can provide additional features that are not part of Dataverse itself, such as data exploration. .. contents:: |toctitle| :local: @@ -9,91 +9,14 @@ External tools can provide additional features that are not part of Dataverse it Inventory of External Tools --------------------------- -The following tools have been successfully integrated with Dataverse: +See :ref:`inventory-of-external-tools`. -- TwoRavens: a system of interlocking statistical tools for data exploration, analysis, and meta-analysis: http://2ra.vn. See the :doc:`/user/data-exploration/tworavens` section of the User Guide for more information on TwoRavens from the user perspective and the :doc:`r-rapache-tworavens` section of the Installation Guide. +Managing External Tools +----------------------- -- Data Explorer: a GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Explorer for the instructions on adding Data Explorer to your Dataverse; and the :doc:`prerequisites` section of the Installation Guide for the instructions on how to set up **basic R configuration required** (specifically, Dataverse uses R to generate .prep metadata files that are needed to run Data Explorer). +See the :doc:`/admin/external-tools` section of the Admin Guide. -- `Whole Tale `_: a platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide `_. +Building External Tools +----------------------- -- `File Previewers `_: A set of tools that display the content of files - including audio, html, `Hypothes.is ` annotations, images, PDF, text, video - allowing them to be viewed without downloading. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. - -- Data Curation Tool: a GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions. - -- [Your tool here! Please get in touch! :) ] - - -Downloading and Adjusting an External Tool Manifest File --------------------------------------------------------- - -In order to make external tools available within Dataverse, you need to configure Dataverse to be aware of them. - -External tools must be expressed in an external tool manifest file, a specific JSON format Dataverse requires. The author of the external tool may be able to provide you with a JSON file and installation instructions. The JSON file might look like this: - -.. literalinclude:: ../_static/installation/files/root/external-tools/awesomeTool.json - -``type`` is required and must be ``explore`` or ``configure`` to make the tool appear under a button called "Explore" or "Configure", respectively. - -``scope`` is required and must be ``file`` or ``dataset`` to make the tool appear at the file level or dataset level. - -File level tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. - -In the example above, a mix of required and optional reserved words appear that can be used to insert dynamic values into tools. The supported values are: - -- ``{fileId}`` (required for file tools, or ``{filePid}``) - The database ID of a file from which the external tool has been launched. -- ``{filePid}`` (required for file tools, or ``{fileId}``) - The Persistent ID (DOI or Handle) of a file (if available) from which the external tool has been launched. -- ``{siteUrl}`` (optional) - The URL of the Dataverse installation from which the tool was launched. -- ``{apiToken}`` (optional) - The Dataverse API token of the user launching the external tool, if available. -- ``{datasetId}`` (required for dataset tools, or ``{datasetPid}``) - The database ID of the dataset. -- ``{datasetPid}`` (required for dataset tools, or ``{datasetId}``) - The Persistent ID (DOI or Handle) of the dataset. -- ``{datasetVersion}`` (optional) - The friendly version number ( or \:draft ) of the dataset version the tool is being launched from. - -Making an External Tool Available in Dataverse ----------------------------------------------- - -If the JSON file were called, for example, :download:`awesomeTool.json <../_static/installation/files/root/external-tools/awesomeTool.json>` you would make any necessary adjustments, as described above, and then make the tool available within Dataverse with the following curl command: - -``curl -X POST -H 'Content-type: application/json' --upload-file awesomeTool.json http://localhost:8080/api/admin/externalTools`` - -Listing all External Tools in Dataverse ---------------------------------------- - -To list all the external tools that are available in Dataverse: - -``curl http://localhost:8080/api/admin/externalTools`` - -Showing an External Tool in Dataverse -------------------------------------- - -To show one of the external tools that are available in Dataverse, pass its database id: - -``export TOOL_ID=1`` - -``curl http://localhost:8080/api/admin/externalTools/$TOOL_ID`` - -Removing an External Tool Available in Dataverse ------------------------------------------------- - -Assuming the external tool database id is "1", remove it with the following command: - -``curl -X DELETE http://localhost:8080/api/admin/externalTools/1`` - -List External Tools for a Dataset ---------------------------------- - -See :ref:`list-external-tools-for-a-dataset-api` - -List External Tools for a File ------------------------------- - -See :ref:`list-external-tools-for-a-file-api` - -Writing Your Own External Tool ------------------------------- - -If you have an idea for an external tool, please let the Dataverse community know by posting about it on the dataverse-community mailing list: https://groups.google.com/forum/#!forum/dataverse-community - -If you need help with your tool, please feel free to post on the dataverse-dev mailing list: https://groups.google.com/forum/#!forum/dataverse-dev - -Once you've gotten your tool working, please make a pull request to update the list of tools above. +See the :doc:`/developers/external-tools` section of the Developer Guide. diff --git a/doc/sphinx-guides/source/user/data-exploration/index.rst b/doc/sphinx-guides/source/user/data-exploration/index.rst index 07e0d986498..52833e3ae50 100755 --- a/doc/sphinx-guides/source/user/data-exploration/index.rst +++ b/doc/sphinx-guides/source/user/data-exploration/index.rst @@ -6,12 +6,7 @@ Data Exploration Guide ======================================================= -Note that the installation of Dataverse you are using may have additional or different tools configured. Developers interested in creating tools should refer to the :doc:`/installation/external-tools` section of the Installation Guide. - -API users can determine if external tools are configured at the dataset for file level by following the instructions below. - -- :ref:`list-external-tools-for-a-dataset-api` -- :ref:`list-external-tools-for-a-file-api` +The installation of Dataverse you are using may have additional or different tools configured than what appears below. For the complete list of available tools, see :ref:`inventory-of-external-tools`. Contents: diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index d54b7764775..212f154ea2f 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -146,7 +146,7 @@ After you've downloaded the Dataverse Package, you may want to double-check that Explore Data ------------ -Please see the :doc:`/user/data-exploration/index`. +Please see the :doc:`/user/data-exploration/index` to get started. .. |image-file-tree-view| image:: ./img/file-tree-view.png :class: img-responsive From 1578595abc6eaa6e43636ad78b13a137ad5ce58c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 10 Sep 2019 11:14:13 -0400 Subject: [PATCH 17/23] Link to the Admin Guide page on external tools #5028 --- doc/sphinx-guides/source/admin/integrations.rst | 6 +++--- doc/sphinx-guides/source/api/apps.rst | 2 +- doc/sphinx-guides/source/api/intro.rst | 4 ++-- doc/sphinx-guides/source/developers/intro.rst | 2 +- doc/sphinx-guides/source/installation/config.rst | 4 ++-- doc/sphinx-guides/source/installation/prep.rst | 3 ++- .../source/installation/r-rapache-tworavens.rst | 2 +- doc/sphinx-guides/source/user/account.rst | 2 +- doc/sphinx-guides/source/user/dataset-management.rst | 2 +- doc/sphinx-guides/source/user/find-use-data.rst | 2 +- 10 files changed, 15 insertions(+), 14 deletions(-) diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst index c7924cd7c20..62a3121ed25 100644 --- a/doc/sphinx-guides/source/admin/integrations.rst +++ b/doc/sphinx-guides/source/admin/integrations.rst @@ -49,14 +49,14 @@ Data Explorer Data Explorer is a GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. -For installation instructions, see the :doc:`/installation/external-tools` section of the Installation Guide. +For installation instructions, see the :doc:`external-tools` section. TwoRavens/Zelig +++++++++++++++ TwoRavens is a web application for tabular data exploration and statistical analysis with Zelig. -For installation instructions, see the :doc:`/installation/external-tools` section of the Installation Guide. +For installation instructions, see the :doc:`external-tools` section. WorldMap ++++++++ @@ -75,7 +75,7 @@ Whole Tale `Whole Tale `_ enables researchers to analyze data using popular tools including Jupyter and RStudio with the ultimate goal of supporting publishing of reproducible research packages. Users can `import data from Dataverse -`_ via identifier (e.g., DOI, URI, etc) or through the External Tools integration. For installation instructions, see the :doc:`/installation/external-tools` section of this Installation Guide or the `Integration `_ section of the Whole Tale User Guide. +`_ via identifier (e.g., DOI, URI, etc) or through the External Tools integration. For installation instructions, see the :doc:`external-tools` section or the `Integration `_ section of the Whole Tale User Guide. Discoverability --------------- diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index f9d8d4c9b02..a7bc1cb3a4c 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -1,7 +1,7 @@ Apps ==== -The introduction of Dataverse APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations` and :doc:`/admin/reporting-tools` sections of the Admin Guide and the :doc:`/installation/external-tools` section of the Installation Guide. +The introduction of Dataverse APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations`, :doc:`/admin/external-tools`, and :doc:`/admin/reporting-tools` sections of the Admin Guide. The apps below are open source and demonstrate how to use Dataverse APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse APIs in Python, R, and Java. diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst index 5fec86fab03..c5b0183f05a 100755 --- a/doc/sphinx-guides/source/api/intro.rst +++ b/doc/sphinx-guides/source/api/intro.rst @@ -135,7 +135,7 @@ Developers of Integrations, External Tools, and Apps One of the primary purposes for Dataverse APIs in the first place is to enable integrations with third party software. Integrations are listed in the following places: - The :doc:`/admin/integrations` section of the Admin Guide. -- The :doc:`/installation/external-tools` section of the Installation Guide. +- The :doc:`/developers/external-tools` section of the Developer Guide. - The :doc:`apps` section of this guide. |Start| Good starting points are the three sections above to get a sense of third-party software that already integrates with Dataverse, followed by the :doc:`getting-started` section. @@ -194,6 +194,7 @@ Please note that some APIs are only documented in other guides that are more sui - Admin Guide + - :doc:`/admin/external-tools` - :doc:`/admin/metadatacustomization` - :doc:`/admin/metadataexport` - :doc:`/admin/make-data-count` @@ -203,7 +204,6 @@ Please note that some APIs are only documented in other guides that are more sui - Installation Guide - :doc:`/installation/config` - - :doc:`/installation/external-tools` Client Libraries ~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index f5a970d772c..4901063ddbc 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -53,7 +53,7 @@ Related Projects As a developer, you also may be interested in these projects related to Dataverse: -- External Tools - add additional features to Dataverse: See the :doc:`/installation/external-tools` section of the Installation Guide. +- External Tools - add additional features to Dataverse without modifying the core: :doc:`external-tools` - Dataverse API client libraries - use Dataverse APIs from various languages: :doc:`/api/client-libraries` - DVUploader - a stand-alone command-line Java application that uses the Dataverse API to support upload of files from local disk to a Dataset: https://github.com/IQSS/dataverse-uploader - dataverse-sample-data - populate your Dataverse installation with sample data: https://github.com/IQSS/dataverse-sample-data diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 79b02fab6da..0f3ee8d5b9a 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1424,12 +1424,12 @@ The relative path URL to which users will be sent for signup. The default settin :TwoRavensUrl +++++++++++++ -The ``:TwoRavensUrl`` option is no longer valid. See :doc:`r-rapache-tworavens` and :doc:`external-tools`. +The ``:TwoRavensUrl`` option is no longer valid. See :doc:`r-rapache-tworavens` and the :doc:`/admin/external-tools` section of the Admin Guide. :TwoRavensTabularView +++++++++++++++++++++ -The ``:TwoRavensTabularView`` option is no longer valid. See :doc:`r-rapache-tworavens` and :doc:`external-tools`. +The ``:TwoRavensTabularView`` option is no longer valid. See :doc:`r-rapache-tworavens` and the :doc:`/admin/external-tools` section of the Admin Guide. :GeoconnectCreateEditMaps +++++++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/installation/prep.rst b/doc/sphinx-guides/source/installation/prep.rst index 2d8beeccdcb..711693125f0 100644 --- a/doc/sphinx-guides/source/installation/prep.rst +++ b/doc/sphinx-guides/source/installation/prep.rst @@ -70,7 +70,8 @@ Optional Components There are a number of optional components you may choose to install or configure, including: -- R, rApache, Zelig, and TwoRavens: :doc:`/user/data-exploration/tworavens` describes the feature and :doc:`r-rapache-tworavens` describes how to install these components. :doc:`external-tools` explains how third-party tools like TwoRavens can be added to Dataverse. +- External Tools: Third party tools for data exploration can be added to Dataverse by following the instructions in the :doc:`/admin/external-tools` section of the Admin Guide. +- R, rApache, Zelig, and TwoRavens: :doc:`/user/data-exploration/tworavens` describes the feature and :doc:`r-rapache-tworavens` describes how to install these components. :doc:`/admin/external-tools` explains how third-party tools like TwoRavens can be added to Dataverse. - Dropbox integration :ref:`dataverse.dropbox.key`: for uploading files from the Dropbox API. - Apache: a web server that can "reverse proxy" Glassfish applications and rewrite HTTP traffic. - Shibboleth: an authentication system described in :doc:`shibboleth`. Its use with Dataverse requires Apache. diff --git a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst index 4857e48dbe0..10e901e1ce5 100644 --- a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst +++ b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst @@ -343,7 +343,7 @@ TwoRavens and Dataverse!)* e. Enable TwoRavens Button in Dataverse --------------------------------------- -Now that you have installed TwoRavens, you can make it available to your users by adding it an "external tool" for your Dataverse installation. (For more on external tools in general, see the :doc:`external-tools` section.) +Now that you have installed TwoRavens, you can make it available to your users by adding it an "external tool" for your Dataverse installation. (For more on external tools in general, see the :doc:`/admin/external-tools` section of the Admin Guide.) First, download :download:`twoRavens.json <../_static/installation/files/root/external-tools/twoRavens.json>` as a starting point and edit ``toolUrl`` in that external tool manifest file to be the URL where you want TwoRavens to run. This is the URL reported by the installer script (as in the example at the end of step ``c.``, above). diff --git a/doc/sphinx-guides/source/user/account.rst b/doc/sphinx-guides/source/user/account.rst index dfb357697b6..eaa86567265 100755 --- a/doc/sphinx-guides/source/user/account.rst +++ b/doc/sphinx-guides/source/user/account.rst @@ -167,7 +167,7 @@ API Token What APIs Are and Why They Are Useful ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -API stands for "Application Programming Interface" and Dataverse APIs allow you to take advantage of integrations with other software that may have been set up by admins of your installation of Dataverse. See the :doc:`/admin/integrations` section of the Admin Guide and the :doc:`/installation/external-tools` section of the Installation Guide for examples of software that is commonly integrated with Dataverse. +API stands for "Application Programming Interface" and Dataverse APIs allow you to take advantage of integrations with other software that may have been set up by admins of your installation of Dataverse. See the :doc:`/admin/external-tools` and :doc:`/admin/integrations` sections of the Admin Guide for examples of software that is commonly integrated with Dataverse. Additionally, if you are willing to write a little code (or find someone to write it for you), APIs provide a way to automate parts of your workflow. See the :doc:`/api/getting-started` section of the API Guide for details. diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index 6df585f5821..3cdc9aec954 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -149,7 +149,7 @@ Certain file types in Dataverse are supported by additional functionality, which Tabular Data Files ------------------ -Files in certain formats - Stata, SPSS, R, Excel(xlsx) and CSV - may be ingested as tabular data (see :doc:`/user/tabulardataingest/index` section of the User Guide for details). Tabular data files can be further explored and manipulated with `TwoRavens <../user/data-exploration/tworavens.html>`_ - a statistical data exploration application integrated with Dataverse, as well as other :doc:`/installation/external-tools` if they have been enabled in the installation of Dataverse you are using. TwoRavens allows the user to run statistical models, view summary statistics, download subsets of variable vectors and more. To start, click on the "Explore" button, found next to each relevant tabular file (the application will be opened in a new window). Create and download your subset using `TwoRavens <../user/data-exploration/tworavens.html>`_. See the `TwoRavens documentation section <../user/data-exploration/tworavens.html>`_ for more information. +Files in certain formats - Stata, SPSS, R, Excel(xlsx) and CSV - may be ingested as tabular data (see :doc:`/user/tabulardataingest/index` section of the User Guide for details). Tabular data files can be further explored and manipulated with `TwoRavens <../user/data-exploration/tworavens.html>`_ - a statistical data exploration application integrated with Dataverse, as well as other :doc:`/admin/external-tools` if they have been enabled in the installation of Dataverse you are using. TwoRavens allows the user to run statistical models, view summary statistics, download subsets of variable vectors and more. To start, click on the "Explore" button, found next to each relevant tabular file (the application will be opened in a new window). Create and download your subset using `TwoRavens <../user/data-exploration/tworavens.html>`_. See the `TwoRavens documentation section <../user/data-exploration/tworavens.html>`_ for more information. Additional download options available for tabular data (found in the same drop-down menu under the "Download" button): diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index 212f154ea2f..2b513d7bb1c 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -96,7 +96,7 @@ Within the Files tab on a dataset page, you can download the files in that datas You may also download a file from its file page by clicking the Download button in the upper right corner of the page, or by :ref:`url_download` under the Metadata tab on the lower half of the page. -Tabular data files offer additional options: You can explore using any data exploration or visualization :doc:`/installation/external-tools` (if they have been enabled) by clicking the Explore button, or choose from a number of tabular-data-specific download options available as a dropdown under the Download button. +Tabular data files offer additional options: You can explore using any data exploration or visualization :doc:`/admin/external-tools` (if they have been enabled) by clicking the Explore button, or choose from a number of tabular-data-specific download options available as a dropdown under the Download button. Tabular Data ^^^^^^^^^^^^ From c89819ac591320b9b45118bd1711fa3ecb175e36 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 10 Sep 2019 17:47:56 -0400 Subject: [PATCH 18/23] move Building External Tools to API Guide #5028 --- doc/sphinx-guides/source/admin/external-tools.rst | 2 +- doc/sphinx-guides/source/{developers => api}/external-tools.rst | 0 doc/sphinx-guides/source/api/index.rst | 1 + doc/sphinx-guides/source/api/intro.rst | 2 +- doc/sphinx-guides/source/developers/index.rst | 1 - doc/sphinx-guides/source/developers/intro.rst | 2 +- doc/sphinx-guides/source/installation/external-tools.rst | 2 +- 7 files changed, 5 insertions(+), 5 deletions(-) rename doc/sphinx-guides/source/{developers => api}/external-tools.rst (100%) diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst index b735ec8d072..d4f859439d2 100644 --- a/doc/sphinx-guides/source/admin/external-tools.rst +++ b/doc/sphinx-guides/source/admin/external-tools.rst @@ -102,6 +102,6 @@ Configure tools at the dataset level are not currently supported. No button appe Writing Your Own External Tool ------------------------------ -If you plan to write a external tool, see the :doc:`/developers/external-tools` section of the Developer Guide. +If you plan to write a external tool, see the :doc:`/api/external-tools` section of the API Guide. If you have an idea for an external tool, please let the Dataverse community know by posting about it on the dataverse-community mailing list: https://groups.google.com/forum/#!forum/dataverse-community diff --git a/doc/sphinx-guides/source/developers/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst similarity index 100% rename from doc/sphinx-guides/source/developers/external-tools.rst rename to doc/sphinx-guides/source/api/external-tools.rst diff --git a/doc/sphinx-guides/source/api/index.rst b/doc/sphinx-guides/source/api/index.rst index 0c0c4c186a3..45cdb1918fe 100755 --- a/doc/sphinx-guides/source/api/index.rst +++ b/doc/sphinx-guides/source/api/index.rst @@ -19,5 +19,6 @@ API Guide metrics sword client-libraries + external-tools apps faq diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst index c5b0183f05a..b498a7c58b9 100755 --- a/doc/sphinx-guides/source/api/intro.rst +++ b/doc/sphinx-guides/source/api/intro.rst @@ -135,7 +135,7 @@ Developers of Integrations, External Tools, and Apps One of the primary purposes for Dataverse APIs in the first place is to enable integrations with third party software. Integrations are listed in the following places: - The :doc:`/admin/integrations` section of the Admin Guide. -- The :doc:`/developers/external-tools` section of the Developer Guide. +- The :doc:`/api/external-tools` section this guide. - The :doc:`apps` section of this guide. |Start| Good starting points are the three sections above to get a sense of third-party software that already integrates with Dataverse, followed by the :doc:`getting-started` section. diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index ffca75238ae..96595220e07 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -33,4 +33,3 @@ Developer Guide selinux big-data-support workflows - external-tools diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index 4901063ddbc..cddc0f97336 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -53,7 +53,7 @@ Related Projects As a developer, you also may be interested in these projects related to Dataverse: -- External Tools - add additional features to Dataverse without modifying the core: :doc:`external-tools` +- External Tools - add additional features to Dataverse without modifying the core: :doc:`/api/external-tools` - Dataverse API client libraries - use Dataverse APIs from various languages: :doc:`/api/client-libraries` - DVUploader - a stand-alone command-line Java application that uses the Dataverse API to support upload of files from local disk to a Dataset: https://github.com/IQSS/dataverse-uploader - dataverse-sample-data - populate your Dataverse installation with sample data: https://github.com/IQSS/dataverse-sample-data diff --git a/doc/sphinx-guides/source/installation/external-tools.rst b/doc/sphinx-guides/source/installation/external-tools.rst index 818020af9cb..028802fd736 100644 --- a/doc/sphinx-guides/source/installation/external-tools.rst +++ b/doc/sphinx-guides/source/installation/external-tools.rst @@ -19,4 +19,4 @@ See the :doc:`/admin/external-tools` section of the Admin Guide. Building External Tools ----------------------- -See the :doc:`/developers/external-tools` section of the Developer Guide. +See the :doc:`/api/external-tools` section of the API Guide. From 4897de43e7e4d74421a05ea9f8dd98a3a16ebde1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 12 Sep 2019 17:16:59 -0400 Subject: [PATCH 19/23] add lots more content for external tool makers #5028 --- .../external-tools/dynamicDatasetTool.json | 4 +- .../root/external-tools/fabulousFileTool.json | 4 +- .../source/api/external-tools.rst | 137 +++++++++++++----- doc/sphinx-guides/source/developers/intro.rst | 2 + 4 files changed, 109 insertions(+), 38 deletions(-) diff --git a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json index eec498acbff..2a9a888dddf 100644 --- a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json +++ b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json @@ -1,9 +1,9 @@ { "displayName": "Dynamic Dataset Tool", "description": "Dazzles! Dizzying!", - "type": "explore", "scope": "dataset", - "toolUrl": "https://dynamicdatasettool.com", + "type": "explore", + "toolUrl": "https://dynamicdatasettool.com/v2", "toolParameters": { "queryParameters": [ { diff --git a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json index ddbc460b694..e1dc0475d28 100644 --- a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json +++ b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json @@ -1,10 +1,10 @@ { "displayName": "Fabulous File Tool", "description": "Fabulous Fun for Files!", - "type": "explore", "scope": "file", - "contentType": "text/tab-separated-values", + "type": "explore", "toolUrl": "https://fabulousfiletool.com", + "contentType": "text/tab-separated-values", "toolParameters": { "queryParameters": [ { diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index 75da9e84773..7662200e061 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -6,10 +6,19 @@ External tools can provide additional features that are not part of Dataverse it .. contents:: |toctitle| :local: +Introduction +------------ + +You can think of a external tool as **a glorified hyperlink** that opens a browser window in a new tab on some other website. The term "external" is used to indicate that the user has left the Dataverse web interface. For example, perhaps the user is looking at a dataset on https://demo.dataverse.org . They click "Explore" and are brought to https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org + +The "other website" (fabulousfiletool.com in the example above) is probably part of the same ecosystem of scholarly publishing that Dataverse itself participates in. Sometimes the other website runs entirely in the browser. Sometimes the other website is a full blown server side web application like Dataverse itself. + +The possibilities for external tools are endless. Let's look at some examples to get your creative juices flowing. + Examples of External Tools -------------------------- -To get your creative juices flowing, here are examples of external tools that are available for Dataverse. +Note: This is the same list that appears in the :doc:`/admin/external-tools` section of the Admin Guide. .. csv-table:: :header: "Tool", "Type", "Scope", "Description" @@ -20,62 +29,122 @@ To get your creative juices flowing, here are examples of external tools that ar How External Tools Are Presented to Users ----------------------------------------- -See :ref:`testing-external-tools` for a description of how various tools are presented to users in the Dataverse web interface. - -How External Tools are Classified (Type Vs. Scope) --------------------------------------------------- - -External tools have a both a "type" and a "scope." Here are defintions for both terms: - -.. table:: - :widths: 10, 20, 70 - - ===== ==================== ========== - Term Possible values Definition - ===== ==================== ========== - type explore or configure The type determines if the tool appears under a button called "Explore" or a button called "Configure". Users must be logged in to Dataverse for configure tools which are expected to write back to Dataverse. Explore tools are typically read-only. - - scope file or dataset What "dvObject" the tool operates on. The third dvOjbect is "dataverse" but this is not supported. - ===== ==================== ========== +In short, an external tool appears under an "Explore" or "Configure" button either on a dataset landing page or a file landing page. See also the :ref:`testing-external-tools` section of the Admin Guide for some perspective on how installations of Dataverse will expect to test your tool before announcing it to their users. Creating an External Tool Manifest ---------------------------------- External tools must be expressed in an external tool manifest file, a specific JSON format Dataverse requires. As the author of an external tool, you are expected to provide this JSON file and installation instructions on a web page for your tool. -Let's look at two examples of external tool manifests before we dive into how they work. +Examples of Manifests ++++++++++++++++++++++ + +Let's look at two examples of external tool manifests (one at the file level and one at the dataset level) before we dive into how they work. + +External Tools for Files +^^^^^^^^^^^^^^^^^^^^^^^^ :download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` is a file level explore tool that operates on tabular files: .. literalinclude:: ../_static/installation/files/root/external-tools/fabulousFileTool.json +External Tools for Datasets +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + :download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` is a dataset level explore tool: .. literalinclude:: ../_static/installation/files/root/external-tools/dynamicDatasetTool.json -``type`` is required and must be ``explore`` or ``configure`` to make the tool appear under a button called "Explore" or "Configure", respectively. Not that a "Configure" button at the dataset level is not currently supported. +Terminology ++++++++++++ + +.. table:: + :widths: 20, 80 + + =========================== ========== + Term Definition + =========================== ========== + external tool manifest A **JSON file** the defines the URL constructed by Dataverse when users click "Explore" or "Configure" buttons. External tool makers are asked to host this JSON file on a website (no app store yet, sorry) and explain how to use install and use the tool. Examples include :download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` and :download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` as well as the real world examples above such as Data Explorer. + + displayName The **name** of the tool in the Dataverse web interface. For example, "Data Explorer". + + description The **description** of the tool, which appears in a popup (for configure tools only) so the user who clicked the tool can learn about the tool before being redirected the tool in a new tab in their browser. HTML is supported. + + scope Whether the external tool appears and operates at the **file** level or the **dataset** level. Note that a file level tool much also specify the type of file it operates on (see "contentType" below). + + type Whether the external tool is an **explore** tool or a **configure** tool. Configure tools require an API token because they make changes to data files (files within datasets). Configure tools are currently not supported at the dataset level (no "Configure" button appears in the GUI for datasets). + + toolUrl The **base URL** of the tool before query parameters are added. + + contentType File level tools operate on a specific **file type** (content type or MIME type). Dataset level tools do not use contentType. -``scope`` is required and must be ``file`` or ``dataset`` to make the tool appear at the file level or dataset level. + toolParameters **Query parameters** are supported and described below. -File level tools can operate on any file, including tabular files that have been created by successful ingestion. (For more on ingest, see the :doc:`/user/tabulardataingest/ingestprocess` of the User Guide.) The optional ``contentType`` entry specifies the mimetype a tool works on. + queryParameters **Key/value combinations** that can be appended to the toolUrl. For example, once substitution takes place (described below) the user may be redirected to ``https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org``. -In the example above, a mix of required and optional reserved words appear that can be used to insert dynamic values into tools. The supported values are: + query parameter keys An **arbitrary string** to associate with a value that is populated with a reserved word (described below). As the author of the tool, you have control over what "key" you would like to be passed to your tool. For example, if you want to have your tool receive and operate on the query parameter "dataverseFileId=42" instead of just "fileId=42", that's fine. -- ``{fileId}`` (required for file tools, or ``{filePid}``) - The database ID of a file from which the external tool has been launched. -- ``{filePid}`` (required for file tools, or ``{fileId}``) - The Persistent ID (DOI or Handle) of a file (if available) from which the external tool has been launched. -- ``{siteUrl}`` (optional) - The URL of the Dataverse installation from which the tool was launched. -- ``{apiToken}`` (optional) - The Dataverse API token of the user launching the external tool, if available. -- ``{datasetId}`` (required for dataset tools, or ``{datasetPid}``) - The database ID of the dataset. -- ``{datasetPid}`` (required for dataset tools, or ``{datasetId}``) - The Persistent ID (DOI or Handle) of the dataset. -- ``{datasetVersion}`` (optional) - The friendly version number ( or \:draft ) of the dataset version the tool is being launched from. + query parameter values A **mechanism for substituting reserved words with dynamic content**. For example, in your manifest file, you can use a reserved word (described below) such as ``{fileId}`` to pass a file's database id to your tool in a query parameter. Your tool might receive this query parameter as "fileId=42". -Getting Help With Writing Your Own External Tool ------------------------------------------------- + reserved words A **set of strings surrounded by curly braces** such as ``{fileId}`` or ``{datasetId}`` that will be inserted into query parameters. See the table below for a complete list. + =========================== ========== -If you need help with your tool, please feel free to post on the dataverse-dev mailing list: https://groups.google.com/forum/#!forum/dataverse-dev +Reserved Words +++++++++++++++ + +.. table:: + :widths: 15, 15, 70 + + =========================== ========== =========== + Reserved word Status Description + =========================== ========== =========== + ``{siteUrl}`` optional The URL of the Dataverse installation from which the tool was launched. For example, ``https://demo.dataverse.org``. + + ``{fileId}`` depends The database ID of a file the user clicks "Explore" or "Configure" on. For example, ``42``. This reserved word is **required for file level tools** unless you use ``{filePid}`` instead. + + ``{filePid}`` depends The Persistent ID (DOI or Handle) of a file the user clicks "Explore" or "Configure" on. For example, ``doi:10.7910/DVN/TJCLKP/3VSTKY``. Note that not all installations of Dataverse have Persistent IDs (PIDs) enabled at the file level. This reserved word is **required for file level tools** unless you use ``{fileId}`` instead. + + ``{apiToken}`` optional The Dataverse API token of the user launching the external tool, if available. Please note that API tokens should be treated with the same care as a password. For example, ``f3465b0c-f830-4bc7-879f-06c0745a5a5c``. + + ``{datasetId}`` depends The database ID of the dataset. For example, ``42``. This reseved word is **required for dataset level tools** unless you use ``{datasetPid}`` instead. + + ``{datasetPid}`` depends The Persistent ID (DOI or Handle) of the dataset. For example, ``doi:10.7910/DVN/TJCLKP``. This reseved word is **required for dataset level tools** unless you use ``{datasetId}`` instead. + + ``{datasetVersion}`` optional The friendly version number ( or \:draft ) of the dataset version the tool is being launched from. For example, ``1.0`` or ``:draft``. + =========================== ========== =========== + +Using Example Manifests to Get Started +++++++++++++++++++++++++++++++++++++++ + +Again, you can use :download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` or :download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` as a starting point for your own manifest file. + +Testing Your External Tool +-------------------------- + +As the author of an external tool, you are not expected to learn how to install and operate Dataverse. There's a very good chance your tool can be added to a server Dataverse developers use for testing if you reach out on any of the channels listed under :ref:`getting-help-developers` in the Developer Guide. + +By all means, if you'd like to install Dataverse yourself, a number of developer-centric options are available. For example, there's a script to spin up Dataverse on EC2 at https://github.com/IQSS/dataverse-sample-data . The process for using curl to add your external tool to a Dataverse installation is documented under :ref:`managing-external-tools` in the Admin Guide. + +Spreading the Word About Your External Tool +------------------------------------------- + +Adding Your Tool to the Inventory of External Tools ++++++++++++++++++++++++++++++++++++++++++++++++++++ Once you've gotten your tool working, please make a pull request to update the list of tools above! You are also welcome to download :download:`dataverse-external-tools.tsv <../_static/admin/dataverse-external-tools.tsv>`, add your tool to the TSV file, create and issue at https://github.com/IQSS/dataverse/issues , and then upload your TSV file there. +Unless your tool runs entirely in a browser, you may have integrated server-side software with Dataverse. If so, please double check that your software is listed in the :doc:`/admin/integrations` section of the Admin Guide and if not, please open an issue or pull request to add it. Thanks! + +If you've thought to yourself that there ought to be an app store for Dataverse external tools, you're not alone. Please see https://github.com/IQSS/dataverse/issues/5688 :) + +Demoing Your External Tool +++++++++++++++++++++++++++ + +https://demo.dataverse.org is the place to play around with Dataverse and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/IQSS/dataverse-ansible which already includes a number of the tools listed above. + +Announcing Your External Tool ++++++++++++++++++++++++++++++ + You are welcome to announce your external tool at https://groups.google.com/forum/#!forum/dataverse-community -Unless your tool runs entirely in a browser, you may have integrated server-side software with Dataverse. If so, please double check that your software is listed in the :doc:`/admin/integrations` section of the Admin Guide and if not, pleas open an issue or pull request to add it. Thanks! +If you're too shy, we'll do it for you. We'll probably tweet about it too. Thank you for your contribution to Dataverse! diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index cddc0f97336..f95e6a97cf8 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -14,6 +14,8 @@ This guide is intended primarily for developers who want to work on the main Dat To get started, you'll want to set up your :doc:`dev-environment` and make sure you understand the branching strategy described in the :doc:`version-control` section and how to make a pull request. :doc:`testing` is expected. Opinions about :doc:`coding-style` are welcome! +.. _getting-help-developers: + Getting Help ------------ From 7828dea911c88c746a5bad5aab5d288f54e4789a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 12 Sep 2019 17:30:11 -0400 Subject: [PATCH 20/23] ignore python virutal environments (venv) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1f518e762f8..2904bc578f2 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,4 @@ scripts/installer/default.config # ignore UI testing related files tests/node_modules tests/package-lock.json +venv From 6dd0cfc0849778f426273dc6d922731493ec5736 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 12 Sep 2019 17:41:15 -0400 Subject: [PATCH 21/23] rename flyway script # 5028 --- ...28-dataset-explore.sql => V4.16.0.2__5028-dataset-explore.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V4.16.1__5028-dataset-explore.sql => V4.16.0.2__5028-dataset-explore.sql} (100%) diff --git a/src/main/resources/db/migration/V4.16.1__5028-dataset-explore.sql b/src/main/resources/db/migration/V4.16.0.2__5028-dataset-explore.sql similarity index 100% rename from src/main/resources/db/migration/V4.16.1__5028-dataset-explore.sql rename to src/main/resources/db/migration/V4.16.0.2__5028-dataset-explore.sql From 519662a7f93734e58baf64a0aacb12eb55e56fae Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 12 Sep 2019 18:08:59 -0400 Subject: [PATCH 22/23] get deployment working again for new installations #6165 --- .../migration/V4.16.0.1__5303-addColumn-to-settingTable.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/resources/db/migration/V4.16.0.1__5303-addColumn-to-settingTable.sql b/src/main/resources/db/migration/V4.16.0.1__5303-addColumn-to-settingTable.sql index d294afe8b56..8309dacf486 100644 --- a/src/main/resources/db/migration/V4.16.0.1__5303-addColumn-to-settingTable.sql +++ b/src/main/resources/db/migration/V4.16.0.1__5303-addColumn-to-settingTable.sql @@ -1,8 +1,8 @@ ALTER TABLE ONLY setting DROP CONSTRAINT setting_pkey ; -ALTER TABLE setting ADD COLUMN ID SERIAL PRIMARY KEY; +ALTER TABLE setting ADD COLUMN IF NOT EXISTS ID SERIAL PRIMARY KEY; -ALTER TABLE setting ADD COLUMN lang text; +ALTER TABLE setting ADD COLUMN IF NOT EXISTS lang text; ALTER TABLE setting ADD CONSTRAINT non_empty_lang @@ -10,4 +10,4 @@ ALTER TABLE setting CREATE UNIQUE INDEX unique_settings ON setting - (name, coalesce(lang, '')); \ No newline at end of file + (name, coalesce(lang, '')); From 06105e85135f35da9a19ee17b5b2c4fc0a94c065 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 12 Sep 2019 18:30:24 -0400 Subject: [PATCH 23/23] move testing methods to /api/admin/test #5028 #4137 --- .../source/admin/external-tools.rst | 4 - doc/sphinx-guides/source/api/native-api.rst | 54 -------------- .../harvard/iq/dataverse/api/Datasets.java | 38 ---------- .../edu/harvard/iq/dataverse/api/Files.java | 31 -------- .../edu/harvard/iq/dataverse/api/TestApi.java | 74 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 4 +- 6 files changed, 76 insertions(+), 129 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/api/TestApi.java diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst index d4f859439d2..6054334e4a6 100644 --- a/doc/sphinx-guides/source/admin/external-tools.rst +++ b/doc/sphinx-guides/source/admin/external-tools.rst @@ -80,8 +80,6 @@ File level explore tools are specific to the file type (content type or MIME typ An "Explore" button will appear (on both the dataset page and the file landing page) for files that match the type that the tool has been built for. When there are multiple explore tools for a filetype, the button becomes a dropdown. -For a specific file, you can check how many tools are available using the GUI as described above or by using the API as described at :ref:`list-external-tools-for-a-file-api`. - File Level Configure Tools ++++++++++++++++++++++++++ @@ -92,8 +90,6 @@ Dataset Level Explore Tools When a dataset level explore tool is added, an "Explore" button on the dataset page will appear. This button becomes a drop down when there are multiple tools. -You can also use the API to list dataset level tools. See :ref:`list-external-tools-for-a-dataset-api` - Dataset Level Configure Tools +++++++++++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 194dfe77daf..bdcde0476ff 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -868,33 +868,6 @@ Normally published datasets should not be deleted, but there exists a "destroy" Calling the destroy endpoint is permanent and irreversible. It will remove the dataset and its datafiles, then re-index the parent dataverse in Solr. This endpoint requires the API token of a superuser. -.. _list-external-tools-for-a-dataset-api: - -List External Tools for a Dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Some installations of Dataverse have external tools enabled for datasets and there are two types: - -- explore -- configure - -.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. - -.. code-block:: bash - - export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export SERVER_URL=https://demo.dataverse.org - export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB - export TYPE=explore - - curl -H X-Dataverse-key:$API_TOKEN \""$SERVER_URL/api/datasets/:persistentId/externalTools?persistentId=$PERSISTENT_ID&type=$TYPE"\" - -The fully expanded example above (without environment variables) looks like this: - -.. code-block:: bash - - curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx "https://demo.dataverse.org/api/datasets/:persistentId/externalTools?persistentId=doi:10.5072/FK2/J8SJZB&type=explore" - Files ----- @@ -1050,33 +1023,6 @@ Starting the release 4.10 the size of the saved original file (for an ingested t Note the optional "limit" parameter. Without it, the API will attempt to populate the sizes for all the saved originals that don't have them in the database yet. Otherwise it will do so for the first N such datafiles. -.. _list-external-tools-for-a-file-api: - -List External Tools for a File -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Some installations of Dataverse have external tools enabled for files and there are two types: - -- explore -- configure - -.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. - -.. code-block:: bash - - export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export SERVER_URL=https://demo.dataverse.org - export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB - export TYPE=explore - - curl -H X-Dataverse-key:$API_TOKEN \""$SERVER_URL/api/files/:persistentId/externalTools?persistentId=$PERSISTENT_ID&type=$TYPE"\" - -The fully expanded example above (without environment variables) looks like this: - -.. code-block:: bash - - curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx "https://demo.dataverse.org/api/files/:persistentId/externalTools?persistentId=doi:10.5072/FK2/J8SJZB&type=explore" - Builtin Users ------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index a51ac707a77..311e746a444 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -77,16 +77,10 @@ import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.S3PackageImporter; import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; -import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; -import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.engine.command.exception.UnforcedCommandException; -import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataFileCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDvObjectPIDMetadataCommand; -import edu.harvard.iq.dataverse.externaltools.ExternalTool; -import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitations; import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitationsServiceBean; import edu.harvard.iq.dataverse.makedatacount.DatasetMetrics; @@ -101,26 +95,20 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.search.IndexServiceBean; -import edu.harvard.iq.dataverse.util.DateUtil; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import java.io.IOException; import java.io.InputStream; import java.io.StringReader; -import java.math.BigDecimal; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.ResourceBundle; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -1896,31 +1884,5 @@ public Response getMakeDataCountMetric(@PathParam("id") String idSupplied, @Path } } - @GET - @Path("{id}/externalTools") - public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam("type") String typeSupplied) { - ExternalTool.Type type; - try { - type = ExternalTool.Type.fromString(typeSupplied); - } catch (IllegalArgumentException ex) { - return error(BAD_REQUEST, ex.getLocalizedMessage()); - } - Dataset dataset; - try { - dataset = findDatasetOrDie(idSupplied); - JsonArrayBuilder tools = Json.createArrayBuilder(); - List datasetTools = externalToolService.findDatasetToolsByType(type); - for (ExternalTool tool : datasetTools) { - ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey()); - ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataset, apiToken); - JsonObjectBuilder toolToJson = externalToolService.getToolAsJsonWithQueryParameters(externalToolHandler); - tools.add(toolToJson); - } - return ok(tools); - } catch (WrappedResponse wr) { - return wr.getResponse(); - } - } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 489d29a5060..8948baec791 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -14,7 +14,6 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.UserNotificationServiceBean; import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; -import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper; @@ -32,8 +31,6 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.export.ExportException; import edu.harvard.iq.dataverse.export.ExportService; -import edu.harvard.iq.dataverse.externaltools.ExternalTool; -import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; import edu.harvard.iq.dataverse.ingest.IngestRequest; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; @@ -51,9 +48,6 @@ import java.util.logging.Logger; import javax.ejb.EJB; import javax.inject.Inject; -import javax.json.Json; -import javax.json.JsonArrayBuilder; -import javax.json.JsonObjectBuilder; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; @@ -621,29 +615,4 @@ private void exportDatasetMetadata(SettingsServiceBean settingsServiceBean, Data } } - @Path("{id}/externalTools") - @GET - public Response getExternalTools(@PathParam("id") String idSupplied, @QueryParam("type") String typeSupplied) { - ExternalTool.Type type; - try { - type = ExternalTool.Type.fromString(typeSupplied); - } catch (IllegalArgumentException ex) { - return error(BAD_REQUEST, ex.getLocalizedMessage()); - } - try { - DataFile dataFile = findDataFileOrDie(idSupplied); - JsonArrayBuilder tools = Json.createArrayBuilder(); - List datasetTools = externalToolService.findFileToolsByTypeAndContentType(type, dataFile.getContentType()); - for (ExternalTool tool : datasetTools) { - ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey()); - ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataFile, apiToken, dataFile.getFileMetadata()); - JsonObjectBuilder toolToJson = externalToolService.getToolAsJsonWithQueryParameters(externalToolHandler); - tools.add(toolToJson); - } - return ok(tools); - } catch (WrappedResponse wr) { - return wr.getResponse(); - } - } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/TestApi.java b/src/main/java/edu/harvard/iq/dataverse/api/TestApi.java new file mode 100644 index 00000000000..f2379276699 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/TestApi.java @@ -0,0 +1,74 @@ +package edu.harvard.iq.dataverse.api; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; +import edu.harvard.iq.dataverse.externaltools.ExternalTool; +import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; +import java.util.List; +import javax.json.Json; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObjectBuilder; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Response; +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; + +@Path("admin/test") +public class TestApi extends AbstractApiBean { + + @GET + @Path("datasets/{id}/externalTools") + public Response getExternalToolsforFile(@PathParam("id") String idSupplied, @QueryParam("type") String typeSupplied) { + ExternalTool.Type type; + try { + type = ExternalTool.Type.fromString(typeSupplied); + } catch (IllegalArgumentException ex) { + return error(BAD_REQUEST, ex.getLocalizedMessage()); + } + Dataset dataset; + try { + dataset = findDatasetOrDie(idSupplied); + JsonArrayBuilder tools = Json.createArrayBuilder(); + List datasetTools = externalToolService.findDatasetToolsByType(type); + for (ExternalTool tool : datasetTools) { + ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey()); + ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataset, apiToken); + JsonObjectBuilder toolToJson = externalToolService.getToolAsJsonWithQueryParameters(externalToolHandler); + tools.add(toolToJson); + } + return ok(tools); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } + + @Path("files/{id}/externalTools") + @GET + public Response getExternalToolsForFile(@PathParam("id") String idSupplied, @QueryParam("type") String typeSupplied) { + ExternalTool.Type type; + try { + type = ExternalTool.Type.fromString(typeSupplied); + } catch (IllegalArgumentException ex) { + return error(BAD_REQUEST, ex.getLocalizedMessage()); + } + try { + DataFile dataFile = findDataFileOrDie(idSupplied); + JsonArrayBuilder tools = Json.createArrayBuilder(); + List datasetTools = externalToolService.findFileToolsByTypeAndContentType(type, dataFile.getContentType()); + for (ExternalTool tool : datasetTools) { + ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey()); + ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataFile, apiToken, dataFile.getFileMetadata()); + JsonObjectBuilder toolToJson = externalToolService.getToolAsJsonWithQueryParameters(externalToolHandler); + tools.add(toolToJson); + } + return ok(tools); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9ecdb8d0f20..840d77d4426 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1778,7 +1778,7 @@ static Response getExternalToolsForDataset(String idOrPersistentIdOfDataset, Str requestSpecification = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); } - return requestSpecification.get("/api/datasets/" + idInPath + "/externalTools?type=" + type + optionalQueryParam); + return requestSpecification.get("/api/admin/test/datasets/" + idInPath + "/externalTools?type=" + type + optionalQueryParam); } static Response getExternalToolsForFile(String idOrPersistentIdOfFile, String type, String apiToken) { @@ -1793,7 +1793,7 @@ static Response getExternalToolsForFile(String idOrPersistentIdOfFile, String ty requestSpecification = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); } - return requestSpecification.get("/api/files/" + idInPath + "/externalTools?type=" + type + optionalQueryParam); + return requestSpecification.get("/api/admin/test/files/" + idInPath + "/externalTools?type=" + type + optionalQueryParam); } static Response submitFeedback(JsonObjectBuilder job) {