diff --git a/doc/release-notes/9002_allow_direct_upload_setting.md b/doc/release-notes/9002_allow_direct_upload_setting.md new file mode 100644 index 00000000000..1e76ed4ad47 --- /dev/null +++ b/doc/release-notes/9002_allow_direct_upload_setting.md @@ -0,0 +1,5 @@ +A Dataverse installation can be now be configured to allow out-of-band upload by setting the `dataverse.files..upload-out-of-band` JVM option to `true`. + +By default, Dataverse supports uploading files via the [add a file to a dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server). + +With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store. diff --git a/doc/release-notes/9589-ds-configure-tool.md b/doc/release-notes/9589-ds-configure-tool.md new file mode 100644 index 00000000000..70ac5fcaa6a --- /dev/null +++ b/doc/release-notes/9589-ds-configure-tool.md @@ -0,0 +1 @@ +Configure tools are now available at the dataset level. They appear under the "Edit Dataset" menu. See also #9589. diff --git a/doc/release-notes/9599-guestbook-at-request.md b/doc/release-notes/9599-guestbook-at-request.md new file mode 100644 index 00000000000..e9554b71fb4 --- /dev/null +++ b/doc/release-notes/9599-guestbook-at-request.md @@ -0,0 +1,2 @@ +Dataverse can now be configured (via the dataverse.files.guestbook-at-request option) to display any configured guestbook to users when they request restricted file(s) or when they download files (the historic default). +The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions. diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst index 67075e986bb..346ca0b15ee 100644 --- a/doc/sphinx-guides/source/admin/external-tools.rst +++ b/doc/sphinx-guides/source/admin/external-tools.rst @@ -115,7 +115,7 @@ Dataset level explore tools allow the user to explore all the files in a dataset Dataset Level Configure Tools +++++++++++++++++++++++++++++ -Configure tools at the dataset level are not currently supported. +Dataset level configure tools can be launched by users who have edit access to the dataset. These tools are found under the "Edit Dataset" menu. Writing Your Own External Tool ------------------------------ diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index 05affaf975e..d12e4b17549 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -40,7 +40,7 @@ How External Tools Are Presented to Users An external tool can appear in your Dataverse installation in a variety of ways: - as an explore, preview, query or configure option for a file -- as an explore option for a dataset +- as an explore or configure option for a dataset - as an embedded preview on the file landing page See also the :ref:`testing-external-tools` section of the Admin Guide for some perspective on how Dataverse installations will expect to test your tool before announcing it to their users. @@ -88,11 +88,11 @@ Terminology displayName The **name** of the tool in the Dataverse installation web interface. For example, "Data Explorer". - description The **description** of the tool, which appears in a popup (for configure tools only) so the user who clicked the tool can learn about the tool before being redirected the tool in a new tab in their browser. HTML is supported. + description The **description** of the tool, which appears in a popup (for configure tools only) so the user who clicked the tool can learn about the tool before being redirected to the tool in a new tab in their browser. HTML is supported. scope Whether the external tool appears and operates at the **file** level or the **dataset** level. Note that a file level tool much also specify the type of file it operates on (see "contentType" below). - types Whether the external tool is an **explore** tool, a **preview** tool, a **query** tool, a **configure** tool or any combination of these (multiple types are supported for a single tool). Configure tools require an API token because they make changes to data files (files within datasets). Configure tools are currently not supported at the dataset level. The older "type" keyword that allows you to pass a single type as a string is deprecated but still supported. + types Whether the external tool is an **explore** tool, a **preview** tool, a **query** tool, a **configure** tool or any combination of these (multiple types are supported for a single tool). Configure tools require an API token because they make changes to data files (files within datasets). The older "type" keyword that allows you to pass a single type as a string is deprecated but still supported. toolUrl The **base URL** of the tool before query parameters are added. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 8cc2a127fe3..cf869d338ca 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -525,10 +525,16 @@ Submit Incomplete Dataset ^^^^^^^^^^^^^^^^^^^^^^^^^ **Note:** This feature requires :ref:`dataverse.api.allow-incomplete-metadata` to be enabled and your Solr -Schema to be up-to-date with the ``datasetValid`` field. +Schema to be up-to-date with the ``datasetValid`` field. If not done yet with the version upgrade, you will +also need to reindex all dataset after enabling the :ref:`dataverse.api.allow-incomplete-metadata` feature. Providing a ``.../datasets?doNotValidate=true`` query parameter turns off the validation of metadata. -In this case, only the "Author Name" is required. For example, a minimal JSON file would look like this: +In this situation, only the "Author Name" is required, except for the case when the setting :ref:`:MetadataLanguages` +is configured and the value of "Dataset Metadata Language" setting of a collection is left with the default +"Chosen at Dataset Creation" value. In that case, a language that is a part of the :ref:`:MetadataLanguages` list must be +declared in the incomplete dataset. + +For example, a minimal JSON file, without the language specification, would look like this: .. code-block:: json :name: dataset-incomplete.json @@ -2322,6 +2328,51 @@ See :ref:`:CustomDatasetSummaryFields` in the Installation Guide for how the lis curl "$SERVER_URL/api/datasets/summaryFieldNames" +.. _guestbook-at-request-api: + +Configure When a Dataset Guestbook Appears (If Enabled) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, users are asked to fill out a configured Guestbook when they down download files from a dataset. If enabled for a given Dataverse instance (see XYZ), users may instead be asked to fill out a Guestbook only when they request access to restricted files. +This is configured by a global default, collection-level settings, or directly at the dataset level via these API calls (superuser access is required to make changes). + +To see the current choice for this dataset: + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG + + curl "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER" + + + The response will be true (guestbook displays when making a request), false (guestbook displays at download), or will indicate that the dataset inherits one of these settings. + +To set the behavior for this dataset: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG + + curl -X PUT -H "X-Dataverse-key:$API_TOKEN" -H Content-type:application/json -d true "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER" + + + This example uses true to set the behavior to guestbook at request. Note that this call will return a 403/Forbidden response if guestbook at request functionality is not enabled for this Dataverse instance. + +The API can also be used to reset the dataset to use the default/inherited value: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG + + curl -X DELETE -H "X-Dataverse-key:$API_TOKEN" -H Content-type:application/json "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER" + + + Files ----- diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst index 4d323455d28..4bf2bbdcc79 100644 --- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -115,7 +115,7 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA" -Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. +Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide. With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above. To add multiple Uploaded Files to the Dataset @@ -146,7 +146,7 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/addFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA" -Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. +Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide. With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above. @@ -176,7 +176,7 @@ Note that the API call does not validate that the file matches the hash value su curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/files/$FILE_IDENTIFIER/replace" -F "jsonData=$JSON_DATA" -Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. +Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide. With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above. Replacing multiple existing files in the Dataset @@ -274,5 +274,5 @@ The JSON object returned as a response from this API call includes a "data" that } -Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. +Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide. With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index bf7f8fa168c..47066aebb10 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -508,6 +508,10 @@ A Dataverse installation can alternately store files in a Swift or S3-compatible A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web-accessible trusted remote store. +A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\.upload-out-of-band`` JVM option to ``true``. +By default, Dataverse supports uploading files via the :ref:`add-file-api`. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server). +With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the :ref:`Adding the Uploaded file to the Dataset ` API call (described in the :doc:`/developers/s3-direct-upload-api` page) used to add metadata and inform Dataverse that a new file has been added to the relevant store. + The following sections describe how to set up various types of stores and how to configure for multiple stores. Multi-store Basics @@ -800,27 +804,28 @@ List of S3 Storage Options .. table:: :align: left - =========================================== ================== ========================================================================== ============= - JVM Option Value Description Default value - =========================================== ================== ========================================================================== ============= - dataverse.files.storage-driver-id Enable as the default storage driver. ``file`` - dataverse.files..type ``s3`` **Required** to mark this storage as S3 based. (none) - dataverse.files..label **Required** label to be shown in the UI for this storage (none) - dataverse.files..bucket-name The bucket name. See above. (none) - dataverse.files..download-redirect ``true``/``false`` Enable direct download or proxy through Dataverse. ``false`` - dataverse.files..upload-redirect ``true``/``false`` Enable direct upload of files added to a dataset to the S3 store. ``false`` - dataverse.files..ingestsizelimit Maximum size of directupload files that should be ingested (none) - dataverse.files..url-expiration-minutes If direct uploads/downloads: time until links expire. Optional. 60 - dataverse.files..min-part-size Multipart direct uploads will occur for files larger than this. Optional. ``1024**3`` - dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none) - dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse`` - dataverse.files..profile Allows the use of AWS profiles for storage spanning multiple AWS accounts. (none) - dataverse.files..proxy-url URL of a proxy protecting the S3 store. Optional. (none) - dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false`` - dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` - dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` - dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` - =========================================== ================== ========================================================================== ============= + =========================================== ================== =================================================================================== ============= + JVM Option Value Description Default value + =========================================== ================== =================================================================================== ============= + dataverse.files.storage-driver-id Enable as the default storage driver. ``file`` + dataverse.files..type ``s3`` **Required** to mark this storage as S3 based. (none) + dataverse.files..label **Required** label to be shown in the UI for this storage (none) + dataverse.files..bucket-name The bucket name. See above. (none) + dataverse.files..download-redirect ``true``/``false`` Enable direct download or proxy through Dataverse. ``false`` + dataverse.files..upload-redirect ``true``/``false`` Enable direct upload of files added to a dataset in the S3 store. ``false`` + dataverse.files..upload-out-of-band ``true``/``false`` Allow upload of files by out-of-band methods (using some tool other than Dataverse) ``false`` + dataverse.files..ingestsizelimit Maximum size of directupload files that should be ingested (none) + dataverse.files..url-expiration-minutes If direct uploads/downloads: time until links expire. Optional. 60 + dataverse.files..min-part-size Multipart direct uploads will occur for files larger than this. Optional. ``1024**3`` + dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none) + dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse`` + dataverse.files..profile Allows the use of AWS profiles for storage spanning multiple AWS accounts. (none) + dataverse.files..proxy-url URL of a proxy protecting the S3 store. Optional. (none) + dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false`` + dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` + dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` + dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` + =========================================== ================== =================================================================================== ============= .. table:: :align: left @@ -2450,6 +2455,17 @@ This setting was added to keep S3 direct upload lightweight. When that feature i See also :ref:`s3-direct-upload-features-disabled`. +.. _dataverse.files.guestbook-at-request: + +dataverse.files.guestbook-at-request +++++++++++++++++++++++++++++++++++++ + +This setting enables functionality to allow guestbooks to be displayed when a user requests access to a restricted data file(s) or when a file is downloaded (the historic default). Providing a true/false value for this setting enables the functionality and provides a global default. The behavior can also be changed at the collection level via the user interface and by a superuser for a give dataset using the API. + +See also :ref:`guestbook-at-request-api` in the API Guide, and . + +Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_GUESTBOOK_AT_REQUEST``. + .. _feature-flags: Feature Flags diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index b5e8d8f4fc9..3d301619105 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -25,6 +25,8 @@ Creating a Dataverse collection is easy but first you must be a registered user * **Category**: Select a category that best describes the type of Dataverse collection this will be. For example, if this is a Dataverse collection for an individual researcher's datasets, select *Researcher*. If this is a Dataverse collection for an institution, select *Organization or Institution*. * **Email**: This is the email address that will be used as the contact for this particular Dataverse collection. You can have more than one contact email address for your Dataverse collection. * **Description**: Provide a description of this Dataverse collection. This will display on the landing page of your Dataverse collection and in the search result list. The description field supports certain HTML tags, if you'd like to format your text (, ,
,
, , ,
,
,
, ,
,

-

, , , ,
  • ,
      ,

      ,

      , , , , , , , 
        ). + * **Dataset Metadata Langauge**: (If enabled) Select which language should be used when entering dataset metadata, or leave that choice to dataset creators. + * **Guestbook Entry Option**: (If enabled) Select whether guestbooks are displayed when a user requests access to restricted file(s) or when they initiate a download. #. **Choose the sets of Metadata Fields for datasets in this Dataverse collection**: * By default the metadata elements will be from the host Dataverse collection that this new Dataverse collection is created in. * The Dataverse Software offers metadata standards for multiple domains. To learn more about the metadata standards in the Dataverse Software please check out the :doc:`/user/appendix`. diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index c023782fb88..44c20ad95ef 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -198,7 +198,7 @@ 1.7.0 - 0.43.0 + 0.43.4 5.0.0 diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index 0f83ae3c5c8..ba13729793e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -18,7 +18,6 @@ import edu.harvard.iq.dataverse.util.ShapefileHandler; import edu.harvard.iq.dataverse.util.StringUtil; import java.io.IOException; -import java.util.Date; import java.util.List; import java.util.ArrayList; import java.util.Objects; @@ -33,7 +32,7 @@ import jakarta.json.JsonArrayBuilder; import jakarta.persistence.*; import jakarta.validation.constraints.Pattern; -import org.hibernate.validator.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; /** * @@ -53,9 +52,9 @@ }) @Entity @Table(indexes = {@Index(columnList="ingeststatus") - , @Index(columnList="checksumvalue") - , @Index(columnList="contenttype") - , @Index(columnList="restricted")}) + , @Index(columnList="checksumvalue") + , @Index(columnList="contenttype") + , @Index(columnList="restricted")}) public class DataFile extends DvObject implements Comparable { private static final Logger logger = Logger.getLogger(DatasetPage.class.getCanonicalName()); private static final long serialVersionUID = 1L; @@ -73,10 +72,6 @@ public class DataFile extends DvObject implements Comparable { @Pattern(regexp = "^.*/.*$", message = "{contenttype.slash}") private String contentType; - public void setFileAccessRequests(List fileAccessRequests) { - this.fileAccessRequests = fileAccessRequests; - } - // @Expose // @SerializedName("storageIdentifier") // @Column( nullable = false ) @@ -200,6 +195,28 @@ public String toString() { @OneToMany(mappedBy="dataFile", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}) private List guestbookResponses; + @OneToMany(mappedBy="dataFile",fetch = FetchType.LAZY,cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST, CascadeType.REFRESH}) + private List fileAccessRequests; + + @ManyToMany + @JoinTable(name = "fileaccessrequests", + joinColumns = @JoinColumn(name = "datafile_id"), + inverseJoinColumns = @JoinColumn(name = "authenticated_user_id")) + private List fileAccessRequesters; + + + public List getFileAccessRequests(){ + return fileAccessRequests; + } + + public List getFileAccessRequests(FileAccessRequest.RequestState state){ + return fileAccessRequests.stream().filter(far -> far.getState() == state).collect(Collectors.toList()); + } + + public void setFileAccessRequests(List fARs){ + this.fileAccessRequests = fARs; + } + public List getGuestbookResponses() { return guestbookResponses; } @@ -750,50 +767,38 @@ public String getUnf() { return null; } - @OneToMany(mappedBy = "dataFile", cascade = {CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}, orphanRemoval = true) - private List fileAccessRequests; + public List getFileAccessRequesters() { + return fileAccessRequesters; + } - public List getFileAccessRequests() { - return fileAccessRequests; + public void setFileAccessRequesters(List fileAccessRequesters) { + this.fileAccessRequesters = fileAccessRequesters; } - public void addFileAccessRequester(AuthenticatedUser authenticatedUser) { + + public void addFileAccessRequest(FileAccessRequest request) { if (this.fileAccessRequests == null) { this.fileAccessRequests = new ArrayList<>(); } - Set existingUsers = this.fileAccessRequests.stream() - .map(FileAccessRequest::getAuthenticatedUser) - .collect(Collectors.toSet()); + this.fileAccessRequests.add(request); + } - if (existingUsers.contains(authenticatedUser)) { - return; + public FileAccessRequest getAccessRequestForAssignee(RoleAssignee roleAssignee) { + if (this.fileAccessRequests == null) { + return null; } - FileAccessRequest request = new FileAccessRequest(); - request.setCreationTime(new Date()); - request.setDataFile(this); - request.setAuthenticatedUser(authenticatedUser); - - FileAccessRequest.FileAccessRequestKey key = new FileAccessRequest.FileAccessRequestKey(); - key.setAuthenticatedUser(authenticatedUser.getId()); - key.setDataFile(this.getId()); - - request.setId(key); - - this.fileAccessRequests.add(request); + return this.fileAccessRequests.stream() + .filter(fileAccessRequest -> fileAccessRequest.getRequester().equals(roleAssignee) && fileAccessRequest.isStateCreated()).findFirst() + .orElse(null); } - public boolean removeFileAccessRequester(RoleAssignee roleAssignee) { + public boolean removeFileAccessRequest(FileAccessRequest request) { if (this.fileAccessRequests == null) { return false; } - FileAccessRequest request = this.fileAccessRequests.stream() - .filter(fileAccessRequest -> fileAccessRequest.getAuthenticatedUser().equals(roleAssignee)) - .findFirst() - .orElse(null); - if (request != null) { this.fileAccessRequests.remove(request); return true; @@ -802,13 +807,13 @@ public boolean removeFileAccessRequester(RoleAssignee roleAssignee) { return false; } - public boolean containsFileAccessRequestFromUser(RoleAssignee roleAssignee) { + public boolean containsActiveFileAccessRequestFromUser(RoleAssignee roleAssignee) { if (this.fileAccessRequests == null) { return false; } - Set existingUsers = this.fileAccessRequests.stream() - .map(FileAccessRequest::getAuthenticatedUser) + Set existingUsers = getFileAccessRequests(FileAccessRequest.RequestState.CREATED).stream() + .map(FileAccessRequest::getRequester) .collect(Collectors.toSet()); return existingUsers.contains(roleAssignee); @@ -975,8 +980,6 @@ public String toJSON(){ public JsonObject asGsonObject(boolean prettyPrint){ - String overarchingKey = "data"; - GsonBuilder builder; if (prettyPrint){ // Add pretty printing builder = new GsonBuilder().excludeFieldsWithoutExposeAnnotation().setPrettyPrinting(); diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 98ee3351458..332a39912d2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.dataaccess.StorageIO; @@ -9,15 +10,19 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileSortFieldAndOrder; import edu.harvard.iq.dataverse.util.FileUtil; +import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; @@ -59,6 +64,8 @@ public class DataFileServiceBean implements java.io.Serializable { @EJB EmbargoServiceBean embargoService; + @EJB SystemConfig systemConfig; + @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; @@ -132,6 +139,39 @@ public class DataFileServiceBean implements java.io.Serializable { */ public static final String MIME_TYPE_PACKAGE_FILE = "application/vnd.dataverse.file-package"; + public class UserStorageQuota { + private Long totalAllocatedInBytes = 0L; + private Long totalUsageInBytes = 0L; + + public UserStorageQuota(Long allocated, Long used) { + this.totalAllocatedInBytes = allocated; + this.totalUsageInBytes = used; + } + + public Long getTotalAllocatedInBytes() { + return totalAllocatedInBytes; + } + + public void setTotalAllocatedInBytes(Long totalAllocatedInBytes) { + this.totalAllocatedInBytes = totalAllocatedInBytes; + } + + public Long getTotalUsageInBytes() { + return totalUsageInBytes; + } + + public void setTotalUsageInBytes(Long totalUsageInBytes) { + this.totalUsageInBytes = totalUsageInBytes; + } + + public Long getRemainingQuotaInBytes() { + if (totalUsageInBytes > totalAllocatedInBytes) { + return 0L; + } + return totalAllocatedInBytes - totalUsageInBytes; + } + } + public DataFile find(Object pk) { return em.find(DataFile.class, pk); } @@ -146,6 +186,27 @@ public DataFile find(Object pk) { }*/ + public List findAll(List fileIds){ + List dataFiles = new ArrayList<>(); + + for (Long fileId : fileIds){ + dataFiles.add(find(fileId)); + } + + return dataFiles; + } + + public List findAll(String fileIdsAsString){ + ArrayList dataFileIds = new ArrayList<>(); + + String[] fileIds = fileIdsAsString.split(","); + for (String fId : fileIds){ + dataFileIds.add(Long.parseLong(fId)); + } + + return findAll(dataFileIds); + } + public DataFile findByGlobalId(String globalId) { return (DataFile) dvObjectService.findByGlobalId(globalId, DvObject.DType.DataFile); } @@ -354,6 +415,18 @@ public FileMetadata findMostRecentVersionFileIsIn(DataFile file) { return fileMetadatas.get(0); } } + + public List findAllCheapAndEasy(String fileIdsAsString){ + //assumption is that the fileIds are separated by ',' + ArrayList dataFilesFound = new ArrayList<>(); + String[] fileIds = fileIdsAsString.split(","); + DataFile df = this.findCheapAndEasy(Long.parseLong(fileIds[0])); + if(df != null){ + dataFilesFound.add(df); + } + + return dataFilesFound; + } public DataFile findCheapAndEasy(Long id) { DataFile dataFile; @@ -566,6 +639,125 @@ public DataFile findCheapAndEasy(Long id) { return dataFile; } + private List retrieveFileAccessRequesters(DataFile fileIn) { + List retList = new ArrayList<>(); + + // List requesters = em.createNativeQuery("select authenticated_user_id + // from fileaccessrequests where datafile_id = + // "+fileIn.getId()).getResultList(); + TypedQuery typedQuery = em.createQuery("select f.user.id from FileAccessRequest f where f.dataFile.id = :file_id and f.requestState= :requestState", Long.class); + typedQuery.setParameter("file_id", fileIn.getId()); + typedQuery.setParameter("requestState", FileAccessRequest.RequestState.CREATED); + List requesters = typedQuery.getResultList(); + for (Long userId : requesters) { + AuthenticatedUser user = userService.find(userId); + if (user != null) { + retList.add(user); + } + } + + return retList; + } + + private List retrieveFileMetadataForVersion(Dataset dataset, DatasetVersion version, List dataFiles, Map filesMap, Map categoryMap) { + List retList = new ArrayList<>(); + Map> categoryMetaMap = new HashMap<>(); + + List categoryResults = em.createNativeQuery("select t0.filecategories_id, t0.filemetadatas_id from filemetadata_datafilecategory t0, filemetadata t1 where (t0.filemetadatas_id = t1.id) AND (t1.datasetversion_id = "+version.getId()+")").getResultList(); + int i = 0; + for (Object[] result : categoryResults) { + Long category_id = (Long) result[0]; + Long filemeta_id = (Long) result[1]; + if (categoryMetaMap.get(filemeta_id) == null) { + categoryMetaMap.put(filemeta_id, new HashSet<>()); + } + categoryMetaMap.get(filemeta_id).add(category_id); + i++; + } + logger.fine("Retrieved and mapped "+i+" file categories attached to files in the version "+version.getId()); + + List metadataResults = em.createNativeQuery("select id, datafile_id, DESCRIPTION, LABEL, RESTRICTED, DIRECTORYLABEL, prov_freeform from FileMetadata where datasetversion_id = "+version.getId() + " ORDER BY LABEL").getResultList(); + + for (Object[] result : metadataResults) { + Integer filemeta_id = (Integer) result[0]; + + if (filemeta_id == null) { + continue; + } + + Long file_id = (Long) result[1]; + if (file_id == null) { + continue; + } + + Integer file_list_id = filesMap.get(file_id); + if (file_list_id == null) { + continue; + } + FileMetadata fileMetadata = new FileMetadata(); + fileMetadata.setId(filemeta_id.longValue()); + fileMetadata.setCategories(new LinkedList<>()); + + if (categoryMetaMap.get(fileMetadata.getId()) != null) { + for (Long cat_id : categoryMetaMap.get(fileMetadata.getId())) { + if (categoryMap.get(cat_id) != null) { + fileMetadata.getCategories().add(dataset.getCategories().get(categoryMap.get(cat_id))); + } + } + } + + fileMetadata.setDatasetVersion(version); + + // Link the FileMetadata object to the DataFile: + fileMetadata.setDataFile(dataFiles.get(file_list_id)); + // ... and the DataFile back to the FileMetadata: + fileMetadata.getDataFile().getFileMetadatas().add(fileMetadata); + + String description = (String) result[2]; + + if (description != null) { + fileMetadata.setDescription(description); + } + + String label = (String) result[3]; + + if (label != null) { + fileMetadata.setLabel(label); + } + + Boolean restricted = (Boolean) result[4]; + if (restricted != null) { + fileMetadata.setRestricted(restricted); + } + + String dirLabel = (String) result[5]; + if (dirLabel != null){ + fileMetadata.setDirectoryLabel(dirLabel); + } + + String provFreeForm = (String) result[6]; + if (provFreeForm != null){ + fileMetadata.setProvFreeForm(provFreeForm); + } + + retList.add(fileMetadata); + } + + logger.fine("Retrieved "+retList.size()+" file metadatas for version "+version.getId()+" (inside the retrieveFileMetadataForVersion method)."); + + + /* + We no longer perform this sort here, just to keep this filemetadata + list as identical as possible to when it's produced by the "traditional" + EJB method. When it's necessary to have the filemetadatas sorted by + FileMetadata.compareByLabel, the DatasetVersion.getFileMetadatasSorted() + method should be called. + + Collections.sort(retList, FileMetadata.compareByLabel); */ + + return retList; + } + public List findIngestsInProgress() { if ( em.isOpen() ) { String qr = "select object(o) from DataFile as o where o.ingestStatus =:scheduledStatusCode or o.ingestStatus =:progressStatusCode order by o.id"; @@ -1203,4 +1395,29 @@ public Embargo findEmbargo(Long id) { DataFile d = find(id); return d.getEmbargo(); } + + public Long getStorageUsageByCreator(AuthenticatedUser user) { + Query query = em.createQuery("SELECT SUM(o.filesize) FROM DataFile o WHERE o.creator.id=:creatorId"); + + try { + Long totalSize = (Long)query.setParameter("creatorId", user.getId()).getSingleResult(); + logger.info("total size for user: "+totalSize); + return totalSize == null ? 0L : totalSize; + } catch (NoResultException nre) { // ? + logger.info("NoResultException, returning 0L"); + return 0L; + } + } + + public UserStorageQuota getUserStorageQuota(AuthenticatedUser user, Dataset dataset) { + // this is for testing only - one pre-set, installation-wide quota limit + // for everybody: + Long totalAllocated = systemConfig.getTestStorageQuotaLimit(); + // again, this is for testing only - we are only counting the total size + // of all the files created by this user; it will likely be a much more + // complex calculation in real life applications: + Long totalUsed = getStorageUsageByCreator(user); + + return new UserStorageQuota(totalAllocated, totalUsed); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index 620e66c6c54..21014d7f99c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -862,6 +862,12 @@ public String getHarvestingDescription() { return null; } + public boolean hasEnabledGuestbook(){ + Guestbook gb = this.getGuestbook(); + + return ( gb != null && gb.isEnabled()); + } + @Override public boolean equals(Object object) { // TODO: Warning - this method won't work in the case the id fields are not set diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index d20175b6e1a..b2d536a0dda 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -376,6 +376,19 @@ public void setShowIngestSuccess(boolean showIngestSuccess) { this.showIngestSuccess = showIngestSuccess; } + private String termsGuestbookPopupAction = ""; + + public void setTermsGuestbookPopupAction(String popupAction){ + if(popupAction != null && popupAction.length() > 0){ + this.termsGuestbookPopupAction = popupAction; + } + + } + + public String getTermsGuestbookPopupAction(){ + return termsGuestbookPopupAction; + } + // TODO: Consider renaming "configureTools" to "fileConfigureTools". List configureTools = new ArrayList<>(); // TODO: Consider renaming "exploreTools" to "fileExploreTools". @@ -391,6 +404,9 @@ public void setShowIngestSuccess(boolean showIngestSuccess) { Map> fileQueryToolsByFileId = new HashMap<>(); List fileQueryTools = new ArrayList<>(); private List datasetExploreTools; + private List datasetConfigureTools; + // The selected dataset-level configure tool + private ExternalTool datasetConfigureTool; public Boolean isHasRsyncScript() { return hasRsyncScript; @@ -2153,6 +2169,7 @@ private String init(boolean initFull) { previewTools = externalToolService.findFileToolsByType(ExternalTool.Type.PREVIEW); fileQueryTools = externalToolService.findFileToolsByType(ExternalTool.Type.QUERY); datasetExploreTools = externalToolService.findDatasetToolsByType(ExternalTool.Type.EXPLORE); + datasetConfigureTools = externalToolService.findDatasetToolsByType(ExternalTool.Type.CONFIGURE); rowsPerPage = 10; if (dataset.getId() != null && canUpdateDataset()) { hasRestrictedFiles = workingVersion.isHasRestrictedFile(); @@ -3143,7 +3160,7 @@ public void startDownloadSelectedOriginal() { private void startDownload(boolean downloadOriginal){ boolean guestbookRequired = isDownloadPopupRequired(); - boolean validate = validateFilesForDownload(guestbookRequired, downloadOriginal); + boolean validate = validateFilesForDownload(downloadOriginal); if (validate) { updateGuestbookResponse(guestbookRequired, downloadOriginal); if(!guestbookRequired && !getValidateFilesOutcome().equals("Mixed")){ @@ -3166,9 +3183,14 @@ public void setValidateFilesOutcome(String validateFilesOutcome) { this.validateFilesOutcome = validateFilesOutcome; } - public boolean validateFilesForDownload(boolean guestbookRequired, boolean downloadOriginal) { - setSelectedDownloadableFiles(new ArrayList<>()); - setSelectedNonDownloadableFiles(new ArrayList<>()); + public boolean validateFilesForDownload(boolean downloadOriginal){ + if (this.selectedFiles.isEmpty()) { + PrimeFaces.current().executeScript("PF('selectFilesForDownload').show()"); + return false; + } else { + this.filterSelectedFiles(); + } + //assume Pass unless something bad happens setValidateFilesOutcome("Pass"); Long bytes = (long) 0; @@ -3178,17 +3200,12 @@ public boolean validateFilesForDownload(boolean guestbookRequired, boolean downl return false; } - for (FileMetadata fmd : this.selectedFiles) { - if (this.fileDownloadHelper.canDownloadFile(fmd)) { - getSelectedDownloadableFiles().add(fmd); - DataFile dataFile = fmd.getDataFile(); - if (downloadOriginal && dataFile.isTabularData()) { - bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize(); - } else { - bytes += dataFile.getFilesize(); - } + for (FileMetadata fmd : getSelectedDownloadableFiles()) { + DataFile dataFile = fmd.getDataFile(); + if (downloadOriginal && dataFile.isTabularData()) { + bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize(); } else { - getSelectedNonDownloadableFiles().add(fmd); + bytes += dataFile.getFilesize(); } } @@ -3212,10 +3229,9 @@ public boolean validateFilesForDownload(boolean guestbookRequired, boolean downl return true; } - if (guestbookRequired) { + if (isTermsPopupRequired() || isGuestbookPopupRequiredAtDownload()) { setValidateFilesOutcome("GuestbookRequired"); } - return true; } @@ -3234,9 +3250,68 @@ private void updateGuestbookResponse (boolean guestbookRequired, boolean downloa } else { guestbookResponse.setFileFormat(""); } - guestbookResponse.setDownloadtype("Download"); + guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD); } + /*helper function to filter the selected files into , + and and for reuse*/ + + /*helper function to filter the selected files into , + and and for reuse*/ + + private boolean filterSelectedFiles(){ + setSelectedDownloadableFiles(new ArrayList<>()); + setSelectedNonDownloadableFiles(new ArrayList<>()); + setSelectedRestrictedFiles(new ArrayList<>()); + setSelectedUnrestrictedFiles(new ArrayList<>()); + + boolean someFiles = false; + for (FileMetadata fmd : this.selectedFiles){ + if(this.fileDownloadHelper.canDownloadFile(fmd)){ + getSelectedDownloadableFiles().add(fmd); + someFiles=true; + } else { + getSelectedNonDownloadableFiles().add(fmd); + } + if(fmd.isRestricted()){ + getSelectedRestrictedFiles().add(fmd); //might be downloadable to user or not + someFiles=true; + } else { + getSelectedUnrestrictedFiles().add(fmd); + someFiles=true; + } + + } + return someFiles; + } + + public void validateFilesForRequestAccess(){ + this.filterSelectedFiles(); + + if(!dataset.isFileAccessRequest()){ //is this needed? wouldn't be able to click Request Access if this !isFileAccessRequest() + return; + } + + if(!this.selectedRestrictedFiles.isEmpty()){ + ArrayList nonDownloadableRestrictedFiles = new ArrayList<>(); + + List userRequestedDataFiles = ((AuthenticatedUser) session.getUser()).getRequestedDataFiles(); + + for(FileMetadata fmd : this.selectedRestrictedFiles){ + if(!this.fileDownloadHelper.canDownloadFile(fmd) && !userRequestedDataFiles.contains(fmd.getDataFile())){ + nonDownloadableRestrictedFiles.add(fmd); + } + } + + if(!nonDownloadableRestrictedFiles.isEmpty()){ + guestbookResponse.setDataFile(null); + guestbookResponse.setSelectedFileIds(this.getFilesIdsString(nonDownloadableRestrictedFiles)); + this.requestAccessMultipleFiles(); + } else { + //popup select data files + } + } + } private boolean selectAllFiles; @@ -3262,26 +3337,23 @@ public void toggleAllSelected(){ // helper Method public String getSelectedFilesIdsString() { - String downloadIdString = ""; - for (FileMetadata fmd : this.selectedFiles){ - if (!StringUtil.isEmpty(downloadIdString)) { - downloadIdString += ","; - } - downloadIdString += fmd.getDataFile().getId(); - } - return downloadIdString; + return this.getFilesIdsString(this.selectedFiles); } - + // helper Method public String getSelectedDownloadableFilesIdsString() { - String downloadIdString = ""; - for (FileMetadata fmd : this.selectedDownloadableFiles){ - if (!StringUtil.isEmpty(downloadIdString)) { - downloadIdString += ","; + return this.getFilesIdsString(this.selectedDownloadableFiles); + } + + public String getFilesIdsString(List fileMetadatas){ //for reuse + String idString = ""; + for (FileMetadata fmd : fileMetadatas){ + if (!StringUtil.isEmpty(idString)) { + idString += ","; } - downloadIdString += fmd.getDataFile().getId(); + idString += fmd.getDataFile().getId(); } - return downloadIdString; + return idString; } @@ -5097,7 +5169,7 @@ public boolean isFileAccessRequestMultiButtonRequired(){ for (FileMetadata fmd : workingVersion.getFileMetadatas()){ AuthenticatedUser authenticatedUser = (AuthenticatedUser) session.getUser(); //Change here so that if all restricted files have pending requests there's no Request Button - if ((!this.fileDownloadHelper.canDownloadFile(fmd) && !fmd.getDataFile().containsFileAccessRequestFromUser(authenticatedUser))) { + if ((!this.fileDownloadHelper.canDownloadFile(fmd) && !fmd.getDataFile().containsActiveFileAccessRequestFromUser(authenticatedUser))) { return true; } } @@ -5108,6 +5180,9 @@ public boolean isFileAccessRequestMultiButtonEnabled(){ if (!isSessionUserAuthenticated() || !dataset.isFileAccessRequest()){ return false; } + //populate file lists + filterSelectedFiles(); + if( this.selectedRestrictedFiles == null || this.selectedRestrictedFiles.isEmpty() ){ return false; } @@ -5190,7 +5265,24 @@ public boolean isDownloadPopupRequired() { public boolean isRequestAccessPopupRequired() { return FileUtil.isRequestAccessPopupRequired(workingVersion); } + + public boolean isGuestbookAndTermsPopupRequired() { + return FileUtil.isGuestbookAndTermsPopupRequired(workingVersion); + } + public boolean isGuestbookPopupRequired(){ + return FileUtil.isGuestbookPopupRequired(workingVersion); + } + + public boolean isTermsPopupRequired(){ + return FileUtil.isTermsPopupRequired(workingVersion); + } + + public boolean isGuestbookPopupRequiredAtDownload(){ + // Only show guestbookAtDownload if guestbook at request is disabled (legacy behavior) + return isGuestbookPopupRequired() && !workingVersion.getDataset().getEffectiveGuestbookEntryAtRequest(); + } + public String requestAccessMultipleFiles() { if (selectedFiles.isEmpty()) { @@ -5205,11 +5297,11 @@ public String requestAccessMultipleFiles() { for (FileMetadata fmd : selectedFiles){ fileDownloadHelper.addMultipleFilesForRequestAccess(fmd.getDataFile()); } - if (isRequestAccessPopupRequired()) { + if (isGuestbookAndTermsPopupRequired()) { //RequestContext requestContext = RequestContext.getCurrentInstance(); - PrimeFaces.current().executeScript("PF('requestAccessPopup').show()"); + PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show()"); //the popup will call writeGuestbookAndRequestAccess(); return ""; - } else { + }else { //No popup required fileDownloadHelper.requestAccessIndirect(); return ""; @@ -5394,6 +5486,14 @@ public FileDownloadServiceBean getFileDownloadService() { public void setFileDownloadService(FileDownloadServiceBean fileDownloadService) { this.fileDownloadService = fileDownloadService; } + + public FileDownloadHelper getFileDownloadHelper() { + return fileDownloadHelper; + } + + public void setFileDownloadHelper(FileDownloadHelper fileDownloadHelper) { + this.fileDownloadHelper = fileDownloadHelper; + } public GuestbookResponseServiceBean getGuestbookResponseService() { @@ -5572,6 +5672,18 @@ public List getDatasetExploreTools() { return datasetExploreTools; } + public List getDatasetConfigureTools() { + return datasetConfigureTools; + } + + public ExternalTool getDatasetConfigureTool() { + return datasetConfigureTool; + } + + public void setDatasetConfigureTool(ExternalTool datasetConfigureTool) { + this.datasetConfigureTool = datasetConfigureTool; + } + Boolean thisLatestReleasedVersion = null; public boolean isThisLatestReleasedVersion() { @@ -5789,6 +5901,16 @@ public void explore(ExternalTool externalTool) { PrimeFaces.current().executeScript(externalToolHandler.getExploreScript()); } + public void configure(ExternalTool externalTool) { + ApiToken apiToken = null; + User user = session.getUser(); + if (user instanceof AuthenticatedUser) { + apiToken = authService.findApiTokenByUser((AuthenticatedUser) user); + } + ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataset, apiToken, session.getLocaleCode()); + PrimeFaces.current().executeScript(externalToolHandler.getConfigureScript()); + } + private FileMetadata fileMetadataForAction; public FileMetadata getFileMetadataForAction() { @@ -5832,7 +5954,7 @@ public String getEffectiveMetadataLanguage() { } public String getEffectiveMetadataLanguage(boolean ofParent) { String mdLang = ofParent ? dataset.getOwner().getEffectiveMetadataLanguage() : dataset.getEffectiveMetadataLanguage(); - if (mdLang.equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE)) { + if (mdLang.equals(DvObjectContainer.UNDEFINED_CODE)) { mdLang = settingsWrapper.getDefaultMetadataLanguage(); } return mdLang; @@ -5840,7 +5962,7 @@ public String getEffectiveMetadataLanguage(boolean ofParent) { public String getLocaleDisplayName(String code) { String displayName = settingsWrapper.getBaseMetadataLanguageMap(false).get(code); - if(displayName==null && !code.equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE)) { + if(displayName==null && !code.equals(DvObjectContainer.UNDEFINED_CODE)) { //Default (for cases such as :when a Dataset has a metadatalanguage code but :MetadataLanguages is no longer defined). displayName = new Locale(code).getDisplayName(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 52eb5868c35..bc79b4a7107 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -1126,5 +1126,5 @@ public void deleteHarvestedDataset(Dataset dataset, DataverseRequest request, Lo hdLogger.warning("Failed to destroy the dataset"); } } - + } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index daf33f444d9..943a74327d5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -1286,4 +1286,7 @@ public String getCurationLabelSetNameLabel() { return setName; } + public Set> getGuestbookEntryOptions() { + return settingsWrapper.getGuestbookEntryOptions(this.dataverse).entrySet(); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java index a322a25103e..f7d361d76f5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java @@ -1,7 +1,11 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.dataaccess.DataAccess; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.SystemConfig; +import java.util.Locale; +import java.util.Optional; + import jakarta.persistence.MappedSuperclass; import org.apache.commons.lang3.StringUtils; @@ -12,10 +16,8 @@ */ @MappedSuperclass public abstract class DvObjectContainer extends DvObject { - - - public static final String UNDEFINED_METADATA_LANGUAGE_CODE = "undefined"; //Used in dataverse.xhtml as a non-null selection option value (indicating inheriting the default) + public static final String UNDEFINED_CODE = "undefined"; //Used in dataverse.xhtml as a non-null selection option value (indicating inheriting the default) public void setOwner(Dataverse owner) { super.setOwner(owner); @@ -37,6 +39,8 @@ public boolean isEffectivelyPermissionRoot() { private String metadataLanguage=null; + private Boolean guestbookAtRequest = null; + public String getEffectiveStorageDriverId() { String id = storageDriver; if (StringUtils.isBlank(id)) { @@ -70,7 +74,7 @@ public String getEffectiveMetadataLanguage() { if (this.getOwner() != null) { ml = this.getOwner().getEffectiveMetadataLanguage(); } else { - ml = UNDEFINED_METADATA_LANGUAGE_CODE; + ml = UNDEFINED_CODE; } } return ml; @@ -78,13 +82,13 @@ public String getEffectiveMetadataLanguage() { public String getMetadataLanguage() { if (metadataLanguage == null) { - return UNDEFINED_METADATA_LANGUAGE_CODE; + return UNDEFINED_CODE; } return metadataLanguage; } public void setMetadataLanguage(String ml) { - if (ml != null && ml.equals(UNDEFINED_METADATA_LANGUAGE_CODE)) { + if (ml != null && ml.equals(UNDEFINED_CODE)) { this.metadataLanguage = null; } else { this.metadataLanguage = ml; @@ -92,7 +96,40 @@ public void setMetadataLanguage(String ml) { } public static boolean isMetadataLanguageSet(String mdLang) { - return mdLang!=null && !mdLang.equals(UNDEFINED_METADATA_LANGUAGE_CODE); + return mdLang!=null && !mdLang.equals(UNDEFINED_CODE); + } + + public boolean getEffectiveGuestbookEntryAtRequest() { + boolean gbAtRequest = false; + if (guestbookAtRequest==null) { + if (this.getOwner() != null) { + gbAtRequest = this.getOwner().getEffectiveGuestbookEntryAtRequest(); + } else { + Optional opt = JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class); + if (opt.isPresent()) { + gbAtRequest = opt.get(); + } + } + } else { + gbAtRequest = guestbookAtRequest; + } + return gbAtRequest; + } + + public String getGuestbookEntryAtRequest() { + if(guestbookAtRequest==null) { + return UNDEFINED_CODE; + } + return Boolean.valueOf(guestbookAtRequest).toString(); + } + + public void setGuestbookEntryAtRequest(String gbAtRequest) { + if (gbAtRequest == null || gbAtRequest.equals(UNDEFINED_CODE)) { + this.guestbookAtRequest = null; + } else { + //Force to true or false + this.guestbookAtRequest = Boolean.valueOf(Boolean.parseBoolean(gbAtRequest)); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 02a148f8cc5..a942830b19e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.provenance.ProvPopupFragmentBean; import edu.harvard.iq.dataverse.DataFile.ChecksumType; +import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota; import edu.harvard.iq.dataverse.api.AbstractApiBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; @@ -28,6 +29,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.RequestRsyncScriptCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetThumbnailCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; +import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDataFilesCommand; import edu.harvard.iq.dataverse.ingest.IngestRequest; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.ingest.IngestUtil; @@ -186,7 +188,13 @@ public enum Referrer { // Used to store results of permissions checks private final Map datasetPermissionMap = new HashMap<>(); // { Permission human_name : Boolean } + // Size limit of an individual file: (set for the storage volume used) private Long maxFileUploadSizeInBytes = null; + // Total amount of data that the user should be allowed to upload. + // Will be calculated in real time based on various level quotas - + // for this user and/or this collection/dataset, etc. We should + // assume that it may change during the user session. + private Long maxTotalUploadSizeInBytes = null; private Long maxIngestSizeInBytes = null; // CSV: 4.8 MB, DTA: 976.6 KB, XLSX: 5.7 MB, etc. private String humanPerFormatTabularLimits = null; @@ -198,6 +206,7 @@ public enum Referrer { private final int NUMBER_OF_SCROLL_ROWS = 25; private DataFile singleFile = null; + private UserStorageQuota userStorageQuota = null; public DataFile getSingleFile() { return singleFile; @@ -340,6 +349,18 @@ public boolean isUnlimitedUploadFileSize() { return this.maxFileUploadSizeInBytes == null; } + + public Long getMaxTotalUploadSizeInBytes() { + return maxTotalUploadSizeInBytes; + } + + public String getHumanMaxTotalUploadSizeInBytes() { + return FileSizeChecker.bytesToHumanReadable(maxTotalUploadSizeInBytes); + } + + public boolean isStorageQuotaEnforced() { + return userStorageQuota != null; + } public Long getMaxIngestSizeInBytes() { return maxIngestSizeInBytes; @@ -508,15 +529,26 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo selectedFiles = selectedFileMetadatasList; this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId()); + if (systemConfig.isStorageQuotasEnforced()) { + this.userStorageQuota = datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset); + this.maxTotalUploadSizeInBytes = userStorageQuota.getRemainingQuotaInBytes(); + } else { + this.maxTotalUploadSizeInBytes = null; + } this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit(); this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits(); this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit(); - + logger.fine("done"); saveEnabled = true; + return null; } + + public boolean isQuotaExceeded() { + return systemConfig.isStorageQuotasEnforced() && userStorageQuota != null && userStorageQuota.getRemainingQuotaInBytes() == 0; + } public String init() { // default mode should be EDIT @@ -559,10 +591,13 @@ public String init() { clone = workingVersion.cloneDatasetVersion(); this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId()); + if (systemConfig.isStorageQuotasEnforced()) { + this.userStorageQuota = datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset); + this.maxTotalUploadSizeInBytes = userStorageQuota.getRemainingQuotaInBytes(); + } this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit(); this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits(); this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit(); - this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId()); hasValidTermsOfAccess = isHasValidTermsOfAccess(); if (!hasValidTermsOfAccess) { @@ -656,7 +691,7 @@ public String init() { if (isHasPublicStore()){ JH.addMessage(FacesMessage.SEVERITY_WARN, getBundleString("dataset.message.label.fileAccess"), getBundleString("dataset.message.publicInstall")); } - + return null; } @@ -1493,14 +1528,16 @@ public void handleDropBoxUpload(ActionEvent event) { // for example, multiple files can be extracted from an uncompressed // zip file. //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream"); - CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream", null, null, systemConfig); + //CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream", null, null, systemConfig); + Command cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, dropBoxStream, fileName, "application/octet-stream", null, userStorageQuota, null); + CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd); datafiles = createDataFilesResult.getDataFiles(); Optional.ofNullable(editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult)).ifPresent(errorMessage -> errorMessages.add(errorMessage)); - } catch (IOException ex) { + } catch (CommandException ex) { this.logger.log(Level.SEVERE, "Error during ingest of DropBox file {0} from link {1}", new Object[]{fileName, fileLink}); continue; - }/*catch (FileExceedsMaxSizeException ex){ + } /*catch (FileExceedsMaxSizeException ex){ this.logger.log(Level.SEVERE, "Error during ingest of DropBox file {0} from link {1}: {2}", new Object[]{fileName, fileLink, ex.getMessage()}); continue; }*/ finally { @@ -2023,7 +2060,21 @@ public void handleFileUpload(FileUploadEvent event) throws IOException { // Note: A single uploaded file may produce multiple datafiles - // for example, multiple files can be extracted from an uncompressed // zip file. - CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null, systemConfig); + ///CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null, systemConfig); + + Command cmd; + if (mode == FileEditMode.CREATE) { + // This is a file upload in the context of creating a brand new + // dataset that does not yet exist in the database. We must + // use the version of the Create New Files constructor that takes + // the parent Dataverse as the extra argument: + cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, userStorageQuota, null, null, null, workingVersion.getDataset().getOwner()); + } else { + cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, userStorageQuota, null); + } + CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd); + + dFileList = createDataFilesResult.getDataFiles(); String createDataFilesError = editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult); if(createDataFilesError != null) { @@ -2032,8 +2083,14 @@ public void handleFileUpload(FileUploadEvent event) throws IOException { } } catch (IOException ioex) { + // shouldn't we try and communicate to the user what happened? logger.warning("Failed to process and/or save the file " + uFile.getFileName() + "; " + ioex.getMessage()); return; + } catch (CommandException cex) { + // shouldn't we try and communicate to the user what happened? + errorMessages.add(cex.getMessage()); + uploadComponentId = event.getComponent().getClientId(); + return; } /*catch (FileExceedsMaxSizeException ex) { logger.warning("Failed to process and/or save the file " + uFile.getFileName() + "; " + ex.getMessage()); @@ -2111,6 +2168,11 @@ public void handleExternalUpload() { - Max size NOT specified in db: default is unlimited - Max size specified in db: check too make sure file is within limits // ---------------------------- */ + /** + * @todo: this size check is probably redundant here, since the new + * CreateNewFilesCommand is going to perform it (and the quota + * checks too, if enabled + */ if ((!this.isUnlimitedUploadFileSize()) && (fileSize > this.getMaxFileUploadSizeInBytes())) { String warningMessage = "Uploaded file \"" + fileName + "\" exceeded the limit of " + fileSize + " bytes and was not uploaded."; sio.delete(); @@ -2130,18 +2192,27 @@ public void handleExternalUpload() { List datafiles = new ArrayList<>(); // ----------------------------------------------------------- - // Send it through the ingest service + // Execute the CreateNewDataFiles command: // ----------------------------------------------------------- + + Dataverse parent = null; + + if (mode == FileEditMode.CREATE) { + // This is a file upload in the context of creating a brand new + // dataset that does not yet exist in the database. We must + // pass the parent Dataverse to the CreateNewFiles command + // constructor. The RequiredPermission on the command in this + // scenario = Permission.AddDataset on the parent dataverse. + parent = workingVersion.getDataset().getOwner(); + } + try { - - // Note: A single uploaded file may produce multiple datafiles - - // for example, multiple files can be extracted from an uncompressed - // zip file. - //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream"); - CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, checksumType, systemConfig); + + Command cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, null, fileName, contentType, fullStorageIdentifier, userStorageQuota, checksumValue, checksumType, fileSize, parent); + CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd); datafiles = createDataFilesResult.getDataFiles(); Optional.ofNullable(editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult)).ifPresent(errorMessage -> errorMessages.add(errorMessage)); - } catch (IOException ex) { + } catch (CommandException ex) { logger.log(Level.SEVERE, "Error during ingest of file {0}", new Object[]{fileName}); } diff --git a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java index 6f68815c2ca..51c67a37a09 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java @@ -1,60 +1,178 @@ package edu.harvard.iq.dataverse; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import java.io.Serializable; +import java.util.Date; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import jakarta.persistence.Column; -import jakarta.persistence.Embeddable; -import jakarta.persistence.EmbeddedId; import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; import jakarta.persistence.JoinColumn; import jakarta.persistence.ManyToOne; -import jakarta.persistence.MapsId; +import jakarta.persistence.NamedQueries; +import jakarta.persistence.NamedQuery; +import jakarta.persistence.OneToOne; import jakarta.persistence.Table; import jakarta.persistence.Temporal; import jakarta.persistence.TemporalType; -import java.io.Serializable; -import java.util.Date; + +/** + * + * @author Marina + */ @Entity @Table(name = "fileaccessrequests") -public class FileAccessRequest { - @EmbeddedId - private FileAccessRequestKey id; + +@NamedQueries({ + @NamedQuery(name = "FileAccessRequest.findByAuthenticatedUserId", + query = "SELECT far FROM FileAccessRequest far WHERE far.user.id=:authenticatedUserId"), + @NamedQuery(name = "FileAccessRequest.findByGuestbookResponseId", + query = "SELECT far FROM FileAccessRequest far WHERE far.guestbookResponse.id=:guestbookResponseId"), + @NamedQuery(name = "FileAccessRequest.findByDataFileId", + query = "SELECT far FROM FileAccessRequest far WHERE far.dataFile.id=:dataFileId"), + @NamedQuery(name = "FileAccessRequest.findByRequestState", + query = "SELECT far FROM FileAccessRequest far WHERE far.requestState=:requestState"), + @NamedQuery(name = "FileAccessRequest.findByAuthenticatedUserIdAndRequestState", + query = "SELECT far FROM FileAccessRequest far WHERE far.user.id=:authenticatedUserId and far.requestState=:requestState"), + @NamedQuery(name = "FileAccessRequest.findByGuestbookResponseIdAndRequestState", + query = "SELECT far FROM FileAccessRequest far WHERE far.guestbookResponse.id=:guestbookResponseId and far.requestState=:requestState"), + @NamedQuery(name = "FileAccessRequest.findByDataFileIdAndRequestState", + query = "SELECT far FROM FileAccessRequest far WHERE far.dataFile.id=:dataFileId and far.requestState=:requestState"), + @NamedQuery(name = "FileAccessRequest.findByAuthenticatedUserIdAndDataFileIdAndRequestState", + query = "SELECT far FROM FileAccessRequest far WHERE far.user.id=:authenticatedUserId and far.dataFile.id=:dataFileId and far.requestState=:requestState") +}) + + +public class FileAccessRequest implements Serializable{ + private static final long serialVersionUID = 1L; + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + @ManyToOne - @MapsId("dataFile") - @JoinColumn(name = "datafile_id") + @JoinColumn(nullable=false) private DataFile dataFile; + @ManyToOne - @MapsId("authenticatedUser") - @JoinColumn(name = "authenticated_user_id") - private AuthenticatedUser authenticatedUser; - + @JoinColumn(name="authenticated_user_id",nullable=false) + private AuthenticatedUser user; + + @OneToOne + @JoinColumn(nullable=true) + private GuestbookResponse guestbookResponse; + + public enum RequestState {CREATED, GRANTED, REJECTED}; + //private RequestState state; + @Enumerated(EnumType.STRING) + @Column(name="request_state", nullable=false ) + private RequestState requestState; + @Temporal(value = TemporalType.TIMESTAMP) @Column(name = "creation_time") private Date creationTime; - - public FileAccessRequestKey getId() { + + public FileAccessRequest(){ + } + + public FileAccessRequest(DataFile df, AuthenticatedUser au){ + setDataFile(df); + setRequester(au); + setState(RequestState.CREATED); + setCreationTime(new Date()); + } + + public FileAccessRequest(DataFile df, AuthenticatedUser au, GuestbookResponse gbr){ + this(df, au); + setGuestbookResponse(gbr); + } + + public Long getId() { return id; } - public void setId(FileAccessRequestKey id) { + public void setId(Long id) { this.id = id; } - - public DataFile getDataFile() { + + public DataFile getDataFile(){ return dataFile; } + + public final void setDataFile(DataFile df){ + this.dataFile = df; + } + + public AuthenticatedUser getRequester(){ + return user; + } + + public final void setRequester(AuthenticatedUser au){ + this.user = au; + } + + public GuestbookResponse getGuestbookResponse(){ + return guestbookResponse; + } + + public final void setGuestbookResponse(GuestbookResponse gbr){ + this.guestbookResponse = gbr; + } + + public RequestState getState() { + return this.requestState; + } + + public void setState(RequestState requestState) { + this.requestState = requestState; + } + + public String getStateLabel() { + if(isStateCreated()){ + return "created"; + } + if(isStateGranted()) { + return "granted"; + } + if(isStateRejected()) { + return "rejected"; + } + return null; + } + + public void setStateCreated() { + this.requestState = RequestState.CREATED; + } + + public void setStateGranted() { + this.requestState = RequestState.GRANTED; + } - public void setDataFile(DataFile dataFile) { - this.dataFile = dataFile; + public void setStateRejected() { + this.requestState = RequestState.REJECTED; } - public AuthenticatedUser getAuthenticatedUser() { - return authenticatedUser; + public boolean isStateCreated() { + return this.requestState == RequestState.CREATED; + } + + public boolean isStateGranted() { + return this.requestState == RequestState.GRANTED; } - public void setAuthenticatedUser(AuthenticatedUser authenticatedUser) { - this.authenticatedUser = authenticatedUser; + public boolean isStateRejected() { + return this.requestState == RequestState.REJECTED; + } + + @Override + public int hashCode() { + int hash = 0; + hash += (id != null ? id.hashCode() : 0); + return hash; } public Date getCreationTime() { @@ -64,28 +182,19 @@ public Date getCreationTime() { public void setCreationTime(Date creationTime) { this.creationTime = creationTime; } - - @Embeddable - public static class FileAccessRequestKey implements Serializable { - @Column(name = "datafile_id") - private Long dataFile; - @Column(name = "authenticated_user_id") - private Long authenticatedUser; - - public Long getDataFile() { - return dataFile; + + @Override + public boolean equals(Object object) { + // TODO: Warning - this method won't work in the case the id fields are not set + if (!(object instanceof FileAccessRequest)) { + return false; } - - public void setDataFile(Long dataFile) { - this.dataFile = dataFile; - } - - public Long getAuthenticatedUser() { - return authenticatedUser; - } - - public void setAuthenticatedUser(Long authenticatedUser) { - this.authenticatedUser = authenticatedUser; + FileAccessRequest other = (FileAccessRequest) object; + if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) { + return false; } + return true; } -} + + +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java new file mode 100644 index 00000000000..af8577fad34 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java @@ -0,0 +1,87 @@ +package edu.harvard.iq.dataverse; + +import java.util.List; +import jakarta.ejb.Stateless; +import jakarta.inject.Named; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; + +/** + * + * @author Marina + */ +@Stateless +@Named +public class FileAccessRequestServiceBean { + + @PersistenceContext(unitName = "VDCNet-ejbPU") + private EntityManager em; + + public FileAccessRequest find(Object pk) { + return em.find(FileAccessRequest.class, pk); + } + + public List findAll() { + return em.createQuery("select object(o) from FileAccessRequest as o order by o.id", FileAccessRequest.class).getResultList(); + } + + public List findAll(Long authenticatedUserId, Long fileId, FileAccessRequest.RequestState requestState){ + return em.createNamedQuery("FileAccessRequest.findByAuthenticatedUserIdAndDataFileIdAndRequestState", FileAccessRequest.class) + .setParameter("authenticatedUserId",authenticatedUserId) + .setParameter("dataFileId",fileId) + .setParameter("requestState",requestState) + .getResultList(); + } + + public List findAllByAuthenticedUserId(Long authenticatedUserId){ + return em.createNamedQuery("FileAccessRequest.findByAuthenticatedUserId", FileAccessRequest.class) + .setParameter("authenticatedUserId", authenticatedUserId) + .getResultList(); + } + + public List findAllByGuestbookResponseId(Long guestbookResponseId){ + return em.createNamedQuery("FileAccessRequest.findByGuestbookResponseId", FileAccessRequest.class) + .setParameter("guestbookResponseId", guestbookResponseId) + .getResultList(); + + } + + public List findAllByDataFileId(Long dataFileId){ + return em.createNamedQuery("FileAccessRequest.findByDataFileId", FileAccessRequest.class) + .setParameter("dataFileId", dataFileId) + .getResultList(); + } + + public List findAllByAuthenticatedUserIdAndRequestState(Long authenticatedUserId, FileAccessRequest.RequestState requestState){ + return em.createNamedQuery("FileAccessRequest.findByAuthenticatedUserIdAndRequestState", FileAccessRequest.class) + .setParameter("authenticatedUserId", authenticatedUserId) + .setParameter("requestState",requestState) + .getResultList(); + } + + public List findAllByGuestbookResponseIdAndRequestState(Long guestbookResponseId, FileAccessRequest.RequestState requestState){ + return em.createNamedQuery("FileAccessRequest.findByGuestbookResponseIdAndRequestState", FileAccessRequest.class) + .setParameter("dataFileId", guestbookResponseId) + .setParameter("requestState",requestState) + .getResultList(); + } + + public List findAllByDataFileIdAndRequestState(Long dataFileId, FileAccessRequest.RequestState requestState){ + return em.createNamedQuery("FileAccessRequest.findByDataFileIdAndRequestState", FileAccessRequest.class) + .setParameter("dataFileId", dataFileId) + .setParameter("requestState",requestState) + .getResultList(); + } + + + public FileAccessRequest save(FileAccessRequest far) { + if (far.getId() == null) { + em.persist(far); + return far; + } else { + return em.merge(far); + } + } + + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownload.java b/src/main/java/edu/harvard/iq/dataverse/FileDownload.java deleted file mode 100644 index a79281f71f0..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownload.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse; - -import java.io.Serializable; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.Temporal; -import jakarta.persistence.TemporalType; -import jakarta.persistence.Transient; -import jakarta.persistence.CascadeType; -import jakarta.persistence.OneToOne; -import jakarta.persistence.MapsId; -import jakarta.persistence.FetchType; -import jakarta.persistence.JoinColumn; -import java.util.Date; - - -/** - * - * @author marina - */ -@Entity -public class FileDownload implements Serializable { - - @Id - private Long id; - - @OneToOne(fetch = FetchType.LAZY) - @MapsId - private GuestbookResponse guestbookResponse; - - @Temporal(value = TemporalType.TIMESTAMP) - private Date downloadTimestamp; - - /* - Transient Values carry non-written information - that will assist in the download process - - selected file ids is a comma delimited list that contains the file ids for multiple download - - fileFormat tells the download api which format a subsettable file should be downloaded as - */ - - @Transient - private String selectedFileIds; - - @Transient - private String fileFormat; - - - /** - * Possible values for downloadType include "Download", "Subset", - * or the displayName of an ExternalTool. - * - * TODO: Types like "Download" and "Subset" should - * be defined once as constants (likely an enum) rather than having these - * strings duplicated in various places when setDownloadtype() is called. - */ - private String downloadtype; - private String sessionId; - - public FileDownload(){ - - } - - public FileDownload(FileDownload source){ - this.setDownloadTimestamp(source.getDownloadTimestamp()); - this.setDownloadtype(source.getDownloadtype()); - this.setFileFormat(source.getFileFormat()); - this.setGuestbookResponse(source.getGuestbookResponse()); - this.setSelectedFileIds(source.getSelectedFileIds()); - this.setSessionId(source.getSessionId()); - } - - public String getFileFormat() { - return fileFormat; - } - - //for download - public void setFileFormat(String downloadFormat) { - this.fileFormat = downloadFormat; - } - - public String getDownloadtype() { - return downloadtype; - } - - public void setDownloadtype(String downloadtype) { - this.downloadtype = downloadtype; - } - - public String getSessionId() { - return sessionId; - } - - public void setSessionId(String sessionId) { - this.sessionId = sessionId; - } - - public String getSelectedFileIds() { - return selectedFileIds; - } - - public void setSelectedFileIds(String selectedFileIds) { - this.selectedFileIds = selectedFileIds; - } - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Date getDownloadTimestamp(){ - return this.downloadTimestamp; - } - - public void setDownloadTimestamp(Date downloadTimestamp){ - this.downloadTimestamp = downloadTimestamp; - } - - - public void setGuestbookResponse(GuestbookResponse gbr){ - this.guestbookResponse = gbr; - } - - public GuestbookResponse getGuestbookResponse(){ - return this.guestbookResponse; - } - - @Override - public int hashCode() { - int hash = 0; - hash += (id != null ? id.hashCode() : 0); - return hash; - } - - @Override - public boolean equals(Object object) { - // TODO: Warning - this method won't work in the case the id fields are not set - if (!(object instanceof FileDownload)) { - return false; - } - FileDownload other = (FileDownload) object; - if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) { - return false; - } - return true; - } - - @Override - public String toString() { - return "edu.harvard.iq.dataverse.FileDownload[ id=" + id + " ]"; - } - - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java index c4b4978e0f8..a6ae7223d9d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java @@ -68,11 +68,11 @@ private boolean testResponseLength(String value) { // This helper method is called from the Download terms/guestbook/etc. popup, // when the user clicks the "ok" button. We use it, instead of calling // downloadServiceBean directly, in order to differentiate between single - // file downloads and multiple (batch) downloads - sice both use the same + // file downloads and multiple (batch) downloads - since both use the same // terms/etc. popup. public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse) { - PrimeFaces.current().executeScript("PF('downloadPopup').hide()"); - guestbookResponse.setDownloadtype("Download"); + PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()"); + guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD); // Note that this method is only ever called from the file-download-popup - // meaning we know for the fact that we DO want to save this // guestbookResponse permanently in the database. @@ -91,9 +91,9 @@ public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse) public void writeGuestbookAndOpenSubset(GuestbookResponse guestbookResponse) { - PrimeFaces.current().executeScript("PF('downloadPopup').hide()"); + PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()"); PrimeFaces.current().executeScript("PF('downloadDataSubsetPopup').show()"); - guestbookResponse.setDownloadtype("Subset"); + guestbookResponse.setEventType(GuestbookResponse.SUBSET); fileDownloadService.writeGuestbookResponseRecord(guestbookResponse); } @@ -132,22 +132,33 @@ public void writeGuestbookAndLaunchExploreTool(GuestbookResponse guestbookRespon fileDownloadService.explore(guestbookResponse, fmd, externalTool); //requestContext.execute("PF('downloadPopup').hide()"); - PrimeFaces.current().executeScript("PF('downloadPopup').hide()"); + PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()"); } public void writeGuestbookAndLaunchPackagePopup(GuestbookResponse guestbookResponse) { - PrimeFaces.current().executeScript("PF('downloadPopup').hide()"); + PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()"); PrimeFaces.current().executeScript("PF('downloadPackagePopup').show()"); PrimeFaces.current().executeScript("handleResizeDialog('downloadPackagePopup')"); fileDownloadService.writeGuestbookResponseRecord(guestbookResponse); } + + public void writeGuestbookResponseAndRequestAccess(GuestbookResponse guestbookResponse) { + + if(!filesForRequestAccess.isEmpty()) { + /* Only for single file requests (i.e. from kebab menu) */ + guestbookResponse.setDataFile(filesForRequestAccess.get(0)); + } + PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()"); + fileDownloadService.writeGuestbookResponseAndRequestAccess(guestbookResponse); + } + /** * Writes a guestbook entry for either popup scenario: guestbook or terms. */ public boolean writeGuestbookAndShowPreview(GuestbookResponse guestbookResponse) { - guestbookResponse.setDownloadtype("Explore"); + guestbookResponse.setEventType(GuestbookResponse.EXPLORE); fileDownloadService.writeGuestbookResponseRecord(guestbookResponse); return true; } @@ -284,7 +295,7 @@ public void handleCommandLinkClick(FileMetadata fmd){ if (FileUtil.isRequestAccessPopupRequired(fmd.getDatasetVersion())){ addFileForRequestAccess(fmd.getDataFile()); - PrimeFaces.current().executeScript("PF('requestAccessPopup').show()"); + PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');"); } else { requestAccess(fmd.getDataFile()); } @@ -299,7 +310,7 @@ public void requestAccessMultiple(List files) { DataFile notificationFile = null; for (DataFile file : files) { //Not sending notification via request method so that - // we can bundle them up into one nofication at dataset level + // we can bundle them up into one notification at dataset level test = processRequestAccess(file, false); succeeded |= test; if (notificationFile == null) { @@ -307,13 +318,15 @@ public void requestAccessMultiple(List files) { } } if (notificationFile != null && succeeded) { - fileDownloadService.sendRequestFileAccessNotification(notificationFile, (AuthenticatedUser) session.getUser()); + fileDownloadService.sendRequestFileAccessNotification(notificationFile.getOwner(), + notificationFile.getId(), (AuthenticatedUser) session.getUser()); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("file.accessRequested.success")); } } public void requestAccessIndirect() { //Called when there are multiple files and no popup - // or there's a popup with sigular or multiple files + // or there's a popup with singular or multiple files // The list of files for Request Access is set in the Dataset Page when // user clicks the request access button in the files fragment // (and has selected one or more files) @@ -325,13 +338,15 @@ private boolean processRequestAccess(DataFile file, Boolean sendNotification) { if (fileDownloadService.requestAccess(file.getId())) { // update the local file object so that the page properly updates AuthenticatedUser user = (AuthenticatedUser) session.getUser(); - file.addFileAccessRequester(user); + //This seems to be required because we don't get the updated file object back from the command called in the fileDownloadService.requestAccess call above + FileAccessRequest request = new FileAccessRequest(file, user); + file.addFileAccessRequest(request); // create notification if necessary if (sendNotification) { - fileDownloadService.sendRequestFileAccessNotification(file, user); - } - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("file.accessRequested.success")); + fileDownloadService.sendRequestFileAccessNotification(file.getOwner(), file.getId(), (AuthenticatedUser) session.getUser()); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("file.accessRequested.success")); + } return true; } JsfHelper.addWarningMessage(BundleUtil.getStringFromBundle("file.accessRequested.alreadyRequested", Arrays.asList(file.getDisplayName()))); diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index 6bf68f908ed..de947ee9058 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -19,7 +19,9 @@ import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; +import edu.harvard.iq.dataverse.util.JsfHelper; import edu.harvard.iq.dataverse.util.StringUtil; import java.io.IOException; import java.sql.Timestamp; @@ -76,6 +78,8 @@ public class FileDownloadServiceBean implements java.io.Serializable { PrivateUrlServiceBean privateUrlService; @EJB SettingsServiceBean settingsService; + @EJB + MailServiceBean mailService; @Inject DataverseSession session; @@ -192,6 +196,42 @@ public void writeGuestbookAndStartFileDownload(GuestbookResponse guestbookRespon redirectToDownloadAPI(guestbookResponse.getFileFormat(), guestbookResponse.getDataFile().getId()); logger.fine("issued file download redirect for datafile "+guestbookResponse.getDataFile().getId()); } + + public void writeGuestbookResponseAndRequestAccess(GuestbookResponse guestbookResponse){ + if (guestbookResponse == null || ( guestbookResponse.getDataFile() == null && guestbookResponse.getSelectedFileIds() == null) ) { + return; + } + + guestbookResponse.setEventType(GuestbookResponse.ACCESS_REQUEST); + + List selectedDataFiles = new ArrayList<>(); //always make sure it's at least an empty List + + if(guestbookResponse.getDataFile() != null ){ //one file 'selected' by 'Request Access' button click + selectedDataFiles.add(datafileService.find(guestbookResponse.getDataFile().getId())); //don't want the findCheapAndEasy + } + + if(guestbookResponse.getSelectedFileIds() != null && !guestbookResponse.getSelectedFileIds().isEmpty()) { //multiple selected through multi-select REquest Access button + selectedDataFiles = datafileService.findAll(guestbookResponse.getSelectedFileIds()); + } + + int countRequestAccessSuccess = 0; + + for(DataFile dataFile : selectedDataFiles){ + guestbookResponse.setDataFile(dataFile); + writeGuestbookResponseRecordForRequestAccess(guestbookResponse); + if(requestAccess(dataFile,guestbookResponse)){ + countRequestAccessSuccess++; + } else { + JsfHelper.addWarningMessage(BundleUtil.getStringFromBundle("file.accessRequested.alreadyRequested", Arrays.asList(dataFile.getDisplayName()))); + } + } + + if(countRequestAccessSuccess > 0){ + DataFile firstDataFile = selectedDataFiles.get(0); + sendRequestFileAccessNotification(firstDataFile.getOwner(), firstDataFile.getId(), (AuthenticatedUser) session.getUser()); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("file.accessRequested.success")); + } + } public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse, FileMetadata fileMetadata, String format) { if(!fileMetadata.getDatasetVersion().isDraft()){ @@ -221,6 +261,18 @@ public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse) { } } + public void writeGuestbookResponseRecordForRequestAccess(GuestbookResponse guestbookResponse) { + try { + CreateGuestbookResponseCommand cmd = new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), guestbookResponse, guestbookResponse.getDataset()); + commandEngine.submit(cmd); + + } catch (CommandException e) { + //if an error occurs here then download won't happen no need for response recs... + logger.info("Failed to writeGuestbookResponseRecord for RequestAccess"); + } + + } + // The "guestBookRecord(s)AlreadyWritten" parameter in the 2 methods // below (redirectToBatchDownloadAPI() and redirectToDownloadAPI(), for the // multiple- and single-file downloads respectively) are passed to the @@ -313,7 +365,7 @@ public void explore(GuestbookResponse guestbookResponse, FileMetadata fmd, Exter String localeCode = session.getLocaleCode(); ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataFile, apiToken, fmd, localeCode); // Persist the name of the tool (i.e. "Data Explorer", etc.) - guestbookResponse.setDownloadtype(externalTool.getDisplayName()); + guestbookResponse.setEventType(externalTool.getDisplayName()); PrimeFaces.current().executeScript(externalToolHandler.getExploreScript()); // This is the old logic from TwoRavens, null checks and all. if (guestbookResponse != null && guestbookResponse.isWriteResponse() @@ -489,7 +541,7 @@ public boolean requestAccess(Long fileId) { return false; } DataFile file = datafileService.find(fileId); - if (!file.containsFileAccessRequestFromUser(session.getUser())) { + if (!file.containsActiveFileAccessRequestFromUser(session.getUser())) { try { commandEngine.submit(new RequestAccessCommand(dvRequestService.getDataverseRequest(), file)); return true; @@ -499,12 +551,33 @@ public boolean requestAccess(Long fileId) { } } return false; - } + } + + public boolean requestAccess(DataFile dataFile, GuestbookResponse gbr){ + boolean accessRequested = false; + if (dvRequestService.getDataverseRequest().getAuthenticatedUser() == null){ + return accessRequested; + } + + if(!dataFile.containsActiveFileAccessRequestFromUser(session.getUser())) { + try { + commandEngine.submit(new RequestAccessCommand(dvRequestService.getDataverseRequest(), dataFile, gbr)); + accessRequested = true; + } catch (CommandException ex) { + logger.info("Unable to request access for file id " + dataFile.getId() + ". Exception: " + ex); + } + } + + return accessRequested; + } - public void sendRequestFileAccessNotification(DataFile datafile, AuthenticatedUser requestor) { - permissionService.getUsersWithPermissionOn(Permission.ManageFilePermissions, datafile).stream().forEach((au) -> { - userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.REQUESTFILEACCESS, datafile.getId(), null, requestor, false); + public void sendRequestFileAccessNotification(Dataset dataset, Long fileId, AuthenticatedUser requestor) { + Timestamp ts = new Timestamp(new Date().getTime()); + permissionService.getUsersWithPermissionOn(Permission.ManageDatasetPermissions, dataset).stream().forEach((au) -> { + userNotificationService.sendNotification(au, ts, UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor, true); }); + //send the user that requested access a notification that they requested the access + userNotificationService.sendNotification(requestor, ts, UserNotification.Type.REQUESTEDFILEACCESS, fileId, null, requestor, true); } diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 49c904c3ac3..bfae80ade27 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -325,6 +325,20 @@ private List sortExternalTools(){ Collections.sort(retList, CompareExternalToolName); return retList; } + + private String termsGuestbookPopupAction = ""; + + public void setTermsGuestbookPopupAction(String popupAction){ + if(popupAction != null && popupAction.length() > 0){ + logger.info("TGPA set to " + popupAction); + this.termsGuestbookPopupAction = popupAction; + } + + } + + public String getTermsGuestbookPopupAction(){ + return termsGuestbookPopupAction; + } public boolean isDownloadPopupRequired() { if(fileMetadata.getId() == null || fileMetadata.getDatasetVersion().getId() == null ){ @@ -340,6 +354,18 @@ public boolean isRequestAccessPopupRequired() { return FileUtil.isRequestAccessPopupRequired(fileMetadata.getDatasetVersion()); } + public boolean isGuestbookAndTermsPopupRequired() { + if(fileMetadata.getId() == null || fileMetadata.getDatasetVersion().getId() == null ){ + return false; + } + return FileUtil.isGuestbookAndTermsPopupRequired(fileMetadata.getDatasetVersion()); + } + + public boolean isGuestbookPopupRequiredAtDownload(){ + // Only show guestbookAtDownload if guestbook at request is disabled (legacy behavior) + DatasetVersion workingVersion = fileMetadata.getDatasetVersion(); + return FileUtil.isGuestbookPopupRequired(workingVersion) && !workingVersion.getDataset().getEffectiveGuestbookEntryAtRequest(); + } public void setFileMetadata(FileMetadata fileMetadata) { this.fileMetadata = fileMetadata; @@ -1245,6 +1271,15 @@ public String getIngestMessage() { public boolean isHasPublicStore() { return settingsWrapper.isTrueForKey(SettingsServiceBean.Key.PublicInstall, StorageIO.isPublicStore(DataAccess.getStorageDriverFromIdentifier(file.getStorageIdentifier()))); } + + //Allows use of fileDownloadHelper in file.xhtml + public FileDownloadHelper getFileDownloadHelper() { + return fileDownloadHelper; + } + + public void setFileDownloadHelper(FileDownloadHelper fileDownloadHelper) { + this.fileDownloadHelper = fileDownloadHelper; + } /** * This method only exists because in file-edit-button-fragment.xhtml we diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java index 0057fbeddab..976f1e084ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java @@ -8,6 +8,8 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.externaltools.ExternalTool; +import edu.harvard.iq.dataverse.util.BundleUtil; + import java.io.Serializable; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -65,8 +67,9 @@ public class GuestbookResponse implements Serializable { @JoinColumn(nullable=true) private AuthenticatedUser authenticatedUser; - @OneToOne(cascade=CascadeType.ALL,mappedBy="guestbookResponse",fetch = FetchType.LAZY, optional = false) - private FileDownload fileDownload; + @OneToMany(mappedBy="guestbookResponse",cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST},fetch = FetchType.LAZY) + //private FileAccessRequest fileAccessRequest; + private List fileAccessRequests; @OneToMany(mappedBy="guestbookResponse",cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST},orphanRemoval=true) @OrderBy ("id") @@ -87,16 +90,37 @@ public class GuestbookResponse implements Serializable { @Temporal(value = TemporalType.TIMESTAMP) private Date responseTime; + + private String sessionId; + private String eventType; + + /** Event Types - there are four pre-defined values in use. + * The type can also be the name of a previewer/explore tool + */ + public static final String ACCESS_REQUEST = "AccessRequest"; + static final String DOWNLOAD = "Download"; + static final String SUBSET = "Subset"; + static final String EXPLORE = "Explore"; + /* Transient Values carry non-written information that will assist in the download process - writeResponse is set to false when dataset version is draft. + - selected file ids is a comma delimited list that contains the file ids for multiple download + - fileFormat tells the download api which format a subsettable file should be downloaded as + */ @Transient private boolean writeResponse = true; + @Transient + private String selectedFileIds; + + @Transient + private String fileFormat; + /** * This transient variable is a place to temporarily retrieve the * ExternalTool object from the popup when the popup is required on the @@ -105,6 +129,7 @@ public class GuestbookResponse implements Serializable { @Transient private ExternalTool externalTool; + public boolean isWriteResponse() { return writeResponse; } @@ -114,19 +139,19 @@ public void setWriteResponse(boolean writeResponse) { } public String getSelectedFileIds(){ - return this.fileDownload.getSelectedFileIds(); + return this.selectedFileIds; } public void setSelectedFileIds(String selectedFileIds) { - this.fileDownload.setSelectedFileIds(selectedFileIds); + this.selectedFileIds = selectedFileIds; } public String getFileFormat() { - return this.fileDownload.getFileFormat(); + return this.fileFormat; } public void setFileFormat(String downloadFormat) { - this.fileDownload.setFileFormat(downloadFormat); + this.fileFormat = downloadFormat; } public ExternalTool getExternalTool() { @@ -138,10 +163,6 @@ public void setExternalTool(ExternalTool externalTool) { } public GuestbookResponse(){ - if(this.getFileDownload() == null){ - this.fileDownload = new FileDownload(); - this.fileDownload.setGuestbookResponse(this); - } } public GuestbookResponse(GuestbookResponse source){ @@ -154,7 +175,7 @@ public GuestbookResponse(GuestbookResponse source){ this.setDataset(source.getDataset()); this.setDatasetVersion(source.getDatasetVersion()); this.setAuthenticatedUser(source.getAuthenticatedUser()); - + this.setSessionId(source.getSessionId()); List customQuestionResponses = new ArrayList<>(); if (!source.getCustomQuestionResponses().isEmpty()){ for (CustomQuestionResponse customQuestionResponse : source.getCustomQuestionResponses() ){ @@ -167,7 +188,6 @@ public GuestbookResponse(GuestbookResponse source){ } this.setCustomQuestionResponses(customQuestionResponses); this.setGuestbook(source.getGuestbook()); - this.setFileDownload(source.getFileDownload()); } @@ -225,17 +245,11 @@ public Date getResponseTime() { public void setResponseTime(Date responseTime) { this.responseTime = responseTime; - this.getFileDownload().setDownloadTimestamp(responseTime); } public String getResponseDate() { return new SimpleDateFormat("MMMM d, yyyy").format(responseTime); } - - public String getResponseDateForDisplay(){ - return null; // SimpleDateFormat("yyyy").format(new Timestamp(new Date().getTime())); - } - public List getCustomQuestionResponses() { return customQuestionResponses; @@ -245,15 +259,14 @@ public void setCustomQuestionResponses(List customQuesti this.customQuestionResponses = customQuestionResponses; } - public FileDownload getFileDownload(){ - return fileDownload; + public List getFileAccessRequests(){ + return fileAccessRequests; } - - public void setFileDownload(FileDownload fDownload){ - this.fileDownload = fDownload; + + public void setFileAccessRequest(List fARs){ + this.fileAccessRequests = fARs; } - public Dataset getDataset() { return dataset; } @@ -286,22 +299,55 @@ public void setAuthenticatedUser(AuthenticatedUser authenticatedUser) { this.authenticatedUser = authenticatedUser; } - public String getDownloadtype() { - return this.fileDownload.getDownloadtype(); + public String getEventType() { + return this.eventType; } - public void setDownloadtype(String downloadtype) { - this.fileDownload.setDownloadtype(downloadtype); + public void setEventType(String eventType) { + this.eventType = eventType; } public String getSessionId() { - return this.fileDownload.getSessionId(); + return this.sessionId; } public void setSessionId(String sessionId) { - this.fileDownload.setSessionId(sessionId); + this.sessionId= sessionId; + } + + public String toHtmlFormattedResponse() { + + StringBuilder sb = new StringBuilder(); + + sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.id") + ": " + getId() + "
        \n"); + sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.date") + ": " + getResponseDate() + "
        \n"); + sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.respondent") + "
          \n
        • " + + BundleUtil.getStringFromBundle("name") + ": " + getName() + "
        • \n
        • "); + sb.append(" " + BundleUtil.getStringFromBundle("email") + ": " + getEmail() + "
        • \n
        • "); + sb.append( + " " + BundleUtil.getStringFromBundle("institution") + ": " + wrapNullAnswer(getInstitution()) + "
        • \n
        • "); + sb.append(" " + BundleUtil.getStringFromBundle("position") + ": " + wrapNullAnswer(getPosition()) + "
        \n"); + sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.guestbook.additionalQuestions") + + ":
          \n"); + + for (CustomQuestionResponse cqr : getCustomQuestionResponses()) { + sb.append("
        • " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.question") + ": " + + cqr.getCustomQuestion().getQuestionString() + "
          " + + BundleUtil.getStringFromBundle("dataset.guestbookResponse.answer") + ": " + + wrapNullAnswer(cqr.getResponse()) + "
        • \n"); + } + sb.append("
        "); + return sb.toString(); + } + + private String wrapNullAnswer(String answer) { + //This assumes we don't have to distinguish null from when the user actually answers "(No Reponse)". The db still has the real value + if (answer == null) { + return BundleUtil.getStringFromBundle("dataset.guestbookResponse.noResponse"); + } + return answer; } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java index bd598d2dca0..b0cc41eb448 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java @@ -63,15 +63,14 @@ public class GuestbookResponseServiceBean { + " and r.dataset_id = o.id " + " and r.guestbook_id = g.id ";*/ - private static final String BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV = "select r.id, g.name, o.id, r.responsetime, f.downloadtype," + private static final String BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV = "select r.id, g.name, o.id, r.responsetime, r.eventtype," + " m.label, r.dataFile_id, r.name, r.email, r.institution, r.position," + " o.protocol, o.authority, o.identifier, d.protocol, d.authority, d.identifier " - + "from guestbookresponse r, filedownload f, filemetadata m, dvobject o, guestbook g, dvobject d " + + "from guestbookresponse r, filemetadata m, dvobject o, guestbook g, dvobject d " + "where " + "m.datasetversion_id = (select max(datasetversion_id) from filemetadata where datafile_id =r.datafile_id ) " + " and m.datafile_id = r.datafile_id " + " and d.id = r.datafile_id " - + " and r.id = f.guestbookresponse_id " + " and r.dataset_id = o.id " + " and r.guestbook_id = g.id "; @@ -79,14 +78,13 @@ public class GuestbookResponseServiceBean { // on the guestbook-results.xhtml page (the info we show on the page is // less detailed than what we let the users download as CSV files, so this // query has fewer fields than the one above). -- L.A. - private static final String BASE_QUERY_STRING_FOR_PAGE_DISPLAY = "select r.id, v.value, r.responsetime, f.downloadtype, m.label, r.name " - + "from guestbookresponse r, filedownload f, datasetfieldvalue v, filemetadata m , dvobject o " + private static final String BASE_QUERY_STRING_FOR_PAGE_DISPLAY = "select r.id, v.value, r.responsetime, r.eventtype, m.label, r.name " + + "from guestbookresponse r, datasetfieldvalue v, filemetadata m , dvobject o " + "where " + " v.datasetfield_id = (select id from datasetfield f where datasetfieldtype_id = 1 " + " and datasetversion_id = (select max(id) from datasetversion where dataset_id =r.dataset_id )) " + " and m.datasetversion_id = (select max(datasetversion_id) from filemetadata where datafile_id =r.datafile_id ) " + " and m.datafile_id = r.datafile_id " - + " and r.id = f.guestbookresponse_id " + " and r.dataset_id = o.id "; // And a custom query for retrieving *all* the custom question responses, for @@ -641,6 +639,9 @@ public GuestbookResponse initGuestbookResponseForFragment(DatasetVersion working GuestbookResponse guestbookResponse = new GuestbookResponse(); + //Not otherwise set for multi-file downloads + guestbookResponse.setDatasetVersion(workingVersion); + if(workingVersion.isDraft()){ guestbookResponse.setWriteResponse(false); } @@ -667,7 +668,7 @@ public GuestbookResponse initGuestbookResponseForFragment(DatasetVersion working if (dataset.getGuestbook() != null && !dataset.getGuestbook().getCustomQuestions().isEmpty()) { initCustomQuestions(guestbookResponse, dataset); } - guestbookResponse.setDownloadtype("Download"); + guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD); guestbookResponse.setDataset(dataset); @@ -721,9 +722,9 @@ public GuestbookResponse initGuestbookResponse(FileMetadata fileMetadata, String if (dataset.getGuestbook() != null && !dataset.getGuestbook().getCustomQuestions().isEmpty()) { initCustomQuestions(guestbookResponse, dataset); } - guestbookResponse.setDownloadtype("Download"); + guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD); if(downloadFormat.toLowerCase().equals("subset")){ - guestbookResponse.setDownloadtype("Subset"); + guestbookResponse.setEventType(GuestbookResponse.SUBSET); } if(downloadFormat.toLowerCase().equals("explore")){ /** @@ -734,12 +735,12 @@ public GuestbookResponse initGuestbookResponse(FileMetadata fileMetadata, String * "externalTool" for all external tools, including TwoRavens. When * clicking "Explore" and then the name of the tool, we want the * name of the exploration tool (i.e. "Data Explorer", - * etc.) to be persisted as the downloadType. We execute - * guestbookResponse.setDownloadtype(externalTool.getDisplayName()) + * etc.) to be persisted as the eventType. We execute + * guestbookResponse.setEventType(externalTool.getDisplayName()) * over in the "explore" method of FileDownloadServiceBean just * before the guestbookResponse is written. */ - guestbookResponse.setDownloadtype("Explore"); + guestbookResponse.setEventType(GuestbookResponse.EXPLORE); } guestbookResponse.setDataset(dataset); @@ -818,7 +819,7 @@ public GuestbookResponse initDefaultGuestbookResponse(Dataset dataset, DataFile guestbookResponse.setDataset(dataset); guestbookResponse.setResponseTime(new Date()); guestbookResponse.setSessionId(session.toString()); - guestbookResponse.setDownloadtype("Download"); + guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD); setUserDefaultResponses(guestbookResponse, session); return guestbookResponse; } @@ -839,7 +840,7 @@ public GuestbookResponse initAPIGuestbookResponse(Dataset dataset, DataFile data guestbookResponse.setDataset(dataset); guestbookResponse.setResponseTime(new Date()); guestbookResponse.setSessionId(session.toString()); - guestbookResponse.setDownloadtype("Download"); + guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD); setUserDefaultResponses(guestbookResponse, session, user); return guestbookResponse; } @@ -903,29 +904,36 @@ public void save(GuestbookResponse guestbookResponse) { em.persist(guestbookResponse); } + + /* + * Metrics - download counts from GuestbookResponses: Any GuestbookResponse that + * is not of eventtype=='AccessRequest' is considered a download. This includes + * actual 'Download's, downloads of 'Subset's, and use by 'Explore' tools and + * previewers (where eventtype is the previewer name) + */ - public Long getCountGuestbookResponsesByDataFileId(Long dataFileId) { + public Long getDownloadCountByDataFileId(Long dataFileId) { // datafile id is null, will return 0 - Query query = em.createNativeQuery("select count(o.id) from GuestbookResponse o where o.datafile_id = " + dataFileId); + Query query = em.createNativeQuery("select count(o.id) from GuestbookResponse o where o.datafile_id = " + dataFileId + "and eventtype != '" + GuestbookResponse.ACCESS_REQUEST +"'"); return (Long) query.getSingleResult(); } - public Long getCountGuestbookResponsesByDatasetId(Long datasetId) { - return getCountGuestbookResponsesByDatasetId(datasetId, null); + public Long getDownloadCountByDatasetId(Long datasetId) { + return getDownloadCountByDatasetId(datasetId, null); } - public Long getCountGuestbookResponsesByDatasetId(Long datasetId, LocalDate date) { + public Long getDownloadCountByDatasetId(Long datasetId, LocalDate date) { // dataset id is null, will return 0 Query query; if(date != null) { - query = em.createNativeQuery("select count(o.id) from GuestbookResponse o where o.dataset_id = " + datasetId + " and responsetime < '" + date.toString() + "'"); + query = em.createNativeQuery("select count(o.id) from GuestbookResponse o where o.dataset_id = " + datasetId + " and responsetime < '" + date.toString() + "' and eventtype != '" + GuestbookResponse.ACCESS_REQUEST +"'"); }else { - query = em.createNativeQuery("select count(o.id) from GuestbookResponse o where o.dataset_id = " + datasetId); + query = em.createNativeQuery("select count(o.id) from GuestbookResponse o where o.dataset_id = " + datasetId+ "and eventtype != '" + GuestbookResponse.ACCESS_REQUEST +"'"); } return (Long) query.getSingleResult(); } - public Long getCountOfAllGuestbookResponses() { + public Long getTotalDownloadCount() { // dataset id is null, will return 0 // "SELECT COUNT(*)" is notoriously expensive in PostgresQL for large @@ -954,10 +962,12 @@ public Long getCountOfAllGuestbookResponses() { } catch (IllegalArgumentException iae) { // Don't do anything, we'll fall back to using "SELECT COUNT()" } - Query query = em.createNativeQuery("select count(o.id) from GuestbookResponse o;"); + Query query = em.createNativeQuery("select count(o.id) from GuestbookResponse o where eventtype != '" + GuestbookResponse.ACCESS_REQUEST +"';"); return (Long) query.getSingleResult(); } + //End Metrics/download counts + public List findByAuthenticatedUserId(AuthenticatedUser user) { Query query = em.createNamedQuery("GuestbookResponse.findByAuthenticatedUserId"); query.setParameter("authenticatedUserId", user.getId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index f17732df7b6..72fc6ee6d64 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -382,12 +382,21 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio logger.fine(dataverseCreatedMessage); return messageText += dataverseCreatedMessage; case REQUESTFILEACCESS: + //Notification to those who can grant file access requests on a dataset when a user makes a request DataFile datafile = (DataFile) targetObject; + pattern = BundleUtil.getStringFromBundle("notification.email.requestFileAccess"); String requestorName = (requestor.getLastName() != null && requestor.getLastName() != null) ? requestor.getFirstName() + " " + requestor.getLastName() : BundleUtil.getStringFromBundle("notification.email.info.unavailable"); String requestorEmail = requestor.getEmail() != null ? requestor.getEmail() : BundleUtil.getStringFromBundle("notification.email.info.unavailable"); String[] paramArrayRequestFileAccess = {datafile.getOwner().getDisplayName(), requestorName, requestorEmail, getDatasetManageFileAccessLink(datafile)}; + messageText = BundleUtil.getStringFromBundle("notification.email.greeting.html"); messageText += MessageFormat.format(pattern, paramArrayRequestFileAccess); + FileAccessRequest far = datafile.getAccessRequestForAssignee(requestor); + GuestbookResponse gbr = far.getGuestbookResponse(); + if (gbr != null) { + messageText += MessageFormat.format( + BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"), gbr.toHtmlFormattedResponse()); + } return messageText; case GRANTFILEACCESS: dataset = (Dataset) targetObject; @@ -630,6 +639,20 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio dataset.getDisplayName()}; messageText = MessageFormat.format(pattern, paramArrayDatasetMentioned); return messageText; + case REQUESTEDFILEACCESS: + //Notification to requestor when they make a request + datafile = (DataFile) targetObject; + + pattern = BundleUtil.getStringFromBundle("notification.email.requestedFileAccess"); + messageText = BundleUtil.getStringFromBundle("notification.email.greeting.html"); + messageText += MessageFormat.format(pattern, getDvObjectLink(datafile), datafile.getOwner().getDisplayName()); + far = datafile.getAccessRequestForAssignee(requestor); + gbr = far.getGuestbookResponse(); + if (gbr != null) { + messageText += MessageFormat.format( + BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"), gbr.toHtmlFormattedResponse()); + } + return messageText; } return ""; @@ -650,6 +673,7 @@ public Object getObjectOfNotification (UserNotification userNotification){ case CREATEDV: return dataverseService.find(userNotification.getObjectId()); case REQUESTFILEACCESS: + case REQUESTEDFILEACCESS: return dataFileService.find(userNotification.getObjectId()); case GRANTFILEACCESS: case REJECTFILEACCESS: diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 1b4af29c915..ca2f6145cba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -27,6 +27,8 @@ import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; + import jakarta.ejb.EJB; import jakarta.faces.application.FacesMessage; import jakarta.faces.event.ActionEvent; @@ -71,6 +73,8 @@ public class ManageFilePermissionsPage implements java.io.Serializable { DataverseRequestServiceBean dvRequestService; @Inject PermissionsWrapper permissionsWrapper; + @EJB + FileAccessRequestServiceBean fileAccessRequestService; @PersistenceContext(unitName = "VDCNet-ejbPU") EntityManager em; @@ -85,6 +89,15 @@ public class ManageFilePermissionsPage implements java.io.Serializable { public TreeMap> getFileAccessRequestMap() { return fileAccessRequestMap; } + + public List getDataFilesForRequestor() { + List fars = fileAccessRequestMap.get(getFileRequester()); + if (fars == null) { + return new ArrayList<>(); + } else { + return fars.stream().map(FileAccessRequest::getDataFile).collect(Collectors.toList()); + } + } private final TreeMap> fileAccessRequestMap = new TreeMap<>(); private boolean showDeleted = true; @@ -177,14 +190,14 @@ private void initMaps() { fileMap.put(file, raList); // populate the file access requests map - for (FileAccessRequest fileAccessRequest : file.getFileAccessRequests()) { - List requestedFiles = fileAccessRequestMap.get(fileAccessRequest.getAuthenticatedUser()); - if (requestedFiles == null) { - requestedFiles = new ArrayList<>(); - AuthenticatedUser withProvider = authenticationService.getAuthenticatedUserWithProvider(fileAccessRequest.getAuthenticatedUser().getUserIdentifier()); - fileAccessRequestMap.put(withProvider, requestedFiles); + for (FileAccessRequest fileAccessRequest : file.getFileAccessRequests(FileAccessRequest.RequestState.CREATED)) { + List fileAccessRequestList = fileAccessRequestMap.get(fileAccessRequest.getRequester()); + if (fileAccessRequestList == null) { + fileAccessRequestList = new ArrayList<>(); + AuthenticatedUser withProvider = authenticationService.getAuthenticatedUserWithProvider(fileAccessRequest.getRequester().getUserIdentifier()); + fileAccessRequestMap.put(withProvider, fileAccessRequestList); } - requestedFiles.add(fileAccessRequest); + fileAccessRequestList.add(fileAccessRequest); } } } @@ -406,16 +419,19 @@ public void grantAccess(ActionEvent evt) { if (file.isReleased()) { sendNotification = true; } - // remove request, if it exist - if (file.removeFileAccessRequester(roleAssignee)) { - datafileService.save(file); + // set request(s) granted, if they exist + for (AuthenticatedUser au : roleAssigneeService.getExplicitUsers(roleAssignee)) { + FileAccessRequest far = file.getAccessRequestForAssignee(au); + far.setStateGranted(); } + datafileService.save(file); } + } if (sendNotification) { for (AuthenticatedUser au : roleAssigneeService.getExplicitUsers(roleAssignee)) { - userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.GRANTFILEACCESS, dataset.getId()); + userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.GRANTFILEACCESS, dataset.getId()); } } } @@ -443,7 +459,9 @@ private void grantAccessToRequests(AuthenticatedUser au, List files) { DataverseRole fileDownloaderRole = roleService.findBuiltinRoleByAlias(DataverseRole.FILE_DOWNLOADER); for (DataFile file : files) { if (assignRole(au, file, fileDownloaderRole)) { - if (file.removeFileAccessRequester(au)) { + FileAccessRequest far = file.getAccessRequestForAssignee(au); + if (far!=null) { + far.setStateGranted(); datafileService.save(file); } actionPerformed = true; @@ -475,9 +493,14 @@ public void rejectAccessToAllRequests(AuthenticatedUser au) { private void rejectAccessToRequests(AuthenticatedUser au, List files) { boolean actionPerformed = false; for (DataFile file : files) { - file.removeFileAccessRequester(au); - datafileService.save(file); - actionPerformed = true; + FileAccessRequest far = file.getAccessRequestForAssignee(au); + if(far!=null) { + far.setStateRejected(); + fileAccessRequestService.save(far); + file.removeFileAccessRequest(far); + datafileService.save(file); + actionPerformed = true; + } } if (actionPerformed) { diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index bf78b9d088f..0e277c5aa32 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -55,6 +55,8 @@ public class ManagePermissionsPage implements java.io.Serializable { @EJB DvObjectServiceBean dvObjectService; @EJB + FileAccessRequestServiceBean fileAccessRequestService; + @EJB DataverseRoleServiceBean roleService; @EJB RoleAssigneeServiceBean roleAssigneeService; diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java index 307301049f0..0a1d0effc03 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.logging.Logger; import java.util.Set; @@ -594,7 +595,7 @@ public Map getBaseMetadataLanguageMap(boolean refresh) { public Map getMetadataLanguages(DvObjectContainer target) { Map currentMap = new HashMap(); currentMap.putAll(getBaseMetadataLanguageMap(false)); - currentMap.put(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE, getDefaultMetadataLanguageLabel(target)); + currentMap.put(DvObjectContainer.UNDEFINED_CODE, getDefaultMetadataLanguageLabel(target)); return currentMap; } @@ -605,7 +606,7 @@ private String getDefaultMetadataLanguageLabel(DvObjectContainer target) { String mlCode = target.getOwner().getEffectiveMetadataLanguage(); // If it's undefined, no parent has a metadata language defined, and the global default should be used. - if (!mlCode.equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE)) { + if (!mlCode.equals(DvObjectContainer.UNDEFINED_CODE)) { // Get the label for the language code found mlLabel = getBaseMetadataLanguageMap(false).get(mlCode); mlLabel = mlLabel + " " + BundleUtil.getStringFromBundle("dataverse.inherited"); @@ -623,7 +624,7 @@ public String getDefaultMetadataLanguage() { return (String) mdMap.keySet().toArray()[0]; } else { //More than one - :MetadataLanguages is set and the default is undefined (users must choose if the collection doesn't override the default) - return DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE; + return DvObjectContainer.UNDEFINED_CODE; } } else { // None - :MetadataLanguages is not set so return null to turn off the display (backward compatibility) @@ -631,6 +632,32 @@ public String getDefaultMetadataLanguage() { } } + public Map getGuestbookEntryOptions(DvObjectContainer target) { + Map currentMap = new HashMap(); + String atDownload = BundleUtil.getStringFromBundle("dataverse.guestbookentry.atdownload"); + String atRequest = BundleUtil.getStringFromBundle("dataverse.guestbookentry.atrequest"); + Optional gbDefault = JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class); + if (gbDefault.isPresent()) { + // Three options - inherited/default option, at Download, at Request + String useDefault = null; + if (target.getOwner() == null) { + boolean defaultOption = gbDefault.get(); + useDefault = (defaultOption ? atRequest : atDownload) + + BundleUtil.getStringFromBundle("dataverse.default"); + } else { + boolean defaultOption = target.getOwner().getEffectiveGuestbookEntryAtRequest(); + useDefault = (defaultOption ? atRequest : atDownload) + + BundleUtil.getStringFromBundle("dataverse.inherited"); + } + currentMap.put(DvObjectContainer.UNDEFINED_CODE, useDefault); + currentMap.put(Boolean.toString(true), atRequest); + currentMap.put(Boolean.toString(false), atDownload); + } else { + // Setting not defined - leave empty + } + return currentMap; + } + public Dataverse getRootDataverse() { if (rootDataverse == null) { rootDataverse = dataverseService.findRootDataverse(); diff --git a/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java b/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java index fff520fd259..44070dcbb41 100644 --- a/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java @@ -148,6 +148,7 @@ public String init() { editMode = TemplatePage.EditMode.CREATE; template = new Template(this.dataverse, settingsWrapper.getSystemMetadataBlocks()); TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); + terms.setFileAccessRequest(true); terms.setTemplate(template); terms.setLicense(licenseServiceBean.getDefault()); template.setTermsOfUseAndAccess(terms); diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java index a87404b69a7..280c2075494 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java @@ -39,7 +39,7 @@ public enum Type { CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED, INGESTCOMPLETED, INGESTCOMPLETEDWITHERRORS, PUBLISHFAILED_PIDREG, WORKFLOW_SUCCESS, WORKFLOW_FAILURE, STATUSUPDATED, DATASETCREATED, DATASETMENTIONED, GLOBUSUPLOADCOMPLETED, GLOBUSUPLOADCOMPLETEDWITHERRORS, - GLOBUSDOWNLOADCOMPLETED, GLOBUSDOWNLOADCOMPLETEDWITHERRORS; + GLOBUSDOWNLOADCOMPLETED, GLOBUSDOWNLOADCOMPLETEDWITHERRORS, REQUESTEDFILEACCESS; public String getDescription() { return BundleUtil.getStringFromBundle("notification.typeDescription." + this.name()); diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java index a2a71ff8b40..228e4b19c38 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java @@ -131,6 +131,7 @@ public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate save(userNotification); } } + public boolean isEmailMuted(UserNotification userNotification) { final Type type = userNotification.getType(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 256af8ec65e..014604bff82 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -1427,7 +1427,7 @@ public Response requestFileAccess(@Context ContainerRequestContext crc, @PathPar return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.invalidRequest")); } - if (dataFile.containsFileAccessRequestFromUser(requestor)) { + if (dataFile.containsActiveFileAccessRequestFromUser(requestor)) { return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.requestExists")); } @@ -1478,17 +1478,17 @@ public Response listFileAccessRequests(@Context ContainerRequestContext crc, @Pa return error(FORBIDDEN, BundleUtil.getStringFromBundle("access.api.rejectAccess.failure.noPermissions")); } - List requests = dataFile.getFileAccessRequests(); + List requests = dataFile.getFileAccessRequests(FileAccessRequest.RequestState.CREATED); if (requests == null || requests.isEmpty()) { List args = Arrays.asList(dataFile.getDisplayName()); - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestList.noRequestsFound", args)); + return error(Response.Status.NOT_FOUND, BundleUtil.getStringFromBundle("access.api.requestList.noRequestsFound", args)); } JsonArrayBuilder userArray = Json.createArrayBuilder(); for (FileAccessRequest fileAccessRequest : requests) { - userArray.add(json(fileAccessRequest.getAuthenticatedUser())); + userArray.add(json(fileAccessRequest.getRequester())); } return ok(userArray); @@ -1534,7 +1534,9 @@ public Response grantFileAccess(@Context ContainerRequestContext crc, @PathParam try { engineSvc.submit(new AssignRoleCommand(ra, fileDownloaderRole, dataFile, dataverseRequest, null)); - if (dataFile.removeFileAccessRequester(ra)) { + FileAccessRequest far = dataFile.getAccessRequestForAssignee(ra); + if(far!=null) { + far.setStateGranted(); dataFileService.save(dataFile); } @@ -1659,20 +1661,21 @@ public Response rejectFileAccess(@Context ContainerRequestContext crc, @PathPara if (!(dataverseRequest.getAuthenticatedUser().isSuperuser() || permissionService.requestOn(dataverseRequest, dataFile).has(Permission.ManageFilePermissions))) { return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.rejectAccess.failure.noPermissions")); } - - if (dataFile.removeFileAccessRequester(ra)) { + FileAccessRequest far = dataFile.getAccessRequestForAssignee(ra); + if (far != null) { + far.setStateRejected(); dataFileService.save(dataFile); try { AuthenticatedUser au = (AuthenticatedUser) ra; - userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.REJECTFILEACCESS, dataFile.getOwner().getId()); + userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), + UserNotification.Type.REJECTFILEACCESS, dataFile.getOwner().getId()); } catch (ClassCastException e) { - //nothing to do here - can only send a notification to an authenticated user + // nothing to do here - can only send a notification to an authenticated user } List args = Arrays.asList(dataFile.getDisplayName()); return ok(BundleUtil.getStringFromBundle("access.api.rejectAccess.success.for.single.file", args)); - } else { List args = Arrays.asList(dataFile.getDisplayName(), ra.getDisplayInfo().getTitle()); return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.fileAccess.rejectFailure.noRequest", args)); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index d082d9c29da..bd0f29463f5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3917,4 +3917,90 @@ public Response getDatasetVersionCitation(@Context ContainerRequestContext crc, return response(req -> ok( getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers).getCitation(true, false)), getRequestUser(crc)); } + + @GET + @AuthRequired + @Path("{identifier}/guestbookEntryAtRequest") + public Response getGuestbookEntryOption(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, + @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + + Dataset dataset; + + try { + dataset = findDatasetOrDie(dvIdtf); + } catch (WrappedResponse ex) { + return error(Response.Status.NOT_FOUND, "No such dataset"); + } + String gbAtRequest = dataset.getGuestbookEntryAtRequest(); + if(gbAtRequest == null || gbAtRequest.equals(DvObjectContainer.UNDEFINED_CODE)) { + return ok("Not set on dataset, using the default: " + dataset.getEffectiveGuestbookEntryAtRequest()); + } + return ok(dataset.getEffectiveGuestbookEntryAtRequest()); + } + + @PUT + @AuthRequired + @Path("{identifier}/guestbookEntryAtRequest") + public Response setguestbookEntryAtRequest(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, + boolean gbAtRequest, + @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + + // Superuser-only: + AuthenticatedUser user; + try { + user = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Response.Status.BAD_REQUEST, "Authentication is required."); + } + if (!user.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "Superusers only."); + } + + Dataset dataset; + + try { + dataset = findDatasetOrDie(dvIdtf); + } catch (WrappedResponse ex) { + return error(Response.Status.NOT_FOUND, "No such dataset"); + } + Optional gbAtRequestOpt = JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class); + if (!gbAtRequestOpt.isPresent()) { + return error(Response.Status.FORBIDDEN, "Guestbook Entry At Request cannot be set. This server is not configured to allow it."); + } + String choice = Boolean.valueOf(gbAtRequest).toString(); + dataset.setGuestbookEntryAtRequest(choice); + datasetService.merge(dataset); + return ok("Guestbook Entry At Request set to: " + choice); + } + + @DELETE + @AuthRequired + @Path("{identifier}/guestbookEntryAtRequest") + public Response resetGuestbookEntryAtRequest(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, + @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + + // Superuser-only: + AuthenticatedUser user; + try { + user = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Response.Status.BAD_REQUEST, "Authentication is required."); + } + if (!user.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "Superusers only."); + } + + Dataset dataset; + + try { + dataset = findDatasetOrDie(dvIdtf); + } catch (WrappedResponse ex) { + return error(Response.Status.NOT_FOUND, "No such dataset"); + } + + dataset.setGuestbookEntryAtRequest(DvObjectContainer.UNDEFINED_CODE); + datasetService.merge(dataset); + return ok("Guestbook Entry At Request reset to default: " + dataset.getEffectiveGuestbookEntryAtRequest()); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 760a6d5526d..d83a55a2914 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -840,7 +840,7 @@ public Response getFixityAlgorithm() { public Response getFileDownloadCount(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) { return response(req -> { DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId))); - return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFile.getId()).toString()); + return ok(guestbookResponseService.getDownloadCountByDataFileId(dataFile.getId()).toString()); }, getRequestUser(crc)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java index 93b7dc96563..15838a09456 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java @@ -6,17 +6,21 @@ import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseRequestServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException; +import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota; +import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDataFilesCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.ConstraintViolationUtil; -import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -66,6 +70,8 @@ public class MediaResourceManagerImpl implements MediaResourceManager { SwordAuth swordAuth; @Inject UrlManager urlManager; + @Inject + DataverseRequestServiceBean dvRequestService; private HttpServletRequest httpRequest; @@ -298,37 +304,42 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au */ String guessContentTypeForMe = null; List dataFiles = new ArrayList<>(); + try { - try { - CreateDataFileResult createDataFilesResponse = FileUtil.createDataFiles(editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null, systemConfig); - dataFiles = createDataFilesResponse.getDataFiles(); - } catch (EJBException ex) { - Throwable cause = ex.getCause(); - if (cause != null) { - if (cause instanceof IllegalArgumentException) { - /** - * @todo should be safe to remove this catch of - * EJBException and IllegalArgumentException once - * this ticket is resolved: - * - * IllegalArgumentException: MALFORMED when - * uploading certain zip files - * https://github.com/IQSS/dataverse/issues/1021 - */ - throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Exception caught calling ingestService.createDataFiles. Problem with zip file, perhaps: " + cause); - } else { - throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Exception caught calling ingestService.createDataFiles: " + cause); - } + //CreateDataFileResult createDataFilesResponse = FileUtil.createDataFiles(editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null, systemConfig); + UserStorageQuota quota = null; + if (systemConfig.isStorageQuotasEnforced()) { + quota = dataFileService.getUserStorageQuota(user, dataset); + } + Command cmd = new CreateNewDataFilesCommand(dvReq, editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, quota, null); + CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd); + dataFiles = createDataFilesResult.getDataFiles(); + } catch (CommandException ex) { + Throwable cause = ex.getCause(); + if (cause != null) { + if (cause instanceof IllegalArgumentException) { + /** + * @todo should be safe to remove this catch of + * EJBException and IllegalArgumentException once this + * ticket is resolved: + * + * IllegalArgumentException: MALFORMED when uploading + * certain zip files + * https://github.com/IQSS/dataverse/issues/1021 + */ + throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset. Problem with zip file, perhaps: " + cause); } else { - throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Exception caught calling ingestService.createDataFiles. No cause: " + ex.getMessage()); + throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + cause); } - } /*TODO: L.A. 4.6! catch (FileExceedsMaxSizeException ex) { + } else { + throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + ex.getMessage()); + } + } + /*TODO: L.A. 4.6! catch (FileExceedsMaxSizeException ex) { throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Exception caught calling ingestService.createDataFiles: " + ex.getMessage()); //Logger.getLogger(MediaResourceManagerImpl.class.getName()).log(Level.SEVERE, null, ex); - }*/ - } catch (IOException ex) { - throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + ex.getMessage()); - } + }*/ + if (!dataFiles.isEmpty()) { Set constraintViolations = editVersion.validate(); if (constraintViolations.size() > 0) { diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index dc4644dfccd..a0e3f899443 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -488,6 +488,7 @@ public void displayNotification() { break; case REQUESTFILEACCESS: + case REQUESTEDFILEACCESS: DataFile file = fileService.find(userNotification.getObjectId()); if (file != null) { userNotification.setTheObject(file.getOwner()); diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index 89429b912f6..3cbfc3cdcac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -1,7 +1,9 @@ package edu.harvard.iq.dataverse.authorization.users; import edu.harvard.iq.dataverse.Cart; +import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DatasetLock; +import edu.harvard.iq.dataverse.FileAccessRequest; import edu.harvard.iq.dataverse.UserNotification.Type; import edu.harvard.iq.dataverse.UserNotification; import edu.harvard.iq.dataverse.validation.ValidateEmail; @@ -17,6 +19,7 @@ import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import java.io.Serializable; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Date; import java.util.HashSet; import java.util.List; @@ -28,6 +31,7 @@ import jakarta.persistence.CascadeType; import jakarta.persistence.Column; import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; @@ -38,8 +42,8 @@ import jakarta.persistence.PostLoad; import jakarta.persistence.PrePersist; import jakarta.persistence.Transient; +import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; -import org.hibernate.validator.constraints.NotBlank; /** * When adding an attribute to this class, be sure to update the following: @@ -202,6 +206,29 @@ public void setDatasetLocks(List datasetLocks) { @OneToMany(mappedBy = "user", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}) private List oAuth2TokenDatas; + /*for many to many fileAccessRequests*/ + @OneToMany(mappedBy = "user", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST, CascadeType.REFRESH}, fetch = FetchType.LAZY) + private List fileAccessRequests; + + public List getFileAccessRequests() { + return fileAccessRequests; + } + + public void setFileAccessRequests(List fARs) { + this.fileAccessRequests = fARs; + } + + public List getRequestedDataFiles(){ + List requestedDataFiles = new ArrayList<>(); + + for(FileAccessRequest far : getFileAccessRequests()){ + if(far.isStateCreated()) { + requestedDataFiles.add(far.getDataFile()); + } + } + return requestedDataFiles; + } + @Override public AuthenticatedUserDisplayInfo getDisplayInfo() { return new AuthenticatedUserDisplayInfo(firstName, lastName, email, affiliation, position); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java index bfd5c5f0d8f..00db98e894e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java @@ -606,7 +606,8 @@ public static String getDriverPrefix(String driverId) { } public static boolean isDirectUploadEnabled(String driverId) { - return Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")); + return (DataAccess.S3.equals(driverId) && Boolean.parseBoolean(System.getProperty("dataverse.files." + DataAccess.S3 + ".upload-redirect"))) || + Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-out-of-band")); } //Check that storageIdentifier is consistent with store's config diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 98d5afc47e6..d44388f39f7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -27,7 +27,6 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.file.CreateDataFileResult; import edu.harvard.iq.dataverse.util.json.JsonPrinter; @@ -61,7 +60,7 @@ import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import org.apache.commons.io.IOUtils; - +import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDataFilesCommand; import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; /** @@ -1204,17 +1203,24 @@ private boolean step_030_createNewFilesViaIngest(){ clone = workingVersion.cloneDatasetVersion(); } try { - CreateDataFileResult result = FileUtil.createDataFiles(workingVersion, + /*CreateDataFileResult result = FileUtil.createDataFiles(workingVersion, this.newFileInputStream, this.newFileName, this.newFileContentType, this.newStorageIdentifier, this.newCheckSum, this.newCheckSumType, - this.systemConfig); - initialFileList = result.getDataFiles(); + this.systemConfig);*/ + + DataFileServiceBean.UserStorageQuota quota = null; + if (systemConfig.isStorageQuotasEnforced()) { + quota = fileService.getUserStorageQuota(dvRequest.getAuthenticatedUser(), dataset); + } + Command cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType); + CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd); + initialFileList = createDataFilesResult.getDataFiles(); - } catch (IOException ex) { + } catch (CommandException ex) { if (ex.getMessage() != null && !ex.getMessage().isEmpty()) { this.addErrorSevere(getBundleErr("ingest_create_file_err") + " " + ex.getMessage()); } else { diff --git a/src/main/java/edu/harvard/iq/dataverse/dataverse/DataverseUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataverse/DataverseUtil.java index 7964c56835e..f45a9058e7c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataverse/DataverseUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataverse/DataverseUtil.java @@ -104,7 +104,7 @@ public static void checkMetadataLangauge(Dataset ds, Dataverse owner, Map { + private static final Logger logger = Logger.getLogger(CreateNewDataFilesCommand.class.getCanonicalName()); + + private final DatasetVersion version; + private final InputStream inputStream; + private final String fileName; + private final String suppliedContentType; + private final UserStorageQuota quota; + // parent Dataverse must be specified when the command is called on Create + // of a new dataset that does not exist in the database yet (for the purposes + // of authorization - see getRequiredPermissions() below): + private final Dataverse parentDataverse; + // With Direct Upload the following values already exist and are passed to the command: + private final String newStorageIdentifier; + private final String newCheckSum; + private DataFile.ChecksumType newCheckSumType; + private final Long newFileSize; + + public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum) { + this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, null); + } + + public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum, DataFile.ChecksumType newCheckSumType) { + this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType, null, null); + } + + // This version of the command must be used when files are created in the + // context of creating a brand new dataset (from the Add Dataset page): + + public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum, DataFile.ChecksumType newCheckSumType, Long newFileSize, Dataverse dataverse) { + super(aRequest, dataverse); + + this.version = version; + this.inputStream = inputStream; + this.fileName = fileName; + this.suppliedContentType = suppliedContentType; + this.newStorageIdentifier = newStorageIdentifier; + this.newCheckSum = newCheckSum; + this.newCheckSumType = newCheckSumType; + this.parentDataverse = dataverse; + this.quota = quota; + this.newFileSize = newFileSize; + } + + + @Override + public CreateDataFileResult execute(CommandContext ctxt) throws CommandException { + List datafiles = new ArrayList<>(); + + //When there is no checksum/checksumtype being sent (normal upload, needs to be calculated), set the type to the current default + if(newCheckSumType == null) { + newCheckSumType = ctxt.systemConfig().getFileFixityChecksumAlgorithm(); + } + + String warningMessage = null; + + // save the file, in the temporary location for now: + Path tempFile = null; + + Long fileSizeLimit = ctxt.systemConfig().getMaxFileUploadSizeForStore(version.getDataset().getEffectiveStorageDriverId()); + Long storageQuotaLimit = null; + + if (ctxt.systemConfig().isStorageQuotasEnforced()) { + if (quota != null) { + storageQuotaLimit = quota.getRemainingQuotaInBytes(); + } + } + String finalType = null; + + if (newStorageIdentifier == null) { + if (getFilesTempDirectory() != null) { + try { + tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload"); + // "temporary" location is the key here; this is why we are not using + // the DataStore framework for this - the assumption is that + // temp files will always be stored on the local filesystem. + // -- L.A. Jul. 2014 + logger.fine("Will attempt to save the file as: " + tempFile.toString()); + Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException ioex) { + throw new CommandExecutionException("Failed to save the upload as a temp file (temp disk space?)", ioex, this); + } + + // A file size check, before we do anything else: + // (note that "no size limit set" = "unlimited") + // (also note, that if this is a zip file, we'll be checking + // the size limit for each of the individual unpacked files) + Long fileSize = tempFile.toFile().length(); + if (fileSizeLimit != null && fileSize > fileSizeLimit) { + try { + tempFile.toFile().delete(); + } catch (Exception ex) { + // ignore - but log a warning + logger.warning("Could not remove temp file " + tempFile.getFileName()); + } + throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit)), this); + } + + } else { + throw new CommandExecutionException("Temp directory is not configured.", this); + } + + logger.fine("mime type supplied: " + suppliedContentType); + + // Let's try our own utilities (Jhove, etc.) to determine the file type + // of the uploaded file. (We may already have a mime type supplied for this + // file - maybe the type that the browser recognized on upload; or, if + // it's a harvest, maybe the remote server has already given us the type + // for this file... with our own type utility we may or may not do better + // than the type supplied: + // -- L.A. + String recognizedType = null; + + try { + recognizedType = determineFileType(tempFile.toFile(), fileName); + logger.fine("File utility recognized the file as " + recognizedType); + if (recognizedType != null && !recognizedType.equals("")) { + if (useRecognizedType(suppliedContentType, recognizedType)) { + finalType = recognizedType; + } + } + + } catch (Exception ex) { + logger.warning("Failed to run the file utility mime type check on file " + fileName); + } + + if (finalType == null) { + finalType = (suppliedContentType == null || suppliedContentType.equals("")) + ? MIME_TYPE_UNDETERMINED_DEFAULT + : suppliedContentType; + } + + // A few special cases: + // if this is a gzipped FITS file, we'll uncompress it, and ingest it as + // a regular FITS file: + if (finalType.equals("application/fits-gzipped")) { + + InputStream uncompressedIn = null; + String finalFileName = fileName; + // if the file name had the ".gz" extension, remove it, + // since we are going to uncompress it: + if (fileName != null && fileName.matches(".*\\.gz$")) { + finalFileName = fileName.replaceAll("\\.gz$", ""); + } + + DataFile datafile = null; + long fileSize = 0L; + try { + uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile())); + File unZippedTempFile = saveInputStreamInTempFile(uncompressedIn, fileSizeLimit, storageQuotaLimit); + fileSize = unZippedTempFile.length(); + datafile = FileUtil.createSingleDataFile(version, unZippedTempFile, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT, ctxt.systemConfig().getFileFixityChecksumAlgorithm()); + } catch (IOException | FileExceedsMaxSizeException | FileExceedsStorageQuotaException ioex) { + // it looks like we simply skip the file silently, if its uncompressed size + // exceeds the limit. we should probably report this in detail instead. + datafile = null; + } finally { + if (uncompressedIn != null) { + try { + uncompressedIn.close(); + } catch (IOException e) { + } + } + } + + // If we were able to produce an uncompressed file, we'll use it + // to create and return a final DataFile; if not, we're not going + // to do anything - and then a new DataFile will be created further + // down, from the original, uncompressed file. + if (datafile != null) { + // remove the compressed temp file: + try { + tempFile.toFile().delete(); + } catch (SecurityException ex) { + // (this is very non-fatal) + logger.warning("Failed to delete temporary file " + tempFile.toString()); + } + + datafiles.add(datafile); + // Update quota if present + if (quota != null) { + quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + fileSize); + } + return CreateDataFileResult.success(fileName, finalType, datafiles); + } + + // If it's a ZIP file, we are going to unpack it and create multiple + // DataFile objects from its contents: + } else if (finalType.equals("application/zip")) { + + ZipFile zipFile = null; + ZipInputStream unZippedIn = null; + ZipEntry zipEntry = null; + + int fileNumberLimit = ctxt.systemConfig().getZipUploadFilesLimit(); + Long combinedUnzippedFileSize = 0L; + + try { + Charset charset = null; + /* + TODO: (?) + We may want to investigate somehow letting the user specify + the charset for the filenames in the zip file... + - otherwise, ZipInputStream bails out if it encounteres a file + name that's not valid in the current charest (i.e., UTF-8, in + our case). It would be a bit trickier than what we're doing for + SPSS tabular ingests - with the lang. encoding pulldown menu - + because this encoding needs to be specified *before* we upload and + attempt to unzip the file. + -- L.A. 4.0 beta12 + logger.info("default charset is "+Charset.defaultCharset().name()); + if (Charset.isSupported("US-ASCII")) { + logger.info("charset US-ASCII is supported."); + charset = Charset.forName("US-ASCII"); + if (charset != null) { + logger.info("was able to obtain charset for US-ASCII"); + } + + } + */ + + /** + * Perform a quick check for how many individual files are + * inside this zip archive. If it's above the limit, we can + * give up right away, without doing any unpacking. + * This should be a fairly inexpensive operation, we just need + * to read the directory at the end of the file. + */ + + if (charset != null) { + zipFile = new ZipFile(tempFile.toFile(), charset); + } else { + zipFile = new ZipFile(tempFile.toFile()); + } + /** + * The ZipFile constructors above will throw ZipException - + * a type of IOException - if there's something wrong + * with this file as a zip. There's no need to intercept it + * here, it will be caught further below, with other IOExceptions, + * at which point we'll give up on trying to unpack it and + * then attempt to save it as is. + */ + + int numberOfUnpackableFiles = 0; + + /** + * Note that we can't just use zipFile.size(), + * unfortunately, since that's the total number of entries, + * some of which can be directories. So we need to go + * through all the individual zipEntries and count the ones + * that are files. + */ + + for (Enumeration entries = zipFile.entries(); entries.hasMoreElements();) { + ZipEntry entry = entries.nextElement(); + logger.fine("inside first zip pass; this entry: "+entry.getName()); + if (!entry.isDirectory()) { + String shortName = entry.getName().replaceFirst("^.*[\\/]", ""); + // ... and, finally, check if it's a "fake" file - a zip archive entry + // created for a MacOS X filesystem element: (these + // start with "._") + if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) { + numberOfUnpackableFiles++; + if (numberOfUnpackableFiles > fileNumberLimit) { + logger.warning("Zip upload - too many files in the zip to process individually."); + warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit + + "); please upload a zip archive with fewer files, if you want them to be ingested " + + "as individual DataFiles."; + throw new IOException(); + } + // In addition to counting the files, we can + // also check the file size while we're here, + // provided the size limit is defined; if a single + // file is above the individual size limit, unzipped, + // we give up on unpacking this zip archive as well: + if (fileSizeLimit != null && entry.getSize() > fileSizeLimit) { + throw new FileExceedsMaxSizeException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(entry.getSize()), bytesToHumanReadable(fileSizeLimit))); + } + // Similarly, we want to check if saving all these unpacked + // files is going to push the disk usage over the + // quota: + if (storageQuotaLimit != null) { + combinedUnzippedFileSize = combinedUnzippedFileSize + entry.getSize(); + if (combinedUnzippedFileSize > storageQuotaLimit) { + //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(combinedUnzippedFileSize), bytesToHumanReadable(storageQuotaLimit))); + // change of plans: if the unzipped content inside exceeds the remaining quota, + // we reject the upload outright, rather than accepting the zip + // file as is. + throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.unzipped.quota_exceeded"), bytesToHumanReadable(storageQuotaLimit)), this); + } + } + } + } + } + + // OK we're still here - that means we can proceed unzipping. + + // Close the ZipFile, re-open as ZipInputStream: + zipFile.close(); + // reset: + combinedUnzippedFileSize = 0L; + + if (charset != null) { + unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset); + } else { + unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile())); + } + + while (true) { + try { + zipEntry = unZippedIn.getNextEntry(); + } catch (IllegalArgumentException iaex) { + // Note: + // ZipInputStream documentation doesn't even mention that + // getNextEntry() throws an IllegalArgumentException! + // but that's what happens if the file name of the next + // entry is not valid in the current CharSet. + // -- L.A. + warningMessage = "Failed to unpack Zip file. (Unknown Character Set used in a file name?) Saving the file as is."; + logger.warning(warningMessage); + throw new IOException(); + } + + if (zipEntry == null) { + break; + } + // Note that some zip entries may be directories - we + // simply skip them: + + if (!zipEntry.isDirectory()) { + if (datafiles.size() > fileNumberLimit) { + logger.warning("Zip upload - too many files."); + warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit + + "); please upload a zip archive with fewer files, if you want them to be ingested " + + "as individual DataFiles."; + throw new IOException(); + } + + String fileEntryName = zipEntry.getName(); + logger.fine("ZipEntry, file: " + fileEntryName); + + if (fileEntryName != null && !fileEntryName.equals("")) { + + String shortName = fileEntryName.replaceFirst("^.*[\\/]", ""); + + // Check if it's a "fake" file - a zip archive entry + // created for a MacOS X filesystem element: (these + // start with "._") + if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) { + // OK, this seems like an OK file entry - we'll try + // to read it and create a DataFile with it: + + String storageIdentifier = FileUtil.generateStorageIdentifier(); + File unzippedFile = new File(getFilesTempDirectory() + "/" + storageIdentifier); + Files.copy(unZippedIn, unzippedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + // No need to check the size of this unpacked file against the size limit, + // since we've already checked for that in the first pass. + + DataFile datafile = FileUtil.createSingleDataFile(version, null, storageIdentifier, shortName, + MIME_TYPE_UNDETERMINED_DEFAULT, + ctxt.systemConfig().getFileFixityChecksumAlgorithm(), null, false); + + if (!fileEntryName.equals(shortName)) { + // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes), + // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all + // the leading, trailing and duplicate slashes; then replace all the characters that + // don't pass our validation rules. + String directoryName = fileEntryName.replaceFirst("[\\\\/][\\\\/]*[^\\\\/]*$", ""); + directoryName = StringUtil.sanitizeFileDirectory(directoryName, true); + // if (!"".equals(directoryName)) { + if (!StringUtil.isEmpty(directoryName)) { + logger.fine("setting the directory label to " + directoryName); + datafile.getFileMetadata().setDirectoryLabel(directoryName); + } + } + + if (datafile != null) { + // We have created this datafile with the mime type "unknown"; + // Now that we have it saved in a temporary location, + // let's try and determine its real type: + + String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(); + + try { + recognizedType = determineFileType(unzippedFile, shortName); + // null the File explicitly, to release any open FDs: + unzippedFile = null; + logger.fine("File utility recognized unzipped file as " + recognizedType); + if (recognizedType != null && !recognizedType.equals("")) { + datafile.setContentType(recognizedType); + } + } catch (Exception ex) { + logger.warning("Failed to run the file utility mime type check on file " + fileName); + } + + datafiles.add(datafile); + combinedUnzippedFileSize += datafile.getFilesize(); + } + } + } + } + unZippedIn.closeEntry(); + + } + + } catch (IOException ioex) { + // just clear the datafiles list and let + // ingest default to creating a single DataFile out + // of the unzipped file. + logger.warning("Unzipping failed; rolling back to saving the file as is."); + if (warningMessage == null) { + warningMessage = BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed"); + } + + datafiles.clear(); + } catch (FileExceedsMaxSizeException femsx) { + logger.warning("One of the unzipped files exceeds the size limit; resorting to saving the file as is. " + femsx.getMessage()); + warningMessage = BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.size", Arrays.asList(FileSizeChecker.bytesToHumanReadable(fileSizeLimit))); + datafiles.clear(); + } /*catch (FileExceedsStorageQuotaException fesqx) { + //logger.warning("One of the unzipped files exceeds the storage quota limit; resorting to saving the file as is. " + fesqx.getMessage()); + //warningMessage = BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.quota", Arrays.asList(FileSizeChecker.bytesToHumanReadable(storageQuotaLimit))); + //datafiles.clear(); + throw new CommandExecutionException(fesqx.getMessage(), fesqx, this); + }*/ finally { + if (zipFile != null) { + try { + zipFile.close(); + } catch (Exception zEx) {} + } + if (unZippedIn != null) { + try { + unZippedIn.close(); + } catch (Exception zEx) {} + } + } + if (!datafiles.isEmpty()) { + // remove the uploaded zip file: + try { + Files.delete(tempFile); + } catch (IOException ioex) { + // do nothing - it's just a temp file. + logger.warning("Could not remove temp file " + tempFile.getFileName().toString()); + } + // update the quota object: + if (quota != null) { + quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + combinedUnzippedFileSize); + } + // and return: + return CreateDataFileResult.success(fileName, finalType, datafiles); + } + + } else if (finalType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)) { + // Shape files may have to be split into multiple files, + // one zip archive per each complete set of shape files: + + // File rezipFolder = new File(this.getFilesTempDirectory()); + File rezipFolder = FileUtil.getShapefileUnzipTempDirectory(); + + IngestServiceShapefileHelper shpIngestHelper; + shpIngestHelper = new IngestServiceShapefileHelper(tempFile.toFile(), rezipFolder); + + boolean didProcessWork = shpIngestHelper.processFile(); + if (!(didProcessWork)) { + logger.severe("Processing of zipped shapefile failed."); + return CreateDataFileResult.error(fileName, finalType); + } + long combinedRezippedFileSize = 0L; + + try { + + for (File finalFile : shpIngestHelper.getFinalRezippedFiles()) { + FileInputStream finalFileInputStream = new FileInputStream(finalFile); + finalType = FileUtil.determineContentType(finalFile); + if (finalType == null) { + logger.warning("Content type is null; but should default to 'MIME_TYPE_UNDETERMINED_DEFAULT'"); + continue; + } + + File unZippedShapeTempFile = saveInputStreamInTempFile(finalFileInputStream, fileSizeLimit, storageQuotaLimit != null ? storageQuotaLimit - combinedRezippedFileSize : null); + DataFile new_datafile = FileUtil.createSingleDataFile(version, unZippedShapeTempFile, finalFile.getName(), finalType, ctxt.systemConfig().getFileFixityChecksumAlgorithm()); + + String directoryName = null; + String absolutePathName = finalFile.getParent(); + if (absolutePathName != null) { + if (absolutePathName.length() > rezipFolder.toString().length()) { + // This file lives in a subfolder - we want to + // preserve it in the FileMetadata: + directoryName = absolutePathName.substring(rezipFolder.toString().length() + 1); + + if (!StringUtil.isEmpty(directoryName)) { + new_datafile.getFileMetadata().setDirectoryLabel(directoryName); + } + } + } + if (new_datafile != null) { + datafiles.add(new_datafile); + combinedRezippedFileSize += unZippedShapeTempFile.length(); + // todo: can this new_datafile be null? + } else { + logger.severe("Could not add part of rezipped shapefile. new_datafile was null: " + finalFile.getName()); + } + try { + finalFileInputStream.close(); + } catch (IOException ioex) { + // this one can be ignored + } + } + } catch (FileExceedsMaxSizeException | FileExceedsStorageQuotaException femsx) { + logger.severe("One of the unzipped shape files exceeded the size limit, or the storage quota; giving up. " + femsx.getMessage()); + datafiles.clear(); + // (or should we throw an exception, instead of skipping it quietly? + } catch (IOException ioex) { + throw new CommandExecutionException("Failed to process one of the components of the unpacked shape file", ioex, this); + // todo? - maybe try to provide a more detailed explanation, of which repackaged component, etc.? + } + + // Delete the temp directory used for unzipping + // The try-catch is due to error encountered in using NFS for stocking file, + // cf. https://github.com/IQSS/dataverse/issues/5909 + try { + FileUtils.deleteDirectory(rezipFolder); + } catch (IOException ioex) { + // do nothing - it's a temp folder. + logger.warning("Could not remove temp folder, error message : " + ioex.getMessage()); + } + + if (!datafiles.isEmpty()) { + // remove the uploaded zip file: + try { + Files.delete(tempFile); + } catch (IOException ioex) { + // ignore - it's just a temp file - but let's log a warning + logger.warning("Could not remove temp file " + tempFile.getFileName().toString()); + } catch (SecurityException se) { + // same + logger.warning("Unable to delete: " + tempFile.toString() + "due to Security Exception: " + + se.getMessage()); + } + // update the quota object: + if (quota != null) { + quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + combinedRezippedFileSize); + } + return CreateDataFileResult.success(fileName, finalType, datafiles); + } else { + logger.severe("No files added from directory of rezipped shapefiles"); + } + return CreateDataFileResult.error(fileName, finalType); + + } else if (finalType.equalsIgnoreCase(BagItFileHandler.FILE_TYPE)) { + + try { + Optional bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler(); + if (bagItFileHandler.isPresent()) { + CreateDataFileResult result = bagItFileHandler.get().handleBagItPackage(ctxt.systemConfig(), version, fileName, tempFile.toFile()); + return result; + } + } catch (IOException ioex) { + throw new CommandExecutionException("Failed to process uploaded BagIt file", ioex, this); + } + } + } else { + // Default to suppliedContentType if set or the overall undetermined default if a contenttype isn't supplied + finalType = StringUtils.isBlank(suppliedContentType) ? FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType; + String type = determineFileTypeByNameAndExtension(fileName); + if (!StringUtils.isBlank(type)) { + //Use rules for deciding when to trust browser supplied type + if (useRecognizedType(finalType, type)) { + finalType = type; + } + logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType); + } + } + // Finally, if none of the special cases above were applicable (or + // if we were unable to unpack an uploaded file, etc.), we'll just + // create and return a single DataFile: + File newFile = null; + long fileSize = -1; + + if (tempFile != null) { + newFile = tempFile.toFile(); + fileSize = newFile.length(); + } else { + // If this is a direct upload, and therefore no temp file associated + // with it, the file size must be explicitly passed to the command + // (note that direct upload relies on knowing the size of the file + // that's being uploaded in advance). + if (newFileSize != null) { + fileSize = newFileSize; + } else { + throw new CommandExecutionException("File size must be explicitly specified when creating DataFiles with Direct Upload", this); + } + } + + // We have already checked that this file does not exceed the individual size limit; + // but if we are processing it as is, as a single file, we need to check if + // its size does not go beyond the allocated storage quota (if specified): + + + if (storageQuotaLimit != null && fileSize > storageQuotaLimit) { + if (newFile != null) { + // Remove the temp. file, if this is a non-direct upload. + // If this is a direct upload, it will be a responsibility of the + // component calling the command to remove the file that may have + // already been saved in the S3 volume. + try { + newFile.delete(); + } catch (Exception ex) { + // ignore - but log a warning + logger.warning("Could not remove temp file " + tempFile.getFileName()); + } + } + throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(fileSize), bytesToHumanReadable(storageQuotaLimit)), this); + } + + DataFile datafile = FileUtil.createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum); + + if (datafile != null && ((newFile != null) || (newStorageIdentifier != null))) { + + if (warningMessage != null) { + createIngestFailureReport(datafile, warningMessage); + datafile.SetIngestProblem(); + } + if (datafile.getFilesize() < 0) { + datafile.setFilesize(fileSize); + } + datafiles.add(datafile); + + // Update quota (may not be necessary in the context of direct upload - ?) + if (quota != null) { + quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + fileSize); + } + return CreateDataFileResult.success(fileName, finalType, datafiles); + } + + return CreateDataFileResult.error(fileName, finalType); + } // end createDataFiles + + @Override + public Map> getRequiredPermissions() { + Map> ret = new HashMap<>(); + + ret.put("", new HashSet<>()); + + if (parentDataverse != null) { + // The command is called in the context of uploading files on + // create of a new dataset + ret.get("").add(Permission.AddDataset); + } else { + // An existing dataset + ret.get("").add(Permission.EditDataset); + } + + return ret; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index b8adc8e23e8..f83041d87bd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -91,7 +91,9 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // we have to merge to update the database but not flush because // we don't want to create two draft versions! Dataset tempDataset = ctxt.em().merge(getDataset()); - + + updateVersion = tempDataset.getLatestVersionForCopy(); + // Look for file metadata changes and update published metadata if needed List pubFmds = updateVersion.getFileMetadatas(); int pubFileCount = pubFmds.size(); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java index e41d70d9804..df0b5d785e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java @@ -212,7 +212,7 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { try { JsonObjectBuilder gbe = Json.createObjectBuilder() .add("id", guestbookResponse.getId()) - .add("downloadType", guestbookResponse.getDownloadtype()) + .add("eventType", guestbookResponse.getEventType()) .add("filename", guestbookResponse.getDataFile().getCurrentName()) .add("date", guestbookResponse.getResponseDate()) .add("guestbookName", guestbookResponse.getGuestbook().getName()); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java index b87b9a73aa5..bf291427341 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java @@ -5,7 +5,13 @@ */ package edu.harvard.iq.dataverse.engine.command.impl; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Logger; + import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.FileAccessRequest; +import edu.harvard.iq.dataverse.GuestbookResponse; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; @@ -22,25 +28,39 @@ */ @RequiredPermissions({}) public class RequestAccessCommand extends AbstractCommand { - + + private static final Logger logger = Logger.getLogger(RequestAccessCommand.class.getName()); + private final DataFile file; private final AuthenticatedUser requester; + private final FileAccessRequest fileAccessRequest; private final Boolean sendNotification; - public RequestAccessCommand(DataverseRequest dvRequest, DataFile file) { // for data file check permission on owning dataset - super(dvRequest, file); - this.file = file; + this(dvRequest, file, false); + } + + public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, Boolean sendNotification) { + // for data file check permission on owning dataset + super(dvRequest, file); + this.file = file; this.requester = (AuthenticatedUser) dvRequest.getUser(); - this.sendNotification = false; + this.fileAccessRequest = new FileAccessRequest(file, requester); + this.sendNotification = sendNotification; + } + + public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr) { + this(dvRequest, file, gbr, false); } - - public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, Boolean sendNotification) { + + public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr, + Boolean sendNotification) { // for data file check permission on owning dataset - super(dvRequest, file); - this.file = file; + super(dvRequest, file); + this.file = file; this.requester = (AuthenticatedUser) dvRequest.getUser(); + this.fileAccessRequest = new FileAccessRequest(file, requester, gbr); this.sendNotification = sendNotification; } @@ -50,21 +70,36 @@ public DataFile execute(CommandContext ctxt) throws CommandException { if (!file.getOwner().isFileAccessRequest()) { throw new CommandException(BundleUtil.getStringFromBundle("file.requestAccess.notAllowed"), this); } - - //if user already has permission to download file or the file is public throw command exception - if (!file.isRestricted() || ctxt.permissions().requestOn(this.getRequest(), file).has(Permission.DownloadFile)) { - throw new CommandException(BundleUtil.getStringFromBundle("file.requestAccess.notAllowed.alreadyHasDownloadPermisssion"), this); + + // if user already has permission to download file or the file is public throw + // command exception + logger.fine("User: " + this.getRequest().getAuthenticatedUser().getName()); + logger.fine("File: " + file.getId() + " : restricted?: " + file.isRestricted()); + logger.fine( + "permission?: " + ctxt.permissions().requestOn(this.getRequest(), file).has(Permission.DownloadFile)); + if (!file.isRestricted() + || ctxt.permissions().requestOn(this.getRequest(), file).has(Permission.DownloadFile)) { + throw new CommandException( + BundleUtil.getStringFromBundle("file.requestAccess.notAllowed.alreadyHasDownloadPermisssion"), + this); } - if(FileUtil.isActivelyEmbargoed(file)) { + if (FileUtil.isActivelyEmbargoed(file)) { throw new CommandException(BundleUtil.getStringFromBundle("file.requestAccess.notAllowed.embargoed"), this); } - file.addFileAccessRequester(requester); + file.addFileAccessRequest(fileAccessRequest); + List fars = requester.getFileAccessRequests(); + if(fars!=null) { + fars.add(fileAccessRequest); + } else { + requester.setFileAccessRequests(Arrays.asList(fileAccessRequest)); + } + DataFile savedFile = ctxt.files().save(file); if (sendNotification) { - ctxt.fileDownload().sendRequestFileAccessNotification(this.file, requester); + logger.fine("ctxt.fileDownload().sendRequestFileAccessNotification(savedFile, requester);"); + ctxt.fileDownload().sendRequestFileAccessNotification(savedFile.getOwner(), savedFile.getId(), requester); } - return ctxt.files().save(file); + return savedFile; } } - diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index a52679deebc..de4317464e6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -253,4 +253,11 @@ public String getExploreScript() { logger.fine("Exploring with " + toolUrl); return getScriptForUrl(toolUrl); } + + // TODO: Consider merging with getExploreScript + public String getConfigureScript() { + String toolUrl = this.getToolUrlWithQueryParams(); + logger.fine("Configuring with " + toolUrl); + return getScriptForUrl(toolUrl); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java index 065b42e5afe..79369207963 100644 --- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.GuestbookResponse; import edu.harvard.iq.dataverse.Metric; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountUtil.MetricType; @@ -424,6 +425,7 @@ public JsonArray downloadsTimeSeries(Dataverse d) { + "select distinct COALESCE(to_char(responsetime, 'YYYY-MM'),'" + earliest + "') as date, count(id)\n" + "from guestbookresponse\n" + ((d == null) ? "" : "where dataset_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ")") + + ((d == null) ? "where ":" and ") + "eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n" + " group by COALESCE(to_char(responsetime, 'YYYY-MM'),'" + earliest + "') order by COALESCE(to_char(responsetime, 'YYYY-MM'),'" + earliest + "');"); logger.log(Level.FINE, "Metric query: {0}", query); @@ -456,6 +458,7 @@ public long downloadsToMonth(String yyyymm, Dataverse d) throws ParseException { + "from guestbookresponse\n" + "where (date_trunc('month', responsetime) <= to_date('" + yyyymm + "','YYYY-MM')" + "or responsetime is NULL)\n" // includes historic guestbook records without date + + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n" + ((d==null) ? ";": "AND dataset_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ");") ); logger.log(Level.FINE, "Metric query: {0}", query); @@ -477,6 +480,7 @@ public long downloadsPastDays(int days, Dataverse d) { + "select count(id)\n" + "from guestbookresponse\n" + "where responsetime > current_date - interval '" + days + "' day\n" + + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n" + ((d==null) ? ";": "AND dataset_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ");") ); logger.log(Level.FINE, "Metric query: {0}", query); @@ -489,6 +493,7 @@ public JsonArray fileDownloadsTimeSeries(Dataverse d, boolean uniqueCounts) { + " FROM guestbookresponse gb, DvObject ob" + " where ob.id = gb.datafile_id " + ((d == null) ? "" : " and ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ")\n") + + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n" + "group by gb.datafile_id, ob.id, ob.protocol, ob.authority, ob.identifier, to_char(gb.responsetime, 'YYYY-MM') order by to_char(gb.responsetime, 'YYYY-MM');"); logger.log(Level.FINE, "Metric query: {0}", query); @@ -503,6 +508,7 @@ public JsonArray fileDownloads(String yyyymm, Dataverse d, boolean uniqueCounts) + " where ob.id = gb.datafile_id " + ((d == null) ? "" : " and ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ")\n") + " and date_trunc('month', gb.responsetime) <= to_date('" + yyyymm + "','YYYY-MM')\n" + + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n" + "group by gb.datafile_id, ob.id, ob.protocol, ob.authority, ob.identifier order by count desc;"); logger.log(Level.FINE, "Metric query: {0}", query); @@ -529,6 +535,7 @@ public JsonArray uniqueDownloadsTimeSeries(Dataverse d) { + " FROM guestbookresponse gb, DvObject ob" + " where ob.id = gb.dataset_id " + ((d == null) ? "" : " and ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n") + + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n" + "group by gb.dataset_id, ob.protocol, ob.authority, ob.identifier, to_char(gb.responsetime, 'YYYY-MM') order by to_char(gb.responsetime, 'YYYY-MM');"); logger.log(Level.FINE, "Metric query: {0}", query); @@ -546,6 +553,7 @@ public JsonArray uniqueDatasetDownloads(String yyyymm, Dataverse d) { + " where ob.id = gb.dataset_id " + ((d == null) ? "" : " and ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n") + " and date_trunc('month', responsetime) <= to_date('" + yyyymm + "','YYYY-MM')\n" + + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n" + "group by gb.dataset_id, ob.protocol, ob.authority, ob.identifier order by count(distinct email) desc;"); JsonArrayBuilder jab = Json.createArrayBuilder(); try { diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 738d63e924f..41ac030bd0a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -49,6 +49,7 @@ public enum JvmSettings { // FILES SETTINGS SCOPE_FILES(PREFIX, "files"), FILES_DIRECTORY(SCOPE_FILES, "directory"), + GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"), // SOLR INDEX SETTINGS SCOPE_SOLR(PREFIX, "solr"), diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 2826df74ed1..0aa403a5116 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -578,7 +578,16 @@ Whether Harvesting (OAI) service is enabled /** * The URL for the DvWebLoader tool (see github.com/gdcc/dvwebloader for details) */ - WebloaderUrl, + WebloaderUrl, + /** + * Enforce storage quotas: + */ + UseStorageQuotas, + /** + * Placeholder storage quota (defines the same quota setting for every user; used to test the concept of a quota. + */ + StorageQuotaSizeInBytes, + /** * A comma-separated list of CategoryName in the desired order for files to be * sorted in the file table display. If not set, files will be sorted diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 5f7643b3115..75f265494b3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -28,6 +28,7 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.Embargo; import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; @@ -94,19 +95,14 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import org.apache.commons.io.FileUtils; import java.util.zip.GZIPInputStream; -import java.util.zip.ZipFile; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import org.apache.commons.io.FilenameUtils; import edu.harvard.iq.dataverse.dataaccess.DataAccessOption; import edu.harvard.iq.dataverse.dataaccess.StorageIO; -import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker; +import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException; import java.util.Arrays; -import java.util.Enumeration; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import ucar.nc2.NetcdfFile; @@ -395,7 +391,7 @@ public static String getUserFriendlyOriginalType(DataFile dataFile) { * Returns a content type string for a FileObject * */ - private static String determineContentType(File fileObject) { + public static String determineContentType(File fileObject) { if (fileObject==null){ return null; } @@ -795,488 +791,6 @@ public static String generateOriginalExtension(String fileType) { } return ""; } - - public static CreateDataFileResult createDataFiles(DatasetVersion version, InputStream inputStream, - String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, - SystemConfig systemConfig) throws IOException { - ChecksumType checkSumType = DataFile.ChecksumType.MD5; - if (newStorageIdentifier == null) { - checkSumType = systemConfig.getFileFixityChecksumAlgorithm(); - } - return createDataFiles(version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, checkSumType, systemConfig); - } - - public static CreateDataFileResult createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, ChecksumType newCheckSumType, SystemConfig systemConfig) throws IOException { - List datafiles = new ArrayList<>(); - - //When there is no checksum/checksumtype being sent (normal upload, needs to be calculated), set the type to the current default - if(newCheckSumType == null) { - newCheckSumType = systemConfig.getFileFixityChecksumAlgorithm(); - } - - String warningMessage = null; - - // save the file, in the temporary location for now: - Path tempFile = null; - - Long fileSizeLimit = systemConfig.getMaxFileUploadSizeForStore(version.getDataset().getEffectiveStorageDriverId()); - String finalType = null; - if (newStorageIdentifier == null) { - if (getFilesTempDirectory() != null) { - tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload"); - // "temporary" location is the key here; this is why we are not using - // the DataStore framework for this - the assumption is that - // temp files will always be stored on the local filesystem. - // -- L.A. Jul. 2014 - logger.fine("Will attempt to save the file as: " + tempFile.toString()); - Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING); - - // A file size check, before we do anything else: - // (note that "no size limit set" = "unlimited") - // (also note, that if this is a zip file, we'll be checking - // the size limit for each of the individual unpacked files) - Long fileSize = tempFile.toFile().length(); - if (fileSizeLimit != null && fileSize > fileSizeLimit) { - try { - tempFile.toFile().delete(); - } catch (Exception ex) { - } - throw new IOException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit))); - } - - } else { - throw new IOException("Temp directory is not configured."); - } - logger.fine("mime type supplied: " + suppliedContentType); - // Let's try our own utilities (Jhove, etc.) to determine the file type - // of the uploaded file. (We may already have a mime type supplied for this - // file - maybe the type that the browser recognized on upload; or, if - // it's a harvest, maybe the remote server has already given us the type - // for this file... with our own type utility we may or may not do better - // than the type supplied: - // -- L.A. - String recognizedType = null; - - try { - recognizedType = determineFileType(tempFile.toFile(), fileName); - logger.fine("File utility recognized the file as " + recognizedType); - if (recognizedType != null && !recognizedType.equals("")) { - if (useRecognizedType(suppliedContentType, recognizedType)) { - finalType = recognizedType; - } - } - - } catch (Exception ex) { - logger.warning("Failed to run the file utility mime type check on file " + fileName); - } - - if (finalType == null) { - finalType = (suppliedContentType == null || suppliedContentType.equals("")) - ? MIME_TYPE_UNDETERMINED_DEFAULT - : suppliedContentType; - } - - // A few special cases: - // if this is a gzipped FITS file, we'll uncompress it, and ingest it as - // a regular FITS file: - if (finalType.equals("application/fits-gzipped")) { - - String finalFileName = fileName; - // if the file name had the ".gz" extension, remove it, - // since we are going to uncompress it: - if (fileName != null && fileName.matches(".*\\.gz$")) { - finalFileName = fileName.replaceAll("\\.gz$", ""); - } - - DataFile datafile = null; - try (InputStream uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile()))){ - File unZippedTempFile = saveInputStreamInTempFile(uncompressedIn, fileSizeLimit); - datafile = createSingleDataFile(version, unZippedTempFile, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT, systemConfig.getFileFixityChecksumAlgorithm()); - } catch (IOException | FileExceedsMaxSizeException ioex) { - datafile = null; - } - - // If we were able to produce an uncompressed file, we'll use it - // to create and return a final DataFile; if not, we're not going - // to do anything - and then a new DataFile will be created further - // down, from the original, uncompressed file. - if (datafile != null) { - // remove the compressed temp file: - try { - tempFile.toFile().delete(); - } catch (SecurityException ex) { - // (this is very non-fatal) - logger.warning("Failed to delete temporary file " + tempFile.toString()); - } - - datafiles.add(datafile); - return CreateDataFileResult.success(fileName, finalType, datafiles); - } - - // If it's a ZIP file, we are going to unpack it and create multiple - // DataFile objects from its contents: - } else if (finalType.equals("application/zip")) { - - ZipFile zipFile = null; - ZipInputStream unZippedIn = null; - ZipEntry zipEntry = null; - - int fileNumberLimit = systemConfig.getZipUploadFilesLimit(); - - try { - Charset charset = null; - /* - TODO: (?) - We may want to investigate somehow letting the user specify - the charset for the filenames in the zip file... - - otherwise, ZipInputStream bails out if it encounteres a file - name that's not valid in the current charest (i.e., UTF-8, in - our case). It would be a bit trickier than what we're doing for - SPSS tabular ingests - with the lang. encoding pulldown menu - - because this encoding needs to be specified *before* we upload and - attempt to unzip the file. - -- L.A. 4.0 beta12 - logger.info("default charset is "+Charset.defaultCharset().name()); - if (Charset.isSupported("US-ASCII")) { - logger.info("charset US-ASCII is supported."); - charset = Charset.forName("US-ASCII"); - if (charset != null) { - logger.info("was able to obtain charset for US-ASCII"); - } - - } - */ - - /** - * Perform a quick check for how many individual files are - * inside this zip archive. If it's above the limit, we can - * give up right away, without doing any unpacking. - * This should be a fairly inexpensive operation, we just need - * to read the directory at the end of the file. - */ - - if (charset != null) { - zipFile = new ZipFile(tempFile.toFile(), charset); - } else { - zipFile = new ZipFile(tempFile.toFile()); - } - /** - * The ZipFile constructors above will throw ZipException - - * a type of IOException - if there's something wrong - * with this file as a zip. There's no need to intercept it - * here, it will be caught further below, with other IOExceptions, - * at which point we'll give up on trying to unpack it and - * then attempt to save it as is. - */ - - int numberOfUnpackableFiles = 0; - /** - * Note that we can't just use zipFile.size(), - * unfortunately, since that's the total number of entries, - * some of which can be directories. So we need to go - * through all the individual zipEntries and count the ones - * that are files. - */ - - for (Enumeration entries = zipFile.entries(); entries.hasMoreElements();) { - ZipEntry entry = entries.nextElement(); - logger.fine("inside first zip pass; this entry: "+entry.getName()); - if (!entry.isDirectory()) { - String shortName = entry.getName().replaceFirst("^.*[\\/]", ""); - // ... and, finally, check if it's a "fake" file - a zip archive entry - // created for a MacOS X filesystem element: (these - // start with "._") - if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) { - numberOfUnpackableFiles++; - if (numberOfUnpackableFiles > fileNumberLimit) { - logger.warning("Zip upload - too many files in the zip to process individually."); - warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit - + "); please upload a zip archive with fewer files, if you want them to be ingested " - + "as individual DataFiles."; - throw new IOException(); - } - // In addition to counting the files, we can - // also check the file size while we're here, - // provided the size limit is defined; if a single - // file is above the individual size limit, unzipped, - // we give up on unpacking this zip archive as well: - if (fileSizeLimit != null && entry.getSize() > fileSizeLimit) { - throw new FileExceedsMaxSizeException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(entry.getSize()), bytesToHumanReadable(fileSizeLimit))); - } - } - } - } - - // OK we're still here - that means we can proceed unzipping. - - // Close the ZipFile, re-open as ZipInputStream: - zipFile.close(); - - if (charset != null) { - unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset); - } else { - unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile())); - } - - while (true) { - try { - zipEntry = unZippedIn.getNextEntry(); - } catch (IllegalArgumentException iaex) { - // Note: - // ZipInputStream documentation doesn't even mention that - // getNextEntry() throws an IllegalArgumentException! - // but that's what happens if the file name of the next - // entry is not valid in the current CharSet. - // -- L.A. - warningMessage = "Failed to unpack Zip file. (Unknown Character Set used in a file name?) Saving the file as is."; - logger.warning(warningMessage); - throw new IOException(); - } - - if (zipEntry == null) { - break; - } - // Note that some zip entries may be directories - we - // simply skip them: - - if (!zipEntry.isDirectory()) { - if (datafiles.size() > fileNumberLimit) { - logger.warning("Zip upload - too many files."); - warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit - + "); please upload a zip archive with fewer files, if you want them to be ingested " - + "as individual DataFiles."; - throw new IOException(); - } - - String fileEntryName = zipEntry.getName(); - logger.fine("ZipEntry, file: " + fileEntryName); - - if (fileEntryName != null && !fileEntryName.equals("")) { - - String shortName = fileEntryName.replaceFirst("^.*[\\/]", ""); - - // Check if it's a "fake" file - a zip archive entry - // created for a MacOS X filesystem element: (these - // start with "._") - if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) { - // OK, this seems like an OK file entry - we'll try - // to read it and create a DataFile with it: - - String storageIdentifier = generateStorageIdentifier(); - File unzippedFile = new File(getFilesTempDirectory() + "/" + storageIdentifier); - Files.copy(unZippedIn, unzippedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - // No need to check the size of this unpacked file against the size limit, - // since we've already checked for that in the first pass. - - DataFile datafile = createSingleDataFile(version, null, storageIdentifier, shortName, - MIME_TYPE_UNDETERMINED_DEFAULT, - systemConfig.getFileFixityChecksumAlgorithm(), null, false); - - if (!fileEntryName.equals(shortName)) { - // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes), - // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all - // the leading, trailing and duplicate slashes; then replace all the characters that - // don't pass our validation rules. - String directoryName = fileEntryName.replaceFirst("[\\\\/][\\\\/]*[^\\\\/]*$", ""); - directoryName = StringUtil.sanitizeFileDirectory(directoryName, true); - // if (!"".equals(directoryName)) { - if (!StringUtil.isEmpty(directoryName)) { - logger.fine("setting the directory label to " + directoryName); - datafile.getFileMetadata().setDirectoryLabel(directoryName); - } - } - - if (datafile != null) { - // We have created this datafile with the mime type "unknown"; - // Now that we have it saved in a temporary location, - // let's try and determine its real type: - - try { - recognizedType = determineFileType(unzippedFile, shortName); - // null the File explicitly, to release any open FDs: - unzippedFile = null; - logger.fine("File utility recognized unzipped file as " + recognizedType); - if (recognizedType != null && !recognizedType.equals("")) { - datafile.setContentType(recognizedType); - } - } catch (Exception ex) { - logger.warning("Failed to run the file utility mime type check on file " + fileName); - } - - datafiles.add(datafile); - } - } - } - } - unZippedIn.closeEntry(); - - } - - } catch (IOException ioex) { - // just clear the datafiles list and let - // ingest default to creating a single DataFile out - // of the unzipped file. - logger.warning("Unzipping failed; rolling back to saving the file as is."); - if (warningMessage == null) { - warningMessage = BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed"); - } - - datafiles.clear(); - } catch (FileExceedsMaxSizeException femsx) { - logger.warning("One of the unzipped files exceeds the size limit; resorting to saving the file as is. " + femsx.getMessage()); - warningMessage = BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.size", Arrays.asList(FileSizeChecker.bytesToHumanReadable(fileSizeLimit))); - datafiles.clear(); - } finally { - if (zipFile != null) { - try { - zipFile.close(); - } catch (Exception zEx) {} - } - if (unZippedIn != null) { - try { - unZippedIn.close(); - } catch (Exception zEx) {} - } - } - if (!datafiles.isEmpty()) { - // remove the uploaded zip file: - try { - Files.delete(tempFile); - } catch (IOException ioex) { - // do nothing - it's just a temp file. - logger.warning("Could not remove temp file " + tempFile.getFileName().toString()); - } - // and return: - return CreateDataFileResult.success(fileName, finalType, datafiles); - } - - } else if (finalType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)) { - // Shape files may have to be split into multiple files, - // one zip archive per each complete set of shape files: - - // File rezipFolder = new File(this.getFilesTempDirectory()); - File rezipFolder = getShapefileUnzipTempDirectory(); - - IngestServiceShapefileHelper shpIngestHelper; - shpIngestHelper = new IngestServiceShapefileHelper(tempFile.toFile(), rezipFolder); - - boolean didProcessWork = shpIngestHelper.processFile(); - if (!(didProcessWork)) { - logger.severe("Processing of zipped shapefile failed."); - return CreateDataFileResult.error(fileName, finalType); - } - - try { - for (File finalFile : shpIngestHelper.getFinalRezippedFiles()) { - FileInputStream finalFileInputStream = new FileInputStream(finalFile); - finalType = determineContentType(finalFile); - if (finalType == null) { - logger.warning("Content type is null; but should default to 'MIME_TYPE_UNDETERMINED_DEFAULT'"); - continue; - } - - File unZippedShapeTempFile = saveInputStreamInTempFile(finalFileInputStream, fileSizeLimit); - DataFile new_datafile = createSingleDataFile(version, unZippedShapeTempFile, finalFile.getName(), finalType, systemConfig.getFileFixityChecksumAlgorithm()); - String directoryName = null; - String absolutePathName = finalFile.getParent(); - if (absolutePathName != null) { - if (absolutePathName.length() > rezipFolder.toString().length()) { - // This file lives in a subfolder - we want to - // preserve it in the FileMetadata: - directoryName = absolutePathName.substring(rezipFolder.toString().length() + 1); - - if (!StringUtil.isEmpty(directoryName)) { - new_datafile.getFileMetadata().setDirectoryLabel(directoryName); - } - } - } - if (new_datafile != null) { - datafiles.add(new_datafile); - } else { - logger.severe("Could not add part of rezipped shapefile. new_datafile was null: " + finalFile.getName()); - } - finalFileInputStream.close(); - - } - } catch (FileExceedsMaxSizeException femsx) { - logger.severe("One of the unzipped shape files exceeded the size limit; giving up. " + femsx.getMessage()); - datafiles.clear(); - } - - // Delete the temp directory used for unzipping - // The try-catch is due to error encountered in using NFS for stocking file, - // cf. https://github.com/IQSS/dataverse/issues/5909 - try { - FileUtils.deleteDirectory(rezipFolder); - } catch (IOException ioex) { - // do nothing - it's a tempo folder. - logger.warning("Could not remove temp folder, error message : " + ioex.getMessage()); - } - - if (datafiles.size() > 0) { - // remove the uploaded zip file: - try { - Files.delete(tempFile); - } catch (IOException ioex) { - // do nothing - it's just a temp file. - logger.warning("Could not remove temp file " + tempFile.getFileName().toString()); - } catch (SecurityException se) { - logger.warning("Unable to delete: " + tempFile.toString() + "due to Security Exception: " - + se.getMessage()); - } - return CreateDataFileResult.success(fileName, finalType, datafiles); - } else { - logger.severe("No files added from directory of rezipped shapefiles"); - } - return CreateDataFileResult.error(fileName, finalType); - - } else if (finalType.equalsIgnoreCase(BagItFileHandler.FILE_TYPE)) { - Optional bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler(); - if (bagItFileHandler.isPresent()) { - CreateDataFileResult result = bagItFileHandler.get().handleBagItPackage(systemConfig, version, fileName, tempFile.toFile()); - return result; - } - } - } else { - // Default to suppliedContentType if set or the overall undetermined default if a contenttype isn't supplied - finalType = StringUtils.isBlank(suppliedContentType) ? FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType; - String type = determineFileTypeByNameAndExtension(fileName); - if (!StringUtils.isBlank(type)) { - //Use rules for deciding when to trust browser supplied type - if (useRecognizedType(finalType, type)) { - finalType = type; - } - logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType); - } - } - // Finally, if none of the special cases above were applicable (or - // if we were unable to unpack an uploaded file, etc.), we'll just - // create and return a single DataFile: - File newFile = null; - if (tempFile != null) { - newFile = tempFile.toFile(); - } - - - DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum); - File f = null; - if (tempFile != null) { - f = tempFile.toFile(); - } - if (datafile != null && ((f != null) || (newStorageIdentifier != null))) { - - if (warningMessage != null) { - createIngestFailureReport(datafile, warningMessage); - datafile.SetIngestProblem(); - } - datafiles.add(datafile); - - return CreateDataFileResult.success(fileName, finalType, datafiles); - } - - return CreateDataFileResult.error(fileName, finalType); - } // end createDataFiles - public static boolean useRecognizedType(String suppliedContentType, String recognizedType) { // is it any better than the type that was supplied to us, @@ -1315,7 +829,12 @@ public static boolean useRecognizedType(String suppliedContentType, String recog } public static File saveInputStreamInTempFile(InputStream inputStream, Long fileSizeLimit) - throws IOException, FileExceedsMaxSizeException { + throws IOException, FileExceedsMaxSizeException, FileExceedsStorageQuotaException { + return saveInputStreamInTempFile(inputStream, fileSizeLimit, null); + } + + public static File saveInputStreamInTempFile(InputStream inputStream, Long fileSizeLimit, Long storageQuotaLimit) + throws IOException, FileExceedsMaxSizeException, FileExceedsStorageQuotaException { Path tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload"); if (inputStream != null && tempFile != null) { @@ -1326,7 +845,12 @@ public static File saveInputStreamInTempFile(InputStream inputStream, Long fileS Long fileSize = tempFile.toFile().length(); if (fileSizeLimit != null && fileSize > fileSizeLimit) { try {tempFile.toFile().delete();} catch (Exception ex) {} - throw new FileExceedsMaxSizeException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit))); + throw new FileExceedsMaxSizeException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit))); + } + + if (storageQuotaLimit != null && fileSize > storageQuotaLimit) { + try {tempFile.toFile().delete();} catch (Exception ex) {} + throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(fileSize), bytesToHumanReadable(storageQuotaLimit))); } return tempFile.toFile(); @@ -1369,7 +893,6 @@ public static DataFile createSingleDataFile(DatasetVersion version, File tempFil datafile.setPermissionModificationTime(new Timestamp(new Date().getTime())); FileMetadata fmd = new FileMetadata(); - // TODO: add directoryLabel? fmd.setLabel(fileName); if (addToDataset) { @@ -1416,7 +939,7 @@ public static DataFile createSingleDataFile(DatasetVersion version, File tempFil Naming convention: getFilesTempDirectory() + "shp_" + "yyyy-MM-dd-hh-mm-ss-SSS" */ - private static File getShapefileUnzipTempDirectory(){ + public static File getShapefileUnzipTempDirectory(){ String tempDirectory = getFilesTempDirectory(); if (tempDirectory == null){ @@ -1600,6 +1123,11 @@ public static boolean isRequestAccessPopupRequired(DatasetVersion datasetVersion if (answer != null) { return answer; } + // 3. Guest Book: + if (datasetVersion.getDataset() != null && datasetVersion.getDataset().getGuestbook() != null && datasetVersion.getDataset().getGuestbook().isEnabled() && datasetVersion.getDataset().getGuestbook().getDataverse() != null) { + logger.fine("Request access popup required because of guestbook."); + return true; + } logger.fine("Request access popup is not required."); return false; } @@ -1641,6 +1169,71 @@ private static Boolean popupDueToStateOrTerms(DatasetVersion datasetVersion) { return null; } + /** + * isGuestbookAndTermsPopupRequired + * meant to replace both isDownloadPopupRequired() and isRequestAccessDownloadPopupRequired() when the guestbook-terms-popup-fragment.xhtml + * replaced file-download-popup-fragment.xhtml and file-request-access-popup-fragment.xhtml + * @param datasetVersion + * @return boolean + */ + + public static boolean isGuestbookAndTermsPopupRequired(DatasetVersion datasetVersion) { + return isGuestbookPopupRequired(datasetVersion) || isTermsPopupRequired(datasetVersion); + } + + public static boolean isGuestbookPopupRequired(DatasetVersion datasetVersion) { + + if (datasetVersion == null) { + logger.fine("GuestbookPopup not required because datasetVersion is null."); + return false; + } + //0. if version is draft then Popup "not required" + if (!datasetVersion.isReleased()) { + logger.fine("GuestbookPopup not required because datasetVersion has not been released."); + return false; + } + + // 3. Guest Book: + if (datasetVersion.getDataset() != null && datasetVersion.getDataset().getGuestbook() != null && datasetVersion.getDataset().getGuestbook().isEnabled() && datasetVersion.getDataset().getGuestbook().getDataverse() != null) { + logger.fine("GuestbookPopup required because an enabled guestbook exists."); + return true; + } + + logger.fine("GuestbookPopup is not required."); + return false; + } + + public static boolean isTermsPopupRequired(DatasetVersion datasetVersion) { + + if (datasetVersion == null) { + logger.fine("TermsPopup not required because datasetVersion is null."); + return false; + } + //0. if version is draft then Popup "not required" + if (!datasetVersion.isReleased()) { + logger.fine("TermsPopup not required because datasetVersion has not been released."); + return false; + } + // 1. License and Terms of Use: + if (datasetVersion.getTermsOfUseAndAccess() != null) { + if (!License.CC0.equals(datasetVersion.getTermsOfUseAndAccess().getLicense()) + && !(datasetVersion.getTermsOfUseAndAccess().getTermsOfUse() == null + || datasetVersion.getTermsOfUseAndAccess().getTermsOfUse().equals(""))) { + logger.fine("TermsPopup required because of license or terms of use."); + return true; + } + + // 2. Terms of Access: + if (!(datasetVersion.getTermsOfUseAndAccess().getTermsOfAccess() == null) && !datasetVersion.getTermsOfUseAndAccess().getTermsOfAccess().equals("")) { + logger.fine("TermsPopup required because of terms of access."); + return true; + } + } + + logger.fine("TermsPopup is not required."); + return false; + } + /** * Provide download URL if no Terms of Use, no guestbook, and not * restricted. diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index dcb6e078df6..0724e53700b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.util; +import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.UserNotification; @@ -39,6 +40,8 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti datasetDisplayName = ((Dataset) objectOfNotification).getDisplayName(); } else if (objectOfNotification instanceof DatasetVersion) { datasetDisplayName = ((DatasetVersion) objectOfNotification).getDataset().getDisplayName(); + } else if (objectOfNotification instanceof DataFile) { + datasetDisplayName = ((DataFile) objectOfNotification).getOwner().getDisplayName(); } } @@ -50,7 +53,9 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti case CREATEDV: return BundleUtil.getStringFromBundle("notification.email.create.dataverse.subject", rootDvNameAsList); case REQUESTFILEACCESS: - return BundleUtil.getStringFromBundle("notification.email.request.file.access.subject", rootDvNameAsList); + return BundleUtil.getStringFromBundle("notification.email.request.file.access.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); + case REQUESTEDFILEACCESS: + return BundleUtil.getStringFromBundle("notification.email.requested.file.access.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); case GRANTFILEACCESS: return BundleUtil.getStringFromBundle("notification.email.grant.file.access.subject", rootDvNameAsList); case REJECTFILEACCESS: diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 4fed3a05976..079cbaa999d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -893,7 +893,7 @@ public String toString() { } } - + public boolean isPublicInstall(){ boolean saneDefault = false; return settingsService.isTrueForKey(SettingsServiceBean.Key.PublicInstall, saneDefault); @@ -1164,4 +1164,18 @@ public boolean isSignupDisabledForRemoteAuthProvider(String providerId) { return !ret; } + + public boolean isStorageQuotasEnforced() { + return settingsService.isTrueForKey(SettingsServiceBean.Key.UseStorageQuotas, false); + } + + /** + * This method should only be used for testing of the new storage quota + * mechanism, temporarily. (it uses the same value as the quota for + * *everybody* regardless of the circumstances, defined as a database + * setting) + */ + public Long getTestStorageQuotaLimit() { + return settingsService.getValueForKeyAsLong(SettingsServiceBean.Key.StorageQuotaSizeInBytes); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/data/FileUtilWrapper.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/data/FileUtilWrapper.java index 2bcac04076a..ecb34bdcfb5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/data/FileUtilWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/data/FileUtilWrapper.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException; +import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException; import edu.harvard.iq.dataverse.util.FileUtil; import java.io.File; @@ -43,7 +44,11 @@ public void deleteFile(Path filePath) { } public File saveInputStreamInTempFile(InputStream inputStream, Long fileSizeLimit) throws IOException, FileExceedsMaxSizeException { - return FileUtil.saveInputStreamInTempFile(inputStream, fileSizeLimit); + try { + return FileUtil.saveInputStreamInTempFile(inputStream, fileSizeLimit); + } catch (FileExceedsStorageQuotaException fesqx) { + return null; + } } public String determineFileType(File file, String fileName) throws IOException { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/file/FileExceedsStorageQuotaException.java b/src/main/java/edu/harvard/iq/dataverse/util/file/FileExceedsStorageQuotaException.java new file mode 100644 index 00000000000..29eeca254f7 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/file/FileExceedsStorageQuotaException.java @@ -0,0 +1,22 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package edu.harvard.iq.dataverse.util.file; + +/** + * + * @author landreev + */ +public class FileExceedsStorageQuotaException extends Exception { + + public FileExceedsStorageQuotaException(String message) { + super(message); + } + + public FileExceedsStorageQuotaException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index e5cd72ff5fc..1fed0b233e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -1,22 +1,6 @@ package edu.harvard.iq.dataverse.util.json; import edu.harvard.iq.dataverse.*; -import edu.harvard.iq.dataverse.AuxiliaryFile; -import edu.harvard.iq.dataverse.ControlledVocabularyValue; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.DataFileTag; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetDistributor; -import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; -import edu.harvard.iq.dataverse.DatasetFieldValue; -import edu.harvard.iq.dataverse.DatasetLock; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseContact; -import edu.harvard.iq.dataverse.DataverseFacet; -import edu.harvard.iq.dataverse.DataverseTheme; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroup; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUser; diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 997f0470cc3..ac725caf1b2 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -55,6 +55,7 @@ affiliation=Affiliation storage=Storage curationLabels=Curation Labels metadataLanguage=Dataset Metadata Language +guestbookEntryOption=Guestbook Entry Option createDataverse=Create Dataverse remove=Remove done=Done @@ -206,6 +207,7 @@ notification.welcome=Welcome to {0}! Get started by adding or finding data. Have notification.welcomeConfirmEmail=Also, check for your welcome email to verify your address. notification.demoSite=Demo Site notification.requestFileAccess=File access requested for dataset: {0} was made by {1} ({2}). +notification.requestedFileAccess=You have requested access to files in dataset: {0}. notification.grantFileAccess=Access granted for files in dataset: {0}. notification.rejectFileAccess=Access rejected for requested files in dataset: {0}. notification.createDataverse={0} was created in {1} . To learn more about what you can do with your dataverse, check out the {2}. @@ -744,7 +746,8 @@ dashboard.card.datamove.dataset.command.error.indexingProblem=Dataset could not notification.email.create.dataverse.subject={0}: Your dataverse has been created notification.email.create.dataset.subject={0}: Dataset "{1}" has been created notification.email.dataset.created.subject={0}: Dataset "{1}" has been created -notification.email.request.file.access.subject={0}: Access has been requested for a restricted file +notification.email.request.file.access.subject={0}: Access has been requested for a restricted file in dataset "{1}" +notification.email.requested.file.access.subject={0}: You have requested access to a restricted file in dataset "{1}" notification.email.grant.file.access.subject={0}: You have been granted access to a restricted file notification.email.rejected.file.access.subject={0}: Your request for access to a restricted file has been rejected notification.email.submit.dataset.subject={0}: Dataset "{1}" has been submitted for review @@ -770,6 +773,7 @@ notification.email.greeting.html=Hello,
        notification.email.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the User Guide at {1}/{2}/user or contact {3} at {4} for assistance. notification.email.welcomeConfirmEmailAddOn=\n\nPlease verify your email address at {0} . Note, the verify link will expire after {1}. Send another verification email by visiting your account page. notification.email.requestFileAccess=File access requested for dataset: {0} by {1} ({2}). Manage permissions at {3}. +notification.email.requestFileAccess.guestbookResponse=

        Guestbook Response:

        {0} notification.email.grantFileAccess=Access granted for files in dataset: {0} (view at {1} ). notification.email.rejectFileAccess=Your request for access was rejected for the requested files in the dataset: {0} (view at {1} ). If you have any questions about why your request was rejected, you may reach the dataset owner using the "Contact" link on the upper right corner of the dataset page. # Bundle file editors, please note that "notification.email.createDataverse" is used in a unit test @@ -788,6 +792,7 @@ notification.email.changeEmail=Hello, {0}.{1}\n\nPlease contact us if you did no notification.email.passwordReset=Hi {0},\n\nSomeone, hopefully you, requested a password reset for {1}.\n\nPlease click the link below to reset your Dataverse account password:\n\n {2} \n\n The link above will only work for the next {3} minutes.\n\n Please contact us if you did not request this password reset or need further help. notification.email.passwordReset.subject=Dataverse Password Reset Requested notification.email.datasetWasCreated=Dataset "{1}" was just created by {2} in the {3} collection. +notification.email.requestedFileAccess=You have requested access to a file(s) in dataset "{1}". Your request has been sent to the managers of this dataset who will grant or reject your request. If you have any questions, you may reach the dataset managers using the "Contact" link on the upper right corner of the dataset page. hours=hours hour=hour minutes=minutes @@ -822,6 +827,7 @@ dataverse.curationLabels.title=A set of curation status labels that are used to dataverse.curationLabels.disabled=Disabled dataverse.category=Category dataverse.category.title=The type that most closely reflects this dataverse. +dataverse.guestbookentryatrequest.title=Whether Guestbooks are displayed to users when they request file access or when they download files. dataverse.type.selectTab.top=Select one... dataverse.type.selectTab.researchers=Researcher dataverse.type.selectTab.researchProjects=Research Project @@ -919,6 +925,8 @@ dataverse.datasize.ioerror=Fatal IO error while trying to determine the total si dataverse.inherited=(inherited from enclosing Dataverse) dataverse.default=(Default) dataverse.metadatalanguage.setatdatasetcreation=Chosen at Dataset Creation +dataverse.guestbookentry.atdownload=Guestbook Entry At Download +dataverse.guestbookentry.atrequest=Guestbook Entry At Access Request # rolesAndPermissionsFragment.xhtml # advanced.xhtml @@ -1360,6 +1368,16 @@ dataset.guestbookResponse.guestbook.additionalQuestions=Additional Questions dataset.guestbookResponse.showPreview.errorMessage=Can't show preview. dataset.guestbookResponse.showPreview.errorDetail=Couldn't write guestbook response. +#GuestbookResponse +dataset.guestbookResponse=Guestbook Response +dataset.guestbookResponse.id=Guestbook Response ID +dataset.guestbookResponse.date=Response Date +dataset.guestbookResponse.respondent=Respondent +dataset.guestbookResponse.question=Q +dataset.guestbookResponse.answer=A +dataset.guestbookResponse.noResponse=(No Response) + + # dataset.xhtml dataset.configureBtn=Configure dataset.pageTitle=Add New Dataset @@ -1367,6 +1385,7 @@ dataset.pageTitle=Add New Dataset dataset.accessBtn=Access Dataset dataset.accessBtn.header.download=Download Options dataset.accessBtn.header.explore=Explore Options +dataset.accessBtn.header.configure=Configure Options dataset.accessBtn.header.compute=Compute Options dataset.accessBtn.download.size=ZIP ({0}) dataset.accessBtn.too.big=The dataset is too large to download. Please select the files you need from the files table. @@ -1677,7 +1696,8 @@ file.select.tooltip=Select Files file.selectAllFiles=Select all {0} files in this dataset. file.dynamicCounter.filesPerPage=Files Per Page file.selectToAddBtn=Select Files to Add -file.selectToAdd.tipLimit=File upload limit is {0} per file. +file.selectToAdd.tipLimit=File upload limit is {0} per file. +file.selectToAdd.tipQuotaRemaining=Storage quota: {0} remaining. file.selectToAdd.tipMaxNumFiles=Maximum of {0} {0, choice, 0#files|1#file|2#files} per upload. file.selectToAdd.tipTabularLimit=Tabular file ingest is limited to {2}. file.selectToAdd.tipPerFileTabularLimit=Ingest is limited to the following file sizes based on their format: {0}. @@ -2185,6 +2205,8 @@ file.message.replaceSuccess=The file has been replaced. file.addreplace.file_size_ok=File size is in range. file.addreplace.error.byte_abrev=B file.addreplace.error.file_exceeds_limit=This file size ({0}) exceeds the size limit of {1}. +file.addreplace.error.quota_exceeded=This file (size {0}) exceeds the remaining storage quota of {1}. +file.addreplace.error.unzipped.quota_exceeded=Unzipped files exceed the remaining storage quota of {0}. file.addreplace.error.dataset_is_null=The dataset cannot be null. file.addreplace.error.dataset_id_is_null=The dataset ID cannot be null. file.addreplace.error.parsing=Error in parsing provided json diff --git a/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql new file mode 100644 index 00000000000..c90ee4a5329 --- /dev/null +++ b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql @@ -0,0 +1,63 @@ +ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64); +ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id SERIAL; +ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fileaccessrequests_pkey; +ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id); +ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbookresponse_id INT; +ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fk_fileaccessrequests_guestbookresponse; +ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookresponse FOREIGN KEY (guestbookresponse_id) REFERENCES guestbookresponse(id); +DROP INDEX IF EXISTS created_requests; +CREATE UNIQUE INDEX created_requests ON fileaccessrequests (datafile_id, authenticated_user_id) WHERE request_state='CREATED'; + +ALTER TABLE dataverse ADD COLUMN IF NOT EXISTS guestbookatrequest bool; +ALTER TABLE dataset ADD COLUMN IF NOT EXISTS guestbookatrequest bool; + +ALTER TABLE guestbookresponse ADD COLUMN IF NOT EXISTS eventtype VARCHAR(255); +ALTER TABLE guestbookresponse ADD COLUMN IF NOT EXISTS sessionid VARCHAR(255); + +DO $$ + BEGIN + IF EXISTS (select 1 from pg_class where relname='filedownload') THEN + + UPDATE guestbookresponse g + SET eventtype = (SELECT downloadtype FROM filedownload f where f.guestbookresponse_id = g.id), + sessionid = (SELECT sessionid FROM filedownload f where f.guestbookresponse_id=g.id); + DROP TABLE filedownload; + END IF; + END + $$ ; + + +-- This creates a function that ESTIMATES the size of the +-- GuestbookResponse table (for the metrics display), instead +-- of relying on straight "SELECT COUNT(*) ..." +-- It uses statistics to estimate the number of guestbook entries +-- and the fraction of them related to downloads, +-- i.e. those that weren't created for 'AccessRequest' events. +-- Significant potential savings for an active installation. +-- See https://github.com/IQSS/dataverse/issues/8840 and +-- https://github.com/IQSS/dataverse/pull/8972 for more details + +CREATE OR REPLACE FUNCTION estimateGuestBookResponseTableSize() +RETURNS bigint AS $$ +DECLARE + estimatedsize bigint; +BEGIN + SELECT CASE WHEN relpages<10 THEN 0 + ELSE ((reltuples / relpages) + * (pg_relation_size('public.guestbookresponse') / current_setting('block_size')::int))::bigint + * (SELECT CASE WHEN ((select count(*) from pg_stats where tablename='guestbookresponse') = 0 + OR (select array_position(most_common_vals::text::text[], 'AccessRequest') + FROM pg_stats WHERE tablename='guestbookresponse' AND attname='eventtype') IS NULL) THEN 1 + ELSE 1 - (SELECT (most_common_freqs::text::text[])[array_position(most_common_vals::text::text[], 'AccessRequest')]::bigint + FROM pg_stats WHERE tablename='guestbookresponse' and attname='eventtype') END) + END + FROM pg_class + WHERE oid = 'public.guestbookresponse'::regclass INTO estimatedsize; + + if estimatedsize = 0 then + SELECT COUNT(id) FROM guestbookresponse WHERE eventtype!= 'AccessRequest' INTO estimatedsize; + END if; + + RETURN estimatedsize; +END; +$$ LANGUAGE plpgsql IMMUTABLE; diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index c5958697a20..c54d94442ea 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -7,6 +7,13 @@ xmlns:o="http://omnifaces.org/ui" xmlns:jsf="http://xmlns.jcp.org/jsf"> + +
        @@ -238,17 +245,13 @@
        -
        +
         
        -
        +
        @@ -267,7 +270,7 @@
        -
        +
        -
        +
        -

        +

        -

        +

        diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 67dcf89c380..1b1d77d3057 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -183,6 +183,7 @@ + @@ -197,6 +198,7 @@ + @@ -453,6 +455,17 @@ + + + + +
      • + + + +
      • +
        +
      • @@ -538,7 +551,7 @@
        - + @@ -560,9 +573,9 @@ data-toggle="tooltip" data-placement="auto top" data-original-title="#{bundle['metrics.dataset.downloads.makedatacount.tip']}"> - + - )
        @@ -990,6 +1003,19 @@
      • + +

        + +

        +
        + + + + +
        +

        @@ -1059,11 +1085,11 @@

        #{bundle['dataset.downloadUnrestricted']}

        + rendered="#{DatasetPage.guestbookAndTermsPopupRequired and !settingsWrapper.rsyncDownload}" + oncomplete="PF('guestbookAndTermsPopup').show();" /> @@ -1505,16 +1531,20 @@
        - + - + - + + + + + @@ -1541,19 +1571,11 @@ - + - - - - - - - -
        @@ -1898,7 +1920,7 @@ PF('downloadInvalid').show(); } if (outcome ==='GuestbookRequired'){ - PF('downloadPopup').show(); + PF('guestbookAndTermsPopup').show(); } } diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index aa3fa535807..41e2807c4fd 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -252,6 +252,8 @@
        +
        +
        #{bundle.description}
        +
        +
        + + #{bundle.guestbookEntryOption} + + +
        + + + + +
        +
        +

        @@ -441,7 +458,7 @@
        - +
        diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index 51f5bfa9f8a..2426cf980d3 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -236,6 +236,14 @@ + + + + + #{item.theObject.displayName} + + + diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 5fac8241f13..fff047f494f 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -91,6 +91,11 @@ rendered="#{!EditDatafilesPage.isUnlimitedUploadFileSize()}"> + + + + @@ -154,7 +159,7 @@ dragDropSupport="true" auto="#{!(systemConfig.directUploadEnabled(EditDatafilesPage.dataset))}" multiple="#{datasetPage || EditDatafilesPage.allowMultipleFileUpload()}" - disabled="#{lockedFromEdits || !(datasetPage || EditDatafilesPage.showFileUploadComponent()) }" + disabled="#{lockedFromEdits || !(datasetPage || EditDatafilesPage.showFileUploadComponent()) || EditDatafilesPage.isQuotaExceeded()}" listener="#{EditDatafilesPage.handleFileUpload}" process="filesTable" update=":datasetForm:filesTable, @([id$=filesButtons])" @@ -166,6 +171,7 @@ fileLimit="#{EditDatafilesPage.getMaxNumberOfFiles()}" invalidSizeMessage="#{bundle['file.edit.error.file_exceeds_limit']}" sequential="true" + previewWidth="-1" widgetVar="fileUploadWidget"> diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index f28efc47705..8ef2af40431 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -24,14 +24,15 @@ - - - #{fileMetadata.dataFile.containsFileAccessRequestFromUser(dataverseSession.user) ? bundle['file.accessRequested'] : bundle['file.requestAccess']} + disabled="#{fileMetadata.dataFile.containsActiveFileAccessRequestFromUser(dataverseSession.user)}"> + + #{fileMetadata.dataFile.containsActiveFileAccessRequestFromUser(dataverseSession.user) ? bundle['file.accessRequested'] : bundle['file.requestAccess']}
      • @@ -60,7 +61,7 @@
      • - #{bundle['file.globus.of']} #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType} - + update="@widgetVar(guestbookAndTermsPopup)" oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');"> + GT: #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType} @@ -85,7 +87,7 @@
      • - #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType} - + update="@widgetVar(guestbookAndTermsPopup)" oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');"> + #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType} - #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType} - + update="@widgetVar(guestbookAndTermsPopup)" oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');"> + #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType} @@ -134,23 +138,24 @@
      • - #{bundle['file.downloadBtn.format.all']} - + update="@widgetVar(guestbookAndTermsPopup)" + oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');"> + #{bundle['file.downloadBtn.format.all']}
      • - @@ -158,12 +163,13 @@ - + update="@widgetVar(guestbookAndTermsPopup)" + oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');"> + @@ -171,35 +177,37 @@
      • - #{bundle['file.downloadBtn.format.tab']} - + update="@widgetVar(guestbookAndTermsPopup)" + oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');"> + #{bundle['file.downloadBtn.format.tab']}
      • - #{bundle['file.downloadBtn.format.rdata']} - + update="@widgetVar(guestbookAndTermsPopup)" + oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');"> + #{bundle['file.downloadBtn.format.rdata']} @@ -215,18 +223,19 @@
      • - #{bundle['file.downloadBtn.format.var']} - + update="@widgetVar(guestbookAndTermsPopup)" + oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');"> + #{bundle['file.downloadBtn.format.var']}
      • @@ -303,20 +312,21 @@
      • - #{tool.getDisplayNameLang()} - + update="@widgetVar(guestbookAndTermsPopup)" + oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');"> + #{tool.getDisplayNameLang()}
      • diff --git a/src/main/webapp/file-download-popup-fragment.xhtml b/src/main/webapp/file-download-popup-fragment.xhtml index 632c2a827ef..3a64ca4a3a2 100644 --- a/src/main/webapp/file-download-popup-fragment.xhtml +++ b/src/main/webapp/file-download-popup-fragment.xhtml @@ -1,3 +1,4 @@ +
        - + diff --git a/src/main/webapp/file-request-access-popup-fragment.xhtml b/src/main/webapp/file-request-access-popup-fragment.xhtml deleted file mode 100644 index 6541d86b686..00000000000 --- a/src/main/webapp/file-request-access-popup-fragment.xhtml +++ /dev/null @@ -1,53 +0,0 @@ - - - -

        - #{someActivelyEmbargoedFiles ? bundle['file.requestAccessTermsDialog.embargoed.tip'] : bundle['file.requestAccessTermsDialog.tip']} -

        -

        - #{bundle['file.requestAccessTermsDialog.embargoed']} -

        -
        -
        - -
        -
        -
        - -
        -
        -
        -
        -
        - -
        -
        -
        - -
        -
        -
        -
        -
        -
        - - - -
        -
        diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 5a60afef60c..f69b5c35afd 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -204,7 +204,7 @@ or FilePage.fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload}"> - + @@ -214,6 +214,7 @@ + @@ -297,7 +298,7 @@
        - + @@ -305,7 +306,7 @@
        - + @@ -353,20 +354,22 @@ - - + + - + + + - + - + + styleClass="btn btn-default btn-request" + update="@form, @([id$=messagePanel])" + action="#{DatasetPage.validateFilesForRequestAccess()}" + disabled="#{DatasetPage.locked or !DatasetPage.fileAccessRequestMultiButtonEnabled}"> + #{bundle['file.requestAccess']}
        -
        +
        #{bundle['file.accessRequested']}  @@ -548,15 +552,17 @@ - - + + + + diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml new file mode 100644 index 00000000000..69cc9fae55c --- /dev/null +++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml @@ -0,0 +1,324 @@ + + + + + +

        + #{someActivelyEmbargoedFiles ? bundle['file.requestAccessTermsDialog.embargoed.tip'] : bundle['file.requestAccessTermsDialog.tip']} +

        +

        + #{bundle['file.requestAccessTermsDialog.embargoed']} +

        +
        + +

        + #{bundle['file.downloadDialog.tip']} +

        +
        +
        +
        + + +
        + + +
        + + +
        +
        + + +
        +
        + + +
        +
        + + +
        +
        + + +
        +
        + + +
        +
        + + +
        +
        + + +
        +
        +
        + + +
        + +
        + +
        + + +
        + +
        + + + +
        +
        +
        + +
        + + + +
        +
        +
        + +
        + + + +
        +
        +
        + +
        + + + +
        +
        +
        + +
        + +
        + + + + + + + + + + + +
        +
        +
        +
        +
        +
        +
        + + + + + + + + + + + + + + + + + + + + + +
        +
        + + + + + + + + + + +
        +
        diff --git a/src/main/webapp/manage-templates.xhtml b/src/main/webapp/manage-templates.xhtml index c9841ace8e8..879cf9e55c2 100644 --- a/src/main/webapp/manage-templates.xhtml +++ b/src/main/webapp/manage-templates.xhtml @@ -139,7 +139,7 @@
        + styleClass="largePopUp" widgetVar="deleteConfirmation" modal="true" focus="contDeleteTemplateBtn">

         

        diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index d3109da69a6..4e4e56f2051 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -322,7 +322,7 @@ diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index 48b6eee38e1..416caa68566 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -546,7 +546,7 @@ public void testRequestAccess() throws InterruptedException { assertEquals(200, revokeFileAccessResponse.getStatusCode()); listAccessRequestResponse = UtilIT.getAccessRequestList(tabFile3IdRestrictedNew.toString(), apiToken); - assertEquals(400, listAccessRequestResponse.getStatusCode()); + assertEquals(404, listAccessRequestResponse.getStatusCode()); } // This is a round trip test of uploading a zipped archive, with some folder diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java index 6f414fb3e24..022747a3cdc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import io.restassured.RestAssured; import io.restassured.path.json.JsonPath; import io.restassured.response.Response; @@ -235,6 +236,84 @@ public void testDatasetLevelTool1() { } + @Test + public void testDatasetLevelToolConfigure() { + + // Delete all external tools before testing. + Response getTools = UtilIT.getExternalTools(); + getTools.prettyPrint(); + getTools.then().assertThat() + .statusCode(OK.getStatusCode()); + String body = getTools.getBody().asString(); + JsonReader bodyObject = Json.createReader(new StringReader(body)); + JsonArray tools = bodyObject.readObject().getJsonArray("data"); + for (int i = 0; i < tools.size(); i++) { + JsonObject tool = tools.getJsonObject(i); + int id = tool.getInt("id"); + Response deleteExternalTool = UtilIT.deleteExternalTool(id); + deleteExternalTool.prettyPrint(); + } + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + createUser.then().assertThat() + .statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + createDataverseResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = JsonPath.from(createDataset.getBody().asString()).getInt("data.id"); + String datasetPid = JsonPath.from(createDataset.getBody().asString()).getString("data.persistentId"); + + String toolManifest = """ +{ + "displayName": "Dataset Configurator", + "description": "Slices! Dices! More info.", + "types": [ + "configure" + ], + "scope": "dataset", + "toolUrl": "https://datasetconfigurator.com", + "toolParameters": { + "queryParameters": [ + { + "datasetPid": "{datasetPid}" + }, + { + "localeCode": "{localeCode}" + } + ] + } + } +"""; + + Response addExternalTool = UtilIT.addExternalTool(JsonUtil.getJsonObject(toolManifest)); + addExternalTool.prettyPrint(); + addExternalTool.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.displayName", CoreMatchers.equalTo("Dataset Configurator")); + + Response getExternalToolsByDatasetId = UtilIT.getExternalToolsForDataset(datasetId.toString(), "configure", apiToken); + getExternalToolsByDatasetId.prettyPrint(); + getExternalToolsByDatasetId.then().assertThat() + .body("data[0].displayName", CoreMatchers.equalTo("Dataset Configurator")) + .body("data[0].scope", CoreMatchers.equalTo("dataset")) + .body("data[0].types[0]", CoreMatchers.equalTo("configure")) + .body("data[0].toolUrlWithQueryParams", CoreMatchers.equalTo("https://datasetconfigurator.com?datasetPid=" + datasetPid)) + .statusCode(OK.getStatusCode()); + + } + @Test public void testAddFilelToolNoFileId() throws IOException { JsonObjectBuilder job = Json.createObjectBuilder(); diff --git a/src/test/java/edu/harvard/iq/dataverse/dataverse/DataverseUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/dataverse/DataverseUtilTest.java index 01e0edd3073..b950d641bf4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataverse/DataverseUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataverse/DataverseUtilTest.java @@ -33,9 +33,9 @@ public void testCheckMetadataLanguageCases() { mLangSettingMap.put("en", "English"); mLangSettingMap.put("fr", "French"); Dataverse undefinedParent = new Dataverse(); - undefinedParent.setMetadataLanguage(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE); + undefinedParent.setMetadataLanguage(DvObjectContainer.UNDEFINED_CODE); Dataset undefinedD = new Dataset(); - undefinedD.setMetadataLanguage(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE); + undefinedD.setMetadataLanguage(DvObjectContainer.UNDEFINED_CODE); Dataverse definedParent = new Dataverse(); definedParent.setMetadataLanguage("en"); Dataset definedEnglishD = new Dataset(); diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java index 39bf96210fc..ad2a24ecdb8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java @@ -1,10 +1,12 @@ package edu.harvard.iq.dataverse.externaltools; +import edu.harvard.iq.dataverse.DOIServiceBean; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.settings.JvmSettings; @@ -15,6 +17,7 @@ import jakarta.json.Json; import jakarta.json.JsonObject; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -234,4 +237,43 @@ public void testGetToolUrlWithAllowedApiCalls() { assertTrue(signedUrl.contains("&token=")); System.out.println(JsonUtil.prettyPrint(jo)); } + + @Test + @JvmSetting(key = JvmSettings.SITE_URL, value = "https://librascholar.org") + public void testDatasetConfigureTool() { + List externalToolTypes = new ArrayList<>(); + var externalToolType = new ExternalToolType(); + externalToolType.setType(ExternalTool.Type.CONFIGURE); + externalToolTypes.add(externalToolType); + var scope = ExternalTool.Scope.DATASET; + String toolUrl = "http://example.com"; + var externalTool = new ExternalTool("displayName", "toolName", "description", externalToolTypes, scope, toolUrl, "{}", DataFileServiceBean.MIME_TYPE_TSV_ALT); + + externalTool.setToolParameters(Json.createObjectBuilder() + .add("queryParameters", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("siteUrl", "{siteUrl}") + ) + .add(Json.createObjectBuilder() + .add("datasetPid", "{datasetPid}") + ) + .add(Json.createObjectBuilder() + .add("localeCode", "{localeCode}") + ) + ) + .build().toString()); + + var dataset = new Dataset(); + dataset.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL, "10.5072", "ABC123", null, DOIServiceBean.DOI_RESOLVER_URL, null)); + ApiToken nullApiToken = null; + String nullLocaleCode = "en"; + var externalToolHandler = new ExternalToolHandler(externalTool, dataset, nullApiToken, nullLocaleCode); + System.out.println("tool: " + externalToolHandler.getToolUrlWithQueryParams()); + assertEquals("http://example.com?siteUrl=https://librascholar.org&datasetPid=doi:10.5072/ABC123&localeCode=en", externalToolHandler.getToolUrlWithQueryParams()); + assertFalse(externalToolHandler.getExternalTool().isExploreTool()); + assertEquals("configure", externalToolHandler.getExternalTool().getExternalToolTypes().get(0).getType().toString()); + assertEquals("dataset", externalToolHandler.getExternalTool().getScope().toString()); + + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java index bbdf5a84fc3..205b1f0bfcf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java @@ -80,7 +80,7 @@ public void testSubjectRevokeRole() { @Test public void testSubjectRequestFileAccess() { userNotification.setType(UserNotification.Type.REQUESTFILEACCESS); - assertEquals("LibraScholar: Access has been requested for a restricted file", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); + assertEquals("LibraScholar: Access has been requested for a restricted file in dataset \"\"", MailUtil.getSubjectTextBasedOnNotification(userNotification, null)); } @Test