Merge branch 'develop' into 6684-dataset-pg-btn-responsive

# Conflicts:
#	src/main/webapp/dataset.xhtml
#	src/main/webapp/file-download-button-fragment.xhtml

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,7 +8,7 @@ Closes #
 
 **Suggestions on how to test this**:
 
-**Does this PR introduce a user interface change?**:
+**Does this PR introduce a user interface change? If mockups are available, please link/include them here**:
 
 **Is there a release notes update needed for this change?**:
 

doc/release-notes/5093-datacite.md

@@ -0,0 +1 @@
+If you are using DataCite as your DOI provider you must add a new JVM option called "doi.baseurlstringnext" with a value of "https://api.datacite.org" for production environments and "https://api.test.datacite.org" for test environments. More information about this JVM option can be found in the Installation Guide.

doc/release-notes/6895-payara-release-note

@@ -0,0 +1,5 @@
+##Payara 5
+
+A major upgrade of the application server will provide security updates, access to new features like MicroProfile Config API, and will enable upgrades to other core technologies.
+
+Note that moving from Glassfish to Payara will be required as part of the move to Dataverse 5.

doc/release-notes/6956-shib-attribute-character-set-conversion.md

@@ -0,0 +1,6 @@
+# Shibboleth attribute character set conversion
+
+By default, all attributes received from Shibboleth are converted from ISO-8859-1 to UTF-8.
+You can disable this behaviour by setting ShibAttributeCharacterSetConversionEnabled to false.
+
+For further documentation on this issue, see: doc/sphinx-guides/source/installation/config.rst

doc/sphinx-guides/source/admin/dataverses-datasets.rst

@@ -47,17 +47,17 @@ To direct new files (uploaded when datasets are created or edited) for all datas
  
     curl -H "X-Dataverse-key: $API_TOKEN" -X PUT -d $storageDriverLabel http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver
     
-The current driver can be seen using:
+The current driver can be seen using::
 
     curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver
 
-and can be reset to the default store with:
+and can be reset to the default store with::
 
     curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver
     
-The available drivers can be listed with:
+The available drivers can be listed with::
 
-    curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/storageDrivers
+    curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/storageDrivers
     
 
 Datasets
@@ -115,6 +115,11 @@ Forces update to metadata provided to the PID provider of a published dataset. O
 
     curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/datasets/$dataset-id/modifyRegistrationMetadata
 
+Check for Unreserved PIDs and Reserve Them
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+See :ref:`pids-api` in the API Guide for details.
+
 Make Metadata Updates Without Changing Dataset Version
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

doc/sphinx-guides/source/api/dataaccess.rst

@@ -67,6 +67,8 @@ Multiple File ("bundle") download
 
 ``/api/access/datafiles/$id1,$id2,...$idN``
 
+Alternate Form: POST to ``/api/access/datafiles`` with a ``fileIds`` input field containing the same comma separated list of file ids. This is most useful when your list of files surpasses the allowed URL length (varies but can be ~2000 characters).  
+
 Returns the files listed, zipped. 
 
 .. note:: If the request can only be completed partially - if only *some* of the requested files can be served (because of the permissions and/or size restrictions), the file MANIFEST.TXT included in the zipped bundle will have entries specifying the reasons the missing files could not be downloaded. IN THE FUTURE the API will return a 207 status code to indicate that the result was a partial success. (As of writing this - v.4.11 - this hasn't been implemented yet)

doc/sphinx-guides/source/api/native-api.rst

@@ -2519,6 +2519,98 @@ Each user can get a dump of their notifications by passing in their API token::
 
     curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/notifications/all
 
+.. _pids-api:
+
+PIDs
+----
+
+PIDs is short for Persistent IDentifiers. Examples include DOI or Handle. There are some additional PID operations listed in the :doc:`/admin/dataverses-datasets` section of the Admin Guide.
+
+Get Info on a PID
+~~~~~~~~~~~~~~~~~
+
+Get information on a PID, especially its "state" such as "draft" or "findable". Currently, this API only works on DataCite DOIs. A superuser API token is required.
+
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PID=doi:10.70122/FK2/9BXT5O
+
+  curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/pids?persistentId=$PID
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/pids?persistentId=doi:10.70122/FK2/9BXT5O
+
+List Unreserved PIDs
+~~~~~~~~~~~~~~~~~~~~
+
+Get a list of PIDs that have not been reserved on the PID provider side. This can happen, for example, if a dataset is created while the PID provider is down. A superuser API token is required.
+
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+
+  curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/pids/unreserved
+
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/pids/unreserved
+
+Reserve a PID
+~~~~~~~~~~~~~
+
+Reserved a PID for a dataset. A superuser API token is required.
+
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PID=doi:10.70122/FK2/9BXT5O
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/pids/:persistentId/reserve?persistentId=$PID
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/pids/:persistentId/reserve?persistentId=doi:10.70122/FK2/9BXT5O
+
+Delete a PID
+~~~~~~~~~~~~
+
+Delete PID from DataCite (this is only possible for PIDs that are in the "draft" state) and within Dataverse, set ``globalidcreatetime`` to null and ``identifierregistered`` to false. A superuser API token is required.
+
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PID=doi:10.70122/FK2/9BXT5O
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE $SERVER_URL/api/pids/:persistentId/delete?persistentId=$PID
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X DELETE https://demo.dataverse.org/api/pids/:persistentId/delete?persistentId=doi:10.70122/FK2/9BXT5O
+
+
 .. _admin:
 
 Admin

doc/sphinx-guides/source/installation/config.rst

@@ -1038,6 +1038,20 @@ See also these related database settings below:
 - :ref:`:Authority`
 - :ref:`:Shoulder`
 
+.. _doi.baseurlstringnext
+
+doi.baseurlstringnext
++++++++++++++++++++++
+
+Dataverse uses multiple APIs from DataCite:
+
+- The DataCite MDS API is older, XML-based, and configured using :ref:`doi.baseurlstring`.
+- The DataCite REST API is newer, JSON-based, and configured using ``doi.baseurlstringnext``.
+
+In production, ``doi.baseurlstringnext`` should be set to ``https://api.datacite.org``
+
+While testing, ``doi.baseurlstringnext`` should be set to ``https://api.test.datacite.org``
+
 .. _doi.mdcbaseurlstring:
 
 doi.mdcbaseurlstring
@@ -1873,6 +1887,17 @@ To check the current value of ``:ShibAffiliationAttribute``:
 
 ``curl -X GET http://localhost:8080/api/admin/settings/:ShibAffiliationAttribute``
 
+:ShibAttributeCharacterSetConversionEnabled
++++++++++++++++++++++++++++++++++++++++++++
+
+It seems that the application server (usually Glassfish or Payara) will interpret all Shibboleth attributes that come through AJP as ISO-8859-1, even if they where originally UTF-8.
+To circumvent that, we re-encode all received Shibboleth attributes manually as UTF-8 by default. 
+In the case you get garbled characters in Shibboleth-supplied fields (e.g. given name, surname, affiliation), you can disable this behaviour by setting ShibAttributeCharacterSetConversionEnabled to false:
+
+``curl -X PUT -d false http://localhost:8080/api/admin/settings/:ShibAttributeCharacterSetConversionEnabled``
+
+If you managed to get correct accented characters from shibboleth while this setting is _false_, please contact us with your application server and Shibboleth configuration!
+
 .. _:ComputeBaseUrl:
 
 :ComputeBaseUrl

src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java

@@ -58,6 +58,15 @@ private DataCiteRESTfullClient getClient() throws IOException {
         return client;
     }
 
+    /**
+     * This method is deprecated and unused. We switched away from this method
+     * when adjusting the code to reserve DOIs from DataCite on dataset create.
+     *
+     * Note that the DOIDataCiteRegisterCache entity/table used in this method
+     * might be a candidate for deprecation as well. Removing it would require
+     * some refactoring as it is used throughout the DataCite code.
+     */
+    @Deprecated
     public String createIdentifierLocal(String identifier, Map<String, String> metadata, DvObject dvObject) {
 
         String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject);
@@ -83,6 +92,49 @@ public String createIdentifierLocal(String identifier, Map<String, String> metad
         return retString;
     }
 
+    /**
+     * This "reserveIdentifier" method is heavily based on the
+     * "registerIdentifier" method below but doesn't, this one doesn't doesn't
+     * register a URL, which causes the "state" of DOI to transition from
+     * "draft" to "findable". Here are some DataCite docs on the matter:
+     *
+     * "DOIs can exist in three states: draft, registered, and findable. DOIs
+     * are in the draft state when metadata have been registered, and will
+     * transition to the findable state when registering a URL." --
+     * https://support.datacite.org/docs/mds-api-guide#doi-states
+     */
+    public String reserveIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject) throws IOException {
+        String retString = "";
+        String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject);
+        DOIDataCiteRegisterCache rc = findByDOI(identifier);
+        String target = metadata.get("_target");
+        if (rc != null) {
+            rc.setDoi(identifier);
+            rc.setXml(xmlMetadata);
+            // DataCite uses the term "draft" instead of "reserved".
+            rc.setStatus("reserved");
+            if (target == null || target.trim().length() == 0) {
+                target = rc.getUrl();
+            } else {
+                rc.setUrl(target);
+            }
+            try {
+                DataCiteRESTfullClient client = getClient();
+                retString = client.postMetadata(xmlMetadata);
+            } catch (UnsupportedEncodingException ex) {
+                Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex);
+            }
+        } else {
+            try {
+                DataCiteRESTfullClient client = getClient();
+                retString = client.postMetadata(xmlMetadata);
+            } catch (UnsupportedEncodingException ex) {
+                Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex);
+            }
+        }
+        return retString;
+    }
+
     public String registerIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject) throws IOException {
         String retString = "";
         String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject);
@@ -116,7 +168,7 @@ public String registerIdentifier(String identifier, Map<String, String> metadata
         return retString;
     }
 
-    public String deactivateIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject) {
+    public String deactivateIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject) throws IOException {
         String retString = "";
 
             String metadataString = getMetadataForDeactivateIdentifier(identifier, metadata, dvObject);

src/main/java/edu/harvard/iq/dataverse/DOIDataCiteServiceBean.java

@@ -1,5 +1,7 @@
 package edu.harvard.iq.dataverse;
 
+import edu.harvard.iq.dataverse.pidproviders.PidUtil;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -31,7 +33,7 @@ public DOIDataCiteServiceBean() {
 
     @Override
     public boolean registerWhenPublished() {
-        return true;
+        return false;
     }
 
     @Override
@@ -72,7 +74,7 @@ public String createIdentifier(DvObject dvObject) throws Exception {
         Map<String, String> metadata = getMetadataForCreateIndicator(dvObject);
         metadata.put("_status", "reserved");
         try {
-            String retString = doiDataCiteRegisterService.createIdentifierLocal(identifier, metadata, dvObject);
+            String retString = doiDataCiteRegisterService.reserveIdentifier(identifier, metadata, dvObject);
             logger.log(Level.FINE, "create DOI identifier retString : " + retString);
             return retString;
         } catch (Exception e) {
@@ -169,40 +171,18 @@ public void deleteRecordFromCache(Dataset datasetIn){
         }
     }
 
+    /**
+     * Deletes DOI from the DataCite side, if possible. Only "draft" DOIs can be
+     * deleted.
+     */
     @Override
-    public void deleteIdentifier(DvObject dvObject) throws Exception {
-        logger.log(Level.FINE,"deleteIdentifier");
-        String identifier = getIdentifier(dvObject);
-        Map<String, String> doiMetadata = new HashMap<>();
-        try {
-            doiMetadata = doiDataCiteRegisterService.getMetadata(identifier);
-        } catch (Exception e) {
-            logger.log(Level.WARNING, "deleteIdentifier: get matadata failed. " + e.getMessage(), e);
-        }
-
-        String idStatus = doiMetadata.get("_status");
-        if ( idStatus != null ) {
-            switch ( idStatus ) {
-                case RESERVED:
-                case DRAFT:    
-                    logger.log(Level.INFO, "Delete status is reserved..");
-                    try {
-                        doiDataCiteRegisterService.deleteIdentifier(identifier);
-                    } catch (Exception e) {
-                        logger.log(Level.WARNING, "delete failed: " + e.getMessage(), e);
-                    }
-                    break;
-
-                case PUBLIC:
-                case FINDABLE:
-                    //if public then it has been released set to unavailable and reset target to n2t url
-                    Map<String, String> metadata = addDOIMetadataForDestroyedDataset(dvObject);
-                    metadata.put("_status", "registered");
-                    metadata.put("_target", getTargetUrl(dvObject));                   
-                    doiDataCiteRegisterService.deactivateIdentifier(identifier, metadata, dvObject);
-                    break;
-            }
-        }
+    public void deleteIdentifier(DvObject dvObject) throws IOException {
+        String baseUrl = System.getProperty("doi.baseurlstringnext");
+        String username = System.getProperty("doi.username");
+        String password = System.getProperty("doi.password");
+        String pid = dvObject.getGlobalId().asString();
+        int result = PidUtil.deleteDoi(pid, baseUrl, username, password);
+        logger.fine("Result of deleteIdentifier for " + pid + ": " + result);
     }
 
     private boolean updateIdentifierStatus(DvObject dvObject, String statusIn) {

src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java

@@ -175,26 +175,18 @@ public boolean testDOIExists(String doi) {
      * @param metadata
      * @return
      */
-    public String postMetadata(String metadata) {
-
+    public String postMetadata(String metadata) throws IOException {
         HttpPost httpPost = new HttpPost(this.url + "/metadata");
         httpPost.setHeader("Content-Type", "application/xml;charset=UTF-8");
-        try {
-            httpPost.setEntity(new StringEntity(metadata, "utf-8"));
-            HttpResponse response = httpClient.execute(httpPost,context);
-
-            String data = EntityUtils.toString(response.getEntity(), encoding);
-            if (response.getStatusLine().getStatusCode() != 201) {
-                String errMsg = "Response code: " + response.getStatusLine().getStatusCode() + ", " + data;
-                logger.log(Level.SEVERE, errMsg);
-                throw new RuntimeException(errMsg);
-            }
-            return data;
-
-        } catch (IOException ioe) {
-            logger.log(Level.SEVERE, "IOException when post metadata");
-            throw new RuntimeException("IOException when post metadata", ioe);
+        httpPost.setEntity(new StringEntity(metadata, "utf-8"));
+        HttpResponse response = httpClient.execute(httpPost, context);
+        String data = EntityUtils.toString(response.getEntity(), encoding);
+        if (response.getStatusLine().getStatusCode() != 201) {
+            String errMsg = "Response code: " + response.getStatusLine().getStatusCode() + ", " + data;
+            logger.log(Level.SEVERE, errMsg);
+            throw new IOException(errMsg);
         }
+        return data;
     }
 
     /**