Merge branch 'develop' into 7424-maildefinition-ng

.env

@@ -2,3 +2,4 @@ APP_IMAGE=gdcc/dataverse:unstable
 POSTGRES_VERSION=13
 DATAVERSE_DB_USER=dataverse
 SOLR_VERSION=9.3.0
+SKIP_DEPLOY=0

README.md

@@ -3,7 +3,7 @@ Dataverse®
 
 Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Dataverse team](https://dataverse.org/about) at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][]).
 
-[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our development [roadmap][], and more.
+[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our [project board][], our development [roadmap][], and more.
 
 We maintain a demo site at [demo.dataverse.org][] which you are welcome to use for testing and evaluating Dataverse.
 
@@ -29,6 +29,7 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi
 [Installation Guide]: https://guides.dataverse.org/en/latest/installation/index.html
 [latest release]: https://github.com/IQSS/dataverse/releases
 [features]: https://dataverse.org/software-features
+[project board]: https://github.com/orgs/IQSS/projects/34
 [roadmap]: https://www.iq.harvard.edu/roadmap-dataverse-project
 [integrations]: https://dataverse.org/integrations
 [REST APIs]: https://guides.dataverse.org/en/latest/api/index.html

conf/proxy/Caddyfile

@@ -0,0 +1,12 @@
+# This configuration is intended to be used with Caddy, a very small high perf proxy.
+# It will serve the application containers Payara Admin GUI via HTTP instead of HTTPS,
+# avoiding the trouble of self signed certificates for local development.
+
+:4848 {
+	reverse_proxy https://dataverse:4848 {
+		transport http {
+			tls_insecure_skip_verify
+		}
+		header_down Location "^https://" "http://"
+	}
+}

conf/solr/9.3.0/schema.xml

@@ -229,6 +229,8 @@
 
     <!-- incomplete datasets issue 8822 -->
     <field name="datasetValid" type="boolean" stored="true" indexed="true" multiValued="false"/>
+
+    <field name="license" type="string" stored="true" indexed="true" multiValued="false"/>
 
     <!--
         METADATA SCHEMA FIELDS
@@ -327,7 +329,7 @@
     <field name="keywordVocabularyURI" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="kindOfData" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="language" type="text_en" multiValued="true" stored="true" indexed="true"/>
-    <field name="northLongitude" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="northLatitude" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="notesText" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="originOfSources" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="otherDataAppraisal" type="text_en" multiValued="false" stored="true" indexed="true"/>
@@ -370,7 +372,7 @@
     <field name="software" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="softwareName" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="softwareVersion" type="text_en" multiValued="true" stored="true" indexed="true"/>
-    <field name="southLongitude" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="southLatitude" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="state" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="studyAssayCellType" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="studyAssayMeasurementType" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -566,7 +568,7 @@
     <copyField source="keywordVocabularyURI" dest="_text_" maxChars="3000"/>
     <copyField source="kindOfData" dest="_text_" maxChars="3000"/>
     <copyField source="language" dest="_text_" maxChars="3000"/>
-    <copyField source="northLongitude" dest="_text_" maxChars="3000"/>
+    <copyField source="northLatitude" dest="_text_" maxChars="3000"/>
     <copyField source="notesText" dest="_text_" maxChars="3000"/>
     <copyField source="originOfSources" dest="_text_" maxChars="3000"/>
     <copyField source="otherDataAppraisal" dest="_text_" maxChars="3000"/>
@@ -609,7 +611,7 @@
     <copyField source="software" dest="_text_" maxChars="3000"/>
     <copyField source="softwareName" dest="_text_" maxChars="3000"/>
     <copyField source="softwareVersion" dest="_text_" maxChars="3000"/>
-    <copyField source="southLongitude" dest="_text_" maxChars="3000"/>
+    <copyField source="southLatitude" dest="_text_" maxChars="3000"/>
     <copyField source="state" dest="_text_" maxChars="3000"/>
     <copyField source="studyAssayCellType" dest="_text_" maxChars="3000"/>
     <copyField source="studyAssayMeasurementType" dest="_text_" maxChars="3000"/>

doc/release-notes/10117-api-metrics-add-user-accounts-number.md

@@ -0,0 +1,3 @@
+### New Accounts Metrics API
+
+Users can retrieve new types of metrics related to user accounts. The new capabilities are [described](https://guides.dataverse.org/en/6.2/api/metrics.html) in the guides.

doc/release-notes/10280-get-file-api-extension.md

@@ -0,0 +1,10 @@
+The API endpoint `api/files/{id}` has been extended to support the following optional query parameters:
+
+- `includeDeaccessioned`: Indicates whether or not to consider deaccessioned dataset versions in the latest file search. (Default: `false`).
+- `returnDatasetVersion`: Indicates whether or not to include the dataset version of the file in the response. (Default: `false`).
+
+A new endpoint `api/files/{id}/versions/{datasetVersionId}` has been created. This endpoint returns the file metadata present in the requested dataset version. To specify the dataset version, you can use ``:latest-published``, or ``:latest``, or ``:draft`` or ``1.0`` or any other available version identifier.
+
+The endpoint supports the `includeDeaccessioned` and `returnDatasetVersion` optional query parameters, as does the `api/files/{id}` endpoint.
+
+`api/files/{id}/draft` endpoint is no longer available in favor of the new endpoint `api/files/{id}/versions/{datasetVersionId}`, which can use the version identifier ``:draft`` (`api/files/{id}/versions/:draft`) to obtain the same result.

doc/release-notes/10286-return-owner-added-to-get-apis.md

@@ -0,0 +1,5 @@
+The API endpoints for getting datasets, Dataverse collections, and datafiles have been extended to support the following optional 'returnOwners' query parameter.
+
+Including the parameter and setting it to true will add a hierarchy showing which dataset and dataverse collection(s) the object is part of to the json object returned. 
+
+

doc/release-notes/10318-uningest-and-reingest.md

@@ -0,0 +1,3 @@
+New Uningest/Reingest options are available in the File Page Edit menu, allowing ingest errors to be cleared (by users who can published the associated dataset)
+and (by superusers) for a successful ingest to be undone or retried (e.g. after a Dataverse version update or if ingest size limits are changed).
+The /api/files/<id>/uningest api also now allows users who can publish the dataset to undo an ingest failure.

doc/release-notes/10342-assign-roles-without-privilege-escalation.md

@@ -0,0 +1 @@
+The permissions required to assign a role have been fixed. It is no longer possible to assign a role that includes permissions that the assigning user doesn't have.

doc/release-notes/10360-binder-redirect.md

@@ -0,0 +1,3 @@
+If your installation is configured to use Binder, you should remove the old "girder_ythub" tool and replace it with the tool described at https://github.com/IQSS/dataverse-binder-redirect
+
+For more information, see #10360.

doc/release-notes/3623-multipid.md

@@ -0,0 +1,37 @@
+This release adds support for using multiple PID (DOI, Handle, PermalLink) providers, multiple PID provider accounts
+(managing a given protocol, authority,separator, shoulder combination), assigning PID provider accounts to specific collections,
+and supporting transferred PIDs (where a PID is managed by an account when it's authority, separator, and/or shoulder don't match
+the combination where the account can mint new PIDs). It also adds the ability for additional provider services beyond the existing
+DataCite, EZId, Handle, and PermaLink providers to be dynamically added as separate jar files.
+
+These changes require per-provider settings rather than the global PID settings previously supported. While backward compatibility 
+for installations using a single PID Provider account is provided, updating to use the new microprofile settings is highly recommended
+and will be required in a future version.
+
+New microprofile settings (where * indicates a provider id indicating which provider the setting is for):
+
+dataverse.pid.providers
+dataverse.pid.default-provider
+dataverse.pid.*.type
+dataverse.pid.*.label
+dataverse.pid.*.authority
+dataverse.pid.*.shoulder
+dataverse.pid.*.identifier-generation-style
+dataverse.pid.*.datafile-pid-format
+dataverse.pid.*.managed-list
+dataverse.pid.*.excluded-list
+dataverse.pid.*.datacite.mds-api-url
+dataverse.pid.*.datacite.rest-api-url
+dataverse.pid.*.datacite.username
+dataverse.pid.*.datacite.password
+dataverse.pid.*.ezid.api-url
+dataverse.pid.*.ezid.username
+dataverse.pid.*.ezid.password
+dataverse.pid.*.permalink.base-url
+dataverse.pid.*.permalink.separator
+dataverse.pid.*.handlenet.index
+dataverse.pid.*.handlenet.independent-service
+dataverse.pid.*.handlenet.auth-handle
+dataverse.pid.*.handlenet.key.path
+dataverse.pid.*.handlenet.key.passphrase
+dataverse.spi.pidproviders.directory

doc/release-notes/3702-return-to-author.md

@@ -0,0 +1,4 @@
+### Return to author
+
+Popup for returning to author now requires a reason that will be sent by email to the author.
+Please note that you can still type a creative and meaningful comment such as "The author would like to modify his dataset", "Files are missing", "Nothing to report" or "A curation report with comments and suggestions/instructions will follow in another email" that suits your situation.

doc/release-notes/5645-geospatial-props-nslong-fix.md

@@ -0,0 +1,4 @@
+Across the application, the Geospatial metadata block fields for north and south were labeled incorrectly as ‘Longitudes,’ as reported on #5645. After updating to this version of Dataverse, users will need to update all the endpoints that used ‘northLongitude’ and ‘southLongitude’ to ‘northLatitude’ and ‘southLatitude,’ respectively.
+
+
+TODO: Whoever puts the release notes together should make sure there is the standard note about updating the schema after upgrading.

doc/release-notes/8134-add-qpj-qmd-extensions.md

@@ -0,0 +1,3 @@
+Add .qpj and .qmd Extensions to Shapefile Handling
+
+- Support for `.qpj` and `.qmd` files in shapefile uploads has been introduced, ensuring that these files are properly recognized and handled as part of geospatial datasets in Dataverse.

doc/release-notes/9060-7482-make-licenses-searchable-faceatable.md

@@ -0,0 +1,6 @@
+### Search by License
+
+A browse/search facet called "License" has been added and will be displayed as long as there is more than one license in datasets and datafiles in browse/search results. This facet allow you to filter by license such as CC0, etc.
+Also, the Search API now handles license filtering using the `fq` parameter, for example : `/api/search?q=*&fq=license%3A%22CC0+1.0%22` for CC0 1.0. See PR #10204
+
+

doc/release-notes/9590-faster-redeploy.md

@@ -0,0 +1,5 @@
+In the Container Guide, documentation for developers on how to quickly redeploy code has been added for Netbeans and improved for IntelliJ.
+
+Also in the context of containers, a new option to skip deployment has been added and the war file is now consistently named "dataverse.war" rather than having a version in the filename, such as "dataverse-6.1.war". This predictability makes tooling easier.
+
+Finally, an option to create tabs in the guides using [Sphinx Tabs](https://sphinx-tabs.readthedocs.io) has been added. (You can see the tabs in action in the "dev usage" page of the Container Guide.) To continue building the guides, you will need to install this new dependency by re-running  `pip install -r requirements.txt`.

doc/sphinx-guides/requirements.txt

@@ -5,3 +5,6 @@ sphinx-icon==0.1.2
 
 # Markdown support
 myst-parser==2.0.0
+
+# tabs
+sphinx-tabs==3.4.5

doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv

@@ -1,7 +1,7 @@
 Tool	Type	Scope	Description
 Data Explorer	explore	file	"A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse."
 Whole Tale	explore	dataset	"A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide <https://wholetale.readthedocs.io/en/stable/users_guide/integration.html>`_."
-Binder	explore	dataset	Binder allows you to spin up custom computing environments in the cloud (including Jupyter notebooks) with the files from your dataset. `Installation instructions <https://github.com/data-exp-lab/girder_ythub/issues/10>`_ are in the Data Exploration Lab girder_ythub project.
+Binder	explore	dataset	Binder allows you to spin up custom computing environments in the cloud (including Jupyter notebooks) with the files from your dataset. See https://github.com/IQSS/dataverse-binder-redirect for installation instructions.
 File Previewers	explore	file	"A set of tools that display the content of files - including audio, html, `Hypothes.is <https://hypothes.is/>`_ annotations, images, PDF, Markdown, text, video, tabular data, spreadsheets, GeoJSON, zip, and NcML files - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers"
 Data Curation Tool	configure	file	"A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions."
 Ask the Data	query	file	Ask the Data is an experimental tool that allows you ask natural language questions about the data contained in Dataverse tables (tabular data). See the README.md file at https://github.com/IQSS/askdataverse/tree/main/askthedata for the instructions on adding Ask the Data to your Dataverse installation. 

doc/sphinx-guides/source/_static/docsdataverse_org.css

@@ -182,3 +182,11 @@ div.form-group .glyphicon.glyphicon-asterisk {font-size: .5em; vertical-align: t
 pre {
     white-space: pre-wrap;
 }
+
+div.sphinx-tabs {
+    width: 100%;
+}
+
+li div.sphinx-tabs {
+    padding-left: 0;
+}

doc/sphinx-guides/source/admin/integrations.rst

@@ -224,6 +224,11 @@ Geodisy
 
 `Geodisy <https://researchdata.library.ubc.ca/find/geodisy>`_ will take your Dataverse installation’s data, search for geospatial metadata and files, and copy them to a new system that allows for visual searching. Your original data and search methods are untouched; you have the benefit of both. For more information, please refer to `Geodisy's GitHub Repository. <https://github.com/ubc-library/geodisy>`_
 
+DataONE
++++++++
+
+`DataONE <https://dataone.org/>`_ is a community driven program providing access to data across multiple `member repositories <https://www.dataone.org/network/>`_, supporting enhanced search and discovery of Earth and environmental data. Membership is free and is most easily achieved by providing schema.org data via `science-on-schema.org <https://science-on-schema.org>`_ metadata markup on dataset landing pages, support for which is native in Dataverse. Dataverse installations are welcome `join the network <https://www.dataone.org/jointhenetwork/>`_ to have their datasets included.
+
 Research Data Preservation
 --------------------------
 

doc/sphinx-guides/source/admin/metadatacustomization.rst

@@ -413,7 +413,7 @@ Setting Up a Dev Environment for Testing
 
 You have several options for setting up a dev environment for testing metadata block changes:
 
-- Docker: See :doc:`/container/index`.
+- Docker: See :doc:`/container/running/metadata-blocks` in the Container Guide.
 - AWS deployment: See the :doc:`/developers/deployment` section of the Developer Guide.
 - Full dev environment: See the :doc:`/developers/dev-environment` section of the Developer Guide.
 

doc/sphinx-guides/source/api/changelog.rst

@@ -10,8 +10,11 @@ This API changelog is experimental and we would love feedback on its usefulness.
 v6.2
 ----
 
+- The fields "northLongitude" and "southLongitude" have been deprecated in favor of "northLatitude" and "southLatitude" in the Geolocation metadata block. After upgrading to 6.2 or later, you will need to use the new fields when creating or updating a dataset.
+
 - **/api/datasets/{id}/versions/{versionId}**: The includeFiles parameter has been renamed to excludeFiles. The default behavior remains the same, which is to include files. However, when excludeFiles is set to true, the files will be excluded. A bug that caused the API to only return a deaccessioned dataset if the user had edit privileges has been fixed.
 - **/api/datasets/{id}/versions**: The includeFiles parameter has been renamed to excludeFiles. The default behavior remains the same, which is to include files. However, when excludeFiles is set to true, the files will be excluded.
+- **/api/files/$ID/uningest**: Can now be used by users with the ability to publish the dataset to undo a failed ingest. (Removing a successful ingest still requires being superuser)
 
 v6.1
 ----

doc/sphinx-guides/source/api/getting-started.rst

@@ -86,7 +86,7 @@ See :ref:`create-dataset-command`.
 Uploading Files
 ~~~~~~~~~~~~~~~
 
-See :ref:`add-file-api`.
+See :ref:`add-file-api`. In addition, when a Dataverse installation is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This facilitates an efficient method to upload big files, but is more complex. The procedure is described in the :doc:`/developers/s3-direct-upload-api` section of the Developer Guide.
 
 Publishing a Dataverse Collection
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~