From aae043c31fbfd643c92969759c22acc041cbebbd Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 6 Oct 2023 17:34:50 -0400 Subject: [PATCH 1/8] add missing db constriants, add cvoc transaction and try/catch --- .../iq/dataverse/DatasetFieldServiceBean.java | 7 ++++++- ...V6.0.0.2__9983-missing-unique-constraints.sql | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 620d4bf3e09..4edc2160979 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -19,6 +19,8 @@ import jakarta.ejb.EJB; import jakarta.ejb.Stateless; +import jakarta.ejb.TransactionAttribute; +import jakarta.ejb.TransactionAttributeType; import jakarta.inject.Named; import jakarta.json.Json; import jakarta.json.JsonArray; @@ -34,6 +36,7 @@ import jakarta.persistence.NoResultException; import jakarta.persistence.NonUniqueResultException; import jakarta.persistence.PersistenceContext; +import jakarta.persistence.PersistenceException; import jakarta.persistence.TypedQuery; import org.apache.commons.codec.digest.DigestUtils; @@ -46,7 +49,6 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; - import edu.harvard.iq.dataverse.settings.SettingsServiceBean; /** @@ -448,6 +450,7 @@ public JsonObject getExternalVocabularyValue(String termUri) { * @param cvocEntry - the configuration for the DatasetFieldType associated with this term * @param term - the term uri as a string */ + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public void registerExternalTerm(JsonObject cvocEntry, String term) { String retrievalUri = cvocEntry.getString("retrieval-uri"); String prefix = cvocEntry.getString("prefix", null); @@ -517,6 +520,8 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep logger.fine("Wrote value for term: " + term); } catch (JsonException je) { logger.severe("Error retrieving: " + retrievalUri + " : " + je.getMessage()); + } catch (PersistenceException e) { + logger.fine("Problem persisting: " + retrievalUri + " : " + e.getMessage()); } } else { logger.severe("Received response code : " + statusCode + " when retrieving " + retrievalUri diff --git a/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql b/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql new file mode 100644 index 00000000000..6cb3a455e4e --- /dev/null +++ b/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql @@ -0,0 +1,16 @@ +DO $$ +BEGIN + + BEGIN + ALTER TABLE externalvocabularyvalue ADD CONSTRAINT externalvocabularvalue_uri_key UNIQUE(uri); + EXCEPTION + WHEN duplicate_table THEN RAISE NOTICE 'Table unique constraint externalvocabularvalue_uri_key already exists'; + END; + + BEGIN + ALTER TABLE oaiset ADD CONSTRAINT oaiset_spec_key UNIQUE(spec); + EXCEPTION + WHEN duplicate_table THEN RAISE NOTICE 'Table unique constraint oaiset_spec_key already exists'; + END; + +END $$; \ No newline at end of file From e29b2b57254a04c631de639a1458a0ffd36ff9a8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 6 Oct 2023 17:46:12 -0400 Subject: [PATCH 2/8] add truncate for vocab table --- .../db/migration/V6.0.0.2__9983-missing-unique-constraints.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql b/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql index 6cb3a455e4e..d867dcde90d 100644 --- a/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql +++ b/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql @@ -2,6 +2,7 @@ DO $$ BEGIN BEGIN + TRUNCATE externalvocabularyvalue ALTER TABLE externalvocabularyvalue ADD CONSTRAINT externalvocabularvalue_uri_key UNIQUE(uri); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Table unique constraint externalvocabularvalue_uri_key already exists'; From 39a2cf5aabd52475bc89d30093c817f05d3d71d6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 6 Oct 2023 17:59:17 -0400 Subject: [PATCH 3/8] add release note, remote truncate --- doc/release-notes/9983-unique-constraints.md | 9 +++++++++ .../V6.0.0.2__9983-missing-unique-constraints.sql | 1 - 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 doc/release-notes/9983-unique-constraints.md diff --git a/doc/release-notes/9983-unique-constraints.md b/doc/release-notes/9983-unique-constraints.md new file mode 100644 index 00000000000..bb3ed200c62 --- /dev/null +++ b/doc/release-notes/9983-unique-constraints.md @@ -0,0 +1,9 @@ +This release adds two missing database constraints that will assure that the externalvocabularyvalue table only has one entry for each uri and that the oaiset table only has one set for each spec. (In the very unlikely case that your existing database has duplicate entries now, install would fail. This can be checked by running + +SELECT uri, count(*) FROM externalvocabularyvaluet group by uri; + +and + +SELECT spec, count(*) FROM oaiset group by spec; + +and then removing any duplicate rows (where count>1). diff --git a/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql b/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql index d867dcde90d..6cb3a455e4e 100644 --- a/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql +++ b/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql @@ -2,7 +2,6 @@ DO $$ BEGIN BEGIN - TRUNCATE externalvocabularyvalue ALTER TABLE externalvocabularyvalue ADD CONSTRAINT externalvocabularvalue_uri_key UNIQUE(uri); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Table unique constraint externalvocabularvalue_uri_key already exists'; From 1c7bc24b487625c805b666bfc1e218236808d33f Mon Sep 17 00:00:00 2001 From: Jan van Mansum Date: Tue, 10 Oct 2023 16:24:17 +0200 Subject: [PATCH 4/8] Changed DatasetFieldType.isControlledVocabulary() to return allowControlledVocabulary instead of looking up complete list of terms to see if it is empty --- src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java index 824b486a42d..01785359e0e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java @@ -284,7 +284,7 @@ public void setDisplayOnCreate(boolean displayOnCreate) { } public boolean isControlledVocabulary() { - return controlledVocabularyValues != null && !controlledVocabularyValues.isEmpty(); + return allowControlledVocabulary; } /** From 43fcf1dfcfaf867d6bb2ef5b6c6c4c9554123936 Mon Sep 17 00:00:00 2001 From: Jan van Mansum Date: Wed, 8 Nov 2023 15:21:06 +0100 Subject: [PATCH 5/8] Amend metadata block TSVs --- doc/release-notes/9983-unique-constraints.md | 5 +++++ scripts/api/data/metadatablocks/astrophysics.tsv | 6 +++--- scripts/api/data/metadatablocks/biomedical.tsv | 2 +- scripts/api/data/metadatablocks/citation.tsv | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/release-notes/9983-unique-constraints.md b/doc/release-notes/9983-unique-constraints.md index bb3ed200c62..1e37d75d88d 100644 --- a/doc/release-notes/9983-unique-constraints.md +++ b/doc/release-notes/9983-unique-constraints.md @@ -7,3 +7,8 @@ and SELECT spec, count(*) FROM oaiset group by spec; and then removing any duplicate rows (where count>1). + + + + +TODO: Add note about reloading metadata blocks after upgrade. \ No newline at end of file diff --git a/scripts/api/data/metadatablocks/astrophysics.tsv b/scripts/api/data/metadatablocks/astrophysics.tsv index 4039d32cb75..92792d404c9 100644 --- a/scripts/api/data/metadatablocks/astrophysics.tsv +++ b/scripts/api/data/metadatablocks/astrophysics.tsv @@ -2,13 +2,13 @@ astrophysics Astronomy and Astrophysics Metadata #datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id astroType Type The nature or genre of the content of the files in the dataset. text 0 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics - astroFacility Facility The observatory or facility where the data was obtained. text 1 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics - astroInstrument Instrument The instrument used to collect the data. text 2 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics + astroFacility Facility The observatory or facility where the data was obtained. text 1 TRUE FALSE TRUE TRUE FALSE FALSE astrophysics + astroInstrument Instrument The instrument used to collect the data. text 2 TRUE FALSE TRUE TRUE FALSE FALSE astrophysics astroObject Object Astronomical Objects represented in the data (Given as SIMBAD recognizable names preferred). text 3 TRUE FALSE TRUE TRUE FALSE FALSE astrophysics resolution.Spatial Spatial Resolution The spatial (angular) resolution that is typical of the observations, in decimal degrees. text 4 TRUE FALSE FALSE TRUE FALSE FALSE astrophysics resolution.Spectral Spectral Resolution The spectral resolution that is typical of the observations, given as the ratio \u03bb/\u0394\u03bb. text 5 TRUE FALSE FALSE TRUE FALSE FALSE astrophysics resolution.Temporal Time Resolution The temporal resolution that is typical of the observations, given in seconds. text 6 FALSE FALSE FALSE FALSE FALSE FALSE astrophysics - coverage.Spectral.Bandpass Bandpass Conventional bandpass name text 7 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics + coverage.Spectral.Bandpass Bandpass Conventional bandpass name text 7 TRUE FALSE TRUE TRUE FALSE FALSE astrophysics coverage.Spectral.CentralWavelength Central Wavelength (m) The central wavelength of the spectral bandpass, in meters. Enter a floating-point number. float 8 TRUE FALSE TRUE TRUE FALSE FALSE astrophysics coverage.Spectral.Wavelength Wavelength Range The minimum and maximum wavelength of the spectral bandpass. Enter a floating-point number. none 9 FALSE FALSE TRUE FALSE FALSE FALSE astrophysics coverage.Spectral.MinimumWavelength Minimum (m) The minimum wavelength of the spectral bandpass, in meters. Enter a floating-point number. float 10 TRUE FALSE FALSE TRUE FALSE FALSE coverage.Spectral.Wavelength astrophysics diff --git a/scripts/api/data/metadatablocks/biomedical.tsv b/scripts/api/data/metadatablocks/biomedical.tsv index 28d59130c34..d70f754336a 100644 --- a/scripts/api/data/metadatablocks/biomedical.tsv +++ b/scripts/api/data/metadatablocks/biomedical.tsv @@ -13,7 +13,7 @@ studyAssayOtherTechnologyType Other Technology Type If Other was selected in Technology Type, list any other technology types that were used in this Dataset. text 9 TRUE FALSE TRUE TRUE FALSE FALSE biomedical studyAssayPlatform Technology Platform The manufacturer and name of the technology platform used in the assay (e.g. Bruker AVANCE). text 10 TRUE TRUE TRUE TRUE FALSE FALSE biomedical studyAssayOtherPlatform Other Technology Platform If Other was selected in Technology Platform, list any other technology platforms that were used in this Dataset. text 11 TRUE FALSE TRUE TRUE FALSE FALSE biomedical - studyAssayCellType Cell Type The name of the cell line from which the source or sample derives. text 12 TRUE TRUE TRUE TRUE FALSE FALSE biomedical + studyAssayCellType Cell Type The name of the cell line from which the source or sample derives. text 12 TRUE FALSE TRUE TRUE FALSE FALSE biomedical #controlledVocabulary DatasetField Value identifier displayOrder studyDesignType Case Control EFO_0001427 0 studyDesignType Cross Sectional EFO_0001428 1 diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index b21b6bcce57..c5af05927dc 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -70,7 +70,7 @@ seriesName Name The name of the dataset series text 66 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE series citation seriesInformation Information Can include 1) a history of the series and 2) a summary of features that apply to the series textbox 67 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE series citation software Software Information about the software used to generate the Dataset none 68 , FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasGeneratedBy - softwareName Name The name of software used to generate the Dataset text 69 #VALUE FALSE TRUE FALSE FALSE FALSE FALSE software citation + softwareName Name The name of software used to generate the Dataset text 69 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation softwareVersion Version The version of the software used to generate the Dataset, e.g. 4.11 text 70 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation relatedMaterial Related Material Information, such as a persistent ID or citation, about the material related to the Dataset, such as appendices or sampling information available outside of the Dataset textbox 71 FALSE FALSE TRUE FALSE FALSE FALSE citation relatedDatasets Related Dataset Information, such as a persistent ID or citation, about a related dataset, such as previous research on the Dataset's subject textbox 72 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/relation From 93de747a3f7c31cbbacd9e4d3895c00f44c0b522 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Jan 2024 16:49:31 -0500 Subject: [PATCH 6/8] Updating flyway name --- ...straints.sql => V6.1.0.4__9983-missing-unique-constraints.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V6.0.0.2__9983-missing-unique-constraints.sql => V6.1.0.4__9983-missing-unique-constraints.sql} (100%) diff --git a/src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql b/src/main/resources/db/migration/V6.1.0.4__9983-missing-unique-constraints.sql similarity index 100% rename from src/main/resources/db/migration/V6.0.0.2__9983-missing-unique-constraints.sql rename to src/main/resources/db/migration/V6.1.0.4__9983-missing-unique-constraints.sql From f612f7a0e3dce2e7e4cb0a5664986adcd2868667 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Feb 2024 13:25:04 -0500 Subject: [PATCH 7/8] change flyway numbering --- ...straints.sql => V6.1.0.3__9983-missing-unique-constraints.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V6.1.0.4__9983-missing-unique-constraints.sql => V6.1.0.3__9983-missing-unique-constraints.sql} (100%) diff --git a/src/main/resources/db/migration/V6.1.0.4__9983-missing-unique-constraints.sql b/src/main/resources/db/migration/V6.1.0.3__9983-missing-unique-constraints.sql similarity index 100% rename from src/main/resources/db/migration/V6.1.0.4__9983-missing-unique-constraints.sql rename to src/main/resources/db/migration/V6.1.0.3__9983-missing-unique-constraints.sql From f3fae4bf754572986fb815324a3a69a2632a87cc Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Feb 2024 14:33:58 -0500 Subject: [PATCH 8/8] add clarity --- doc/release-notes/9983-unique-constraints.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/9983-unique-constraints.md b/doc/release-notes/9983-unique-constraints.md index 1e37d75d88d..d889beb0718 100644 --- a/doc/release-notes/9983-unique-constraints.md +++ b/doc/release-notes/9983-unique-constraints.md @@ -11,4 +11,4 @@ and then removing any duplicate rows (where count>1). -TODO: Add note about reloading metadata blocks after upgrade. \ No newline at end of file +TODO: Whoever puts the release notes together should make sure there is the standard note about reloading metadata blocks for the citation, astrophysics, and biomedical blocks (plus any others from other PRs) after upgrading. \ No newline at end of file