Fix bard14 spectra (#443)

* new test to find duplicate spectra * spectra fixes
SIMPLE-AstroDB · Jan 2, 2024 · 7449241 · 7449241
1 parent b8b81dc
commit 7449241
Show file tree

Hide file tree

Showing 16 changed files with 63 additions and 46 deletions.
diff --git a/data/2mass_j04285096-2253227.json b/data/2mass_j04285096-2253227.json
@@ -275,8 +275,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J04285096-2253227_2012-09-25.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/0428-2253.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/u10286.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/u10286.fits",
             "regime": "optical",
             "telescope": "KPNO 4m",

diff --git a/data/2mass_j04510093-3402150.json b/data/2mass_j04510093-3402150.json
@@ -314,8 +314,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J04510093-3402150_2012-09-27.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/0451-3402.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/U10334.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/U10334.fits",
             "regime": "optical",
             "telescope": "CTIO 4m",

diff --git a/data/2mass_j05002100+0330501.json b/data/2mass_j05002100+0330501.json
@@ -232,8 +232,8 @@
     ],
     "Spectra": [
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J05002100%2b0330501_2012-12-27.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/0500%2B0330.txt",
+            "spectrum": "https://bdnyc.s3.amazonaws.com/xshooter/2M0500_total.fits",
+            "original_spectrum": null,
             "local_spectrum": null,
             "regime": "em.IR.NIR",
             "telescope": "ESO VLT",
@@ -245,8 +245,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J05002100%2b0330501_2012-12-27.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/0500%2B0330.txt",
+            "spectrum": "https://bdnyc.s3.amazonaws.com/xshooter/2M0500_total.fits",
+            "original_spectrum": null,
             "local_spectrum": null,
             "regime": "em.opt",
             "telescope": "ESO VLT",
@@ -271,8 +271,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J05002100%2b0330501_2012-12-27.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/0500%2B0330.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/u20197.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/u20197.fits",
             "regime": "optical",
             "telescope": "KPNO 4m",

diff --git a/data/2mass_j06411840-4322329.json b/data/2mass_j06411840-4322329.json
@@ -233,8 +233,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J06411840-4322329_2012-12-27.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/0641-4322.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/U20252.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/U20252.fits",
             "regime": "optical",
             "telescope": "CTIO 4m",

diff --git a/data/2mass_j06521977-2534505.json b/data/2mass_j06521977-2534505.json
@@ -274,8 +274,8 @@
     ],
     "Spectra": [
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J06521977-2534505_2009-01-25.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/U50078_JHK_2009jan25.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/SpeX/Prism/U50078_0652-2534_060902.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/SpeX/Prism/U50078_0652-2534_060902.fits",
             "regime": "nir",
             "telescope": "IRTF",
@@ -300,8 +300,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J06521977-2534505_2009-01-25.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/U50078_JHK_2009jan25.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/SpeX/Prism/U50078_0652-2534_091104.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/SpeX/Prism/U50078_0652-2534_091104.fits",
             "regime": "nir",
             "telescope": "IRTF",

diff --git a/data/2mass_j13204427+0409045.json b/data/2mass_j13204427+0409045.json
@@ -275,8 +275,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J13204427%2b0409045_2013-04-24.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/1320%2B0409.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/optical_spectra/U20475.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/U20475.fits",
             "regime": "optical",
             "telescope": "CTIO 4m",

diff --git a/data/2mass_j13595510-4034582.json b/data/2mass_j13595510-4034582.json
@@ -238,8 +238,8 @@
     ],
     "Spectra": [
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J13595510-4034582_2013-04-25.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/1359-4034_2013-04-25.txt",
+            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J13595510-4034582_2012-03-30.fits",
+            "original_spectrum": "https://s3.amazonaws.com/bdnyc/SpeX/Prism/1359-4034_2012-03-30.txt",
             "local_spectrum": "$BDNYC_spectra/SpeX/Prism/1359-4034_2012-03-30.txt",
             "regime": "nir",
             "telescope": "IRTF",
@@ -264,8 +264,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J13595510-4034582_2013-04-25.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/1359-4034_2013-04-25.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/U20543.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/U20543.fits",
             "regime": "optical",
             "telescope": "CTIO 4m",

diff --git a/data/2mass_j14283132+5923354.json b/data/2mass_j14283132+5923354.json
@@ -200,9 +200,9 @@
             "other_references": null
         },
         {
-            "spectrum": "https://s3.amazonaws.com/bdnyc/optical_spectra/U20752.fits",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/optical_spectra/U20570.fits",
             "original_spectrum": null,
-            "local_spectrum": "$BDNYC_spectra/optical_spectra/U20752.fits",
+            "local_spectrum": "$BDNYC_spectra/optical_spectra/U20570.fits",
             "regime": "optical",
             "telescope": "Gemini North",
             "instrument": "GMOS-N",

diff --git a/data/2mass_j14402293+1339230.json b/data/2mass_j14402293+1339230.json
@@ -293,8 +293,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J14402293%2b1339230_2013-03-27.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/1440%2B1339.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/U11230.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/U11230.fits",
             "regime": "optical",
             "telescope": "KPNO 2.1m",

diff --git a/data/2mass_j14563831-2809473.json b/data/2mass_j14563831-2809473.json
@@ -277,8 +277,8 @@
     ],
     "Spectra": [
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J14563831-2809473_2008-07-29.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/spex_prism_lhs3003_080729.txt",
+            "spectrum": "https://bdnyc.s3.amazonaws.com/IRS/2MASS+J14563831-2809473.fits",
+            "original_spectrum": "https://bdnyc.s3.amazonaws.com/IRS/1456-2809_IRS_spectrum.dat",
             "local_spectrum": null,
             "regime": "mir",
             "telescope": "Spitzer",
@@ -290,8 +290,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J14563831-2809473_2008-07-29.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/spex_prism_lhs3003_080729.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/LHS3003_SL.txt",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/IRS/IRS_DimSuns_S12/LHS3003_SL.txt",
             "regime": "mir",
             "telescope": "Spitzer",
@@ -316,8 +316,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J14563831-2809473_2008-07-29.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/spex_prism_lhs3003_080729.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/U11264.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/U11264.fits",
             "regime": "optical",
             "telescope": "KPNO 2.1m",

diff --git a/data/2massi_j0953212-101420.json b/data/2massi_j0953212-101420.json
@@ -229,8 +229,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASSI+J0953212-101420_2008-01-12.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/SpeX/Prism/U10857_JHK_2008jan12.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/optical_spectra/U10857_0953-1014.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/U10857_0953-1014.fits",
             "regime": "optical",
             "telescope": "Magellan II Clay",

diff --git a/data/lhs_2021.json b/data/lhs_2021.json
@@ -338,8 +338,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/LHS+2021_2012-12-27.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/0830%2B0947.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/u20302.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/u20302.fits",
             "regime": "optical",
             "telescope": "KPNO 4m",

diff --git a/data/lp__938-71.json b/data/lp__938-71.json
@@ -278,8 +278,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/LP++938-71_2008-09-07.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/LHS132.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/u20049.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/u20049.fits",
             "regime": "optical",
             "telescope": "CTIO 1.5m",

diff --git a/data/tvlm_831-161058.json b/data/tvlm_831-161058.json
@@ -284,8 +284,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/TVLM+831-161058_2013-09-03.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/0251%2B0047.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/u20130.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/u20130.fits",
             "regime": "optical",
             "telescope": "KPNO 4m",

diff --git a/data/tvlm_868-110639.json b/data/tvlm_868-110639.json
@@ -314,8 +314,8 @@
             "other_references": null
         },
         {
-            "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/TVLM+868-110639_2009-06-29.fits",
-            "original_spectrum": "https://s3.amazonaws.com/bdnyc/1510-0241.txt",
+            "spectrum": "https://s3.amazonaws.com/bdnyc/U20602.fits",
+            "original_spectrum": null,
             "local_spectrum": "$BDNYC_spectra/optical_spectra/U20602.fits",
             "regime": "optical",
             "telescope": "KPNO 2.1m",

diff --git a/tests/test_integrity.py b/tests/test_integrity.py
@@ -1,8 +1,5 @@
 # Test to verify database integrity
-
 import os
-from operator import and_
-
 import pytest
 from . import REFERENCE_TABLES
 from sqlalchemy import func, and_  # , select, except_
@@ -672,6 +669,26 @@ def test_spectra(db):
         print(t)
     assert len(t) == 0
 
+    # All spectra should have a unique filename
+    sql_text = (
+        "SELECT Spectra.spectrum, Spectra.source "
+        "FROM Spectra "
+        "GROUP BY spectrum "
+        "HAVING (Count(*) > 1)"
+    )
+    duplicate_spectra = db.sql_query(sql_text, fmt="astropy")
+
+    # if duplicate spectra is non_zero, print out duplicate names
+    if len(duplicate_spectra) > 0:
+        print(f"\n{len(duplicate_spectra)} duplicated spectra")
+        print(duplicate_spectra)
+        print(duplicate_spectra["source"])
+
+    assert len(duplicate_spectra) == 22
+    # 21 are xshooter spectra which correctly have two entires
+    # 1 (W1542%2B22.csv) is an incorrect duplicate and the topic of
+    # https://github.com/SIMPLE-AstroDB/SIMPLE-db/issues/442
+
 
 def test_special_characters(db):
     # This test asserts that no special unicode characters are in the database
@@ -683,7 +700,7 @@ def test_special_characters(db):
         "\u00ed",
         "\u00e1",
         "\u00fa",
-        "\u0000"
+        "\u0000",
     ]
     for char in bad_characters:
         data = db.search_string(char)