From 7449241e6c19a4015b3fbd12cea5fad06b378ec1 Mon Sep 17 00:00:00 2001 From: Kelle Cruz Date: Tue, 2 Jan 2024 13:22:29 -0500 Subject: [PATCH] Fix bard14 spectra (#443) * new test to find duplicate spectra * spectra fixes --- data/2mass_j04285096-2253227.json | 4 ++-- data/2mass_j04510093-3402150.json | 4 ++-- data/2mass_j05002100+0330501.json | 12 ++++++------ data/2mass_j06411840-4322329.json | 4 ++-- data/2mass_j06521977-2534505.json | 8 ++++---- data/2mass_j13204427+0409045.json | 4 ++-- data/2mass_j13595510-4034582.json | 8 ++++---- data/2mass_j14283132+5923354.json | 4 ++-- data/2mass_j14402293+1339230.json | 4 ++-- data/2mass_j14563831-2809473.json | 12 ++++++------ data/2massi_j0953212-101420.json | 4 ++-- data/lhs_2021.json | 4 ++-- data/lp__938-71.json | 4 ++-- data/tvlm_831-161058.json | 4 ++-- data/tvlm_868-110639.json | 4 ++-- tests/test_integrity.py | 25 +++++++++++++++++++++---- 16 files changed, 63 insertions(+), 46 deletions(-) diff --git a/data/2mass_j04285096-2253227.json b/data/2mass_j04285096-2253227.json index 32bbe548f..c8a888586 100644 --- a/data/2mass_j04285096-2253227.json +++ b/data/2mass_j04285096-2253227.json @@ -275,8 +275,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J04285096-2253227_2012-09-25.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/0428-2253.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/u10286.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/u10286.fits", "regime": "optical", "telescope": "KPNO 4m", diff --git a/data/2mass_j04510093-3402150.json b/data/2mass_j04510093-3402150.json index 86ac0d413..5ac0003b6 100644 --- a/data/2mass_j04510093-3402150.json +++ b/data/2mass_j04510093-3402150.json @@ -314,8 +314,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J04510093-3402150_2012-09-27.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/0451-3402.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/U10334.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/U10334.fits", "regime": "optical", "telescope": "CTIO 4m", diff --git a/data/2mass_j05002100+0330501.json b/data/2mass_j05002100+0330501.json index 940caf095..9a9e96195 100644 --- a/data/2mass_j05002100+0330501.json +++ b/data/2mass_j05002100+0330501.json @@ -232,8 +232,8 @@ ], "Spectra": [ { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J05002100%2b0330501_2012-12-27.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/0500%2B0330.txt", + "spectrum": "https://bdnyc.s3.amazonaws.com/xshooter/2M0500_total.fits", + "original_spectrum": null, "local_spectrum": null, "regime": "em.IR.NIR", "telescope": "ESO VLT", @@ -245,8 +245,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J05002100%2b0330501_2012-12-27.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/0500%2B0330.txt", + "spectrum": "https://bdnyc.s3.amazonaws.com/xshooter/2M0500_total.fits", + "original_spectrum": null, "local_spectrum": null, "regime": "em.opt", "telescope": "ESO VLT", @@ -271,8 +271,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J05002100%2b0330501_2012-12-27.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/0500%2B0330.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/u20197.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/u20197.fits", "regime": "optical", "telescope": "KPNO 4m", diff --git a/data/2mass_j06411840-4322329.json b/data/2mass_j06411840-4322329.json index 458dbb39a..ee9be49ff 100644 --- a/data/2mass_j06411840-4322329.json +++ b/data/2mass_j06411840-4322329.json @@ -233,8 +233,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J06411840-4322329_2012-12-27.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/0641-4322.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/U20252.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/U20252.fits", "regime": "optical", "telescope": "CTIO 4m", diff --git a/data/2mass_j06521977-2534505.json b/data/2mass_j06521977-2534505.json index 7ca808a1e..98dfdc57a 100644 --- a/data/2mass_j06521977-2534505.json +++ b/data/2mass_j06521977-2534505.json @@ -274,8 +274,8 @@ ], "Spectra": [ { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J06521977-2534505_2009-01-25.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/U50078_JHK_2009jan25.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/SpeX/Prism/U50078_0652-2534_060902.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/SpeX/Prism/U50078_0652-2534_060902.fits", "regime": "nir", "telescope": "IRTF", @@ -300,8 +300,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J06521977-2534505_2009-01-25.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/U50078_JHK_2009jan25.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/SpeX/Prism/U50078_0652-2534_091104.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/SpeX/Prism/U50078_0652-2534_091104.fits", "regime": "nir", "telescope": "IRTF", diff --git a/data/2mass_j13204427+0409045.json b/data/2mass_j13204427+0409045.json index 1be178485..4b1907802 100644 --- a/data/2mass_j13204427+0409045.json +++ b/data/2mass_j13204427+0409045.json @@ -275,8 +275,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J13204427%2b0409045_2013-04-24.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/1320%2B0409.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/optical_spectra/U20475.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/U20475.fits", "regime": "optical", "telescope": "CTIO 4m", diff --git a/data/2mass_j13595510-4034582.json b/data/2mass_j13595510-4034582.json index 41b2a7ff1..933f76426 100644 --- a/data/2mass_j13595510-4034582.json +++ b/data/2mass_j13595510-4034582.json @@ -238,8 +238,8 @@ ], "Spectra": [ { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J13595510-4034582_2013-04-25.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/1359-4034_2013-04-25.txt", + "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J13595510-4034582_2012-03-30.fits", + "original_spectrum": "https://s3.amazonaws.com/bdnyc/SpeX/Prism/1359-4034_2012-03-30.txt", "local_spectrum": "$BDNYC_spectra/SpeX/Prism/1359-4034_2012-03-30.txt", "regime": "nir", "telescope": "IRTF", @@ -264,8 +264,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J13595510-4034582_2013-04-25.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/1359-4034_2013-04-25.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/U20543.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/U20543.fits", "regime": "optical", "telescope": "CTIO 4m", diff --git a/data/2mass_j14283132+5923354.json b/data/2mass_j14283132+5923354.json index 6d667792f..146faff95 100644 --- a/data/2mass_j14283132+5923354.json +++ b/data/2mass_j14283132+5923354.json @@ -200,9 +200,9 @@ "other_references": null }, { - "spectrum": "https://s3.amazonaws.com/bdnyc/optical_spectra/U20752.fits", + "spectrum": "https://s3.amazonaws.com/bdnyc/optical_spectra/U20570.fits", "original_spectrum": null, - "local_spectrum": "$BDNYC_spectra/optical_spectra/U20752.fits", + "local_spectrum": "$BDNYC_spectra/optical_spectra/U20570.fits", "regime": "optical", "telescope": "Gemini North", "instrument": "GMOS-N", diff --git a/data/2mass_j14402293+1339230.json b/data/2mass_j14402293+1339230.json index fd9270050..66344a02c 100644 --- a/data/2mass_j14402293+1339230.json +++ b/data/2mass_j14402293+1339230.json @@ -293,8 +293,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J14402293%2b1339230_2013-03-27.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/1440%2B1339.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/U11230.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/U11230.fits", "regime": "optical", "telescope": "KPNO 2.1m", diff --git a/data/2mass_j14563831-2809473.json b/data/2mass_j14563831-2809473.json index 5f471abf2..693f5da2d 100644 --- a/data/2mass_j14563831-2809473.json +++ b/data/2mass_j14563831-2809473.json @@ -277,8 +277,8 @@ ], "Spectra": [ { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J14563831-2809473_2008-07-29.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/spex_prism_lhs3003_080729.txt", + "spectrum": "https://bdnyc.s3.amazonaws.com/IRS/2MASS+J14563831-2809473.fits", + "original_spectrum": "https://bdnyc.s3.amazonaws.com/IRS/1456-2809_IRS_spectrum.dat", "local_spectrum": null, "regime": "mir", "telescope": "Spitzer", @@ -290,8 +290,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J14563831-2809473_2008-07-29.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/spex_prism_lhs3003_080729.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/LHS3003_SL.txt", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/IRS/IRS_DimSuns_S12/LHS3003_SL.txt", "regime": "mir", "telescope": "Spitzer", @@ -316,8 +316,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASS+J14563831-2809473_2008-07-29.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/spex_prism_lhs3003_080729.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/U11264.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/U11264.fits", "regime": "optical", "telescope": "KPNO 2.1m", diff --git a/data/2massi_j0953212-101420.json b/data/2massi_j0953212-101420.json index 585c93b40..05ee6064f 100644 --- a/data/2massi_j0953212-101420.json +++ b/data/2massi_j0953212-101420.json @@ -229,8 +229,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/2MASSI+J0953212-101420_2008-01-12.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/SpeX/Prism/U10857_JHK_2008jan12.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/optical_spectra/U10857_0953-1014.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/U10857_0953-1014.fits", "regime": "optical", "telescope": "Magellan II Clay", diff --git a/data/lhs_2021.json b/data/lhs_2021.json index e1a6778f4..b9f42dff3 100644 --- a/data/lhs_2021.json +++ b/data/lhs_2021.json @@ -338,8 +338,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/LHS+2021_2012-12-27.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/0830%2B0947.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/u20302.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/u20302.fits", "regime": "optical", "telescope": "KPNO 4m", diff --git a/data/lp__938-71.json b/data/lp__938-71.json index ff06d5d1b..1aa12724a 100644 --- a/data/lp__938-71.json +++ b/data/lp__938-71.json @@ -278,8 +278,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/LP++938-71_2008-09-07.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/LHS132.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/u20049.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/u20049.fits", "regime": "optical", "telescope": "CTIO 1.5m", diff --git a/data/tvlm_831-161058.json b/data/tvlm_831-161058.json index 61e443cf7..26dc6f41b 100644 --- a/data/tvlm_831-161058.json +++ b/data/tvlm_831-161058.json @@ -284,8 +284,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/TVLM+831-161058_2013-09-03.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/0251%2B0047.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/u20130.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/u20130.fits", "regime": "optical", "telescope": "KPNO 4m", diff --git a/data/tvlm_868-110639.json b/data/tvlm_868-110639.json index 554df3a8d..c45ac2a10 100644 --- a/data/tvlm_868-110639.json +++ b/data/tvlm_868-110639.json @@ -314,8 +314,8 @@ "other_references": null }, { - "spectrum": "https://bdnyc.s3.amazonaws.com/SpeX/Prism/TVLM+868-110639_2009-06-29.fits", - "original_spectrum": "https://s3.amazonaws.com/bdnyc/1510-0241.txt", + "spectrum": "https://s3.amazonaws.com/bdnyc/U20602.fits", + "original_spectrum": null, "local_spectrum": "$BDNYC_spectra/optical_spectra/U20602.fits", "regime": "optical", "telescope": "KPNO 2.1m", diff --git a/tests/test_integrity.py b/tests/test_integrity.py index 53ac7db76..2ebf7154d 100644 --- a/tests/test_integrity.py +++ b/tests/test_integrity.py @@ -1,8 +1,5 @@ # Test to verify database integrity - import os -from operator import and_ - import pytest from . import REFERENCE_TABLES from sqlalchemy import func, and_ # , select, except_ @@ -672,6 +669,26 @@ def test_spectra(db): print(t) assert len(t) == 0 + # All spectra should have a unique filename + sql_text = ( + "SELECT Spectra.spectrum, Spectra.source " + "FROM Spectra " + "GROUP BY spectrum " + "HAVING (Count(*) > 1)" + ) + duplicate_spectra = db.sql_query(sql_text, fmt="astropy") + + # if duplicate spectra is non_zero, print out duplicate names + if len(duplicate_spectra) > 0: + print(f"\n{len(duplicate_spectra)} duplicated spectra") + print(duplicate_spectra) + print(duplicate_spectra["source"]) + + assert len(duplicate_spectra) == 22 + # 21 are xshooter spectra which correctly have two entires + # 1 (W1542%2B22.csv) is an incorrect duplicate and the topic of + # https://github.com/SIMPLE-AstroDB/SIMPLE-db/issues/442 + def test_special_characters(db): # This test asserts that no special unicode characters are in the database @@ -683,7 +700,7 @@ def test_special_characters(db): "\u00ed", "\u00e1", "\u00fa", - "\u0000" + "\u0000", ] for char in bad_characters: data = db.search_string(char)