Merge branch 'main' into fix-irs-urls

.github/workflows/scheduled-tests.yml

@@ -0,0 +1,32 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Scheduled Checks
+
+on:
+  schedule:
+    - cron: '30 1 1 * *'
+  workflow_dispatch:  # manual execution
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.10'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest ads
+        pip install astrodbkit2
+
+    - name: Test with pytest
+      run: |
+        pytest -s tests/scheduled_checks.py

tests/scheduled_checks.py

@@ -0,0 +1,66 @@
+import os
+import pytest
+import sys
+import requests
+from tqdm import tqdm
+from astrodbkit2.astrodb import create_database, Database
+from scripts.ingests.utils import check_internet_connection
+
+sys.path.append(".")
+from simple.schema import *
+from . import REFERENCE_TABLES
+
+
+DB_NAME = "temp.db"
+DB_PATH = "data"
+
+
+# Load the database for use in individual tests
+@pytest.fixture(scope="module")
+def db():
+    # Create a fresh temporary database and assert it exists
+    # Because we've imported simple.schema, we will be using that schema for the database
+
+    if os.path.exists(DB_NAME):
+        os.remove(DB_NAME)
+    connection_string = "sqlite:///" + DB_NAME
+    create_database(connection_string)
+    assert os.path.exists(DB_NAME)
+
+    # Connect to the new database and confirm it has the Sources table
+    db = Database(connection_string, reference_tables=REFERENCE_TABLES)
+    assert db
+    assert "source" in [c.name for c in db.Sources.columns]
+
+    # Load data into an in-memory sqlite database first, for performance
+
+    # create and connects to a temporary in-memory database
+    temp_db = Database("sqlite://", reference_tables=REFERENCE_TABLES)
+
+    # load the data from the data files into the database
+    temp_db.load_database(DB_PATH, verbose=False)
+
+    # dump in-memory database to file
+    temp_db.dump_sqlite(DB_NAME)
+    # replace database object with new file version
+    db = Database("sqlite:///" + DB_NAME, reference_tables=REFERENCE_TABLES)
+
+    return db
+
+
+def test_spectra_urls(db):
+    spectra_urls = db.query(db.Spectra.c.spectrum).astropy()
+    broken_urls = []
+    codes = []
+    internet = check_internet_connection()
+    if internet:
+        for spectrum_url in tqdm(spectra_urls["spectrum"]):
+            request_response = requests.head(spectrum_url)
+            status_code = request_response.status_code
+            # The website is up if the status code is 200
+            # cuny academic commons links give 301 status code
+            if status_code != 200 and status_code != 301:
+                broken_urls.append(spectrum_url)
+                codes.append(status_code)
+    assert (149 <= len(broken_urls) <= 150
+    ), f"found {len(broken_urls)} broken spectra urls: {broken_urls}, {codes}"

tests/test_data.py

@@ -3,13 +3,14 @@
 import os
 import pytest
 import sys
+from astrodbkit2.astrodb import create_database, Database
+from sqlalchemy import except_, select, and_
 
 sys.path.append('.')
 from simple.schema import *
-from astrodbkit2.astrodb import create_database, Database
-from sqlalchemy import except_, select, and_
 from . import REFERENCE_TABLES
 
+
 DB_NAME = 'temp.db'
 DB_PATH = 'data'
 
@@ -205,84 +206,35 @@ def test_parallax_refs(db):
     assert len(t) == 1104, f'found {len(t)} adopted parallax reference entries for {ref}'
 
 
-def test_photometry_bands(db):
+@pytest.mark.parametrize('band, value', [
+    ('GAIA2.G', 1266),
+    ('GAIA2.Grp', 1106),
+    ('GAIA3.G', 1256),
+    ('GAIA3.Grp', 1261),
+    ('WISE.W1', 460),
+    ('WISE.W2', 460),
+    ('WISE.W3', 457),
+    ('WISE.W4', 450),
+    ('2MASS.J', 1802),
+    ('2MASS.H', 1791),
+    ('2MASS.Ks', 1762),
+    ('GPI.Y', 1),
+    ('NIRI.Y', 21),
+    ('UFTI.Y', 13),
+    ('Wircam.Y', 29),
+    ('WFCAM.Y', 854),
+    ('VisAO.Ys', 1),
+    ('VISTA.Y', 59),
+])
+def test_photometry_bands(db, band, value):
     # To refresh these counts:
     # from sqlalchemy import func
     # db.query(db.Photometry.c.band, func.count(db.Photometry.c.band).label('count')).\
     #     group_by(db.Photometry.c.band).\
     #     astropy()
 
-    band = 'GAIA2.G'
     t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 1266, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'GAIA2.Grp'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 1106, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'GAIA3.G'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 1256, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'GAIA3.Grp'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 1261, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'WISE.W1'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 460, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'WISE.W2'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 460, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'WISE.W3'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 457, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'WISE.W4'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 450, f'found {len(t)} photometry measurements for {band}'
-
-    band = '2MASS.J'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 1802, f'found {len(t)} photometry measurements for {band}'
-
-    band = '2MASS.H'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 1791, f'found {len(t)} photometry measurements for {band}'
-
-    band = '2MASS.Ks'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 1762, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'GPI.Y'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 1, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'NIRI.Y'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 21, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'UFTI.Y'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 13, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'Wircam.Y'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 29, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'WFCAM.Y'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 854, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'VisAO.Ys'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 1, f'found {len(t)} photometry measurements for {band}'
-
-    band = 'VISTA.Y'
-    t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy()
-    assert len(t) == 59, f'found {len(t)} photometry measurements for {band}'
+    assert len(t) == value, f'found {len(t)} photometry measurements for {band}'
 
 
 def test_missions(db):
@@ -448,6 +400,7 @@ def test_spectral_types(db):
     n_spectral_types = db.query(db.SpectralTypes).count()
     assert len(m_dwarfs) + len(l_dwarfs) + len(t_dwarfs) + len(y_dwarfs) == n_spectral_types
 
+
 # Individual ingest tests
 # -----------------------------------------------------------------------------------------
 def test_Manj19_data(db):
@@ -533,7 +486,7 @@ def test_Kirk19_ingest(db):
 
     # Test spectral types added
 
-    # Test parallaxes 
+    # Test parallaxes
     ref = 'Kirk19'
     t = db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == ref).astropy()
     assert len(t) == 23, f'found {len(t)} parallax entries for {ref}'
@@ -597,6 +550,7 @@ def test_suar22_ingest(db):
     t = db.query(db.Spectra).filter(db.Spectra.c.reference == ref).astropy()
     assert len(t) == 112, f'found {len(t)} spectra entries for {ref}'
 
+
 def test_modeledparameters(db):
     # Test to verify existing counts of modeled parameters
     ref = 'Fili15'