add test for spectrum urls

.github/workflows/scheduled-tests.yml

@@ -0,0 +1,30 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Scheduled Checks
+
+on:
+  schedule:
+    - cron: '30 1 1 * *'
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.10'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest ads
+        pip install astrodbkit2
+
+    - name: Test with pytest
+      run: |
+        pytest -s tests/scheduled_tests.py

tests/scheduled_checks.py

@@ -0,0 +1,66 @@
+import os
+import pytest
+import sys
+import requests
+from astrodbkit2.astrodb import create_database, Database
+from scripts.ingests.utils import check_internet_connection
+
+sys.path.append(".")
+from simple.schema import *
+from . import REFERENCE_TABLES
+
+
+DB_NAME = "temp.db"
+DB_PATH = "data"
+
+
+# Load the database for use in individual tests
+@pytest.fixture(scope="module")
+def db():
+    # Create a fresh temporary database and assert it exists
+    # Because we've imported simple.schema, we will be using that schema for the database
+
+    if os.path.exists(DB_NAME):
+        os.remove(DB_NAME)
+    connection_string = "sqlite:///" + DB_NAME
+    create_database(connection_string)
+    assert os.path.exists(DB_NAME)
+
+    # Connect to the new database and confirm it has the Sources table
+    db = Database(connection_string, reference_tables=REFERENCE_TABLES)
+    assert db
+    assert "source" in [c.name for c in db.Sources.columns]
+
+    # Load data into an in-memory sqlite database first, for performance
+
+    # create and connects to a temporary in-memory database
+    temp_db = Database("sqlite://", reference_tables=REFERENCE_TABLES)
+
+    # load the data from the data files into the database
+    temp_db.load_database(DB_PATH, verbose=False)
+
+    # dump in-memory database to file
+    temp_db.dump_sqlite(DB_NAME)
+    # replace database object with new file version
+    db = Database("sqlite:///" + DB_NAME, reference_tables=REFERENCE_TABLES)
+
+    return db
+
+
+def test_spectra_urls(db):
+    spectra_urls = db.query(db.Spectra.c.spectrum).astropy()
+    broken_urls = []
+    codes = []
+    internet = check_internet_connection()
+    if internet:
+        for spectrum_url in spectra_urls["spectrum"]:
+            request_response = requests.head(spectrum_url)
+            status_code = request_response.status_code
+            # The website is up if the status code is 200
+            # cuny academic commons links give 301 status code
+            if status_code != 200 and status_code != 301:
+                broken_urls.append(spectrum_url)
+                codes.append(status_code)
+    assert (
+        len(broken_urls) == 149
+    ), f"found {len(broken_urls)} broken spectra urls: {broken_urls}, {codes}"

tests/test_data.py

@@ -3,13 +3,14 @@
 import os
 import pytest
 import sys
+from astrodbkit2.astrodb import create_database, Database
+from sqlalchemy import except_, select, and_
 
 sys.path.append('.')
 from simple.schema import *
-from astrodbkit2.astrodb import create_database, Database
-from sqlalchemy import except_, select, and_
 from . import REFERENCE_TABLES
 
+
 DB_NAME = 'temp.db'
 DB_PATH = 'data'
 
@@ -448,6 +449,7 @@ def test_spectral_types(db):
     n_spectral_types = db.query(db.SpectralTypes).count()
     assert len(m_dwarfs) + len(l_dwarfs) + len(t_dwarfs) + len(y_dwarfs) == n_spectral_types
 
+
 # Individual ingest tests
 # -----------------------------------------------------------------------------------------
 def test_Manj19_data(db):
@@ -533,7 +535,7 @@ def test_Kirk19_ingest(db):
 
     # Test spectral types added
 
-    # Test parallaxes 
+    # Test parallaxes
     ref = 'Kirk19'
     t = db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == ref).astropy()
     assert len(t) == 23, f'found {len(t)} parallax entries for {ref}'
@@ -597,6 +599,7 @@ def test_suar22_ingest(db):
     t = db.query(db.Spectra).filter(db.Spectra.c.reference == ref).astropy()
     assert len(t) == 112, f'found {len(t)} spectra entries for {ref}'
 
+
 def test_modeledparameters(db):
     # Test to verify existing counts of modeled parameters
     ref = 'Fili15'