diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml new file mode 100644 index 000000000..db51bb06e --- /dev/null +++ b/.github/workflows/scheduled-tests.yml @@ -0,0 +1,30 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Scheduled Checks + +on: + schedule: + - cron: '30 1 1 * *' +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest ads + pip install astrodbkit2 + + - name: Test with pytest + run: | + pytest -s tests/scheduled_tests.py diff --git a/tests/scheduled_checks.py b/tests/scheduled_checks.py new file mode 100644 index 000000000..2e2927889 --- /dev/null +++ b/tests/scheduled_checks.py @@ -0,0 +1,66 @@ +import os +import pytest +import sys +import requests +from astrodbkit2.astrodb import create_database, Database +from scripts.ingests.utils import check_internet_connection + +sys.path.append(".") +from simple.schema import * +from . import REFERENCE_TABLES + + +DB_NAME = "temp.db" +DB_PATH = "data" + + +# Load the database for use in individual tests +@pytest.fixture(scope="module") +def db(): + # Create a fresh temporary database and assert it exists + # Because we've imported simple.schema, we will be using that schema for the database + + if os.path.exists(DB_NAME): + os.remove(DB_NAME) + connection_string = "sqlite:///" + DB_NAME + create_database(connection_string) + assert os.path.exists(DB_NAME) + + # Connect to the new database and confirm it has the Sources table + db = Database(connection_string, reference_tables=REFERENCE_TABLES) + assert db + assert "source" in [c.name for c in db.Sources.columns] + + # Load data into an in-memory sqlite database first, for performance + + # create and connects to a temporary in-memory database + temp_db = Database("sqlite://", reference_tables=REFERENCE_TABLES) + + # load the data from the data files into the database + temp_db.load_database(DB_PATH, verbose=False) + + # dump in-memory database to file + temp_db.dump_sqlite(DB_NAME) + # replace database object with new file version + db = Database("sqlite:///" + DB_NAME, reference_tables=REFERENCE_TABLES) + + return db + + +def test_spectra_urls(db): + spectra_urls = db.query(db.Spectra.c.spectrum).astropy() + broken_urls = [] + codes = [] + internet = check_internet_connection() + if internet: + for spectrum_url in spectra_urls["spectrum"]: + request_response = requests.head(spectrum_url) + status_code = request_response.status_code + # The website is up if the status code is 200 + # cuny academic commons links give 301 status code + if status_code != 200 and status_code != 301: + broken_urls.append(spectrum_url) + codes.append(status_code) + assert ( + len(broken_urls) == 149 + ), f"found {len(broken_urls)} broken spectra urls: {broken_urls}, {codes}" diff --git a/tests/test_data.py b/tests/test_data.py index 81eb22a3d..788ec0596 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -3,13 +3,14 @@ import os import pytest import sys +from astrodbkit2.astrodb import create_database, Database +from sqlalchemy import except_, select, and_ sys.path.append('.') from simple.schema import * -from astrodbkit2.astrodb import create_database, Database -from sqlalchemy import except_, select, and_ from . import REFERENCE_TABLES + DB_NAME = 'temp.db' DB_PATH = 'data' @@ -448,6 +449,7 @@ def test_spectral_types(db): n_spectral_types = db.query(db.SpectralTypes).count() assert len(m_dwarfs) + len(l_dwarfs) + len(t_dwarfs) + len(y_dwarfs) == n_spectral_types + # Individual ingest tests # ----------------------------------------------------------------------------------------- def test_Manj19_data(db): @@ -533,7 +535,7 @@ def test_Kirk19_ingest(db): # Test spectral types added - # Test parallaxes + # Test parallaxes ref = 'Kirk19' t = db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == ref).astropy() assert len(t) == 23, f'found {len(t)} parallax entries for {ref}' @@ -597,6 +599,7 @@ def test_suar22_ingest(db): t = db.query(db.Spectra).filter(db.Spectra.c.reference == ref).astropy() assert len(t) == 112, f'found {len(t)} spectra entries for {ref}' + def test_modeledparameters(db): # Test to verify existing counts of modeled parameters ref = 'Fili15'