From 65e36e381e23991a4092d3c10a7eecbb4a9badb8 Mon Sep 17 00:00:00 2001 From: Kelle Cruz Date: Tue, 17 Oct 2023 16:26:38 -0400 Subject: [PATCH 1/4] add test for spectrum urls (#404) * add test for spectrum urls * create scheduled_checks for URLs * polish of the test scripts. * add scheduled test to run once per month * change name scheduled-tests.yml --- .github/workflows/scheduled-tests.yml | 30 ++++++++++++ tests/scheduled_checks.py | 66 +++++++++++++++++++++++++++ tests/test_data.py | 9 ++-- 3 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/scheduled-tests.yml create mode 100644 tests/scheduled_checks.py diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml new file mode 100644 index 000000000..db51bb06e --- /dev/null +++ b/.github/workflows/scheduled-tests.yml @@ -0,0 +1,30 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Scheduled Checks + +on: + schedule: + - cron: '30 1 1 * *' +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest ads + pip install astrodbkit2 + + - name: Test with pytest + run: | + pytest -s tests/scheduled_tests.py diff --git a/tests/scheduled_checks.py b/tests/scheduled_checks.py new file mode 100644 index 000000000..2e2927889 --- /dev/null +++ b/tests/scheduled_checks.py @@ -0,0 +1,66 @@ +import os +import pytest +import sys +import requests +from astrodbkit2.astrodb import create_database, Database +from scripts.ingests.utils import check_internet_connection + +sys.path.append(".") +from simple.schema import * +from . import REFERENCE_TABLES + + +DB_NAME = "temp.db" +DB_PATH = "data" + + +# Load the database for use in individual tests +@pytest.fixture(scope="module") +def db(): + # Create a fresh temporary database and assert it exists + # Because we've imported simple.schema, we will be using that schema for the database + + if os.path.exists(DB_NAME): + os.remove(DB_NAME) + connection_string = "sqlite:///" + DB_NAME + create_database(connection_string) + assert os.path.exists(DB_NAME) + + # Connect to the new database and confirm it has the Sources table + db = Database(connection_string, reference_tables=REFERENCE_TABLES) + assert db + assert "source" in [c.name for c in db.Sources.columns] + + # Load data into an in-memory sqlite database first, for performance + + # create and connects to a temporary in-memory database + temp_db = Database("sqlite://", reference_tables=REFERENCE_TABLES) + + # load the data from the data files into the database + temp_db.load_database(DB_PATH, verbose=False) + + # dump in-memory database to file + temp_db.dump_sqlite(DB_NAME) + # replace database object with new file version + db = Database("sqlite:///" + DB_NAME, reference_tables=REFERENCE_TABLES) + + return db + + +def test_spectra_urls(db): + spectra_urls = db.query(db.Spectra.c.spectrum).astropy() + broken_urls = [] + codes = [] + internet = check_internet_connection() + if internet: + for spectrum_url in spectra_urls["spectrum"]: + request_response = requests.head(spectrum_url) + status_code = request_response.status_code + # The website is up if the status code is 200 + # cuny academic commons links give 301 status code + if status_code != 200 and status_code != 301: + broken_urls.append(spectrum_url) + codes.append(status_code) + assert ( + len(broken_urls) == 149 + ), f"found {len(broken_urls)} broken spectra urls: {broken_urls}, {codes}" diff --git a/tests/test_data.py b/tests/test_data.py index 81eb22a3d..788ec0596 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -3,13 +3,14 @@ import os import pytest import sys +from astrodbkit2.astrodb import create_database, Database +from sqlalchemy import except_, select, and_ sys.path.append('.') from simple.schema import * -from astrodbkit2.astrodb import create_database, Database -from sqlalchemy import except_, select, and_ from . import REFERENCE_TABLES + DB_NAME = 'temp.db' DB_PATH = 'data' @@ -448,6 +449,7 @@ def test_spectral_types(db): n_spectral_types = db.query(db.SpectralTypes).count() assert len(m_dwarfs) + len(l_dwarfs) + len(t_dwarfs) + len(y_dwarfs) == n_spectral_types + # Individual ingest tests # ----------------------------------------------------------------------------------------- def test_Manj19_data(db): @@ -533,7 +535,7 @@ def test_Kirk19_ingest(db): # Test spectral types added - # Test parallaxes + # Test parallaxes ref = 'Kirk19' t = db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == ref).astropy() assert len(t) == 23, f'found {len(t)} parallax entries for {ref}' @@ -597,6 +599,7 @@ def test_suar22_ingest(db): t = db.query(db.Spectra).filter(db.Spectra.c.reference == ref).astropy() assert len(t) == 112, f'found {len(t)} spectra entries for {ref}' + def test_modeledparameters(db): # Test to verify existing counts of modeled parameters ref = 'Fili15' From c0458cd5937e4ca040db4a9288dc446da6b38de2 Mon Sep 17 00:00:00 2001 From: Kelle Cruz Date: Tue, 17 Oct 2023 16:32:09 -0400 Subject: [PATCH 2/4] Add manual option run for scheduled tests (#410) --- .github/workflows/scheduled-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml index db51bb06e..90936250c 100644 --- a/.github/workflows/scheduled-tests.yml +++ b/.github/workflows/scheduled-tests.yml @@ -6,6 +6,8 @@ name: Scheduled Checks on: schedule: - cron: '30 1 1 * *' + workflow_dispatch: # manual execution + jobs: build: From 461aa0ccb3b29df220f8e743196b33d684364a67 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Tue, 17 Oct 2023 16:36:04 -0400 Subject: [PATCH 3/4] Parametrizing the photometry tests for clearer updates (#407) --- tests/test_data.py | 93 +++++++++++----------------------------------- 1 file changed, 22 insertions(+), 71 deletions(-) diff --git a/tests/test_data.py b/tests/test_data.py index 788ec0596..7c096d476 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -206,84 +206,35 @@ def test_parallax_refs(db): assert len(t) == 1104, f'found {len(t)} adopted parallax reference entries for {ref}' -def test_photometry_bands(db): +@pytest.mark.parametrize('band, value', [ + ('GAIA2.G', 1266), + ('GAIA2.Grp', 1106), + ('GAIA3.G', 1256), + ('GAIA3.Grp', 1261), + ('WISE.W1', 460), + ('WISE.W2', 460), + ('WISE.W3', 457), + ('WISE.W4', 450), + ('2MASS.J', 1802), + ('2MASS.H', 1791), + ('2MASS.Ks', 1762), + ('GPI.Y', 1), + ('NIRI.Y', 21), + ('UFTI.Y', 13), + ('Wircam.Y', 29), + ('WFCAM.Y', 854), + ('VisAO.Ys', 1), + ('VISTA.Y', 59), +]) +def test_photometry_bands(db, band, value): # To refresh these counts: # from sqlalchemy import func # db.query(db.Photometry.c.band, func.count(db.Photometry.c.band).label('count')).\ # group_by(db.Photometry.c.band).\ # astropy() - band = 'GAIA2.G' t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 1266, f'found {len(t)} photometry measurements for {band}' - - band = 'GAIA2.Grp' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 1106, f'found {len(t)} photometry measurements for {band}' - - band = 'GAIA3.G' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 1256, f'found {len(t)} photometry measurements for {band}' - - band = 'GAIA3.Grp' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 1261, f'found {len(t)} photometry measurements for {band}' - - band = 'WISE.W1' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 460, f'found {len(t)} photometry measurements for {band}' - - band = 'WISE.W2' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 460, f'found {len(t)} photometry measurements for {band}' - - band = 'WISE.W3' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 457, f'found {len(t)} photometry measurements for {band}' - - band = 'WISE.W4' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 450, f'found {len(t)} photometry measurements for {band}' - - band = '2MASS.J' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 1802, f'found {len(t)} photometry measurements for {band}' - - band = '2MASS.H' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 1791, f'found {len(t)} photometry measurements for {band}' - - band = '2MASS.Ks' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 1762, f'found {len(t)} photometry measurements for {band}' - - band = 'GPI.Y' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 1, f'found {len(t)} photometry measurements for {band}' - - band = 'NIRI.Y' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 21, f'found {len(t)} photometry measurements for {band}' - - band = 'UFTI.Y' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 13, f'found {len(t)} photometry measurements for {band}' - - band = 'Wircam.Y' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 29, f'found {len(t)} photometry measurements for {band}' - - band = 'WFCAM.Y' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 854, f'found {len(t)} photometry measurements for {band}' - - band = 'VisAO.Ys' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 1, f'found {len(t)} photometry measurements for {band}' - - band = 'VISTA.Y' - t = db.query(db.Photometry).filter(db.Photometry.c.band == band).astropy() - assert len(t) == 59, f'found {len(t)} photometry measurements for {band}' + assert len(t) == value, f'found {len(t)} photometry measurements for {band}' def test_missions(db): From 532b6591907ee70399b4bf357eb88b750a8822d3 Mon Sep 17 00:00:00 2001 From: Kelle Cruz Date: Tue, 17 Oct 2023 17:25:59 -0400 Subject: [PATCH 4/4] Fix type scheduled-tests.yml (#411) * Fix name of the scheduled-tests.yml * tqdm and number of broken spectra * between 149 and 150 --------- Co-authored-by: David Rodriguez --- .github/workflows/scheduled-tests.yml | 2 +- tests/scheduled_checks.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml index 90936250c..38246c4e6 100644 --- a/.github/workflows/scheduled-tests.yml +++ b/.github/workflows/scheduled-tests.yml @@ -29,4 +29,4 @@ jobs: - name: Test with pytest run: | - pytest -s tests/scheduled_tests.py + pytest -s tests/scheduled_checks.py diff --git a/tests/scheduled_checks.py b/tests/scheduled_checks.py index 2e2927889..92cad769f 100644 --- a/tests/scheduled_checks.py +++ b/tests/scheduled_checks.py @@ -2,6 +2,7 @@ import pytest import sys import requests +from tqdm import tqdm from astrodbkit2.astrodb import create_database, Database from scripts.ingests.utils import check_internet_connection @@ -53,7 +54,7 @@ def test_spectra_urls(db): codes = [] internet = check_internet_connection() if internet: - for spectrum_url in spectra_urls["spectrum"]: + for spectrum_url in tqdm(spectra_urls["spectrum"]): request_response = requests.head(spectrum_url) status_code = request_response.status_code # The website is up if the status code is 200 @@ -61,6 +62,5 @@ def test_spectra_urls(db): if status_code != 200 and status_code != 301: broken_urls.append(spectrum_url) codes.append(status_code) - assert ( - len(broken_urls) == 149 + assert (149 <= len(broken_urls) <= 150 ), f"found {len(broken_urls)} broken spectra urls: {broken_urls}, {codes}"