From 1ffae071b7ddfff51c1bbc6c61472a330f71479c Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Wed, 13 Dec 2023 15:47:53 -0500 Subject: [PATCH 01/35] test --- scripts/ingests/ingest_backyard_worlds_benchmarks.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 scripts/ingests/ingest_backyard_worlds_benchmarks.py diff --git a/scripts/ingests/ingest_backyard_worlds_benchmarks.py b/scripts/ingests/ingest_backyard_worlds_benchmarks.py new file mode 100644 index 000000000..8ccafb210 --- /dev/null +++ b/scripts/ingests/ingest_backyard_worlds_benchmarks.py @@ -0,0 +1,3 @@ +test + + From a9b095138c9bef09c9c0633ae466a552674c761c Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Wed, 13 Dec 2023 16:02:50 -0500 Subject: [PATCH 02/35] first commit --- scripts/ingests/ingest_BYW_2024.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 scripts/ingests/ingest_BYW_2024.py diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py new file mode 100644 index 000000000..8ccafb210 --- /dev/null +++ b/scripts/ingests/ingest_BYW_2024.py @@ -0,0 +1,3 @@ +test + + From 03b9ffd3f7bc1f03c172ab550f7d18b605590a32 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Wed, 13 Dec 2023 16:09:17 -0500 Subject: [PATCH 03/35] update gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 946d05521..1c9421839 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ .vscode .pytest_cache .ds_store -bdnyc/ \ No newline at end of file +bdnyc/ +*.code-workspace From 1b048b01c082463e6106022f4b051aede0009cdb Mon Sep 17 00:00:00 2001 From: kelle Date: Wed, 13 Dec 2023 17:13:27 -0500 Subject: [PATCH 04/35] modular source ingest script --- scripts/ingests/ingest_utils.py | 362 ++++++++++++++++++-------------- 1 file changed, 205 insertions(+), 157 deletions(-) diff --git a/scripts/ingests/ingest_utils.py b/scripts/ingests/ingest_utils.py index 0174d3588..b6adcd874 100644 --- a/scripts/ingests/ingest_utils.py +++ b/scripts/ingests/ingest_utils.py @@ -92,9 +92,9 @@ def ingest_sources(db, sources, references=None, ras=None, decs=None, comments=N # SETUP INPUTS if ras is None and decs is None: - coords = False + coords_provided = False else: - coords = True + coords_provided = True if isinstance(sources, str): n_sources = 1 @@ -121,165 +121,42 @@ def ingest_sources(db, sources, references=None, ras=None, decs=None, comments=N logger.info(f"Trying to add {n_sources} sources") # Loop over each source and decide to ingest, skip, or add alt name - for i, source in enumerate(sources): - # Find out if source is already in database or not - if coords and search_db: - name_matches = find_source_in_db(db, source, ra=ras[i], dec=decs[i]) - elif search_db: - name_matches = find_source_in_db(db, source) - elif not search_db: - name_matches = [] - else: - name_matches = None - ra = None - dec = None - - if len(name_matches) == 1 and search_db: # Source is already in database - n_existing += 1 - msg1 = f"{i}: Skipping {source}. Already in database as {name_matches[0]}. \n " - logger.debug(msg1) - - # Figure out if ingest name is an alternate name and add - db_matches = db.search_object(source, output_table='Sources', fuzzy_search=False) - if len(db_matches) == 0: - #add other name to names table - ingest_names(db, name_matches[0], source) - n_alt_names += 1 - continue - elif len(name_matches) > 1 and search_db: # Multiple source matches in the database - n_multiples += 1 - msg1 = f"{i} Skipping {source} " - msg = f"{i} More than one match for {source}\n {name_matches}\n" - logger.warning(msg1 + msg) - if raise_error: - raise SimpleError(msg) - else: - continue - elif len(name_matches) == 0 or not search_db: # No match in the database, INGEST! - if coords: # Coordinates were provided as input - ra = ras[i] - dec = decs[i] - epoch = None if ma.is_masked(epochs[i]) else epochs[i] - equinox = None if ma.is_masked(equinoxes[i]) else equinoxes[i] - else: # Try to get coordinates from SIMBAD - simbad_result_table = Simbad.query_object(source) - if simbad_result_table is None: - n_skipped += 1 - ra = None - dec = None - msg = f"{i}: Skipping: {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" - logger.warning(msg) - if raise_error: - raise SimpleError(msg) - else: - continue - elif len(simbad_result_table) == 1: - simbad_coords = simbad_result_table['RA'][0] + ' ' + simbad_result_table['DEC'][0] - simbad_skycoord = SkyCoord(simbad_coords, unit=(u.hourangle, u.deg)) - ra = simbad_skycoord.to_string(style='decimal').split()[0] - dec = simbad_skycoord.to_string(style='decimal').split()[1] - epoch = '2000' # Default coordinates from SIMBAD are epoch 2000. - equinox = 'J2000' # Default frame from SIMBAD is IRCS and J2000. - msg = f"Coordinates retrieved from SIMBAD {ra}, {dec}" - logger.debug(msg) - else: - n_skipped += 1 - ra = None - dec = None - msg = f"{i}: Skipping: {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" - logger.warning(msg) - if raise_error: - raise SimpleError(msg) - else: - continue - - logger.debug(f"{i}: Ingesting {source}. Not already in database. ") + for source_counter, source in enumerate(sources): + # ingest_source function starts here + # TODO: figure out counter + logger.debug(f"{source_counter}: Trying to ingest {source}") + + reference = references[source_counter] + other_reference = other_references[source_counter] + comment = None if ma.is_masked(comments[source_counter]) else comments[source_counter] + + if coords_provided: + ra = ras[source_counter] + dec = decs[source_counter] + epoch = None if ma.is_masked(epochs[source_counter]) else epochs[source_counter] + equinox = None if ma.is_masked(equinoxes[source_counter]) else equinoxes[source_counter] + + ingest_source(db, source, reference=reference, ra=ra, dec=dec, epoch=epoch, equinox=equinox, + other_reference=other_reference, comment=comment, raise_error=raise_error, search_db=search_db) else: - msg = f"{i}: unexpected condition encountered ingesting {source}" - logger.error(msg) - raise SimpleError(msg) + ingest_source(db, source, reference=reference, other_reference=other_reference, comment=comment, + raise_error=raise_error, search_db=search_db) - # Construct data to be added - source_data = [{'source': source, - 'ra': ra, - 'dec': dec, - 'reference': references[i], - 'epoch': epoch, - 'equinox': equinox, - 'other_references': other_references[i], - 'comments': None if ma.is_masked(comments[i]) else comments[i]}] - names_data = [{'source': source, - 'other_name': source}] - - # Try to add the source to the database - try: - with db.engine.connect() as conn: - conn.execute(db.Sources.insert().values(source_data)) - conn.commit() - n_added += 1 - msg = f"Added {str(source_data)}" - logger.debug(msg) - except sqlalchemy.exc.IntegrityError: - if ma.is_masked(source_data[0]['reference']): # check if reference is blank - msg = f"{i}: Skipping: {source}. Discovery reference is blank. \n" - msg2 = f"\n {str(source_data)}\n" - logger.warning(msg) - logger.debug(msg2) - n_skipped += 1 - if raise_error: - raise SimpleError(msg + msg2) - else: - continue - elif db.query(db.Publications).filter(db.Publications.c.publication == references[i]).count() == 0: - # check if reference is in Publications table - msg = f"{i}: Skipping: {source}. Discovery reference {references[i]} is not in Publications table. \n" \ - f"(Add it with add_publication function.) \n " - msg2 = f"\n {str(source_data)}\n" - logger.warning(msg) - logger.debug(msg2) - n_skipped += 1 - if raise_error: - raise SimpleError(msg + msg2) - else: - continue - else: - msg = f"{i}: Skipping: {source}. Not sure why." - msg2 = f"\n {str(source_data)} " - logger.warning(msg) - logger.debug(msg2) - n_skipped += 1 - if raise_error: - raise SimpleError(msg + msg2) - else: - continue + # if n_sources > 1: + # logger.info(f"Sources added to database: {n_added}") + # logger.info(f"Names added to database: {n_names} \n") + # logger.info(f"Sources already in database: {n_existing}") + # logger.info(f"Alt Names added to database: {n_alt_names}") + # logger.info(f"Sources NOT added to database because multiple matches: {n_multiples}") + # logger.info(f"Sources NOT added to database: {n_skipped} \n") - # Try to add the source name to the Names table - try: - ingest_names(db, source, source) - n_names += 1 - except sqlalchemy.exc.IntegrityError: - msg = f"{i}: Could not add {names_data} to database" - logger.warning(msg) - if raise_error: - raise SimpleError(msg) - else: - continue + # if n_added != n_names: + # msg = f"Number added should equal names added." + # raise SimpleError(msg) - if n_sources > 1: - logger.info(f"Sources added to database: {n_added}") - logger.info(f"Names added to database: {n_names} \n") - logger.info(f"Sources already in database: {n_existing}") - logger.info(f"Alt Names added to database: {n_alt_names}") - logger.info(f"Sources NOT added to database because multiple matches: {n_multiples}") - logger.info(f"Sources NOT added to database: {n_skipped} \n") - - if n_added != n_names: - msg = f"Number added should equal names added." - raise SimpleError(msg) - - if n_added + n_existing + n_multiples + n_skipped != n_sources: - msg = f"Number added + Number skipped doesn't add up to total sources" - raise SimpleError(msg) + # if n_added + n_existing + n_multiples + n_skipped != n_sources: + # msg = f"Number added + Number skipped doesn't add up to total sources" + # raise SimpleError(msg) return @@ -1580,3 +1457,174 @@ def ingest_companion_relationships(db, source, companion_name, relationship, raise SimpleError(msg) +def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equinox=None, + raise_error=True, search_db=True, other_reference=None, comment=None): + # TODO: ADD DOCSTRING + + if ra is None and dec is None: + coords_provided = False + else: + coords_provided = True + ra = ra + dec = dec + epoch = epoch + equinox = equinox + + logger.debug(f"coords_provided:{coords_provided}") + + # Find out if source is already in database or not + if coords_provided and search_db: + name_matches = find_source_in_db(db, source, ra=ra, dec=dec) + elif search_db: + name_matches = find_source_in_db(db, source) + elif not search_db: + name_matches = [] + else: + name_matches = None + + logger.debug(f"name_matches: {name_matches}") + + # Source is already in database + if len(name_matches) == 1 and search_db: + msg = f"Not ingesting {source}. Already in database as {name_matches[0]}. \n " + logger.info(msg) + + # Figure out if source name provided is an alternate name + db_source_matches = db.search_object(source, output_table='Sources', fuzzy_search=False) + + # Try to add alternate source name to Names table + if len(db_source_matches) == 0: + alt_names_data = [{'source': name_matches[0], 'other_name': source}] + try: + with db.engine.connect() as conn: + conn.execute(db.Names.insert().values(alt_names_data)) + conn.commit() + logger.debug(f" Name added to database: {alt_names_data}\n") + except sqlalchemy.exc.IntegrityError as e: + msg = f" Could not add {alt_names_data} to database" + logger.warning(msg) + if raise_error: + raise SimpleError(msg + '\n' + str(e)) + else: + return + return # Source is already in database, nothing new to ingest + + # Multiple source matches in the database so unable to ingest source + elif len(name_matches) > 1 and search_db: + msg1 = f" Not ingesting {source}." + msg = f" More than one match for {source}\n {name_matches}\n" + logger.warning(msg1 + msg) + if raise_error: + raise SimpleError(msg) + else: + return + + # No match in the database, INGEST! + elif len(name_matches) == 0 or not search_db: + + # Make sure reference is provided and in References table + if reference is None or ma.is_masked(reference): + msg = f"Not ingesting {source}. Discovery reference is blank. \n" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + + ref_check = find_publication(db, name=reference) + logger.debug(f'ref_check: {ref_check}') + + if ref_check is False: + msg = f"Skipping: {source}. Discovery reference {reference} is not in Publications table. \n" \ + f"(Add it with ingest_publication function.)" + logger.warning(msg) + if raise_error: + raise SimpleError(msg + msg2) + else: + return + + # Try to get coordinates from SIMBAD if they were not provided + if not coords_provided: + # Try to get coordinates from SIMBAD + simbad_result_table = Simbad.query_object(source) + + if simbad_result_table is None: + msg = f"Not ingesting {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + # One SIMBAD match! Using those coordinates for source. + elif len(simbad_result_table) == 1: + simbad_coords = simbad_result_table['RA'][0] + ' ' + simbad_result_table['DEC'][0] + simbad_skycoord = SkyCoord(simbad_coords, unit=(u.hourangle, u.deg)) + ra = simbad_skycoord.to_string(style='decimal').split()[0] + dec = simbad_skycoord.to_string(style='decimal').split()[1] + epoch = '2000' # Default coordinates from SIMBAD are epoch 2000. + equinox = 'J2000' # Default frame from SIMBAD is IRCS and J2000. + msg = f"Coordinates retrieved from SIMBAD {ra}, {dec}" + logger.debug(msg) + else: + msg = f"Not ingesting {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + + + + # Just in case other conditionals not met + else: + msg = f"Unexpected condition encountered ingesting {source}" + logger.error(msg) + raise SimpleError(msg) + + logger.debug(f" Ingesting {source}.") + + # Construct data to be added + source_data = [{'source': source, + 'ra': ra, + 'dec': dec, + 'reference': reference, + 'epoch': epoch, + 'equinox': equinox, + 'other_references': other_reference, + 'comments': comment }] + names_data = [{'source': source, + 'other_name': source}] + + # Try to add the source to the database + try: + with db.engine.connect() as conn: + conn.execute(db.Sources.insert().values(source_data)) + conn.commit() + msg = f"Added {str(source_data)}" + logger.info(f"Added {source}") + logger.debug(msg) + except sqlalchemy.exc.IntegrityError: + msg = f"Not ingesting {source}. Not sure why. \n" + msg2 = f" {str(source_data)} " + logger.warning(msg) + logger.debug(msg2) + if raise_error: + raise SimpleError(msg + msg2) + else: + return + + # Try to add the source name to the Names table + try: + with db.engine.connect() as conn: + conn.execute(db.Names.insert().values(names_data)) + conn.commit() + logger.debug(f" Name added to database: {names_data}\n") + except sqlalchemy.exc.IntegrityError: + msg = f" Could not add {names_data} to database" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + + return \ No newline at end of file From 55402ce757c5da728bb1822dff1edd3e7a53ebcd Mon Sep 17 00:00:00 2001 From: kelle Date: Wed, 13 Dec 2023 17:15:05 -0500 Subject: [PATCH 05/35] update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 1c9421839..8c38f80e3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Ignore the database file and misc *.db +*.sqlite *.pyc .idea .vscode @@ -7,3 +8,4 @@ .ds_store bdnyc/ *.code-workspace + From 4ff5a7f05fdd07cb294c6f67884f116b17fde723 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Wed, 13 Dec 2023 22:55:15 -0500 Subject: [PATCH 06/35] First ingest-bwy-2024 --- scripts/ingests/ingest_BYW_2024.py | 202 ++++++++++++++++++++++++++++- 1 file changed, 201 insertions(+), 1 deletion(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 8ccafb210..197ad01fe 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -1,3 +1,203 @@ -test +from scripts.ingests.ingest_utils import * +from scripts.ingests.utils import * +from astropy.table import Table +import astropy.units as u +from astropy.coordinates import Angle +SAVE_DB = True # save the data files in addition to modifying the .db file +RECREATE_DB = True # recreates the .db file from the data files +# LOAD THE DATABASE +db = load_simpledb('SIMPLE.db', recreatedb=RECREATE_DB) + +# live google sheet +link = "https://docs.google.com/spreadsheets/d/1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs/edit#gid=0" + +columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] +byw_table = Table.read(link, format='ascii', data_start=2, data_end=90, header_start=1, names=columns, guess=False, + fast_reader=False, delimiter=',') + +data_columns = ['CWISE J000021.45-481314.9', '0.0893808', '-48.2208077', '2015.4041', 'ICRS', 'CWISE 0000-4813', 'Rothermich', 'None', 'None'] # columns with wanted data values + +print(byw_table) + +# replacing empty values ('cdots') with None +for column in data_columns: + byw_table[column][np.where(byw_table[column] == 'cdots')] = None + +def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equinox=None, + raise_error=True, search_db=True, other_reference=None, comment=None): + # TODO: ADD DOCSTRING + + if ra is None and dec is None: + coords_provided = False + else: + coords_provided = True + ra = ra + dec = dec + epoch = epoch + equinox = equinox + + logger.debug(f"coords_provided:{coords_provided}") + + # Find out if source is already in database or not + if coords_provided and search_db: + name_matches = find_source_in_db(db, source, ra=ra, dec=dec) + elif search_db: + name_matches = find_source_in_db(db, source) + elif not search_db: + name_matches = [] + else: + name_matches = None + + logger.debug(f"name_matches: {name_matches}") + + # Source is already in database + if len(name_matches) == 1 and search_db: + msg = f"Not ingesting {source}. Already in database as {name_matches[0]}. \n " + logger.info(msg) + + # Figure out if source name provided is an alternate name + db_source_matches = db.search_object(source, output_table='Sources', fuzzy_search=False) + + # Try to add alternate source name to Names table + if len(db_source_matches) == 0: + alt_names_data = [{'source': name_matches[0], 'other_name': source}] + try: + with db.engine.connect() as conn: + conn.execute(db.Names.insert().values(alt_names_data)) + conn.commit() + logger.debug(f" Name added to database: {alt_names_data}\n") + except sqlalchemy.exc.IntegrityError as e: + msg = f" Could not add {alt_names_data} to database" + logger.warning(msg) + if raise_error: + raise SimpleError(msg + '\n' + str(e)) + else: + return + return # Source is already in database, nothing new to ingest + + # Multiple source matches in the database so unable to ingest source + elif len(name_matches) > 1 and search_db: + msg1 = f" Not ingesting {source}." + msg = f" More than one match for {source}\n {name_matches}\n" + logger.warning(msg1 + msg) + if raise_error: + raise SimpleError(msg) + else: + return + + # No match in the database, INGEST! + elif len(name_matches) == 0 or not search_db: + + # Make sure reference is provided and in References table + if reference is None or ma.is_masked(reference): + msg = f"Not ingesting {source}. Discovery reference is blank. \n" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + + ref_check = find_publication(db, name=reference) + logger.debug(f'ref_check: {ref_check}') + + if ref_check is False: + msg = f"Skipping: {source}. Discovery reference {reference} is not in Publications table. \n" \ + f"(Add it with ingest_publication function.)" + logger.warning(msg) + if raise_error: + raise SimpleError(msg + msg2) + else: + return + + # Try to get coordinates from SIMBAD if they were not provided + if not coords_provided: + # Try to get coordinates from SIMBAD + simbad_result_table = Simbad.query_object(source) + + if simbad_result_table is None: + msg = f"Not ingesting {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + # One SIMBAD match! Using those coordinates for source. + elif len(simbad_result_table) == 1: + simbad_coords = simbad_result_table['RA'][0] + ' ' + simbad_result_table['DEC'][0] + simbad_skycoord = SkyCoord(simbad_coords, unit=(u.hourangle, u.deg)) + ra = simbad_skycoord.to_string(style='decimal').split()[0] + dec = simbad_skycoord.to_string(style='decimal').split()[1] + epoch = '2000' # Default coordinates from SIMBAD are epoch 2000. + equinox = 'J2000' # Default frame from SIMBAD is IRCS and J2000. + msg = f"Coordinates retrieved from SIMBAD {ra}, {dec}" + logger.debug(msg) + else: + msg = f"Not ingesting {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + + + + # Just in case other conditionals not met + else: + msg = f"Unexpected condition encountered ingesting {source}" + logger.error(msg) + raise SimpleError(msg) + + logger.debug(f" Ingesting {source}.") + + # Construct data to be added + source_data = [{'source': source, + 'ra': ra, + 'dec': dec, + 'reference': reference, + 'epoch': epoch, + 'equinox': equinox, + 'other_references': other_reference, + 'comments': comment }] + names_data = [{'source': source, + 'other_name': source}] + + # Try to add the source to the database + try: + with db.engine.connect() as conn: + conn.execute(db.Sources.insert().values(source_data)) + conn.commit() + msg = f"Added {str(source_data)}" + logger.info(f"Added {source}") + logger.debug(msg) + except sqlalchemy.exc.IntegrityError: + msg = f"Not ingesting {source}. Not sure why. \n" + msg2 = f" {str(source_data)} " + logger.warning(msg) + logger.debug(msg2) + if raise_error: + raise SimpleError(msg + msg2) + else: + return + + # Try to add the source name to the Names table + try: + with db.engine.connect() as conn: + conn.execute(db.Names.insert().values(names_data)) + conn.commit() + logger.debug(f" Name added to database: {names_data}\n") + except sqlalchemy.exc.IntegrityError: + msg = f" Could not add {names_data} to database" + logger.warning(msg) + if raise_error: + raise SimpleError(msg) + else: + return + + return + + +# WRITE THE JSON FILES +if SAVE_DB: + db.save_database(directory='data/') \ No newline at end of file From 11d78f87dca282b4995427f290998cb948c51acd Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Fri, 15 Dec 2023 01:05:37 -0500 Subject: [PATCH 07/35] Removed lines +added source --- scripts/ingests/ingest_BYW_2024.py | 187 ++--------------------------- 1 file changed, 11 insertions(+), 176 deletions(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 197ad01fe..b38de2ded 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -1,6 +1,6 @@ from scripts.ingests.ingest_utils import * from scripts.ingests.utils import * -from astropy.table import Table +from astropy.io import ascii import astropy.units as u from astropy.coordinates import Angle @@ -14,189 +14,24 @@ link = "https://docs.google.com/spreadsheets/d/1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs/edit#gid=0" columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] -byw_table = Table.read(link, format='ascii', data_start=2, data_end=90, header_start=1, names=columns, guess=False, +byw_table = ascii.read(link, format='csv', data_start=2, data_end=90, header_start=1, names=columns, guess=False, fast_reader=False, delimiter=',') -data_columns = ['CWISE J000021.45-481314.9', '0.0893808', '-48.2208077', '2015.4041', 'ICRS', 'CWISE 0000-4813', 'Rothermich', 'None', 'None'] # columns with wanted data values - -print(byw_table) +data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values # replacing empty values ('cdots') with None for column in data_columns: byw_table[column][np.where(byw_table[column] == 'cdots')] = None -def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equinox=None, - raise_error=True, search_db=True, other_reference=None, comment=None): - # TODO: ADD DOCSTRING - - if ra is None and dec is None: - coords_provided = False - else: - coords_provided = True - ra = ra - dec = dec - epoch = epoch - equinox = equinox - - logger.debug(f"coords_provided:{coords_provided}") - - # Find out if source is already in database or not - if coords_provided and search_db: - name_matches = find_source_in_db(db, source, ra=ra, dec=dec) - elif search_db: - name_matches = find_source_in_db(db, source) - elif not search_db: - name_matches = [] - else: - name_matches = None - - logger.debug(f"name_matches: {name_matches}") - - # Source is already in database - if len(name_matches) == 1 and search_db: - msg = f"Not ingesting {source}. Already in database as {name_matches[0]}. \n " - logger.info(msg) - - # Figure out if source name provided is an alternate name - db_source_matches = db.search_object(source, output_table='Sources', fuzzy_search=False) - - # Try to add alternate source name to Names table - if len(db_source_matches) == 0: - alt_names_data = [{'source': name_matches[0], 'other_name': source}] - try: - with db.engine.connect() as conn: - conn.execute(db.Names.insert().values(alt_names_data)) - conn.commit() - logger.debug(f" Name added to database: {alt_names_data}\n") - except sqlalchemy.exc.IntegrityError as e: - msg = f" Could not add {alt_names_data} to database" - logger.warning(msg) - if raise_error: - raise SimpleError(msg + '\n' + str(e)) - else: - return - return # Source is already in database, nothing new to ingest - - # Multiple source matches in the database so unable to ingest source - elif len(name_matches) > 1 and search_db: - msg1 = f" Not ingesting {source}." - msg = f" More than one match for {source}\n {name_matches}\n" - logger.warning(msg1 + msg) - if raise_error: - raise SimpleError(msg) - else: - return - - # No match in the database, INGEST! - elif len(name_matches) == 0 or not search_db: - - # Make sure reference is provided and in References table - if reference is None or ma.is_masked(reference): - msg = f"Not ingesting {source}. Discovery reference is blank. \n" - logger.warning(msg) - if raise_error: - raise SimpleError(msg) - else: - return - - ref_check = find_publication(db, name=reference) - logger.debug(f'ref_check: {ref_check}') - - if ref_check is False: - msg = f"Skipping: {source}. Discovery reference {reference} is not in Publications table. \n" \ - f"(Add it with ingest_publication function.)" - logger.warning(msg) - if raise_error: - raise SimpleError(msg + msg2) - else: - return - - # Try to get coordinates from SIMBAD if they were not provided - if not coords_provided: - # Try to get coordinates from SIMBAD - simbad_result_table = Simbad.query_object(source) - - if simbad_result_table is None: - msg = f"Not ingesting {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" - logger.warning(msg) - if raise_error: - raise SimpleError(msg) - else: - return - # One SIMBAD match! Using those coordinates for source. - elif len(simbad_result_table) == 1: - simbad_coords = simbad_result_table['RA'][0] + ' ' + simbad_result_table['DEC'][0] - simbad_skycoord = SkyCoord(simbad_coords, unit=(u.hourangle, u.deg)) - ra = simbad_skycoord.to_string(style='decimal').split()[0] - dec = simbad_skycoord.to_string(style='decimal').split()[1] - epoch = '2000' # Default coordinates from SIMBAD are epoch 2000. - equinox = 'J2000' # Default frame from SIMBAD is IRCS and J2000. - msg = f"Coordinates retrieved from SIMBAD {ra}, {dec}" - logger.debug(msg) - else: - msg = f"Not ingesting {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" - logger.warning(msg) - if raise_error: - raise SimpleError(msg) - else: - return - - - - # Just in case other conditionals not met - else: - msg = f"Unexpected condition encountered ingesting {source}" - logger.error(msg) - raise SimpleError(msg) - - logger.debug(f" Ingesting {source}.") - - # Construct data to be added - source_data = [{'source': source, - 'ra': ra, - 'dec': dec, - 'reference': reference, - 'epoch': epoch, - 'equinox': equinox, - 'other_references': other_reference, - 'comments': comment }] - names_data = [{'source': source, - 'other_name': source}] - - # Try to add the source to the database - try: - with db.engine.connect() as conn: - conn.execute(db.Sources.insert().values(source_data)) - conn.commit() - msg = f"Added {str(source_data)}" - logger.info(f"Added {source}") - logger.debug(msg) - except sqlalchemy.exc.IntegrityError: - msg = f"Not ingesting {source}. Not sure why. \n" - msg2 = f" {str(source_data)} " - logger.warning(msg) - logger.debug(msg2) - if raise_error: - raise SimpleError(msg + msg2) - else: - return - - # Try to add the source name to the Names table - try: - with db.engine.connect() as conn: - conn.execute(db.Names.insert().values(names_data)) - conn.commit() - logger.debug(f" Name added to database: {names_data}\n") - except sqlalchemy.exc.IntegrityError: - msg = f" Could not add {names_data} to database" - logger.warning(msg) - if raise_error: - raise SimpleError(msg) - else: - return - - return +def ingest_source(db): + ingest_source(db, source = ["CWISE J000021.45-481314.9"], + reference = "Rothermich", + ra = [0.0893808], + dec = [-48.2208077], + epoch = [2015.4041], + equinox = "ICRS") + # WRITE THE JSON FILES if SAVE_DB: From b5b8befdb6fdfccccd1357017f501382145b5ecb Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Fri, 15 Dec 2023 01:10:58 -0500 Subject: [PATCH 08/35] ascii.read change --- scripts/ingests/ingest_BYW_2024.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index b38de2ded..acd02a224 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -14,7 +14,7 @@ link = "https://docs.google.com/spreadsheets/d/1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs/edit#gid=0" columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] -byw_table = ascii.read(link, format='csv', data_start=2, data_end=90, header_start=1, names=columns, guess=False, +byw_table = ascii.read(link, format='ascii', data_start=2, data_end=90, header_start=1, names=columns, guess=False, fast_reader=False, delimiter=',') data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values From 83d99b36f50ec5e4464288ebe6d6fdccdaa382f7 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Fri, 15 Dec 2023 01:15:25 -0500 Subject: [PATCH 09/35] Changed incorrect import name --- scripts/ingests/ingest_BYW_2024.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index acd02a224..489ba27ca 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -1,6 +1,6 @@ from scripts.ingests.ingest_utils import * from scripts.ingests.utils import * -from astropy.io import ascii +from astropy.io.ascii import ascii import astropy.units as u from astropy.coordinates import Angle From aaa7a409a306f35e7d2cbaa30dec7f36b620cc68 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Sat, 16 Dec 2023 02:09:30 -0500 Subject: [PATCH 10/35] Import error: cannot import 'ascii' from astropy.io.ascii (Line 3) --- scripts/ingests/ingest_BYW_2024.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 489ba27ca..059b66173 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -11,17 +11,14 @@ # live google sheet -link = "https://docs.google.com/spreadsheets/d/1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs/edit#gid=0" +link = "https://docs.google.com/spreadsheets/d/1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs/pub?output=csv" columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] -byw_table = ascii.read(link, format='ascii', data_start=2, data_end=90, header_start=1, names=columns, guess=False, +byw_table = ascii.read(link, format='ecsv', data_start=2, data_end=90, header_start=1, names=columns, guess=False, fast_reader=False, delimiter=',') data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values -# replacing empty values ('cdots') with None -for column in data_columns: - byw_table[column][np.where(byw_table[column] == 'cdots')] = None def ingest_source(db): From 720a41d9082f8b9c2c6124a74a3f71ce600e5fc7 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Mon, 18 Dec 2023 12:33:29 -0500 Subject: [PATCH 11/35] Update csv viewing --- scripts/ingests/ingest_BYW_2024.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 059b66173..cb29a573f 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -2,6 +2,7 @@ from scripts.ingests.utils import * from astropy.io.ascii import ascii import astropy.units as u +import pandas as pd from astropy.coordinates import Angle SAVE_DB = True # save the data files in addition to modifying the .db file @@ -11,16 +12,22 @@ # live google sheet -link = "https://docs.google.com/spreadsheets/d/1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs/pub?output=csv" +sheet_id = '1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs' -columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] -byw_table = ascii.read(link, format='ecsv', data_start=2, data_end=90, header_start=1, names=columns, guess=False, - fast_reader=False, delimiter=',') +df = pd.read_csv(f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv") -data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values +print(df) +#link = "https://docs.google.com/spreadsheets/d/1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs/pub?output=csv" -def ingest_source(db): +#columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] +#byw_table = ascii.read(link, format='ecsv', data_start=2, data_end=90, header_start=1, names=columns, guess=False, +# fast_reader=False, delimiter=',') + +#data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values + + +def ingest_source_one(db): ingest_source(db, source = ["CWISE J000021.45-481314.9"], reference = "Rothermich", @@ -28,7 +35,8 @@ def ingest_source(db): dec = [-48.2208077], epoch = [2015.4041], equinox = "ICRS") - + + db.inventory('CWISE J000021.45-481314.9', pretty_print=True) # WRITE THE JSON FILES if SAVE_DB: From 7ded76120423bb25c0ed2ecec9ddc71626830c12 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Mon, 18 Dec 2023 14:09:34 -0500 Subject: [PATCH 12/35] update google sheet file not found error --- scripts/ingests/ingest_BYW_2024.py | 41 ++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index cb29a573f..84e764305 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -1,8 +1,9 @@ from scripts.ingests.ingest_utils import * from scripts.ingests.utils import * -from astropy.io.ascii import ascii +from astropy.io import ascii import astropy.units as u import pandas as pd +import csv from astropy.coordinates import Angle SAVE_DB = True # save the data files in addition to modifying the .db file @@ -11,32 +12,50 @@ db = load_simpledb('SIMPLE.db', recreatedb=RECREATE_DB) +#csv_file_path = "C:\Users\LRamon\Documents\Austin-BYW-Benchmark-Sources.-.Sheet1.csv" + +#with open(csv_file_path, 'r') as file: + # Create a CSV reader object with dictionary-like behavior + #csv_reader = csv.DictReader(file) + + # Iterate through each row in the CSV file + #for row in csv_reader: + # Each 'row' variable represents a dictionary with column names as keys + # print(row) + + + # live google sheet -sheet_id = '1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs' +#sheet_id = '1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs' +#df = pd.read_csv(f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv") +#print(df) -df = pd.read_csv(f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv") -print(df) +sheet_id = '1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs' +link = f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv" -#link = "https://docs.google.com/spreadsheets/d/1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs/pub?output=csv" +#define the column names +columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] +#read the csv data into an astropy table +byw_table = ascii.read(link, format='csv', data_start=2, data_end=90, header_start=1, names=columns, guess=False, delimiter=',') -#columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] -#byw_table = ascii.read(link, format='ecsv', data_start=2, data_end=90, header_start=1, names=columns, guess=False, -# fast_reader=False, delimiter=',') +data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values -#data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values +byw_table.rename_columns(data_columns) +#print result astropy table +print(byw_table) def ingest_source_one(db): - ingest_source(db, source = ["CWISE J000021.45-481314.9"], + ingest_source_one(db, source = ["CWISE J000021.45-481314.9"], reference = "Rothermich", ra = [0.0893808], dec = [-48.2208077], epoch = [2015.4041], equinox = "ICRS") - db.inventory('CWISE J000021.45-481314.9', pretty_print=True) +db.inventory('CWISE J000021.45-481314.9', pretty_print=True) # WRITE THE JSON FILES if SAVE_DB: From 055feb347c818c9a3f892fb38bd14f64e905b1af Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Mon, 18 Dec 2023 14:21:35 -0500 Subject: [PATCH 13/35] update google sheet table prints. Accidental json file changes --- data/2mass_j12560183-1257276.json | 8 ++++---- data/Versions.json | 2 +- scripts/ingests/ingest_BYW_2024.py | 20 +++++++++++++++----- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/data/2mass_j12560183-1257276.json b/data/2mass_j12560183-1257276.json index eed3a0b12..6f15633b6 100644 --- a/data/2mass_j12560183-1257276.json +++ b/data/2mass_j12560183-1257276.json @@ -30,6 +30,9 @@ { "other_name": "VHS 1256-1257b" }, + { + "other_name": "VHS 1256b" + }, { "other_name": "VHS J125601.92-125723.9 b" }, @@ -38,9 +41,6 @@ }, { "other_name": "WISEA J125601.66-125728.7" - }, - { - "other_name": "VHS 1256b" } ], "Parallaxes": [ @@ -267,4 +267,4 @@ "reference": "Gauz15" } ] -} +} \ No newline at end of file diff --git a/data/Versions.json b/data/Versions.json index 9ad0796a2..9df07db7e 100644 --- a/data/Versions.json +++ b/data/Versions.json @@ -57,6 +57,6 @@ "version": "latest", "start_date": "2023-12-12", "end_date": null, - "description": "Version in development" + "description": "Version in development" } ] \ No newline at end of file diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 84e764305..28560368d 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -4,6 +4,8 @@ import astropy.units as u import pandas as pd import csv +import requests +from io import StringIO from astropy.coordinates import Angle SAVE_DB = True # save the data files in addition to modifying the .db file @@ -11,7 +13,7 @@ # LOAD THE DATABASE db = load_simpledb('SIMPLE.db', recreatedb=RECREATE_DB) - +#Attempt 1 #csv_file_path = "C:\Users\LRamon\Documents\Austin-BYW-Benchmark-Sources.-.Sheet1.csv" #with open(csv_file_path, 'r') as file: @@ -25,23 +27,31 @@ -# live google sheet +# live google sheet (Attempt 2) #sheet_id = '1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs' #df = pd.read_csv(f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv") #print(df) +#(Attempt 3) sheet_id = '1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs' link = f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv" +# Fetch the content from the URL +response = requests.get(link) +data = response.text + +# Use io.StringIO to create a file-like object +data_file = StringIO(data) + #define the column names columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] #read the csv data into an astropy table -byw_table = ascii.read(link, format='csv', data_start=2, data_end=90, header_start=1, names=columns, guess=False, delimiter=',') +byw_table = ascii.read(link, format='csv', data_start=1, data_end=90, header_start=1, names=columns, guess=False, fast_reader=False, delimiter=',') -data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values +#data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values +#byw_table.rename_columns(data_columns) -byw_table.rename_columns(data_columns) #print result astropy table print(byw_table) From ca4cfbfa4f9fc86014806035216c57657c8ccfda Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Mon, 18 Dec 2023 14:22:47 -0500 Subject: [PATCH 14/35] Updated import modules --- scripts/ingests/ingest_BYW_2024.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 28560368d..31b8d238f 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -2,8 +2,6 @@ from scripts.ingests.utils import * from astropy.io import ascii import astropy.units as u -import pandas as pd -import csv import requests from io import StringIO from astropy.coordinates import Angle From a912394a1673aec57c07d711691139f63fc5ed74 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Mon, 18 Dec 2023 16:04:23 -0500 Subject: [PATCH 15/35] Source ingest layout attempt --- scripts/ingests/ingest_BYW_2024.py | 33 +++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 31b8d238f..44e7d1713 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -6,7 +6,7 @@ from io import StringIO from astropy.coordinates import Angle -SAVE_DB = True # save the data files in addition to modifying the .db file +SAVE_DB = False # save the data files in addition to modifying the .db file RECREATE_DB = True # recreates the .db file from the data files # LOAD THE DATABASE db = load_simpledb('SIMPLE.db', recreatedb=RECREATE_DB) @@ -44,7 +44,9 @@ #define the column names columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] + #read the csv data into an astropy table +#ascii.read attempts to read data from local files rather from URLs so using a library like requests helps get data and create object that can be passed to ascii.read byw_table = ascii.read(link, format='csv', data_start=1, data_end=90, header_start=1, names=columns, guess=False, fast_reader=False, delimiter=',') #data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values @@ -54,14 +56,39 @@ print(byw_table) +#Individual source ingest attempt... def ingest_source_one(db): - ingest_source_one(db, source = ["CWISE J000021.45-481314.9"], + ingest_source(db, source = ["CWISE J000021.45-481314.9"], reference = "Rothermich", ra = [0.0893808], dec = [-48.2208077], epoch = [2015.4041], - equinox = "ICRS") + equinox = "ICRS", + raise_error=True, + search_db=True, + other_reference=None, + comment=None) + + + +#All sources ingest attempt... +def ingest_all_sources(db): + ingest_sources(db, sources = [" "], + references=None, + ras=None, + decs=None, + comments=None, + epochs=None, + equinoxes=None, + other_references=None, + raise_error=True, + search_db=True) + +#calling functions +ingest_source_one(db) +OR +#ingest_all_sources(db) db.inventory('CWISE J000021.45-481314.9', pretty_print=True) From 0bc26b47081aafe2f08bb690ed10d2b2ee31c727 Mon Sep 17 00:00:00 2001 From: kelle Date: Mon, 18 Dec 2023 16:35:21 -0500 Subject: [PATCH 16/35] Ingest_source: added docstring --- scripts/ingests/ingest_utils.py | 39 ++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/scripts/ingests/ingest_utils.py b/scripts/ingests/ingest_utils.py index b6adcd874..54ca3b0d0 100644 --- a/scripts/ingests/ingest_utils.py +++ b/scripts/ingests/ingest_utils.py @@ -1457,9 +1457,42 @@ def ingest_companion_relationships(db, source, companion_name, relationship, raise SimpleError(msg) -def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equinox=None, - raise_error=True, search_db=True, other_reference=None, comment=None): - # TODO: ADD DOCSTRING +def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equinox=None, + other_reference=None, comment=None, + raise_error=True, search_db=True): + """ + Parameters + ---------- + db: astrodbkit2.astrodb.Database + Database object created by astrodbkit2 + sources: str + Names of sources + references: str + Discovery references of sources + ras: float, optional + Right ascensions of sources. Decimal degrees. + decs: float, optional + Declinations of sources. Decimal degrees. + comments: string, optional + Comments + epochs: str, optional + Epochs of coordinates + equinoxes: str, optional + Equinoxes of coordinates + other_references: str + raise_error: bool, optional + True (default): Raise an error if a source cannot be ingested + False: Log a warning but skip sources which cannot be ingested + search_db: bool, optional + True (default): Search database to see if source is already ingested + False: Ingest source without searching the database + + Returns + ------- + + None + + """ if ra is None and dec is None: coords_provided = False From f0de2360005d50be8f2d364246a654239e0e86e5 Mon Sep 17 00:00:00 2001 From: kelle Date: Mon, 18 Dec 2023 17:08:22 -0500 Subject: [PATCH 17/35] ingest source linting --- scripts/ingests/ingest_utils.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/ingests/ingest_utils.py b/scripts/ingests/ingest_utils.py index 54ca3b0d0..82f2e9c54 100644 --- a/scripts/ingests/ingest_utils.py +++ b/scripts/ingests/ingest_utils.py @@ -1457,9 +1457,10 @@ def ingest_companion_relationships(db, source, companion_name, relationship, raise SimpleError(msg) -def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equinox=None, - other_reference=None, comment=None, - raise_error=True, search_db=True): +def ingest_source(db, source, reference: str = None, ra: float = None, dec: float = None, + epoch: str = None, equinox: str = None, + other_reference: str = None, comment: str = None, + raise_error: bool = True, search_db: bool = True): """ Parameters ---------- @@ -1493,7 +1494,7 @@ def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equ None """ - + if ra is None and dec is None: coords_provided = False else: @@ -1501,8 +1502,7 @@ def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equ ra = ra dec = dec epoch = epoch - equinox = equinox - + equinox = equinox logger.debug(f"coords_provided:{coords_provided}") # Find out if source is already in database or not @@ -1524,7 +1524,7 @@ def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equ # Figure out if source name provided is an alternate name db_source_matches = db.search_object(source, output_table='Sources', fuzzy_search=False) - + # Try to add alternate source name to Names table if len(db_source_matches) == 0: alt_names_data = [{'source': name_matches[0], 'other_name': source}] @@ -1541,7 +1541,6 @@ def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equ else: return return # Source is already in database, nothing new to ingest - # Multiple source matches in the database so unable to ingest source elif len(name_matches) > 1 and search_db: msg1 = f" Not ingesting {source}." @@ -1551,8 +1550,8 @@ def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equ raise SimpleError(msg) else: return - - # No match in the database, INGEST! + + # No match in the database, INGEST! elif len(name_matches) == 0 or not search_db: # Make sure reference is provided and in References table @@ -1580,7 +1579,7 @@ def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equ if not coords_provided: # Try to get coordinates from SIMBAD simbad_result_table = Simbad.query_object(source) - + if simbad_result_table is None: msg = f"Not ingesting {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" logger.warning(msg) @@ -1607,7 +1606,6 @@ def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equ return - # Just in case other conditionals not met else: msg = f"Unexpected condition encountered ingesting {source}" @@ -1624,7 +1622,7 @@ def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equ 'epoch': epoch, 'equinox': equinox, 'other_references': other_reference, - 'comments': comment }] + 'comments': comment}] names_data = [{'source': source, 'other_name': source}] @@ -1637,7 +1635,9 @@ def ingest_source(db, source, reference=None, ra=None, dec=None, epoch=None, equ logger.info(f"Added {source}") logger.debug(msg) except sqlalchemy.exc.IntegrityError: - msg = f"Not ingesting {source}. Not sure why. \n" + msg = f"Not ingesting {source}. Not sure why. \n" \ + "The reference may not exist in Publications table. \n" \ + "Add it with ingest_publication function. \n" msg2 = f" {str(source_data)} " logger.warning(msg) logger.debug(msg2) From aaff895631ad9c0135e773b43f529af6cb3c5394 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Mon, 18 Dec 2023 17:08:30 -0500 Subject: [PATCH 18/35] New changes --- scripts/ingests/ingest_BYW_2024.py | 46 +++++++++++++++++++----------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 44e7d1713..6e849c61f 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -55,35 +55,49 @@ #print result astropy table print(byw_table) +#Loop over the byw table and print source name ra, dec, at each row. +#Change all references to Roth; search and replace +#Individually ingest other_ref sources as publications, then use those 3 to modify the 3 sources to be ingested in the loop + #Individual source ingest attempt... def ingest_source_one(db): - ingest_source(db, source = ["CWISE J000021.45-481314.9"], - reference = "Rothermich", - ra = [0.0893808], - dec = [-48.2208077], - epoch = [2015.4041], + ingest_source(db, source = "CWISE J000021.45-481314.9", + reference = "Roth", + ra = 0.0893808, + dec = -48.2208077, + epoch = 2015.4041, equinox = "ICRS", raise_error=True, search_db=True, other_reference=None, comment=None) + ingest_source(db, source = "CWISE J002029.72-153527.5", + reference = "Roth", + ra = 5.1238388, + dec = -15.5909815, + epoch = 2015.4041, + equinox = "ICRS", + raise_error=True, + search_db=True, + other_reference=None, + comment=None) #All sources ingest attempt... -def ingest_all_sources(db): - ingest_sources(db, sources = [" "], - references=None, - ras=None, - decs=None, - comments=None, - epochs=None, - equinoxes=None, - other_references=None, - raise_error=True, - search_db=True) +#def ingest_all_sources(db): +# ingest_sources(db, sources = [" "], +# references=None, +# ras=None, +# decs=None, +# comments=None, +# epochs=None, +# equinoxes=None, +# other_references=None, +# raise_error=True, +# search_db=True) #calling functions ingest_source_one(db) From f7dd534ffcece03c8ea51d7d0a9811c81656028e Mon Sep 17 00:00:00 2001 From: kelle Date: Mon, 18 Dec 2023 17:48:19 -0500 Subject: [PATCH 19/35] black formatting --- scripts/ingests/ingest_utils.py | 1184 +++++++++++++++++++++---------- 1 file changed, 797 insertions(+), 387 deletions(-) diff --git a/scripts/ingests/ingest_utils.py b/scripts/ingests/ingest_utils.py index 82f2e9c54..c7dd87cef 100644 --- a/scripts/ingests/ingest_utils.py +++ b/scripts/ingests/ingest_utils.py @@ -5,6 +5,7 @@ from astropy.coordinates import SkyCoord import astropy.units as u from astroquery.gaia import Gaia +from astropy.table import Table from typing import List, Union, Optional import numpy as np import numpy.ma as ma @@ -14,14 +15,17 @@ import dateutil import re import requests +import logging +from sqlalchemy import or_, and_ +import sqlalchemy.exc +from scripts.ingests.utils import SimpleError, find_source_in_db, find_publication, check_internet_connection -from scripts.ingests.utils import * +logger = logging.getLogger("SIMPLE") -logger = logging.getLogger('SIMPLE') # NAMES def ingest_names(db, source, other_name): - ''' + """ This function ingests an other name into the Names table Parameters @@ -37,8 +41,8 @@ def ingest_names(db, source, other_name): Returns ------- None - ''' - names_data = [{'source': source, 'other_name': other_name}] + """ + names_data = [{"source": source, "other_name": other_name}] try: with db.engine.connect() as conn: conn.execute(db.Names.insert().values(names_data)) @@ -47,12 +51,23 @@ def ingest_names(db, source, other_name): except sqlalchemy.exc.IntegrityError as e: msg = f"Could not add {names_data} to database. Name is likely a duplicate." logger.warning(msg) - raise SimpleError(msg + '\n' + str(e) + '\n') + raise SimpleError(msg + "\n" + str(e) + "\n") # SOURCES -def ingest_sources(db, sources, references=None, ras=None, decs=None, comments=None, epochs=None, - equinoxes=None, other_references=None, raise_error=True, search_db=True): +def ingest_sources( + db, + sources, + references=None, + ras=None, + decs=None, + comments=None, + epochs=None, + equinoxes=None, + other_references=None, + raise_error=True, + search_db=True, +): """ Script to ingest sources TODO: better support references=None @@ -102,13 +117,31 @@ def ingest_sources(db, sources, references=None, ras=None, decs=None, comments=N n_sources = len(sources) # Convert single element input values into lists - input_values = [sources, references, ras, decs, epochs, equinoxes, comments, other_references] + input_values = [ + sources, + references, + ras, + decs, + epochs, + equinoxes, + comments, + other_references, + ] for i, input_value in enumerate(input_values): if input_value is None: input_values[i] = [None] * n_sources elif isinstance(input_value, (str, float)): input_values[i] = [input_value] * n_sources - sources, references, ras, decs, epochs, equinoxes, comments, other_references = input_values + ( + sources, + references, + ras, + decs, + epochs, + equinoxes, + comments, + other_references, + ) = input_values n_added = 0 n_existing = 0 @@ -125,22 +158,48 @@ def ingest_sources(db, sources, references=None, ras=None, decs=None, comments=N # ingest_source function starts here # TODO: figure out counter logger.debug(f"{source_counter}: Trying to ingest {source}") - - reference = references[source_counter] + + reference = references[source_counter] other_reference = other_references[source_counter] - comment = None if ma.is_masked(comments[source_counter]) else comments[source_counter] - - if coords_provided: - ra = ras[source_counter] - dec = decs[source_counter] - epoch = None if ma.is_masked(epochs[source_counter]) else epochs[source_counter] - equinox = None if ma.is_masked(equinoxes[source_counter]) else equinoxes[source_counter] + comment = ( + None if ma.is_masked(comments[source_counter]) else comments[source_counter] + ) - ingest_source(db, source, reference=reference, ra=ra, dec=dec, epoch=epoch, equinox=equinox, - other_reference=other_reference, comment=comment, raise_error=raise_error, search_db=search_db) + if coords_provided: + ra = ras[source_counter] + dec = decs[source_counter] + epoch = ( + None if ma.is_masked(epochs[source_counter]) else epochs[source_counter] + ) + equinox = ( + None + if ma.is_masked(equinoxes[source_counter]) + else equinoxes[source_counter] + ) + + ingest_source( + db, + source, + reference=reference, + ra=ra, + dec=dec, + epoch=epoch, + equinox=equinox, + other_reference=other_reference, + comment=comment, + raise_error=raise_error, + search_db=search_db, + ) else: - ingest_source(db, source, reference=reference, other_reference=other_reference, comment=comment, - raise_error=raise_error, search_db=search_db) + ingest_source( + db, + source, + reference=reference, + other_reference=other_reference, + comment=comment, + raise_error=raise_error, + search_db=search_db, + ) # if n_sources > 1: # logger.info(f"Sources added to database: {n_added}") @@ -184,54 +243,66 @@ def find_survey_name_in_simbad(sources, desig_prefix, source_id_index=None): n_sources = len(sources) Simbad.reset_votable_fields() - Simbad.add_votable_fields('typed_id') # keep search term in result table - Simbad.add_votable_fields('ids') # add all SIMBAD identifiers as an output column + Simbad.add_votable_fields("typed_id") # keep search term in result table + Simbad.add_votable_fields("ids") # add all SIMBAD identifiers as an output column logger.info("simbad query started") - result_table = Simbad.query_objects(sources['source']) + result_table = Simbad.query_objects(sources["source"]) logger.info("simbad query ended") - ind = result_table['SCRIPT_NUMBER_ID'] > 0 # find indexes which contain results - simbad_ids = result_table['TYPED_ID', 'IDS'][ind] + ind = result_table["SCRIPT_NUMBER_ID"] > 0 # find indexes which contain results + simbad_ids = result_table["TYPED_ID", "IDS"][ind] db_names = [] simbad_designations = [] source_ids = [] for row in simbad_ids: - db_name = row['TYPED_ID'] - ids = row['IDS'].split('|') + db_name = row["TYPED_ID"] + ids = row["IDS"].split("|") designation = [i for i in ids if desig_prefix in i] if designation: - logger.debug(f'{db_name}, {designation[0]}') + logger.debug(f"{db_name}, {designation[0]}") db_names.append(db_name) if len(designation) == 1: simbad_designations.append(designation[0]) else: simbad_designations.append(designation[0]) - logger.warning(f'more than one designation matched, {designation}') + logger.warning(f"more than one designation matched, {designation}") if source_id_index is not None: source_id = designation[0].split()[source_id_index] source_ids.append(int(source_id)) # convert to int since long in Gaia n_matches = len(db_names) - logger.info(f"Found, {n_matches}, {desig_prefix}, sources for, {n_sources}, sources") + logger.info( + f"Found, {n_matches}, {desig_prefix}, sources for, {n_sources}, sources" + ) if source_id_index is not None: - result_table = Table([db_names, simbad_designations, source_ids], - names=('db_names', 'designation', 'source_id')) + result_table = Table( + [db_names, simbad_designations, source_ids], + names=("db_names", "designation", "source_id"), + ) else: - result_table = Table([db_names, simbad_designations], - names=('db_names', 'designation')) + result_table = Table( + [db_names, simbad_designations], names=("db_names", "designation") + ) return result_table # SPECTRAL TYPES -def ingest_spectral_types(db, sources, spectral_types, references, regimes, spectral_type_error=None, - comments=None): +def ingest_spectral_types( + db, + sources, + spectral_types, + references, + regimes, + spectral_type_error=None, + comments=None, +): """ Script to ingest spectral types Parameters @@ -260,14 +331,28 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec n_sources = len(sources) # Convert single element input value to list - input_values = [sources, spectral_types, spectral_type_error, regimes, comments, references] + input_values = [ + sources, + spectral_types, + spectral_type_error, + regimes, + comments, + references, + ] for i, input_value in enumerate(input_values): if input_value is None: input_values[i] = [None] * n_sources elif isinstance(input_value, str): input_values[i] = [input_value] * n_sources # Convert single element input value to list - sources, spectral_types, spectral_type_error, regimes, comments, references = input_values + ( + sources, + spectral_types, + spectral_type_error, + regimes, + comments, + references, + ) = input_values n_added = 0 n_skipped = 0 @@ -279,21 +364,27 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec # Spectral Type data is in the database if len(db_name) != 1: - msg = f"No unique source match for {source} in the database " \ - f"(with SpT: {spectral_types[i]} from {references[i]})" + msg = ( + f"No unique source match for {source} in the database " + f"(with SpT: {spectral_types[i]} from {references[i]})" + ) raise SimpleError(msg) else: db_name = db_name[0] adopted = None - source_spt_data = db.query(db.SpectralTypes).filter(db.SpectralTypes.c.source == db_name).table() + source_spt_data = ( + db.query(db.SpectralTypes) + .filter(db.SpectralTypes.c.source == db_name) + .table() + ) if source_spt_data is None or len(source_spt_data) == 0: adopted: True logger.debug("No Spectral Type data for this source in the database") elif len(source_spt_data) > 0: # Spectral Type Data already exists - dupe_ind = source_spt_data['reference'] == references[i] + dupe_ind = source_spt_data["reference"] == references[i] if sum(dupe_ind): logger.debug(f"Duplicate measurement\n, {source_spt_data[dupe_ind]}") else: @@ -301,26 +392,40 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec if logger.level == 10: source_spt_data.pprint_all() - adopted_ind = source_spt_data['adopted'] == 1 + adopted_ind = source_spt_data["adopted"] == 1 if sum(adopted_ind): old_adopted = source_spt_data[adopted_ind] - if spectral_type_error[i] < min(source_spt_data['spectral_type_error']): + if spectral_type_error[i] < min(source_spt_data["spectral_type_error"]): adopted = True if old_adopted: with db.engine.connect() as conn: conn.execute( - db.SpectralTypes. \ - update(). \ - where(and_(db.SpectralTypes.c.source == old_adopted['source'][0], - db.SpectralTypes.c.reference == old_adopted['reference'][0])). \ - values(adopted=False) + db.SpectralTypes.update() + .where( + and_( + db.SpectralTypes.c.source + == old_adopted["source"][0], + db.SpectralTypes.c.reference + == old_adopted["reference"][0], + ) ) + .values(adopted=False) + ) conn.commit() # check that adopted flag is successfully changed - old_adopted_data = db.query(db.SpectralTypes).filter( - and_(db.SpectralTypes.c.source == old_adopted['source'][0], - db.SpectralTypes.c.reference == old_adopted['reference'][0])).table() + old_adopted_data = ( + db.query(db.SpectralTypes) + .filter( + and_( + db.SpectralTypes.c.source + == old_adopted["source"][0], + db.SpectralTypes.c.reference + == old_adopted["reference"][0], + ) + ) + .table() + ) logger.debug("Old adopted measurement unset") if logger.level == 10: old_adopted_data.pprint_all() @@ -337,23 +442,37 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec logger.debug(msg) # Construct the data to be added - spt_data = [{'source': db_name, - 'spectral_type_string': spectral_types[i], - 'spectral_type_code': spectral_type_code, - 'spectral_type_error': spectral_type_error[i], - 'regime': regimes[i], - 'adopted': adopted, - 'comments': comments[i], - 'reference': references[i]}] + spt_data = [ + { + "source": db_name, + "spectral_type_string": spectral_types[i], + "spectral_type_code": spectral_type_code, + "spectral_type_error": spectral_type_error[i], + "regime": regimes[i], + "adopted": adopted, + "comments": comments[i], + "reference": references[i], + } + ] # Check if the entry already exists; if so: skip adding it - check = db.query(db.SpectralTypes.c.source).filter(and_(db.SpectralTypes.c.source == db_name, - db.SpectralTypes.c.regime == regimes[i], - db.SpectralTypes.c.reference == references[i])).count() + check = ( + db.query(db.SpectralTypes.c.source) + .filter( + and_( + db.SpectralTypes.c.source == db_name, + db.SpectralTypes.c.regime == regimes[i], + db.SpectralTypes.c.reference == references[i], + ) + ) + .count() + ) if check == 1: n_skipped += 1 - logger.info(f'Spectral type for {db_name} already in the database: skipping insert ' - f'{spt_data}') + logger.info( + f"Spectral type for {db_name} already in the database: skipping insert " + f"{spt_data}" + ) continue logger.debug(f"Trying to insert {spt_data} into Spectral Types table ") @@ -365,8 +484,13 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec msg = f"Added {str(spt_data)}" logger.debug(msg) except sqlalchemy.exc.IntegrityError as e: - if db.query(db.Publications).filter(db.Publications.c.reference == references[i]).count() == 0: - msg = f"The publication does not exist in the database" + if ( + db.query(db.Publications) + .filter(db.Publications.c.reference == references[i]) + .count() + == 0 + ): + msg = f"The publication {references[i]} does not exist in the database" msg1 = f"Add it with ingest_publication function." logger.debug(msg + msg1) raise SimpleError(msg) @@ -379,8 +503,7 @@ def ingest_spectral_types(db, sources, spectral_types, references, regimes, spec logger.error(msg) raise SimpleError(msg) - msg = f"Spectral types added: {n_added} \n" \ - f"Spectral Types skipped: {n_skipped}" + msg = f"Spectral types added: {n_added} \n" f"Spectral Types skipped: {n_skipped}" logger.info(msg) @@ -410,25 +533,25 @@ def convert_spt_string_to_code(spectral_types): continue # identify main spectral class, loop over any prefix text to identify MLTY for i, item in enumerate(spt): - if item == 'M': + if item == "M": spt_code = 60 break - elif item == 'L': + elif item == "L": spt_code = 70 break - elif item == 'T': + elif item == "T": spt_code = 80 break - elif item == 'Y': + elif item == "Y": spt_code = 90 break else: # only trigger if not MLTY i = 0 # find integer or decimal subclass and add to spt_code - if re.search('\d*\.?\d+', spt[i+1:]) is None: + if re.search("\d*\.?\d+", spt[i + 1 :]) is None: spt_code = spt_code else: - spt_code += float(re.findall('\d*\.?\d+', spt[i + 1:])[0]) + spt_code += float(re.findall("\d*\.?\d+", spt[i + 1 :])[0]) spectral_type_codes.append(spt_code) return spectral_type_codes @@ -453,21 +576,21 @@ def convert_spt_code_to_string_to_code(spectral_codes, decimals=1): spectral_types = [] for spt in spectral_codes: - spt_type = '' + spt_type = "" # Identify major type if 60 <= spt < 70: - spt_type = 'M' + spt_type = "M" elif 70 <= spt < 80: - spt_type = 'L' + spt_type = "L" elif 80 <= spt < 90: - spt_type = 'T' + spt_type = "T" elif 90 <= spt < 100: - spt_type = 'Y' + spt_type = "Y" # Numeric part of type - format = f'.{decimals}f' - spt_type = f'{spt_type}{spt % 10:{format}}' + format = f".{decimals}f" + spt_type = f"{spt_type}{spt % 10:{format}}" logger.debug(f"Converting: {spt} -> {spt_type}") spectral_types.append(spt_type) @@ -523,7 +646,8 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): n_added = 0 - for i, source in enumerate(sources): # loop through sources with parallax data to ingest + # loop through sources with parallax data to ingest + for i, source in enumerate(sources): db_name = find_source_in_db(db, source) if len(db_name) != 1: @@ -535,16 +659,19 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): # Search for existing parallax data and determine if this is the best # If no previous measurement exists, set the new one to the Adopted measurement adopted = None - source_plx_data: Table = db.query(db.Parallaxes).filter(db.Parallaxes.c.source == db_name).table() + source_plx_data: Table = ( + db.query(db.Parallaxes).filter(db.Parallaxes.c.source == db_name).table() + ) if source_plx_data is None or len(source_plx_data) == 0: - # if there's no other measurements in the database, set new data Adopted = True + # if there's no other measurements in the database, + # set new data Adopted = True adopted = True # old_adopted = None # not used logger.debug("No other measurement") elif len(source_plx_data) > 0: # Parallax data already exists # check for duplicate measurement - dupe_ind = source_plx_data['reference'] == plx_refs[i] + dupe_ind = source_plx_data["reference"] == plx_refs[i] if sum(dupe_ind): logger.debug(f"Duplicate measurement\n, {source_plx_data[dupe_ind]}") continue @@ -554,28 +681,41 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): source_plx_data.pprint_all() # check for previous adopted measurement and find new adopted - adopted_ind = source_plx_data['adopted'] == 1 + adopted_ind = source_plx_data["adopted"] == 1 if sum(adopted_ind): old_adopted = source_plx_data[adopted_ind] # if errors of new data are less than other measurements, set Adopted = True. - if plx_errs[i] < min(source_plx_data['parallax_error']): + if plx_errs[i] < min(source_plx_data["parallax_error"]): adopted = True # unset old adopted if old_adopted: with db.engine.connect() as conn: conn.execute( - db.Parallaxes. \ - update(). \ - where(and_(db.Parallaxes.c.source == old_adopted['source'][0], - db.Parallaxes.c.reference == old_adopted['reference'][0])). \ - values(adopted=False) + db.Parallaxes.update() + .where( + and_( + db.Parallaxes.c.source + == old_adopted["source"][0], + db.Parallaxes.c.reference + == old_adopted["reference"][0], + ) ) + .values(adopted=False) + ) conn.commit() # check that adopted flag is successfully changed - old_adopted_data = db.query(db.Parallaxes).filter( - and_(db.Parallaxes.c.source == old_adopted['source'][0], - db.Parallaxes.c.reference == old_adopted['reference'][0])).table() + old_adopted_data = ( + db.query(db.Parallaxes) + .filter( + and_( + db.Parallaxes.c.source == old_adopted["source"][0], + db.Parallaxes.c.reference + == old_adopted["reference"][0], + ) + ) + .table() + ) logger.debug("Old adopted measurement unset") if logger.level == 10: old_adopted_data.pprint_all() @@ -583,17 +723,21 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): adopted = False logger.debug(f"The new measurement's adopted flag is:, {adopted}") else: - msg = 'Unexpected state' + msg = "Unexpected state" logger.error(msg) raise RuntimeError(msg) # Construct data to be added - parallax_data = [{'source': db_name, - 'parallax': plxs[i], - 'parallax_error': plx_errs[i], - 'reference': plx_refs[i], - 'adopted': adopted, - 'comments': comments[i]}] + parallax_data = [ + { + "source": db_name, + "parallax": plxs[i], + "parallax_error": plx_errs[i], + "reference": plx_refs[i], + "adopted": adopted, + "comments": comments[i], + } + ] logger.debug(f"{parallax_data}") @@ -602,13 +746,14 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): conn.execute(db.Parallaxes.insert().values(parallax_data)) conn.commit() n_added += 1 - logger.info(f"Parallax added to database: \n " - f"{parallax_data}") + logger.info(f"Parallax added to database: \n " f"{parallax_data}") except sqlalchemy.exc.IntegrityError: - msg = "The source may not exist in Sources table.\n" \ - "The parallax reference may not exist in Publications table. " \ - "Add it with add_publication function. \n" \ - "The parallax measurement may be a duplicate." + msg = ( + "The source may not exist in Sources table.\n" + "The parallax reference may not exist in Publications table. " + "Add it with add_publication function. \n" + "The parallax measurement may be a duplicate." + ) logger.error(msg) raise SimpleError(msg) @@ -618,7 +763,9 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): # PROPER MOTIONS -def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, pm_references): +def ingest_proper_motions( + db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, pm_references +): """ Parameters @@ -661,7 +808,6 @@ def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, n_added = 0 for i, source in enumerate(sources): - db_name = find_source_in_db(db, source) if len(db_name) != 1: @@ -673,52 +819,73 @@ def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, # Search for existing proper motion data and determine if this is the best # If no previous measurement exists, set the new one to the Adopted measurement # adopted = None # not used - source_pm_data = db.query(db.ProperMotions).filter(db.ProperMotions.c.source == db_name).table() + source_pm_data = ( + db.query(db.ProperMotions) + .filter(db.ProperMotions.c.source == db_name) + .table() + ) if source_pm_data is None or len(source_pm_data) == 0: - # if there's no other measurements in the database, set new data Adopted = True + # if there's no other measurements in the database, + # set new data Adopted = True adopted = True elif len(source_pm_data) > 0: - # check to see if other measurement is a duplicate of the new data - dupe_ind = source_pm_data['reference'] == pm_references[i] + dupe_ind = source_pm_data["reference"] == pm_references[i] if sum(dupe_ind): logger.debug(f"Duplicate measurement\n, {source_pm_data}") continue # check for previous adopted measurement - adopted_ind = source_pm_data['adopted'] == 1 + adopted_ind = source_pm_data["adopted"] == 1 if sum(adopted_ind): old_adopted = source_pm_data[adopted_ind] else: old_adopted = None # if errors of new data are less than other measurements, set Adopted = True. - if pm_ra_errs[i] < min(source_pm_data['mu_ra_error']) and pm_dec_errs[i] < min( - source_pm_data['mu_dec_error']): + if pm_ra_errs[i] < min(source_pm_data["mu_ra_error"]) and pm_dec_errs[ + i + ] < min(source_pm_data["mu_dec_error"]): adopted = True # unset old adopted if it exists if old_adopted: with db.engine.connect() as conn: conn.execute( - db.ProperMotions. \ - update(). \ - where(and_(db.ProperMotions.c.source == old_adopted['source'][0], - db.ProperMotions.c.reference == old_adopted['reference'][0])). \ - values(adopted=False) + db.ProperMotions.update() + .where( + and_( + db.ProperMotions.c.source + == old_adopted["source"][0], + db.ProperMotions.c.reference + == old_adopted["reference"][0], + ) ) + .values(adopted=False) + ) conn.commit() else: adopted = False - # if no previous adopted measurement, set adopted to the measurement with the smallest errors - if not adopted and not old_adopted and \ - min(source_pm_data['mu_ra_error']) < pm_ra_errs[i] and \ - min(source_pm_data['mu_dec_error']) < pm_dec_errs[i]: - adopted_pm = db.ProperMotions.update().where(and_(db.ProperMotions.c.source == db_name, - db.ProperMotions.c.mu_ra_error == min( - source_pm_data['mu_ra_error']), - db.ProperMotions.c.mu_dec_error == min( - source_pm_data['mu_dec_error']))). \ - values(adopted=True) + # if no previous adopted measurement, + # set adopted to the measurement with the smallest errors + if ( + not adopted + and not old_adopted + and min(source_pm_data["mu_ra_error"]) < pm_ra_errs[i] + and min(source_pm_data["mu_dec_error"]) < pm_dec_errs[i] + ): + adopted_pm = ( + db.ProperMotions.update() + .where( + and_( + db.ProperMotions.c.source == db_name, + db.ProperMotions.c.mu_ra_error + == min(source_pm_data["mu_ra_error"]), + db.ProperMotions.c.mu_dec_error + == min(source_pm_data["mu_dec_error"]), + ) + ) + .values(adopted=True) + ) with db.engine.connect() as conn: conn.execute(adopted_pm) conn.commit() @@ -727,19 +894,23 @@ def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, source_pm_data.pprint_all() else: - msg = 'Unexpected state' + msg = "Unexpected state" logger.error(msg) raise RuntimeError(msg) # Construct data to be added - pm_data = [{'source': db_name, - 'mu_ra': pm_ras[i], - 'mu_ra_error': pm_ra_errs[i], - 'mu_dec': pm_decs[i], - 'mu_dec_error': pm_dec_errs[i], - 'adopted': adopted, - 'reference': pm_references[i]}] - logger.debug(f'Proper motion data to add: {pm_data}') + pm_data = [ + { + "source": db_name, + "mu_ra": pm_ras[i], + "mu_ra_error": pm_ra_errs[i], + "mu_dec": pm_decs[i], + "mu_dec_error": pm_dec_errs[i], + "adopted": adopted, + "reference": pm_references[i], + } + ] + logger.debug(f"Proper motion data to add: {pm_data}") try: with db.engine.connect() as conn: @@ -747,15 +918,21 @@ def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, conn.commit() n_added += 1 except sqlalchemy.exc.IntegrityError: - msg = "The source may not exist in Sources table.\n" \ - "The proper motion reference may not exist in Publications table. " \ - "Add it with add_publication function. \n" \ - "The proper motion measurement may be a duplicate." + msg = ( + "The source may not exist in Sources table.\n" + "The proper motion reference may not exist in Publications table. " + "Add it with add_publication function. \n" + "The proper motion measurement may be a duplicate." + ) logger.error(msg) raise SimpleError(msg) - updated_source_pm_data = db.query(db.ProperMotions).filter(db.ProperMotions.c.source == db_name).table() - logger.info('Updated proper motion data:') + updated_source_pm_data = ( + db.query(db.ProperMotions) + .filter(db.ProperMotions.c.source == db_name) + .table() + ) + logger.info("Updated proper motion data:") if logger.level == 20: # Info = 20, Debug = 10 updated_source_pm_data.pprint_all() @@ -763,8 +940,18 @@ def ingest_proper_motions(db, sources, pm_ras, pm_ra_errs, pm_decs, pm_dec_errs, # PHOTOMETRY -def ingest_photometry(db, sources, bands, magnitudes, magnitude_errors, reference, - telescope=None, epoch=None, comments=None, raise_error=True): +def ingest_photometry( + db, + sources, + bands, + magnitudes, + magnitude_errors, + reference, + telescope=None, + epoch=None, + comments=None, + raise_error=True, +): """ TODO: Write Docstring @@ -813,13 +1000,19 @@ def ingest_photometry(db, sources, bands, magnitudes, magnitude_errors, referenc magnitudes, magnitude_errors = input_float_values if n_sources != len(magnitudes) or n_sources != len(magnitude_errors): - msg = f"N Sources: {len(sources)}, " \ - f"N Magnitudes: {len(magnitudes)}, N Mag errors: {len(magnitude_errors)}," \ - f"\nSources, magnitudes, and magnitude error lists should all be same length" + msg = ( + f"N Sources: {len(sources)}, " + f"N Magnitudes: {len(magnitudes)}, N Mag errors: {len(magnitude_errors)}," + f"\nSources, magnitudes, and magnitude error lists should all be same length" + ) logger.error(msg) raise RuntimeError(msg) - if n_sources != len(reference) or n_sources != len(telescope) or n_sources != len(bands): + if ( + n_sources != len(reference) + or n_sources != len(telescope) + or n_sources != len(bands) + ): msg = "All lists should be same length" logger.error(msg) raise RuntimeError(msg) @@ -842,25 +1035,30 @@ def ingest_photometry(db, sources, bands, magnitudes, magnitude_errors, referenc mag_error = str(magnitude_errors[i]) # Construct data to be added - photometry_data = [{'source': db_name, - 'band': bands[i], - 'magnitude': str(magnitudes[i]), # Convert to string to maintain significant digits - 'magnitude_error': mag_error, - 'telescope': telescope[i], - 'epoch': epoch, - 'comments': comments, - 'reference': reference[i]}] - logger.debug(f'Photometry data: {photometry_data}') + photometry_data = [ + { + "source": db_name, + "band": bands[i], + "magnitude": str( + magnitudes[i] + ), # Convert to string to maintain significant digits + "magnitude_error": mag_error, + "telescope": telescope[i], + "epoch": epoch, + "comments": comments, + "reference": reference[i], + } + ] + logger.debug(f"Photometry data: {photometry_data}") try: with db.engine.connect() as conn: conn.execute(db.Photometry.insert().values(photometry_data)) conn.commit() n_added += 1 - logger.info(f"Photometry measurement added: \n" - f"{photometry_data}") + logger.info(f"Photometry measurement added: \n" f"{photometry_data}") except sqlalchemy.exc.IntegrityError as e: - if 'UNIQUE constraint failed:' in str(e): + if "UNIQUE constraint failed:" in str(e): msg = "The measurement may be a duplicate." if raise_error: logger.error(msg) @@ -869,9 +1067,11 @@ def ingest_photometry(db, sources, bands, magnitudes, magnitude_errors, referenc logger.warning(msg) continue else: - msg = "The source may not exist in Sources table.\n" \ - "The reference may not exist in the Publications table. " \ - "Add it with add_publication function." + msg = ( + "The source may not exist in Sources table.\n" + "The reference may not exist in the Publications table. " + "Add it with add_publication function." + ) logger.error(msg) raise SimpleError(msg) @@ -881,9 +1081,24 @@ def ingest_photometry(db, sources, bands, magnitudes, magnitude_errors, referenc # SPECTRA -def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes, obs_dates, references,original_spectra=None, - wavelength_units=None, flux_units=None, wavelength_order=None, - comments=None, other_references=None, raise_error=True): +def ingest_spectra( + db, + sources, + spectra, + regimes, + telescopes, + instruments, + modes, + obs_dates, + references, + original_spectra=None, + wavelength_units=None, + flux_units=None, + wavelength_order=None, + comments=None, + other_references=None, + raise_error=True, +): """ Parameters @@ -927,15 +1142,37 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes if isinstance(spectra, str): spectra = [spectra] - input_values = [regimes, telescopes, instruments, modes, obs_dates, wavelength_order, wavelength_units, flux_units, - references,comments, other_references] + input_values = [ + regimes, + telescopes, + instruments, + modes, + obs_dates, + wavelength_order, + wavelength_units, + flux_units, + references, + comments, + other_references, + ] for i, input_value in enumerate(input_values): if isinstance(input_value, str): input_values[i] = [input_value] * len(sources) elif isinstance(input_value, type(None)): input_values[i] = [None] * len(sources) - regimes, telescopes, instruments, modes, obs_dates, wavelength_order, wavelength_units, flux_units, \ - references, comments, other_references = input_values + ( + regimes, + telescopes, + instruments, + modes, + obs_dates, + wavelength_order, + wavelength_units, + flux_units, + references, + comments, + other_references, + ) = input_values n_spectra = len(spectra) n_skipped = 0 @@ -944,7 +1181,7 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes n_added = 0 n_blank = 0 - msg = f'Trying to add {n_spectra} spectra' + msg = f"Trying to add {n_spectra} spectra" logger.info(msg) for i, source in enumerate(sources): @@ -964,12 +1201,16 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes internet = check_internet_connection() if internet: request_response = requests.head(spectra[i]) - status_code = request_response.status_code # The website is up if the status code is 200 + status_code = ( + request_response.status_code + ) # The website is up if the status code is 200 if status_code != 200: n_skipped += 1 - msg = "The spectrum location does not appear to be valid: \n" \ - f'spectrum: {spectra[i]} \n' \ - f'status code: {status_code}' + msg = ( + "The spectrum location does not appear to be valid: \n" + f"spectrum: {spectra[i]} \n" + f"status code: {status_code}" + ) logger.error(msg) if raise_error: raise SimpleError(msg) @@ -983,9 +1224,11 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes status_code1 = request_response1.status_code if status_code1 != 200: n_skipped += 1 - msg = "The spectrum location does not appear to be valid: \n" \ - f'spectrum: {original_spectra[i]} \n' \ - f'status code: {status_code1}' + msg = ( + "The spectrum location does not appear to be valid: \n" + f"spectrum: {original_spectra[i]} \n" + f"status code: {status_code1}" + ) logger.error(msg) if raise_error: raise SimpleError(msg) @@ -999,13 +1242,17 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes raise SimpleError(msg) # Find what spectra already exists in database for this source - source_spec_data = db.query(db.Spectra).filter(db.Spectra.c.source == db_name).table() + source_spec_data = ( + db.query(db.Spectra).filter(db.Spectra.c.source == db_name).table() + ) # SKIP if observation date is blank # TODO: try to populate obs date from meta data in spectrum file - if ma.is_masked(obs_dates[i]) or obs_dates[i] == '': + if ma.is_masked(obs_dates[i]) or obs_dates[i] == "": obs_date = None - missing_obs_msg = f"Skipping spectrum with missing observation date: {source} \n" + missing_obs_msg = ( + f"Skipping spectrum with missing observation date: {source} \n" + ) missing_row_spe = f"{source, obs_dates[i], references[i]} \n" logger.info(missing_obs_msg) logger.debug(missing_row_spe) @@ -1013,7 +1260,9 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes continue else: try: - obs_date = pd.to_datetime(obs_dates[i]) # TODO: Another method that doesn't require pandas? + obs_date = pd.to_datetime( + obs_dates[i] + ) # TODO: Another method that doesn't require pandas? except ValueError: n_skipped += 1 if raise_error: @@ -1032,22 +1281,32 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes continue # TODO: make it possible to ingest units and order - row_data = [{'source': db_name, - 'spectrum': spectra[i], - 'original_spectrum': None, # if ma.is_masked(original_spectra[i]) or isinstance(original_spectra,None) - # else original_spectra[i], - 'local_spectrum': None, # if ma.is_masked(local_spectra[i]) else local_spectra[i], - 'regime': regimes[i], - 'telescope': telescopes[i], - 'instrument': None if ma.is_masked(instruments[i]) else instruments[i], - 'mode': None if ma.is_masked(modes[i]) else modes[i], - 'observation_date': obs_date, - 'wavelength_units': None if ma.is_masked(wavelength_units[i]) else wavelength_units[i], - 'flux_units': None if ma.is_masked(flux_units[i]) else flux_units[i], - 'wavelength_order': None if ma.is_masked(wavelength_order[i]) else wavelength_order[i], - 'comments': None if ma.is_masked(comments[i]) else comments[i], - 'reference': references[i], - 'other_references': None if ma.is_masked(other_references[i]) else other_references[i]}] + row_data = [ + { + "source": db_name, + "spectrum": spectra[i], + "original_spectrum": None, # if ma.is_masked(original_spectra[i]) or isinstance(original_spectra,None) + # else original_spectra[i], + "local_spectrum": None, # if ma.is_masked(local_spectra[i]) else local_spectra[i], + "regime": regimes[i], + "telescope": telescopes[i], + "instrument": None if ma.is_masked(instruments[i]) else instruments[i], + "mode": None if ma.is_masked(modes[i]) else modes[i], + "observation_date": obs_date, + "wavelength_units": None + if ma.is_masked(wavelength_units[i]) + else wavelength_units[i], + "flux_units": None if ma.is_masked(flux_units[i]) else flux_units[i], + "wavelength_order": None + if ma.is_masked(wavelength_order[i]) + else wavelength_order[i], + "comments": None if ma.is_masked(comments[i]) else comments[i], + "reference": references[i], + "other_references": None + if ma.is_masked(other_references[i]) + else other_references[i], + } + ] logger.debug(row_data) try: @@ -1056,7 +1315,6 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes conn.commit() n_added += 1 except sqlalchemy.exc.IntegrityError as e: - if "CHECK constraint failed: regime" in str(e): msg = f"Regime provided is not in schema: {regimes[i]}" logger.error(msg) @@ -1064,26 +1322,50 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes raise SimpleError(msg) else: continue - if db.query(db.Publications).filter(db.Publications.c.publication == references[i]).count() == 0: - msg = f"Spectrum for {source} could not be added to the database because the reference {references[i]} is not in Publications table. \n" \ - f"(Add it with ingest_publication function.) \n " + if ( + db.query(db.Publications) + .filter(db.Publications.c.publication == references[i]) + .count() + == 0 + ): + msg = ( + f"Spectrum for {source} could not be added to the database because the reference {references[i]} is not in Publications table. \n" + f"(Add it with ingest_publication function.) \n " + ) logger.warning(msg) if raise_error: raise SimpleError(msg) else: continue # check telescope, instrument, mode exists - telescope = db.query(db.Telescopes).filter(db.Telescopes.c.name == row_data[0]['telescope']).table() - instrument = db.query(db.Instruments).filter(db.Instruments.c.name == row_data[0]['instrument']).table() - mode = db.query(db.Modes).filter(db.Modes.c.name == row_data[0]['mode']).table() + telescope = ( + db.query(db.Telescopes) + .filter(db.Telescopes.c.name == row_data[0]["telescope"]) + .table() + ) + instrument = ( + db.query(db.Instruments) + .filter(db.Instruments.c.name == row_data[0]["instrument"]) + .table() + ) + mode = ( + db.query(db.Modes) + .filter(db.Modes.c.name == row_data[0]["mode"]) + .table() + ) if len(source_spec_data) > 0: # Spectra data already exists # check for duplicate measurement - ref_dupe_ind = source_spec_data['reference'] == references[i] - date_dupe_ind = source_spec_data['observation_date'] == obs_date - instrument_dupe_ind = source_spec_data['instrument'] == instruments[i] - mode_dupe_ind = source_spec_data['mode'] == modes[i] - if sum(ref_dupe_ind) and sum(date_dupe_ind) and sum(instrument_dupe_ind) and sum(mode_dupe_ind): + ref_dupe_ind = source_spec_data["reference"] == references[i] + date_dupe_ind = source_spec_data["observation_date"] == obs_date + instrument_dupe_ind = source_spec_data["instrument"] == instruments[i] + mode_dupe_ind = source_spec_data["mode"] == modes[i] + if ( + sum(ref_dupe_ind) + and sum(date_dupe_ind) + and sum(instrument_dupe_ind) + and sum(mode_dupe_ind) + ): msg = f"Skipping suspected duplicate measurement\n{source}\n" msg2 = f"{source_spec_data[ref_dupe_ind]['source', 'instrument', 'mode', 'observation_date', 'reference']}" msg3 = f"{instruments[i], modes[i], obs_date, references[i], spectra[i]} \n" @@ -1107,11 +1389,13 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes # n_skipped += 1 # continue if len(instrument) == 0 or len(mode) == 0 or len(telescope) == 0: - msg = f'Spectrum for {source} could not be added to the database. \n' \ - f' Telescope, Instrument, and/or Mode need to be added to the appropriate table. \n' \ - f" Trying to find telescope: {row_data[0]['telescope']}, instrument: {row_data[0]['instrument']}, " \ - f" mode: {row_data[0]['mode']} \n" \ - f" Telescope: {telescope}, Instrument: {instrument}, Mode: {mode} \n" + msg = ( + f"Spectrum for {source} could not be added to the database. \n" + f" Telescope, Instrument, and/or Mode need to be added to the appropriate table. \n" + f" Trying to find telescope: {row_data[0]['telescope']}, instrument: {row_data[0]['instrument']}, " + f" mode: {row_data[0]['mode']} \n" + f" Telescope: {telescope}, Instrument: {instrument}, Mode: {mode} \n" + ) logger.error(msg) n_missing_instrument += 1 if raise_error: @@ -1119,34 +1403,48 @@ def ingest_spectra(db, sources, spectra, regimes, telescopes, instruments, modes else: continue else: - msg = f'Spectrum for {source} could not be added to the database for unknown reason: \n {row_data} \n ' + msg = f"Spectrum for {source} could not be added to the database for unknown reason: \n {row_data} \n " logger.error(msg) raise SimpleError(msg) - msg = f"SPECTRA ADDED: {n_added} \n" \ - f" Spectra with blank obs_date: {n_blank} \n" \ - f" Suspected duplicates skipped: {n_dupes}\n" \ - f" Missing Telescope/Instrument/Mode: {n_missing_instrument} \n" \ - f" Spectra skipped for unknown reason: {n_skipped} \n" + msg = ( + f"SPECTRA ADDED: {n_added} \n" + f" Spectra with blank obs_date: {n_blank} \n" + f" Suspected duplicates skipped: {n_dupes}\n" + f" Missing Telescope/Instrument/Mode: {n_missing_instrument} \n" + f" Spectra skipped for unknown reason: {n_skipped} \n" + ) if n_spectra == 1: - logger.info(f"Added {source} : \n" - f"{row_data}") + logger.info(f"Added {source} : \n" f"{row_data}") else: logger.info(msg) - if n_added + n_dupes + n_blank + n_skipped + n_missing_instrument != n_spectra: msg = "Numbers don't add up: " logger.error(msg) raise SimpleError(msg) - spec_count = db.query(Spectra.regime, func.count(Spectra.regime)).group_by(Spectra.regime).all() - - spec_ref_count = db.query(Spectra.reference, func.count(Spectra.reference)). \ - group_by(Spectra.reference).order_by(func.count(Spectra.reference).desc()).limit(20).all() - - telescope_spec_count = db.query(Spectra.telescope, func.count(Spectra.telescope)). \ - group_by(Spectra.telescope).order_by(func.count(Spectra.telescope).desc()).limit(20).all() + spec_count = ( + db.query(Spectra.regime, func.count(Spectra.regime)) + .group_by(Spectra.regime) + .all() + ) + + spec_ref_count = ( + db.query(Spectra.reference, func.count(Spectra.reference)) + .group_by(Spectra.reference) + .order_by(func.count(Spectra.reference).desc()) + .limit(20) + .all() + ) + + telescope_spec_count = ( + db.query(Spectra.telescope, func.count(Spectra.telescope)) + .group_by(Spectra.telescope) + .order_by(func.count(Spectra.telescope).desc()) + .limit(20) + .all() + ) # logger.info(f'Spectra in the database: \n {spec_count} \n {spec_ref_count} \n {telescope_spec_count}') @@ -1179,49 +1477,66 @@ def ingest_instrument(db, telescope=None, instrument=None, mode=None): logger.error(msg) raise SimpleError(msg) - msg_search = f'Searching for {telescope}, {instrument}, {mode} in database' + msg_search = f"Searching for {telescope}, {instrument}, {mode} in database" logger.info(msg_search) # Search for the inputs in the database - telescope_db = db.query(db.Telescopes).filter(db.Telescopes.c.telescope == telescope).table() - mode_db = db.query(db.Instruments).filter(and_(db.Instruments.c.mode == mode, - db.Instruments.c.instrument == instrument, - db.Instruments.c.telescope == telescope)).table() + telescope_db = ( + db.query(db.Telescopes).filter(db.Telescopes.c.telescope == telescope).table() + ) + mode_db = ( + db.query(db.Instruments) + .filter( + and_( + db.Instruments.c.mode == mode, + db.Instruments.c.instrument == instrument, + db.Instruments.c.telescope == telescope, + ) + ) + .table() + ) if len(telescope_db) == 1 and len(mode_db) == 1: - msg_found = f'{telescope}, {instrument}, and {mode} are already in the database.' + msg_found = ( + f"{telescope}, {instrument}, and {mode} are already in the database." + ) logger.info(msg_found) return # Ingest telescope entry if not already present if telescope is not None and len(telescope_db) == 0: - telescope_add = [{'telescope': telescope}] + telescope_add = [{"telescope": telescope}] try: with db.engine.connect() as conn: conn.execute(db.Telescopes.insert().values(telescope_add)) conn.commit() - msg_telescope = f'{telescope} was successfully ingested in the database' + msg_telescope = f"{telescope} was successfully ingested in the database" logger.info(msg_telescope) except sqlalchemy.exc.IntegrityError as e: # pylint: disable=invalid-name - msg = 'Telescope could not be ingested' + msg = "Telescope could not be ingested" logger.error(msg) - raise SimpleError(msg + '\n' + str(e)) + raise SimpleError(msg + "\n" + str(e)) # Ingest instrument+mode (requires telescope) if not already present - if telescope is not None and instrument is not None and mode is not None and len(mode_db) == 0: - instrument_add = [{'instrument': instrument, - 'mode': mode, - 'telescope': telescope}] + if ( + telescope is not None + and instrument is not None + and mode is not None + and len(mode_db) == 0 + ): + instrument_add = [ + {"instrument": instrument, "mode": mode, "telescope": telescope} + ] try: with db.engine.connect() as conn: conn.execute(db.Instruments.insert().values(instrument_add)) conn.commit() - msg_instrument = f'{instrument} was successfully ingested in the database.' + msg_instrument = f"{instrument} was successfully ingested in the database." logger.info(msg_instrument) except sqlalchemy.exc.IntegrityError as e: # pylint: disable=invalid-name - msg = 'Instrument/Mode could not be ingested' + msg = "Instrument/Mode could not be ingested" logger.error(msg) - raise SimpleError(msg + '\n' + str(e)) + raise SimpleError(msg + "\n" + str(e)) return @@ -1241,13 +1556,15 @@ def get_gaiadr3(gaia_id, verbose=True): Table of Gaia data """ - gaia_query_string = f"SELECT " \ - f"parallax, parallax_error, " \ - f"pmra, pmra_error, pmdec, pmdec_error, " \ - f"phot_g_mean_flux, phot_g_mean_flux_error, phot_g_mean_mag, " \ - f"phot_rp_mean_flux, phot_rp_mean_flux_error, phot_rp_mean_mag " \ - f"FROM gaiadr3.gaia_source WHERE " \ - f"gaiadr3.gaia_source.source_id = '{gaia_id}'" + gaia_query_string = ( + f"SELECT " + f"parallax, parallax_error, " + f"pmra, pmra_error, pmdec, pmdec_error, " + f"phot_g_mean_flux, phot_g_mean_flux_error, phot_g_mean_mag, " + f"phot_rp_mean_flux, phot_rp_mean_flux_error, phot_rp_mean_mag " + f"FROM gaiadr3.gaia_source WHERE " + f"gaiadr3.gaia_source.source_id = '{gaia_id}'" + ) job_gaia_query = Gaia.launch_job(gaia_query_string, verbose=verbose) gaia_data = job_gaia_query.get_results() @@ -1257,57 +1574,91 @@ def get_gaiadr3(gaia_id, verbose=True): def ingest_gaia_photometry(db, sources, gaia_data, ref): # TODO write some tests - unmasked_gphot = np.logical_not(gaia_data['phot_g_mean_mag'].mask).nonzero() - gaia_g_phot = gaia_data[unmasked_gphot]['phot_g_mean_flux', 'phot_g_mean_flux_error', - 'phot_g_mean_mag'] + unmasked_gphot = np.logical_not(gaia_data["phot_g_mean_mag"].mask).nonzero() + gaia_g_phot = gaia_data[unmasked_gphot][ + "phot_g_mean_flux", "phot_g_mean_flux_error", "phot_g_mean_mag" + ] - unmased_rpphot = np.logical_not(gaia_data['phot_rp_mean_mag'].mask).nonzero() - gaia_rp_phot = gaia_data[unmased_rpphot]['phot_rp_mean_flux', 'phot_rp_mean_flux_error', - 'phot_rp_mean_mag'] + unmased_rpphot = np.logical_not(gaia_data["phot_rp_mean_mag"].mask).nonzero() + gaia_rp_phot = gaia_data[unmased_rpphot][ + "phot_rp_mean_flux", "phot_rp_mean_flux_error", "phot_rp_mean_mag" + ] # e_Gmag=abs(-2.5/ln(10)*e_FG/FG) from Vizier Note 37 on Gaia DR2 (I/345/gaia2) - gaia_g_phot['g_unc'] = np.abs( - -2.5 / np.log(10) * gaia_g_phot['phot_g_mean_flux_error'] / gaia_g_phot['phot_g_mean_flux']) - gaia_rp_phot['rp_unc'] = np.abs( - -2.5 / np.log(10) * gaia_rp_phot['phot_rp_mean_flux_error'] / gaia_rp_phot['phot_rp_mean_flux']) - - if ref == 'GaiaDR2': - g_band_name = 'GAIA2.G' - rp_band_name = 'GAIA2.Grp' - elif ref == 'GaiaEDR3' or ref == 'GaiaDR3': - g_band_name = 'GAIA3.G' - rp_band_name = 'GAIA3.Grp' + gaia_g_phot["g_unc"] = np.abs( + -2.5 + / np.log(10) + * gaia_g_phot["phot_g_mean_flux_error"] + / gaia_g_phot["phot_g_mean_flux"] + ) + gaia_rp_phot["rp_unc"] = np.abs( + -2.5 + / np.log(10) + * gaia_rp_phot["phot_rp_mean_flux_error"] + / gaia_rp_phot["phot_rp_mean_flux"] + ) + + if ref == "GaiaDR2": + g_band_name = "GAIA2.G" + rp_band_name = "GAIA2.Grp" + elif ref == "GaiaEDR3" or ref == "GaiaDR3": + g_band_name = "GAIA3.G" + rp_band_name = "GAIA3.Grp" else: raise Exception - ingest_photometry(db, sources, g_band_name, gaia_g_phot['phot_g_mean_mag'], gaia_g_phot['g_unc'], - ref, ucds='em.opt', telescope='Gaia', instrument='Gaia') - - ingest_photometry(db, sources, rp_band_name, gaia_rp_phot['phot_rp_mean_mag'], - gaia_rp_phot['rp_unc'], ref, ucds='em.opt.R', telescope='Gaia', instrument='Gaia') + ingest_photometry( + db, + sources, + g_band_name, + gaia_g_phot["phot_g_mean_mag"], + gaia_g_phot["g_unc"], + ref, + ucds="em.opt", + telescope="Gaia", + instrument="Gaia", + ) + + ingest_photometry( + db, + sources, + rp_band_name, + gaia_rp_phot["phot_rp_mean_mag"], + gaia_rp_phot["rp_unc"], + ref, + ucds="em.opt.R", + telescope="Gaia", + instrument="Gaia", + ) return def ingest_gaia_parallaxes(db, sources, gaia_data, ref): # TODO write some tests - unmasked_pi = np.logical_not(gaia_data['parallax'].mask).nonzero() - gaia_parallaxes = gaia_data[unmasked_pi]['parallax', 'parallax_error'] + unmasked_pi = np.logical_not(gaia_data["parallax"].mask).nonzero() + gaia_parallaxes = gaia_data[unmasked_pi]["parallax", "parallax_error"] - ingest_parallaxes(db, sources, gaia_parallaxes['parallax'], - gaia_parallaxes['parallax_error'], ref) + ingest_parallaxes( + db, sources, gaia_parallaxes["parallax"], gaia_parallaxes["parallax_error"], ref + ) def ingest_gaia_pms(db, sources, gaia_data, ref): # TODO write some tests - unmasked_pms = np.logical_not(gaia_data['pmra'].mask).nonzero() - pms = gaia_data[unmasked_pms]['pmra', 'pmra_error', 'pmdec', 'pmdec_error'] + unmasked_pms = np.logical_not(gaia_data["pmra"].mask).nonzero() + pms = gaia_data[unmasked_pms]["pmra", "pmra_error", "pmdec", "pmdec_error"] refs = [ref] * len(pms) - ingest_proper_motions(db, sources, - pms['pmra'], pms['pmra_error'], - pms['pmdec'], pms['pmdec_error'], - refs) + ingest_proper_motions( + db, + sources, + pms["pmra"], + pms["pmra_error"], + pms["pmdec"], + pms["pmdec_error"], + refs, + ) def ingest_spectrum_from_fits(db, source, spectrum_fits_file): @@ -1322,31 +1673,55 @@ def ingest_spectrum_from_fits(db, source, spectrum_fits_file): """ header = fits.getheader(spectrum_fits_file) - regime = header['SPECBAND'] - if regime == 'opt': - regime = 'optical' - telescope = header['TELESCOP'] - instrument = header['INSTRUME'] + regime = header["SPECBAND"] + if regime == "opt": + regime = "optical" + telescope = header["TELESCOP"] + instrument = header["INSTRUME"] try: - mode = header['MODE'] + mode = header["MODE"] except KeyError: mode = None - obs_date = header['DATE-OBS'] - doi = header['REFERENC'] + obs_date = header["DATE-OBS"] + doi = header["REFERENC"] data_header = fits.getheader(spectrum_fits_file, 1) - w_unit = data_header['TUNIT1'] - flux_unit = data_header['TUNIT2'] - - reference_match = db.query(db.Publications.c.publication).filter(db.Publications.c.doi == doi).table() - reference = reference_match['publication'][0] - - ingest_spectra(db, source, spectrum_fits_file, regime, telescope, instrument, None, obs_date, reference, - wavelength_units=w_unit, flux_units=flux_unit) - -#COMPANION RELATIONSHIP -def ingest_companion_relationships(db, source, companion_name, relationship, - projected_separation_arcsec = None, projected_separation_error = None, - comment = None, ref = None, other_companion_names = None): + w_unit = data_header["TUNIT1"] + flux_unit = data_header["TUNIT2"] + + reference_match = ( + db.query(db.Publications.c.publication) + .filter(db.Publications.c.doi == doi) + .table() + ) + reference = reference_match["publication"][0] + + ingest_spectra( + db, + source, + spectrum_fits_file, + regime, + telescope, + instrument, + None, + obs_date, + reference, + wavelength_units=w_unit, + flux_units=flux_unit, + ) + + +# COMPANION RELATIONSHIP +def ingest_companion_relationships( + db, + source, + companion_name, + relationship, + projected_separation_arcsec=None, + projected_separation_error=None, + comment=None, + ref=None, + other_companion_names=None, +): """ This function ingests a single row in to the CompanionRelationship table @@ -1356,7 +1731,7 @@ def ingest_companion_relationships(db, source, companion_name, relationship, Database object created by astrodbkit2 source: str Name of source as it appears in sources table - relationship: str + relationship: str relationship is of the souce to its companion should be one of the following: Child, Sibling, Parent, or Unresolved Parent see note @@ -1385,29 +1760,31 @@ def ingest_companion_relationships(db, source, companion_name, relationship, - *Unresolved Parent*: The source is the unresolved, combined light source of an unresolved multiple system which includes the companion - """ + """ # checking relationship entered - possible_relationships = ['Child', 'Sibling', 'Parent', 'Unresolved Parent', None] + possible_relationships = ["Child", "Sibling", "Parent", "Unresolved Parent", None] # check captialization if relationship.title() != relationship: - logger.info(f"Relationship captilization changed from {relationship} to {relationship.title()} ") + logger.info( + f"Relationship captilization changed from {relationship} to {relationship.title()} " + ) relationship = relationship.title() if relationship not in possible_relationships: msg = f"Relationship given for {source}, {companion_name}: {relationship} NOT one of the constrained relationships \n {possible_relationships}" logger.error(msg) raise SimpleError(msg) - + # source canot be same as companion if source == companion_name: msg = f"{source}: Source cannot be the same as companion name" logger.error(msg) raise SimpleError(msg) - + if source == companion_name: msg = f"{source}: Source cannot be the same as companion name" logger.error(msg) raise SimpleError(msg) - + if projected_separation_arcsec != None and projected_separation_arcsec < 0: msg = f"Projected separation: {projected_separation_arcsec}, cannot be negative" logger.error(msg) @@ -1418,49 +1795,73 @@ def ingest_companion_relationships(db, source, companion_name, relationship, raise SimpleError(msg) # check other names - ## make sure companion name is included in the list + # make sure companion name is included in the list if other_companion_names == None: other_companion_names = companion_name else: - companion_name_list = other_companion_names.split(', ') + companion_name_list = other_companion_names.split(", ") if companion_name not in companion_name_list: companion_name_list.append(companion_name) - other_companion_names = (', ').join(companion_name_list) - + other_companion_names = (", ").join(companion_name_list) try: with db.engine.connect() as conn: - conn.execute(db.CompanionRelationships.insert().values( - {'source': source, - 'companion_name': companion_name, - 'projected_separation_arcsec':projected_separation_arcsec, - 'projected_separation_error':projected_separation_error, - 'relationship':relationship, - 'reference': ref, - 'comments': comment, - 'other_companion_names': other_companion_names})) + conn.execute( + db.CompanionRelationships.insert().values( + { + "source": source, + "companion_name": companion_name, + "projected_separation_arcsec": projected_separation_arcsec, + "projected_separation_error": projected_separation_error, + "relationship": relationship, + "reference": ref, + "comments": comment, + "other_companion_names": other_companion_names, + } + ) + ) conn.commit() - logger.info(f"ComapnionRelationship added: ", - [source, companion_name, relationship, projected_separation_arcsec, \ - projected_separation_error, comment, ref]) + logger.info( + f"ComapnionRelationship added: ", + [ + source, + companion_name, + relationship, + projected_separation_arcsec, + projected_separation_error, + comment, + ref, + ], + ) except sqlalchemy.exc.IntegrityError as e: - if 'UNIQUE constraint failed:' in str(e): + if "UNIQUE constraint failed:" in str(e): msg = "The companion may be a duplicate." logger.error(msg) raise SimpleError(msg) - + else: - msg = ("Make sure all require parameters are provided. \\" - "Other possible errors: source may not exist in Sources table \\" \ - "or the reference may not exist in the Publications table. " ) + msg = ( + "Make sure all require parameters are provided. \\" + "Other possible errors: source may not exist in Sources table \\" + "or the reference may not exist in the Publications table. " + ) logger.error(msg) raise SimpleError(msg) -def ingest_source(db, source, reference: str = None, ra: float = None, dec: float = None, - epoch: str = None, equinox: str = None, - other_reference: str = None, comment: str = None, - raise_error: bool = True, search_db: bool = True): +def ingest_source( + db, + source, + reference: str = None, + ra: float = None, + dec: float = None, + epoch: str = None, + equinox: str = None, + other_reference: str = None, + comment: str = None, + raise_error: bool = True, + search_db: bool = True, +): """ Parameters ---------- @@ -1468,7 +1869,7 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa Database object created by astrodbkit2 sources: str Names of sources - references: str + references: str Discovery references of sources ras: float, optional Right ascensions of sources. Decimal degrees. @@ -1502,7 +1903,7 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa ra = ra dec = dec epoch = epoch - equinox = equinox + equinox = equinox logger.debug(f"coords_provided:{coords_provided}") # Find out if source is already in database or not @@ -1518,16 +1919,18 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa logger.debug(f"name_matches: {name_matches}") # Source is already in database - if len(name_matches) == 1 and search_db: + if len(name_matches) == 1 and search_db: msg = f"Not ingesting {source}. Already in database as {name_matches[0]}. \n " logger.info(msg) # Figure out if source name provided is an alternate name - db_source_matches = db.search_object(source, output_table='Sources', fuzzy_search=False) + db_source_matches = db.search_object( + source, output_table="Sources", fuzzy_search=False + ) # Try to add alternate source name to Names table if len(db_source_matches) == 0: - alt_names_data = [{'source': name_matches[0], 'other_name': source}] + alt_names_data = [{"source": name_matches[0], "other_name": source}] try: with db.engine.connect() as conn: conn.execute(db.Names.insert().values(alt_names_data)) @@ -1537,14 +1940,14 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa msg = f" Could not add {alt_names_data} to database" logger.warning(msg) if raise_error: - raise SimpleError(msg + '\n' + str(e)) + raise SimpleError(msg + "\n" + str(e)) else: return return # Source is already in database, nothing new to ingest # Multiple source matches in the database so unable to ingest source - elif len(name_matches) > 1 and search_db: + elif len(name_matches) > 1 and search_db: msg1 = f" Not ingesting {source}." - msg = f" More than one match for {source}\n {name_matches}\n" + msg = f" More than one match for {source}\n {name_matches}\n" logger.warning(msg1 + msg) if raise_error: raise SimpleError(msg) @@ -1552,9 +1955,8 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa return # No match in the database, INGEST! - elif len(name_matches) == 0 or not search_db: - - # Make sure reference is provided and in References table + elif len(name_matches) == 0 or not search_db: + # Make sure reference is provided and in References table if reference is None or ma.is_masked(reference): msg = f"Not ingesting {source}. Discovery reference is blank. \n" logger.warning(msg) @@ -1564,11 +1966,13 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa return ref_check = find_publication(db, name=reference) - logger.debug(f'ref_check: {ref_check}') + logger.debug(f"ref_check: {ref_check}") if ref_check is False: - msg = f"Skipping: {source}. Discovery reference {reference} is not in Publications table. \n" \ - f"(Add it with ingest_publication function.)" + msg = ( + f"Skipping: {source}. Discovery reference {reference} is not in Publications table. \n" + f"(Add it with ingest_publication function.)" + ) logger.warning(msg) if raise_error: raise SimpleError(msg + msg2) @@ -1589,15 +1993,17 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa return # One SIMBAD match! Using those coordinates for source. elif len(simbad_result_table) == 1: - simbad_coords = simbad_result_table['RA'][0] + ' ' + simbad_result_table['DEC'][0] + simbad_coords = ( + simbad_result_table["RA"][0] + " " + simbad_result_table["DEC"][0] + ) simbad_skycoord = SkyCoord(simbad_coords, unit=(u.hourangle, u.deg)) - ra = simbad_skycoord.to_string(style='decimal').split()[0] - dec = simbad_skycoord.to_string(style='decimal').split()[1] - epoch = '2000' # Default coordinates from SIMBAD are epoch 2000. - equinox = 'J2000' # Default frame from SIMBAD is IRCS and J2000. - msg = f"Coordinates retrieved from SIMBAD {ra}, {dec}" + ra = simbad_skycoord.to_string(style="decimal").split()[0] + dec = simbad_skycoord.to_string(style="decimal").split()[1] + epoch = "2000" # Default coordinates from SIMBAD are epoch 2000. + equinox = "J2000" # Default frame from SIMBAD is IRCS and J2000. + msg = f"Coordinates retrieved from SIMBAD {ra}, {dec}" logger.debug(msg) - else: + else: msg = f"Not ingesting {source}. Coordinates are needed and could not be retrieved from SIMBAD. \n" logger.warning(msg) if raise_error: @@ -1605,8 +2011,7 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa else: return - - # Just in case other conditionals not met + # Just in case other conditionals not met else: msg = f"Unexpected condition encountered ingesting {source}" logger.error(msg) @@ -1615,16 +2020,19 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa logger.debug(f" Ingesting {source}.") # Construct data to be added - source_data = [{'source': source, - 'ra': ra, - 'dec': dec, - 'reference': reference, - 'epoch': epoch, - 'equinox': equinox, - 'other_references': other_reference, - 'comments': comment}] - names_data = [{'source': source, - 'other_name': source}] + source_data = [ + { + "source": source, + "ra": ra, + "dec": dec, + "reference": reference, + "epoch": epoch, + "equinox": equinox, + "other_references": other_reference, + "comments": comment, + } + ] + names_data = [{"source": source, "other_name": source}] # Try to add the source to the database try: @@ -1635,9 +2043,11 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa logger.info(f"Added {source}") logger.debug(msg) except sqlalchemy.exc.IntegrityError: - msg = f"Not ingesting {source}. Not sure why. \n" \ - "The reference may not exist in Publications table. \n" \ - "Add it with ingest_publication function. \n" + msg = ( + f"Not ingesting {source}. Not sure why. \n" + "The reference may not exist in Publications table. \n" + "Add it with ingest_publication function. \n" + ) msg2 = f" {str(source_data)} " logger.warning(msg) logger.debug(msg2) @@ -1659,5 +2069,5 @@ def ingest_source(db, source, reference: str = None, ra: float = None, dec: floa raise SimpleError(msg) else: return - - return \ No newline at end of file + + return From d38f92a877d3fde258a622aeb62cf6da7072c262 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Wed, 20 Dec 2023 15:12:17 -0500 Subject: [PATCH 20/35] Cleaned script + json files --- data/cwise_j000021.45-481314.9.json | 20 ++++ data/cwise_j002029.72-153527.5.json | 20 ++++ data/cwise_j002101.45-334631.5.json | 20 ++++ data/cwise_j002159.02-451434.4.json | 20 ++++ data/cwise_j002414.30-403053.7.json | 20 ++++ data/cwise_j002658.73-260859.6.json | 20 ++++ data/cwise_j004218.67+291730.6.json | 20 ++++ data/cwise_j004945.38+423619.3.json | 20 ++++ data/cwise_j005635.48-240401.9.json | 20 ++++ data/cwise_j010424.50+133949.8.json | 20 ++++ data/cwise_j011003.61-592650.8.json | 20 ++++ data/cwise_j012715.52-300246.9.json | 20 ++++ data/cwise_j013719.04+342837.6.json | 20 ++++ data/cwise_j015905.58+105551.5.json | 20 ++++ data/cwise_j020538.20+594452.2.json | 20 ++++ data/cwise_j022454.80+152629.5.json | 20 ++++ data/cwise_j022737.75+083008.8.json | 20 ++++ data/cwise_j025645.16-335008.9.json | 20 ++++ data/cwise_j030005.73-062218.6.json | 20 ++++ data/cwise_j032853.32-112332.6.json | 20 ++++ data/cwise_j055515.83+510514.0.json | 20 ++++ data/cwise_j055909.00-384219.8.json | 20 ++++ data/cwise_j060202.17-462447.8.json | 20 ++++ data/cwise_j062000.01+344641.3.json | 20 ++++ data/cwise_j062648.96+594129.2.json | 20 ++++ data/cwise_j062727.34-002826.8.json | 20 ++++ data/cwise_j062909.21+395701.6.json | 20 ++++ data/cwise_j065752.45+163350.2.json | 20 ++++ data/cwise_j073831.31+525453.7.json | 20 ++++ data/cwise_j080912.95+741704.3.json | 20 ++++ data/cwise_j085131.24-603056.2.json | 20 ++++ data/cwise_j091558.53+254713.0.json | 20 ++++ data/cwise_j091902.55-574837.3.json | 20 ++++ data/cwise_j094352.22+335639.1.json | 20 ++++ data/cwise_j095259.54-505418.9.json | 20 ++++ data/cwise_j101017.43-242300.4.json | 20 ++++ data/cwise_j101317.51-010313.6.json | 20 ++++ data/cwise_j101523.92-111539.6.json | 20 ++++ data/cwise_j103734.29-050749.8.json | 20 ++++ data/cwise_j104053.42-355029.7.json | 20 ++++ data/cwise_j115229.05-351105.9.json | 20 ++++ data/cwise_j124033.48+460544.7.json | 20 ++++ data/cwise_j125858.20+001133.2.json | 20 ++++ data/cwise_j130329.90+512754.0.json | 20 ++++ data/cwise_j130446.64-120023.6.json | 20 ++++ data/cwise_j131355.15+223005.6.json | 20 ++++ data/cwise_j132539.70+022309.4.json | 20 ++++ data/cwise_j132857.58-063747.4.json | 20 ++++ data/cwise_j133211.93-374837.9.json | 20 ++++ data/cwise_j133427.70-273053.1.json | 20 ++++ data/cwise_j141737.21+041847.2.json | 20 ++++ data/cwise_j143935.94-804851.5.json | 20 ++++ data/cwise_j143951.51+260000.3.json | 20 ++++ data/cwise_j144057.87+380432.8.json | 20 ++++ data/cwise_j151939.68-485820.4.json | 20 ++++ data/cwise_j152742.44-121528.2.json | 20 ++++ data/cwise_j153910.07+722757.2.json | 20 ++++ data/cwise_j160654.19-103214.7.json | 20 ++++ data/cwise_j162511.27+774946.8.json | 20 ++++ data/cwise_j165143.63+695259.4.json | 20 ++++ data/cwise_j165325.10+490439.7.json | 20 ++++ data/cwise_j171308.00+644220.8.json | 20 ++++ data/cwise_j171700.57+724437.3.json | 20 ++++ data/cwise_j173859.73+200501.2.json | 20 ++++ data/cwise_j174426.85+230355.1.json | 20 ++++ data/cwise_j174509.03+380733.2.json | 20 ++++ data/cwise_j190909.62+553027.1.json | 20 ++++ data/cwise_j192425.03-532837.1.json | 20 ++++ data/cwise_j195956.16-644318.1.json | 20 ++++ data/cwise_j200304.70-142228.9.json | 20 ++++ data/cwise_j200813.66-124502.4.json | 20 ++++ data/cwise_j202934.80-791013.1.json | 20 ++++ data/cwise_j205247.07+053855.7.json | 20 ++++ data/cwise_j212342.88+655615.6.json | 20 ++++ data/cwise_j214129.80-024623.6.json | 20 ++++ data/cwise_j214213.99-204659.8.json | 20 ++++ data/cwise_j221418.15+253432.2.json | 20 ++++ data/cwise_j224452.18-362946.0.json | 20 ++++ data/cwise_j225525.22-302320.8.json | 20 ++++ data/cwise_j231824.58+052142.3.json | 20 ++++ data/cwise_j233531.55+014219.6.json | 20 ++++ data/cwise_j235827.96-521813.4.json | 20 ++++ data/cwisep_j040351.00-491605.6.json | 3 + data/cwisep_j085938.95+534908.7.json | 3 + data/ulas_j103131.49+123736.4.json | 20 ++++ data/ulas_j134403.78+083951.0.json | 3 + data/wisea_j040702.42+190945.8.json | 3 + scripts/ingests/ingest_BYW_2024.py | 135 ++++++++++----------------- 88 files changed, 1720 insertions(+), 87 deletions(-) create mode 100644 data/cwise_j000021.45-481314.9.json create mode 100644 data/cwise_j002029.72-153527.5.json create mode 100644 data/cwise_j002101.45-334631.5.json create mode 100644 data/cwise_j002159.02-451434.4.json create mode 100644 data/cwise_j002414.30-403053.7.json create mode 100644 data/cwise_j002658.73-260859.6.json create mode 100644 data/cwise_j004218.67+291730.6.json create mode 100644 data/cwise_j004945.38+423619.3.json create mode 100644 data/cwise_j005635.48-240401.9.json create mode 100644 data/cwise_j010424.50+133949.8.json create mode 100644 data/cwise_j011003.61-592650.8.json create mode 100644 data/cwise_j012715.52-300246.9.json create mode 100644 data/cwise_j013719.04+342837.6.json create mode 100644 data/cwise_j015905.58+105551.5.json create mode 100644 data/cwise_j020538.20+594452.2.json create mode 100644 data/cwise_j022454.80+152629.5.json create mode 100644 data/cwise_j022737.75+083008.8.json create mode 100644 data/cwise_j025645.16-335008.9.json create mode 100644 data/cwise_j030005.73-062218.6.json create mode 100644 data/cwise_j032853.32-112332.6.json create mode 100644 data/cwise_j055515.83+510514.0.json create mode 100644 data/cwise_j055909.00-384219.8.json create mode 100644 data/cwise_j060202.17-462447.8.json create mode 100644 data/cwise_j062000.01+344641.3.json create mode 100644 data/cwise_j062648.96+594129.2.json create mode 100644 data/cwise_j062727.34-002826.8.json create mode 100644 data/cwise_j062909.21+395701.6.json create mode 100644 data/cwise_j065752.45+163350.2.json create mode 100644 data/cwise_j073831.31+525453.7.json create mode 100644 data/cwise_j080912.95+741704.3.json create mode 100644 data/cwise_j085131.24-603056.2.json create mode 100644 data/cwise_j091558.53+254713.0.json create mode 100644 data/cwise_j091902.55-574837.3.json create mode 100644 data/cwise_j094352.22+335639.1.json create mode 100644 data/cwise_j095259.54-505418.9.json create mode 100644 data/cwise_j101017.43-242300.4.json create mode 100644 data/cwise_j101317.51-010313.6.json create mode 100644 data/cwise_j101523.92-111539.6.json create mode 100644 data/cwise_j103734.29-050749.8.json create mode 100644 data/cwise_j104053.42-355029.7.json create mode 100644 data/cwise_j115229.05-351105.9.json create mode 100644 data/cwise_j124033.48+460544.7.json create mode 100644 data/cwise_j125858.20+001133.2.json create mode 100644 data/cwise_j130329.90+512754.0.json create mode 100644 data/cwise_j130446.64-120023.6.json create mode 100644 data/cwise_j131355.15+223005.6.json create mode 100644 data/cwise_j132539.70+022309.4.json create mode 100644 data/cwise_j132857.58-063747.4.json create mode 100644 data/cwise_j133211.93-374837.9.json create mode 100644 data/cwise_j133427.70-273053.1.json create mode 100644 data/cwise_j141737.21+041847.2.json create mode 100644 data/cwise_j143935.94-804851.5.json create mode 100644 data/cwise_j143951.51+260000.3.json create mode 100644 data/cwise_j144057.87+380432.8.json create mode 100644 data/cwise_j151939.68-485820.4.json create mode 100644 data/cwise_j152742.44-121528.2.json create mode 100644 data/cwise_j153910.07+722757.2.json create mode 100644 data/cwise_j160654.19-103214.7.json create mode 100644 data/cwise_j162511.27+774946.8.json create mode 100644 data/cwise_j165143.63+695259.4.json create mode 100644 data/cwise_j165325.10+490439.7.json create mode 100644 data/cwise_j171308.00+644220.8.json create mode 100644 data/cwise_j171700.57+724437.3.json create mode 100644 data/cwise_j173859.73+200501.2.json create mode 100644 data/cwise_j174426.85+230355.1.json create mode 100644 data/cwise_j174509.03+380733.2.json create mode 100644 data/cwise_j190909.62+553027.1.json create mode 100644 data/cwise_j192425.03-532837.1.json create mode 100644 data/cwise_j195956.16-644318.1.json create mode 100644 data/cwise_j200304.70-142228.9.json create mode 100644 data/cwise_j200813.66-124502.4.json create mode 100644 data/cwise_j202934.80-791013.1.json create mode 100644 data/cwise_j205247.07+053855.7.json create mode 100644 data/cwise_j212342.88+655615.6.json create mode 100644 data/cwise_j214129.80-024623.6.json create mode 100644 data/cwise_j214213.99-204659.8.json create mode 100644 data/cwise_j221418.15+253432.2.json create mode 100644 data/cwise_j224452.18-362946.0.json create mode 100644 data/cwise_j225525.22-302320.8.json create mode 100644 data/cwise_j231824.58+052142.3.json create mode 100644 data/cwise_j233531.55+014219.6.json create mode 100644 data/cwise_j235827.96-521813.4.json create mode 100644 data/ulas_j103131.49+123736.4.json diff --git a/data/cwise_j000021.45-481314.9.json b/data/cwise_j000021.45-481314.9.json new file mode 100644 index 000000000..1eb836967 --- /dev/null +++ b/data/cwise_j000021.45-481314.9.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J000021.45-481314.9", + "ra": 0.0893808, + "dec": -48.2208077, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J000021.45-481314.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j002029.72-153527.5.json b/data/cwise_j002029.72-153527.5.json new file mode 100644 index 000000000..306e7fb2f --- /dev/null +++ b/data/cwise_j002029.72-153527.5.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J002029.72-153527.5", + "ra": 5.1238388, + "dec": -15.5909815, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J002029.72-153527.5" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j002101.45-334631.5.json b/data/cwise_j002101.45-334631.5.json new file mode 100644 index 000000000..f8576f887 --- /dev/null +++ b/data/cwise_j002101.45-334631.5.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J002101.45-334631.5", + "ra": 5.2560473, + "dec": -33.7754218, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J002101.45-334631.5" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j002159.02-451434.4.json b/data/cwise_j002159.02-451434.4.json new file mode 100644 index 000000000..22d15e0bd --- /dev/null +++ b/data/cwise_j002159.02-451434.4.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J002159.02-451434.4", + "ra": 5.4959547, + "dec": -45.2428987, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J002159.02-451434.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j002414.30-403053.7.json b/data/cwise_j002414.30-403053.7.json new file mode 100644 index 000000000..9dab9d390 --- /dev/null +++ b/data/cwise_j002414.30-403053.7.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J002414.30-403053.7", + "ra": 6.0596207, + "dec": -40.5149259, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J002414.30-403053.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j002658.73-260859.6.json b/data/cwise_j002658.73-260859.6.json new file mode 100644 index 000000000..b65abdc3d --- /dev/null +++ b/data/cwise_j002658.73-260859.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J002658.73-260859.6", + "ra": 6.7447362, + "dec": -26.1499067, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J002658.73-260859.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j004218.67+291730.6.json b/data/cwise_j004218.67+291730.6.json new file mode 100644 index 000000000..c2d20efea --- /dev/null +++ b/data/cwise_j004218.67+291730.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J004218.67+291730.6", + "ra": 10.5777904, + "dec": 29.2918566, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J004218.67+291730.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j004945.38+423619.3.json b/data/cwise_j004945.38+423619.3.json new file mode 100644 index 000000000..92d3d696d --- /dev/null +++ b/data/cwise_j004945.38+423619.3.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J004945.38+423619.3", + "ra": 12.439107, + "dec": 42.6053806, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J004945.38+423619.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j005635.48-240401.9.json b/data/cwise_j005635.48-240401.9.json new file mode 100644 index 000000000..614aadc71 --- /dev/null +++ b/data/cwise_j005635.48-240401.9.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J005635.48-240401.9", + "ra": 14.1478506, + "dec": -24.0672083, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J005635.48-240401.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j010424.50+133949.8.json b/data/cwise_j010424.50+133949.8.json new file mode 100644 index 000000000..d980cbd8f --- /dev/null +++ b/data/cwise_j010424.50+133949.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J010424.50+133949.8", + "ra": 16.1020944, + "dec": 13.6638352, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J010424.50+133949.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j011003.61-592650.8.json b/data/cwise_j011003.61-592650.8.json new file mode 100644 index 000000000..a1ad9d2cf --- /dev/null +++ b/data/cwise_j011003.61-592650.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J011003.61-592650.8", + "ra": 17.5150429, + "dec": -59.4474481, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J011003.61-592650.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j012715.52-300246.9.json b/data/cwise_j012715.52-300246.9.json new file mode 100644 index 000000000..a601125fb --- /dev/null +++ b/data/cwise_j012715.52-300246.9.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J012715.52-300246.9", + "ra": 21.8146967, + "dec": -30.0463841, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J012715.52-300246.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j013719.04+342837.6.json b/data/cwise_j013719.04+342837.6.json new file mode 100644 index 000000000..e2979731a --- /dev/null +++ b/data/cwise_j013719.04+342837.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J013719.04+342837.6", + "ra": 24.3293435, + "dec": 34.4771182, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J013719.04+342837.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j015905.58+105551.5.json b/data/cwise_j015905.58+105551.5.json new file mode 100644 index 000000000..e1b7a53da --- /dev/null +++ b/data/cwise_j015905.58+105551.5.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J015905.58+105551.5", + "ra": 29.773276, + "dec": 10.9309815, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J015905.58+105551.5" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j020538.20+594452.2.json b/data/cwise_j020538.20+594452.2.json new file mode 100644 index 000000000..c30e7114e --- /dev/null +++ b/data/cwise_j020538.20+594452.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J020538.20+594452.2", + "ra": 31.40993946, + "dec": 59.74890954, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J020538.20+594452.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j022454.80+152629.5.json b/data/cwise_j022454.80+152629.5.json new file mode 100644 index 000000000..b64f4c6a6 --- /dev/null +++ b/data/cwise_j022454.80+152629.5.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J022454.80+152629.5", + "ra": 36.2283563, + "dec": 15.441534, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J022454.80+152629.5" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j022737.75+083008.8.json b/data/cwise_j022737.75+083008.8.json new file mode 100644 index 000000000..414b00d35 --- /dev/null +++ b/data/cwise_j022737.75+083008.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J022737.75+083008.8", + "ra": 36.9072967, + "dec": 8.5024629, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J022737.75+083008.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j025645.16-335008.9.json b/data/cwise_j025645.16-335008.9.json new file mode 100644 index 000000000..4bc015178 --- /dev/null +++ b/data/cwise_j025645.16-335008.9.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J025645.16-335008.9", + "ra": 44.188202, + "dec": -33.8358082, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J025645.16-335008.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j030005.73-062218.6.json b/data/cwise_j030005.73-062218.6.json new file mode 100644 index 000000000..2e046ee29 --- /dev/null +++ b/data/cwise_j030005.73-062218.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J030005.73-062218.6", + "ra": 45.0238923, + "dec": -6.371848, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J030005.73-062218.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j032853.32-112332.6.json b/data/cwise_j032853.32-112332.6.json new file mode 100644 index 000000000..b6eb589cd --- /dev/null +++ b/data/cwise_j032853.32-112332.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J032853.32-112332.6", + "ra": 52.2221958, + "dec": -11.3924108, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J032853.32-112332.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j055515.83+510514.0.json b/data/cwise_j055515.83+510514.0.json new file mode 100644 index 000000000..3e76fe9ae --- /dev/null +++ b/data/cwise_j055515.83+510514.0.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J055515.83+510514.0", + "ra": 88.8159856, + "dec": 51.0872307, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J055515.83+510514.0" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j055909.00-384219.8.json b/data/cwise_j055909.00-384219.8.json new file mode 100644 index 000000000..147eda74f --- /dev/null +++ b/data/cwise_j055909.00-384219.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J055909.00-384219.8", + "ra": 89.787502, + "dec": -38.7055027, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J055909.00-384219.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j060202.17-462447.8.json b/data/cwise_j060202.17-462447.8.json new file mode 100644 index 000000000..8d01af33f --- /dev/null +++ b/data/cwise_j060202.17-462447.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J060202.17-462447.8", + "ra": 90.509052, + "dec": -46.4132863, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J060202.17-462447.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j062000.01+344641.3.json b/data/cwise_j062000.01+344641.3.json new file mode 100644 index 000000000..3bee1fa8b --- /dev/null +++ b/data/cwise_j062000.01+344641.3.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J062000.01+344641.3", + "ra": 95.0000552, + "dec": 34.7781567, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J062000.01+344641.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j062648.96+594129.2.json b/data/cwise_j062648.96+594129.2.json new file mode 100644 index 000000000..e8aabeac8 --- /dev/null +++ b/data/cwise_j062648.96+594129.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J062648.96+594129.2", + "ra": 96.7040383, + "dec": 59.6914571, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J062648.96+594129.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j062727.34-002826.8.json b/data/cwise_j062727.34-002826.8.json new file mode 100644 index 000000000..faafab9e3 --- /dev/null +++ b/data/cwise_j062727.34-002826.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J062727.34-002826.8", + "ra": 96.8639415, + "dec": -0.474135, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J062727.34-002826.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j062909.21+395701.6.json b/data/cwise_j062909.21+395701.6.json new file mode 100644 index 000000000..284b643e2 --- /dev/null +++ b/data/cwise_j062909.21+395701.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J062909.21+395701.6", + "ra": 97.2884074, + "dec": 39.950463, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J062909.21+395701.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j065752.45+163350.2.json b/data/cwise_j065752.45+163350.2.json new file mode 100644 index 000000000..d56e5d859 --- /dev/null +++ b/data/cwise_j065752.45+163350.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J065752.45+163350.2", + "ra": 104.4685682, + "dec": 16.5639658, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J065752.45+163350.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j073831.31+525453.7.json b/data/cwise_j073831.31+525453.7.json new file mode 100644 index 000000000..4a52eec10 --- /dev/null +++ b/data/cwise_j073831.31+525453.7.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J073831.31+525453.7", + "ra": 114.6304785, + "dec": 52.9149251, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J073831.31+525453.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j080912.95+741704.3.json b/data/cwise_j080912.95+741704.3.json new file mode 100644 index 000000000..1d24bee0f --- /dev/null +++ b/data/cwise_j080912.95+741704.3.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J080912.95+741704.3", + "ra": 122.3039975, + "dec": 74.2845438, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J080912.95+741704.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j085131.24-603056.2.json b/data/cwise_j085131.24-603056.2.json new file mode 100644 index 000000000..f884217bc --- /dev/null +++ b/data/cwise_j085131.24-603056.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J085131.24-603056.2", + "ra": 132.8801846, + "dec": -60.5156128, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J085131.24-603056.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j091558.53+254713.0.json b/data/cwise_j091558.53+254713.0.json new file mode 100644 index 000000000..009f09ac2 --- /dev/null +++ b/data/cwise_j091558.53+254713.0.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J091558.53+254713.0", + "ra": 138.9939062, + "dec": 25.7869589, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J091558.53+254713.0" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j091902.55-574837.3.json b/data/cwise_j091902.55-574837.3.json new file mode 100644 index 000000000..98c0605da --- /dev/null +++ b/data/cwise_j091902.55-574837.3.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J091902.55-574837.3", + "ra": 139.7606504, + "dec": -57.8103837, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J091902.55-574837.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j094352.22+335639.1.json b/data/cwise_j094352.22+335639.1.json new file mode 100644 index 000000000..d078b80cb --- /dev/null +++ b/data/cwise_j094352.22+335639.1.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J094352.22+335639.1", + "ra": 145.9676104, + "dec": 33.9442094, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J094352.22+335639.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j095259.54-505418.9.json b/data/cwise_j095259.54-505418.9.json new file mode 100644 index 000000000..ab6487182 --- /dev/null +++ b/data/cwise_j095259.54-505418.9.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J095259.54-505418.9", + "ra": 148.2480941, + "dec": -50.905256, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J095259.54-505418.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j101017.43-242300.4.json b/data/cwise_j101017.43-242300.4.json new file mode 100644 index 000000000..ae92cc8bc --- /dev/null +++ b/data/cwise_j101017.43-242300.4.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J101017.43-242300.4", + "ra": 152.5726326, + "dec": -24.3834649, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J101017.43-242300.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j101317.51-010313.6.json b/data/cwise_j101317.51-010313.6.json new file mode 100644 index 000000000..1a77fc1fa --- /dev/null +++ b/data/cwise_j101317.51-010313.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J101317.51-010313.6", + "ra": 153.3229961, + "dec": -1.0537976, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J101317.51-010313.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j101523.92-111539.6.json b/data/cwise_j101523.92-111539.6.json new file mode 100644 index 000000000..c7e862dd7 --- /dev/null +++ b/data/cwise_j101523.92-111539.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J101523.92-111539.6", + "ra": 153.849691, + "dec": -11.2610035, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J101523.92-111539.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j103734.29-050749.8.json b/data/cwise_j103734.29-050749.8.json new file mode 100644 index 000000000..58dcf64e3 --- /dev/null +++ b/data/cwise_j103734.29-050749.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J103734.29-050749.8", + "ra": 159.3929093, + "dec": -5.1305155, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J103734.29-050749.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j104053.42-355029.7.json b/data/cwise_j104053.42-355029.7.json new file mode 100644 index 000000000..60e315937 --- /dev/null +++ b/data/cwise_j104053.42-355029.7.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J104053.42-355029.7", + "ra": 160.2226104, + "dec": -35.8415843, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J104053.42-355029.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j115229.05-351105.9.json b/data/cwise_j115229.05-351105.9.json new file mode 100644 index 000000000..fabeb376f --- /dev/null +++ b/data/cwise_j115229.05-351105.9.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J115229.05-351105.9", + "ra": 178.1210618, + "dec": -35.1849737, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J115229.05-351105.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j124033.48+460544.7.json b/data/cwise_j124033.48+460544.7.json new file mode 100644 index 000000000..ed64555b5 --- /dev/null +++ b/data/cwise_j124033.48+460544.7.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J124033.48+460544.7", + "ra": 190.1395201, + "dec": 46.0957506, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J124033.48+460544.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j125858.20+001133.2.json b/data/cwise_j125858.20+001133.2.json new file mode 100644 index 000000000..80b50fc0b --- /dev/null +++ b/data/cwise_j125858.20+001133.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J125858.20+001133.2", + "ra": 194.7425134, + "dec": 0.1925822, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J125858.20+001133.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j130329.90+512754.0.json b/data/cwise_j130329.90+512754.0.json new file mode 100644 index 000000000..a70a2575b --- /dev/null +++ b/data/cwise_j130329.90+512754.0.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J130329.90+512754.0", + "ra": 195.8745856, + "dec": 51.4650208, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J130329.90+512754.0" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j130446.64-120023.6.json b/data/cwise_j130446.64-120023.6.json new file mode 100644 index 000000000..5bc7a76da --- /dev/null +++ b/data/cwise_j130446.64-120023.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J130446.64-120023.6", + "ra": 196.1944474, + "dec": -12.00636689, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J130446.64-120023.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j131355.15+223005.6.json b/data/cwise_j131355.15+223005.6.json new file mode 100644 index 000000000..ca605917a --- /dev/null +++ b/data/cwise_j131355.15+223005.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J131355.15+223005.6", + "ra": 198.4798067, + "dec": 22.5015715, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J131355.15+223005.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j132539.70+022309.4.json b/data/cwise_j132539.70+022309.4.json new file mode 100644 index 000000000..906db9ad1 --- /dev/null +++ b/data/cwise_j132539.70+022309.4.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J132539.70+022309.4", + "ra": 201.4154343, + "dec": 2.3859476, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J132539.70+022309.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j132857.58-063747.4.json b/data/cwise_j132857.58-063747.4.json new file mode 100644 index 000000000..58dbdd0c3 --- /dev/null +++ b/data/cwise_j132857.58-063747.4.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J132857.58-063747.4", + "ra": 202.2399559, + "dec": -6.6298609, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J132857.58-063747.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j133211.93-374837.9.json b/data/cwise_j133211.93-374837.9.json new file mode 100644 index 000000000..9adbe5619 --- /dev/null +++ b/data/cwise_j133211.93-374837.9.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J133211.93-374837.9", + "ra": 203.049724, + "dec": -37.8105289, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J133211.93-374837.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j133427.70-273053.1.json b/data/cwise_j133427.70-273053.1.json new file mode 100644 index 000000000..6da66825d --- /dev/null +++ b/data/cwise_j133427.70-273053.1.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J133427.70-273053.1", + "ra": 203.6154266, + "dec": -27.5147656, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J133427.70-273053.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j141737.21+041847.2.json b/data/cwise_j141737.21+041847.2.json new file mode 100644 index 000000000..8342deff5 --- /dev/null +++ b/data/cwise_j141737.21+041847.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J141737.21+041847.2", + "ra": 214.4050608, + "dec": 4.3131162, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J141737.21+041847.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j143935.94-804851.5.json b/data/cwise_j143935.94-804851.5.json new file mode 100644 index 000000000..c73249076 --- /dev/null +++ b/data/cwise_j143935.94-804851.5.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J143935.94-804851.5", + "ra": 219.8997509, + "dec": -80.8143328, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J143935.94-804851.5" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j143951.51+260000.3.json b/data/cwise_j143951.51+260000.3.json new file mode 100644 index 000000000..f64bd49e7 --- /dev/null +++ b/data/cwise_j143951.51+260000.3.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J143951.51+260000.3", + "ra": 219.964641, + "dec": 26.0000984, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J143951.51+260000.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j144057.87+380432.8.json b/data/cwise_j144057.87+380432.8.json new file mode 100644 index 000000000..e2432757f --- /dev/null +++ b/data/cwise_j144057.87+380432.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J144057.87+380432.8", + "ra": 220.2411279, + "dec": 38.0757925, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J144057.87+380432.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j151939.68-485820.4.json b/data/cwise_j151939.68-485820.4.json new file mode 100644 index 000000000..6f7ce264a --- /dev/null +++ b/data/cwise_j151939.68-485820.4.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J151939.68-485820.4", + "ra": 229.9153563, + "dec": -48.972359, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J151939.68-485820.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j152742.44-121528.2.json b/data/cwise_j152742.44-121528.2.json new file mode 100644 index 000000000..af216a6b6 --- /dev/null +++ b/data/cwise_j152742.44-121528.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J152742.44-121528.2", + "ra": 231.9268706, + "dec": -12.2578539, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J152742.44-121528.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j153910.07+722757.2.json b/data/cwise_j153910.07+722757.2.json new file mode 100644 index 000000000..3205a747c --- /dev/null +++ b/data/cwise_j153910.07+722757.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J153910.07+722757.2", + "ra": 234.7919959, + "dec": 72.4658937, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J153910.07+722757.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j160654.19-103214.7.json b/data/cwise_j160654.19-103214.7.json new file mode 100644 index 000000000..190898030 --- /dev/null +++ b/data/cwise_j160654.19-103214.7.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J160654.19-103214.7", + "ra": 241.7258072, + "dec": -10.5374259, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J160654.19-103214.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j162511.27+774946.8.json b/data/cwise_j162511.27+774946.8.json new file mode 100644 index 000000000..0f76780e6 --- /dev/null +++ b/data/cwise_j162511.27+774946.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J162511.27+774946.8", + "ra": 246.2969812, + "dec": 77.8296817, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J162511.27+774946.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j165143.63+695259.4.json b/data/cwise_j165143.63+695259.4.json new file mode 100644 index 000000000..9722bad31 --- /dev/null +++ b/data/cwise_j165143.63+695259.4.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J165143.63+695259.4", + "ra": 252.9318288, + "dec": 69.8831753, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J165143.63+695259.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j165325.10+490439.7.json b/data/cwise_j165325.10+490439.7.json new file mode 100644 index 000000000..494f5a971 --- /dev/null +++ b/data/cwise_j165325.10+490439.7.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J165325.10+490439.7", + "ra": 253.3545821, + "dec": 49.0777014, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J165325.10+490439.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j171308.00+644220.8.json b/data/cwise_j171308.00+644220.8.json new file mode 100644 index 000000000..fc08f5678 --- /dev/null +++ b/data/cwise_j171308.00+644220.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J171308.00+644220.8", + "ra": 258.2833578, + "dec": 64.7058005, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J171308.00+644220.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j171700.57+724437.3.json b/data/cwise_j171700.57+724437.3.json new file mode 100644 index 000000000..70ada9956 --- /dev/null +++ b/data/cwise_j171700.57+724437.3.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J171700.57+724437.3", + "ra": 259.2523353, + "dec": 72.74369143, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J171700.57+724437.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j173859.73+200501.2.json b/data/cwise_j173859.73+200501.2.json new file mode 100644 index 000000000..d26143a91 --- /dev/null +++ b/data/cwise_j173859.73+200501.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J173859.73+200501.2", + "ra": 264.7489139, + "dec": 20.0836743, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J173859.73+200501.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j174426.85+230355.1.json b/data/cwise_j174426.85+230355.1.json new file mode 100644 index 000000000..4947273e1 --- /dev/null +++ b/data/cwise_j174426.85+230355.1.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J174426.85+230355.1", + "ra": 266.1118811, + "dec": 23.0653259, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J174426.85+230355.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j174509.03+380733.2.json b/data/cwise_j174509.03+380733.2.json new file mode 100644 index 000000000..6a217c8d9 --- /dev/null +++ b/data/cwise_j174509.03+380733.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J174509.03+380733.2", + "ra": 266.2876616, + "dec": 38.1259034, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J174509.03+380733.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j190909.62+553027.1.json b/data/cwise_j190909.62+553027.1.json new file mode 100644 index 000000000..85c4661b6 --- /dev/null +++ b/data/cwise_j190909.62+553027.1.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J190909.62+553027.1", + "ra": 287.2900976, + "dec": 55.5075306, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J190909.62+553027.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j192425.03-532837.1.json b/data/cwise_j192425.03-532837.1.json new file mode 100644 index 000000000..14f254256 --- /dev/null +++ b/data/cwise_j192425.03-532837.1.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J192425.03-532837.1", + "ra": 291.1043308, + "dec": -53.4769769, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J192425.03-532837.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j195956.16-644318.1.json b/data/cwise_j195956.16-644318.1.json new file mode 100644 index 000000000..d7cf70704 --- /dev/null +++ b/data/cwise_j195956.16-644318.1.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J195956.16-644318.1", + "ra": 299.9840118, + "dec": -64.7217168, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J195956.16-644318.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j200304.70-142228.9.json b/data/cwise_j200304.70-142228.9.json new file mode 100644 index 000000000..c0b3d68cc --- /dev/null +++ b/data/cwise_j200304.70-142228.9.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J200304.70-142228.9", + "ra": 300.7696229, + "dec": -14.3747026, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J200304.70-142228.9" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j200813.66-124502.4.json b/data/cwise_j200813.66-124502.4.json new file mode 100644 index 000000000..6db0ba784 --- /dev/null +++ b/data/cwise_j200813.66-124502.4.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J200813.66-124502.4", + "ra": 302.0569331, + "dec": -12.7506846, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J200813.66-124502.4" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j202934.80-791013.1.json b/data/cwise_j202934.80-791013.1.json new file mode 100644 index 000000000..cde5c69f6 --- /dev/null +++ b/data/cwise_j202934.80-791013.1.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J202934.80-791013.1", + "ra": 307.3950501, + "dec": -79.17033373, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J202934.80-791013.1" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j205247.07+053855.7.json b/data/cwise_j205247.07+053855.7.json new file mode 100644 index 000000000..de6ed9f89 --- /dev/null +++ b/data/cwise_j205247.07+053855.7.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J205247.07+053855.7", + "ra": 313.196131, + "dec": 5.6488179, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J205247.07+053855.7" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j212342.88+655615.6.json b/data/cwise_j212342.88+655615.6.json new file mode 100644 index 000000000..0bf6a8e2d --- /dev/null +++ b/data/cwise_j212342.88+655615.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J212342.88+655615.6", + "ra": 320.9286959, + "dec": 65.9376939, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J212342.88+655615.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j214129.80-024623.6.json b/data/cwise_j214129.80-024623.6.json new file mode 100644 index 000000000..a0f37a135 --- /dev/null +++ b/data/cwise_j214129.80-024623.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J214129.80-024623.6", + "ra": 325.3741677, + "dec": -2.7732463, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J214129.80-024623.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j214213.99-204659.8.json b/data/cwise_j214213.99-204659.8.json new file mode 100644 index 000000000..b43598f41 --- /dev/null +++ b/data/cwise_j214213.99-204659.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J214213.99-204659.8", + "ra": 325.558323, + "dec": -20.7832923, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J214213.99-204659.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j221418.15+253432.2.json b/data/cwise_j221418.15+253432.2.json new file mode 100644 index 000000000..4fb39ec77 --- /dev/null +++ b/data/cwise_j221418.15+253432.2.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J221418.15+253432.2", + "ra": 333.5756522, + "dec": 25.5756384, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J221418.15+253432.2" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j224452.18-362946.0.json b/data/cwise_j224452.18-362946.0.json new file mode 100644 index 000000000..3e07fa4e0 --- /dev/null +++ b/data/cwise_j224452.18-362946.0.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J224452.18-362946.0", + "ra": 341.2174357, + "dec": -36.4961287, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J224452.18-362946.0" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j225525.22-302320.8.json b/data/cwise_j225525.22-302320.8.json new file mode 100644 index 000000000..92ba40af8 --- /dev/null +++ b/data/cwise_j225525.22-302320.8.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J225525.22-302320.8", + "ra": 343.8550923, + "dec": -30.3891146, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J225525.22-302320.8" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j231824.58+052142.3.json b/data/cwise_j231824.58+052142.3.json new file mode 100644 index 000000000..f652b83f0 --- /dev/null +++ b/data/cwise_j231824.58+052142.3.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J231824.58+052142.3", + "ra": 349.6024391, + "dec": 5.3617605, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J231824.58+052142.3" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j233531.55+014219.6.json b/data/cwise_j233531.55+014219.6.json new file mode 100644 index 000000000..a71345823 --- /dev/null +++ b/data/cwise_j233531.55+014219.6.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J233531.55+014219.6", + "ra": 353.8814851, + "dec": 1.7054667, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J233531.55+014219.6" + } + ] +} \ No newline at end of file diff --git a/data/cwise_j235827.96-521813.4.json b/data/cwise_j235827.96-521813.4.json new file mode 100644 index 000000000..c559c35ae --- /dev/null +++ b/data/cwise_j235827.96-521813.4.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "CWISE J235827.96-521813.4", + "ra": 359.6165101, + "dec": -52.3037451, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Roth", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "CWISE J235827.96-521813.4" + } + ] +} \ No newline at end of file diff --git a/data/cwisep_j040351.00-491605.6.json b/data/cwisep_j040351.00-491605.6.json index 222ebc0fc..8a0920ae4 100644 --- a/data/cwisep_j040351.00-491605.6.json +++ b/data/cwisep_j040351.00-491605.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE J040351.12-491605.4" + }, { "other_name": "CWISEP J040351.00-491605.6" } diff --git a/data/cwisep_j085938.95+534908.7.json b/data/cwisep_j085938.95+534908.7.json index b0d9b0d89..409a9abe4 100644 --- a/data/cwisep_j085938.95+534908.7.json +++ b/data/cwisep_j085938.95+534908.7.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE J085938.91+534908.4" + }, { "other_name": "CWISEP J085938.95+534908.7" } diff --git a/data/ulas_j103131.49+123736.4.json b/data/ulas_j103131.49+123736.4.json new file mode 100644 index 000000000..bf2c4c878 --- /dev/null +++ b/data/ulas_j103131.49+123736.4.json @@ -0,0 +1,20 @@ +{ + "Sources": [ + { + "source": "ULAS J103131.49+123736.4", + "ra": 157.8811199, + "dec": 12.6267923, + "epoch": 2015.4041, + "equinox": "ICRS", + "shortname": null, + "reference": "Skrz16", + "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + } + ], + "Names": [ + { + "other_name": "ULAS J103131.49+123736.4" + } + ] +} \ No newline at end of file diff --git a/data/ulas_j134403.78+083951.0.json b/data/ulas_j134403.78+083951.0.json index 594e0c29d..cb06fb469 100644 --- a/data/ulas_j134403.78+083951.0.json +++ b/data/ulas_j134403.78+083951.0.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "SDSS J134403.83+083950.9" + }, { "other_name": "ULAS J134403.78+083951.0" } diff --git a/data/wisea_j040702.42+190945.8.json b/data/wisea_j040702.42+190945.8.json index 39e3d19e0..973331fd8 100644 --- a/data/wisea_j040702.42+190945.8.json +++ b/data/wisea_j040702.42+190945.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE J040702.50+190944.6" + }, { "other_name": "WISEA J040702.42+190945.8" } diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 6e849c61f..3a10d2b21 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -1,108 +1,69 @@ from scripts.ingests.ingest_utils import * from scripts.ingests.utils import * from astropy.io import ascii -import astropy.units as u -import requests -from io import StringIO -from astropy.coordinates import Angle -SAVE_DB = False # save the data files in addition to modifying the .db file +SAVE_DB = True # save the data files in addition to modifying the .db file RECREATE_DB = True # recreates the .db file from the data files # LOAD THE DATABASE db = load_simpledb('SIMPLE.db', recreatedb=RECREATE_DB) -#Attempt 1 -#csv_file_path = "C:\Users\LRamon\Documents\Austin-BYW-Benchmark-Sources.-.Sheet1.csv" - -#with open(csv_file_path, 'r') as file: - # Create a CSV reader object with dictionary-like behavior - #csv_reader = csv.DictReader(file) - - # Iterate through each row in the CSV file - #for row in csv_reader: - # Each 'row' variable represents a dictionary with column names as keys - # print(row) - - - -# live google sheet (Attempt 2) -#sheet_id = '1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs' -#df = pd.read_csv(f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv") -#print(df) - - -#(Attempt 3) +# Load Google sheet sheet_id = '1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs' link = f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv" -# Fetch the content from the URL -response = requests.get(link) -data = response.text - -# Use io.StringIO to create a file-like object -data_file = StringIO(data) - -#define the column names -columns = ['source', 'ra', 'dec', 'epoch', 'equinox', 'shortname', 'reference', 'other_ref', 'comments'] - #read the csv data into an astropy table #ascii.read attempts to read data from local files rather from URLs so using a library like requests helps get data and create object that can be passed to ascii.read -byw_table = ascii.read(link, format='csv', data_start=1, data_end=90, header_start=1, names=columns, guess=False, fast_reader=False, delimiter=',') - -#data_columns = ['Source', 'RA', 'Dec', 'Epoch', 'Equinox', 'Shortname', 'Reference', 'Other_ref', 'Comments'] # columns with wanted data values -#byw_table.rename_columns(data_columns) +byw_table = ascii.read(link, format='csv', data_start=1, header_start=0, guess=False, fast_reader=False, delimiter=',') #print result astropy table -print(byw_table) - -#Loop over the byw table and print source name ra, dec, at each row. -#Change all references to Roth; search and replace -#Individually ingest other_ref sources as publications, then use those 3 to modify the 3 sources to be ingested in the loop - - -#Individual source ingest attempt... -def ingest_source_one(db): - - ingest_source(db, source = "CWISE J000021.45-481314.9", - reference = "Roth", - ra = 0.0893808, - dec = -48.2208077, - epoch = 2015.4041, - equinox = "ICRS", - raise_error=True, - search_db=True, - other_reference=None, - comment=None) +print(byw_table.info) + + +#Loop through each row in byw table and print data: source name ra, dec. +def ingest_all_sources(db): + for row in byw_table: # skip the header row - [1:10]runs only first 10 rows + + # Print byw source information + print("BYW Source Information:") + #for key, value in row_dict.items(): + #print(f"{key}: {value}") + + for col_name in row.colnames: + print(f"{col_name}: {row[col_name]}") + + + ingest_source(db, source = row['Source'], + reference= row['Reference'], + ra= row['RA'], + dec= row['Dec'], + comment= row['Comments'], + epoch= row['Epoch'], + equinox= row['Equinox'], + other_reference= row['Other_ref'], + raise_error=True, + search_db=True) + #print(row['ra']) + #print(row_dict['ra']) + + # Add a separator between rows for better readability + print("-" * 20) + + +#Ingested other_ref sources as publications +# Skrzypek et al. 2016, Marocco et al. 2015(Online Catalog), Kirkpatrick et al. 2021 +#Ingest reference name: Skrz16, Maro15, Kirk21 - ingest_source(db, source = "CWISE J002029.72-153527.5", - reference = "Roth", - ra = 5.1238388, - dec = -15.5909815, - epoch = 2015.4041, - equinox = "ICRS", - raise_error=True, - search_db=True, - other_reference=None, - comment=None) - - -#All sources ingest attempt... -#def ingest_all_sources(db): -# ingest_sources(db, sources = [" "], -# references=None, -# ras=None, -# decs=None, -# comments=None, -# epochs=None, -# equinoxes=None, -# other_references=None, -# raise_error=True, -# search_db=True) +def add_publication(db): + + ingest_publication(db, doi = "10.26093/cds/vizier.35890049", bibcode = "2016yCat..35890049S") + + ingest_publication(db, doi = "10.1093/mnras/stv530", bibcode = "2015MNRAS.449.3651M") + + ingest_publication(db, doi = "10.3847/1538-4365/abd107", bibcode = "2021ApJS..253....7K") #calling functions -ingest_source_one(db) -OR -#ingest_all_sources(db) +#add_publication(db) +ingest_all_sources(db) db.inventory('CWISE J000021.45-481314.9', pretty_print=True) From 7450f6a84b995a47d8c6ae4e5e88c737bfe273d9 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Wed, 20 Dec 2023 15:37:52 -0500 Subject: [PATCH 21/35] Updated suggestions --- scripts/ingests/ingest_BYW_2024.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 3a10d2b21..3033ceac7 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -35,11 +35,9 @@ def ingest_all_sources(db): ingest_source(db, source = row['Source'], reference= row['Reference'], ra= row['RA'], - dec= row['Dec'], - comment= row['Comments'], + dec= row['Dec'], epoch= row['Epoch'], - equinox= row['Equinox'], - other_reference= row['Other_ref'], + equinox= row['Equinox'], raise_error=True, search_db=True) #print(row['ra']) From 6f4f05b0d9e5c3e17169e81c0c640b557ef841fb Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Wed, 20 Dec 2023 16:00:59 -0500 Subject: [PATCH 22/35] New Roth test --- tests/test_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index f4976813e..22b48cda3 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -116,6 +116,10 @@ def test_discovery_references(db): t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() assert len(t) == 91, f'found {len(t)} discovery reference entries for {ref}' + ref = 'Roth' + t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() + assert len(t) == 83, f'found {len(t)} discovery reference entries for {ref}' + def test_proper_motion_refs(db): """ From 27346f68286f28398462a4f23870142c10ab9b09 Mon Sep 17 00:00:00 2001 From: kelle Date: Wed, 20 Dec 2023 16:15:18 -0500 Subject: [PATCH 23/35] mostly black and pylance fixes --- scripts/ingests/ingest_utils.py | 15 +- scripts/ingests/utils.py | 251 +++++++---- tests/test_integrity.py | 710 ++++++++++++++++++++------------ tests/test_utils.py | 452 ++++++++++++++------ 4 files changed, 949 insertions(+), 479 deletions(-) diff --git a/scripts/ingests/ingest_utils.py b/scripts/ingests/ingest_utils.py index c7dd87cef..3e0b80969 100644 --- a/scripts/ingests/ingest_utils.py +++ b/scripts/ingests/ingest_utils.py @@ -18,7 +18,14 @@ import logging from sqlalchemy import or_, and_ import sqlalchemy.exc -from scripts.ingests.utils import SimpleError, find_source_in_db, find_publication, check_internet_connection +from scripts.ingests.utils import ( + SimpleError, + find_source_in_db, + find_publication, + check_internet_connection, +) + +# TODO: add __all__ to all modules logger = logging.getLogger("SIMPLE") @@ -647,7 +654,7 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): n_added = 0 # loop through sources with parallax data to ingest - for i, source in enumerate(sources): + for i, source in enumerate(sources): db_name = find_source_in_db(db, source) if len(db_name) != 1: @@ -664,7 +671,7 @@ def ingest_parallaxes(db, sources, plxs, plx_errs, plx_refs, comments=None): ) if source_plx_data is None or len(source_plx_data) == 0: - # if there's no other measurements in the database, + # if there's no other measurements in the database, # set new data Adopted = True adopted = True # old_adopted = None # not used @@ -865,7 +872,7 @@ def ingest_proper_motions( conn.commit() else: adopted = False - # if no previous adopted measurement, + # if no previous adopted measurement, # set adopted to the measurement with the smallest errors if ( not adopted diff --git a/scripts/ingests/utils.py b/scripts/ingests/utils.py index b926f08b5..84dffe8ca 100644 --- a/scripts/ingests/utils.py +++ b/scripts/ingests/utils.py @@ -21,20 +21,30 @@ import socket from scripts import REFERENCE_TABLES -warnings.filterwarnings("ignore", module='astroquery.simbad') -logger = logging.getLogger('SIMPLE') +# TODO: add __all__ to limit what is imported with * +__all__ = ["SimpleError", + "load_simpledb", + "find_source_in_db", + "find_publication", + "ingest_publication", + "check_internet_connection"] + +warnings.filterwarnings("ignore", module="astroquery.simbad") +logger = logging.getLogger("SIMPLE") # Logger setup # This will stream all logger messages to the standard output and apply formatting for that logger.propagate = False # prevents duplicated logging messages -LOGFORMAT = logging.Formatter('%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %I:%M:%S%p') +LOGFORMAT = logging.Formatter( + "%(asctime)s %(levelname)s: %(message)s", datefmt="%m/%d/%Y %I:%M:%S%p" +) ch = logging.StreamHandler(stream=sys.stdout) ch.setFormatter(LOGFORMAT) # To prevent duplicate handlers, only add if they haven't been set previously if not len(logger.handlers): logger.addHandler(ch) logger.setLevel(logging.INFO) - + class SimpleError(Exception): pass @@ -56,29 +66,43 @@ def load_simpledb(db_file, recreatedb=True, reference_tables=REFERENCE_TABLES): # Utility function to load the database db_file_path = Path(db_file) - db_connection_string = 'sqlite:///' + db_file + db_connection_string = "sqlite:///" + db_file if recreatedb and db_file_path.exists(): os.remove(db_file) # removes the current .db file if one already exists if not db_file_path.exists(): - try: # Use fancy in-memory database, if supported by astrodbkit2 - db = Database('sqlite://', reference_tables=REFERENCE_TABLES) # creates and connects to a temporary in-memory database - db.load_database('data/') # loads the data from the data files into the database + try: # Use fancy in-memory database, if supported by astrodbkit2 + db = Database( + "sqlite://", reference_tables=REFERENCE_TABLES + ) # creates and connects to a temporary in-memory database + db.load_database( + "data/" + ) # loads the data from the data files into the database db.dump_sqlite(db_file) # dump in-memory database to file - db = Database(db_connection_string, reference_tables=REFERENCE_TABLES) # replace database object with new file version + db = Database( + db_connection_string, reference_tables=REFERENCE_TABLES + ) # replace database object with new file version except RuntimeError: # use in-file database - create_database(db_connection_string) # creates empty database based on the simple schema - db = Database(db_connection_string, reference_tables=REFERENCE_TABLES) # connects to the empty database - db.load_database('data/') # loads the data from the data files into the database + create_database( + db_connection_string + ) # creates empty database based on the simple schema + db = Database( + db_connection_string, reference_tables=REFERENCE_TABLES + ) # connects to the empty database + db.load_database( + "data/" + ) # loads the data from the data files into the database else: - db = Database(db_connection_string, reference_tables=REFERENCE_TABLES) # if database already exists, connects to .db file + db = Database( + db_connection_string, reference_tables=REFERENCE_TABLES + ) # if database already exists, connects to .db file return db -def find_source_in_db(db, source, ra=None, dec=None, search_radius=60.): +def find_source_in_db(db, source, ra=None, dec=None, search_radius=60.0): """ Find a source in the database given a source name and optional coordinates. @@ -113,54 +137,64 @@ def find_source_in_db(db, source, ra=None, dec=None, search_radius=60.): source = source.strip() - logger.debug(f'{source}: Searching for match in database.') + logger.debug(f"{source}: Searching for match in database.") - db_name_matches = db.search_object(source, output_table='Sources', fuzzy_search=False, verbose=False) + db_name_matches = db.search_object( + source, output_table="Sources", fuzzy_search=False, verbose=False + ) # NO MATCHES # If no matches, try fuzzy search if len(db_name_matches) == 0: logger.debug(f"{source}: No name matches, trying fuzzy search") - db_name_matches = db.search_object(source, output_table='Sources', fuzzy_search=True, verbose=False) + db_name_matches = db.search_object( + source, output_table="Sources", fuzzy_search=True, verbose=False + ) # If still no matches, try to resolve the name with Simbad if len(db_name_matches) == 0: logger.debug(f"{source}: No name matches, trying Simbad search") - db_name_matches = db.search_object(source, resolve_simbad=True, fuzzy_search=False, verbose=False) + db_name_matches = db.search_object( + source, resolve_simbad=True, fuzzy_search=False, verbose=False + ) # if still no matches, try spatial search using coordinates, if provided if len(db_name_matches) == 0 and coords: - location = SkyCoord(ra, dec, frame='icrs', unit='deg') - radius = u.Quantity(search_radius, unit='arcsec') - logger.info(f"{source}: No Simbad match, trying coord search around {location.ra.degree}, {location.dec}") + location = SkyCoord(ra, dec, frame="icrs", unit="deg") + radius = u.Quantity(search_radius, unit="arcsec") + logger.info( + f"{source}: No Simbad match, trying coord search around {location.ra.degree}, {location.dec}" + ) db_name_matches = db.query_region(location, radius=radius) # If still no matches, try to get the coords from SIMBAD if len(db_name_matches) == 0: simbad_result_table = Simbad.query_object(source) if simbad_result_table is not None and len(simbad_result_table) == 1: - simbad_coords = simbad_result_table['RA'][0] + ' ' + simbad_result_table['DEC'][0] + simbad_coords = ( + simbad_result_table["RA"][0] + " " + simbad_result_table["DEC"][0] + ) simbad_skycoord = SkyCoord(simbad_coords, unit=(u.hourangle, u.deg)) - ra = simbad_skycoord.to_string(style='decimal').split()[0] - dec = simbad_skycoord.to_string(style='decimal').split()[1] + ra = simbad_skycoord.to_string(style="decimal").split()[0] + dec = simbad_skycoord.to_string(style="decimal").split()[1] msg = f"Coordinates retrieved from SIMBAD {ra}, {dec}" logger.debug(msg) # Search database around that coordinate - radius = u.Quantity(search_radius, unit='arcsec') + radius = u.Quantity(search_radius, unit="arcsec") db_name_matches = db.query_region(simbad_skycoord, radius=radius) if len(db_name_matches) == 1: - db_names = db_name_matches['source'].tolist() - logger.debug(f'One match found for {source}: {db_names[0]}') + db_names = db_name_matches["source"].tolist() + logger.debug(f"One match found for {source}: {db_names[0]}") elif len(db_name_matches) > 1: - db_names = db_name_matches['source'].tolist() - logger.debug(f'More than match found for {source}: {db_names}') + db_names = db_name_matches["source"].tolist() + logger.debug(f"More than match found for {source}: {db_names}") # TODO: Find way for user to choose correct match elif len(db_name_matches) == 0: db_names = [] - logger.debug(f' {source}: No match found') + logger.debug(f" {source}: No match found") else: - raise SimpleError(f'Unexpected condition searching for {source}') + raise SimpleError(f"Unexpected condition searching for {source}") return db_names @@ -227,19 +261,25 @@ def find_publication(db, name: str = None, doi: str = None, bibcode: str = None) not_null_pub_filters.append(db.Publications.c.bibcode.ilike(bibcode)) pub_search_table = Table() if len(not_null_pub_filters) > 0: - pub_search_table = db.query(db.Publications).filter(or_(*not_null_pub_filters)).table() + pub_search_table = ( + db.query(db.Publications).filter(or_(*not_null_pub_filters)).table() + ) n_pubs_found = len(pub_search_table) if n_pubs_found == 1: - logger.info(f'Found {n_pubs_found} matching publications for ' - f"{name} or {doi} or {bibcode}: {pub_search_table['reference'].data}") + logger.info( + f"Found {n_pubs_found} matching publications for " + f"{name} or {doi} or {bibcode}: {pub_search_table['reference'].data}" + ) if logger.level <= 10: # debug pub_search_table.pprint_all() - return True, pub_search_table['reference'].data[0] + return True, pub_search_table["reference"].data[0] if n_pubs_found > 1: - logger.warning(f'Found {n_pubs_found} matching publications for {name} or {doi} or {bibcode}') + logger.warning( + f"Found {n_pubs_found} matching publications for {name} or {doi} or {bibcode}" + ) if logger.level <= 30: # warning pub_search_table.pprint_all() return False, n_pubs_found @@ -247,26 +287,31 @@ def find_publication(db, name: str = None, doi: str = None, bibcode: str = None) # If no matches found, search using first four characters of input name if n_pubs_found == 0 and name: shorter_name = name[:4] - logger.debug(f'No matching publications for {name}, Trying {shorter_name}.') - fuzzy_query_shorter_name = '%' + shorter_name + '%' - pub_search_table = db.query(db.Publications).filter( - db.Publications.c.reference.ilike(fuzzy_query_shorter_name)).table() + logger.debug(f"No matching publications for {name}, Trying {shorter_name}.") + fuzzy_query_shorter_name = "%" + shorter_name + "%" + pub_search_table = ( + db.query(db.Publications) + .filter(db.Publications.c.reference.ilike(fuzzy_query_shorter_name)) + .table() + ) n_pubs_found_short = len(pub_search_table) if n_pubs_found_short == 0: - logger.warning(f'No matching publications for {name} or {shorter_name}') - logger.warning('Use add_publication() to add it to the database.') + logger.warning(f"No matching publications for {name} or {shorter_name}") + logger.warning("Use add_publication() to add it to the database.") return False, 0 if n_pubs_found_short > 0: - logger.debug(f'Found {n_pubs_found_short} matching publications for {shorter_name}') + logger.debug( + f"Found {n_pubs_found_short} matching publications for {shorter_name}" + ) if logger.level == 10: # debug pub_search_table.pprint_all() # Try to find numbers in the reference which might be a date - dates = re.findall(r'\d+', name) + dates = re.findall(r"\d+", name) # try to find a two digit date if len(dates) == 0: - logger.debug(f'Could not find a date in {name}') + logger.debug(f"Could not find a date in {name}") two_digit_date = None elif len(dates) == 1: if len(dates[0]) == 4: @@ -274,29 +319,33 @@ def find_publication(db, name: str = None, doi: str = None, bibcode: str = None) elif len(dates[0]) == 2: two_digit_date = dates[0] else: - logger.debug(f'Could not find a two digit date using {dates}') + logger.debug(f"Could not find a two digit date using {dates}") two_digit_date = None else: - logger.debug(f'Could not find a two digit date using {dates}') + logger.debug(f"Could not find a two digit date using {dates}") two_digit_date = None if two_digit_date: - logger.debug(f'Trying to limit using {two_digit_date}') + logger.debug(f"Trying to limit using {two_digit_date}") n_pubs_found_short_date = 0 pubs_found_short_date = [] - for pub in pub_search_table['reference']: + for pub in pub_search_table["reference"]: if pub.find(two_digit_date) != -1: n_pubs_found_short_date += 1 pubs_found_short_date.append(pub) if n_pubs_found_short_date == 1: - logger.debug(f'Found {n_pubs_found_short_date} matching publications for ' - f'{name} using {shorter_name} and {two_digit_date}') - logger.debug(f'{pubs_found_short_date}') + logger.debug( + f"Found {n_pubs_found_short_date} matching publications for " + f"{name} using {shorter_name} and {two_digit_date}" + ) + logger.debug(f"{pubs_found_short_date}") return True, pubs_found_short_date[0] else: - logger.warning(f'Found {n_pubs_found_short_date} matching publications for ' - f'{name} using {shorter_name} and {two_digit_date}') - logger.warning(f'{pubs_found_short_date}') + logger.warning( + f"Found {n_pubs_found_short_date} matching publications for " + f"{name} using {shorter_name} and {two_digit_date}" + ) + logger.warning(f"{pubs_found_short_date}") return False, n_pubs_found_short_date else: return False, n_pubs_found_short @@ -306,8 +355,14 @@ def find_publication(db, name: str = None, doi: str = None, bibcode: str = None) return -def ingest_publication(db, doi: str = None, bibcode: str = None, publication: str = None, description: str = None, - ignore_ads: bool = False): +def ingest_publication( + db, + doi: str = None, + bibcode: str = None, + publication: str = None, + description: str = None, + ignore_ads: bool = False, +): """ Adds publication to the database using DOI or ADS Bibcode, including metadata found with ADS. @@ -335,14 +390,16 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st """ if not (publication or doi or bibcode): - logger.error('Publication, DOI, or Bibcode is required input') + logger.error("Publication, DOI, or Bibcode is required input") return - ads.config.token = os.getenv('ADS_TOKEN') + ads.config.token = os.getenv("ADS_TOKEN") if not ads.config.token and (not publication and (not doi or not bibcode)): - logger.error("An ADS_TOKEN environment variable must be set in order to auto-populate the fields.\n" - "Without an ADS_TOKEN, name and bibcode or DOI must be set explicity.") + logger.error( + "An ADS_TOKEN environment variable must be set in order to auto-populate the fields.\n" + "Without an ADS_TOKEN, name and bibcode or DOI must be set explicity." + ) return if ads.config.token and not ignore_ads: @@ -352,7 +409,7 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st logger.debug(f"Use ADS set to {use_ads}") if bibcode: - if 'arXiv' in bibcode: + if "arXiv" in bibcode: arxiv_id = bibcode bibcode = None else: @@ -360,21 +417,25 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st else: arxiv_id = None - name_add, bibcode_add, doi_add = '', '', '' + name_add, bibcode_add, doi_add = "", "", "" # Search ADS uing a provided arxiv id if arxiv_id and use_ads: - arxiv_matches = ads.SearchQuery(q=arxiv_id, fl=['id', 'bibcode', 'title', 'first_author', 'year', 'doi']) + arxiv_matches = ads.SearchQuery( + q=arxiv_id, fl=["id", "bibcode", "title", "first_author", "year", "doi"] + ) arxiv_matches_list = list(arxiv_matches) if len(arxiv_matches_list) != 1: - logger.error('should only be one matching arxiv id') + logger.error("should only be one matching arxiv id") return if len(arxiv_matches_list) == 1: logger.debug(f"Publication found in ADS using arxiv id: , {arxiv_id}") article = arxiv_matches_list[0] - logger.debug(f"{article.first_author}, {article.year}, {article.bibcode}, {article.title}") + logger.debug( + f"{article.first_author}, {article.year}, {article.bibcode}, {article.title}" + ) if not publication: # generate the name if it was not provided - name_stub = article.first_author.replace(',', '').replace(' ', '') + name_stub = article.first_author.replace(",", "").replace(" ", "") name_add = name_stub[0:4] + article.year[-2:] else: name_add = publication @@ -389,19 +450,23 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st # Search ADS using a provided DOI if doi and use_ads: - doi_matches = ads.SearchQuery(doi=doi, fl=['id', 'bibcode', 'title', 'first_author', 'year', 'doi']) + doi_matches = ads.SearchQuery( + doi=doi, fl=["id", "bibcode", "title", "first_author", "year", "doi"] + ) doi_matches_list = list(doi_matches) if len(doi_matches_list) != 1: - logger.error('should only be one matching DOI') + logger.error("should only be one matching DOI") return if len(doi_matches_list) == 1: logger.debug(f"Publication found in ADS using DOI: {doi}") using = doi article = doi_matches_list[0] - logger.debug(f"{article.first_author}, {article.year}, {article.bibcode}, {article.title}") + logger.debug( + f"{article.first_author}, {article.year}, {article.bibcode}, {article.title}" + ) if not publication: # generate the name if it was not provided - name_stub = article.first_author.replace(',', '').replace(' ', '') + name_stub = article.first_author.replace(",", "").replace(" ", "") name_add = name_stub[0:4] + article.year[-2:] else: name_add = publication @@ -414,25 +479,30 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st doi_add = doi if bibcode and use_ads: - bibcode_matches = ads.SearchQuery(bibcode=bibcode, fl=['id', 'bibcode', 'title', 'first_author', 'year', 'doi']) + bibcode_matches = ads.SearchQuery( + bibcode=bibcode, + fl=["id", "bibcode", "title", "first_author", "year", "doi"], + ) bibcode_matches_list = list(bibcode_matches) if len(bibcode_matches_list) == 0: - logger.error('not a valid bibcode:' + str(bibcode)) - logger.error('nothing added') + logger.error("not a valid bibcode:" + str(bibcode)) + logger.error("nothing added") raise elif len(bibcode_matches_list) > 1: - logger.error('should only be one matching bibcode for:' + str(bibcode)) - logger.error('nothing added') + logger.error("should only be one matching bibcode for:" + str(bibcode)) + logger.error("nothing added") raise elif len(bibcode_matches_list) == 1: logger.debug("Publication found in ADS using bibcode: " + str(bibcode)) using = str(bibcode) article = bibcode_matches_list[0] - logger.debug(f"{article.first_author}, {article.year}, {article.bibcode}, {article.doi}, {article.title}") + logger.debug( + f"{article.first_author}, {article.year}, {article.bibcode}, {article.doi}, {article.title}" + ) if not publication: # generate the name if it was not provided - name_stub = article.first_author.replace(',', '').replace(' ', '') + name_stub = article.first_author.replace(",", "").replace(" ", "") name_add = name_stub[0:4] + article.year[-2:] else: name_add = publication @@ -449,33 +519,40 @@ def ingest_publication(db, doi: str = None, bibcode: str = None, publication: st if publication and not bibcode and not doi: name_add = publication - using = 'user input' + using = "user input" - new_ref = [{'reference': name_add, 'bibcode': bibcode_add, 'doi': doi_add, 'description': description}] + new_ref = [ + { + "reference": name_add, + "bibcode": bibcode_add, + "doi": doi_add, + "description": description, + } + ] try: with db.engine.connect() as conn: conn.execute(db.Publications.insert().values(new_ref)) conn.commit() - logger.info(f'Added {name_add} to Publications table using {using}') + logger.info(f"Added {name_add} to Publications table using {using}") except sqlalchemy.exc.IntegrityError as error: - msg = f"Not able to add {new_ref} to the database. " \ - f"It's possible that a similar publication already exists in database\n"\ - "Use find_publication function before adding a new record" + msg = ( + f"Not able to add {new_ref} to the database. " + f"It's possible that a similar publication already exists in database\n" + "Use find_publication function before adding a new record" + ) logger.error(msg) raise SimpleError(msg + str(error)) return - def check_internet_connection(): # get current IP address of system ipaddress = socket.gethostbyname(socket.gethostname()) # checking system IP is the same as "127.0.0.1" or not. - if ipaddress == "127.0.0.1": # no internet + if ipaddress == "127.0.0.1": # no internet return False, ipaddress else: return True, ipaddress - diff --git a/tests/test_integrity.py b/tests/test_integrity.py index 43b6f1eac..53ac7db76 100644 --- a/tests/test_integrity.py +++ b/tests/test_integrity.py @@ -5,7 +5,7 @@ import pytest from . import REFERENCE_TABLES -from sqlalchemy import func, select, except_ +from sqlalchemy import func, and_ # , select, except_ from simple.schema import * from astrodbkit2.astrodb import create_database, Database, or_ from astropy.table import unique @@ -14,8 +14,8 @@ from astrodbkit2.utils import _name_formatter -DB_NAME = 'temp.db' -DB_PATH = 'data' +DB_NAME = "temp.db" +DB_PATH = "data" # Load the database for use in individual tests @@ -26,20 +26,26 @@ def db(): if os.path.exists(DB_NAME): os.remove(DB_NAME) - connection_string = 'sqlite:///' + DB_NAME + connection_string = "sqlite:///" + DB_NAME create_database(connection_string) assert os.path.exists(DB_NAME) # Connect to the new database and confirm it has the Sources table db = Database(connection_string, reference_tables=REFERENCE_TABLES) assert db - assert 'source' in [c.name for c in db.Sources.columns] + assert "source" in [c.name for c in db.Sources.columns] # Load data into an in-memory sqlite database first, for performance - temp_db = Database('sqlite://', reference_tables=REFERENCE_TABLES) # creates and connects to a temporary in-memory database - temp_db.load_database(DB_PATH, verbose=False) # loads the data from the data files into the database + temp_db = Database( + "sqlite://", reference_tables=REFERENCE_TABLES + ) # creates and connects to a temporary in-memory database + temp_db.load_database( + DB_PATH, verbose=False + ) # loads the data from the data files into the database temp_db.dump_sqlite(DB_NAME) # dump in-memory database to file - db = Database('sqlite:///' + DB_NAME, reference_tables=REFERENCE_TABLES) # replace database object with new file version + db = Database( + "sqlite:///" + DB_NAME, reference_tables=REFERENCE_TABLES + ) # replace database object with new file version return db @@ -47,18 +53,26 @@ def db(): def test_reference_uniqueness(db): # Verify that all Publications.name values are unique t = db.query(db.Publications.c.reference).astropy() - assert len(t) == len(unique(t, keys='reference')), 'duplicated publications found' + assert len(t) == len(unique(t, keys="reference")), "duplicated publications found" # Verify that DOI are supplied - t = db.query(db.Publications.c.reference).filter(db.Publications.c.doi.is_(None)).astropy() + t = ( + db.query(db.Publications.c.reference) + .filter(db.Publications.c.doi.is_(None)) + .astropy() + ) if len(t) > 0: - print(f'\n{len(t)} publications lacking DOI:') + print(f"\n{len(t)} publications lacking DOI:") print(t) # Verify that Bibcodes are supplied - t = db.query(db.Publications.c.reference).filter(db.Publications.c.bibcode.is_(None)).astropy() + t = ( + db.query(db.Publications.c.reference) + .filter(db.Publications.c.bibcode.is_(None)) + .astropy() + ) if len(t) > 0: - print(f'\n{len(t)} publications lacking ADS bibcodes:') + print(f"\n{len(t)} publications lacking ADS bibcodes:") print(t) @@ -66,38 +80,60 @@ def test_references(db): # Verify that all data point to an existing Publication ref_list = [] - table_list = ['Sources', 'Photometry', 'Parallaxes', 'ProperMotions', 'Spectra'] + table_list = ["Sources", "Photometry", "Parallaxes", "ProperMotions", "Spectra"] for table in table_list: # Get list of unique references t = db.query(db.metadata.tables[table].c.reference).distinct().astropy() - ref_list = ref_list + t['reference'].tolist() + ref_list = ref_list + t["reference"].tolist() # Getting unique set ref_list = list(set(ref_list)) # Confirm that all are in Publications - t = db.query(db.Publications.c.reference).filter(db.Publications.c.reference.in_(ref_list)).astropy() - assert len(t) == len(ref_list), 'Some references were not matched' + t = ( + db.query(db.Publications.c.reference) + .filter(db.Publications.c.reference.in_(ref_list)) + .astropy() + ) + assert len(t) == len(ref_list), "Some references were not matched" # List out publications that have not been used - t = db.query(db.Publications.c.reference).filter(db.Publications.c.reference.notin_(ref_list)).astropy() - assert len(t) <= 606, f'{len(t)} unused references' + t = ( + db.query(db.Publications.c.reference) + .filter(db.Publications.c.reference.notin_(ref_list)) + .astropy() + ) + assert len(t) <= 606, f"{len(t)} unused references" def test_publications(db): # Find unused references in the Sources Table - # stm = except_(select([db.Publications.c.reference]), select([db.Sources.c.reference])) + # stm = except_(select([db.Publications.c.reference]), + # select([db.Sources.c.reference])) # result = db.session.execute(stm) # s = result.scalars().all() # assert len(s) == 720, f'found {len(s)} unused references' # Find references with no doi or bibcode - t = db.query(db.Publications.c.reference).filter( - or_(and_(db.Publications.c.doi.is_(None), db.Publications.c.bibcode.is_(None)), - and_(db.Publications.c.doi.is_(''), db.Publications.c.bibcode.is_(None)), - and_(db.Publications.c.doi.is_(None), db.Publications.c.bibcode.is_('')), - and_(db.Publications.c.doi.is_(''), db.Publications.c.bibcode.is_('')))).astropy() - assert len(t) == 28, f'found {len(t)} publications with missing bibcode and doi' + t = ( + db.query(db.Publications.c.reference) + .filter( + or_( + and_( + db.Publications.c.doi.is_(None), db.Publications.c.bibcode.is_(None) + ), + and_( + db.Publications.c.doi.is_(""), db.Publications.c.bibcode.is_(None) + ), + and_( + db.Publications.c.doi.is_(None), db.Publications.c.bibcode.is_("") + ), + and_(db.Publications.c.doi.is_(""), db.Publications.c.bibcode.is_("")), + ) + ) + .astropy() + ) + assert len(t) == 28, f"found {len(t)} publications with missing bibcode and doi" def test_parameters(db): @@ -106,17 +142,21 @@ def test_parameters(db): """ t = db.query(db.Parameters).astropy() - assert len(t) > 0, 'Parameters table is empty' + assert len(t) > 0, "Parameters table is empty" # Check usage of Parameters param_list = db.query(db.ModeledParameters.c.parameter).astropy() if len(param_list) > 0: # Get unique values - param_list = list(param_list['parameter']) + param_list = list(param_list["parameter"]) param_list = list(set(param_list)) - t = db.query(db.Parameters).filter(db.Parameters.c.parameter.notin_(param_list)).astropy() + t = ( + db.query(db.Parameters) + .filter(db.Parameters.c.parameter.notin_(param_list)) + .astropy() + ) if len(t) > 0: - print('The following parameters are not being used:') + print("The following parameters are not being used:") print(t) # Skipping actual assertion test # assert len(t) == 0, f'{len(t)} unused parameters' @@ -124,22 +164,35 @@ def test_parameters(db): def test_coordinates(db): # Verify that all sources have valid coordinates - t = db.query(db.Sources.c.source, db.Sources.c.ra, db.Sources.c.dec).filter( - or_(db.Sources.c.ra.is_(None), db.Sources.c.ra < 0, db.Sources.c.ra > 360, - db.Sources.c.dec.is_(None), db.Sources.c.dec < -90, db.Sources.c.dec > 90)).astropy() + t = ( + db.query(db.Sources.c.source, db.Sources.c.ra, db.Sources.c.dec) + .filter( + or_( + db.Sources.c.ra.is_(None), + db.Sources.c.ra < 0, + db.Sources.c.ra > 360, + db.Sources.c.dec.is_(None), + db.Sources.c.dec < -90, + db.Sources.c.dec > 90, + ) + ) + .astropy() + ) if len(t) > 0: - print(f'\n{len(t)} Sources failed coordinate checks') + print(f"\n{len(t)} Sources failed coordinate checks") print(t) - assert len(t) == 0, f'{len(t)} Sources failed coordinate checks' + assert len(t) == 0, f"{len(t)} Sources failed coordinate checks" def test_source_names(db): # Verify that all sources have at least one entry in Names table - sql_text = "SELECT Sources.source FROM Sources LEFT JOIN Names " \ - "ON Names.source=Sources.source WHERE Names.source IS NULL" - missing_names = db.sql_query(sql_text, fmt='astropy') + sql_text = ( + "SELECT Sources.source FROM Sources LEFT JOIN Names " + "ON Names.source=Sources.source WHERE Names.source IS NULL" + ) + missing_names = db.sql_query(sql_text, fmt="astropy") assert len(missing_names) == 0 @@ -150,13 +203,14 @@ def test_source_uniqueness(db): assert len(source_names) == len(unique_source_names) # Another method to find the duplicates - sql_text = "SELECT Sources.source FROM Sources GROUP BY source " \ - "HAVING (Count(*) > 1)" - duplicate_names = db.sql_query(sql_text, fmt='astropy') + sql_text = ( + "SELECT Sources.source FROM Sources GROUP BY source " "HAVING (Count(*) > 1)" + ) + duplicate_names = db.sql_query(sql_text, fmt="astropy") # if duplicate_names is non_zero, print out duplicate names if len(duplicate_names) > 0: - print(f'\n{len(duplicate_names)} duplicated names') + print(f"\n{len(duplicate_names)} duplicated names") print(duplicate_names) assert len(duplicate_names) == 0 @@ -165,51 +219,69 @@ def test_source_uniqueness(db): def test_names_table(db): # Verify that all Sources contain at least one entry in the Names table name_list = db.query(db.Sources.c.source).astropy() - name_list = name_list['source'].tolist() - source_name_counts = db.query(db.Names.c.source). \ - filter(db.Names.c.source.in_(name_list)). \ - distinct(). \ - count() - assert len(name_list) == source_name_counts, 'ERROR: There are Sources without entries in the Names table' - - # Verify that each Source contains an entry in Names with Names.source = Names.other_source - valid_name_counts = db.query(db.Names.c.source). \ - filter(db.Names.c.source == db.Names.c.other_name). \ - distinct(). \ - count() - - # If the number of valid names don't match the number of sources, then there are cases that are missing + name_list = name_list["source"].tolist() + source_name_counts = ( + db.query(db.Names.c.source) + .filter(db.Names.c.source.in_(name_list)) + .distinct() + .count() + ) + assert ( + len(name_list) == source_name_counts + ), "ERROR: There are Sources without entries in the Names table" + + # Verify that each Source contains an entry in + # Names with Names.source = Names.other_source + valid_name_counts = ( + db.query(db.Names.c.source) + .filter(db.Names.c.source == db.Names.c.other_name) + .distinct() + .count() + ) + + # If the number of valid names don't match the number of sources, + # then there are cases that are missing # The script below will gather them and print them out if len(name_list) != valid_name_counts: - # Create a temporary table that groups entries in the Names table by their source name + # Create a temporary table that groups entries in the + # Names table by their source name # with a column containing a concatenation of all known names - t = db.query(db.Names.c.source, - func.group_concat(db.Names.c.other_name).label('names')). \ - group_by(db.Names.c.source). \ - astropy() - - # Get the list of entries whose source name are not present in the 'other_names' column - # Then return the Names table results so we can see what the DB has for these entries - results = [row['source'] for row in t if row['source'] not in row['names'].split(',')] - print('\nEntries in Names without Names.source == Names.other_name:') - print(db.query(db.Names). - filter(db.Names.c.source.in_(results)). - astropy()) - - assert len(name_list) == valid_name_counts, \ - 'ERROR: There are entries in Names without Names.source == Names.other_name' + t = ( + db.query( + db.Names.c.source, + func.group_concat(db.Names.c.other_name).label("names"), + ) + .group_by(db.Names.c.source) + .astropy() + ) + + # Get the list of entries whose source name + # are not present in the 'other_names' column + # Then return the Names table results + # so we can see what the DB has for these entries + results = [ + row["source"] for row in t if row["source"] not in row["names"].split(",") + ] + print("\nEntries in Names without Names.source == Names.other_name:") + print(db.query(db.Names).filter(db.Names.c.source.in_(results)).astropy()) + + assert ( + len(name_list) == valid_name_counts + ), "ERROR: There are entries in Names without Names.source == Names.other_name" # Verify that there are no empty strings as other_names in Names - blank_names = db.query(db.Names).filter(db.Names.c.other_name == '').astropy() - assert len(blank_names) == 0, \ - 'ERROR: There are entries in Names which are empty strings' + blank_names = db.query(db.Names).filter(db.Names.c.other_name == "").astropy() + assert ( + len(blank_names) == 0 + ), "ERROR: There are entries in Names which are empty strings" def test_source_uniqueness2(db): # Verify that all Sources.source values are unique and find the duplicates - sql_text = "SELECT Sources.source FROM Sources GROUP BY source " \ - "HAVING (Count(*) > 1)" - duplicate_names = db.sql_query(sql_text, fmt='astropy') + sql_text = ( + "SELECT Sources.source FROM Sources GROUP BY source " "HAVING (Count(*) > 1)" + ) + duplicate_names = db.sql_query(sql_text, fmt="astropy") # if duplicate_names is non_zero, print out duplicate names assert len(duplicate_names) == 0 @@ -223,34 +295,41 @@ def test_source_simbad(db): name_list = [s[0] for s in results] # Add all IDS to the Simbad output as well as the user-provided id - Simbad.add_votable_fields('ids') - Simbad.add_votable_fields('typed_id') + Simbad.add_votable_fields("ids") + Simbad.add_votable_fields("typed_id") simbad_results = Simbad.query_objects(name_list) # Get a nicely formatted list of Simbad names for each input row duplicate_count = 0 - for row in simbad_results[['TYPED_ID', 'IDS']].iterrows(): + for row in simbad_results[["TYPED_ID", "IDS"]].iterrows(): try: name, ids = row[0].decode("utf-8"), row[1].decode("utf-8") except AttributeError: # Catch decoding error name, ids = row[0], row[1] - simbad_names = [_name_formatter(s) for s in ids.split('|') - if _name_formatter(s) != '' and _name_formatter(s) is not None] + simbad_names = [ + _name_formatter(s) + for s in ids.split("|") + if _name_formatter(s) != "" and _name_formatter(s) is not None + ] if len(simbad_names) == 0: - print(f'No Simbad names for {name}') + print(f"No Simbad names for {name}") continue # Examine DB for each input, displaying results when more than one source matches - t = db.search_object(simbad_names, output_table='Sources', fmt='astropy', fuzzy_search=False) + t = db.search_object( + simbad_names, output_table="Sources", fmt="astropy", fuzzy_search=False + ) if len(t) > 1: - print(f'Multiple matches for {name}: {simbad_names}') - print(db.query(db.Names).filter(db.Names.c.source.in_(t['source'])).astropy()) + print(f"Multiple matches for {name}: {simbad_names}") + print( + db.query(db.Names).filter(db.Names.c.source.in_(t["source"])).astropy() + ) duplicate_count += 1 - assert duplicate_count == 0, 'Duplicate sources identified via Simbad queries' + assert duplicate_count == 0, "Duplicate sources identified via Simbad queries" def test_photometry(db): @@ -258,21 +337,26 @@ def test_photometry(db): # Check that no negative magnitudes have been provided, # nor any that are larger than 99 (if missing/limits, just use None) - t = db.query(db.Photometry). \ - filter(or_(db.Photometry.c.magnitude < 0, - db.Photometry.c.magnitude >= 99)). \ - astropy() + t = ( + db.query(db.Photometry) + .filter(or_(db.Photometry.c.magnitude < 0, db.Photometry.c.magnitude >= 99)) + .astropy() + ) if len(t) > 0: - print('\nInvalid magnitudes present') + print("\nInvalid magnitudes present") print(t) assert len(t) == 0 def test_photometry_filters(db): bands_in_use = db.query(db.Photometry.c.band).distinct().astropy() - for band_in_use in bands_in_use['band']: - check = db.query(db.PhotometryFilters).filter(db.PhotometryFilters.c.band == band_in_use).astropy() - assert len(check) == 1, f'{band_in_use} not in PhotometryFilters' + for band_in_use in bands_in_use["band"]: + check = ( + db.query(db.PhotometryFilters) + .filter(db.PhotometryFilters.c.band == band_in_use) + .astropy() + ) + assert len(check) == 1, f"{band_in_use} not in PhotometryFilters" def test_parallaxes(db): @@ -280,11 +364,15 @@ def test_parallaxes(db): # While there may be many parallax measurements for a single source, # there should be only one marked as adopted - t = db.query(db.Parallaxes.c.source, - func.sum(db.Parallaxes.c.adopted).label('adopted_counts')). \ - group_by(db.Parallaxes.c.source). \ - having(func.sum(db.Parallaxes.c.adopted) > 1). \ - astropy() + t = ( + db.query( + db.Parallaxes.c.source, + func.sum(db.Parallaxes.c.adopted).label("adopted_counts"), + ) + .group_by(db.Parallaxes.c.source) + .having(func.sum(db.Parallaxes.c.adopted) > 1) + .astropy() + ) if len(t) > 0: print("\nParallax entries with incorrect 'adopted' labels") print(t) @@ -295,22 +383,29 @@ def test_propermotions(db): # Tests against the ProperMotions table # There should be no entries in the ProperMotions table without both mu_ra and mu_dec - t = db.query(db.ProperMotions.c.source). \ - filter(or_(db.ProperMotions.c.mu_ra.is_(None), - db.ProperMotions.c.mu_dec.is_(None))). \ - astropy() + t = ( + db.query(db.ProperMotions.c.source) + .filter( + or_(db.ProperMotions.c.mu_ra.is_(None), db.ProperMotions.c.mu_dec.is_(None)) + ) + .astropy() + ) if len(t) > 0: - print('\nEntries found without proper motion values') + print("\nEntries found without proper motion values") print(t) assert len(t) == 0 # While there may be many proper motion measurements for a single source, # there should be only one marked as adopted - t = db.query(db.ProperMotions.c.source, - func.sum(db.ProperMotions.c.adopted).label('adopted_counts')). \ - group_by(db.ProperMotions.c.source). \ - having(func.sum(db.ProperMotions.c.adopted) > 1). \ - astropy() + t = ( + db.query( + db.ProperMotions.c.source, + func.sum(db.ProperMotions.c.adopted).label("adopted_counts"), + ) + .group_by(db.ProperMotions.c.source) + .having(func.sum(db.ProperMotions.c.adopted) > 1) + .astropy() + ) if len(t) > 0: print("\nProper Motion measurements with incorrect 'adopted' labels") print(t) @@ -321,21 +416,27 @@ def test_radialvelocities(db): # Tests against the RadialVelocities table # There should be no entries in the RadialVelocities table without rv values - t = db.query(db.RadialVelocities.c.source). \ - filter(db.RadialVelocities.c.radial_velocity.is_(None)). \ - astropy() + t = ( + db.query(db.RadialVelocities.c.source) + .filter(db.RadialVelocities.c.radial_velocity.is_(None)) + .astropy() + ) if len(t) > 0: - print('\nEntries found without radial velocity values') + print("\nEntries found without radial velocity values") print(t) assert len(t) == 0 # While there may be many radial velocity measurements for a single source, # there should be only one marked as adopted - t = db.query(db.RadialVelocities.c.source, - func.sum(db.RadialVelocities.c.adopted).label('adopted_counts')). \ - group_by(db.RadialVelocities.c.source). \ - having(func.sum(db.RadialVelocities.c.adopted) > 1). \ - astropy() + t = ( + db.query( + db.RadialVelocities.c.source, + func.sum(db.RadialVelocities.c.adopted).label("adopted_counts"), + ) + .group_by(db.RadialVelocities.c.source) + .having(func.sum(db.RadialVelocities.c.adopted) > 1) + .astropy() + ) if len(t) > 0: print("\nRadial velocity measurements with incorrect 'adopted' labels") print(t) @@ -346,30 +447,38 @@ def test_spectraltypes(db): # Tests against the SpectralTypes table # There should be no entries in the SpectralTypes table without a spectral type string - t = db.query(db.SpectralTypes.c.source). \ - filter(db.SpectralTypes.c.spectral_type_string.is_(None)). \ - astropy() + t = ( + db.query(db.SpectralTypes.c.source) + .filter(db.SpectralTypes.c.spectral_type_string.is_(None)) + .astropy() + ) if len(t) > 0: - print('\nEntries found without spectral type strings') + print("\nEntries found without spectral type strings") print(t) assert len(t) == 0 # There should be no entries in the SpectralTypes table without a spectral type code - t = db.query(db.SpectralTypes.c.source). \ - filter(db.SpectralTypes.c.spectral_type_code.is_(None)). \ - astropy() + t = ( + db.query(db.SpectralTypes.c.source) + .filter(db.SpectralTypes.c.spectral_type_code.is_(None)) + .astropy() + ) if len(t) > 0: - print('\nEntries found without spectral type codes') + print("\nEntries found without spectral type codes") print(t) assert len(t) == 0 # While there may be many spectral type measurements for a single source, # there should be only one marked as adopted - t = db.query(db.SpectralTypes.c.source, - func.sum(db.SpectralTypes.c.adopted).label('adopted_counts')). \ - group_by(db.SpectralTypes.c.source). \ - having(func.sum(db.SpectralTypes.c.adopted) > 1). \ - astropy() + t = ( + db.query( + db.SpectralTypes.c.source, + func.sum(db.SpectralTypes.c.adopted).label("adopted_counts"), + ) + .group_by(db.SpectralTypes.c.source) + .having(func.sum(db.SpectralTypes.c.adopted) > 1) + .astropy() + ) if len(t) > 0: print("\nSpectral Type entries with incorrect 'adopted' labels") print(t) @@ -380,139 +489,175 @@ def test_gravities(db): # Tests against the Gravities table # There should be no entries in the Gravities table without a gravity measurement - t = db.query(db.Gravities.c.source). \ - filter(db.Gravities.c.gravity.is_(None)). \ - astropy() + t = ( + db.query(db.Gravities.c.source) + .filter(db.Gravities.c.gravity.is_(None)) + .astropy() + ) if len(t) > 0: - print('\nEntries found without gravity values') + print("\nEntries found without gravity values") print(t) assert len(t) == 0 def test_sources(db): # Counting the top 20 references in the Sources Table - spec_ref_count = db.query(Sources.reference, func.count(Sources.reference)). \ - group_by(Sources.reference).order_by(func.count(Sources.reference).desc()).limit(20).all() + # spec_ref_count = ( + # db.query(Sources.reference, func.count(Sources.reference)) + # .group_by(Sources.reference) + # .order_by(func.count(Sources.reference).desc()) + # .limit(20) + # .all() + # ) # Top 20 References in the Sources Table - ref = 'Schm10.1808' + ref = "Schm10.1808" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 208, f'found {len(t)} sources from {ref}' + assert len(t) == 208, f"found {len(t)} sources from {ref}" - ref = 'Reid08.1290' + ref = "Reid08.1290" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 206, f'found {len(t)} sources from {ref}' + assert len(t) == 206, f"found {len(t)} sources from {ref}" - ref = 'West08' + ref = "West08" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 192, f'found {len(t)} sources from {ref}' + assert len(t) == 192, f"found {len(t)} sources from {ref}" - ref = 'Cruz03' + ref = "Cruz03" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 165, f'found {len(t)} sources from {ref}' + assert len(t) == 165, f"found {len(t)} sources from {ref}" - ref = 'Maro15' + ref = "Maro15" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 113, f'found {len(t)} sources from {ref}' + assert len(t) == 113, f"found {len(t)} sources from {ref}" - ref = 'Best15' + ref = "Best15" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 101, f'found {len(t)} sources from {ref}' + assert len(t) == 101, f"found {len(t)} sources from {ref}" - ref = 'Kirk11' + ref = "Kirk11" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 100, f'found {len(t)} sources from {ref}' + assert len(t) == 100, f"found {len(t)} sources from {ref}" - ref = 'Mace13.6' + ref = "Mace13.6" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 93, f'found {len(t)} sources from {ref}' + assert len(t) == 93, f"found {len(t)} sources from {ref}" - ref = 'Cruz07' + ref = "Cruz07" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 91, f'found {len(t)} sources from {ref}' + assert len(t) == 91, f"found {len(t)} sources from {ref}" - ref = 'Burn13' + ref = "Burn13" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 69, f'found {len(t)} sources from {ref}' + assert len(t) == 69, f"found {len(t)} sources from {ref}" - ref = 'Gagn15.33' + ref = "Gagn15.33" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 68, f'found {len(t)} sources from {ref}' + assert len(t) == 68, f"found {len(t)} sources from {ref}" - ref = 'Chiu06' + ref = "Chiu06" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 62, f'found {len(t)} sources from {ref}' + assert len(t) == 62, f"found {len(t)} sources from {ref}" - ref = 'Kirk00' + ref = "Kirk00" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 61, f'found {len(t)} sources from {ref}' + assert len(t) == 61, f"found {len(t)} sources from {ref}" - ref = 'DayJ13' + ref = "DayJ13" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 61, f'found {len(t)} sources from {ref}' + assert len(t) == 61, f"found {len(t)} sources from {ref}" - ref = 'Kirk10' + ref = "Kirk10" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 56, f'found {len(t)} sources from {ref}' + assert len(t) == 56, f"found {len(t)} sources from {ref}" - ref = 'Deac14.119' + ref = "Deac14.119" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 52, f'found {len(t)} sources from {ref}' + assert len(t) == 52, f"found {len(t)} sources from {ref}" - ref = 'Hawl02' + ref = "Hawl02" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 51, f'found {len(t)} sources from {ref}' + assert len(t) == 51, f"found {len(t)} sources from {ref}" - ref = 'Card15' + ref = "Card15" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 45, f'found {len(t)} sources from {ref}' + assert len(t) == 45, f"found {len(t)} sources from {ref}" - ref = 'Burn10.1885' + ref = "Burn10.1885" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 43, f'found {len(t)} sources from {ref}' + assert len(t) == 43, f"found {len(t)} sources from {ref}" - ref = 'Albe11' + ref = "Albe11" t = db.query(db.Sources).filter(db.Sources.c.reference == ref).astropy() - assert len(t) == 37, f'found {len(t)} sources from {ref}' + assert len(t) == 37, f"found {len(t)} sources from {ref}" def test_modeled_parameters(db): # There should be no entries in the modeled parameters table without parameter - t = db.query(db.ModeledParameters).filter(db.ModeledParameters.c.parameter.is_(None)).astropy() + t = ( + db.query(db.ModeledParameters) + .filter(db.ModeledParameters.c.parameter.is_(None)) + .astropy() + ) if len(t) > 0: - print('\nEntries found without a parameter') + print("\nEntries found without a parameter") print(t) assert len(t) == 0 # Test units are astropy.unit resolvable - t = db.query(db.ModeledParameters).filter(db.ModeledParameters.c.unit.is_not(None)).distinct().astropy() + t = ( + db.query(db.ModeledParameters) + .filter(db.ModeledParameters.c.unit.is_not(None)) + .distinct() + .astropy() + ) unit_fail = [] for x in t: - unit = x['unit'] + unit = x["unit"] try: - assert u.Unit(unit, parse_strict='raise') + assert u.Unit(unit, parse_strict="raise") except ValueError: - print(f'{unit} is not a recognized astropy unit') - counts = db.query(db.ModeledParameters).filter(db.ModeledParameters.c.unit == unit).count() + print(f"{unit} is not a recognized astropy unit") + counts = ( + db.query(db.ModeledParameters) + .filter(db.ModeledParameters.c.unit == unit) + .count() + ) unit_fail.append({unit: counts}) # count of how many of that unit there is - assert len(unit_fail) == 0, f'Some parameter units did not resolve: {unit_fail}' + assert len(unit_fail) == 0, f"Some parameter units did not resolve: {unit_fail}" # check no negative Mass, Radius, or Teff - t = db.query(db.ModeledParameters).filter(and_(db.ModeledParameters.c.parameter.in_(["radius", "mass", "Teff"]), - db.ModeledParameters.c.value < 0)).astropy() + t = ( + db.query(db.ModeledParameters) + .filter( + and_( + db.ModeledParameters.c.parameter.in_(["radius", "mass", "Teff"]), + db.ModeledParameters.c.value < 0, + ) + ) + .astropy() + ) if len(t) > 0: - print('\n Negative value for Radius, Mass, or Teff not allowed.\n') + print("\n Negative value for Radius, Mass, or Teff not allowed.\n") print(t) assert len(t) == 0 - # check no negative value error - t = db.query(db.ModeledParameters).filter(and_(db.ModeledParameters.c.value_error != None, - db.ModeledParameters.c.value_error < 0)).astropy() - + # check no negative value error + t = ( + db.query(db.ModeledParameters) + .filter( + and_( + db.ModeledParameters.c.value_error is not None, + db.ModeledParameters.c.value_error < 0, + ) + ) + .astropy() + ) + if len(t) > 0: - print('\n Negative projected separations') + print("\n Negative projected separations") print(t) assert len(t) == 0 @@ -521,11 +666,9 @@ def test_spectra(db): # Tests against the Spectra table # There should be no entries in the Spectra table without a spectrum - t = db.query(db.Spectra.c.source). \ - filter(db.Spectra.c.spectrum.is_(None)). \ - astropy() + t = db.query(db.Spectra.c.source).filter(db.Spectra.c.spectrum.is_(None)).astropy() if len(t) > 0: - print('\nEntries found without spectrum') + print("\nEntries found without spectrum") print(t) assert len(t) == 0 @@ -533,24 +676,48 @@ def test_spectra(db): def test_special_characters(db): # This test asserts that no special unicode characters are in the database # This can be expanded with additional characters we want to avoid - bad_characters = ['\u2013', '\u00f3', '\u00e9', '\u00ed', '\u00e1', '\u00fa'] + bad_characters = [ + "\u2013", + "\u00f3", + "\u00e9", + "\u00ed", + "\u00e1", + "\u00fa", + "\u0000" + ] for char in bad_characters: data = db.search_string(char) - # Check tables individually, want to make sure primary/foreign keys are verified but not comments/descriptions + # Make sure primary/foreign keys don't have unicode + # but not checking comments/descriptions if len(data) > 0: for table_name in data.keys(): - if table_name == 'Publications': - check = [char not in data[table_name]['reference']] - assert all(check), f'{char} in {table_name}' - elif table_name == 'Spectra': - check = [char not in data[table_name]['spectrum']] - assert all(check), f'{char} in {table_name}' - elif table_name == 'Names': - check = [char not in data[table_name]['other_name']] - assert all(check), f'{char} in {table_name}' + if table_name == "Publications": + check = [char not in data[table_name]["reference"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Spectra": + check = [char not in data[table_name]["spectrum"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Names": + check = [char not in data[table_name]["other_name"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Instruments": + check = [char not in data[table_name]["instrument"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Telescopes": + check = [char not in data[table_name]["telescope"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Parameters": + check = [char not in data[table_name]["parameter"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "PhotometryFilters": + check = [char not in data[table_name]["band"]] + assert all(check), f"{char} in {table_name}" + elif table_name == "Versions": + check = [char not in data[table_name]["version"]] + assert all(check), f"{char} in {table_name}" else: - check = [char not in data[table_name]['source']] - assert all(check), f'{char} in {table_name}' + check = [char not in data[table_name]["source"]] + assert all(check), f"{char} in {table_name}" def test_database_views(db): @@ -563,93 +730,124 @@ def test_database_views(db): assert len(t) > 0 # Check view is not part of inventory - assert 'ParallaxView' not in db.inventory('2MASSI J0019457+521317').keys() - + assert "ParallaxView" not in db.inventory("2MASSI J0019457+521317").keys() + + def test_companion_relationship(db): # There should be no entries without a companion name - t = db.query(db.CompanionRelationships.c.source). \ - filter(db.CompanionRelationships.c.companion_name.is_(None)). \ - astropy() + t = ( + db.query(db.CompanionRelationships.c.source) + .filter(db.CompanionRelationships.c.companion_name.is_(None)) + .astropy() + ) if len(t) > 0: - print('\n Entries found without a companion name') + print("\n Entries found without a companion name") print(t) assert len(t) == 0 # There should be no entries a companion name thats the same as the source - t = db.query(db.CompanionRelationships.c.source). \ - filter(db.CompanionRelationships.c.companion_name == db.CompanionRelationships.c.source). \ - astropy() + t = ( + db.query(db.CompanionRelationships.c.source) + .filter( + db.CompanionRelationships.c.companion_name + == db.CompanionRelationships.c.source + ) + .astropy() + ) if len(t) > 0: - print('\nCompanion name cannot be source name') + print("\nCompanion name cannot be source name") print(t) assert len(t) == 0 # check no negative separations or error - ## first separtation - t = db.query(db.CompanionRelationships). \ - filter(and_(db.CompanionRelationships.c.projected_separation_arcsec != None, - db.CompanionRelationships.c.projected_separation_arcsec < 0)).astropy() + # first separtation + t = ( + db.query(db.CompanionRelationships) + .filter( + and_( + db.CompanionRelationships.c.projected_separation_arcsec is not None, + db.CompanionRelationships.c.projected_separation_arcsec < 0, + ) + ) + .astropy() + ) if len(t) > 0: - print('\n Negative projected separations') + print("\n Negative projected separations") print(t) assert len(t) == 0 - ## separation error - t = db.query(db.CompanionRelationships). \ - filter(and_(db.CompanionRelationships.c.projected_separation_error != None, - db.CompanionRelationships.c.projected_separation_error < 0)). \ - astropy() - + # separation error + t = ( + db.query(db.CompanionRelationships) + .filter( + and_( + db.CompanionRelationships.c.projected_separation_error is not None, + db.CompanionRelationships.c.projected_separation_error < 0, + ) + ) + .astropy() + ) + if len(t) > 0: - print('\n Negative projected separations') + print("\n Negative projected separations") print(t) assert len(t) == 0 - # test correct relationship - possible_relationships = ['Child', 'Sibling', 'Parent', 'Unresolved Parent'] - t = db.query(db.CompanionRelationships). \ - filter(~db.CompanionRelationships.c.relationship.in_(possible_relationships)). \ - astropy() + # test correct relationship + possible_relationships = ["Child", "Sibling", "Parent", "Unresolved Parent"] + t = ( + db.query(db.CompanionRelationships) + .filter(~db.CompanionRelationships.c.relationship.in_(possible_relationships)) + .astropy() + ) if len(t) > 0: - print('\n relationship is of the souce to its companion \ - should be one of the following: Child, Sibling, Parent, or Unresolved Parent') + print( + "\n relationship is of the souce to its companion \ + should be one of the following: Child, Sibling, Parent, or Unresolved Parent" + ) print(t) assert len(t) == 0 def test_companion_relationship_uniqueness(db): # Verify that all souces and companion_names values are unique combinations - ## first finding duplicate sources - sql_text = "SELECT CompanionRelationships.source FROM CompanionRelationships GROUP BY source " \ - "HAVING (Count(*) > 1)" - duplicate_sources = db.sql_query(sql_text, fmt='astropy') - - ##checking duplicate sources have different companions + # first finding duplicate sources + sql_text = ( + "SELECT CompanionRelationships.source " + "FROM CompanionRelationships GROUP BY source " + "HAVING (Count(*) > 1)" + ) + duplicate_sources = db.sql_query(sql_text, fmt="astropy") + + # checking duplicate sources have different companions non_unique = [] for source in duplicate_sources: t = db.query(db.CompanionRelationships.c.companion_name) - filter(db.CompanionRelationships.c.source == source). \ - astropy() - duplicate_companions = [n for n, companion in enumerate(t) if companion in t[:n]] + filter(db.CompanionRelationships.c.source == source).astropy() + duplicate_companions = [ + n for n, companion in enumerate(t) if companion in t[:n] + ] if len(duplicate_companions) > 0: non_unique.append(f"{source} and {duplicate_companions}") if len(non_unique) > 0: - print('\n Non-unique companion combination(s)') + print("\n Non-unique companion combination(s)") print(non_unique) assert len(non_unique) == 0 -def test_names_uniqueness(db): +def test_names_uniqueness(db): # Verify that all Names.other_name values are unique - sql_text = "SELECT Names.other_name FROM Names GROUP BY other_name " \ - "HAVING (Count(*) > 1)" - duplicate_names = db.sql_query(sql_text, fmt='astropy') + sql_text = ( + "SELECT Names.other_name FROM Names GROUP BY other_name " + "HAVING (Count(*) > 1)" + ) + duplicate_names = db.sql_query(sql_text, fmt="astropy") # if duplicate_names is non_zero, print out duplicate other names if len(duplicate_names) > 0: - print(f'\n{len(duplicate_names)} possibly a duplicated other_name.') + print(f"\n{len(duplicate_names)} possibly a duplicated other_name.") print(duplicate_names) assert len(duplicate_names) == 0 diff --git a/tests/test_utils.py b/tests/test_utils.py index 347f995d3..ce17dc150 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,10 +2,8 @@ import pytest import sys import math - -# pylint: disable=all - -sys.path.append('.') +import os +sys.path.append(".") from scripts.ingests.utils import * from scripts.ingests.ingest_utils import * from simple.schema import * @@ -13,8 +11,8 @@ from astropy.table import Table from sqlalchemy import and_ -DB_NAME = 'temp.db' -DB_PATH = 'data' +DB_NAME = "temp.db" +DB_PATH = "data" logger.setLevel(logging.INFO) @@ -26,14 +24,14 @@ def db(): if os.path.exists(DB_NAME): os.remove(DB_NAME) - connection_string = 'sqlite:///' + DB_NAME + connection_string = "sqlite:///" + DB_NAME create_database(connection_string) assert os.path.exists(DB_NAME) # Connect to the new database and confirm it has the Sources table db = Database(connection_string) assert db - assert 'source' in [c.name for c in db.Sources.columns] + assert "source" in [c.name for c in db.Sources.columns] return db @@ -41,10 +39,13 @@ def db(): # Create fake astropy Table of data to load @pytest.fixture(scope="module") def t_plx(): - t_plx = Table([{'source': 'Fake 1', 'plx': 113., 'plx_err': 0.3, 'plx_ref': 'Ref 1'}, - {'source': 'Fake 2', 'plx': 145., 'plx_err': 0.5, 'plx_ref': 'Ref 1'}, - {'source': 'Fake 3', 'plx': 155., 'plx_err': 0.6, 'plx_ref': 'Ref 2'}, - ]) + t_plx = Table( + [ + {"source": "Fake 1", "plx": 113.0, "plx_err": 0.3, "plx_ref": "Ref 1"}, + {"source": "Fake 2", "plx": 145.0, "plx_err": 0.5, "plx_ref": "Ref 1"}, + {"source": "Fake 3", "plx": 155.0, "plx_err": 0.6, "plx_ref": "Ref 2"}, + ] + ) return t_plx @@ -52,23 +53,53 @@ def t_plx(): @pytest.fixture(scope="module") def t_pm(): t_pm = Table( - [{'source': 'Fake 1', 'mu_ra': 113., 'mu_ra_err': 0.3, 'mu_dec': 113., 'mu_dec_err': 0.3, 'reference': 'Ref 1'}, - {'source': 'Fake 2', 'mu_ra': 145., 'mu_ra_err': 0.5, 'mu_dec': 113., 'mu_dec_err': 0.3, 'reference': 'Ref 1'}, - {'source': 'Fake 3', 'mu_ra': 55., 'mu_ra_err': 0.23, 'mu_dec': 113., 'mu_dec_err': 0.3, 'reference': 'Ref 2'}, - ]) + [ + { + "source": "Fake 1", + "mu_ra": 113.0, + "mu_ra_err": 0.3, + "mu_dec": 113.0, + "mu_dec_err": 0.3, + "reference": "Ref 1", + }, + { + "source": "Fake 2", + "mu_ra": 145.0, + "mu_ra_err": 0.5, + "mu_dec": 113.0, + "mu_dec_err": 0.3, + "reference": "Ref 1", + }, + { + "source": "Fake 3", + "mu_ra": 55.0, + "mu_ra_err": 0.23, + "mu_dec": 113.0, + "mu_dec_err": 0.3, + "reference": "Ref 2", + }, + ] + ) return t_pm def test_setup_db(db): # Some setup tasks to ensure some data exists in the database first - ref_data = [{'reference': 'Ref 1', 'doi': '10.1093/mnras/staa1522', 'bibcode': '2020MNRAS.496.1922B'}, - {'reference': 'Ref 2', 'doi': 'Doi2', 'bibcode': '2012yCat.2311....0C'}, - {'reference': 'Burn08', 'doi': 'Doi3', 'bibcode': '2008MNRAS.391..320B'}] - - source_data = [{'source': 'Fake 1', 'ra': 9.0673755, 'dec': 18.352889, 'reference': 'Ref 1'}, - {'source': 'Fake 2', 'ra': 9.0673755, 'dec': 18.352889, 'reference': 'Ref 1'}, - {'source': 'Fake 3', 'ra': 9.0673755, 'dec': 18.352889, 'reference': 'Ref 2'}, - ] + ref_data = [ + { + "reference": "Ref 1", + "doi": "10.1093/mnras/staa1522", + "bibcode": "2020MNRAS.496.1922B", + }, + {"reference": "Ref 2", "doi": "Doi2", "bibcode": "2012yCat.2311....0C"}, + {"reference": "Burn08", "doi": "Doi3", "bibcode": "2008MNRAS.391..320B"}, + ] + + source_data = [ + {"source": "Fake 1", "ra": 9.0673755, "dec": 18.352889, "reference": "Ref 1"}, + {"source": "Fake 2", "ra": 9.0673755, "dec": 18.352889, "reference": "Ref 1"}, + {"source": "Fake 3", "ra": 9.0673755, "dec": 18.352889, "reference": "Ref 2"}, + ] with db.engine.connect() as conn: conn.execute(db.Publications.insert().values(ref_data)) @@ -80,121 +111,249 @@ def test_setup_db(db): def test_ingest_sources(db): # TODO: Test adding an alt name - source_data1 = Table([{'source': 'Fake 1', 'ra': 9.0673755, 'dec': 18.352889, 'reference': 'Ref 1'}, - {'source': 'Fake 6', 'ra': 10.0673755, 'dec': 18.352889, 'reference': 'Ref 2'}, - {'source': 'Fake 7', 'ra': 11.0673755, 'dec': 18.352889, 'reference': 'Ref 1'}]) - source_data5 = Table([{'source': 'Fake 5', 'ra': 9.06799, 'dec': 18.352889, 'reference': ''}]) - source_data8 = Table([{'source': 'Fake 8', 'ra': 9.06799, 'dec': 18.352889, 'reference': 'Ref 4'}]) - - ingest_sources(db, source_data1['source'], ras=source_data1['ra'], decs=source_data1['dec'], - references=source_data1['reference'], raise_error=True) - - ingest_sources(db, ['Barnard Star'], references='Ref 2', raise_error=True) - Barnard_star = db.query(db.Sources).filter(db.Sources.c.source == 'Barnard Star').astropy() + source_data1 = Table( + [ + { + "source": "Fake 1", + "ra": 9.0673755, + "dec": 18.352889, + "reference": "Ref 1", + }, + { + "source": "Fake 6", + "ra": 10.0673755, + "dec": 18.352889, + "reference": "Ref 2", + }, + { + "source": "Fake 7", + "ra": 11.0673755, + "dec": 18.352889, + "reference": "Ref 1", + }, + ] + ) + source_data5 = Table( + [{"source": "Fake 5", "ra": 9.06799, "dec": 18.352889, "reference": ""}] + ) + source_data8 = Table( + [{"source": "Fake 8", "ra": 9.06799, "dec": 18.352889, "reference": "Ref 4"}] + ) + + ingest_sources( + db, + source_data1["source"], + ras=source_data1["ra"], + decs=source_data1["dec"], + references=source_data1["reference"], + raise_error=True, + ) + + ingest_sources(db, ["Barnard Star"], references="Ref 2", raise_error=True) + Barnard_star = ( + db.query(db.Sources).filter(db.Sources.c.source == "Barnard Star").astropy() + ) assert len(Barnard_star) == 1 - assert math.isclose(Barnard_star['ra'], 269.452, abs_tol=0.001) - assert math.isclose(Barnard_star['dec'], 4.6933, abs_tol=0.001) + assert math.isclose(Barnard_star["ra"], 269.452, abs_tol=0.001) + assert math.isclose(Barnard_star["dec"], 4.6933, abs_tol=0.001) - assert db.query(db.Sources).filter(db.Sources.c.source == 'Fake 1').count() == 1 - assert db.query(db.Sources).filter(db.Sources.c.source == 'Fake 6').count() == 1 - assert db.query(db.Sources).filter(db.Sources.c.source == 'Fake 7').count() == 1 + assert db.query(db.Sources).filter(db.Sources.c.source == "Fake 1").count() == 1 + assert db.query(db.Sources).filter(db.Sources.c.source == "Fake 6").count() == 1 + assert db.query(db.Sources).filter(db.Sources.c.source == "Fake 7").count() == 1 with pytest.raises(SimpleError) as error_message: - ingest_sources(db, source_data8['source'], ras=source_data8['ra'], decs=source_data5['dec'], - references=source_data8['reference'], raise_error=True) - assert 'not in Publications table' in str(error_message.value) + ingest_sources( + db, + source_data8["source"], + ras=source_data8["ra"], + decs=source_data5["dec"], + references=source_data8["reference"], + raise_error=True, + ) + assert "not in Publications table" in str(error_message.value) with pytest.raises(SimpleError) as error_message: - ingest_sources(db, source_data5['source'], ras=source_data5['ra'], decs=source_data5['dec'], - references=source_data5['reference'], raise_error=True) - assert 'blank' in str(error_message.value) + ingest_sources( + db, + source_data5["source"], + ras=source_data5["ra"], + decs=source_data5["dec"], + references=source_data5["reference"], + raise_error=True, + ) + assert "blank" in str(error_message.value) with pytest.raises(SimpleError) as error_message: - ingest_sources(db, ['NotinSimbad'], references='Ref 1', raise_error=True) - assert 'Coordinates are needed' in str(error_message.value) + ingest_sources(db, ["NotinSimbad"], references="Ref 1", raise_error=True) + assert "Coordinates are needed" in str(error_message.value) def test_convert_spt_string_to_code(): # Test conversion of spectral types into numeric values - assert convert_spt_string_to_code(['M5.6']) == [65.6] - assert convert_spt_string_to_code(['T0.1']) == [80.1] - assert convert_spt_string_to_code(['Y2pec']) == [92] + assert convert_spt_string_to_code(["M5.6"]) == [65.6] + assert convert_spt_string_to_code(["T0.1"]) == [80.1] + assert convert_spt_string_to_code(["Y2pec"]) == [92] def test_ingest_parallaxes(db, t_plx): # Test ingest of parallax data - ingest_parallaxes(db, t_plx['source'], t_plx['plx'], t_plx['plx_err'], t_plx['plx_ref']) + ingest_parallaxes( + db, t_plx["source"], t_plx["plx"], t_plx["plx_err"], t_plx["plx_ref"] + ) - results = db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == 'Ref 1').table() + results = ( + db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == "Ref 1").table() + ) assert len(results) == 2 - results = db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == 'Ref 2').table() + results = ( + db.query(db.Parallaxes).filter(db.Parallaxes.c.reference == "Ref 2").table() + ) assert len(results) == 1 - assert results['source'][0] == 'Fake 3' - assert results['parallax'][0] == 155 - assert results['parallax_error'][0] == 0.6 + assert results["source"][0] == "Fake 3" + assert results["parallax"][0] == 155 + assert results["parallax_error"][0] == 0.6 def test_ingest_proper_motions(db, t_pm): - ingest_proper_motions(db, t_pm['source'], t_pm['mu_ra'], t_pm['mu_ra_err'], - t_pm['mu_dec'], t_pm['mu_dec_err'], t_pm['reference']) - assert db.query(db.ProperMotions).filter(db.ProperMotions.c.reference == 'Ref 1').count() == 2 - results = db.query(db.ProperMotions).filter(db.ProperMotions.c.reference == 'Ref 2').table() + ingest_proper_motions( + db, + t_pm["source"], + t_pm["mu_ra"], + t_pm["mu_ra_err"], + t_pm["mu_dec"], + t_pm["mu_dec_err"], + t_pm["reference"], + ) + assert ( + db.query(db.ProperMotions) + .filter(db.ProperMotions.c.reference == "Ref 1") + .count() + == 2 + ) + results = ( + db.query(db.ProperMotions) + .filter(db.ProperMotions.c.reference == "Ref 2") + .table() + ) assert len(results) == 1 - assert results['source'][0] == 'Fake 3' - assert results['mu_ra'][0] == 55 - assert results['mu_ra_error'][0] == 0.23 + assert results["source"][0] == "Fake 3" + assert results["mu_ra"][0] == 55 + assert results["mu_ra_error"][0] == 0.23 def test_ingest_spectral_types(db): - data1 = Table([{'source': 'Fake 1', 'spectral_type': 'M5.6', 'regime': 'nir', 'reference': 'Ref 1'}, - {'source': 'Fake 2', 'spectral_type': 'T0.1', 'regime': 'nir', 'reference': 'Ref 1'}, - {'source': 'Fake 3', 'spectral_type': 'Y2pec', 'regime': 'nir', 'reference': 'Ref 2'}, - ]) - - data2 = Table([{'source': 'Fake 1', 'spectral_type': 'M5.6', 'reference': 'Ref 1'}, - {'source': 'Fake 2', 'spectral_type': 'T0.1', 'reference': 'Ref 1'}, - {'source': 'Fake 3', 'spectral_type': 'Y2pec', 'reference': 'Ref 2'}, - ]) - - data3 = Table([{'source': 'Fake 1', 'spectral_type': 'M5.6', 'regime': 'nir', 'reference': 'Ref 1'}, - {'source': 'Fake 2', 'spectral_type': 'T0.1', 'regime': 'nir', 'reference': 'Ref 1'}, - {'source': 'Fake 3', 'spectral_type': 'Y2pec', 'regime': 'nir', 'reference': 'Ref 4'}, - ]) - ingest_spectral_types(db, data1['source'], data1['spectral_type'], data1['reference'], data1['regime']) - assert db.query(db.SpectralTypes).filter(db.SpectralTypes.c.reference == 'Ref 1').count() == 2 - results = db.query(db.SpectralTypes).filter(db.SpectralTypes.c.reference == 'Ref 2').table() + data1 = Table( + [ + { + "source": "Fake 1", + "spectral_type": "M5.6", + "regime": "nir", + "reference": "Ref 1", + }, + { + "source": "Fake 2", + "spectral_type": "T0.1", + "regime": "nir", + "reference": "Ref 1", + }, + { + "source": "Fake 3", + "spectral_type": "Y2pec", + "regime": "nir", + "reference": "Ref 2", + }, + ] + ) + + data2 = Table( + [ + {"source": "Fake 1", "spectral_type": "M5.6", "reference": "Ref 1"}, + {"source": "Fake 2", "spectral_type": "T0.1", "reference": "Ref 1"}, + {"source": "Fake 3", "spectral_type": "Y2pec", "reference": "Ref 2"}, + ] + ) + + data3 = Table( + [ + { + "source": "Fake 1", + "spectral_type": "M5.6", + "regime": "nir", + "reference": "Ref 1", + }, + { + "source": "Fake 2", + "spectral_type": "T0.1", + "regime": "nir", + "reference": "Ref 1", + }, + { + "source": "Fake 3", + "spectral_type": "Y2pec", + "regime": "nir", + "reference": "Ref 4", + }, + ] + ) + ingest_spectral_types( + db, data1["source"], data1["spectral_type"], data1["reference"], data1["regime"] + ) + assert ( + db.query(db.SpectralTypes) + .filter(db.SpectralTypes.c.reference == "Ref 1") + .count() + == 2 + ) + results = ( + db.query(db.SpectralTypes) + .filter(db.SpectralTypes.c.reference == "Ref 2") + .table() + ) assert len(results) == 1 - assert results['source'][0] == 'Fake 3' - assert results['spectral_type_string'][0] == 'Y2pec' - assert results['spectral_type_code'][0] == [92] + assert results["source"][0] == "Fake 3" + assert results["spectral_type_string"][0] == "Y2pec" + assert results["spectral_type_code"][0] == [92] # testing for publication error with pytest.raises(SimpleError) as error_message: - ingest_spectral_types(db, data3['source'], data3['spectral_type'], data3['regime'], data3['reference']) - assert 'The publication does not exist in the database' in str(error_message.value) + ingest_spectral_types( + db, + data3["source"], + data3["spectral_type"], + data3["regime"], + data3["reference"], + ) + assert "The publication does not exist in the database" in str( + error_message.value + ) def test_find_publication(db): assert not find_publication(db)[0] # False - assert find_publication(db, name='Ref 1')[0] # True - assert find_publication(db, name='Ref 1', doi='10.1093/mnras/staa1522')[0] # True - doi_search = find_publication(db, doi='10.1093/mnras/staa1522') + assert find_publication(db, name="Ref 1")[0] # True + assert find_publication(db, name="Ref 1", doi="10.1093/mnras/staa1522")[0] # True + doi_search = find_publication(db, doi="10.1093/mnras/staa1522") assert doi_search[0] # True - assert doi_search[1] == 'Ref 1' - bibcode_search = find_publication(db, bibcode='2020MNRAS.496.1922B') + assert doi_search[1] == "Ref 1" + bibcode_search = find_publication(db, bibcode="2020MNRAS.496.1922B") assert bibcode_search[0] # True - assert bibcode_search[1] == 'Ref 1' - multiple_matches = find_publication(db, name='Ref') + assert bibcode_search[1] == "Ref 1" + multiple_matches = find_publication(db, name="Ref") assert not multiple_matches[0] # False, multiple matches assert multiple_matches[1] == 2 # multiple matches - assert not find_publication(db, name='Ref 2', doi='10.1093/mnras/staa1522')[0] # False - assert not find_publication(db, name='Ref 2', bibcode='2020MNRAS.496.1922B')[0] # False - assert find_publication(db, name='Burningham_2008') == (1, 'Burn08') + assert not find_publication(db, name="Ref 2", doi="10.1093/mnras/staa1522")[ + 0 + ] # False + assert not find_publication(db, name="Ref 2", bibcode="2020MNRAS.496.1922B")[ + 0 + ] # False + assert find_publication(db, name="Burningham_2008") == (1, "Burn08") + def test_ingest_publication(db): # should fail if trying to add a duplicate record with pytest.raises(SimpleError) as error_message: - ingest_publication(db, publication='Ref 1', bibcode='2020MNRAS.496.1922B') - assert ' similar publication already exists' in str(error_message.value) + ingest_publication(db, publication="Ref 1", bibcode="2020MNRAS.496.1922B") + assert " similar publication already exists" in str(error_message.value) # TODO - Mock environment where ADS_TOKEN is not set. #117 @@ -202,10 +361,12 @@ def test_ingest_instrument(db): # TESTS WHICH SHOULD WORK # test adding just telescope - ingest_instrument(db, telescope='test') - telescope_db = db.query(db.Telescopes).filter(db.Telescopes.c.telescope == 'test').table() + ingest_instrument(db, telescope="test") + telescope_db = ( + db.query(db.Telescopes).filter(db.Telescopes.c.telescope == "test").table() + ) assert len(telescope_db) == 1 - assert telescope_db['telescope'][0] == 'test' + assert telescope_db["telescope"][0] == "test" # No longer supported just adding an instrument without a mode # test adding telescope and instrument @@ -220,67 +381,94 @@ def test_ingest_instrument(db): # assert instrument_db['instrument'][0] == inst_test # test adding new telescope, instrument, and mode - tel_test = 'test4' - inst_test = 'test5' - mode_test = 'test6' + tel_test = "test4" + inst_test = "test5" + mode_test = "test6" ingest_instrument(db, telescope=tel_test, instrument=inst_test, mode=mode_test) - telescope_db = db.query(db.Telescopes).filter(db.Telescopes.c.telescope == tel_test).table() - instrument_db = db.query(db.Instruments).filter(and_(db.Instruments.c.mode == mode_test, - db.Instruments.c.instrument == inst_test, - db.Instruments.c.telescope == tel_test)).table() - assert len(telescope_db) == 1, 'Missing telescope insert' - assert telescope_db['telescope'][0] == tel_test + telescope_db = ( + db.query(db.Telescopes).filter(db.Telescopes.c.telescope == tel_test).table() + ) + instrument_db = ( + db.query(db.Instruments) + .filter( + and_( + db.Instruments.c.mode == mode_test, + db.Instruments.c.instrument == inst_test, + db.Instruments.c.telescope == tel_test, + ) + ) + .table() + ) + assert len(telescope_db) == 1, "Missing telescope insert" + assert telescope_db["telescope"][0] == tel_test assert len(instrument_db) == 1 - assert instrument_db['instrument'][0] == inst_test - assert instrument_db['mode'][0] == mode_test + assert instrument_db["instrument"][0] == inst_test + assert instrument_db["mode"][0] == mode_test # test adding common mode name for new telescope, instrument - tel_test = 'test4' - inst_test = 'test5' - mode_test = 'Prism' + tel_test = "test4" + inst_test = "test5" + mode_test = "Prism" print(db.query(db.Telescopes).table()) print(db.query(db.Instruments).table()) ingest_instrument(db, telescope=tel_test, instrument=inst_test, mode=mode_test) - mode_db = db.query(db.Instruments).filter(and_(db.Instruments.c.mode == mode_test, - db.Instruments.c.instrument == inst_test, - db.Instruments.c.telescope == tel_test)).table() + mode_db = ( + db.query(db.Instruments) + .filter( + and_( + db.Instruments.c.mode == mode_test, + db.Instruments.c.instrument == inst_test, + db.Instruments.c.telescope == tel_test, + ) + ) + .table() + ) assert len(mode_db) == 1 - assert mode_db['mode'][0] == mode_test + assert mode_db["mode"][0] == mode_test # TESTS WHICH SHOULD FAIL # test with no variables provided with pytest.raises(SimpleError) as error_message: ingest_instrument(db) - assert 'Telescope, Instrument, and Mode must be provided' in str(error_message.value) + assert "Telescope, Instrument, and Mode must be provided" in str( + error_message.value + ) # test with mode but no instrument or telescope with pytest.raises(SimpleError) as error_message: - ingest_instrument(db, mode='test') - assert 'Telescope, Instrument, and Mode must be provided' in str(error_message.value) + ingest_instrument(db, mode="test") + assert "Telescope, Instrument, and Mode must be provided" in str( + error_message.value + ) + # TODO: test for ingest_photometry # TODO: test for ingest_spectra + def test_companion_relationships(db): -# testing companion ingest -## trying no companion + # testing companion ingest + ## trying no companion with pytest.raises(SimpleError) as error_message: - ingest_companion_relationships(db, 'Fake 1', None, "Sibling" ) + ingest_companion_relationships(db, "Fake 1", None, "Sibling") assert "Make sure all require parameters are provided." in str(error_message.value) ## trying companion == source with pytest.raises(SimpleError) as error_message: - ingest_companion_relationships(db, 'Fake 1', 'Fake 1', "Sibling" ) - assert "Source cannot be the same as companion name" in str(error_message.value) + ingest_companion_relationships(db, "Fake 1", "Fake 1", "Sibling") + assert "Source cannot be the same as companion name" in str(error_message.value) ## trying negative separation with pytest.raises(SimpleError) as error_message: - ingest_companion_relationships(db, 'Fake 1', 'Bad Companion', "Sibling", projected_separation_arcsec = -5) - assert 'cannot be negative' in str(error_message.value) + ingest_companion_relationships( + db, "Fake 1", "Bad Companion", "Sibling", projected_separation_arcsec=-5 + ) + assert "cannot be negative" in str(error_message.value) ## trying negative separation error with pytest.raises(SimpleError) as error_message: - ingest_companion_relationships(db, 'Fake 1', 'Bad Companion', "Sibling" , projected_separation_error = -5) - assert 'cannot be negative' in str(error_message.value) - + ingest_companion_relationships( + db, "Fake 1", "Bad Companion", "Sibling", projected_separation_error=-5 + ) + assert "cannot be negative" in str(error_message.value) From 4e43af06ca1c03dd481b84f3c1db4dbaa2c55e85 Mon Sep 17 00:00:00 2001 From: kelle Date: Wed, 20 Dec 2023 16:38:07 -0500 Subject: [PATCH 24/35] fixed raise_error bug --- scripts/ingests/ingest_utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/ingests/ingest_utils.py b/scripts/ingests/ingest_utils.py index 3e0b80969..0080f9e52 100644 --- a/scripts/ingests/ingest_utils.py +++ b/scripts/ingests/ingest_utils.py @@ -1926,10 +1926,8 @@ def ingest_source( logger.debug(f"name_matches: {name_matches}") # Source is already in database + # Checking out alternate names if len(name_matches) == 1 and search_db: - msg = f"Not ingesting {source}. Already in database as {name_matches[0]}. \n " - logger.info(msg) - # Figure out if source name provided is an alternate name db_source_matches = db.search_object( source, output_table="Sources", fuzzy_search=False @@ -1950,7 +1948,14 @@ def ingest_source( raise SimpleError(msg + "\n" + str(e)) else: return - return # Source is already in database, nothing new to ingest + + msg = f"Not ingesting {source}. Already in database as {name_matches[0]}. \n " + if raise_error: + raise SimpleError(msg) + else: + logger.info(msg) + return # Source is already in database, nothing new to ingest + # Multiple source matches in the database so unable to ingest source elif len(name_matches) > 1 and search_db: msg1 = f" Not ingesting {source}." From 6a48f3e5fa74f7225d8daf8275a151ed5145f2e5 Mon Sep 17 00:00:00 2001 From: kelle Date: Wed, 20 Dec 2023 16:45:44 -0500 Subject: [PATCH 25/35] changes blobs to None --- scripts/ingests/ingest_BYW_2024.py | 99 ++++++++++++++++++------------ 1 file changed, 61 insertions(+), 38 deletions(-) diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 3033ceac7..a563ffa83 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -2,69 +2,92 @@ from scripts.ingests.utils import * from astropy.io import ascii -SAVE_DB = True # save the data files in addition to modifying the .db file -RECREATE_DB = True # recreates the .db file from the data files +SAVE_DB = False # save the data files in addition to modifying the .db file +RECREATE_DB = False # recreates the .db file from the data files # LOAD THE DATABASE -db = load_simpledb('SIMPLE.db', recreatedb=RECREATE_DB) +db = load_simpledb("SIMPLE.db", recreatedb=RECREATE_DB) # Load Google sheet -sheet_id = '1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs' +sheet_id = "1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs" link = f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv" -#read the csv data into an astropy table -#ascii.read attempts to read data from local files rather from URLs so using a library like requests helps get data and create object that can be passed to ascii.read -byw_table = ascii.read(link, format='csv', data_start=1, header_start=0, guess=False, fast_reader=False, delimiter=',') - -#print result astropy table +# read the csv data into an astropy table +# ascii.read attempts to read data from local files rather from URLs so using a library like requests helps get data and create object that can be passed to ascii.read +byw_table = ascii.read( + link, + format="csv", + data_start=1, + header_start=0, + guess=False, + fast_reader=False, + delimiter=",", +) + +# print result astropy table print(byw_table.info) -#Loop through each row in byw table and print data: source name ra, dec. -def ingest_all_sources(db): +# Loop through each row in byw table and print data: source name ra, dec. +def ingest_all_sources(db): for row in byw_table: # skip the header row - [1:10]runs only first 10 rows - # Print byw source information print("BYW Source Information:") - #for key, value in row_dict.items(): - #print(f"{key}: {value}") + # for key, value in row_dict.items(): + # print(f"{key}: {value}") for col_name in row.colnames: print(f"{col_name}: {row[col_name]}") - - - ingest_source(db, source = row['Source'], - reference= row['Reference'], - ra= row['RA'], - dec= row['Dec'], - epoch= row['Epoch'], - equinox= row['Equinox'], - raise_error=True, - search_db=True) - #print(row['ra']) - #print(row_dict['ra']) + + ingest_source( + db, + source=row["Source"], + reference=row["Reference"], + ra=row["RA"], + dec=row["Dec"], + epoch=row["Epoch"], + equinox=row["Equinox"], + raise_error=True, + search_db=True, + ) + # print(row['ra']) + # print(row_dict['ra']) # Add a separator between rows for better readability print("-" * 20) -#Ingested other_ref sources as publications +# Ingested other_ref sources as publications # Skrzypek et al. 2016, Marocco et al. 2015(Online Catalog), Kirkpatrick et al. 2021 -#Ingest reference name: Skrz16, Maro15, Kirk21 - +# Ingest reference name: Skrz16, Maro15, Kirk21 + + def add_publication(db): + ingest_publication( + db, doi="10.26093/cds/vizier.35890049", bibcode="2016yCat..35890049S" + ) + + ingest_publication(db, doi="10.1093/mnras/stv530", bibcode="2015MNRAS.449.3651M") + + ingest_publication( + db, doi="10.3847/1538-4365/abd107", bibcode="2021ApJS..253....7K" + ) + - ingest_publication(db, doi = "10.26093/cds/vizier.35890049", bibcode = "2016yCat..35890049S") +def fix_blobs(db): + with db.engine.begin() as conn: + conn.execute( + db.Sources.update() + .where(db.Sources.c.reference == "Roth") + .values(other_references=None, comments=None) + ) - ingest_publication(db, doi = "10.1093/mnras/stv530", bibcode = "2015MNRAS.449.3651M") - ingest_publication(db, doi = "10.3847/1538-4365/abd107", bibcode = "2021ApJS..253....7K") +# calling functions +# add_publication(db) +ingest_all_sources(db) -#calling functions -#add_publication(db) -ingest_all_sources(db) - -db.inventory('CWISE J000021.45-481314.9', pretty_print=True) +db.inventory("CWISE J000021.45-481314.9", pretty_print=True) # WRITE THE JSON FILES if SAVE_DB: - db.save_database(directory='data/') \ No newline at end of file + db.save_database(directory="data/") From 1b31552e52adc72be94885129d2c3ef802fb9bbe Mon Sep 17 00:00:00 2001 From: kelle Date: Wed, 20 Dec 2023 16:46:39 -0500 Subject: [PATCH 26/35] fixed JSON files --- data/cwise_j000021.45-481314.9.json | 4 ++-- data/cwise_j002029.72-153527.5.json | 4 ++-- data/cwise_j002101.45-334631.5.json | 4 ++-- data/cwise_j002159.02-451434.4.json | 4 ++-- data/cwise_j002414.30-403053.7.json | 4 ++-- data/cwise_j002658.73-260859.6.json | 4 ++-- data/cwise_j004218.67+291730.6.json | 4 ++-- data/cwise_j004945.38+423619.3.json | 4 ++-- data/cwise_j005635.48-240401.9.json | 4 ++-- data/cwise_j010424.50+133949.8.json | 4 ++-- data/cwise_j011003.61-592650.8.json | 4 ++-- data/cwise_j012715.52-300246.9.json | 4 ++-- data/cwise_j013719.04+342837.6.json | 4 ++-- data/cwise_j015905.58+105551.5.json | 4 ++-- data/cwise_j020538.20+594452.2.json | 4 ++-- data/cwise_j022454.80+152629.5.json | 4 ++-- data/cwise_j022737.75+083008.8.json | 4 ++-- data/cwise_j025645.16-335008.9.json | 4 ++-- data/cwise_j030005.73-062218.6.json | 4 ++-- data/cwise_j032853.32-112332.6.json | 4 ++-- data/cwise_j055515.83+510514.0.json | 4 ++-- data/cwise_j055909.00-384219.8.json | 4 ++-- data/cwise_j060202.17-462447.8.json | 4 ++-- data/cwise_j062000.01+344641.3.json | 4 ++-- data/cwise_j062648.96+594129.2.json | 4 ++-- data/cwise_j062727.34-002826.8.json | 4 ++-- data/cwise_j062909.21+395701.6.json | 4 ++-- data/cwise_j065752.45+163350.2.json | 4 ++-- data/cwise_j073831.31+525453.7.json | 4 ++-- data/cwise_j080912.95+741704.3.json | 4 ++-- data/cwise_j085131.24-603056.2.json | 4 ++-- data/cwise_j091558.53+254713.0.json | 4 ++-- data/cwise_j091902.55-574837.3.json | 4 ++-- data/cwise_j094352.22+335639.1.json | 4 ++-- data/cwise_j095259.54-505418.9.json | 4 ++-- data/cwise_j101017.43-242300.4.json | 4 ++-- data/cwise_j101317.51-010313.6.json | 4 ++-- data/cwise_j101523.92-111539.6.json | 4 ++-- data/cwise_j103734.29-050749.8.json | 4 ++-- data/cwise_j104053.42-355029.7.json | 4 ++-- data/cwise_j115229.05-351105.9.json | 4 ++-- data/cwise_j124033.48+460544.7.json | 4 ++-- data/cwise_j125858.20+001133.2.json | 4 ++-- data/cwise_j130329.90+512754.0.json | 4 ++-- data/cwise_j130446.64-120023.6.json | 4 ++-- data/cwise_j131355.15+223005.6.json | 4 ++-- data/cwise_j132539.70+022309.4.json | 4 ++-- data/cwise_j132857.58-063747.4.json | 4 ++-- data/cwise_j133211.93-374837.9.json | 4 ++-- data/cwise_j133427.70-273053.1.json | 4 ++-- data/cwise_j141737.21+041847.2.json | 4 ++-- data/cwise_j143935.94-804851.5.json | 4 ++-- data/cwise_j143951.51+260000.3.json | 4 ++-- data/cwise_j144057.87+380432.8.json | 4 ++-- data/cwise_j151939.68-485820.4.json | 4 ++-- data/cwise_j152742.44-121528.2.json | 4 ++-- data/cwise_j153910.07+722757.2.json | 4 ++-- data/cwise_j160654.19-103214.7.json | 4 ++-- data/cwise_j162511.27+774946.8.json | 4 ++-- data/cwise_j165143.63+695259.4.json | 4 ++-- data/cwise_j165325.10+490439.7.json | 4 ++-- data/cwise_j171308.00+644220.8.json | 4 ++-- data/cwise_j171700.57+724437.3.json | 4 ++-- data/cwise_j173859.73+200501.2.json | 4 ++-- data/cwise_j174426.85+230355.1.json | 4 ++-- data/cwise_j174509.03+380733.2.json | 4 ++-- data/cwise_j190909.62+553027.1.json | 4 ++-- data/cwise_j192425.03-532837.1.json | 4 ++-- data/cwise_j195956.16-644318.1.json | 4 ++-- data/cwise_j200304.70-142228.9.json | 4 ++-- data/cwise_j200813.66-124502.4.json | 4 ++-- data/cwise_j202934.80-791013.1.json | 4 ++-- data/cwise_j205247.07+053855.7.json | 4 ++-- data/cwise_j212342.88+655615.6.json | 4 ++-- data/cwise_j214129.80-024623.6.json | 4 ++-- data/cwise_j214213.99-204659.8.json | 4 ++-- data/cwise_j221418.15+253432.2.json | 4 ++-- data/cwise_j224452.18-362946.0.json | 4 ++-- data/cwise_j225525.22-302320.8.json | 4 ++-- data/cwise_j231824.58+052142.3.json | 4 ++-- data/cwise_j233531.55+014219.6.json | 4 ++-- data/cwise_j235827.96-521813.4.json | 4 ++-- 82 files changed, 164 insertions(+), 164 deletions(-) diff --git a/data/cwise_j000021.45-481314.9.json b/data/cwise_j000021.45-481314.9.json index 1eb836967..f941bef11 100644 --- a/data/cwise_j000021.45-481314.9.json +++ b/data/cwise_j000021.45-481314.9.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j002029.72-153527.5.json b/data/cwise_j002029.72-153527.5.json index 306e7fb2f..c05725078 100644 --- a/data/cwise_j002029.72-153527.5.json +++ b/data/cwise_j002029.72-153527.5.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j002101.45-334631.5.json b/data/cwise_j002101.45-334631.5.json index f8576f887..0e08b742b 100644 --- a/data/cwise_j002101.45-334631.5.json +++ b/data/cwise_j002101.45-334631.5.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j002159.02-451434.4.json b/data/cwise_j002159.02-451434.4.json index 22d15e0bd..5e8951ad7 100644 --- a/data/cwise_j002159.02-451434.4.json +++ b/data/cwise_j002159.02-451434.4.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j002414.30-403053.7.json b/data/cwise_j002414.30-403053.7.json index 9dab9d390..04c9f9631 100644 --- a/data/cwise_j002414.30-403053.7.json +++ b/data/cwise_j002414.30-403053.7.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j002658.73-260859.6.json b/data/cwise_j002658.73-260859.6.json index b65abdc3d..40e76eadf 100644 --- a/data/cwise_j002658.73-260859.6.json +++ b/data/cwise_j002658.73-260859.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j004218.67+291730.6.json b/data/cwise_j004218.67+291730.6.json index c2d20efea..24ad31360 100644 --- a/data/cwise_j004218.67+291730.6.json +++ b/data/cwise_j004218.67+291730.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j004945.38+423619.3.json b/data/cwise_j004945.38+423619.3.json index 92d3d696d..60c798189 100644 --- a/data/cwise_j004945.38+423619.3.json +++ b/data/cwise_j004945.38+423619.3.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j005635.48-240401.9.json b/data/cwise_j005635.48-240401.9.json index 614aadc71..8841443a3 100644 --- a/data/cwise_j005635.48-240401.9.json +++ b/data/cwise_j005635.48-240401.9.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j010424.50+133949.8.json b/data/cwise_j010424.50+133949.8.json index d980cbd8f..61ce8ae9e 100644 --- a/data/cwise_j010424.50+133949.8.json +++ b/data/cwise_j010424.50+133949.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j011003.61-592650.8.json b/data/cwise_j011003.61-592650.8.json index a1ad9d2cf..57fa5cdc4 100644 --- a/data/cwise_j011003.61-592650.8.json +++ b/data/cwise_j011003.61-592650.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j012715.52-300246.9.json b/data/cwise_j012715.52-300246.9.json index a601125fb..144abad73 100644 --- a/data/cwise_j012715.52-300246.9.json +++ b/data/cwise_j012715.52-300246.9.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j013719.04+342837.6.json b/data/cwise_j013719.04+342837.6.json index e2979731a..9793e13fa 100644 --- a/data/cwise_j013719.04+342837.6.json +++ b/data/cwise_j013719.04+342837.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j015905.58+105551.5.json b/data/cwise_j015905.58+105551.5.json index e1b7a53da..23e09081f 100644 --- a/data/cwise_j015905.58+105551.5.json +++ b/data/cwise_j015905.58+105551.5.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j020538.20+594452.2.json b/data/cwise_j020538.20+594452.2.json index c30e7114e..3cb73888a 100644 --- a/data/cwise_j020538.20+594452.2.json +++ b/data/cwise_j020538.20+594452.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j022454.80+152629.5.json b/data/cwise_j022454.80+152629.5.json index b64f4c6a6..db2792faa 100644 --- a/data/cwise_j022454.80+152629.5.json +++ b/data/cwise_j022454.80+152629.5.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j022737.75+083008.8.json b/data/cwise_j022737.75+083008.8.json index 414b00d35..f0d9d3022 100644 --- a/data/cwise_j022737.75+083008.8.json +++ b/data/cwise_j022737.75+083008.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j025645.16-335008.9.json b/data/cwise_j025645.16-335008.9.json index 4bc015178..7866f35f6 100644 --- a/data/cwise_j025645.16-335008.9.json +++ b/data/cwise_j025645.16-335008.9.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j030005.73-062218.6.json b/data/cwise_j030005.73-062218.6.json index 2e046ee29..231744941 100644 --- a/data/cwise_j030005.73-062218.6.json +++ b/data/cwise_j030005.73-062218.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j032853.32-112332.6.json b/data/cwise_j032853.32-112332.6.json index b6eb589cd..75c296ccf 100644 --- a/data/cwise_j032853.32-112332.6.json +++ b/data/cwise_j032853.32-112332.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j055515.83+510514.0.json b/data/cwise_j055515.83+510514.0.json index 3e76fe9ae..1298c6bd3 100644 --- a/data/cwise_j055515.83+510514.0.json +++ b/data/cwise_j055515.83+510514.0.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j055909.00-384219.8.json b/data/cwise_j055909.00-384219.8.json index 147eda74f..c580ca25c 100644 --- a/data/cwise_j055909.00-384219.8.json +++ b/data/cwise_j055909.00-384219.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j060202.17-462447.8.json b/data/cwise_j060202.17-462447.8.json index 8d01af33f..363f5aea1 100644 --- a/data/cwise_j060202.17-462447.8.json +++ b/data/cwise_j060202.17-462447.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j062000.01+344641.3.json b/data/cwise_j062000.01+344641.3.json index 3bee1fa8b..09db9679c 100644 --- a/data/cwise_j062000.01+344641.3.json +++ b/data/cwise_j062000.01+344641.3.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j062648.96+594129.2.json b/data/cwise_j062648.96+594129.2.json index e8aabeac8..b98dc2954 100644 --- a/data/cwise_j062648.96+594129.2.json +++ b/data/cwise_j062648.96+594129.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j062727.34-002826.8.json b/data/cwise_j062727.34-002826.8.json index faafab9e3..525ddc1b0 100644 --- a/data/cwise_j062727.34-002826.8.json +++ b/data/cwise_j062727.34-002826.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j062909.21+395701.6.json b/data/cwise_j062909.21+395701.6.json index 284b643e2..fe18e0695 100644 --- a/data/cwise_j062909.21+395701.6.json +++ b/data/cwise_j062909.21+395701.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j065752.45+163350.2.json b/data/cwise_j065752.45+163350.2.json index d56e5d859..c6b7c49c6 100644 --- a/data/cwise_j065752.45+163350.2.json +++ b/data/cwise_j065752.45+163350.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j073831.31+525453.7.json b/data/cwise_j073831.31+525453.7.json index 4a52eec10..f30adfd3c 100644 --- a/data/cwise_j073831.31+525453.7.json +++ b/data/cwise_j073831.31+525453.7.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j080912.95+741704.3.json b/data/cwise_j080912.95+741704.3.json index 1d24bee0f..8196f0c75 100644 --- a/data/cwise_j080912.95+741704.3.json +++ b/data/cwise_j080912.95+741704.3.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j085131.24-603056.2.json b/data/cwise_j085131.24-603056.2.json index f884217bc..248e8d91f 100644 --- a/data/cwise_j085131.24-603056.2.json +++ b/data/cwise_j085131.24-603056.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j091558.53+254713.0.json b/data/cwise_j091558.53+254713.0.json index 009f09ac2..1da4f652b 100644 --- a/data/cwise_j091558.53+254713.0.json +++ b/data/cwise_j091558.53+254713.0.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j091902.55-574837.3.json b/data/cwise_j091902.55-574837.3.json index 98c0605da..7a92447cd 100644 --- a/data/cwise_j091902.55-574837.3.json +++ b/data/cwise_j091902.55-574837.3.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j094352.22+335639.1.json b/data/cwise_j094352.22+335639.1.json index d078b80cb..38aaddcce 100644 --- a/data/cwise_j094352.22+335639.1.json +++ b/data/cwise_j094352.22+335639.1.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j095259.54-505418.9.json b/data/cwise_j095259.54-505418.9.json index ab6487182..dd1d23c4e 100644 --- a/data/cwise_j095259.54-505418.9.json +++ b/data/cwise_j095259.54-505418.9.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j101017.43-242300.4.json b/data/cwise_j101017.43-242300.4.json index ae92cc8bc..46c9eccf7 100644 --- a/data/cwise_j101017.43-242300.4.json +++ b/data/cwise_j101017.43-242300.4.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j101317.51-010313.6.json b/data/cwise_j101317.51-010313.6.json index 1a77fc1fa..7f3b79e8f 100644 --- a/data/cwise_j101317.51-010313.6.json +++ b/data/cwise_j101317.51-010313.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j101523.92-111539.6.json b/data/cwise_j101523.92-111539.6.json index c7e862dd7..c893f5ea3 100644 --- a/data/cwise_j101523.92-111539.6.json +++ b/data/cwise_j101523.92-111539.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j103734.29-050749.8.json b/data/cwise_j103734.29-050749.8.json index 58dcf64e3..3807219f3 100644 --- a/data/cwise_j103734.29-050749.8.json +++ b/data/cwise_j103734.29-050749.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j104053.42-355029.7.json b/data/cwise_j104053.42-355029.7.json index 60e315937..068d86277 100644 --- a/data/cwise_j104053.42-355029.7.json +++ b/data/cwise_j104053.42-355029.7.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j115229.05-351105.9.json b/data/cwise_j115229.05-351105.9.json index fabeb376f..ce3b2d2e8 100644 --- a/data/cwise_j115229.05-351105.9.json +++ b/data/cwise_j115229.05-351105.9.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j124033.48+460544.7.json b/data/cwise_j124033.48+460544.7.json index ed64555b5..d3840403a 100644 --- a/data/cwise_j124033.48+460544.7.json +++ b/data/cwise_j124033.48+460544.7.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j125858.20+001133.2.json b/data/cwise_j125858.20+001133.2.json index 80b50fc0b..45b6adb92 100644 --- a/data/cwise_j125858.20+001133.2.json +++ b/data/cwise_j125858.20+001133.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j130329.90+512754.0.json b/data/cwise_j130329.90+512754.0.json index a70a2575b..a5114a2c9 100644 --- a/data/cwise_j130329.90+512754.0.json +++ b/data/cwise_j130329.90+512754.0.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j130446.64-120023.6.json b/data/cwise_j130446.64-120023.6.json index 5bc7a76da..ba6e5bfa0 100644 --- a/data/cwise_j130446.64-120023.6.json +++ b/data/cwise_j130446.64-120023.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j131355.15+223005.6.json b/data/cwise_j131355.15+223005.6.json index ca605917a..5ed2ccd0d 100644 --- a/data/cwise_j131355.15+223005.6.json +++ b/data/cwise_j131355.15+223005.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j132539.70+022309.4.json b/data/cwise_j132539.70+022309.4.json index 906db9ad1..c1a3ac793 100644 --- a/data/cwise_j132539.70+022309.4.json +++ b/data/cwise_j132539.70+022309.4.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j132857.58-063747.4.json b/data/cwise_j132857.58-063747.4.json index 58dbdd0c3..3caea3529 100644 --- a/data/cwise_j132857.58-063747.4.json +++ b/data/cwise_j132857.58-063747.4.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j133211.93-374837.9.json b/data/cwise_j133211.93-374837.9.json index 9adbe5619..e9cc58e26 100644 --- a/data/cwise_j133211.93-374837.9.json +++ b/data/cwise_j133211.93-374837.9.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j133427.70-273053.1.json b/data/cwise_j133427.70-273053.1.json index 6da66825d..11100e4c0 100644 --- a/data/cwise_j133427.70-273053.1.json +++ b/data/cwise_j133427.70-273053.1.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j141737.21+041847.2.json b/data/cwise_j141737.21+041847.2.json index 8342deff5..5a1f02cf6 100644 --- a/data/cwise_j141737.21+041847.2.json +++ b/data/cwise_j141737.21+041847.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j143935.94-804851.5.json b/data/cwise_j143935.94-804851.5.json index c73249076..932753a2a 100644 --- a/data/cwise_j143935.94-804851.5.json +++ b/data/cwise_j143935.94-804851.5.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j143951.51+260000.3.json b/data/cwise_j143951.51+260000.3.json index f64bd49e7..d8b71aad7 100644 --- a/data/cwise_j143951.51+260000.3.json +++ b/data/cwise_j143951.51+260000.3.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j144057.87+380432.8.json b/data/cwise_j144057.87+380432.8.json index e2432757f..3b5a5f92e 100644 --- a/data/cwise_j144057.87+380432.8.json +++ b/data/cwise_j144057.87+380432.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j151939.68-485820.4.json b/data/cwise_j151939.68-485820.4.json index 6f7ce264a..950723d0e 100644 --- a/data/cwise_j151939.68-485820.4.json +++ b/data/cwise_j151939.68-485820.4.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j152742.44-121528.2.json b/data/cwise_j152742.44-121528.2.json index af216a6b6..7bc35849e 100644 --- a/data/cwise_j152742.44-121528.2.json +++ b/data/cwise_j152742.44-121528.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j153910.07+722757.2.json b/data/cwise_j153910.07+722757.2.json index 3205a747c..c83970491 100644 --- a/data/cwise_j153910.07+722757.2.json +++ b/data/cwise_j153910.07+722757.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j160654.19-103214.7.json b/data/cwise_j160654.19-103214.7.json index 190898030..ca35e94bd 100644 --- a/data/cwise_j160654.19-103214.7.json +++ b/data/cwise_j160654.19-103214.7.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j162511.27+774946.8.json b/data/cwise_j162511.27+774946.8.json index 0f76780e6..f7f0a2cc8 100644 --- a/data/cwise_j162511.27+774946.8.json +++ b/data/cwise_j162511.27+774946.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j165143.63+695259.4.json b/data/cwise_j165143.63+695259.4.json index 9722bad31..4d2af0ee6 100644 --- a/data/cwise_j165143.63+695259.4.json +++ b/data/cwise_j165143.63+695259.4.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j165325.10+490439.7.json b/data/cwise_j165325.10+490439.7.json index 494f5a971..4343905a3 100644 --- a/data/cwise_j165325.10+490439.7.json +++ b/data/cwise_j165325.10+490439.7.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j171308.00+644220.8.json b/data/cwise_j171308.00+644220.8.json index fc08f5678..664d60338 100644 --- a/data/cwise_j171308.00+644220.8.json +++ b/data/cwise_j171308.00+644220.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j171700.57+724437.3.json b/data/cwise_j171700.57+724437.3.json index 70ada9956..fbeaefd01 100644 --- a/data/cwise_j171700.57+724437.3.json +++ b/data/cwise_j171700.57+724437.3.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j173859.73+200501.2.json b/data/cwise_j173859.73+200501.2.json index d26143a91..deba6ae25 100644 --- a/data/cwise_j173859.73+200501.2.json +++ b/data/cwise_j173859.73+200501.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j174426.85+230355.1.json b/data/cwise_j174426.85+230355.1.json index 4947273e1..eaded33d4 100644 --- a/data/cwise_j174426.85+230355.1.json +++ b/data/cwise_j174426.85+230355.1.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j174509.03+380733.2.json b/data/cwise_j174509.03+380733.2.json index 6a217c8d9..8ab0b3006 100644 --- a/data/cwise_j174509.03+380733.2.json +++ b/data/cwise_j174509.03+380733.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j190909.62+553027.1.json b/data/cwise_j190909.62+553027.1.json index 85c4661b6..4b85a0c4e 100644 --- a/data/cwise_j190909.62+553027.1.json +++ b/data/cwise_j190909.62+553027.1.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j192425.03-532837.1.json b/data/cwise_j192425.03-532837.1.json index 14f254256..c9277e064 100644 --- a/data/cwise_j192425.03-532837.1.json +++ b/data/cwise_j192425.03-532837.1.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j195956.16-644318.1.json b/data/cwise_j195956.16-644318.1.json index d7cf70704..c5aba18f0 100644 --- a/data/cwise_j195956.16-644318.1.json +++ b/data/cwise_j195956.16-644318.1.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j200304.70-142228.9.json b/data/cwise_j200304.70-142228.9.json index c0b3d68cc..3bdd654c3 100644 --- a/data/cwise_j200304.70-142228.9.json +++ b/data/cwise_j200304.70-142228.9.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j200813.66-124502.4.json b/data/cwise_j200813.66-124502.4.json index 6db0ba784..22d1784e8 100644 --- a/data/cwise_j200813.66-124502.4.json +++ b/data/cwise_j200813.66-124502.4.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j202934.80-791013.1.json b/data/cwise_j202934.80-791013.1.json index cde5c69f6..c1c21ff89 100644 --- a/data/cwise_j202934.80-791013.1.json +++ b/data/cwise_j202934.80-791013.1.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j205247.07+053855.7.json b/data/cwise_j205247.07+053855.7.json index de6ed9f89..606f50bf9 100644 --- a/data/cwise_j205247.07+053855.7.json +++ b/data/cwise_j205247.07+053855.7.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j212342.88+655615.6.json b/data/cwise_j212342.88+655615.6.json index 0bf6a8e2d..5ffef336b 100644 --- a/data/cwise_j212342.88+655615.6.json +++ b/data/cwise_j212342.88+655615.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j214129.80-024623.6.json b/data/cwise_j214129.80-024623.6.json index a0f37a135..bfc7e173b 100644 --- a/data/cwise_j214129.80-024623.6.json +++ b/data/cwise_j214129.80-024623.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j214213.99-204659.8.json b/data/cwise_j214213.99-204659.8.json index b43598f41..45fb3f31a 100644 --- a/data/cwise_j214213.99-204659.8.json +++ b/data/cwise_j214213.99-204659.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j221418.15+253432.2.json b/data/cwise_j221418.15+253432.2.json index 4fb39ec77..7db1e6672 100644 --- a/data/cwise_j221418.15+253432.2.json +++ b/data/cwise_j221418.15+253432.2.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j224452.18-362946.0.json b/data/cwise_j224452.18-362946.0.json index 3e07fa4e0..9f585212b 100644 --- a/data/cwise_j224452.18-362946.0.json +++ b/data/cwise_j224452.18-362946.0.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j225525.22-302320.8.json b/data/cwise_j225525.22-302320.8.json index 92ba40af8..f0e9ff15f 100644 --- a/data/cwise_j225525.22-302320.8.json +++ b/data/cwise_j225525.22-302320.8.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j231824.58+052142.3.json b/data/cwise_j231824.58+052142.3.json index f652b83f0..4cd8a1ad7 100644 --- a/data/cwise_j231824.58+052142.3.json +++ b/data/cwise_j231824.58+052142.3.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j233531.55+014219.6.json b/data/cwise_j233531.55+014219.6.json index a71345823..3b47950e4 100644 --- a/data/cwise_j233531.55+014219.6.json +++ b/data/cwise_j233531.55+014219.6.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ diff --git a/data/cwise_j235827.96-521813.4.json b/data/cwise_j235827.96-521813.4.json index c559c35ae..9f6d071d8 100644 --- a/data/cwise_j235827.96-521813.4.json +++ b/data/cwise_j235827.96-521813.4.json @@ -8,8 +8,8 @@ "equinox": "ICRS", "shortname": null, "reference": "Roth", - "other_references": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "comments": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "other_references": null, + "comments": null } ], "Names": [ From 91c49676b95e1a520bdebb911557fab3246acd1d Mon Sep 17 00:00:00 2001 From: kelle Date: Wed, 20 Dec 2023 16:47:18 -0500 Subject: [PATCH 27/35] add _all_ --- scripts/ingests/utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/ingests/utils.py b/scripts/ingests/utils.py index 84dffe8ca..9da64bc90 100644 --- a/scripts/ingests/utils.py +++ b/scripts/ingests/utils.py @@ -21,13 +21,14 @@ import socket from scripts import REFERENCE_TABLES -# TODO: add __all__ to limit what is imported with * -__all__ = ["SimpleError", - "load_simpledb", - "find_source_in_db", - "find_publication", - "ingest_publication", - "check_internet_connection"] +__all__ = [ + "SimpleError", + "load_simpledb", + "find_source_in_db", + "find_publication", + "ingest_publication", + "check_internet_connection", +] warnings.filterwarnings("ignore", module="astroquery.simbad") logger = logging.getLogger("SIMPLE") From 6d77346f52a559569d5df937747d073a4f248682 Mon Sep 17 00:00:00 2001 From: kelle Date: Wed, 20 Dec 2023 16:49:04 -0500 Subject: [PATCH 28/35] add ingest_source test --- tests/test_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index ce17dc150..5ba0c9199 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,6 +3,7 @@ import sys import math import os + sys.path.append(".") from scripts.ingests.utils import * from scripts.ingests.ingest_utils import * @@ -188,6 +189,19 @@ def test_ingest_sources(db): assert "Coordinates are needed" in str(error_message.value) +def test_ingest_source(): + with pytest.raises(SimpleError) as error_message: + ingest_source( + db, + "Fake7", + ra=11.0673755, + dec=18.352889, + reference="Ref 1", + raise_error=True, + ) + assert "already exists" in str(error_message.value) + + def test_convert_spt_string_to_code(): # Test conversion of spectral types into numeric values assert convert_spt_string_to_code(["M5.6"]) == [65.6] From be9272625c6f25649045fa9b6b15b7b80add7009 Mon Sep 17 00:00:00 2001 From: kelle Date: Wed, 20 Dec 2023 16:51:50 -0500 Subject: [PATCH 29/35] clean up .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8c38f80e3..bd62c4b79 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,5 @@ .vscode .pytest_cache .ds_store -bdnyc/ *.code-workspace From cc83b203ee6ab438594b2fa7353765b96ae5eb3b Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Wed, 20 Dec 2023 17:10:49 -0500 Subject: [PATCH 30/35] Corrected 2mass_j12560183-1257276.json file --- data/2mass_j12560183-1257276.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/2mass_j12560183-1257276.json b/data/2mass_j12560183-1257276.json index 6f15633b6..eed3a0b12 100644 --- a/data/2mass_j12560183-1257276.json +++ b/data/2mass_j12560183-1257276.json @@ -30,9 +30,6 @@ { "other_name": "VHS 1256-1257b" }, - { - "other_name": "VHS 1256b" - }, { "other_name": "VHS J125601.92-125723.9 b" }, @@ -41,6 +38,9 @@ }, { "other_name": "WISEA J125601.66-125728.7" + }, + { + "other_name": "VHS 1256b" } ], "Parallaxes": [ @@ -267,4 +267,4 @@ "reference": "Gauz15" } ] -} \ No newline at end of file +} From 12e1dfdb35bf39d5d2913fb90aad386592e07962 Mon Sep 17 00:00:00 2001 From: kelle Date: Wed, 20 Dec 2023 17:50:06 -0500 Subject: [PATCH 31/35] improved ingest --- data/cwise_j210640.16+250729.0.json | 10 +++++----- scripts/ingests/ingest_BYW_2024.py | 7 ++++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/data/cwise_j210640.16+250729.0.json b/data/cwise_j210640.16+250729.0.json index f4201c39c..deef990d9 100644 --- a/data/cwise_j210640.16+250729.0.json +++ b/data/cwise_j210640.16+250729.0.json @@ -2,17 +2,17 @@ "Sources": [ { "source": "CWISE J210640.16+250729.0", - "ra": 316.66736, - "dec": 25.124719444444445, - "epoch": null, - "equinox": null, + "ra": 316.6673638, + "dec": 25.1247223, + "epoch": 2015.4041, + "equinox": "ICRS", "shortname": null, "reference": "Roth", "other_references": null, "comments": null } ], - "CompanionRelationships": [ +"CompanionRelationships": [ { "companion_name": "BD+24 4329", "projected_separation_arcsec": 1124.0, diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index a563ffa83..c88115800 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -29,7 +29,7 @@ # Loop through each row in byw table and print data: source name ra, dec. def ingest_all_sources(db): - for row in byw_table: # skip the header row - [1:10]runs only first 10 rows + for row in byw_table[72:]: # skip the header row - [1:10]runs only first 10 rows # Print byw source information print("BYW Source Information:") # for key, value in row_dict.items(): @@ -82,6 +82,11 @@ def fix_blobs(db): ) +def delete_roth_sources(db): + with db.engine.begin() as conn: + conn.execute(db.Sources.delete().where(db.Sources.c.reference == "Roth")) + + # calling functions # add_publication(db) ingest_all_sources(db) From 04caf219bbd82c9ee60229de434f9b5733152ca0 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Thu, 21 Dec 2023 22:10:15 -0500 Subject: [PATCH 32/35] Added shortnames in names table --- data/2mass_j12560183-1257276.json | 8 +++---- data/cwise_j002029.72-153527.5.json | 3 +++ data/cwise_j002101.45-334631.5.json | 3 +++ data/cwise_j002159.02-451434.4.json | 3 +++ data/cwise_j002414.30-403053.7.json | 3 +++ data/cwise_j002658.73-260859.6.json | 3 +++ data/cwise_j004218.67+291730.6.json | 3 +++ data/cwise_j004945.38+423619.3.json | 3 +++ data/cwise_j005635.48-240401.9.json | 3 +++ data/cwise_j010424.50+133949.8.json | 3 +++ data/cwise_j011003.61-592650.8.json | 3 +++ data/cwise_j012715.52-300246.9.json | 3 +++ data/cwise_j013719.04+342837.6.json | 3 +++ data/cwise_j015905.58+105551.5.json | 3 +++ data/cwise_j020538.20+594452.2.json | 3 +++ data/cwise_j022454.80+152629.5.json | 3 +++ data/cwise_j022737.75+083008.8.json | 3 +++ data/cwise_j025645.16-335008.9.json | 3 +++ data/cwise_j030005.73-062218.6.json | 3 +++ data/cwise_j032853.32-112332.6.json | 3 +++ data/cwise_j055515.83+510514.0.json | 3 +++ data/cwise_j055909.00-384219.8.json | 3 +++ data/cwise_j060202.17-462447.8.json | 3 +++ data/cwise_j062000.01+344641.3.json | 3 +++ data/cwise_j062648.96+594129.2.json | 3 +++ data/cwise_j062727.34-002826.8.json | 3 +++ data/cwise_j062909.21+395701.6.json | 3 +++ data/cwise_j065752.45+163350.2.json | 3 +++ data/cwise_j073831.31+525453.7.json | 3 +++ data/cwise_j080912.95+741704.3.json | 3 +++ data/cwise_j085131.24-603056.2.json | 3 +++ data/cwise_j091558.53+254713.0.json | 3 +++ data/cwise_j091902.55-574837.3.json | 3 +++ data/cwise_j094352.22+335639.1.json | 3 +++ data/cwise_j095259.54-505418.9.json | 3 +++ data/cwise_j101017.43-242300.4.json | 3 +++ data/cwise_j101317.51-010313.6.json | 3 +++ data/cwise_j101523.92-111539.6.json | 3 +++ data/cwise_j103734.29-050749.8.json | 3 +++ data/cwise_j104053.42-355029.7.json | 3 +++ data/cwise_j115229.05-351105.9.json | 3 +++ data/cwise_j124033.48+460544.7.json | 3 +++ data/cwise_j125858.20+001133.2.json | 3 +++ data/cwise_j130329.90+512754.0.json | 3 +++ data/cwise_j130446.64-120023.6.json | 3 +++ data/cwise_j131355.15+223005.6.json | 3 +++ data/cwise_j132539.70+022309.4.json | 3 +++ data/cwise_j132857.58-063747.4.json | 3 +++ data/cwise_j133211.93-374837.9.json | 3 +++ data/cwise_j133427.70-273053.1.json | 3 +++ data/cwise_j141737.21+041847.2.json | 3 +++ data/cwise_j143935.94-804851.5.json | 3 +++ data/cwise_j143951.51+260000.3.json | 3 +++ data/cwise_j144057.87+380432.8.json | 3 +++ data/cwise_j151939.68-485820.4.json | 3 +++ data/cwise_j152742.44-121528.2.json | 3 +++ data/cwise_j153910.07+722757.2.json | 3 +++ data/cwise_j160654.19-103214.7.json | 3 +++ data/cwise_j162511.27+774946.8.json | 3 +++ data/cwise_j165143.63+695259.4.json | 3 +++ data/cwise_j165325.10+490439.7.json | 3 +++ data/cwise_j171308.00+644220.8.json | 3 +++ data/cwise_j171700.57+724437.3.json | 3 +++ data/cwise_j173859.73+200501.2.json | 3 +++ data/cwise_j174426.85+230355.1.json | 3 +++ data/cwise_j174509.03+380733.2.json | 3 +++ data/cwise_j183207.94-540943.3.json | 3 +++ data/cwise_j190909.62+553027.1.json | 3 +++ data/cwise_j192425.03-532837.1.json | 3 +++ data/cwise_j195956.16-644318.1.json | 3 +++ data/cwise_j200304.70-142228.9.json | 3 +++ data/cwise_j200813.66-124502.4.json | 3 +++ data/cwise_j202934.80-791013.1.json | 3 +++ data/cwise_j205247.07+053855.7.json | 3 +++ data/cwise_j210640.16+250729.0.json | 5 +++- data/cwise_j212342.88+655615.6.json | 3 +++ data/cwise_j214129.80-024623.6.json | 3 +++ data/cwise_j214213.99-204659.8.json | 3 +++ data/cwise_j221418.15+253432.2.json | 3 +++ data/cwise_j224452.18-362946.0.json | 3 +++ data/cwise_j225525.22-302320.8.json | 3 +++ data/cwise_j231824.58+052142.3.json | 3 +++ data/cwise_j233531.55+014219.6.json | 3 +++ data/cwise_j235827.96-521813.4.json | 3 +++ data/ulas_j103131.49+123736.4.json | 3 +++ scripts/ingests/ingest_BYW_2024.py | 37 +++++++++++++++++++++++------ 86 files changed, 287 insertions(+), 12 deletions(-) diff --git a/data/2mass_j12560183-1257276.json b/data/2mass_j12560183-1257276.json index eed3a0b12..6f15633b6 100644 --- a/data/2mass_j12560183-1257276.json +++ b/data/2mass_j12560183-1257276.json @@ -30,6 +30,9 @@ { "other_name": "VHS 1256-1257b" }, + { + "other_name": "VHS 1256b" + }, { "other_name": "VHS J125601.92-125723.9 b" }, @@ -38,9 +41,6 @@ }, { "other_name": "WISEA J125601.66-125728.7" - }, - { - "other_name": "VHS 1256b" } ], "Parallaxes": [ @@ -267,4 +267,4 @@ "reference": "Gauz15" } ] -} +} \ No newline at end of file diff --git a/data/cwise_j002029.72-153527.5.json b/data/cwise_j002029.72-153527.5.json index c05725078..676b23f61 100644 --- a/data/cwise_j002029.72-153527.5.json +++ b/data/cwise_j002029.72-153527.5.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0020-1535" + }, { "other_name": "CWISE J002029.72-153527.5" } diff --git a/data/cwise_j002101.45-334631.5.json b/data/cwise_j002101.45-334631.5.json index 0e08b742b..130530cda 100644 --- a/data/cwise_j002101.45-334631.5.json +++ b/data/cwise_j002101.45-334631.5.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0021-3346" + }, { "other_name": "CWISE J002101.45-334631.5" } diff --git a/data/cwise_j002159.02-451434.4.json b/data/cwise_j002159.02-451434.4.json index 5e8951ad7..f2daec136 100644 --- a/data/cwise_j002159.02-451434.4.json +++ b/data/cwise_j002159.02-451434.4.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0021-4514" + }, { "other_name": "CWISE J002159.02-451434.4" } diff --git a/data/cwise_j002414.30-403053.7.json b/data/cwise_j002414.30-403053.7.json index 04c9f9631..2c80b2394 100644 --- a/data/cwise_j002414.30-403053.7.json +++ b/data/cwise_j002414.30-403053.7.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0024-4030" + }, { "other_name": "CWISE J002414.30-403053.7" } diff --git a/data/cwise_j002658.73-260859.6.json b/data/cwise_j002658.73-260859.6.json index 40e76eadf..e955e40c9 100644 --- a/data/cwise_j002658.73-260859.6.json +++ b/data/cwise_j002658.73-260859.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0026-2608" + }, { "other_name": "CWISE J002658.73-260859.6" } diff --git a/data/cwise_j004218.67+291730.6.json b/data/cwise_j004218.67+291730.6.json index 24ad31360..3de7e3c0b 100644 --- a/data/cwise_j004218.67+291730.6.json +++ b/data/cwise_j004218.67+291730.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0042+2917" + }, { "other_name": "CWISE J004218.67+291730.6" } diff --git a/data/cwise_j004945.38+423619.3.json b/data/cwise_j004945.38+423619.3.json index 60c798189..857156570 100644 --- a/data/cwise_j004945.38+423619.3.json +++ b/data/cwise_j004945.38+423619.3.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0049+4236" + }, { "other_name": "CWISE J004945.38+423619.3" } diff --git a/data/cwise_j005635.48-240401.9.json b/data/cwise_j005635.48-240401.9.json index 8841443a3..c48c8e4f9 100644 --- a/data/cwise_j005635.48-240401.9.json +++ b/data/cwise_j005635.48-240401.9.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0056-2404" + }, { "other_name": "CWISE J005635.48-240401.9" } diff --git a/data/cwise_j010424.50+133949.8.json b/data/cwise_j010424.50+133949.8.json index 61ce8ae9e..801ee5170 100644 --- a/data/cwise_j010424.50+133949.8.json +++ b/data/cwise_j010424.50+133949.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0104+1339" + }, { "other_name": "CWISE J010424.50+133949.8" } diff --git a/data/cwise_j011003.61-592650.8.json b/data/cwise_j011003.61-592650.8.json index 57fa5cdc4..e9b9ef794 100644 --- a/data/cwise_j011003.61-592650.8.json +++ b/data/cwise_j011003.61-592650.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0110-5926" + }, { "other_name": "CWISE J011003.61-592650.8" } diff --git a/data/cwise_j012715.52-300246.9.json b/data/cwise_j012715.52-300246.9.json index 144abad73..12cfbad71 100644 --- a/data/cwise_j012715.52-300246.9.json +++ b/data/cwise_j012715.52-300246.9.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0127-3002" + }, { "other_name": "CWISE J012715.52-300246.9" } diff --git a/data/cwise_j013719.04+342837.6.json b/data/cwise_j013719.04+342837.6.json index 9793e13fa..a09eb9b9c 100644 --- a/data/cwise_j013719.04+342837.6.json +++ b/data/cwise_j013719.04+342837.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0137+3428" + }, { "other_name": "CWISE J013719.04+342837.6" } diff --git a/data/cwise_j015905.58+105551.5.json b/data/cwise_j015905.58+105551.5.json index 23e09081f..30b052924 100644 --- a/data/cwise_j015905.58+105551.5.json +++ b/data/cwise_j015905.58+105551.5.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0159+1055" + }, { "other_name": "CWISE J015905.58+105551.5" } diff --git a/data/cwise_j020538.20+594452.2.json b/data/cwise_j020538.20+594452.2.json index 3cb73888a..c653a2423 100644 --- a/data/cwise_j020538.20+594452.2.json +++ b/data/cwise_j020538.20+594452.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0205+5944" + }, { "other_name": "CWISE J020538.20+594452.2" } diff --git a/data/cwise_j022454.80+152629.5.json b/data/cwise_j022454.80+152629.5.json index db2792faa..1a9fca912 100644 --- a/data/cwise_j022454.80+152629.5.json +++ b/data/cwise_j022454.80+152629.5.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0224+1526*" + }, { "other_name": "CWISE J022454.80+152629.5" } diff --git a/data/cwise_j022737.75+083008.8.json b/data/cwise_j022737.75+083008.8.json index f0d9d3022..b6f8a8972 100644 --- a/data/cwise_j022737.75+083008.8.json +++ b/data/cwise_j022737.75+083008.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0227+0830" + }, { "other_name": "CWISE J022737.75+083008.8" } diff --git a/data/cwise_j025645.16-335008.9.json b/data/cwise_j025645.16-335008.9.json index 7866f35f6..f0261db06 100644 --- a/data/cwise_j025645.16-335008.9.json +++ b/data/cwise_j025645.16-335008.9.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0256-3350" + }, { "other_name": "CWISE J025645.16-335008.9" } diff --git a/data/cwise_j030005.73-062218.6.json b/data/cwise_j030005.73-062218.6.json index 231744941..685a2ca83 100644 --- a/data/cwise_j030005.73-062218.6.json +++ b/data/cwise_j030005.73-062218.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0300-0622" + }, { "other_name": "CWISE J030005.73-062218.6" } diff --git a/data/cwise_j032853.32-112332.6.json b/data/cwise_j032853.32-112332.6.json index 75c296ccf..41db3dc03 100644 --- a/data/cwise_j032853.32-112332.6.json +++ b/data/cwise_j032853.32-112332.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0328-1123" + }, { "other_name": "CWISE J032853.32-112332.6" } diff --git a/data/cwise_j055515.83+510514.0.json b/data/cwise_j055515.83+510514.0.json index 1298c6bd3..df388498e 100644 --- a/data/cwise_j055515.83+510514.0.json +++ b/data/cwise_j055515.83+510514.0.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0555+5105" + }, { "other_name": "CWISE J055515.83+510514.0" } diff --git a/data/cwise_j055909.00-384219.8.json b/data/cwise_j055909.00-384219.8.json index c580ca25c..27037ec53 100644 --- a/data/cwise_j055909.00-384219.8.json +++ b/data/cwise_j055909.00-384219.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0559-3842" + }, { "other_name": "CWISE J055909.00-384219.8" } diff --git a/data/cwise_j060202.17-462447.8.json b/data/cwise_j060202.17-462447.8.json index 363f5aea1..d8108901c 100644 --- a/data/cwise_j060202.17-462447.8.json +++ b/data/cwise_j060202.17-462447.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0602-4624" + }, { "other_name": "CWISE J060202.17-462447.8" } diff --git a/data/cwise_j062000.01+344641.3.json b/data/cwise_j062000.01+344641.3.json index 09db9679c..ec485010d 100644 --- a/data/cwise_j062000.01+344641.3.json +++ b/data/cwise_j062000.01+344641.3.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0620+3446" + }, { "other_name": "CWISE J062000.01+344641.3" } diff --git a/data/cwise_j062648.96+594129.2.json b/data/cwise_j062648.96+594129.2.json index b98dc2954..d6d971a68 100644 --- a/data/cwise_j062648.96+594129.2.json +++ b/data/cwise_j062648.96+594129.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0626+5941" + }, { "other_name": "CWISE J062648.96+594129.2" } diff --git a/data/cwise_j062727.34-002826.8.json b/data/cwise_j062727.34-002826.8.json index 525ddc1b0..3588662b2 100644 --- a/data/cwise_j062727.34-002826.8.json +++ b/data/cwise_j062727.34-002826.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0627-0028A" + }, { "other_name": "CWISE J062727.34-002826.8" } diff --git a/data/cwise_j062909.21+395701.6.json b/data/cwise_j062909.21+395701.6.json index fe18e0695..498af5976 100644 --- a/data/cwise_j062909.21+395701.6.json +++ b/data/cwise_j062909.21+395701.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0629+3957" + }, { "other_name": "CWISE J062909.21+395701.6" } diff --git a/data/cwise_j065752.45+163350.2.json b/data/cwise_j065752.45+163350.2.json index c6b7c49c6..e71656093 100644 --- a/data/cwise_j065752.45+163350.2.json +++ b/data/cwise_j065752.45+163350.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0657+1633" + }, { "other_name": "CWISE J065752.45+163350.2" } diff --git a/data/cwise_j073831.31+525453.7.json b/data/cwise_j073831.31+525453.7.json index f30adfd3c..f27e65755 100644 --- a/data/cwise_j073831.31+525453.7.json +++ b/data/cwise_j073831.31+525453.7.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0738+5254" + }, { "other_name": "CWISE J073831.31+525453.7" } diff --git a/data/cwise_j080912.95+741704.3.json b/data/cwise_j080912.95+741704.3.json index 8196f0c75..50c789409 100644 --- a/data/cwise_j080912.95+741704.3.json +++ b/data/cwise_j080912.95+741704.3.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0809+7417" + }, { "other_name": "CWISE J080912.95+741704.3" } diff --git a/data/cwise_j085131.24-603056.2.json b/data/cwise_j085131.24-603056.2.json index 248e8d91f..60025a2da 100644 --- a/data/cwise_j085131.24-603056.2.json +++ b/data/cwise_j085131.24-603056.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0851-6030" + }, { "other_name": "CWISE J085131.24-603056.2" } diff --git a/data/cwise_j091558.53+254713.0.json b/data/cwise_j091558.53+254713.0.json index 1da4f652b..4109a8c7a 100644 --- a/data/cwise_j091558.53+254713.0.json +++ b/data/cwise_j091558.53+254713.0.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0915+2547" + }, { "other_name": "CWISE J091558.53+254713.0" } diff --git a/data/cwise_j091902.55-574837.3.json b/data/cwise_j091902.55-574837.3.json index 7a92447cd..6b2912691 100644 --- a/data/cwise_j091902.55-574837.3.json +++ b/data/cwise_j091902.55-574837.3.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0919-5748" + }, { "other_name": "CWISE J091902.55-574837.3" } diff --git a/data/cwise_j094352.22+335639.1.json b/data/cwise_j094352.22+335639.1.json index 38aaddcce..4616b7ca6 100644 --- a/data/cwise_j094352.22+335639.1.json +++ b/data/cwise_j094352.22+335639.1.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0943+3356" + }, { "other_name": "CWISE J094352.22+335639.1" } diff --git a/data/cwise_j095259.54-505418.9.json b/data/cwise_j095259.54-505418.9.json index dd1d23c4e..b9bb92901 100644 --- a/data/cwise_j095259.54-505418.9.json +++ b/data/cwise_j095259.54-505418.9.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0952-5054" + }, { "other_name": "CWISE J095259.54-505418.9" } diff --git a/data/cwise_j101017.43-242300.4.json b/data/cwise_j101017.43-242300.4.json index 46c9eccf7..d59e2597d 100644 --- a/data/cwise_j101017.43-242300.4.json +++ b/data/cwise_j101017.43-242300.4.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1010-2423" + }, { "other_name": "CWISE J101017.43-242300.4" } diff --git a/data/cwise_j101317.51-010313.6.json b/data/cwise_j101317.51-010313.6.json index 7f3b79e8f..9cbed6b15 100644 --- a/data/cwise_j101317.51-010313.6.json +++ b/data/cwise_j101317.51-010313.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1013-0103" + }, { "other_name": "CWISE J101317.51-010313.6" } diff --git a/data/cwise_j101523.92-111539.6.json b/data/cwise_j101523.92-111539.6.json index c893f5ea3..ca1fb8ae1 100644 --- a/data/cwise_j101523.92-111539.6.json +++ b/data/cwise_j101523.92-111539.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1015-1115" + }, { "other_name": "CWISE J101523.92-111539.6" } diff --git a/data/cwise_j103734.29-050749.8.json b/data/cwise_j103734.29-050749.8.json index 3807219f3..cdb162fbe 100644 --- a/data/cwise_j103734.29-050749.8.json +++ b/data/cwise_j103734.29-050749.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1037-0507" + }, { "other_name": "CWISE J103734.29-050749.8" } diff --git a/data/cwise_j104053.42-355029.7.json b/data/cwise_j104053.42-355029.7.json index 068d86277..8891640db 100644 --- a/data/cwise_j104053.42-355029.7.json +++ b/data/cwise_j104053.42-355029.7.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1040-3550" + }, { "other_name": "CWISE J104053.42-355029.7" } diff --git a/data/cwise_j115229.05-351105.9.json b/data/cwise_j115229.05-351105.9.json index ce3b2d2e8..9b2ec85b5 100644 --- a/data/cwise_j115229.05-351105.9.json +++ b/data/cwise_j115229.05-351105.9.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1152-3511" + }, { "other_name": "CWISE J115229.05-351105.9" } diff --git a/data/cwise_j124033.48+460544.7.json b/data/cwise_j124033.48+460544.7.json index d3840403a..3838f23b4 100644 --- a/data/cwise_j124033.48+460544.7.json +++ b/data/cwise_j124033.48+460544.7.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1240+4605" + }, { "other_name": "CWISE J124033.48+460544.7" } diff --git a/data/cwise_j125858.20+001133.2.json b/data/cwise_j125858.20+001133.2.json index 45b6adb92..2aa99ad16 100644 --- a/data/cwise_j125858.20+001133.2.json +++ b/data/cwise_j125858.20+001133.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1258+0011" + }, { "other_name": "CWISE J125858.20+001133.2" } diff --git a/data/cwise_j130329.90+512754.0.json b/data/cwise_j130329.90+512754.0.json index a5114a2c9..21027628d 100644 --- a/data/cwise_j130329.90+512754.0.json +++ b/data/cwise_j130329.90+512754.0.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1303+5127" + }, { "other_name": "CWISE J130329.90+512754.0" } diff --git a/data/cwise_j130446.64-120023.6.json b/data/cwise_j130446.64-120023.6.json index ba6e5bfa0..f133da069 100644 --- a/data/cwise_j130446.64-120023.6.json +++ b/data/cwise_j130446.64-120023.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1304-1200" + }, { "other_name": "CWISE J130446.64-120023.6" } diff --git a/data/cwise_j131355.15+223005.6.json b/data/cwise_j131355.15+223005.6.json index 5ed2ccd0d..a86894775 100644 --- a/data/cwise_j131355.15+223005.6.json +++ b/data/cwise_j131355.15+223005.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1313+2230" + }, { "other_name": "CWISE J131355.15+223005.6" } diff --git a/data/cwise_j132539.70+022309.4.json b/data/cwise_j132539.70+022309.4.json index c1a3ac793..713b05053 100644 --- a/data/cwise_j132539.70+022309.4.json +++ b/data/cwise_j132539.70+022309.4.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1325+0223" + }, { "other_name": "CWISE J132539.70+022309.4" } diff --git a/data/cwise_j132857.58-063747.4.json b/data/cwise_j132857.58-063747.4.json index 3caea3529..8838331c5 100644 --- a/data/cwise_j132857.58-063747.4.json +++ b/data/cwise_j132857.58-063747.4.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1328-0637" + }, { "other_name": "CWISE J132857.58-063747.4" } diff --git a/data/cwise_j133211.93-374837.9.json b/data/cwise_j133211.93-374837.9.json index e9cc58e26..605287722 100644 --- a/data/cwise_j133211.93-374837.9.json +++ b/data/cwise_j133211.93-374837.9.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1332-3748" + }, { "other_name": "CWISE J133211.93-374837.9" } diff --git a/data/cwise_j133427.70-273053.1.json b/data/cwise_j133427.70-273053.1.json index 11100e4c0..6377fd1a2 100644 --- a/data/cwise_j133427.70-273053.1.json +++ b/data/cwise_j133427.70-273053.1.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1334-2730" + }, { "other_name": "CWISE J133427.70-273053.1" } diff --git a/data/cwise_j141737.21+041847.2.json b/data/cwise_j141737.21+041847.2.json index 5a1f02cf6..bdc58c9e3 100644 --- a/data/cwise_j141737.21+041847.2.json +++ b/data/cwise_j141737.21+041847.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1417+0418" + }, { "other_name": "CWISE J141737.21+041847.2" } diff --git a/data/cwise_j143935.94-804851.5.json b/data/cwise_j143935.94-804851.5.json index 932753a2a..93cf82d05 100644 --- a/data/cwise_j143935.94-804851.5.json +++ b/data/cwise_j143935.94-804851.5.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1439-8048" + }, { "other_name": "CWISE J143935.94-804851.5" } diff --git a/data/cwise_j143951.51+260000.3.json b/data/cwise_j143951.51+260000.3.json index d8b71aad7..44bb7d7be 100644 --- a/data/cwise_j143951.51+260000.3.json +++ b/data/cwise_j143951.51+260000.3.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1439+2600" + }, { "other_name": "CWISE J143951.51+260000.3" } diff --git a/data/cwise_j144057.87+380432.8.json b/data/cwise_j144057.87+380432.8.json index 3b5a5f92e..b8128b4dd 100644 --- a/data/cwise_j144057.87+380432.8.json +++ b/data/cwise_j144057.87+380432.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1440+3804" + }, { "other_name": "CWISE J144057.87+380432.8" } diff --git a/data/cwise_j151939.68-485820.4.json b/data/cwise_j151939.68-485820.4.json index 950723d0e..1536af124 100644 --- a/data/cwise_j151939.68-485820.4.json +++ b/data/cwise_j151939.68-485820.4.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1519-4858" + }, { "other_name": "CWISE J151939.68-485820.4" } diff --git a/data/cwise_j152742.44-121528.2.json b/data/cwise_j152742.44-121528.2.json index 7bc35849e..1c5e5e016 100644 --- a/data/cwise_j152742.44-121528.2.json +++ b/data/cwise_j152742.44-121528.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1527-1215" + }, { "other_name": "CWISE J152742.44-121528.2" } diff --git a/data/cwise_j153910.07+722757.2.json b/data/cwise_j153910.07+722757.2.json index c83970491..93ff3055a 100644 --- a/data/cwise_j153910.07+722757.2.json +++ b/data/cwise_j153910.07+722757.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1539+7227" + }, { "other_name": "CWISE J153910.07+722757.2" } diff --git a/data/cwise_j160654.19-103214.7.json b/data/cwise_j160654.19-103214.7.json index ca35e94bd..cd8036bb7 100644 --- a/data/cwise_j160654.19-103214.7.json +++ b/data/cwise_j160654.19-103214.7.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1606-1032" + }, { "other_name": "CWISE J160654.19-103214.7" } diff --git a/data/cwise_j162511.27+774946.8.json b/data/cwise_j162511.27+774946.8.json index f7f0a2cc8..1764e2e7a 100644 --- a/data/cwise_j162511.27+774946.8.json +++ b/data/cwise_j162511.27+774946.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1625+7749" + }, { "other_name": "CWISE J162511.27+774946.8" } diff --git a/data/cwise_j165143.63+695259.4.json b/data/cwise_j165143.63+695259.4.json index 4d2af0ee6..591c3064d 100644 --- a/data/cwise_j165143.63+695259.4.json +++ b/data/cwise_j165143.63+695259.4.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1651+6952" + }, { "other_name": "CWISE J165143.63+695259.4" } diff --git a/data/cwise_j165325.10+490439.7.json b/data/cwise_j165325.10+490439.7.json index 4343905a3..383d0810e 100644 --- a/data/cwise_j165325.10+490439.7.json +++ b/data/cwise_j165325.10+490439.7.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1653+4904" + }, { "other_name": "CWISE J165325.10+490439.7" } diff --git a/data/cwise_j171308.00+644220.8.json b/data/cwise_j171308.00+644220.8.json index 664d60338..9817b4580 100644 --- a/data/cwise_j171308.00+644220.8.json +++ b/data/cwise_j171308.00+644220.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1713+6442" + }, { "other_name": "CWISE J171308.00+644220.8" } diff --git a/data/cwise_j171700.57+724437.3.json b/data/cwise_j171700.57+724437.3.json index fbeaefd01..fe46adb1a 100644 --- a/data/cwise_j171700.57+724437.3.json +++ b/data/cwise_j171700.57+724437.3.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1717+7244" + }, { "other_name": "CWISE J171700.57+724437.3" } diff --git a/data/cwise_j173859.73+200501.2.json b/data/cwise_j173859.73+200501.2.json index deba6ae25..3a96409a6 100644 --- a/data/cwise_j173859.73+200501.2.json +++ b/data/cwise_j173859.73+200501.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1738+2005" + }, { "other_name": "CWISE J173859.73+200501.2" } diff --git a/data/cwise_j174426.85+230355.1.json b/data/cwise_j174426.85+230355.1.json index eaded33d4..55c1ab0f4 100644 --- a/data/cwise_j174426.85+230355.1.json +++ b/data/cwise_j174426.85+230355.1.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1744+2303" + }, { "other_name": "CWISE J174426.85+230355.1" } diff --git a/data/cwise_j174509.03+380733.2.json b/data/cwise_j174509.03+380733.2.json index 8ab0b3006..518468bf6 100644 --- a/data/cwise_j174509.03+380733.2.json +++ b/data/cwise_j174509.03+380733.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1745+3807" + }, { "other_name": "CWISE J174509.03+380733.2" } diff --git a/data/cwise_j183207.94-540943.3.json b/data/cwise_j183207.94-540943.3.json index e2c0035f4..e4509b9c1 100644 --- a/data/cwise_j183207.94-540943.3.json +++ b/data/cwise_j183207.94-540943.3.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1832-5409" + }, { "other_name": "CWISE J183207.94-540943.3" } diff --git a/data/cwise_j190909.62+553027.1.json b/data/cwise_j190909.62+553027.1.json index 4b85a0c4e..50d0ac073 100644 --- a/data/cwise_j190909.62+553027.1.json +++ b/data/cwise_j190909.62+553027.1.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1909+5530" + }, { "other_name": "CWISE J190909.62+553027.1" } diff --git a/data/cwise_j192425.03-532837.1.json b/data/cwise_j192425.03-532837.1.json index c9277e064..2759e1541 100644 --- a/data/cwise_j192425.03-532837.1.json +++ b/data/cwise_j192425.03-532837.1.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1924-5328" + }, { "other_name": "CWISE J192425.03-532837.1" } diff --git a/data/cwise_j195956.16-644318.1.json b/data/cwise_j195956.16-644318.1.json index c5aba18f0..3f88d6a83 100644 --- a/data/cwise_j195956.16-644318.1.json +++ b/data/cwise_j195956.16-644318.1.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 1959-6443" + }, { "other_name": "CWISE J195956.16-644318.1" } diff --git a/data/cwise_j200304.70-142228.9.json b/data/cwise_j200304.70-142228.9.json index 3bdd654c3..039066a8c 100644 --- a/data/cwise_j200304.70-142228.9.json +++ b/data/cwise_j200304.70-142228.9.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2003-1422" + }, { "other_name": "CWISE J200304.70-142228.9" } diff --git a/data/cwise_j200813.66-124502.4.json b/data/cwise_j200813.66-124502.4.json index 22d1784e8..36adb088e 100644 --- a/data/cwise_j200813.66-124502.4.json +++ b/data/cwise_j200813.66-124502.4.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2008-1245" + }, { "other_name": "CWISE J200813.66-124502.4" } diff --git a/data/cwise_j202934.80-791013.1.json b/data/cwise_j202934.80-791013.1.json index c1c21ff89..0ac35b85c 100644 --- a/data/cwise_j202934.80-791013.1.json +++ b/data/cwise_j202934.80-791013.1.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2029-7910" + }, { "other_name": "CWISE J202934.80-791013.1" } diff --git a/data/cwise_j205247.07+053855.7.json b/data/cwise_j205247.07+053855.7.json index 606f50bf9..d7e983671 100644 --- a/data/cwise_j205247.07+053855.7.json +++ b/data/cwise_j205247.07+053855.7.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2052+0538" + }, { "other_name": "CWISE J205247.07+053855.7" } diff --git a/data/cwise_j210640.16+250729.0.json b/data/cwise_j210640.16+250729.0.json index deef990d9..e31f6592e 100644 --- a/data/cwise_j210640.16+250729.0.json +++ b/data/cwise_j210640.16+250729.0.json @@ -12,7 +12,7 @@ "comments": null } ], -"CompanionRelationships": [ + "CompanionRelationships": [ { "companion_name": "BD+24 4329", "projected_separation_arcsec": 1124.0, @@ -24,6 +24,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2106+2507" + }, { "other_name": "CWISE J210640.16+250729.0" } diff --git a/data/cwise_j212342.88+655615.6.json b/data/cwise_j212342.88+655615.6.json index 5ffef336b..992bafd38 100644 --- a/data/cwise_j212342.88+655615.6.json +++ b/data/cwise_j212342.88+655615.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2123+6556" + }, { "other_name": "CWISE J212342.88+655615.6" } diff --git a/data/cwise_j214129.80-024623.6.json b/data/cwise_j214129.80-024623.6.json index bfc7e173b..a4f1937d4 100644 --- a/data/cwise_j214129.80-024623.6.json +++ b/data/cwise_j214129.80-024623.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2141-0246" + }, { "other_name": "CWISE J214129.80-024623.6" } diff --git a/data/cwise_j214213.99-204659.8.json b/data/cwise_j214213.99-204659.8.json index 45fb3f31a..f4d3c4a67 100644 --- a/data/cwise_j214213.99-204659.8.json +++ b/data/cwise_j214213.99-204659.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2142-2046" + }, { "other_name": "CWISE J214213.99-204659.8" } diff --git a/data/cwise_j221418.15+253432.2.json b/data/cwise_j221418.15+253432.2.json index 7db1e6672..9af6909ed 100644 --- a/data/cwise_j221418.15+253432.2.json +++ b/data/cwise_j221418.15+253432.2.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2214+2535" + }, { "other_name": "CWISE J221418.15+253432.2" } diff --git a/data/cwise_j224452.18-362946.0.json b/data/cwise_j224452.18-362946.0.json index 9f585212b..00e6cd9eb 100644 --- a/data/cwise_j224452.18-362946.0.json +++ b/data/cwise_j224452.18-362946.0.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2244-3629" + }, { "other_name": "CWISE J224452.18-362946.0" } diff --git a/data/cwise_j225525.22-302320.8.json b/data/cwise_j225525.22-302320.8.json index f0e9ff15f..a6060c28f 100644 --- a/data/cwise_j225525.22-302320.8.json +++ b/data/cwise_j225525.22-302320.8.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2255-3023" + }, { "other_name": "CWISE J225525.22-302320.8" } diff --git a/data/cwise_j231824.58+052142.3.json b/data/cwise_j231824.58+052142.3.json index 4cd8a1ad7..03805bc7e 100644 --- a/data/cwise_j231824.58+052142.3.json +++ b/data/cwise_j231824.58+052142.3.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2318+0521" + }, { "other_name": "CWISE J231824.58+052142.3" } diff --git a/data/cwise_j233531.55+014219.6.json b/data/cwise_j233531.55+014219.6.json index 3b47950e4..942d92ab2 100644 --- a/data/cwise_j233531.55+014219.6.json +++ b/data/cwise_j233531.55+014219.6.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2335+0142" + }, { "other_name": "CWISE J233531.55+014219.6" } diff --git a/data/cwise_j235827.96-521813.4.json b/data/cwise_j235827.96-521813.4.json index 9f6d071d8..8b973bf6b 100644 --- a/data/cwise_j235827.96-521813.4.json +++ b/data/cwise_j235827.96-521813.4.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 2358-5218" + }, { "other_name": "CWISE J235827.96-521813.4" } diff --git a/data/ulas_j103131.49+123736.4.json b/data/ulas_j103131.49+123736.4.json index bf2c4c878..02bc4d897 100644 --- a/data/ulas_j103131.49+123736.4.json +++ b/data/ulas_j103131.49+123736.4.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "ULAS 1031+1237" + }, { "other_name": "ULAS J103131.49+123736.4" } diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index c88115800..99e790761 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -3,7 +3,7 @@ from astropy.io import ascii SAVE_DB = False # save the data files in addition to modifying the .db file -RECREATE_DB = False # recreates the .db file from the data files +RECREATE_DB = True # recreates the .db file from the data files # LOAD THE DATABASE db = load_simpledb("SIMPLE.db", recreatedb=RECREATE_DB) @@ -29,7 +29,7 @@ # Loop through each row in byw table and print data: source name ra, dec. def ingest_all_sources(db): - for row in byw_table[72:]: # skip the header row - [1:10]runs only first 10 rows + for row in byw_table[1:90]: # skip the header row - [1:10]runs only first 10 rows # Print byw source information print("BYW Source Information:") # for key, value in row_dict.items(): @@ -55,12 +55,35 @@ def ingest_all_sources(db): # Add a separator between rows for better readability print("-" * 20) +#Call sources function +ingest_all_sources(db) + + +#Ingest shortnames as other names to source +#Loop through data +def ingest_all_shortnames(db): + for row in byw_table[55:]: # skip the header row - [1:10]runs only first 10 rows + + # Print byw source information + print("BYW Source Information:") + + for col_name in row.colnames: + print(f"{col_name}: {row[col_name]}") + + ingest_names(db, + source=row["Source"], + other_name=row["Shortname"] + ) + + print("-" * 20) + +#Call shortnames function +ingest_all_shortnames(db) + # Ingested other_ref sources as publications # Skrzypek et al. 2016, Marocco et al. 2015(Online Catalog), Kirkpatrick et al. 2021 # Ingest reference name: Skrz16, Maro15, Kirk21 - - def add_publication(db): ingest_publication( db, doi="10.26093/cds/vizier.35890049", bibcode="2016yCat..35890049S" @@ -72,6 +95,8 @@ def add_publication(db): db, doi="10.3847/1538-4365/abd107", bibcode="2021ApJS..253....7K" ) +#Call publications function +# add_publication(db) def fix_blobs(db): with db.engine.begin() as conn: @@ -87,9 +112,7 @@ def delete_roth_sources(db): conn.execute(db.Sources.delete().where(db.Sources.c.reference == "Roth")) -# calling functions -# add_publication(db) -ingest_all_sources(db) + db.inventory("CWISE J000021.45-481314.9", pretty_print=True) From b21d984c96f14512e8d2482fbaf798ffc5d546b6 Mon Sep 17 00:00:00 2001 From: kelle Date: Thu, 21 Dec 2023 22:26:55 -0500 Subject: [PATCH 33/35] clean up of tests and utils --- scripts/ingests/ingest_utils.py | 39 ++++++--- scripts/ingests/utils.py | 4 + tests/test_utils.py | 140 +++++++++++++++++++------------- 3 files changed, 115 insertions(+), 68 deletions(-) diff --git a/scripts/ingests/ingest_utils.py b/scripts/ingests/ingest_utils.py index 0080f9e52..5ce4ab7eb 100644 --- a/scripts/ingests/ingest_utils.py +++ b/scripts/ingests/ingest_utils.py @@ -25,7 +25,18 @@ check_internet_connection, ) -# TODO: add __all__ to all modules +__all__ = [ + "ingest_names", + "ingest_source", + "ingest_sources", + "ingest_spectral_types", + "ingest_parallaxes", + "ingest_proper_motions", + "ingest_photometry", + "ingest_spectra", + "ingest_instrument", + "find_survey_name_in_simbad", +] logger = logging.getLogger("SIMPLE") @@ -150,20 +161,20 @@ def ingest_sources( other_references, ) = input_values - n_added = 0 - n_existing = 0 - n_names = 0 - n_alt_names = 0 - n_skipped = 0 - n_multiples = 0 + # TODO: figure out counting + # n_added = 0 + # n_existing = 0 + # n_names = 0 + # n_alt_names = 0 + # n_skipped = 0 + # n_multiples = 0 if n_sources > 1: logger.info(f"Trying to add {n_sources} sources") # Loop over each source and decide to ingest, skip, or add alt name for source_counter, source in enumerate(sources): - # ingest_source function starts here - # TODO: figure out counter + logger.debug(f"{source_counter}: Trying to ingest {source}") reference = references[source_counter] @@ -213,7 +224,9 @@ def ingest_sources( # logger.info(f"Names added to database: {n_names} \n") # logger.info(f"Sources already in database: {n_existing}") # logger.info(f"Alt Names added to database: {n_alt_names}") - # logger.info(f"Sources NOT added to database because multiple matches: {n_multiples}") + # logger.info( + # f"Sources NOT added to database because multiple matches: {n_multiples}" + # ) # logger.info(f"Sources NOT added to database: {n_skipped} \n") # if n_added != n_names: @@ -1915,15 +1928,17 @@ def ingest_source( # Find out if source is already in database or not if coords_provided and search_db: + logger.debug(f"Checking database for: {source} at ra: {ra}, dec: {dec}") name_matches = find_source_in_db(db, source, ra=ra, dec=dec) elif search_db: + logger.debug(f"Checking database for: {source}") name_matches = find_source_in_db(db, source) elif not search_db: name_matches = [] else: name_matches = None - logger.debug(f"name_matches: {name_matches}") + logger.debug(f"Source matches in database: {name_matches}") # Source is already in database # Checking out alternate names @@ -1940,7 +1955,7 @@ def ingest_source( with db.engine.connect() as conn: conn.execute(db.Names.insert().values(alt_names_data)) conn.commit() - logger.debug(f" Name added to database: {alt_names_data}\n") + logger.info(f" Name added to database: {alt_names_data}\n") except sqlalchemy.exc.IntegrityError as e: msg = f" Could not add {alt_names_data} to database" logger.warning(msg) diff --git a/scripts/ingests/utils.py b/scripts/ingests/utils.py index 9da64bc90..205a8927b 100644 --- a/scripts/ingests/utils.py +++ b/scripts/ingests/utils.py @@ -182,6 +182,10 @@ def find_source_in_db(db, source, ra=None, dec=None, search_radius=60.0): logger.debug(msg) # Search database around that coordinate radius = u.Quantity(search_radius, unit="arcsec") + msg2 = ( + f"Finding SIMBAD matches around {simbad_skycoord} with radius {radius}" + ) + logger.debug(msg2) db_name_matches = db.query_region(simbad_skycoord, radius=radius) if len(db_name_matches) == 1: diff --git a/tests/test_utils.py b/tests/test_utils.py index 5ba0c9199..92a96a53b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,20 +1,34 @@ -# Test to verify functions in utils import pytest -import sys import math import os - -sys.path.append(".") -from scripts.ingests.utils import * -from scripts.ingests.ingest_utils import * -from simple.schema import * from astrodbkit2.astrodb import create_database, Database from astropy.table import Table from sqlalchemy import and_ +from scripts.ingests.utils import ( + SimpleError, + find_publication, + ingest_publication, +) +from scripts.ingests.ingest_utils import ( + convert_spt_string_to_code, + ingest_companion_relationships, + ingest_source, + ingest_sources, + ingest_parallaxes, + ingest_spectral_types, + ingest_instrument, + ingest_proper_motions, +) +from simple.schema import * +import logging + + +logger = logging.getLogger("SIMPLE") +logger.setLevel(logging.DEBUG) + DB_NAME = "temp.db" DB_PATH = "data" -logger.setLevel(logging.INFO) # Load the database for use in individual tests @@ -110,36 +124,33 @@ def test_setup_db(db): return db +@pytest.mark.filterwarnings( + "ignore::UserWarning" +) # suppress astroquery SIMBAD warnings def test_ingest_sources(db): # TODO: Test adding an alt name source_data1 = Table( [ { - "source": "Fake 1", - "ra": 9.0673755, - "dec": 18.352889, + "source": "Apple", + "ra": 10.0673755, + "dec": 17.352889, "reference": "Ref 1", }, { - "source": "Fake 6", - "ra": 10.0673755, - "dec": 18.352889, + "source": "Orange", + "ra": 12.0673755, + "dec": -15.352889, "reference": "Ref 2", }, { - "source": "Fake 7", - "ra": 11.0673755, - "dec": 18.352889, + "source": "Banana", + "ra": 119.0673755, + "dec": -28.352889, "reference": "Ref 1", }, ] ) - source_data5 = Table( - [{"source": "Fake 5", "ra": 9.06799, "dec": 18.352889, "reference": ""}] - ) - source_data8 = Table( - [{"source": "Fake 8", "ra": 9.06799, "dec": 18.352889, "reference": "Ref 4"}] - ) ingest_sources( db, @@ -149,51 +160,66 @@ def test_ingest_sources(db): references=source_data1["reference"], raise_error=True, ) + assert db.query(db.Sources).filter(db.Sources.c.source == "Apple").count() == 1 + assert db.query(db.Sources).filter(db.Sources.c.source == "Orange").count() == 1 + assert db.query(db.Sources).filter(db.Sources.c.source == "Banana").count() == 1 + + +@pytest.mark.filterwarnings( + "ignore::UserWarning" +) # suppress astroquery SIMBAD warnings +def test_ingest_source(db): + ingest_source(db, "Barnard Star", reference="Ref 2", raise_error=True) - ingest_sources(db, ["Barnard Star"], references="Ref 2", raise_error=True) Barnard_star = ( db.query(db.Sources).filter(db.Sources.c.source == "Barnard Star").astropy() ) assert len(Barnard_star) == 1 - assert math.isclose(Barnard_star["ra"], 269.452, abs_tol=0.001) - assert math.isclose(Barnard_star["dec"], 4.6933, abs_tol=0.001) - - assert db.query(db.Sources).filter(db.Sources.c.source == "Fake 1").count() == 1 - assert db.query(db.Sources).filter(db.Sources.c.source == "Fake 6").count() == 1 - assert db.query(db.Sources).filter(db.Sources.c.source == "Fake 7").count() == 1 - + assert math.isclose(Barnard_star["ra"][0], 269.452, abs_tol=0.001) + assert math.isclose(Barnard_star["dec"][0], 4.6933, abs_tol=0.001) + + source_data8 = { + "source": "Fake 8", + "ra": 9.06799, + "dec": 18.352889, + "reference": "Ref 4", + } with pytest.raises(SimpleError) as error_message: - ingest_sources( + ingest_source( db, source_data8["source"], - ras=source_data8["ra"], - decs=source_data5["dec"], - references=source_data8["reference"], + ra=source_data8["ra"], + dec=source_data8["dec"], + reference=source_data8["reference"], raise_error=True, ) assert "not in Publications table" in str(error_message.value) + source_data5 = { + "source": "Fake 5", + "ra": 9.06799, + "dec": 18.352889, + "reference": "", + } with pytest.raises(SimpleError) as error_message: - ingest_sources( + ingest_source( db, source_data5["source"], - ras=source_data5["ra"], - decs=source_data5["dec"], - references=source_data5["reference"], + ra=source_data5["ra"], + dec=source_data5["dec"], + reference=source_data5["reference"], raise_error=True, ) assert "blank" in str(error_message.value) with pytest.raises(SimpleError) as error_message: - ingest_sources(db, ["NotinSimbad"], references="Ref 1", raise_error=True) + ingest_source(db, "NotinSimbad", reference="Ref 1", raise_error=True) assert "Coordinates are needed" in str(error_message.value) - -def test_ingest_source(): with pytest.raises(SimpleError) as error_message: ingest_source( db, - "Fake7", + "Fake 1", ra=11.0673755, dec=18.352889, reference="Ref 1", @@ -279,13 +305,13 @@ def test_ingest_spectral_types(db): ] ) - data2 = Table( - [ - {"source": "Fake 1", "spectral_type": "M5.6", "reference": "Ref 1"}, - {"source": "Fake 2", "spectral_type": "T0.1", "reference": "Ref 1"}, - {"source": "Fake 3", "spectral_type": "Y2pec", "reference": "Ref 2"}, - ] - ) + # data2 = Table( + # [ + # {"source": "Fake 1", "spectral_type": "M5.6", "reference": "Ref 1"}, + # {"source": "Fake 2", "spectral_type": "T0.1", "reference": "Ref 1"}, + # {"source": "Fake 3", "spectral_type": "Y2pec", "reference": "Ref 2"}, + # ] + # ) data3 = Table( [ @@ -387,8 +413,10 @@ def test_ingest_instrument(db): # tel_test = 'test2' # inst_test = 'test3' # ingest_instrument(db, telescope=tel_test, instrument=inst_test) - # telescope_db = db.query(db.Telescopes).filter(db.Telescopes.c.telescope == tel_test).table() - # instrument_db = db.query(db.Instruments).filter(db.Instruments.c.instrument == inst_test).table() + # telescope_db = db.query(db.Telescopes). + # filter(db.Telescopes.c.telescope == tel_test).table() + # instrument_db = db.query(db.Instruments). + # filter(db.Instruments.c.instrument == inst_test).table() # assert len(telescope_db) == 1 # assert telescope_db['telescope'][0] == tel_test # assert len(instrument_db) == 1 @@ -463,24 +491,24 @@ def test_ingest_instrument(db): def test_companion_relationships(db): # testing companion ingest - ## trying no companion + # trying no companion with pytest.raises(SimpleError) as error_message: ingest_companion_relationships(db, "Fake 1", None, "Sibling") assert "Make sure all require parameters are provided." in str(error_message.value) - ## trying companion == source + # trying companion == source with pytest.raises(SimpleError) as error_message: ingest_companion_relationships(db, "Fake 1", "Fake 1", "Sibling") assert "Source cannot be the same as companion name" in str(error_message.value) - ## trying negative separation + # trying negative separation with pytest.raises(SimpleError) as error_message: ingest_companion_relationships( db, "Fake 1", "Bad Companion", "Sibling", projected_separation_arcsec=-5 ) assert "cannot be negative" in str(error_message.value) - ## trying negative separation error + # trying negative separation error with pytest.raises(SimpleError) as error_message: ingest_companion_relationships( db, "Fake 1", "Bad Companion", "Sibling", projected_separation_error=-5 From 78f5611e00af92295214940d602ec797a1c2f9c0 Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Fri, 22 Dec 2023 09:04:22 -0500 Subject: [PATCH 34/35] Corrected 2mass-j1256 json again --- data/2mass_j12560183-1257276.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/2mass_j12560183-1257276.json b/data/2mass_j12560183-1257276.json index 6f15633b6..eed3a0b12 100644 --- a/data/2mass_j12560183-1257276.json +++ b/data/2mass_j12560183-1257276.json @@ -30,9 +30,6 @@ { "other_name": "VHS 1256-1257b" }, - { - "other_name": "VHS 1256b" - }, { "other_name": "VHS J125601.92-125723.9 b" }, @@ -41,6 +38,9 @@ }, { "other_name": "WISEA J125601.66-125728.7" + }, + { + "other_name": "VHS 1256b" } ], "Parallaxes": [ @@ -267,4 +267,4 @@ "reference": "Gauz15" } ] -} \ No newline at end of file +} From 5f42f96319a2bd8306670e6e7c92b937d51697cb Mon Sep 17 00:00:00 2001 From: Lisha Ramon Date: Fri, 22 Dec 2023 13:39:53 -0500 Subject: [PATCH 35/35] Ingest json shortname + deleted extra file --- data/cwise_j000021.45-481314.9.json | 3 +++ scripts/ingests/ingest_BYW_2024.py | 8 +++----- scripts/ingests/ingest_backyard_worlds_benchmarks.py | 3 --- 3 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 scripts/ingests/ingest_backyard_worlds_benchmarks.py diff --git a/data/cwise_j000021.45-481314.9.json b/data/cwise_j000021.45-481314.9.json index f941bef11..d79e1a87a 100644 --- a/data/cwise_j000021.45-481314.9.json +++ b/data/cwise_j000021.45-481314.9.json @@ -13,6 +13,9 @@ } ], "Names": [ + { + "other_name": "CWISE 0000-4813" + }, { "other_name": "CWISE J000021.45-481314.9" } diff --git a/scripts/ingests/ingest_BYW_2024.py b/scripts/ingests/ingest_BYW_2024.py index 99e790761..37b27b972 100644 --- a/scripts/ingests/ingest_BYW_2024.py +++ b/scripts/ingests/ingest_BYW_2024.py @@ -32,8 +32,7 @@ def ingest_all_sources(db): for row in byw_table[1:90]: # skip the header row - [1:10]runs only first 10 rows # Print byw source information print("BYW Source Information:") - # for key, value in row_dict.items(): - # print(f"{key}: {value}") + for col_name in row.colnames: print(f"{col_name}: {row[col_name]}") @@ -49,8 +48,7 @@ def ingest_all_sources(db): raise_error=True, search_db=True, ) - # print(row['ra']) - # print(row_dict['ra']) + # Add a separator between rows for better readability print("-" * 20) @@ -62,7 +60,7 @@ def ingest_all_sources(db): #Ingest shortnames as other names to source #Loop through data def ingest_all_shortnames(db): - for row in byw_table[55:]: # skip the header row - [1:10]runs only first 10 rows + for row in byw_table[1:90]: # Print byw source information print("BYW Source Information:") diff --git a/scripts/ingests/ingest_backyard_worlds_benchmarks.py b/scripts/ingests/ingest_backyard_worlds_benchmarks.py deleted file mode 100644 index 8ccafb210..000000000 --- a/scripts/ingests/ingest_backyard_worlds_benchmarks.py +++ /dev/null @@ -1,3 +0,0 @@ -test - -