From 538871b40f83ec9c5de15106a0d0e55efeae11aa Mon Sep 17 00:00:00 2001 From: mdlpstsci Date: Wed, 24 May 2023 11:20:04 -0400 Subject: [PATCH] Newcandidate 360rc2 (#1562) Co-authored-by: Zach Burnett Co-authored-by: Steve Goldman <32876747+s-goldman@users.noreply.github.com> --- drizzlepac/hapsequencer.py | 4 +- drizzlepac/haputils/catalog_utils.py | 66 +++++++++++++++++++-------- drizzlepac/pars/allsky_cells.fits | Bin 8640 -> 8640 bytes requirements-dev.txt | 2 +- tests/acs/test_asn_regress.py | 3 ++ tests/stis/test_stis.py | 4 ++ tests/wfc3/test_wfc3.py | 6 +++ tests/wfpc2/test_wfpc2.py | 3 ++ tox.ini | 4 +- 9 files changed, 69 insertions(+), 23 deletions(-) diff --git a/drizzlepac/hapsequencer.py b/drizzlepac/hapsequencer.py index d15cc8292..ecfa331d6 100755 --- a/drizzlepac/hapsequencer.py +++ b/drizzlepac/hapsequencer.py @@ -302,11 +302,13 @@ def create_catalog_products(total_obj_list, log_level, diagnostic_mode=False, ph # Determine whether any catalogs should be written out at all based on comparison to expected # rate of cosmic-ray contamination for the total detection product + reject_catalogs = {} reject_catalogs = total_product_catalogs.verify_crthresh(n1_exposure_time) if diagnostic_mode: # If diagnostic mode, we want to inspect the original full source catalogs - reject_catalogs = False + for cat_type in total_product_catalogs.catalogs.keys(): + reject_catalogs[cat_type] = False for filter_product_obj in total_product_obj.fdp_list: filter_product_catalogs = filter_catalogs[filter_product_obj.drizzle_filename] diff --git a/drizzlepac/haputils/catalog_utils.py b/drizzlepac/haputils/catalog_utils.py index ffd390e3b..178a337fb 100755 --- a/drizzlepac/haputils/catalog_utils.py +++ b/drizzlepac/haputils/catalog_utils.py @@ -625,14 +625,17 @@ def __init__(self, fitsfile, param_dict, param_dict_qc, num_images_mask, log_lev # This does NOT identify or measure sources to create the catalogs at this point... # The syntax here is EXTREMELY cludgy, but until a more compact way to do this is found, # it will have to do... + self.reject_cats = {} self.catalogs = {} if 'segment' in self.types: self.catalogs['segment'] = HAPSegmentCatalog(self.image, self.param_dict, self.param_dict_qc, self.diagnostic_mode, tp_sources=tp_sources) + self.reject_cats['segment'] = False if 'aperture' in self.types: self.catalogs['aperture'] = HAPPointCatalog(self.image, self.param_dict, self.param_dict_qc, self.diagnostic_mode, tp_sources=tp_sources) + self.reject_cats['aperture'] = False self.filters = {} @@ -654,10 +657,23 @@ def identify(self, **pars): def verify_crthresh(self, n1_exposure_time): """Verify whether catalogs meet cosmic-ray threshold limits. - ... note : If either catalog fails the following test, then both are rejected. - n_cat < thresh - where - thresh = crfactor * n1_exposure_time**2 / texptime + ... note : This algorithm has been modified from the original implementation + where if either catalog failed the cosmic ray criterion test, then + both catalogs would be rejected. + + Instead... + an empty catalog (n_sources = 0) should not get rejected by the CR + contamination test (since it is empty, it clearly is not contaminated + by cosmic rays!) Also, if a catalog is empty because it is rejected + for some other reason, it should never trigger the rejection of the + other type of catalog. + + thresh = crfactor * n1_exposure_time**2 / texptime + + Since catalog output (*.ecsv) files are always written, rejection + means that all of the rows of measurements are deleted from the + output ECSV file. + """ for cat_type in self.catalogs: crthresh_mask = None @@ -673,8 +689,6 @@ def verify_crthresh(self, n1_exposure_time): crthresh_mask = np.bitwise_or(crthresh_mask, catalog_crmask) source_cat.sources_num_good = len(np.where(crthresh_mask)[0]) - reject_catalogs = False - log.info("Determining whether point and/or segment catalogs meet cosmic-ray threshold") log.info(" based on EXPTIME = {}sec for the n=1 filters".format(n1_exposure_time)) @@ -686,13 +700,11 @@ def verify_crthresh(self, n1_exposure_time): n_sources = source_cat.sources_num_good # len(source_cat) all_sources = len(source_cat) log.info("{} catalog with {} good sources out of {} total sources : CR threshold = {}".format(cat_type, n_sources, all_sources, thresh)) - # n_sources == 0 should never get here, but just to cover the past case of 0 < 0.0 - if n_sources < thresh or n_sources == 0: - reject_catalogs = True - log.info("{} catalog FAILED CR threshold. Rejecting both catalogs...".format(cat_type)) - break + if n_sources < thresh and 0 < n_sources: + self.reject_cats[cat_type] = True + log.info("{} catalog FAILED CR threshold.".format(cat_type)) - return reject_catalogs + return self.reject_cats def measure(self, filter_name, **pars): """Perform photometry and other measurements on sources for this image. @@ -719,8 +731,10 @@ def write(self, reject_catalogs, **pars): Parameters ---------- - reject_catalogs : bool - Indicator as to whether or not the catalogs (*.ecsv) should be written. + reject_catalogs : dictionary + Dictionary where the keys are the types of catalogs, and the values are + bools indicating whether or not the catalogs (*.ecsv) should be written + with their full contents or as empty catalogs. types : list List of catalog types to be generated. If None, build all available catalogs. @@ -1334,8 +1348,10 @@ def write_catalog(self, reject_catalogs): Parameters ---------- - reject_catalogs : bool - Indicator as to whether or not the catalogs (*.ecsv) should be written. + reject_catalogs : dictionary + Dictionary where the keys are the types of catalogs, and the values are + bools indicating whether or not the catalogs (*.ecsv) should be written + with their full contents or as empty catalogs. Returns ------- @@ -1349,11 +1365,15 @@ def write_catalog(self, reject_catalogs): written to output ECSV files, they are written as empty ("") strings. In order for the catalogs to be ingested into a database by possible downstream processing, the masked values will be replaced by a numeric indicator. + + ... note : A catalog can have no rows of measurements because no sources were + found OR because the catalog was "rejected" according to the cosmic ray rejection + criterion. """ # Insure catalog has all necessary metadata self.source_cat = self.annotate_table(self.source_cat, self.param_dict_qc, proc_type="aperture", product=self.image.ghd_product) - if reject_catalogs: + if reject_catalogs[self.catalog_type]: # We still want to write out empty files # This will delete all rows from the existing table self.source_cat.remove_rows(slice(0, None)) @@ -2796,8 +2816,10 @@ def write_catalog(self, reject_catalogs): Parameters ---------- - reject_catalogs : bool - Indicator as to whether or not the catalogs (*.ecsv) should be written. + reject_catalogs : dictionary + Dictionary where the keys are the types of catalogs, and the values are + bools indicating whether or not the catalogs (*.ecsv) should be written + with their full contents or as empty catalogs. Returns ------- @@ -2811,11 +2833,15 @@ def write_catalog(self, reject_catalogs): written to output ECSV files, they are written as empty ("") strings. In order for the catalogs to be ingested into a database by possible downstream processing, the masked values will be replaced by a numeric indicator. + + ... note : A catalog can have no rows of measurements because no sources were + found OR because the catalog was "rejected" according to the cosmic ray rejection + criterion. """ self.source_cat = self.annotate_table(self.source_cat, self.param_dict_qc, proc_type="segment", product=self.image.ghd_product) - if reject_catalogs: + if reject_catalogs[self.catalog_type]: # We still want to write out empty files # This will delete all rows from the existing table self.source_cat.remove_rows(slice(0, None)) diff --git a/drizzlepac/pars/allsky_cells.fits b/drizzlepac/pars/allsky_cells.fits index fb6accbdd64420a8f56d47da2325976d4b7e486b..71a4cef45f5be7262e4b93d0469c73decb91a859 100644 GIT binary patch delta 31 fcmX@$e871_pBP*He<0A^JVi{76UN>l&%p!$2uu&p delta 31 ecmX@$e871_pBP&M0|;!MA|}TPWABjXU;+S;n+JaY diff --git a/requirements-dev.txt b/requirements-dev.txt index b4fb129da..188b3bcee 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ git+https://github.com/astropy/photutils.git#egg=photutils git+https://github.com/spacetelescope/stsci.tools.git#egg=stsci.tools -astropy>=0.0.dev0 +git+https://github.com/astropy/astropy.git#egg=astropy git+https://github.com/spacetelescope/stwcs.git#egg=stwcs git+https://github.com/astropy/astroquery.git#egg=astroquery numpy>=0.0.dev0 diff --git a/tests/acs/test_asn_regress.py b/tests/acs/test_asn_regress.py index bf325813e..f7430d0c6 100644 --- a/tests/acs/test_asn_regress.py +++ b/tests/acs/test_asn_regress.py @@ -11,6 +11,9 @@ class TestAsnRegress(BaseACS): def test_hrc_asn(self): + # Customized tolerances as Linux and Mac would need different truth files. + self.rtol = 1e-4 + self.atol = 1e-5 rootname = 'j8bt06010' asn_file = rootname + '_asn.fits' diff --git a/tests/stis/test_stis.py b/tests/stis/test_stis.py index b3b8c7a94..c34bd1c1f 100644 --- a/tests/stis/test_stis.py +++ b/tests/stis/test_stis.py @@ -107,6 +107,10 @@ def test_stis_ccd(self): distortion model for STIS CCD data and create a combined product. """ + # Customized tolerances as Linux and Mac would need different truth files. + self.rtol = 1e-3 + self.atol = 1e-4 + # Prepare input files. raw_inputs = ['o6cl10arq_flt.fits', 'o6cl10asq_flt.fits', 'o6cl10atq_flt.fits', 'o6cl10auq_flt.fits', diff --git a/tests/wfc3/test_wfc3.py b/tests/wfc3/test_wfc3.py index a8f62d575..e05bf72ec 100644 --- a/tests/wfc3/test_wfc3.py +++ b/tests/wfc3/test_wfc3.py @@ -13,6 +13,9 @@ class TestWFC3(BaseWFC3): def test_binned_single(self): + # Customized tolerances as Linux and Mac would need different truth files. + self.rtol = 1e-5 + self.atol = 1e-5 rootname = 'iacs01t9q' input_name = '{}_flt.fits'.format(rootname) output = '{}_drz.fits'.format(rootname) @@ -41,6 +44,9 @@ def test_binned_single(self): def test_uvis_single(self): + # Customized tolerances as Linux and Mac would need different truth files. + self.rtol = 1e-3 + self.atol = 1e-3 rootname = 'iacr51ohq' input_name = '{}_flt.fits'.format(rootname) output = '{}_drz.fits'.format(rootname) diff --git a/tests/wfpc2/test_wfpc2.py b/tests/wfpc2/test_wfpc2.py index 5306d191b..c7026f1ca 100644 --- a/tests/wfpc2/test_wfpc2.py +++ b/tests/wfpc2/test_wfpc2.py @@ -132,6 +132,9 @@ def test_mef_asn(self): WFPC2 data stored in Multi-extensions FITS(MEF) format. """ + # Customized tolerances as Linux and Mac would need different truth files. + self.rtol = 1e-6 + self.atol = 1e-4 # Prepare input files. raw_inputs = ['u9yq0703m_c0m.fits', 'u9yq0704m_c0m.fits', 'u9yq0707m_c0m.fits', 'u9yq0708m_c0m.fits', diff --git a/tox.ini b/tox.ini index 05def4dc5..b946b7a54 100644 --- a/tox.ini +++ b/tox.ini @@ -47,13 +47,15 @@ description = cov: with coverage xdist: using parallel processing package = editable +set_env = + devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scipy-wheels-nightly/simple deps = pytest ci_watson xdist: pytest-xdist cov: pytest-cov - devdeps: -rrequirements-dev.txt commands_pre = + devdeps: pip install -r requirements-dev.txt -U --upgrade-strategy eager pip freeze commands = pytest -s --basetemp=test_outputs tests \