Newcandidate 360rc2 (#1562)

Co-authored-by: Zach Burnett <[email protected]>
Co-authored-by: Steve Goldman <[email protected]>

drizzlepac/hapsequencer.py

@@ -302,11 +302,13 @@ def create_catalog_products(total_obj_list, log_level, diagnostic_mode=False, ph
 
             # Determine whether any catalogs should be written out at all based on comparison to expected
             # rate of cosmic-ray contamination for the total detection product
+            reject_catalogs = {}
             reject_catalogs = total_product_catalogs.verify_crthresh(n1_exposure_time)
 
             if diagnostic_mode:
                 # If diagnostic mode, we want to inspect the original full source catalogs
-                reject_catalogs = False
+                for cat_type in total_product_catalogs.catalogs.keys():
+                    reject_catalogs[cat_type] = False
 
             for filter_product_obj in total_product_obj.fdp_list:
                 filter_product_catalogs = filter_catalogs[filter_product_obj.drizzle_filename]

drizzlepac/haputils/catalog_utils.py

@@ -625,14 +625,17 @@ def __init__(self, fitsfile, param_dict, param_dict_qc, num_images_mask, log_lev
         # This does NOT identify or measure sources to create the catalogs at this point...
         # The syntax here is EXTREMELY cludgy, but until a more compact way to do this is found,
         #  it will have to do...
+        self.reject_cats = {}
         self.catalogs = {}
         if 'segment' in self.types:
             self.catalogs['segment'] = HAPSegmentCatalog(self.image, self.param_dict, self.param_dict_qc,
                                                          self.diagnostic_mode, tp_sources=tp_sources)
+            self.reject_cats['segment'] = False
 
         if 'aperture' in self.types:
             self.catalogs['aperture'] = HAPPointCatalog(self.image, self.param_dict, self.param_dict_qc,
                                                         self.diagnostic_mode, tp_sources=tp_sources)
+            self.reject_cats['aperture'] = False
 
         self.filters = {}
 
@@ -654,10 +657,23 @@ def identify(self, **pars):
     def verify_crthresh(self, n1_exposure_time):
         """Verify whether catalogs meet cosmic-ray threshold limits.
 
-        ... note : If either catalog fails the following test, then both are rejected.
-                        n_cat < thresh
-                   where
-                        thresh = crfactor * n1_exposure_time**2 / texptime
+        ... note : This algorithm has been modified from the original implementation
+                   where if either catalog failed the cosmic ray criterion test, then
+                   both catalogs would be rejected.
+
+                   Instead...
+                   an empty catalog (n_sources = 0) should not get rejected by the CR
+                   contamination test (since it is empty, it clearly is not contaminated
+                   by cosmic rays!) Also, if a catalog is empty because it is rejected
+                   for some other reason, it should never trigger the rejection of the
+                   other type of catalog.
+
+                   thresh = crfactor * n1_exposure_time**2 / texptime
+
+                   Since catalog output (*.ecsv) files are always written, rejection
+                   means that all of the rows of measurements are deleted from the
+                   output ECSV file.
+
         """
         for cat_type in self.catalogs:
             crthresh_mask = None
@@ -673,8 +689,6 @@ def verify_crthresh(self, n1_exposure_time):
                     crthresh_mask = np.bitwise_or(crthresh_mask, catalog_crmask)
             source_cat.sources_num_good = len(np.where(crthresh_mask)[0])
 
-        reject_catalogs = False
-
         log.info("Determining whether point and/or segment catalogs meet cosmic-ray threshold")
         log.info("  based on EXPTIME = {}sec for the n=1 filters".format(n1_exposure_time))
 
@@ -686,13 +700,11 @@ def verify_crthresh(self, n1_exposure_time):
                 n_sources = source_cat.sources_num_good  # len(source_cat)
                 all_sources = len(source_cat)
                 log.info("{} catalog with {} good sources out of {} total sources :  CR threshold = {}".format(cat_type, n_sources, all_sources, thresh))
-                # n_sources == 0 should never get here, but just to cover the past case of 0 < 0.0
-                if n_sources < thresh or n_sources == 0:
-                    reject_catalogs = True
-                    log.info("{} catalog FAILED CR threshold.  Rejecting both catalogs...".format(cat_type))
-                    break
+                if n_sources < thresh and 0 < n_sources:
+                    self.reject_cats[cat_type] = True
+                    log.info("{} catalog FAILED CR threshold.".format(cat_type))
 
-        return reject_catalogs
+        return self.reject_cats
 
     def measure(self, filter_name, **pars):
         """Perform photometry and other measurements on sources for this image.
@@ -719,8 +731,10 @@ def write(self, reject_catalogs, **pars):
 
         Parameters
         ----------
-        reject_catalogs : bool
-            Indicator as to whether or not the catalogs (*.ecsv) should be written.
+        reject_catalogs : dictionary
+            Dictionary where the keys are the types of catalogs, and the values are
+            bools indicating whether or not the catalogs (*.ecsv) should be written
+            with their full contents or as empty catalogs.
 
         types : list
             List of catalog types to be generated.  If None, build all available catalogs.
@@ -1334,8 +1348,10 @@ def write_catalog(self, reject_catalogs):
 
         Parameters
         ----------
-        reject_catalogs : bool
-            Indicator as to whether or not the catalogs (*.ecsv) should be written.
+        reject_catalogs : dictionary
+            Dictionary where the keys are the types of catalogs, and the values are
+            bools indicating whether or not the catalogs (*.ecsv) should be written
+            with their full contents or as empty catalogs.
 
         Returns
         -------
@@ -1349,11 +1365,15 @@ def write_catalog(self, reject_catalogs):
         written to output ECSV files, they are written as empty ("") strings.  In order for the catalogs
         to be ingested into a database by possible downstream processing, the masked values will be
         replaced by a numeric indicator.
+
+        ... note : A catalog can have no rows of measurements because no sources were
+        found OR because the catalog was "rejected" according to the cosmic ray rejection
+        criterion.
         """
         # Insure catalog has all necessary metadata
         self.source_cat = self.annotate_table(self.source_cat, self.param_dict_qc, proc_type="aperture",
                                               product=self.image.ghd_product)
-        if reject_catalogs:
+        if reject_catalogs[self.catalog_type]:
             # We still want to write out empty files
             # This will delete all rows from the existing table
             self.source_cat.remove_rows(slice(0, None))
@@ -2796,8 +2816,10 @@ def write_catalog(self, reject_catalogs):
 
         Parameters
         ----------
-        reject_catalogs : bool
-            Indicator as to whether or not the catalogs (*.ecsv) should be written.
+        reject_catalogs : dictionary
+            Dictionary where the keys are the types of catalogs, and the values are
+            bools indicating whether or not the catalogs (*.ecsv) should be written
+            with their full contents or as empty catalogs.
 
         Returns
         -------
@@ -2811,11 +2833,15 @@ def write_catalog(self, reject_catalogs):
         written to output ECSV files, they are written as empty ("") strings.  In order for the catalogs
         to be ingested into a database by possible downstream processing, the masked values will be
         replaced by a numeric indicator.
+
+        ... note : A catalog can have no rows of measurements because no sources were
+        found OR because the catalog was "rejected" according to the cosmic ray rejection
+        criterion.
         """
         self.source_cat = self.annotate_table(self.source_cat, self.param_dict_qc, proc_type="segment",
                                               product=self.image.ghd_product)
 
-        if reject_catalogs:
+        if reject_catalogs[self.catalog_type]:
             # We still want to write out empty files
             # This will delete all rows from the existing table
             self.source_cat.remove_rows(slice(0, None))

requirements-dev.txt

@@ -1,6 +1,6 @@
 git+https://github.com/astropy/photutils.git#egg=photutils
 git+https://github.com/spacetelescope/stsci.tools.git#egg=stsci.tools
-astropy>=0.0.dev0
+git+https://github.com/astropy/astropy.git#egg=astropy
 git+https://github.com/spacetelescope/stwcs.git#egg=stwcs
 git+https://github.com/astropy/astroquery.git#egg=astroquery
 numpy>=0.0.dev0

tests/acs/test_asn_regress.py

@@ -11,6 +11,9 @@
 class TestAsnRegress(BaseACS):
 
     def test_hrc_asn(self):
+        # Customized tolerances as Linux and Mac would need different truth files.
+        self.rtol = 1e-4
+        self.atol = 1e-5
         rootname = 'j8bt06010'
         asn_file = rootname + '_asn.fits'
 

tests/stis/test_stis.py

@@ -107,6 +107,10 @@ def test_stis_ccd(self):
         distortion model for STIS CCD data and create a combined product.
         """
 
+        # Customized tolerances as Linux and Mac would need different truth files.
+        self.rtol = 1e-3
+        self.atol = 1e-4
+
         # Prepare input files.
         raw_inputs = ['o6cl10arq_flt.fits', 'o6cl10asq_flt.fits',
                       'o6cl10atq_flt.fits', 'o6cl10auq_flt.fits',

tests/wfc3/test_wfc3.py

@@ -13,6 +13,9 @@
 class TestWFC3(BaseWFC3):
 
     def test_binned_single(self):
+        # Customized tolerances as Linux and Mac would need different truth files.
+        self.rtol = 1e-5
+        self.atol = 1e-5
         rootname = 'iacs01t9q'
         input_name = '{}_flt.fits'.format(rootname)
         output = '{}_drz.fits'.format(rootname)
@@ -41,6 +44,9 @@ def test_binned_single(self):
 
 
     def test_uvis_single(self):
+        # Customized tolerances as Linux and Mac would need different truth files.
+        self.rtol = 1e-3
+        self.atol = 1e-3
         rootname = 'iacr51ohq'
         input_name = '{}_flt.fits'.format(rootname)
         output = '{}_drz.fits'.format(rootname)

tests/wfpc2/test_wfpc2.py

@@ -132,6 +132,9 @@ def test_mef_asn(self):
         WFPC2 data stored in Multi-extensions FITS(MEF) format.
         """
 
+        # Customized tolerances as Linux and Mac would need different truth files.
+        self.rtol = 1e-6
+        self.atol = 1e-4
         # Prepare input files.
         raw_inputs = ['u9yq0703m_c0m.fits', 'u9yq0704m_c0m.fits',
                       'u9yq0707m_c0m.fits', 'u9yq0708m_c0m.fits',

tox.ini

@@ -47,13 +47,15 @@ description =
     cov: with coverage
     xdist: using parallel processing
 package = editable
+set_env =
+    devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scipy-wheels-nightly/simple
 deps =
     pytest
     ci_watson
     xdist: pytest-xdist
     cov: pytest-cov
-    devdeps: -rrequirements-dev.txt
 commands_pre =
+    devdeps: pip install -r requirements-dev.txt -U --upgrade-strategy eager
     pip freeze
 commands =
     pytest -s --basetemp=test_outputs tests \