diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 055966c6a..86b1863ea 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,16 +18,14 @@ number of the code change for that issue. These PRs can be viewed at: https://github.com/spacetelescope/drizzlepac/pulls -3.7.2 (unreleased) -================== -- Added python 3.12 to testing matrix for Jenkins and github actions. [#1843] - -- ``manageInputCopies`` now copies successfully even if the original files were - defined by full paths rather than being in the current working directory. [#1835] - - 3.7.1 (unreleased) ================== +- Corrected the way the n1_exposure_time and tot_exposure_time values + are computed as these values are used in the computation for rejecting + catalog creation based on expected cosmic ray detections. Generalized + the crfactor dictionary for all detectors. Ensure if any catalog type + is rejected, all the catalog types are rejected. [#1853] + - Modified the call to the hamming function in the deconvolve_utils.py module as SciPy deprecated the way window filtering functions can be invoked. These functions can no longer be imported from the scipy.signal namespace but need diff --git a/drizzlepac/hapsequencer.py b/drizzlepac/hapsequencer.py index 578ff1143..40cde0189 100755 --- a/drizzlepac/hapsequencer.py +++ b/drizzlepac/hapsequencer.py @@ -179,6 +179,57 @@ def create_catalog_products(total_obj_list, log_level, diagnostic_mode=False, ph # Get parameter from config files for CR rejection of catalogs cr_residual = total_product_obj.configobj_pars.get_pars('catalog generation')['cr_residual'] n1_exposure_time = 0 + tot_exposure_time = 0 + n1_factor = 0.0 + n1_dict = {} + + # Compute the n1_exposure_time for the total detection image as + # this value is used as a criterion for rejecting catalog creation + if total_product_obj.detector.upper() not in ['IR', 'SBC']: + + # Create a dictionary based on ALL of the exposures available to be + # used for the creation of this particular total detection image. + # Accummulate the number of exposures per filter and the corresponding + # exposure time. + #pdb.set_trace() + for edp in total_product_obj.edp_list: + if edp.filters not in n1_dict: + n1_dict[edp.filters] = {'n': 1, 'texptime': edp.exptime} + else: + n1_dict[edp.filters]['n'] += 1 + n1_dict[edp.filters]['texptime'] += edp.exptime + + # Since single filter exposures are excluded from the total + # detection image, these exposures should not be included in + # the computation of the tot_exposure_time. The exception to + # this statement is when there are only single exposures from + # all the filters which comprise the total detection image. + + # If the total detection image is comprised of more than one + # exposure in at least one filter... + filter_info = n1_dict.values() + if max([x['n'] for x in n1_dict.values()]) > 1: + for values in filter_info: + if values['n'] > 1: + tot_exposure_time += values['texptime'] + # ...else if the the detection images is comprised of single + # filter exposures + else: + for values in filter_info: + if values['n'] == 1: + tot_exposure_time += values['texptime'] + n1_exposure_time += values['texptime'] + n1_factor += cr_residual + + # Insure that n1_factor only improves the threshold, not make it worse. + n1_factor = min(n1_factor, 1.0) + + # Account for the influence of the single-image cosmic-ray identification + # This fraction represents the residual number of cosmic-rays after single-image identification + if n1_exposure_time < tot_exposure_time: + n1_exposure_time *= n1_factor + + log.info(f"Total exposure time of {tot_exposure_time} for total detection image: {total_product_obj.drizzle_filename}") log.info("Generating filter product source catalogs") for filter_product_obj in total_product_obj.fdp_list: @@ -209,33 +260,6 @@ def create_catalog_products(total_obj_list, log_level, diagnostic_mode=False, ph log_level, diagnostic_mode) - if total_product_obj.detector.upper() not in ['IR', 'SBC']: - # Apply cosmic-ray threshold criteria used by HLA to determine whether or not to reject - # the catalogs. - tot_exposure_time = 0 - n1_factor = 0.0 - n1_dict = {} - for edp in total_product_obj.edp_list: - if edp.filters not in n1_dict: - n1_dict[edp.filters] = {'n': 1, 'texptime': edp.exptime} - else: - n1_dict[edp.filters]['n'] += 1 - n1_dict[edp.filters]['texptime'] += edp.exptime - - for edp in total_product_obj.edp_list: - tot_exposure_time += edp.exptime - if n1_dict[edp.filters]['n'] == 1: - n1_exposure_time += edp.exptime - n1_factor += cr_residual - - # Insure that n1_factor only improves the threshold, not make it worse. - n1_factor = min(n1_factor, 1.0) - - # Account for the influence of the single-image cosmic-ray identification - # This fraction represents the residual number of cosmic-rays after single-image identification - if n1_exposure_time < tot_exposure_time: - n1_exposure_time *= n1_factor - # write out CI and FWHM values to file (if IRAFStarFinder was used instead of DAOStarFinder) for hla_flag_filter parameter optimization. if diagnostic_mode and phot_mode in ['aperture', 'both']: if "fwhm" in total_product_catalogs.catalogs['aperture'].sources.colnames: diff --git a/drizzlepac/haputils/catalog_utils.py b/drizzlepac/haputils/catalog_utils.py index 7caa5ad28..75d0a0100 100755 --- a/drizzlepac/haputils/catalog_utils.py +++ b/drizzlepac/haputils/catalog_utils.py @@ -556,7 +556,16 @@ def _get_max_key_value(self, header, root_of_keyword): class HAPCatalogs: """Generate photometric sourcelist for specified TOTAL or FILTER product image. """ - crfactor = {'aperture': 300, 'segment': 150} # CRs / hr / 4kx4k pixels + + # This reference for the crfactor is Rick White. It is based upon the physical + # area of the detector. From the instrument handbooks: + # ACS/WFC: 4096**2 pixels, pixel size (15 um)**2 + # ACS/HRC: 1024**2 pixels, pixel size (21 um)**2 + # WFC3/UVIS: 4096**2 pixels, pixel size (15 um)**2 + # WFPC2: 1600**2 pixels, pixel size (15um)**2 + # acs/wfc-> crfactor = {'aperture': 300, 'segment': 150} # CRs / hr / 4kx4k pixels + crfactor = {'WFC': {'aperture': 300, 'segment': 150}, 'HRC': {'aperture': 37, 'segment': 18.5}, + 'UVIS': {'aperture': 300, 'segment': 150}, 'PC': {'aperture': 46, 'segment': 23}} def __init__(self, fitsfile, param_dict, param_dict_qc, num_images_mask, log_level, diagnostic_mode=False, types=None, tp_sources=None): @@ -664,6 +673,9 @@ def verify_crthresh(self, n1_exposure_time): crthresh_mask = None source_cat = self.catalogs[cat_type].sources if cat_type == 'aperture' else self.catalogs[cat_type].source_cat + # Collect up the names of all the columns which start with "Flag" - there will be + # a flag column for each filter in the visit - even the filters which did not end up + # contributing to the total detection image. flag_cols = [colname for colname in source_cat.colnames if colname.startswith('Flag')] for colname in flag_cols: catalog_crmask = source_cat[colname] < 2 @@ -677,10 +689,11 @@ def verify_crthresh(self, n1_exposure_time): log.info("Determining whether point and/or segment catalogs meet cosmic-ray threshold") log.info(" based on EXPTIME = {}sec for the n=1 filters".format(n1_exposure_time)) + detector = self.image.keyword_dict["detector"].upper() for cat_type in self.catalogs: source_cat = self.catalogs[cat_type] if source_cat.sources: - thresh = self.crfactor[cat_type] * n1_exposure_time**2 / self.image.keyword_dict['texpo_time'] + thresh = self.crfactor[detector][cat_type] * n1_exposure_time**2 / self.image.keyword_dict['texpo_time'] source_cat = source_cat.sources if cat_type == 'aperture' else source_cat.source_cat n_sources = source_cat.sources_num_good # len(source_cat) all_sources = len(source_cat) @@ -689,6 +702,11 @@ def verify_crthresh(self, n1_exposure_time): self.reject_cats[cat_type] = True log.info("{} catalog FAILED CR threshold.".format(cat_type)) + # Ensure if any catalog is rejected, the remaining catalogs are also rejected + if any([True for k,v in self.reject_cats.items() if v == True]): + for k in self.reject_cats.keys(): + self.reject_cats[k] = True + return self.reject_cats def measure(self, filter_name, **pars):