diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eb2f70037..98b48c761 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,9 @@ number of the code change for that issue. These PRs can be viewed at: 3.7.2 (unreleased) ================== +- Introduce warnings for fits extensions with science data of all zeros, and ensure + data with zeros in all science extensions are not processed. [#998] + - Change to the algorithm which chooses which background determination algorithm to use for processing when working on the output source catalogs. If the RMS from the Photutils Background2D is greater than the RMS from the astropy.stats diff --git a/drizzlepac/haputils/analyze.py b/drizzlepac/haputils/analyze.py index 8751f2f9b..7b206dd36 100644 --- a/drizzlepac/haputils/analyze.py +++ b/drizzlepac/haputils/analyze.py @@ -16,7 +16,7 @@ from enum import Enum from astropy.io import fits -from astropy.io.fits import getheader +from astropy.io.fits import getheader, getdata from astropy.table import Table from astropy.stats import sigma_clipped_stats import numpy as np @@ -31,6 +31,7 @@ from stsci.tools.bitmask import bitfield_to_boolean_mask from .astrometric_utils import classify_sources +from ..util import count_sci_extensions __taskname__ = 'analyze' @@ -85,7 +86,7 @@ MIN_LINES = 4 # Minimum number of detected lines for consideration of bad guiding -# Return codes +# Return codes class Ret_code(Enum): """ Define return status codes for Operations @@ -304,7 +305,7 @@ def analyze_data(input_file_list, log_level=logutil.logging.DEBUG, type=""): log.setLevel(log_level) analyze_data_good_index = [] - + acs_filt_name_list = [DEFAULT_KEYS['FILKEY1'], DEFAULT_KEYS['FILKEY2']] # Interpret input filenames and adjust size of column accordingly @@ -425,13 +426,32 @@ def analyze_data(input_file_list, log_level=logutil.logging.DEBUG, type=""): # Determine if the image has one of these conditions. The routine # will exit processing upon the first satisfied condition. + # Check if all science image arrays in the RAW file are filled with zero values + non_zero_data_in_array = False # start assuming data is zeros + science_ext_ind_array = count_sci_extensions(input_file, return_ind=True) + # make sure science extension exists + if len(science_ext_ind_array)>0: + for sci_ext_ind in science_ext_ind_array: + science_data = getdata(input_file, sci_ext_ind) + # change flag if good data in any science extension array + if not np.all(science_data==0): + non_zero_data_in_array = True + else: + log.warning( + f"{input_file} (SCI, {sci_ext_ind}) is all zeros, but processing will continue with the other science extensions." + ) + + else: + log.warning(f'No science extension in file: {input_file}') + # Compute if the exposure time is very close to zero as it will be # needed when deciding whether or not to use the particular Grism/Prism data - is_zero = True if math.isclose(exptime, 0.0, abs_tol=1e-5) else False + exposure_time_near_zero = True if math.isclose(exptime, 0.0, abs_tol=1e-5) else False no_proc_key = None no_proc_value = None do_process = True + # Imaging vs spectroscopic or coronagraphic if obstype != 'IMAGING': no_proc_key = hdr_keys['OBSKEY'] @@ -494,7 +514,7 @@ def analyze_data(input_file_list, log_level=logutil.logging.DEBUG, type=""): split_sfilter = sfilter.upper().split('_') for item in split_sfilter: # This is the only circumstance when Grism/Prism data WILL be processed. - if item.startswith(('G', 'PR')) and not is_zero and type.upper() == "SVM": + if item.startswith(('G', 'PR')) and not exposure_time_near_zero and type.upper() == "SVM": no_proc_key = None no_proc_value = None log.info("The Grism/Prism data, {}, will be processed.".format(input_file)) @@ -503,10 +523,10 @@ def analyze_data(input_file_list, log_level=logutil.logging.DEBUG, type=""): if type.upper() == "MVM": no_proc_value += ", Grism/Prism data and MVM processing" log.warning("The Grism/Prism data {} with MVM processing will be ignored.".format(input_file)) - elif is_zero: + elif exposure_time_near_zero: no_proc_value += ", Grism/Prism data and EXPTIME = 0.0" log.warning("The Grism/Prism data {} with zero exposure time will be ignored.".format(input_file)) - + if item.startswith(('BLOCK')): no_proc_key = hdr_keys['FILKEY'] no_proc_value = sfilter @@ -532,7 +552,11 @@ def analyze_data(input_file_list, log_level=logutil.logging.DEBUG, type=""): # processing should be allowed, but there may be some issue with the result (e.g., # GYROS mode so some drift) generate_msg(input_file, msg_type, no_proc_key, no_proc_value) - + elif non_zero_data_in_array==False: + do_process=False + process_msg="SCI data all zeros" + log.warning(f'Science data for {input_file} filled with zeros. Dataset cannot be aligned.') + else: analyze_data_good_index.append(i) diff --git a/drizzlepac/util.py b/drizzlepac/util.py index f62b6e182..abdeaadb1 100644 --- a/drizzlepac/util.py +++ b/drizzlepac/util.py @@ -436,17 +436,38 @@ def updateNEXTENDKw(fobj): if 'nextend' in fobj[0].header: fobj[0].header['nextend'] = len(fobj) - 1 -def count_sci_extensions(filename): +def count_sci_extensions(filename, return_ind=False): """ Return the number of SCI extensions and the EXTNAME from a input MEF file. - """ + + Parameters + ---------- + filename : str + Filename of the file you would like to count the extensions of. + return_ind : bool, optional + Whether to return a list of the indices of the true "SCI" science extensions, + by default False. + + Returns + ------- + tuple + Science extension and number of extensions. + + or (if return_ind=True) + + list + indices of the "SCI" science extensions. + """ + num_sci = 0 + index=[] extname = 'SCI' hdu_list = fileutil.openImage(filename, memmap=False) - for extn in hdu_list: + for i, extn in enumerate(hdu_list): if 'extname' in extn.header and extn.header['extname'] == extname: num_sci += 1 + index.append(i) if num_sci == 0: extname = 'PRIMARY' @@ -454,7 +475,11 @@ def count_sci_extensions(filename): hdu_list.close() - return num_sci, extname + if return_ind: + return index + + else: + return num_sci, extname def verifyUniqueWcsname(fname, wcsname, include_primary=True):