diff --git a/docs/jwst/outlier_detection/arguments.rst b/docs/jwst/outlier_detection/arguments.rst deleted file mode 100644 index b2acf485d4..0000000000 --- a/docs/jwst/outlier_detection/arguments.rst +++ /dev/null @@ -1,114 +0,0 @@ -.. _outlier_detection_step_args: - -For more details about step arguments (including datatypes, possible values -and defaults) see :py:obj:`jwst.outlier_detection.OutlierDetectionStep.spec`. - -Step Arguments for Non-IFU data -=============================== -The `outlier_detection` step for non-IFU data has the following optional arguments -that control the behavior of the processing: - -``--weight_type`` - The type of data weighting to use during resampling. - -``--pixfrac`` - The pixel fraction used during resampling; - valid values go from 0.0 to 1.0. - -``--kernel`` - The form of the kernel function used to distribute flux onto a - resampled image. - -``--fillval`` - The value to assign to resampled image pixels that have zero weight or - do not receive any flux from any input pixels during drizzling. - Any floating-point value, given as a string, is valid. - A value of 'INDEF' will use the last zero weight flux. - -``--nlow`` - Deprecated and has no effect. This parameter will be removed - in a future version. - -``--nhigh`` - Deprecated and has no effect. This parameter will be removed - in a future version. - -``--maskpt`` - The percent of maximum weight to use as lower-limit for valid data; - valid values go from 0.0 to 1.0. - -``--snr`` - The signal-to-noise values to use for bad pixel identification. - Since cosmic rays often extend across several pixels the user - must specify two cut-off values for determining whether a pixel should - be masked: the first for detecting the primary cosmic ray, and the - second (typically lower threshold) for masking lower-level bad pixels - adjacent to those found in the first pass. Valid values are a pair of - floating-point values in a single string (for example "5.0 4.0"). - -``--scale`` - The scaling factor applied to derivative used to identify bad pixels. - Since cosmic rays often extend across several pixels the user - must specify two cut-off values for determining whether a pixel should - be masked: the first for detecting the primary cosmic ray, and the - second (typically lower threshold) for masking lower-level bad pixels - adjacent to those found in the first pass. Valid values are a pair of - floating-point values in a single string (for example "1.2 0.7"). - -``--backg`` - User-specified background value to apply to the median image. - -``--rolling_window_width`` - Number of integrations over which to take the median when using rolling-window - median for TSO observations. - -``--save_intermediate_results`` - Specifies whether or not to save any intermediate products created - during step processing. - -``--resample_data`` - Specifies whether or not to resample the input images when - performing outlier detection. - -``--good_bits`` - The DQ bit values from the input image DQ arrays - that should be considered 'good' when building the weight mask. See - DQ flag :ref:`dq_parameter_specification` for details. - -``--allowed_memory`` - Specifies the fractional amount of - free memory to allow when creating the resampled image. If ``None``, the - environment variable ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no - check is made. If the resampled image would be larger than specified, an - ``OutputTooLargeError`` exception will be generated. - - For example, if set to ``0.5``, only resampled images that use less than half - the available memory can be created. - -``--in_memory`` - Specifies whether or not to load and create all images that are used during - processing into memory. If ``False``, input files are loaded from disk when - needed and all intermediate files are stored on disk, rather than in memory. - -Step Arguments for IFU data -=========================== -The `outlier_detection` step for IFU data has the following optional arguments -that control the behavior of the processing: - -``--kernel_size`` - The size of the kernel to use to normalize the pixel differences. The kernel size - must only contain odd values. Valid values are a pair of ints in a single string - (for example "7 7"). - -``--threshold_percent`` - The threshold (in percent) of the normalized minimum pixel difference used to identify bad pixels. - Pixels with a normalized minimum pixel difference above this percentage are flagged as a outlier. - -``--save_intermediate_results`` - Specifies whether or not to save any intermediate products created - during step processing. - -``--in_memory`` - Specifies whether or not to load and create all images that are used during - processing into memory. If ``False``, input files are loaded from disk when - needed and all intermediate files are stored on disk, rather than in memory. diff --git a/docs/jwst/outlier_detection/index.rst b/docs/jwst/outlier_detection/index.rst index e78ff7c57f..2d70eed896 100644 --- a/docs/jwst/outlier_detection/index.rst +++ b/docs/jwst/outlier_detection/index.rst @@ -1,15 +1,9 @@ .. _outlier_detection_step: -================= -Outlier Detection -================= - .. toctree:: :maxdepth: 2 main.rst - arguments.rst - outlier_detection_step.rst outlier_detection_imaging.rst outlier_detection_coron.rst outlier_detection_tso.rst diff --git a/docs/jwst/outlier_detection/main.rst b/docs/jwst/outlier_detection/main.rst index 4e53369f27..7d929ed687 100644 --- a/docs/jwst/outlier_detection/main.rst +++ b/docs/jwst/outlier_detection/main.rst @@ -1,58 +1,16 @@ -Description -=========== +================= +Outlier Detection +================= -:Classes: `jwst.outlier_detection.OutlierDetectionStep` -:Aliases: outlier_detection +Outlier detection uses different algorithms based on the input data type, and so is handled +in separate steps, as follows: -Processing multiple datasets together allows for the identification of bad pixels -or cosmic-rays that remain in each of the input images, many times at levels which -were not detectable by the :ref:`jump ` step. The ``outlier_detection`` step -implements the following algorithm to identify and flag any remaining cosmic-rays or -other artifacts left over from previous calibrations: +#. **Image modes**: 'FGS_IMAGE', 'MIR_IMAGE', 'NRC_IMAGE', 'NIS_IMAGE' +#. **Spectroscopic modes**: 'MIR_LRS-FIXEDSLIT', 'NRS_FIXEDSLIT', 'NRS_MSASPEC' +#. **Time-Series-Observation(TSO) Spectroscopic modes**: 'MIR_LRS-SLITLESS', 'NRC_TSGRISM', 'NIS_SOSS', 'NRS_BRIGHTOBJ' +#. **IFU Spectroscopic modes**: 'MIR_MRS', 'NRS_IFU' +#. **TSO Image modes**: 'NRC_TSIMAGE' +#. **Coronagraphic Image modes**: 'MIR_LYOT', 'MIR_4QPM', 'NRC_CORON' -#. build a stack of input data - - - all inputs will need to have the same WCS since outlier detection assumes - the same flux for each point on the sky, and variations from one image to - the next would indicate a problem with the detector during readout of that pixel - - if needed, each input will be resampled to a common output WCS - -#. create a median image from the stack of input data - - this median operation will ignore any input pixels which have a weight - which is too low (<70% max weight) - -#. create "blotted" data from the median image to exactly match each original - input dataset - -#. perform a statistical comparison (pixel-by-pixel) between the median blotted - data with the original input data to look for pixels with values that are - different from the mean value by more than some specified sigma - based on the noise model - - - the noise model used relies on the error array computed by previous - calibration steps based on the readnoise and calibration errors - -#. flag the DQ array for the input data for any pixel (or affected neighboring - pixels) identified as a statistical outlier - -The outlier detection step serves as a single interface to apply this general -process to any JWST data, with specific variations of this algorithm for each -type of data. Sub-classes of the outlier detection algorithm have been developed -specifically for: - -#. Imaging data -#. IFU spectroscopic data -#. TSO data -#. coronagraphic data -#. spectroscopic data - -This allows the outlier_detection step to be tuned to the variations in each type -of JWST data. - -Reference Files -=============== - -The ``outlier_detection`` step uses the PARS-OUTLIERDETECTIONSTEP parameter reference file. - -.. include:: ../references_general/pars-outlierdetectionstep_reffile.inc +FIXME: add links to the docs for each of these \ No newline at end of file diff --git a/docs/jwst/outlier_detection/outlier_detection_coron.rst b/docs/jwst/outlier_detection/outlier_detection_coron.rst index 72cbd79b80..ba18bb827a 100644 --- a/docs/jwst/outlier_detection/outlier_detection_coron.rst +++ b/docs/jwst/outlier_detection/outlier_detection_coron.rst @@ -3,14 +3,15 @@ Outlier Detection for Coronagraphic Data ======================================== -This module serves as the interface for applying ``outlier_detection`` to coronagraphic +:Classes: `jwst.outlier_detection.OutlierDetectionCoronStep` +:Aliases: outlier_detection_coron + +This module serves as the interface for applying outlier detection to coronagraphic image observations. Specifically, this routine performs the following operations: #. Extract parameter settings from input model and merge them with any user-provided values. - See :ref:`outlier detection arguments ` for the full list - of parameters. #. Convert input data, as needed, to make sure it is in a format that can be processed. A :py:class:`~jwst.datamodels.CubeModel` serves as the basic format for all processing @@ -30,4 +31,33 @@ Specifically, this routine performs the following operations: #. Update input data model DQ arrays with mask of detected outliers. -.. automodapi:: jwst.outlier_detection.coron + +Step Arguments +============== + +``--maskpt`` + The percent of maximum weight to use as lower-limit for valid data; + valid values go from 0.0 to 1.0. + +``--snr`` + The signal-to-noise values to use for bad pixel identification. + Valid values are any positive float. + +``--save_intermediate_results`` + Specifies whether or not to save any intermediate products created + during step processing. + +``--good_bits`` + The DQ bit values from the input image DQ arrays + that should be considered 'good' when building the weight mask. See + DQ flag :ref:`dq_parameter_specification` for details. + + +Reference Files +=============== + +The ``outlier_detection_coron`` step uses the PARS-OUTLIERDETECTIONCORONSTEP parameter reference file. + +.. include:: ../references_general/pars-outlierdetectioncoronstep_reffile.inc + +.. automodapi:: jwst.outlier_detection.outlier_detection_coron_step diff --git a/docs/jwst/outlier_detection/outlier_detection_ifu.rst b/docs/jwst/outlier_detection/outlier_detection_ifu.rst index f8a4c4d66e..ace1116880 100644 --- a/docs/jwst/outlier_detection/outlier_detection_ifu.rst +++ b/docs/jwst/outlier_detection/outlier_detection_ifu.rst @@ -3,6 +3,9 @@ Outlier Detection for IFU Data ============================== +:Classes: `jwst.outlier_detection.OutlierDetectionIFUStep` +:Aliases: outlier_detection_ifu + This module serves as the interface for applying ``outlier_detection`` to IFU observations, like those taken with NIRSpec and MIRI. The code implements the basic outlier detection algorithm searching for pixels that are consistent outliers @@ -30,5 +33,31 @@ through this stack is determined and normalized. The normalization uses a local is greater than the input threshold percentage. Pixels that are found to be outliers are flaged in in the DQ array. +Step Arguments +============== + +``--kernel_size`` + The size of the kernel to use to normalize the pixel differences. The kernel size + must only contain odd values. Valid values are a pair of ints in a single string + (for example "7 7"). + +``--threshold_percent`` + The threshold (in percent) of the normalized minimum pixel difference used to identify bad pixels. + Pixels with a normalized minimum pixel difference above this percentage are flagged as a outlier. + +``--ifu_second_check`` + FIXME: WHAT IS THIS? + +``--save_intermediate_results`` + Specifies whether or not to save any intermediate products created + during step processing. + + +Reference Files +=============== + +The ``outlier_detection_ifu`` step uses the PARS-OUTLIERDETECTIONIFUSTEP parameter reference file. + +.. include:: ../references_general/pars-outlierdetectionifustep_reffile.inc -.. automodapi:: jwst.outlier_detection.ifu +.. automodapi:: jwst.outlier_detection.outlier_detection_ifu_step diff --git a/docs/jwst/outlier_detection/outlier_detection_imaging.rst b/docs/jwst/outlier_detection/outlier_detection_imaging.rst index ced7d9599f..a0ea2bd1a8 100644 --- a/docs/jwst/outlier_detection/outlier_detection_imaging.rst +++ b/docs/jwst/outlier_detection/outlier_detection_imaging.rst @@ -3,15 +3,16 @@ Outlier Detection Algorithm for Imaging Data ============================================ -This module serves as the interface for applying ``outlier_detection`` to direct +:Classes: `jwst.outlier_detection.OutlierDetectionImagingStep` +:Aliases: outlier_detection_imaging + +This module serves as the interface for applying outlier detection to direct image observations, like those taken with MIRI, NIRCam and NIRISS. The code implements the basic outlier detection algorithm used with HST data, as adapted to JWST. Specifically, this routine performs the following operations: #. Extract parameter settings from input model and merge them with any user-provided values. - See :ref:`outlier detection arguments ` for the full list - of parameters. #. Convert input data, as needed, to make sure it is in a format that can be processed. @@ -152,12 +153,6 @@ memory usage at the expense of file I/O. The control over this memory model hap with the use of the ``in_memory`` parameter. The full impact of this parameter during processing includes: -#. The ``save_open`` parameter gets set to `False` - when opening the input :py:class:`~jwst.datamodels.ModelContainer` object. - This forces all input models in the input :py:class:`~jwst.datamodels.ModelContainer` or - :py:class:`~jwst.datamodels.CubeModel` to get written out to disk. The ModelContainer - then uses the filename of the input model during subsequent processing. - #. The ``in_memory`` parameter gets passed to the :py:class:`~jwst.resample.ResampleStep` to set whether or not to keep the resampled images in memory or not. By default, the outlier detection processing sets this parameter to `False` so that each resampled @@ -174,34 +169,85 @@ These changes result in a minimum amount of memory usage during processing at th expense of reading and writing the products from disk. -Outlier Detection for Coronagraphic Data ----------------------------------------- -Coronagraphic data is processed in a near-identical manner to direct imaging data, but -no resampling occurs. +Step Arguments +============== + +``--weight_type`` + The type of data weighting to use during resampling. + +``--pixfrac`` + The pixel fraction used during resampling; + valid values go from 0.0 to 1.0. + +``--kernel`` + The form of the kernel function used to distribute flux onto a + resampled image. + +``--fillval`` + The value to assign to resampled image pixels that have zero weight or + do not receive any flux from any input pixels during drizzling. + Any floating-point value, given as a string, is valid. + A value of 'INDEF' will use the last zero weight flux. + +``--maskpt`` + The percent of maximum weight to use as lower-limit for valid data; + valid values go from 0.0 to 1.0. + +``--snr`` + The signal-to-noise values to use for bad pixel identification. + Since cosmic rays often extend across several pixels the user + must specify two cut-off values for determining whether a pixel should + be masked: the first for detecting the primary cosmic ray, and the + second (typically lower threshold) for masking lower-level bad pixels + adjacent to those found in the first pass. Valid values are a pair of + floating-point values in a single string (for example "5.0 4.0"). + +``--scale`` + The scaling factor applied to derivative used to identify bad pixels. + Since cosmic rays often extend across several pixels the user + must specify two cut-off values for determining whether a pixel should + be masked: the first for detecting the primary cosmic ray, and the + second (typically lower threshold) for masking lower-level bad pixels + adjacent to those found in the first pass. Valid values are a pair of + floating-point values in a single string (for example "1.2 0.7"). + +``--backg`` + User-specified background value to apply to the median image. + +``--save_intermediate_results`` + Specifies whether or not to save any intermediate products created + during step processing. + +``--resample_data`` + Specifies whether or not to resample the input images when + performing outlier detection. + +``--good_bits`` + The DQ bit values from the input image DQ arrays + that should be considered 'good' when building the weight mask. See + DQ flag :ref:`dq_parameter_specification` for details. + +``--allowed_memory`` + Specifies the fractional amount of + free memory to allow when creating the resampled image. If ``None``, the + environment variable ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no + check is made. If the resampled image would be larger than specified, an + ``OutputTooLargeError`` exception will be generated. + For example, if set to ``0.5``, only resampled images that use less than half + the available memory can be created. -Outlier Detection for TSO data -------------------------------- -Normal imaging data benefit from combining all integrations into a -single image. TSO data's value, however, comes from looking for variations from one -integration to the next. The outlier detection algorithm, therefore, gets run with -a few variations to accomodate the nature of these 3D data. See the -:ref:`TSO outlier detection ` documentation for details. +``--in_memory`` + Specifies whether or not to load and create all images that are used during + processing into memory. If ``False``, input files are loaded from disk when + needed and all intermediate files are stored on disk, rather than in memory. -Outlier Detection for IFU data ------------------------------- -Integral Field Unit (IFU) data is handled as 2D images, similar to direct -imaging modes. The nature of the detection algorithm, however, is quite -different and involves measuring the differences between neighboring pixels -in the spatial (cross-dispersion) direction within the IFU slice images. -See the :ref:`IFU outlier detection ` documentation for -all the details. +Reference Files +=============== +The ``outlier_detection_imaging`` step uses the PARS-OUTLIERDETECTIONIMAGINGSTEP parameter reference file. -Outlier Detection for Slit data -------------------------------- -See the :ref:`IFU outlier detection ` documentation for -details. +.. include:: ../references_general/pars-outlierdetectionimagingstep_reffile.inc -.. automodapi:: jwst.outlier_detection.imaging +.. automodapi:: jwst.outlier_detection.outlier_detection_imaging_step diff --git a/docs/jwst/outlier_detection/outlier_detection_spec.rst b/docs/jwst/outlier_detection/outlier_detection_spec.rst index 6d8cd13de5..8674277827 100644 --- a/docs/jwst/outlier_detection/outlier_detection_spec.rst +++ b/docs/jwst/outlier_detection/outlier_detection_spec.rst @@ -3,6 +3,9 @@ Outlier Detection for Slit-like Spectroscopic Data ================================================== +:Classes: `jwst.outlier_detection.OutlierDetectionSpecStep` +:Aliases: outlier_detection_spec + This module serves as the interface for applying ``outlier_detection`` to slit-like spectroscopic observations. The code implements the basic outlier detection algorithm used with HST data, as adapted to JWST @@ -45,4 +48,75 @@ Specifically, this routine performs the following operations (modified from the #. Update input data model DQ arrays with mask of detected outliers -.. automodapi:: jwst.outlier_detection.spec +Step Arguments +============== + +``--weight_type`` + The type of data weighting to use during resampling. + +``--pixfrac`` + The pixel fraction used during resampling; + valid values go from 0.0 to 1.0. + +``--kernel`` + The form of the kernel function used to distribute flux onto a + resampled image. + +``--fillval`` + The value to assign to resampled image pixels that have zero weight or + do not receive any flux from any input pixels during drizzling. + Any floating-point value, given as a string, is valid. + A value of 'INDEF' will use the last zero weight flux. + +``--maskpt`` + The percent of maximum weight to use as lower-limit for valid data; + valid values go from 0.0 to 1.0. + +``--snr`` + The signal-to-noise values to use for bad pixel identification. + Since cosmic rays often extend across several pixels the user + must specify two cut-off values for determining whether a pixel should + be masked: the first for detecting the primary cosmic ray, and the + second (typically lower threshold) for masking lower-level bad pixels + adjacent to those found in the first pass. Valid values are a pair of + floating-point values in a single string (for example "5.0 4.0"). + +``--scale`` + The scaling factor applied to derivative used to identify bad pixels. + Since cosmic rays often extend across several pixels the user + must specify two cut-off values for determining whether a pixel should + be masked: the first for detecting the primary cosmic ray, and the + second (typically lower threshold) for masking lower-level bad pixels + adjacent to those found in the first pass. Valid values are a pair of + floating-point values in a single string (for example "1.2 0.7"). + +``--backg`` + User-specified background value to apply to the median image. + +``--save_intermediate_results`` + Specifies whether or not to save any intermediate products created + during step processing. + +``--resample_data`` + Specifies whether or not to resample the input images when + performing outlier detection. + +``--good_bits`` + The DQ bit values from the input image DQ arrays + that should be considered 'good' when building the weight mask. See + DQ flag :ref:`dq_parameter_specification` for details. + +``--in_memory`` + Specifies whether or not to load and create all images that are used during + processing into memory. If ``False``, input files are loaded from disk when + needed and all intermediate files are stored on disk, rather than in memory. + + +Reference Files +=============== + +The ``outlier_detection_spec`` step uses the PARS-OUTLIERDETECTIONSPECSTEP parameter reference file. + +.. include:: ../references_general/pars-outlierdetectionspecstep_reffile.inc + +.. automodapi:: jwst.outlier_detection.outlier_detection_spec_step diff --git a/docs/jwst/outlier_detection/outlier_detection_step.rst b/docs/jwst/outlier_detection/outlier_detection_step.rst deleted file mode 100644 index 0713147a31..0000000000 --- a/docs/jwst/outlier_detection/outlier_detection_step.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. _outlier_design: - -Python Step Design: OutlierDetectionStep ------------------------------------------ - -This module provides the sole interface to all methods of performing outlier -detection on JWST observations. The ``outlier_detection`` step supports multiple -algorithms and determines the appropriate algorithm for the type of observation -being processed. This step supports: - -#. **Image modes**: 'FGS_IMAGE', 'MIR_IMAGE', 'NRC_IMAGE', 'NIS_IMAGE' -#. **Spectroscopic modes**: 'MIR_LRS-FIXEDSLIT', 'NRS_FIXEDSLIT', 'NRS_MSASPEC' -#. **Time-Series-Observation(TSO) Spectroscopic modes**: 'MIR_LRS-SLITLESS', 'NRC_TSGRISM', 'NIS_SOSS', 'NRS_BRIGHTOBJ' -#. **IFU Spectroscopic modes**: 'MIR_MRS', 'NRS_IFU' -#. **TSO Image modes**: 'NRC_TSIMAGE' -#. **Coronagraphic Image modes**: 'MIR_LYOT', 'MIR_4QPM', 'NRC_CORON' - - -This step uses the following logic to apply the appropriate algorithm to the -input data: - -#. Interpret inputs (ASN table, ModelContainer or CubeModel) - to identify all input observations to be processed - -#. Read in type of exposures in input by interpreting ``meta.exposure.type`` from inputs - -#. Read in parameters set by user - -#. Select outlier detection algorithm based on exposure type - - - **Images**: like those taken with NIRCam, will use - :py:mod:`~jwst.outlier_detection.outlier_detection.OutlierDetection` as described - in :ref:`outlier-detection-imaging` - - **Coronagraphic observations**: - use :py:mod:`~jwst.outlier_detection.coron` - as described in :ref:`outlier-detection-coron` - - **Time-Series Observations(TSO)**: both imaging and spectroscopic modes, use - :py:mod:`~jwst.outlier_detection.tso` - as described in :ref:`outlier-detection-tso` - - **IFU observations**: use - :py:mod:`~jwst.outlier_detection.ifu` as - described in :ref:`outlier-detection-ifu` - - **Long-slit spectroscopic observations**: use - :py:mod:`~jwst.outlier_detection.spec` as - described in :ref:`outlier-detection-spec` - -#. Instantiate and run outlier detection class determined for the exposure type - using parameter values interpreted from inputs. - -#. Return input models with DQ arrays updated with flags for identified outliers - - -.. automodapi:: jwst.outlier_detection.outlier_detection_step diff --git a/docs/jwst/outlier_detection/outlier_detection_tso.rst b/docs/jwst/outlier_detection/outlier_detection_tso.rst index 843ad95941..f2235ad241 100644 --- a/docs/jwst/outlier_detection/outlier_detection_tso.rst +++ b/docs/jwst/outlier_detection/outlier_detection_tso.rst @@ -3,6 +3,9 @@ Outlier Detection for TSO Data ============================== +:Classes: `jwst.outlier_detection.OutlierDetectionTSOStep` +:Aliases: outlier_detection_tso + Time-series observations (TSO) result in input data stored as a 3D CubeModel where each plane in the cube represents a separate integration without changing the pointing. Normal imaging data benefit from combining all integrations into a @@ -36,4 +39,37 @@ a few variations to accomodate the nature of these 3D data. #. The input data model DQ arrays are updated with the mask of detected outliers. -.. automodapi:: jwst.outlier_detection.tso + +Step Arguments +============== + +``--maskpt`` + The percent of maximum weight to use as lower-limit for valid data; + valid values go from 0.0 to 1.0. + +``--snr`` + The signal-to-noise values to use for bad pixel identification. + Valid values are any positive float. + +``--rolling_window_width`` + Number of integrations over which to take the median when using rolling-window + median for TSO observations. + +``--save_intermediate_results`` + Specifies whether or not to save any intermediate products created + during step processing. + +``--good_bits`` + The DQ bit values from the input image DQ arrays + that should be considered 'good' when building the weight mask. See + DQ flag :ref:`dq_parameter_specification` for details. + + +Reference Files +=============== + +The ``outlier_detection_tso`` step uses the PARS-OUTLIERDETECTIONTSOSTEP parameter reference file. + +.. include:: ../references_general/pars-outlierdetectiontsostep_reffile.inc + +.. automodapi:: jwst.outlier_detection.outlier_detection_tso_step diff --git a/docs/jwst/references_general/pars-outlierdetectioncoronstep_reffile.inc b/docs/jwst/references_general/pars-outlierdetectioncoronstep_reffile.inc new file mode 100644 index 0000000000..815f319dc0 --- /dev/null +++ b/docs/jwst/references_general/pars-outlierdetectioncoronstep_reffile.inc @@ -0,0 +1,22 @@ +.. _pars_outlierdetectioncoronstep_reffile: + +PARS-OUTLIERDETECTIONCORONSTEP Parameter Reference File +------------------------------------------------------- + +:REFTYPE: PARS-OUTLIERDETECTIONCORONSTEP +:Data model: N/A + +Reference Selection Keywords +++++++++++++++++++++++++++++ +CRDS selects appropriate pars-outlierdetectionstep references based on the following keywords. + +========== ==================================== +Instrument Keywords +========== ==================================== +FGS EXP_TYPE +MIRI EXP_TYPE, FILTER, SUBARRAY, TSOVISIT +NIRCAM EXP_TYPE, FILTER, PUPIL, TSOVISIT +NIRISS EXP_TYPE, FILTER, PUPIL, TSOVISIT +========== ==================================== + +.. include:: ../includes/standard_keywords.inc diff --git a/docs/jwst/references_general/pars-outlierdetectionstep_reffile.inc b/docs/jwst/references_general/pars-outlierdetectionifustep_reffile.inc similarity index 74% rename from docs/jwst/references_general/pars-outlierdetectionstep_reffile.inc rename to docs/jwst/references_general/pars-outlierdetectionifustep_reffile.inc index 65c93a7d53..9292006162 100644 --- a/docs/jwst/references_general/pars-outlierdetectionstep_reffile.inc +++ b/docs/jwst/references_general/pars-outlierdetectionifustep_reffile.inc @@ -1,9 +1,9 @@ -.. _pars_outlierdetectionstep_reffile: +.. _pars_outlierdetectionifustep_reffile: -PARS-OUTLIERDETECTIONSTEP Parameter Reference File --------------------------------------------------- +PARS-OUTLIERDETECTIONIFUSTEP Parameter Reference File +----------------------------------------------------- -:REFTYPE: PARS-OUTLIERDETECTIONSTEP +:REFTYPE: PARS-OUTLIERDETECTIONIFUSTEP :Data model: N/A Reference Selection Keywords diff --git a/docs/jwst/references_general/pars-outlierdetectionimagingstep_reffile.inc b/docs/jwst/references_general/pars-outlierdetectionimagingstep_reffile.inc new file mode 100644 index 0000000000..25bf8beace --- /dev/null +++ b/docs/jwst/references_general/pars-outlierdetectionimagingstep_reffile.inc @@ -0,0 +1,22 @@ +.. _pars_outlierdetectionimagingstep_reffile: + +PARS-OUTLIERDETECTIONIMAGINGSTEP Parameter Reference File +--------------------------------------------------------- + +:REFTYPE: PARS-OUTLIERDETECTIONIMAGINGSTEP +:Data model: N/A + +Reference Selection Keywords +++++++++++++++++++++++++++++ +CRDS selects appropriate pars-outlierdetectionstep references based on the following keywords. + +========== ==================================== +Instrument Keywords +========== ==================================== +FGS EXP_TYPE +MIRI EXP_TYPE, FILTER, SUBARRAY, TSOVISIT +NIRCAM EXP_TYPE, FILTER, PUPIL, TSOVISIT +NIRISS EXP_TYPE, FILTER, PUPIL, TSOVISIT +========== ==================================== + +.. include:: ../includes/standard_keywords.inc diff --git a/docs/jwst/references_general/pars-outlierdetectionspecstep_reffile.inc b/docs/jwst/references_general/pars-outlierdetectionspecstep_reffile.inc new file mode 100644 index 0000000000..95f849a4bf --- /dev/null +++ b/docs/jwst/references_general/pars-outlierdetectionspecstep_reffile.inc @@ -0,0 +1,22 @@ +.. _pars_outlierdetectionspecstep_reffile: + +PARS-OUTLIERDETECTIONSPECSTEP Parameter Reference File +------------------------------------------------------ + +:REFTYPE: PARS-OUTLIERDETECTIONSPECSTEP +:Data model: N/A + +Reference Selection Keywords +++++++++++++++++++++++++++++ +CRDS selects appropriate pars-outlierdetectionstep references based on the following keywords. + +========== ==================================== +Instrument Keywords +========== ==================================== +FGS EXP_TYPE +MIRI EXP_TYPE, FILTER, SUBARRAY, TSOVISIT +NIRCAM EXP_TYPE, FILTER, PUPIL, TSOVISIT +NIRISS EXP_TYPE, FILTER, PUPIL, TSOVISIT +========== ==================================== + +.. include:: ../includes/standard_keywords.inc diff --git a/docs/jwst/references_general/pars-outlierdetectiontsostep_reffile.inc b/docs/jwst/references_general/pars-outlierdetectiontsostep_reffile.inc new file mode 100644 index 0000000000..b1efa53961 --- /dev/null +++ b/docs/jwst/references_general/pars-outlierdetectiontsostep_reffile.inc @@ -0,0 +1,22 @@ +.. _pars_outlierdetectiontsostep_reffile: + +PARS-OUTLIERDETECTIONTSOSTEP Parameter Reference File +----------------------------------------------------- + +:REFTYPE: PARS-OUTLIERDETECTIONTSOSTEP +:Data model: N/A + +Reference Selection Keywords +++++++++++++++++++++++++++++ +CRDS selects appropriate pars-outlierdetectionstep references based on the following keywords. + +========== ==================================== +Instrument Keywords +========== ==================================== +FGS EXP_TYPE +MIRI EXP_TYPE, FILTER, SUBARRAY, TSOVISIT +NIRCAM EXP_TYPE, FILTER, PUPIL, TSOVISIT +NIRISS EXP_TYPE, FILTER, PUPIL, TSOVISIT +========== ==================================== + +.. include:: ../includes/standard_keywords.inc diff --git a/jwst/outlier_detection/__init__.py b/jwst/outlier_detection/__init__.py index c012c72cbf..42c6635247 100644 --- a/jwst/outlier_detection/__init__.py +++ b/jwst/outlier_detection/__init__.py @@ -1,3 +1,11 @@ -from .outlier_detection_step import OutlierDetectionStep +from .outlier_detection_coron_step import OutlierDetectionCoronStep +from .outlier_detection_ifu_step import OutlierDetectionIFUStep +from .outlier_detection_spec_step import OutlierDetectionSpecStep +from .outlier_detection_imaging_step import OutlierDetectionImagingStep +from .outlier_detection_tso_step import OutlierDetectionTSOStep -__all__ = ['OutlierDetectionStep'] +__all__ = ['OutlierDetectionCoronStep', + 'OutlierDetectionIFUStep', + 'OutlierDetectionSpecStep', + 'OutlierDetectionImagingStep', + 'OutlierDetectionTSOStep'] diff --git a/jwst/outlier_detection/coron.py b/jwst/outlier_detection/coron.py deleted file mode 100644 index 500ad49602..0000000000 --- a/jwst/outlier_detection/coron.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Submodule for performing outlier detection on coronagraphy data. -""" - - -import logging - -import numpy as np - -from stdatamodels.jwst import datamodels - -from jwst.resample.resample_utils import build_mask - -from .utils import create_cube_median, flag_model_crs -from ._fileio import save_median - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - - -__all__ = ["detect_outliers"] - - -def detect_outliers( - input_model, - save_intermediate_results, - good_bits, - maskpt, - snr, - asn_id, - make_output_path, -): - """ - Flag outliers in coronography data. - - See `OutlierDetectionStep.spec` for documentation of these arguments. - """ - if not isinstance(input_model, datamodels.JwstDataModel): - input_model = datamodels.open(input_model) - - if not isinstance(input_model, datamodels.CubeModel): - raise TypeError(f"Input must be a CubeModel: {input_model}") - - # FIXME weight_type could now be used here. Similar to tso data coron - # data was previously losing var_rnoise due to the conversion from a cube - # to a ModelContainer (which makes the default ivm weight ignore var_rnoise). - # Now that it's handled as a cube we could use the var_rnoise. - input_model.wht = build_mask(input_model.dq, good_bits).astype(np.float32) - - # Perform median combination on set of drizzled mosaics - median_data = create_cube_median(input_model, maskpt) - - if save_intermediate_results: - # make a median model - median_model = datamodels.ImageModel(median_data) - median_model.update(input_model) - median_model.meta.wcs = input_model.meta.wcs - - save_median(median_model, make_output_path, asn_id) - del median_model - - # Perform outlier detection using statistical comparisons between - # each original input image and its blotted version of the median image - flag_model_crs( - input_model, - median_data, - snr, - ) - return input_model diff --git a/jwst/outlier_detection/imaging.py b/jwst/outlier_detection/imaging.py deleted file mode 100644 index c6ba99da13..0000000000 --- a/jwst/outlier_detection/imaging.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -Submodule for performing outlier detection on imaging data. -""" - -import copy -import logging -import os - -from stdatamodels.jwst import datamodels - -from jwst.datamodels import ModelContainer -from jwst.resample import resample -from jwst.resample.resample_utils import build_driz_weight -from jwst.stpipe.utilities import record_step_status - -from .utils import create_median, flag_crs_in_models, flag_crs_in_models_with_resampling -from ._fileio import remove_file, save_median - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - - -__all__ = ["detect_outliers"] - - -def detect_outliers( - input_models, - save_intermediate_results, - good_bits, - maskpt, - snr1, - snr2, - scale1, - scale2, - backg, - resample_data, - weight_type, - pixfrac, - kernel, - fillval, - allowed_memory, - in_memory, - asn_id, - make_output_path, -): - """ - Flag outliers in imaging data. - - See `OutlierDetectionStep.spec` for documentation of these arguments. - """ - if not isinstance(input_models, ModelContainer): - input_models = ModelContainer(input_models, save_open=in_memory) - - if len(input_models) < 2: - log.warning(f"Input only contains {len(input_models)} exposures") - log.warning("Outlier detection will be skipped") - record_step_status(input_models, "outlier_detection", False) - return input_models - - if resample_data: - # Start by creating resampled/mosaic images for - # each group of exposures - output_path = make_output_path(basepath=input_models[0].meta.filename, - suffix='') - output_path = os.path.dirname(output_path) - resamp = resample.ResampleData( - input_models, - output=output_path, - single=True, - blendheaders=False, - wht_type=weight_type, - pixfrac=pixfrac, - kernel=kernel, - fillval=fillval, - good_bits=good_bits, - in_memory=in_memory, - asn_id=asn_id, - allowed_memory=allowed_memory, - ) - median_wcs = resamp.output_wcs - drizzled_models = resamp.do_drizzle(input_models) - else: - # for non-dithered data, the resampled image is just the original image - drizzled_models = input_models - for i in range(len(input_models)): - drizzled_models[i].wht = build_driz_weight( - input_models[i], - weight_type=weight_type, - good_bits=good_bits) - # copy for when saving median and input is a filename? - median_wcs = copy.deepcopy(input_models[0].meta.wcs) - - # Perform median combination on set of drizzled mosaics - median_data = create_median(drizzled_models, maskpt) - - if save_intermediate_results: - # make a median model - with datamodels.open(drizzled_models[0]) as dm0: - median_model = datamodels.ImageModel(median_data) - median_model.update(dm0) - median_model.meta.wcs = median_wcs - - save_median(median_model, make_output_path, asn_id) - del median_model - else: - # since we're not saving intermediate results if the drizzled models - # were written to disk, remove them - if not in_memory: - for fn in drizzled_models._models: - remove_file(fn) - - # Perform outlier detection using statistical comparisons between - # each original input image and its blotted version of the median image - if resample_data: - flag_crs_in_models_with_resampling( - input_models, - median_data, - median_wcs, - snr1, - snr2, - scale1, - scale2, - backg, - ) - else: - flag_crs_in_models(input_models, median_data, snr1) - return input_models diff --git a/jwst/outlier_detection/outlier_detection_coron_step.py b/jwst/outlier_detection/outlier_detection_coron_step.py new file mode 100644 index 0000000000..940236a5f8 --- /dev/null +++ b/jwst/outlier_detection/outlier_detection_coron_step.py @@ -0,0 +1,86 @@ +""" +Submodule for performing outlier detection on coronagraphy data. +""" + + +import logging + +import numpy as np + +from stdatamodels.jwst import datamodels +from jwst.stpipe import Step + +from jwst.resample.resample_utils import build_mask + +from .utils import create_cube_median, flag_model_crs, OutlierDetectionStepBase +from ._fileio import save_median + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + + +__all__ = ["OutlierDetectionCoronStep"] + + +class OutlierDetectionCoronStep(Step, OutlierDetectionStepBase): + """Flag outlier bad pixels and cosmic rays in DQ array of each input image. + + Input images can be listed in an input association file or already opened + with a ModelContainer. DQ arrays are modified in place. + + Parameters + ----------- + input_model : ~jwst.datamodels.CubeModel + CubeModel or filename pointing to a CubeModel + + """ + + class_alias = "outlier_detection_coron" + + spec = """ + maskpt = float(default=0.7) + snr = float(default=5.0) + save_intermediate_results = boolean(default=False) + good_bits = string(default="~DO_NOT_USE") # DQ flags to allow + """ + + def process(self, input_model): + """Perform outlier detection processing on input data.""" + + # determine the asn_id (if not set by the pipeline) + asn_id = self._get_asn_id(input_model) + self.log.info(f"Outlier Detection asn_id: {asn_id}") + + + if not isinstance(input_model, datamodels.JwstDataModel): + input_model = datamodels.open(input_model) + + if not isinstance(input_model, datamodels.CubeModel): + raise TypeError(f"Input must be a CubeModel: {input_model}") + + # FIXME weight_type could now be used here. Similar to tso data coron + # data was previously losing var_rnoise due to the conversion from a cube + # to a ModelContainer (which makes the default ivm weight ignore var_rnoise). + # Now that it's handled as a cube we could use the var_rnoise. + input_model.wht = build_mask(input_model.dq, self.good_bits).astype(np.float32) + + # Perform median combination on set of drizzled mosaics + median_data = create_cube_median(input_model, self.maskpt) + + if self.save_intermediate_results: + # make a median model + median_model = datamodels.ImageModel(median_data) + median_model.update(input_model) + median_model.meta.wcs = input_model.meta.wcs + + save_median(median_model, self.make_output_path, asn_id) + del median_model + + # Perform outlier detection using statistical comparisons between + # each original input image and its blotted version of the median image + flag_model_crs( + input_model, + median_data, + self.snr, + ) + return input_model diff --git a/jwst/outlier_detection/ifu.py b/jwst/outlier_detection/outlier_detection_ifu_step.py similarity index 68% rename from jwst/outlier_detection/ifu.py rename to jwst/outlier_detection/outlier_detection_ifu_step.py index dfd7c7f6ed..93318ff155 100644 --- a/jwst/outlier_detection/ifu.py +++ b/jwst/outlier_detection/outlier_detection_ifu_step.py @@ -1,30 +1,5 @@ """ -Submodule defined for performing outlier detection on IFU data. - -This is the controlling routine for the outlier detection process. -It loads and sets the various input data and parameters needed to flag -outliers. Pixel are flagged as outliers based on the MINIMUM difference -a pixel has with its neighbor across all the input cal files. - -Notes ------ -This routine performs the following operations:: - - 1. Extracts parameter settings from input ModelContainer and merges - them with any user-provided values - 2. Loop over cal files - a. read in science data - b. Store computed neighbor differences for all the pixels. - The neighbor pixel differences are defined by the dispersion axis. - For MIRI, with the dispersion axis along the y axis, the neighbors that are used to - to find the differences are to the left and right of each pixel being examined. - For NIRSpec, with the dispersion along the x axis, the neighbors that are used to - find the differences are above and below the pixel being examined. - 3. For each input file store the minimum of the pixel neighbor differences - 4. Comparing all the differences from all the input data find the minimum neighbor difference - 5. Normalize minimum difference to local median of difference array - 6. select outliers by flagging those normailzed minimum values > threshold_percent - 7. Updates input ImageModel DQ arrays with mask of detected outliers. +Module defined for performing outlier detection on IFU data. """ import logging @@ -33,77 +8,111 @@ from jwst.datamodels import ModelContainer from jwst.stpipe.utilities import record_step_status +from jwst.stpipe import Step from stdatamodels.jwst import datamodels from stdatamodels.jwst.datamodels import dqflags from stcal.outlier_detection.utils import medfilt +from .utils import OutlierDetectionStepBase log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) -__all__ = ["detect_outliers"] +__all__ = ["OutlierDetectionIFUStep"] -def detect_outliers( - input_models, - save_intermediate_results, - kernel_size, - ifu_second_check, - threshold_percent, - make_output_path, -): +class OutlierDetectionIFUStep(Step, OutlierDetectionStepBase): + """Flag outlier bad pixels and cosmic rays in DQ array of each input image. + + Input images can be listed in an input association file or already opened + with a ModelContainer. DQ arrays are modified in place. + + Parameters + ----------- + input_models : asn file or ~jwst.datamodels.ModelContainer + Single filename association table, or a datamodels.ModelContainer. + + Notes + ----- + This routine performs the following operations. + + 1. Extracts parameter settings from input ModelContainer and merges + them with any user-provided values + 2. Loop over cal files + a. read in science data + b. Store computed neighbor differences for all the pixels. + The neighbor pixel differences are defined by the dispersion axis. + For MIRI, with the dispersion axis along the y axis, the neighbors that are used to + to find the differences are to the left and right of each pixel being examined. + For NIRSpec, with the dispersion along the x axis, the neighbors that are used to + find the differences are above and below the pixel being examined. + 3. For each input file store the minimum of the pixel neighbor differences + 4. Comparing all the differences from all the input data find the minimum neighbor difference + 5. Normalize minimum difference to local median of difference array + 6. select outliers by flagging those normailzed minimum values > threshold_percent + 7. Updates input ImageModel DQ arrays with mask of detected outliers. """ - Flag outliers in ifu data. - See `OutlierDetectionStep.spec` for documentation of these arguments. + class_alias = "outlier_detection_ifu" + + spec = """ + kernel_size = string(default='7 7') + threshold_percent = float(default=99.8) + ifu_second_check = boolean(default=False) + save_intermediate_results = boolean(default=False) """ - if not isinstance(input_models, ModelContainer): - input_models = ModelContainer(input_models) - if len(input_models) < 2: - log.warning(f"Input only contains {len(input_models)} exposures") - log.warning("Outlier detection will be skipped") - record_step_status(input_models, "outlier_detection", False) + def process(self, input_models): + """Perform outlier detection processing on input data.""" + + if not isinstance(input_models, ModelContainer): + input_models = ModelContainer(input_models) + + if len(input_models) < 2: + log.warning(f"Input only contains {len(input_models)} exposures") + log.warning("Outlier detection will be skipped") + record_step_status(input_models, "outlier_detection", False) + return input_models + + sizex, sizey = [int(val) for val in self.kernel_size.split()] + kern_size = np.zeros(2, dtype=int) + kern_size[0] = sizex + kern_size[1] = sizey + + # check if kernel size is an odd value + if kern_size[0] % 2 == 0: + log.info("X kernel size is given as an even number. This value must be an odd number. Increasing number by 1") + kern_size[0] = kern_size[0] + 1 + log.info("New x kernel size is {}: ".format(kern_size[0])) + if kern_size[1] % 2 == 0: + log.info("Y kernel size is given as an even number. This value must be an odd number. Increasing number by 1") + kern_size[1] = kern_size[1] + 1 + log.info("New y kernel size is {}: ".format(kern_size[1])) + + (diffaxis, ny, nx) = _find_detector_parameters(input_models) + + nfiles = len(input_models) + detector = np.empty(nfiles, dtype=' 1) and (rolling_window_width < weighted_cube.shape[0]): - medians = compute_rolling_median(weighted_cube, weight_threshold, w=rolling_window_width) - - else: - medians = np.nanmedian(weighted_cube.data, axis=0) - # this is a 2-D array, need to repeat it into the time axis - # for consistent shape with rolling median case - medians = np.broadcast_to(medians, weighted_cube.shape) - - # Save median model if pars['save_intermediate_results'] is True - # this will be a CubeModel with rolling median values. - if save_intermediate_results: - median_model = dm.CubeModel(data=medians) - with dm.open(weighted_cube) as dm0: - median_model.update(dm0) - save_median(median_model, make_output_path, asn_id) - del median_model - - # no need for blotting, resample is turned off for TSO - # go straight to outlier detection - log.info("Flagging outliers") - flag_model_crs( - input_model, - medians, - snr, - ) - return input_model + + def process(self, input_model): + """Perform outlier detection processing on input data.""" + + # determine the asn_id (if not set by the pipeline) + asn_id = self._get_asn_id(input_model) + self.log.info(f"Outlier Detection asn_id: {asn_id}") + + if not isinstance(input_model, dm.JwstDataModel): + input_model = dm.open(input_model) + if isinstance(input_model, dm.ModelContainer): + raise TypeError("OutlierDetectionTSO does not support ModelContainer input.") + weighted_cube = weight_no_resample(input_model, self.good_bits) + + weight_threshold = compute_weight_threshold(weighted_cube.wht,self. maskpt) + + if (self.rolling_window_width > 1) and (self.rolling_window_width < weighted_cube.shape[0]): + medians = compute_rolling_median(weighted_cube, weight_threshold, w=self.rolling_window_width) + + else: + medians = np.nanmedian(weighted_cube.data, axis=0) + # this is a 2-D array, need to repeat it into the time axis + # for consistent shape with rolling median case + medians = np.broadcast_to(medians, weighted_cube.shape) + + # Save median model if pars['save_intermediate_results'] is True + # this will be a CubeModel with rolling median values. + if self.save_intermediate_results: + median_model = dm.CubeModel(data=medians) + with dm.open(weighted_cube) as dm0: + median_model.update(dm0) + save_median(median_model, self.make_output_path, asn_id) + del median_model + + # no need for blotting, resample is turned off for TSO + # go straight to outlier detection + log.info("Flagging outliers") + flag_model_crs( + input_model, + medians, + self.snr, + ) + return input_model def weight_no_resample(input_model, good_bits): diff --git a/jwst/outlier_detection/spec.py b/jwst/outlier_detection/spec.py deleted file mode 100644 index 7f35733aa6..0000000000 --- a/jwst/outlier_detection/spec.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -Submodule for performing outlier detection on spectra. -""" -import copy - -from stdatamodels.jwst import datamodels - -from jwst.datamodels import ModelContainer -from jwst.stpipe.utilities import record_step_status - -from ..resample import resample_spec, resample_utils -from .utils import create_median, flag_crs_in_models, flag_crs_in_models_with_resampling -from ._fileio import remove_file - -import logging -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - - -__all__ = ["detect_outliers"] - - -def detect_outliers( - input_models, - save_intermediate_results, - good_bits, - maskpt, - snr1, - snr2, - scale1, - scale2, - backg, - resample_data, - weight_type, - pixfrac, - kernel, - fillval, - in_memory, - asn_id, - make_output_path, -): - """ - Flag outliers in spec data. - - See `OutlierDetectionStep.spec` for documentation of these arguments. - """ - if not isinstance(input_models, ModelContainer): - input_models = ModelContainer(input_models, save_open=in_memory) - - if len(input_models) < 2: - log.warning(f"Input only contains {len(input_models)} exposures") - log.warning("Outlier detection will be skipped") - record_step_status(input_models, "outlier_detection", False) - return input_models - - if resample_data is True: - # Start by creating resampled/mosaic images for - # each group of exposures - resamp = resample_spec.ResampleSpecData( - input_models, - single=True, - blendheaders=False, - wht_type=weight_type, - pixfrac=pixfrac, - kernel=kernel, - fillval=fillval, - good_bits=good_bits, - in_memory=in_memory, - asn_id=asn_id, - ) - median_wcs = resamp.output_wcs - drizzled_models = resamp.do_drizzle(input_models) - if save_intermediate_results: - for model in drizzled_models: - model.meta.filename = make_output_path( - basepath=model.meta.filename, - suffix="_outlier_s2d.fits", - ) - log.info("Writing out resampled spectra...") - model.save(model.meta.filename) - else: - drizzled_models = input_models - for i in range(len(input_models)): - drizzled_models[i].wht = resample_utils.build_driz_weight( - input_models[i], - weight_type=weight_type, - good_bits=good_bits) - # copy for when saving median and input is a filename? - median_wcs = copy.deepcopy(input_models[0].meta.wcs) - - # Perform median combination on set of drizzled mosaics - # create_median should be called as a method from parent class - median_data = create_median(drizzled_models, maskpt) - - if save_intermediate_results: - # Initialize intermediate products used in the outlier detection - median_model = datamodels.ImageModel(median_data) - median_model.meta = drizzled_models[0].meta - median_model.meta.filename = make_output_path( - basepath=input_models[0].meta.filename, - suffix='median' - ) - - log.info("Writing out MEDIAN image to: {}".format( - median_model.meta.filename)) - median_model.save(median_model.meta.filename) - - del median_model - else: - # since we're not saving intermediate results if the drizzled models - # were written to disk, remove them - if not in_memory: - for fn in drizzled_models._models: - remove_file(fn) - log.info(f"Removing file {fn}") - - # Perform outlier detection using statistical comparisons between - # each original input image and its blotted version of the median image - if resample_data: - flag_crs_in_models_with_resampling( - input_models, - median_data, - median_wcs, - snr1, - snr2, - scale1, - scale2, - backg, - ) - else: - flag_crs_in_models(input_models, median_data, snr1) - return input_models diff --git a/jwst/outlier_detection/tests/test_algorithms.py b/jwst/outlier_detection/tests/test_algorithms.py index b3b9255bb4..05686acea0 100644 --- a/jwst/outlier_detection/tests/test_algorithms.py +++ b/jwst/outlier_detection/tests/test_algorithms.py @@ -1,6 +1,6 @@ import numpy as np -from jwst.outlier_detection.tso import moving_median_over_zeroth_axis +from jwst.outlier_detection.outlier_detection_tso_step import moving_median_over_zeroth_axis def test_rolling_median(): diff --git a/jwst/outlier_detection/tests/test_outlier_detection.py b/jwst/outlier_detection/tests/test_outlier_detection.py index 622fc791ff..06a6b2f3ae 100644 --- a/jwst/outlier_detection/tests/test_outlier_detection.py +++ b/jwst/outlier_detection/tests/test_outlier_detection.py @@ -7,13 +7,11 @@ from stdatamodels.jwst import datamodels from jwst.datamodels import ModelContainer -from jwst.outlier_detection import OutlierDetectionStep from jwst.outlier_detection.utils import flag_resampled_model_crs -from jwst.outlier_detection.outlier_detection_step import ( - IMAGE_MODES, - TSO_SPEC_MODES, - TSO_IMAGE_MODES, - CORON_IMAGE_MODES, +from jwst.outlier_detection import ( + OutlierDetectionTSOStep, + OutlierDetectionCoronStep, + OutlierDetectionImagingStep, ) from jwst.assign_wcs.pointing import create_fitswcs @@ -21,10 +19,13 @@ datamodels.dqflags.pixel["DO_NOT_USE"], datamodels.dqflags.pixel["OUTLIER"] ) -# TSO types to test -exptypes_tso = [(exptype, True) for exptype in TSO_SPEC_MODES + TSO_IMAGE_MODES] -exptypes_tso.append(("MIR_IMAGE", True)) -# CORON types to test +IMAGE_MODES = ['NRC_IMAGE', 'MIR_IMAGE', 'NRS_IMAGE', 'NIS_IMAGE', 'FGS_IMAGE'] + +TSO_MODES = ['NIS_SOSS', 'MIR_LRS-SLITLESS', 'NRC_TSGRISM', + 'NRS_BRIGHTOBJ', 'NRC_TSIMAGE', 'MIR_IMAGE'] +exptypes_tso = [(exptype, True) for exptype in TSO_MODES] + +CORON_IMAGE_MODES = ['NRC_CORON', 'MIR_LYOT', 'MIR_4QPM'] exptypes_coron = [(exptype, False) for exptype in CORON_IMAGE_MODES] @@ -195,7 +196,7 @@ def test_outlier_step_no_outliers(we_three_sci, tmp_cwd): """Test whole step, no outliers""" container = ModelContainer(list(we_three_sci)) pristine = ModelContainer([m.copy() for m in container]) - OutlierDetectionStep.call(container) + OutlierDetectionImagingStep.call(container) # Make sure nothing changed in SCI and DQ arrays for image, uncorrected in zip(pristine, container): @@ -211,13 +212,13 @@ def test_outlier_step(we_three_sci, tmp_cwd): container[0].data[12, 12] += 1 # Verify that intermediary files are removed - OutlierDetectionStep.call(container) + OutlierDetectionImagingStep.call(container) i2d_files = glob(os.path.join(tmp_cwd, '*i2d.fits')) median_files = glob(os.path.join(tmp_cwd, '*median.fits')) assert len(i2d_files) == 0 assert len(median_files) == 0 - result = OutlierDetectionStep.call( + result = OutlierDetectionImagingStep.call( container, save_results=True, save_intermediate_results=True ) @@ -253,7 +254,7 @@ def test_outlier_step_on_disk(we_three_sci, tmp_cwd): # Initialize inputs for the test based on filenames only container = ModelContainer(filenames) - result = OutlierDetectionStep.call( + result = OutlierDetectionImagingStep.call( container, save_results=True, save_intermediate_results=True ) @@ -278,7 +279,7 @@ def test_outlier_step_square_source_no_outliers(we_three_sci, tmp_cwd): ccont.data[5:15, 5:15] += 1e3 pristine = container.copy() - result = OutlierDetectionStep.call(container) + result = OutlierDetectionImagingStep.call(container) # Make sure nothing changed in SCI and DQ arrays for image, uncorrected in zip(pristine, container): @@ -307,7 +308,7 @@ def test_outlier_step_image_weak_CR_dither(exptype, tmp_cwd): # no noise so it should always be above the default threshold of 5 container[0].data[12, 12] = bkg + sig * 10 - result = OutlierDetectionStep.call(container) + result = OutlierDetectionImagingStep.call(container) # Make sure nothing changed in SCI array for image, corrected in zip(container, result): @@ -339,7 +340,7 @@ def test_outlier_step_image_weak_CR_coron(exptype, tsovisit, tmp_cwd): # coron3 will provide a CubeModel so convert the container to a cube cube = container_to_cube(container) - result = OutlierDetectionStep.call(cube) + result = OutlierDetectionCoronStep.call(cube) # Make sure nothing changed in SCI array for i, image in enumerate(container): @@ -379,7 +380,7 @@ def test_outlier_step_weak_cr_tso(exptype, tsovisit): cube = container_to_cube(im) - result = OutlierDetectionStep.call(cube, rolling_window_width=rolling_window_width) + result = OutlierDetectionTSOStep.call(cube, rolling_window_width=rolling_window_width) # Make sure nothing changed in SCI array for i, model in enumerate(im): diff --git a/jwst/outlier_detection/utils.py b/jwst/outlier_detection/utils.py index f8c6cc1333..ce53c3f97c 100644 --- a/jwst/outlier_detection/utils.py +++ b/jwst/outlier_detection/utils.py @@ -1,6 +1,8 @@ """ The ever-present utils sub-module. A home for all... """ +from abc import ABC, abstractmethod +from functools import partial import warnings import numpy as np @@ -17,6 +19,41 @@ DO_NOT_USE = datamodels.dqflags.pixel['DO_NOT_USE'] OUTLIER = datamodels.dqflags.pixel['OUTLIER'] +class OutlierDetectionStepBase(ABC): + """Minimal base class holding common methods for outlier detection steps.""" + + @abstractmethod + def search_attr(self, attr, **kwargs): + pass + + @abstractmethod + def _make_output_path(self): + pass + + def _get_asn_id(self, input_models): + # handle if input_models isn't open + if not isinstance(input_models, datamodels.JwstDataModel): + input_models = datamodels.open(input_models, asn_n_members=1) + + # Setup output path naming if associations are involved. + asn_id = None + try: + asn_id = input_models.meta.asn_table.asn_id + except (AttributeError, KeyError): + pass + if asn_id is None: + asn_id = self.search_attr('asn_id') + if asn_id is not None: + _make_output_path = self.search_attr( + '_make_output_path', parent_first=True + ) + + self._make_output_path = partial( + _make_output_path, + asn_id=asn_id + ) + return asn_id + def create_cube_median(cube_model, maskpt): log.info("Computing median") diff --git a/jwst/pipeline/calwebb_coron3.py b/jwst/pipeline/calwebb_coron3.py index a309fcc14d..5a7fb9b6b1 100644 --- a/jwst/pipeline/calwebb_coron3.py +++ b/jwst/pipeline/calwebb_coron3.py @@ -13,7 +13,7 @@ from ..coron import stack_refs_step from ..coron import align_refs_step from ..coron import klip_step -from ..outlier_detection import outlier_detection_step +from ..outlier_detection import outlier_detection_coron_step from ..resample import resample_step __all__ = ['Coron3Pipeline'] @@ -67,7 +67,7 @@ class Coron3Pipeline(Pipeline): 'stack_refs': stack_refs_step.StackRefsStep, 'align_refs': align_refs_step.AlignRefsStep, 'klip': klip_step.KlipStep, - 'outlier_detection': outlier_detection_step.OutlierDetectionStep, + 'outlier_detection': outlier_detection_coron_step.OutlierDetectionCoronStep, 'resample': resample_step.ResampleStep } @@ -104,7 +104,6 @@ def process(self, user_input): # Set up required output products and formats self.outlier_detection.suffix = 'crfints' - self.outlier_detection.mode = 'coron' self.outlier_detection.save_results = self.save_results self.resample.blendheaders = False diff --git a/jwst/pipeline/calwebb_image3.py b/jwst/pipeline/calwebb_image3.py index 9407d85913..4fa98cc58e 100644 --- a/jwst/pipeline/calwebb_image3.py +++ b/jwst/pipeline/calwebb_image3.py @@ -9,7 +9,7 @@ from ..tweakreg import tweakreg_step from ..skymatch import skymatch_step from ..resample import resample_step -from ..outlier_detection import outlier_detection_step +from ..outlier_detection import outlier_detection_imaging_step from ..source_catalog import source_catalog_step __all__ = ['Image3Pipeline'] @@ -39,7 +39,7 @@ class Image3Pipeline(Pipeline): 'assign_mtwcs': assign_mtwcs_step.AssignMTWcsStep, 'tweakreg': tweakreg_step.TweakRegStep, 'skymatch': skymatch_step.SkyMatchStep, - 'outlier_detection': outlier_detection_step.OutlierDetectionStep, + 'outlier_detection': outlier_detection_imaging_step.OutlierDetectionImagingStep, 'resample': resample_step.ResampleStep, 'source_catalog': source_catalog_step.SourceCatalogStep } @@ -61,7 +61,6 @@ def process(self, input_data): # Configure settings for saving results files self.outlier_detection.suffix = 'crf' - self.outlier_detection.mode = 'imaging' self.outlier_detection.save_results = self.save_results self.resample.suffix = 'i2d' diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index 96a5b25d68..4f2f92a551 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -20,7 +20,7 @@ from ..extract_1d import extract_1d_step from ..master_background import master_background_step from ..mrs_imatch import mrs_imatch_step -from ..outlier_detection import outlier_detection_step +from ..outlier_detection import outlier_detection_spec_step, outlier_detection_ifu_step from ..resample import resample_spec_step from ..combine_1d import combine_1d_step from ..photom import photom_step @@ -43,7 +43,7 @@ class Spec3Pipeline(Pipeline): assign moving target wcs (assign_mtwcs) master background subtraction (master_background) MIRI MRS background matching (mrs_imatch) - outlier detection (outlier_detection) + outlier detection (outlier_detection_spec_step or outlier_detection_ifu_step) 2-D spectroscopic resampling (resample_spec) 3-D spectroscopic resampling (cube_build) 1-D spectral extraction (extract_1d) @@ -61,7 +61,8 @@ class Spec3Pipeline(Pipeline): 'assign_mtwcs': assign_mtwcs_step.AssignMTWcsStep, 'master_background': master_background_step.MasterBackgroundStep, 'mrs_imatch': mrs_imatch_step.MRSIMatchStep, - 'outlier_detection': outlier_detection_step.OutlierDetectionStep, + 'outlier_detection_spec': outlier_detection_spec_step.OutlierDetectionSpecStep, + 'outlier_detection_ifu': outlier_detection_ifu_step.OutlierDetectionIFUStep, 'pixel_replace': pixel_replace_step.PixelReplaceStep, 'resample_spec': resample_spec_step.ResampleSpecStep, 'cube_build': cube_build_step.CubeBuildStep, @@ -86,8 +87,10 @@ def process(self, input): # Setup sub-step defaults self.master_background.suffix = 'mbsub' self.mrs_imatch.suffix = 'mrs_imatch' - self.outlier_detection.suffix = 'crf' - self.outlier_detection.save_results = self.save_results + self.outlier_detection_spec.suffix = 'crf' + self.outlier_detection_spec.save_results = self.save_results + self.outlier_detection_ifu.suffix = 'crf' + self.outlier_detection_ifu.save_results = self.save_results self.resample_spec.suffix = 's2d' self.resample_spec.save_results = self.save_results self.cube_build.suffix = 's3d' @@ -106,7 +109,8 @@ def process(self, input): # These steps save intermediate files, resulting in meta.filename # being modified. This can affect the filenames of subsequent # steps. - self.outlier_detection.save_model = invariant_filename(self.outlier_detection.save_model) + self.outlier_detection_spec.save_model = invariant_filename(self.outlier_detection_spec.save_model) + self.outlier_detection_ifu.save_model = invariant_filename(self.outlier_detection_ifu.save_model) self.pixel_replace.save_model = invariant_filename(self.pixel_replace.save_model) # Retrieve the inputs: @@ -228,10 +232,10 @@ def process(self, input): for cal_array in result: cal_array.meta.asn.table_name = op.basename(input_models.asn_table_name) if exptype in IFU_EXPTYPES: - self.outlier_detection.mode = 'ifu' + result = self.outlier_detection_ifu(result) else: - self.outlier_detection.mode = 'spec' - result = self.outlier_detection(result) + result = self.outlier_detection_spec(result) + # interpolate pixels that have a NaN value or are flagged # as DO_NOT_USE or NON_SCIENCE. diff --git a/jwst/pipeline/calwebb_tso3.py b/jwst/pipeline/calwebb_tso3.py index 2b9e14ae86..df173b02dd 100644 --- a/jwst/pipeline/calwebb_tso3.py +++ b/jwst/pipeline/calwebb_tso3.py @@ -8,7 +8,7 @@ from ..stpipe import Pipeline -from ..outlier_detection import outlier_detection_step +from ..outlier_detection import outlier_detection_tso_step from ..tso_photometry import tso_photometry_step from ..extract_1d import extract_1d_step from ..white_light import white_light_step @@ -41,7 +41,7 @@ class Tso3Pipeline(Pipeline): # Define alias to steps step_defs = {'outlier_detection': - outlier_detection_step.OutlierDetectionStep, + outlier_detection_tso_step.OutlierDetectionTSOStep, 'tso_photometry': tso_photometry_step.TSOPhotometryStep, 'pixel_replace': pixel_replace_step.PixelReplaceStep, 'extract_1d': extract_1d_step.Extract1dStep, @@ -77,7 +77,6 @@ def process(self, input): # This asn_id assignment is important as it allows outlier detection # to know the asn_id since that step receives the cube as input. self.asn_id = input_models.meta.asn_table.asn_id - self.outlier_detection.mode = 'tso' # Input may consist of multiple exposures, so loop over each of them input_exptype = None diff --git a/jwst/step.py b/jwst/step.py index 1c9b67b58f..a5e053b7af 100644 --- a/jwst/step.py +++ b/jwst/step.py @@ -35,7 +35,11 @@ from .mrs_imatch.mrs_imatch_step import MRSIMatchStep from .msaflagopen.msaflagopen_step import MSAFlagOpenStep from .nsclean.nsclean_step import NSCleanStep -from .outlier_detection.outlier_detection_step import OutlierDetectionStep +from .outlier_detection.outlier_detection_coron_step import OutlierDetectionCoronStep +from .outlier_detection.outlier_detection_ifu_step import OutlierDetectionIFUStep +from .outlier_detection.outlier_detection_imaging_step import OutlierDetectionImagingStep +from .outlier_detection.outlier_detection_spec_step import OutlierDetectionSpecStep +from .outlier_detection.outlier_detection_tso_step import OutlierDetectionTSOStep from .pathloss.pathloss_step import PathLossStep from .persistence.persistence_step import PersistenceStep from .photom.photom_step import PhotomStep @@ -99,7 +103,11 @@ "MRSIMatchStep", "MSAFlagOpenStep", "NSCleanStep", - "OutlierDetectionStep", + "OutlierDetectionCoronStep", + "OutlierDetectionIFUStep", + "OutlierDetectionImagingStep", + "OutlierDetectionSpecStep", + "OutlierDetectionTSOStep", "PathLossStep", "PersistenceStep", "PhotomStep", diff --git a/jwst/stpipe/integration.py b/jwst/stpipe/integration.py index 597a6afe3e..1ad6bc8a4d 100644 --- a/jwst/stpipe/integration.py +++ b/jwst/stpipe/integration.py @@ -67,7 +67,11 @@ def get_steps(): ("jwst.step.MRSIMatchStep", 'mrs_imatch', False), ("jwst.step.MSAFlagOpenStep", 'msa_flagging', False), ("jwst.step.NSCleanStep", 'nsclean', False), - ("jwst.step.OutlierDetectionStep", 'outlier_detection', False), + ("jwst.step.OutlierDetectionCoronStep", 'outlier_detection_coron', False), + ("jwst.step.OutlierDetectionIFUStep", 'outlier_detection_ifu', False), + ("jwst.step.OutlierDetectionImagingStep", 'outlier_detection_imaging', False), + ("jwst.step.OutlierDetectionSpecStep", 'outlier_detection_spec', False), + ("jwst.step.OutlierDetectionTSOStep", 'outlier_detection_tso', False), ("jwst.step.PathLossStep", 'pathloss', False), ("jwst.step.PersistenceStep", 'persistence', False), ("jwst.step.PhotomStep", 'photom', False),