diff --git a/CHANGES.rst b/CHANGES.rst index 14a97a26b6..6672bf030a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -61,14 +61,18 @@ outlier_detection - Fixed failures due to a missing ``wcs.array_shape`` attribute when the ``outlier_detection`` step was run standalone using e.g. ``strun`` [#8645] +- Refactored separate modes into submodules instead of inheriting from a base class. + Moved non-JWST-specific code to stcal. [#8613] + +- Fixed file names and output directory for intermediate resampled spectral + images. Intermediate files now have suffix ``outlier_s2d`` and are saved to + the output directory alongside final products. [#8735] + set_telescope_pointing ---------------------- - replace usage of ``copy_arrays=True`` with ``memmap=False`` [#8660] -- Refactored separate modes into submodules instead of inheriting from a base class. - Moved non-JWST-specific code to stcal. [#8613] - resample_spec ------------- diff --git a/docs/jwst/outlier_detection/outlier_detection_spec.rst b/docs/jwst/outlier_detection/outlier_detection_spec.rst index 6d8cd13de5..f66df77854 100644 --- a/docs/jwst/outlier_detection/outlier_detection_spec.rst +++ b/docs/jwst/outlier_detection/outlier_detection_spec.rst @@ -29,14 +29,14 @@ Specifically, this routine performs the following operations (modified from the #. Resample all input images into a :py:class:`~jwst.datamodels.ModelContainer` using :py:class:`~jwst.resample.resample_spec.ResampleSpecData` - - Resampled images are written out to disk if the ``save_intermediate_results`` - parameter is set to `True` + - Resampled images are written out to disk with suffix "outlier_s2d" + if the ``save_intermediate_results`` parameter is set to `True`. - **If resampling is turned off**, the original unrectified inputs are used to create - the median image for cosmic-ray detection + the median image for cosmic-ray detection. #. Create a median image from (possibly) resampled :py:class:`~jwst.datamodels.ModelContainer` - - The median image is written out to disk if the ``save_intermediate_results`` - parameter is set to `True` + - The median image is written out to disk with suffix "median" + if the ``save_intermediate_results`` parameter is set to `True` #. Blot median image to match each original input image - **If resampling is turned off**, the median image is used for comparison diff --git a/jwst/outlier_detection/imaging.py b/jwst/outlier_detection/imaging.py index c6ba99da13..ad6466de82 100644 --- a/jwst/outlier_detection/imaging.py +++ b/jwst/outlier_detection/imaging.py @@ -60,8 +60,8 @@ def detect_outliers( if resample_data: # Start by creating resampled/mosaic images for # each group of exposures - output_path = make_output_path(basepath=input_models[0].meta.filename, - suffix='') + output_path = make_output_path( + basepath=input_models[0].meta.filename, suffix='') output_path = os.path.dirname(output_path) resamp = resample.ResampleData( input_models, diff --git a/jwst/outlier_detection/outlier_detection_step.py b/jwst/outlier_detection/outlier_detection_step.py index 936ec3767b..a099509e22 100644 --- a/jwst/outlier_detection/outlier_detection_step.py +++ b/jwst/outlier_detection/outlier_detection_step.py @@ -193,7 +193,7 @@ def _guess_mode(self, input_models): if exptype in IFU_SPEC_MODES: return 'ifu' - self.log.error("Outlier detection failed for unknown/unsupported ", + self.log.error(f"Outlier detection failed for unknown/unsupported " f"exposure type: {exptype}") return None diff --git a/jwst/outlier_detection/spec.py b/jwst/outlier_detection/spec.py index 7f35733aa6..88f541a4d9 100644 --- a/jwst/outlier_detection/spec.py +++ b/jwst/outlier_detection/spec.py @@ -2,6 +2,7 @@ Submodule for performing outlier detection on spectra. """ import copy +import os from stdatamodels.jwst import datamodels @@ -56,8 +57,12 @@ def detect_outliers( if resample_data is True: # Start by creating resampled/mosaic images for # each group of exposures + output_path = make_output_path( + basepath=input_models[0].meta.filename, suffix='') + output_path = os.path.dirname(output_path) resamp = resample_spec.ResampleSpecData( input_models, + output=output_path, single=True, blendheaders=False, wht_type=weight_type, @@ -70,14 +75,6 @@ def detect_outliers( ) median_wcs = resamp.output_wcs drizzled_models = resamp.do_drizzle(input_models) - if save_intermediate_results: - for model in drizzled_models: - model.meta.filename = make_output_path( - basepath=model.meta.filename, - suffix="_outlier_s2d.fits", - ) - log.info("Writing out resampled spectra...") - model.save(model.meta.filename) else: drizzled_models = input_models for i in range(len(input_models)): diff --git a/jwst/outlier_detection/tests/test_outlier_detection.py b/jwst/outlier_detection/tests/test_outlier_detection.py index 622fc791ff..1c7bc0e9e4 100644 --- a/jwst/outlier_detection/tests/test_outlier_detection.py +++ b/jwst/outlier_detection/tests/test_outlier_detection.py @@ -6,6 +6,8 @@ from stdatamodels.jwst import datamodels +from jwst.assign_wcs import AssignWcsStep +from jwst.assign_wcs.pointing import create_fitswcs from jwst.datamodels import ModelContainer from jwst.outlier_detection import OutlierDetectionStep from jwst.outlier_detection.utils import flag_resampled_model_crs @@ -15,7 +17,7 @@ TSO_IMAGE_MODES, CORON_IMAGE_MODES, ) -from jwst.assign_wcs.pointing import create_fitswcs +from jwst.resample.tests.test_resample_step import miri_rate_model OUTLIER_DO_NOT_USE = np.bitwise_or( datamodels.dqflags.pixel["DO_NOT_USE"], datamodels.dqflags.pixel["OUTLIER"] @@ -210,7 +212,7 @@ def test_outlier_step(we_three_sci, tmp_cwd): # Drop a CR on the science array container[0].data[12, 12] += 1 - # Verify that intermediary files are removed + # Verify that intermediate files are removed OutlierDetectionStep.call(container) i2d_files = glob(os.path.join(tmp_cwd, '*i2d.fits')) median_files = glob(os.path.join(tmp_cwd, '*median.fits')) @@ -232,13 +234,104 @@ def test_outlier_step(we_three_sci, tmp_cwd): # Verify CR is flagged assert result[0].dq[12, 12] == OUTLIER_DO_NOT_USE - # Verify that intermediary files are saved at the specified location + # Verify that intermediate files are saved at the specified location i2d_files = glob(os.path.join(tmp_cwd, '*i2d.fits')) median_files = glob(os.path.join(tmp_cwd, '*median.fits')) assert len(i2d_files) != 0 assert len(median_files) != 0 +def test_outlier_step_spec(tmp_cwd, tmp_path): + """Test outlier step for spec data including saving intermediate results.""" + output_dir = tmp_path / 'output' + output_dir.mkdir(exist_ok=True) + output_dir = str(output_dir) + + # Make a MIRI model and assign a spectral wcs + miri_rate = miri_rate_model() + miri_cal = AssignWcsStep.call(miri_rate) + + # Make it an exposure type outlier detection expects + miri_cal.meta.exposure.type = "MIR_LRS-FIXEDSLIT" + + # Make a couple copies + container = ModelContainer([miri_cal, miri_cal.copy(), miri_cal.copy()]) + + # Give each image a unique name so output files don't overwrite + for i, model in enumerate(container): + model.meta.filename = f'test_{i}_cal.fits' + + # Drop a CR on the science array in the first image + container[0].data[209, 37] += 1 + + # Verify that intermediate files are removed when not saved + # (s2d files are expected, i2d files are not, but we'll check + # for them to make sure the imaging extension didn't creep back in) + OutlierDetectionStep.call(container, output_dir=output_dir, save_results=True) + for dirname in [output_dir, tmp_cwd]: + result_files = glob(os.path.join(dirname, '*outlierdetectionstep.fits')) + i2d_files = glob(os.path.join(dirname, '*i2d*.fits')) + s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits')) + median_files = glob(os.path.join(dirname, '*median.fits')) + + # intermediate files are removed + assert len(i2d_files) == 0 + assert len(s2d_files) == 0 + assert len(median_files) == 0 + + # result files are written to the output directory + if dirname == output_dir: + assert len(result_files) == len(container) + else: + assert len(result_files) == 0 + + # Call again, but save intermediate to the output path + result = OutlierDetectionStep.call( + container, save_results=True, save_intermediate_results=True, + output_dir=output_dir + ) + + # Make sure nothing changed in SCI array + for image, corrected in zip(container, result): + np.testing.assert_allclose(image.data, corrected.data) + + # Verify CR is flagged + assert result[0].dq[209, 37] == OUTLIER_DO_NOT_USE + + # Verify that intermediate files are saved at the specified location + for dirname in [output_dir, tmp_cwd]: + all_files = glob(os.path.join(dirname, '*.fits')) + result_files = glob(os.path.join(dirname, '*outlierdetectionstep.fits')) + i2d_files = glob(os.path.join(dirname, '*i2d*.fits')) + s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits')) + median_files = glob(os.path.join(dirname, '*median.fits')) + if dirname == output_dir: + # result files are written to the output directory + assert len(result_files) == len(container) + + # s2d and median files are written to the output directory + assert len(s2d_files) == len(container) + assert len(median_files) == 1 + + # i2d files not written + assert len(i2d_files) == 0 + + # nothing else was written + assert len(all_files) == len(s2d_files) + len(median_files) + len(result_files) + else: + # nothing should be written to the current directory + assert len(result_files) == 0 + assert len(s2d_files) == 0 + assert len(median_files) == 0 + assert len(i2d_files) == 0 + assert len(all_files) == 0 + + miri_rate.close() + result.close() + for model in container: + model.close() + + def test_outlier_step_on_disk(we_three_sci, tmp_cwd): """Test whole step with an outlier including saving intermediate and results files""" diff --git a/jwst/resample/resample.py b/jwst/resample/resample.py index 245102b4b0..681592f3aa 100644 --- a/jwst/resample/resample.py +++ b/jwst/resample/resample.py @@ -73,6 +73,7 @@ def __init__(self, input_models, output=None, single=False, blendheaders=True, if output is not None and '.fits' not in str(output): self.output_dir = output self.output_filename = None + self.intermediate_suffix = 'outlier_i2d' self.pscale_ratio = pscale_ratio self.single = single @@ -276,6 +277,7 @@ def resample_many_to_many(self, input_models): """ for exposure in input_models.models_grouped: output_model = self.blank_output + # Determine output file type from input exposure filenames # Use this for defining the output filename indx = exposure[0].meta.filename.rfind('.') @@ -283,9 +285,13 @@ def resample_many_to_many(self, input_models): output_root = '_'.join(exposure[0].meta.filename.replace( output_type, '').split('_')[:-1]) if self.asn_id is not None: - output_model.meta.filename = f'{output_root}_{self.asn_id}_outlier_i2d{output_type}' + output_model.meta.filename = ( + f'{output_root}_{self.asn_id}_' + f'{self.intermediate_suffix}{output_type}') else: - output_model.meta.filename = f'{output_root}_outlier_i2d{output_type}' + output_model.meta.filename = ( + f'{output_root}_' + f'{self.intermediate_suffix}{output_type}') # Initialize the output with the wcs driz = gwcs_drizzle.GWCSDrizzle(output_model, pixfrac=self.pixfrac, diff --git a/jwst/resample/resample_spec.py b/jwst/resample/resample_spec.py index 871068ba7f..e75d1c898d 100644 --- a/jwst/resample/resample_spec.py +++ b/jwst/resample/resample_spec.py @@ -66,6 +66,7 @@ def __init__(self, input_models, output=None, single=False, blendheaders=False, if output is not None and '.fits' not in str(output): self.output_dir = output self.output_filename = None + self.intermediate_suffix = 'outlier_s2d' self.pscale_ratio = pscale_ratio self.single = single @@ -78,7 +79,7 @@ def __init__(self, input_models, output=None, single=False, blendheaders=False, self.in_memory = kwargs.get('in_memory', True) self._recalc_pscale_ratio = False - log.info(f"Driz parameter kernal: {self.kernel}") + log.info(f"Driz parameter kernel: {self.kernel}") log.info(f"Driz parameter pixfrac: {self.pixfrac}") log.info(f"Driz parameter fillval: {self.fillval}") log.info(f"Driz parameter weight_type: {self.weight_type}") diff --git a/jwst/resample/tests/test_resample_step.py b/jwst/resample/tests/test_resample_step.py index 15261f63d2..fcd380a4b8 100644 --- a/jwst/resample/tests/test_resample_step.py +++ b/jwst/resample/tests/test_resample_step.py @@ -37,8 +37,7 @@ def _set_photom_kwd(im): ) -@pytest.fixture -def miri_rate(): +def miri_rate_model(): xsize = 72 ysize = 416 shape = (ysize, xsize) @@ -87,6 +86,11 @@ def miri_rate(): 'start_time': 58119.8333, 'type': 'MIR_LRS-SLITLESS', 'zero_frame': False} + return im + +@pytest.fixture +def miri_rate(): + im = miri_rate_model() yield im im.close()