diff --git a/CHANGES.rst b/CHANGES.rst index 296e5f2851..acfee15c11 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,11 @@ 1.12.4 (unreleased) =================== -- +cube_build +---------- + +- Keep data models opened by cube build open until the step completes. [#7998] + 1.12.3 (2023-10-03) =================== diff --git a/jwst/cube_build/cube_build_step.py b/jwst/cube_build/cube_build_step.py index c751a13ad0..15a9dde45d 100755 --- a/jwst/cube_build/cube_build_step.py +++ b/jwst/cube_build/cube_build_step.py @@ -241,6 +241,7 @@ def process(self, input): # Check for a valid reference file if par_filename == 'N/A': self.log.warning('No default cube parameters reference file found') + input_table.close() return # ________________________________________________________________________________ # shove the input parameters in to pars to pull out in general cube_build.py @@ -299,6 +300,7 @@ def process(self, input): if instrument == 'MIRI' and self.coord_system == 'internal_cal': self.log.warning('The output coordinate system of internal_cal is not valid for MIRI') self.log.warning('use output_coord = ifualign instead') + input_table.close() return filenames = master_table.FileMap['filename'] @@ -402,6 +404,7 @@ def process(self, input): if status_cube == 1: self.skip = True + input_table.close() return cube_container # ****************************************************************************** diff --git a/jwst/cube_build/data_types.py b/jwst/cube_build/data_types.py index e519cebe92..d7217087a4 100644 --- a/jwst/cube_build/data_types.py +++ b/jwst/cube_build/data_types.py @@ -69,30 +69,34 @@ def __init__(self, input, single, output_file, output_dir): self.input_models = [] self.output_name = None - + # open the input with datamodels # if input is filename or model when it is opened it is a model # if input if an association name or ModelContainer then it is opened as a container - with datamodels.open(input) as input_models: - - if isinstance(input_models, datamodels.IFUImageModel): - # It's a single image that's been passed in as a model - # input is a model - filename = input_models.meta.filename - self.input_models.append(input_models) - self.output_name = self.build_product_name(filename) - - elif isinstance(input_models, ModelContainer): - self.output_name = 'Temp' - self.input_models = input_models - if not single: # find the name of the output file from the association - self.output_name = input_models.meta.asn_table.products[0].name - else: - raise TypeError("Failed to process file type {}".format(type(input_models))) - # if the user has set the output name - strip out *.fits - # later suffixes will be added to this name to designate the - # channel, subchannel or grating,filter the data is covers. + input_models = datamodels.open(input) + # if input is a filename, we will need to close the opened file + self._opened = [input_models] + + if isinstance(input_models, datamodels.IFUImageModel): + # It's a single image that's been passed in as a model + # input is a model + filename = input_models.meta.filename + self.input_models.append(input_models) + self.output_name = self.build_product_name(filename) + + elif isinstance(input_models, ModelContainer): + self.output_name = 'Temp' + self.input_models = input_models + if not single: # find the name of the output file from the association + self.output_name = input_models.meta.asn_table.products[0].name + else: + # close files opened above + self.close() + raise TypeError("Failed to process file type {}".format(type(input_models))) + # If the user has set the output name, strip off *.fits. + # Suffixes will be added to this name later, to designate the + # channel+subchannel (MIRI MRS) or grating+filter (NRS IFU) the output cube covers. if output_file is not None: @@ -102,6 +106,12 @@ def __init__(self, input, single, output_file, output_dir): if output_dir is not None: self.output_name = output_dir + '/' + self.output_name + def close(self): + """ + Close any files opened by this instance + """ + [f.close() for f in self._opened] + # _______________________________________________________________________________ def build_product_name(self, filename): """ Determine the base of output name if an input data is a fits filename. diff --git a/jwst/cube_build/file_table.py b/jwst/cube_build/file_table.py index 69902b46ea..73863a83d5 100644 --- a/jwst/cube_build/file_table.py +++ b/jwst/cube_build/file_table.py @@ -113,7 +113,6 @@ def set_file_table(self, # ________________________________________________________________________________ # Loop over input list of files and assign fill in the MasterTable with filename # for the correct (channel-subchannel) or (grating-subchannel) - for model in input_models: instrument = model.meta.instrument.name.upper() assign_wcs = model.meta.cal_step.assign_wcs @@ -143,9 +142,6 @@ def set_file_table(self, log.info('Instrument not valid for cube') pass - model.close() - del model - return instrument diff --git a/jwst/cube_build/tests/test_cube_build_step.py b/jwst/cube_build/tests/test_cube_build_step.py index 215772207a..3d318fae4a 100644 --- a/jwst/cube_build/tests/test_cube_build_step.py +++ b/jwst/cube_build/tests/test_cube_build_step.py @@ -6,8 +6,10 @@ import pytest from astropy.io import fits +from gwcs import WCS from stdatamodels.jwst.datamodels import IFUImageModel +from jwst import assign_wcs from jwst.cube_build import CubeBuildStep from jwst.cube_build.file_table import ErrorNoAssignWCS from jwst.cube_build.cube_build import ErrorNoChannels @@ -100,8 +102,15 @@ def miri_image(): return image -def test_call_cube_build(_jail, miri_cube_pars, miri_image): +@pytest.mark.parametrize("as_filename", [True, False]) +def test_call_cube_build(_jail, miri_cube_pars, miri_image, tmp_path, as_filename): """ test defaults of step are set up and user input are defined correctly """ + if as_filename: + fn = tmp_path / 'miri.fits' + miri_image.save(fn) + step_input = fn + else: + step_input = miri_image # we do not want to run the CubeBuild through to completion because # the image needs to be a full image and this take too much time @@ -112,7 +121,7 @@ def test_call_cube_build(_jail, miri_cube_pars, miri_image): step = CubeBuildStep() step.override_cubepar = miri_cube_pars step.channel = '3' - step.run(miri_image) + step.run(step_input) # Test some defaults to step are setup correctly and # is user specifies channel is set up correctly @@ -121,7 +130,7 @@ def test_call_cube_build(_jail, miri_cube_pars, miri_image): step.channel = '1' try: - step.run(miri_image) + step.run(step_input) except ErrorNoAssignWCS: pass @@ -132,8 +141,53 @@ def test_call_cube_build(_jail, miri_cube_pars, miri_image): # Set Assign WCS has been run but the user input to channels is wrong miri_image.meta.cal_step.assign_wcs = 'COMPLETE' + # save file with modifications + if as_filename: + miri_image.save(step_input) with pytest.raises(ErrorNoChannels): step = CubeBuildStep() step.override_cubepar = miri_cube_pars step.channel = '3' - step.run(miri_image) + step.run(step_input) + + +@pytest.fixture(scope='function') +def nirspec_data(): + image = IFUImageModel((2048, 2048)) + image.data = np.random.random((2048, 2048)) + image.meta.instrument.name = 'NIRSPEC' + image.meta.instrument.detector = 'NRS1' + image.meta.exposure.type = 'NRS_IFU' + image.meta.filename = 'test_nirspec.fits' + image.meta.observation.date = '2023-10-06' + image.meta.observation.time = '00:00:00.000' + # below values taken from regtest using file + # jw01249005001_03101_00004_nrs1_cal.fits + image.meta.instrument.filter = 'F290LP' + image.meta.instrument.grating = 'G395H' + image.meta.wcsinfo.v2_ref = 299.83548 + image.meta.wcsinfo.v3_ref = -498.256805 + image.meta.wcsinfo.ra_ref = 358.0647567841019 + image.meta.wcsinfo.dec_ref = -2.167207258876695 + image.meta.cal_step.assign_wcs = 'COMPLETE' + step = assign_wcs.assign_wcs_step.AssignWcsStep() + refs = {} + for reftype in assign_wcs.assign_wcs_step.AssignWcsStep.reference_file_types: + refs[reftype] = step.get_reference_file(image, reftype) + pipe = assign_wcs.nirspec.create_pipeline(image, refs, slit_y_range=[-.5, .5]) + image.meta.wcs = WCS(pipe) + return image + + +@pytest.mark.parametrize("as_filename", [True, False]) +def test_call_cube_build_nirspec(_jail, nirspec_data, tmp_path, as_filename): + if as_filename: + fn = tmp_path / 'test_nirspec.fits' + nirspec_data.save(fn) + step_input = fn + else: + step_input = nirspec_data + step = CubeBuildStep() + step.channel = '1' + step.coord_system = 'internal_cal' + step.run(step_input)