From 6fc485bfab70171b2f554aebed08f89cfa415d06 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 25 Nov 2024 07:36:35 -0800 Subject: [PATCH] JP-3742: Remove allowed_memory and associated environment variable (#8975) --- changes/8975.resample.rst | 1 + docs/jwst/outlier_detection/arguments.rst | 9 ----- docs/jwst/resample/arguments.rst | 9 ----- jwst/outlier_detection/imaging.py | 2 -- .../outlier_detection_step.py | 2 -- .../tests/test_outlier_detection.py | 1 - jwst/regtest/test_fgs_image3.py | 21 ------------ jwst/resample/resample.py | 34 +------------------ jwst/resample/resample_step.py | 2 -- pyproject.toml | 1 - 10 files changed, 2 insertions(+), 80 deletions(-) create mode 100644 changes/8975.resample.rst diff --git a/changes/8975.resample.rst b/changes/8975.resample.rst new file mode 100644 index 0000000000..95d48e855e --- /dev/null +++ b/changes/8975.resample.rst @@ -0,0 +1 @@ +Removed allowed_memory parameter and DMODEL_ALLOWED_MEMORY environment variable diff --git a/docs/jwst/outlier_detection/arguments.rst b/docs/jwst/outlier_detection/arguments.rst index 4deae0e47b..be41db97d4 100644 --- a/docs/jwst/outlier_detection/arguments.rst +++ b/docs/jwst/outlier_detection/arguments.rst @@ -79,15 +79,6 @@ Step Arguments for Imaging and Slit-like Spectroscopic data Specifies whether or not to resample the input images when performing outlier detection. -``--allowed_memory`` - Specifies the fractional amount of - free memory to allow when creating the resampled image. If ``None``, the - environment variable ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no - check is made. If the resampled image would be larger than specified, an - ``OutputTooLargeError`` exception will be generated. - For example, if set to ``0.5``, only resampled images that use less than half - the available memory can be created. - ``--in_memory`` Specifies whether or not to load and create all images that are used during processing into memory. If ``False``, input files are loaded from disk when diff --git a/docs/jwst/resample/arguments.rst b/docs/jwst/resample/arguments.rst index 9332bc6dcd..2b2a47a6a0 100644 --- a/docs/jwst/resample/arguments.rst +++ b/docs/jwst/resample/arguments.rst @@ -108,15 +108,6 @@ image. ``--blendheaders`` (bool, default=True) Blend metadata from all input images into the resampled output image. -``--allowed_memory`` (float, default=None) - Specifies the fractional amount of free memory to allow when creating the - resampled image. If ``None``, the environment variable - ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no check is made. If the - resampled image would be larger than specified, an ``OutputTooLargeError`` - exception will be generated. - - For example, if set to ``0.5``, only resampled images that use less than - half the available memory can be created. ``--in_memory`` (boolean, default=True) Specifies whether or not to load and create all images that are used during diff --git a/jwst/outlier_detection/imaging.py b/jwst/outlier_detection/imaging.py index c964f3e9ad..0c5d7d163a 100644 --- a/jwst/outlier_detection/imaging.py +++ b/jwst/outlier_detection/imaging.py @@ -35,7 +35,6 @@ def detect_outliers( pixfrac, kernel, fillval, - allowed_memory, in_memory, make_output_path, ): @@ -65,7 +64,6 @@ def detect_outliers( kernel=kernel, fillval=fillval, good_bits=good_bits, - allowed_memory=allowed_memory, ) median_data, median_wcs = median_with_resampling(input_models, resamp, diff --git a/jwst/outlier_detection/outlier_detection_step.py b/jwst/outlier_detection/outlier_detection_step.py index 1b6b42f733..3f5e634522 100644 --- a/jwst/outlier_detection/outlier_detection_step.py +++ b/jwst/outlier_detection/outlier_detection_step.py @@ -59,7 +59,6 @@ class OutlierDetectionStep(Step): resample_data = boolean(default=True) good_bits = string(default="~DO_NOT_USE") # DQ flags to allow search_output_file = boolean(default=False) - allowed_memory = float(default=None) # Fraction of memory to use for the combined image in_memory = boolean(default=False) # ignored if run within the pipeline; set at pipeline level instead """ @@ -113,7 +112,6 @@ def process(self, input_data): self.pixfrac, self.kernel, self.fillval, - self.allowed_memory, self.in_memory, self.make_output_path, ) diff --git a/jwst/outlier_detection/tests/test_outlier_detection.py b/jwst/outlier_detection/tests/test_outlier_detection.py index 7d9849fc46..008ae19c64 100644 --- a/jwst/outlier_detection/tests/test_outlier_detection.py +++ b/jwst/outlier_detection/tests/test_outlier_detection.py @@ -677,6 +677,5 @@ def make_resamp(input_models): good_bits="~DO_NOT_USE", in_memory=in_memory, asn_id="test", - allowed_memory=None, ) return resamp diff --git a/jwst/regtest/test_fgs_image3.py b/jwst/regtest/test_fgs_image3.py index b9f574e5dc..d6679d6a56 100644 --- a/jwst/regtest/test_fgs_image3.py +++ b/jwst/regtest/test_fgs_image3.py @@ -1,7 +1,6 @@ import pytest from astropy.io.fits.diff import FITSDiff -from jwst.resample.resample import OutputTooLargeError from jwst.stpipe import Step @@ -34,23 +33,3 @@ def test_fgs_image3_catalog(run_fgs_image3, rtdata_module, diff_astropy_tables): rtdata.get_truth("truth/test_fgs_image3/jw01029-o001_t009_fgs_clear_cat.ecsv") assert diff_astropy_tables(rtdata.output, rtdata.truth, rtol=1e-3, atol=1e-4) - - -@pytest.mark.bigdata -def test_fgs_toobig(rtdata, fitsdiff_default_kwargs, caplog, monkeypatch): - """Test for the situation where the combined mosaic is too large""" - - # Set the environment to not allow the resultant too-large image. - # Note: this test was originally run on two pre-flight images - # with WCSs from very different parts of the sky. - # This condition should hopefully never be encountered in reductions - # of in-flight data. To test the software failsafe, we now use real data - # that makes a reasonable sized mosaic, but set the allowed memory to a - # small value. - monkeypatch.setenv('DMODEL_ALLOWED_MEMORY', "0.0001") - - rtdata.get_asn('fgs/image3/jw01029-o001_20240716t172128_image3_00001_asn.json') - - args = ['jwst.resample.ResampleStep', rtdata.input] - with pytest.raises(OutputTooLargeError): - Step.from_cmdline(args) diff --git a/jwst/resample/resample.py b/jwst/resample/resample.py index 72b325cbd3..673c4c7f07 100644 --- a/jwst/resample/resample.py +++ b/jwst/resample/resample.py @@ -4,12 +4,10 @@ import json import numpy as np -import psutil from drizzle.resample import Drizzle from spherical_geometry.polygon import SphericalPolygon from stdatamodels.jwst import datamodels -from stdatamodels.jwst.library.basic_utils import bytes2human from jwst.datamodels import ModelLibrary from jwst.associations.asn_from_list import asn_from_list @@ -20,11 +18,7 @@ log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) -__all__ = ["OutputTooLargeError", "ResampleData"] - - -class OutputTooLargeError(RuntimeError): - """Raised when the output is too large for in-memory instantiation""" +__all__ = ["ResampleData"] class ResampleData: @@ -154,32 +148,6 @@ def __init__(self, input_models, output=None, single=False, blendheaders=True, log.debug(f"Output mosaic size: {tuple(self.output_wcs.pixel_shape)}") - allowed_memory = kwargs['allowed_memory'] - if allowed_memory is None: - allowed_memory = os.environ.get('DMODEL_ALLOWED_MEMORY', allowed_memory) - if allowed_memory: - allowed_memory = float(allowed_memory) - # make a small image model to get the dtype - dtype = datamodels.ImageModel((1, 1)).data.dtype - - # get the available memory - available_memory = psutil.virtual_memory().available + psutil.swap_memory().total - - # compute the output array size - required_memory = np.prod(self.output_array_shape) * dtype.itemsize - - # compare used to available - used_fraction = required_memory / available_memory - can_allocate = used_fraction <= allowed_memory - else: - can_allocate = True - - if not can_allocate: - raise OutputTooLargeError( - f'Combined ImageModel size {self.output_array_shape} ' - f'requires {bytes2human(required_memory)}. ' - f'Model cannot be instantiated.' - ) def _create_output_model(self, ref_input_model=None): """ Create a new blank model and update it's meta with info from ``ref_input_model``. """ diff --git a/jwst/resample/resample_step.py b/jwst/resample/resample_step.py index 4132850918..0909227bbb 100755 --- a/jwst/resample/resample_step.py +++ b/jwst/resample/resample_step.py @@ -52,7 +52,6 @@ class ResampleStep(Step): output_wcs = string(default='') # Custom output WCS single = boolean(default=False) # Resample each input to its own output grid blendheaders = boolean(default=True) # Blend metadata from inputs into output - allowed_memory = float(default=None) # Fraction of memory to use for the combined image in_memory = boolean(default=True) # Keep images in memory """ @@ -234,7 +233,6 @@ def get_drizpars(self): good_bits=GOOD_BITS, single=self.single, blendheaders=self.blendheaders, - allowed_memory=self.allowed_memory, in_memory=self.in_memory ) diff --git a/pyproject.toml b/pyproject.toml index 155e2ea531..5c487d65b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,6 @@ dependencies = [ "numpy>=1.22,<2.0", "opencv-python-headless>=4.6.0.66", "photutils>=1.5.0", - "psutil>=5.7.2", "poppy>=1.0.2", "pyparsing>=2.2.1", "requests>=2.22",