JP-3742: Remove allowed_memory and associated environment variable (s…

…pacetelescope#8975)

changes/8975.resample.rst

@@ -0,0 +1 @@
+Removed allowed_memory parameter and DMODEL_ALLOWED_MEMORY environment variable

docs/jwst/outlier_detection/arguments.rst

@@ -79,15 +79,6 @@ Step Arguments for Imaging and Slit-like Spectroscopic data
   Specifies whether or not to resample the input images when
   performing outlier detection.
 
-``--allowed_memory``
-  Specifies the fractional amount of
-  free memory to allow when creating the resampled image. If ``None``, the
-  environment variable ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no
-  check is made. If the resampled image would be larger than specified, an
-  ``OutputTooLargeError`` exception will be generated.
-  For example, if set to ``0.5``, only resampled images that use less than half
-  the available memory can be created.
-
 ``--in_memory``
   Specifies whether or not to load and create all images that are used during
   processing into memory. If ``False``, input files are loaded from disk when

docs/jwst/resample/arguments.rst

@@ -108,15 +108,6 @@ image.
 ``--blendheaders`` (bool, default=True)
     Blend metadata from all input images into the resampled output image.
 
-``--allowed_memory`` (float, default=None)
-    Specifies the fractional amount of free memory to allow when creating the
-    resampled image. If ``None``, the environment variable
-    ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no check is made. If the
-    resampled image would be larger than specified, an ``OutputTooLargeError``
-    exception will be generated.
-
-    For example, if set to ``0.5``, only resampled images that use less than
-    half the available memory can be created.
 
 ``--in_memory`` (boolean, default=True)
   Specifies whether or not to load and create all images that are used during

jwst/outlier_detection/imaging.py

@@ -35,7 +35,6 @@ def detect_outliers(
     pixfrac,
     kernel,
     fillval,
-    allowed_memory,
     in_memory,
     make_output_path,
 ):
@@ -65,7 +64,6 @@ def detect_outliers(
             kernel=kernel,
             fillval=fillval,
             good_bits=good_bits,
-            allowed_memory=allowed_memory,
         )
         median_data, median_wcs = median_with_resampling(input_models,
                                                     resamp,

jwst/outlier_detection/outlier_detection_step.py

@@ -59,7 +59,6 @@ class OutlierDetectionStep(Step):
         resample_data = boolean(default=True)
         good_bits = string(default="~DO_NOT_USE")  # DQ flags to allow
         search_output_file = boolean(default=False)
-        allowed_memory = float(default=None)  # Fraction of memory to use for the combined image
         in_memory = boolean(default=False) # ignored if run within the pipeline; set at pipeline level instead
     """
 
@@ -113,7 +112,6 @@ def process(self, input_data):
                 self.pixfrac,
                 self.kernel,
                 self.fillval,
-                self.allowed_memory,
                 self.in_memory,
                 self.make_output_path,
             )

jwst/outlier_detection/tests/test_outlier_detection.py

@@ -677,6 +677,5 @@ def make_resamp(input_models):
         good_bits="~DO_NOT_USE",
         in_memory=in_memory,
         asn_id="test",
-        allowed_memory=None,
     )
     return resamp

jwst/regtest/test_fgs_image3.py

@@ -1,7 +1,6 @@
 import pytest
 from astropy.io.fits.diff import FITSDiff
 
-from jwst.resample.resample import OutputTooLargeError
 from jwst.stpipe import Step
 
 
@@ -34,23 +33,3 @@ def test_fgs_image3_catalog(run_fgs_image3, rtdata_module, diff_astropy_tables):
     rtdata.get_truth("truth/test_fgs_image3/jw01029-o001_t009_fgs_clear_cat.ecsv")
 
     assert diff_astropy_tables(rtdata.output, rtdata.truth, rtol=1e-3, atol=1e-4)
-
-
-@pytest.mark.bigdata
-def test_fgs_toobig(rtdata, fitsdiff_default_kwargs, caplog, monkeypatch):
-    """Test for the situation where the combined mosaic is too large"""
-
-    # Set the environment to not allow the resultant too-large image.
-    # Note: this test was originally run on two pre-flight images
-    # with WCSs from very different parts of the sky.
-    # This condition should hopefully never be encountered in reductions
-    # of in-flight data.  To test the software failsafe, we now use real data
-    # that makes a reasonable sized mosaic, but set the allowed memory to a
-    # small value.
-    monkeypatch.setenv('DMODEL_ALLOWED_MEMORY', "0.0001")
-
-    rtdata.get_asn('fgs/image3/jw01029-o001_20240716t172128_image3_00001_asn.json')
-
-    args = ['jwst.resample.ResampleStep', rtdata.input]
-    with pytest.raises(OutputTooLargeError):
-        Step.from_cmdline(args)

jwst/resample/resample.py

@@ -4,12 +4,10 @@
 import json
 
 import numpy as np
-import psutil
 from drizzle.resample import Drizzle
 from spherical_geometry.polygon import SphericalPolygon
 
 from stdatamodels.jwst import datamodels
-from stdatamodels.jwst.library.basic_utils import bytes2human
 
 from jwst.datamodels import ModelLibrary
 from jwst.associations.asn_from_list import asn_from_list
@@ -20,11 +18,7 @@
 log = logging.getLogger(__name__)
 log.setLevel(logging.DEBUG)
 
-__all__ = ["OutputTooLargeError", "ResampleData"]
-
-
-class OutputTooLargeError(RuntimeError):
-    """Raised when the output is too large for in-memory instantiation"""
+__all__ = ["ResampleData"]
 
 
 class ResampleData:
@@ -154,32 +148,6 @@ def __init__(self, input_models, output=None, single=False, blendheaders=True,
 
         log.debug(f"Output mosaic size: {tuple(self.output_wcs.pixel_shape)}")
 
-        allowed_memory = kwargs['allowed_memory']
-        if allowed_memory is None:
-            allowed_memory = os.environ.get('DMODEL_ALLOWED_MEMORY', allowed_memory)
-        if allowed_memory:
-            allowed_memory = float(allowed_memory)
-            # make a small image model to get the dtype
-            dtype = datamodels.ImageModel((1, 1)).data.dtype
-
-            # get the available memory
-            available_memory = psutil.virtual_memory().available + psutil.swap_memory().total
-
-            # compute the output array size
-            required_memory = np.prod(self.output_array_shape) * dtype.itemsize
-
-            # compare used to available
-            used_fraction = required_memory / available_memory
-            can_allocate = used_fraction <= allowed_memory
-        else:
-            can_allocate = True
-
-        if not can_allocate:
-            raise OutputTooLargeError(
-                f'Combined ImageModel size {self.output_array_shape} '
-                f'requires {bytes2human(required_memory)}. '
-                f'Model cannot be instantiated.'
-            )
 
     def _create_output_model(self, ref_input_model=None):
         """ Create a new blank model and update it's meta with info from ``ref_input_model``. """

jwst/resample/resample_step.py

@@ -52,7 +52,6 @@ class ResampleStep(Step):
         output_wcs = string(default='')  # Custom output WCS
         single = boolean(default=False)  # Resample each input to its own output grid
         blendheaders = boolean(default=True)  # Blend metadata from inputs into output
-        allowed_memory = float(default=None)  # Fraction of memory to use for the combined image
         in_memory = boolean(default=True)  # Keep images in memory
     """
 
@@ -234,7 +233,6 @@ def get_drizpars(self):
             good_bits=GOOD_BITS,
             single=self.single,
             blendheaders=self.blendheaders,
-            allowed_memory=self.allowed_memory,
             in_memory=self.in_memory
         )
 

pyproject.toml

@@ -27,7 +27,6 @@ dependencies = [
     "numpy>=1.22,<2.0",
     "opencv-python-headless>=4.6.0.66",
     "photutils>=1.5.0",
-    "psutil>=5.7.2",
     "poppy>=1.0.2",
     "pyparsing>=2.2.1",
     "requests>=2.22",