From 5299646bd96ccce9322cda441d9046ba3d65623f Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Wed, 30 Oct 2024 09:04:37 -0700 Subject: [PATCH] AL-875: Add memory saving options to compute_weight_threshold sigma_clip call (#312) * Added memory-saving options to compute_weight_threshold sigma clip --- src/stcal/outlier_detection/utils.py | 7 ++++++- tests/outlier_detection/test_utils.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/stcal/outlier_detection/utils.py b/src/stcal/outlier_detection/utils.py index bc0485f3..88d27ce0 100644 --- a/src/stcal/outlier_detection/utils.py +++ b/src/stcal/outlier_detection/utils.py @@ -77,7 +77,12 @@ def compute_weight_threshold(weight, maskpt): weight_masked = np.ma.array(weight, mask=np.logical_or( mask_zero_weight, mask_nans)) # Sigma-clip the unmasked data - weight_masked = sigma_clip(weight_masked, sigma=3, maxiters=5) + weight_masked = sigma_clip(weight_masked, + sigma=3, + maxiters=5, + masked=False, + copy=False, + ) mean_weight = np.mean(weight_masked) # Mask pixels where weight falls below maskpt percent weight_threshold = mean_weight * maskpt diff --git a/tests/outlier_detection/test_utils.py b/tests/outlier_detection/test_utils.py index 0a59f3c2..4112391b 100644 --- a/tests/outlier_detection/test_utils.py +++ b/tests/outlier_detection/test_utils.py @@ -16,6 +16,7 @@ reproject, medfilt, ) +from stcal.testing_helpers import MemoryThreshold @pytest.mark.parametrize("shape,diff", [ @@ -72,6 +73,23 @@ def test_compute_weight_threshold_zeros(): np.testing.assert_allclose(result, 21) +def test_compute_weight_threshold_memory(): + """Test that weight threshold function modifies + the weight array in place""" + arr = np.zeros([500, 500], dtype=np.float32) + arr[:250, :250] = 42 + arr[10,10] = 0 + arr[-10,-10] = np.nan + + # buffer to account for memory overhead needs to be small enough + # to ensure that the array was not copied + fractional_memory_buffer = 1.9 + expected_mem = int(arr.nbytes*fractional_memory_buffer) + with MemoryThreshold(str(expected_mem) + " B"): + result = compute_weight_threshold(arr, 0.5) + np.testing.assert_allclose(result, 21) + + def test_flag_crs(): sci = np.zeros((10, 10), dtype=np.float32) err = np.ones_like(sci)