From e82661010ec2110393487c0ab76dcf02ff983f26 Mon Sep 17 00:00:00 2001 From: David Law Date: Wed, 27 Nov 2024 13:26:32 -0500 Subject: [PATCH 1/5] Change prior-group flagging to DO_NOT_USE instead of SATURATION --- src/stcal/saturation/saturation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/stcal/saturation/saturation.py b/src/stcal/saturation/saturation.py index c607c64a..5f97cadc 100644 --- a/src/stcal/saturation/saturation.py +++ b/src/stcal/saturation/saturation.py @@ -133,11 +133,11 @@ def flag_saturated_pixels( # Reset flag array to only pixels passing this gauntlet flagarray[:] = 0 - flagarray[indx] = 2 + flagarray[indx] = dnu # Grow the newly-flagged saturating pixels if n_pix_grow_sat > 0: - flagarray = adjacent_pixels(flagarray, saturated, n_pix_grow_sat) + flagarray = adjacent_pixels(flagarray, dnu, n_pix_grow_sat) # Add them to the gdq array np.bitwise_or(gdq[ints, group, :, :], flagarray, gdq[ints, group, :, :]) @@ -166,10 +166,10 @@ def flag_saturated_pixels( # Flag the 2nd group for the pixels passing that gauntlet flagarray = np.zeros_like(mask,dtype='uint8') - flagarray[mask] = saturated + flagarray[mask] = dnu # flag any pixels that border these new pixels if n_pix_grow_sat > 0: - flagarray = adjacent_pixels(flagarray, saturated, n_pix_grow_sat) + flagarray = adjacent_pixels(flagarray, dnu, n_pix_grow_sat) # Add them to the gdq array np.bitwise_or(gdq[ints, 1, :, :], flagarray, gdq[ints, 1, :, :]) From 0089d93354dd7336b5c8c747c3d717f4858a0138 Mon Sep 17 00:00:00 2001 From: David Law Date: Wed, 27 Nov 2024 14:40:24 -0500 Subject: [PATCH 2/5] Fix boundary issue --- src/stcal/saturation/saturation.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/stcal/saturation/saturation.py b/src/stcal/saturation/saturation.py index 5f97cadc..dcccf0ec 100644 --- a/src/stcal/saturation/saturation.py +++ b/src/stcal/saturation/saturation.py @@ -167,9 +167,6 @@ def flag_saturated_pixels( # Flag the 2nd group for the pixels passing that gauntlet flagarray = np.zeros_like(mask,dtype='uint8') flagarray[mask] = dnu - # flag any pixels that border these new pixels - if n_pix_grow_sat > 0: - flagarray = adjacent_pixels(flagarray, dnu, n_pix_grow_sat) # Add them to the gdq array np.bitwise_or(gdq[ints, 1, :, :], flagarray, gdq[ints, 1, :, :]) From a6646a5bad8dce523e02404c9e3ecb0c589485d4 Mon Sep 17 00:00:00 2001 From: David Law Date: Wed, 27 Nov 2024 14:44:28 -0500 Subject: [PATCH 3/5] Add change log entry --- changes/321.saturation.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/321.saturation.rst diff --git a/changes/321.saturation.rst b/changes/321.saturation.rst new file mode 100644 index 00000000..cc087d1f --- /dev/null +++ b/changes/321.saturation.rst @@ -0,0 +1 @@ +Change flagging of 'pre-saturation' grouped data to use DO_NOT_USE instead of SATURATION flag to avoid confusing the snowball routine downstream. From 36d1cc6eb82fbe5c06197be7d567ba3b2e6e6b41 Mon Sep 17 00:00:00 2001 From: David Law Date: Wed, 27 Nov 2024 15:17:01 -0500 Subject: [PATCH 4/5] Move change log entry --- changes/{321.saturation.rst => 321.bugfix.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{321.saturation.rst => 321.bugfix.rst} (100%) diff --git a/changes/321.saturation.rst b/changes/321.bugfix.rst similarity index 100% rename from changes/321.saturation.rst rename to changes/321.bugfix.rst From 1b49a43b80de861d4ab3476313694eaebc3efafb Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Mon, 16 Dec 2024 14:54:10 -0500 Subject: [PATCH 5/5] xfail unit test, add coverage for this change --- tests/test_saturation.py | 42 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/test_saturation.py b/tests/test_saturation.py index cc1465a3..a3869600 100644 --- a/tests/test_saturation.py +++ b/tests/test_saturation.py @@ -4,7 +4,7 @@ """ from enum import IntEnum - +import pytest import numpy as np from stcal.saturation.saturation import flag_saturated_pixels @@ -40,6 +40,7 @@ def test_basic_saturation_flagging(): assert np.all(gdq[0, satindex:, 5, 5] == DQFLAGS["SATURATED"]) +@pytest.mark.xfail(reason="stcal PR#321 broke this test") def test_read_pattern_saturation_flagging(): """Check that the saturation threshold varies depending on how the reads are allocated into resultants.""" @@ -78,6 +79,45 @@ def test_read_pattern_saturation_flagging(): assert np.all(gdq[0, 2:, 5, 5] == DQFLAGS["SATURATED"]) +def test_read_pattern_saturation_flagging_dnu(): + """Check that the saturation threshold varies depending on how the reads + are allocated into resultants. First resultant expected to have DNU while + PR#321 band-aid still in place.""" + + # Create inputs, data, and saturation maps + data = np.zeros((1, 5, 20, 20)).astype("float32") + gdq = np.zeros((1, 5, 20, 20)).astype("uint32") + pdq = np.zeros((20, 20)).astype("uint32") + sat_thresh = np.ones((20, 20)) * 100000.0 + sat_dq = np.zeros((20, 20)).astype("uint32") + + # Add ramp values up to the saturation limit + data[0, 0, 5, 5] = 0 + data[0, 1, 5, 5] = 20000 + data[0, 2, 5, 5] = 40000 + data[0, 3, 5, 5] = 60000 # Signal reaches saturation limit + data[0, 4, 5, 5] = 62000 + + # Set saturation value in the saturation model + satvalue = 60000 + sat_thresh[5, 5] = satvalue + + # set read_pattern to have many reads in the third resultant, so that + # its mean exposure time is much smaller than its last read time + # (in this case, the ratio is 13 / 20). + # This means that the effective saturation for the third resultant + # is 60000 * 13 / 20 = 39000 and the third resultant should be marked + # saturated. + read_pattern = [[1], [2], [3, 4, 5, 6, 7, 8, 9, 10], [11], [12], [13]] + + gdq, pdq, _ = flag_saturated_pixels( + data, gdq, pdq, sat_thresh, sat_dq, ATOD_LIMIT, DQFLAGS, read_pattern=read_pattern + ) + + # Make sure that groups after the third get flagged + assert np.all(gdq[0, 2:, 5, 5] == [DQFLAGS["DO_NOT_USE"], DQFLAGS["SATURATED"], DQFLAGS["SATURATED"]]) + + def test_no_sat_check_at_limit(): """Test to verify that pixels at the A-to-D limit (65535), but flagged with NO_SAT_CHECK do NOT get flagged as saturated, and that their neighbors