From dfcd6e6c1f11852d5548a687f196fa46d9c38dde Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Tue, 7 Nov 2023 14:09:32 -0500 Subject: [PATCH 01/30] Start moving ramp related stuff to its own module. --- setup.py | 6 ++ src/stcal/ramp_fitting/ols_cas22/_core.pxd | 9 +-- src/stcal/ramp_fitting/ols_cas22/_core.pyx | 61 ----------------- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 6 +- src/stcal/ramp_fitting/ols_cas22/_pixel.pxd | 6 +- src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 5 +- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 14 ++++ src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 68 +++++++++++++++++++ tests/test_jump_cas22.py | 3 +- 9 files changed, 102 insertions(+), 76 deletions(-) create mode 100644 src/stcal/ramp_fitting/ols_cas22/_ramp.pxd create mode 100644 src/stcal/ramp_fitting/ols_cas22/_ramp.pyx diff --git a/setup.py b/setup.py index b34e7dfa..3224c493 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,12 @@ include_dirs=[np.get_include()], language='c++' ), + Extension( + 'stcal.ramp_fitting.ols_cas22._ramp', + ['src/stcal/ramp_fitting/ols_cas22/_ramp.pyx'], + include_dirs=[np.get_include()], + language='c++' + ), Extension( 'stcal.ramp_fitting.ols_cas22._fit_ramps', ['src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx'], diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pxd b/src/stcal/ramp_fitting/ols_cas22/_core.pxd index 641c206e..6517576e 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pxd @@ -1,9 +1,5 @@ from libcpp.vector cimport vector - - -cdef struct RampIndex: - int start - int end +from stcal.ramp_fitting.ols_cas22._ramp cimport RampQueue cdef struct RampFit: @@ -16,7 +12,7 @@ cdef struct RampFits: RampFit average vector[int] jumps vector[RampFit] fits - vector[RampIndex] index + RampQueue index cdef struct ReadPatternMetadata: @@ -52,5 +48,4 @@ cpdef enum RampJumpDQ: cpdef float threshold(Thresh thresh, float slope) cdef float get_power(float s) -cpdef vector[RampIndex] init_ramps(int[:, :] dq, int n_resultants, int index_pixel) cpdef ReadPatternMetadata metadata_from_read_pattern(list[list[int]] read_pattern, float read_time) diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pyx b/src/stcal/ramp_fitting/ols_cas22/_core.pyx index 0d312261..104cd6ca 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pyx @@ -134,67 +134,6 @@ cpdef inline float threshold(Thresh thresh, float slope): return thresh.intercept - thresh.constant * log10(slope) -@cython.boundscheck(False) -@cython.wraparound(False) -cpdef inline vector[RampIndex] init_ramps(int[:, :] dq, int n_resultants, int index_pixel): - """ - Create the initial ramp stack for each pixel - if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp - otherwise, the resultant is not in a ramp - - Parameters - ---------- - dq : int[n_resultants, n_pixel] - DQ array - n_resultants : int - Number of resultants - index_pixel : int - The index of the pixel to find ramps for - - Returns - ------- - vector of RampIndex objects - - vector with entry for each ramp found (last entry is last ramp found) - - RampIndex with start and end indices of the ramp in the resultants - """ - cdef vector[RampIndex] ramps = vector[RampIndex]() - - # Note: if start/end are -1, then no value has been assigned - # ramp.start == -1 means we have not started a ramp - # dq[index_resultant, index_pixel] == 0 means resultant is in ramp - cdef RampIndex ramp = RampIndex(-1, -1) - for index_resultant in range(n_resultants): - if ramp.start == -1: - # Looking for the start of a ramp - if dq[index_resultant, index_pixel] == 0: - # We have found the start of a ramp! - ramp.start = index_resultant - else: - # This is not the start of the ramp yet - continue - else: - # Looking for the end of a ramp - if dq[index_resultant, index_pixel] == 0: - # This pixel is in the ramp do nothing - continue - else: - # This pixel is not in the ramp - # => index_resultant - 1 is the end of the ramp - ramp.end = index_resultant - 1 - - # Add completed ramp to stack and reset ramp - ramps.push_back(ramp) - ramp = RampIndex(-1, -1) - - # Handle case where last resultant is in ramp (so no end has been set) - if ramp.start != -1 and ramp.end == -1: - # Last resultant is end of the ramp => set then add to stack - ramp.end = n_resultants - 1 - ramps.push_back(ramp) - - return ramps - - @cython.boundscheck(False) @cython.wraparound(False) cpdef ReadPatternMetadata metadata_from_read_pattern(list[list[int]] read_pattern, float read_time): diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index 4b2e39e3..cf3d74e1 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -4,12 +4,14 @@ from libcpp cimport bool from libcpp.list cimport list as cpp_list cimport cython -from stcal.ramp_fitting.ols_cas22._core cimport (RampFits, RampIndex, Thresh, - metadata_from_read_pattern, init_ramps, +from stcal.ramp_fitting.ols_cas22._core cimport (RampFits, Thresh, + metadata_from_read_pattern, Parameter, Variance, RampJumpDQ) from stcal.ramp_fitting.ols_cas22._fixed cimport fixed_values_from_metadata, FixedValues from stcal.ramp_fitting.ols_cas22._pixel cimport make_pixel +from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps + from typing import NamedTuple, Optional diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd index 1539724b..219e2759 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd @@ -1,8 +1,8 @@ from libcpp cimport bool -from libcpp.vector cimport vector -from stcal.ramp_fitting.ols_cas22._core cimport RampFit, RampFits, RampIndex +from stcal.ramp_fitting.ols_cas22._core cimport RampFit, RampFits from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues +from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue cdef class Pixel: cdef FixedValues fixed @@ -18,7 +18,7 @@ cdef class Pixel: cdef float correction(Pixel self, RampIndex ramp, float slope) cdef float stat(Pixel self, float slope, RampIndex ramp, int index, int diff) cdef float[:] stats(Pixel self, float slope, RampIndex ramp) - cdef RampFits fit_ramps(Pixel self, vector[RampIndex] ramps, bool include_diagnostic) + cdef RampFits fit_ramps(Pixel self, RampQueue ramps, bool include_diagnostic) cpdef Pixel make_pixel(FixedValues fixed, float read_noise, float [:] resultants) diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx index ddf04971..bfe6b63d 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx @@ -23,9 +23,10 @@ cimport numpy as np cimport cython -from stcal.ramp_fitting.ols_cas22._core cimport get_power, threshold, RampFit, RampFits, RampIndex, Diff +from stcal.ramp_fitting.ols_cas22._core cimport get_power, threshold, RampFit, RampFits, Diff from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel +from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue cdef class Pixel: @@ -330,7 +331,7 @@ cdef class Pixel: @cython.boundscheck(False) @cython.wraparound(False) @cython.cdivision(True) - cdef inline RampFits fit_ramps(Pixel self, vector[RampIndex] ramps, bool include_diagnostic): + cdef inline RampFits fit_ramps(Pixel self, RampQueue ramps, bool include_diagnostic): """ Compute all the ramps for a single pixel using the Casertano+22 algorithm with jump detection. diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd new file mode 100644 index 00000000..cad21197 --- /dev/null +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -0,0 +1,14 @@ +cimport numpy as cnp + +from libcpp.vector cimport vector + + +cdef struct RampIndex: + cnp.int32_t start + cnp.int32_t end + + +ctypedef vector[RampIndex] RampQueue + + +cpdef RampQueue init_ramps(cnp.int32_t[:, :] dq, cnp.int32_t n_resultants, cnp.int32_t index_pixel) diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx new file mode 100644 index 00000000..7eeb29ed --- /dev/null +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -0,0 +1,68 @@ +cimport cython +cimport numpy as cnp + +from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue + + +cnp.import_array() + +@cython.boundscheck(False) +@cython.wraparound(False) +cpdef inline RampQueue init_ramps(cnp.int32_t[:, :] dq, cnp.int32_t n_resultants, cnp.int32_t index_pixel): + """ + Create the initial ramp stack for each pixel + if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp + otherwise, the resultant is not in a ramp + + Parameters + ---------- + dq : int[n_resultants, n_pixel] + DQ array + n_resultants : int + Number of resultants + index_pixel : int + The index of the pixel to find ramps for + + Returns + ------- + vector of RampIndex objects + - vector with entry for each ramp found (last entry is last ramp found) + - RampIndex with start and end indices of the ramp in the resultants + """ + cdef RampQueue ramps = RampQueue() + + # Note: if start/end are -1, then no value has been assigned + # ramp.start == -1 means we have not started a ramp + # dq[index_resultant, index_pixel] == 0 means resultant is in ramp + cdef RampIndex ramp = RampIndex(-1, -1) + cdef int index_resultant + for index_resultant in range(n_resultants): + if ramp.start == -1: + # Looking for the start of a ramp + if dq[index_resultant, index_pixel] == 0: + # We have found the start of a ramp! + ramp.start = index_resultant + else: + # This is not the start of the ramp yet + continue + else: + # Looking for the end of a ramp + if dq[index_resultant, index_pixel] == 0: + # This pixel is in the ramp do nothing + continue + else: + # This pixel is not in the ramp + # => index_resultant - 1 is the end of the ramp + ramp.end = index_resultant - 1 + + # Add completed ramp to stack and reset ramp + ramps.push_back(ramp) + ramp = RampIndex(-1, -1) + + # Handle case where last resultant is in ramp (so no end has been set) + if ramp.start != -1 and ramp.end == -1: + # Last resultant is end of the ramp => set then add to stack + ramp.end = n_resultants - 1 + ramps.push_back(ramp) + + return ramps \ No newline at end of file diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 0bf1b6ed..284b050f 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,9 +2,10 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22._core import init_ramps, metadata_from_read_pattern, threshold +from stcal.ramp_fitting.ols_cas22._core import metadata_from_read_pattern, threshold from stcal.ramp_fitting.ols_cas22._fixed import fixed_values_from_metadata from stcal.ramp_fitting.ols_cas22._pixel import make_pixel +from stcal.ramp_fitting.ols_cas22._ramp import init_ramps from stcal.ramp_fitting.ols_cas22 import fit_ramps, Parameter, Variance, Diff, RampJumpDQ From 01c535d144df45e70d87fccad01192ec438b94db Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Tue, 7 Nov 2023 14:25:34 -0500 Subject: [PATCH 02/30] Begin moving jump related items into module --- setup.py | 6 +++++ src/stcal/ramp_fitting/ols_cas22/_core.pxd | 6 ----- src/stcal/ramp_fitting/ols_cas22/_core.pyx | 23 +--------------- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 5 ++-- src/stcal/ramp_fitting/ols_cas22/_fixed.pxd | 3 ++- src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 3 ++- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 9 +++++++ src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 27 +++++++++++++++++++ src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 3 ++- tests/test_jump_cas22.py | 3 ++- 10 files changed, 54 insertions(+), 34 deletions(-) create mode 100644 src/stcal/ramp_fitting/ols_cas22/_jump.pxd create mode 100644 src/stcal/ramp_fitting/ols_cas22/_jump.pyx diff --git a/setup.py b/setup.py index 3224c493..809ad21a 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,12 @@ include_dirs=[np.get_include()], language='c++' ), + Extension( + 'stcal.ramp_fitting.ols_cas22._jump', + ['src/stcal/ramp_fitting/ols_cas22/_jump.pyx'], + include_dirs=[np.get_include()], + language='c++' + ), Extension( 'stcal.ramp_fitting.ols_cas22._ramp', ['src/stcal/ramp_fitting/ols_cas22/_ramp.pyx'], diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pxd b/src/stcal/ramp_fitting/ols_cas22/_core.pxd index 6517576e..e504193f 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pxd @@ -21,11 +21,6 @@ cdef struct ReadPatternMetadata: vector[int] n_reads -cdef struct Thresh: - float intercept - float constant - - cpdef enum Diff: single = 0 double = 1 @@ -46,6 +41,5 @@ cpdef enum RampJumpDQ: JUMP_DET = 4 -cpdef float threshold(Thresh thresh, float slope) cdef float get_power(float s) cpdef ReadPatternMetadata metadata_from_read_pattern(list[list[int]] read_pattern, float read_time) diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pyx b/src/stcal/ramp_fitting/ols_cas22/_core.pyx index 104cd6ca..e087e4b3 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pyx @@ -81,7 +81,7 @@ import numpy as np cimport numpy as np cimport cython -from stcal.ramp_fitting.ols_cas22._core cimport Thresh, ReadPatternMetadata +from stcal.ramp_fitting.ols_cas22._core cimport ReadPatternMetadata # Casertano+2022, Table 2 @@ -113,27 +113,6 @@ cdef inline float get_power(float signal): return PTABLE[1][i] -cpdef inline float threshold(Thresh thresh, float slope): - """ - Compute jump threshold - - Parameters - ---------- - thresh : Thresh - threshold parameters struct - slope : float - slope of the ramp in question - - Returns - ------- - intercept - constant * log10(slope) - """ - # clip slope in 1, 1e4 - slope = slope if slope > 1 else 1 - slope = slope if slope < 1e4 else 1e4 - return thresh.intercept - thresh.constant * log10(slope) - - @cython.boundscheck(False) @cython.wraparound(False) cpdef ReadPatternMetadata metadata_from_read_pattern(list[list[int]] read_pattern, float read_time): diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index cf3d74e1..1532bb6a 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -4,12 +4,13 @@ from libcpp cimport bool from libcpp.list cimport list as cpp_list cimport cython -from stcal.ramp_fitting.ols_cas22._core cimport (RampFits, Thresh, +from stcal.ramp_fitting.ols_cas22._core cimport (RampFits, metadata_from_read_pattern, Parameter, Variance, RampJumpDQ) from stcal.ramp_fitting.ols_cas22._fixed cimport fixed_values_from_metadata, FixedValues from stcal.ramp_fitting.ols_cas22._pixel cimport make_pixel +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps from typing import NamedTuple, Optional @@ -102,7 +103,7 @@ def fit_ramps(np.ndarray[float, ndim=2] resultants, # Pre-compute data for all pixels cdef FixedValues fixed = fixed_values_from_metadata(metadata_from_read_pattern(read_pattern, read_time), - Thresh(intercept, constant), + Thresh(np.float32(intercept), np.float32(constant)), use_jump) # Use list because this might grow very large which would require constant diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd index 72087051..5dfec4d2 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd @@ -1,6 +1,7 @@ from libcpp cimport bool -from stcal.ramp_fitting.ols_cas22._core cimport Thresh, ReadPatternMetadata +from stcal.ramp_fitting.ols_cas22._core cimport ReadPatternMetadata +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh cdef class FixedValues: diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx index 8417507b..a84ea815 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx @@ -20,8 +20,9 @@ cimport numpy as np cimport cython from libcpp cimport bool -from stcal.ramp_fitting.ols_cas22._core cimport Thresh, ReadPatternMetadata, Diff +from stcal.ramp_fitting.ols_cas22._core cimport ReadPatternMetadata, Diff from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh cdef class FixedValues: """ diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd new file mode 100644 index 00000000..a1674123 --- /dev/null +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -0,0 +1,9 @@ +cimport numpy as cnp + + +cdef struct Thresh: + cnp.float32_t intercept + cnp.float32_t constant + + +cpdef cnp.float32_t threshold(Thresh thresh, cnp.float32_t slope) \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx new file mode 100644 index 00000000..32e7d522 --- /dev/null +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -0,0 +1,27 @@ +import numpy as np +cimport numpy as cnp + +from libc.math cimport log10 + + +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh + +cpdef inline cnp.float32_t threshold(Thresh thresh, cnp.float32_t slope): + """ + Compute jump threshold + + Parameters + ---------- + thresh : Thresh + threshold parameters struct + slope : float + slope of the ramp in question + + Returns + ------- + intercept - constant * log10(slope) + """ + slope = slope if slope > 1 else np.float32(1) + slope = slope if slope < 1e4 else np.float32(1e4) + + return thresh.intercept - thresh.constant * log10(slope) \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx index bfe6b63d..6200ff10 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx @@ -23,8 +23,9 @@ cimport numpy as np cimport cython -from stcal.ramp_fitting.ols_cas22._core cimport get_power, threshold, RampFit, RampFits, Diff +from stcal.ramp_fitting.ols_cas22._core cimport get_power, RampFit, RampFits, Diff from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues +from stcal.ramp_fitting.ols_cas22._jump cimport threshold from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 284b050f..fcbb9ca1 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,8 +2,9 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22._core import metadata_from_read_pattern, threshold +from stcal.ramp_fitting.ols_cas22._core import metadata_from_read_pattern from stcal.ramp_fitting.ols_cas22._fixed import fixed_values_from_metadata +from stcal.ramp_fitting.ols_cas22._jump import threshold from stcal.ramp_fitting.ols_cas22._pixel import make_pixel from stcal.ramp_fitting.ols_cas22._ramp import init_ramps From b2fff9f6c810e2a310eb6e84034413cdf45bce50 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Tue, 7 Nov 2023 15:17:18 -0500 Subject: [PATCH 03/30] Move read pattern stuff to its own module --- setup.py | 6 + src/stcal/ramp_fitting/ols_cas22/_core.pxd | 7 - src/stcal/ramp_fitting/ols_cas22/_core.pyx | 45 ----- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 4 +- src/stcal/ramp_fitting/ols_cas22/_fixed.pxd | 10 +- src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 182 ++++++++---------- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 9 +- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 7 +- src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 31 +-- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 8 +- src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 2 +- .../ramp_fitting/ols_cas22/_read_pattern.pxd | 9 + .../ramp_fitting/ols_cas22/_read_pattern.pyx | 78 ++++++++ tests/test_jump_cas22.py | 83 +++----- 14 files changed, 229 insertions(+), 252 deletions(-) create mode 100644 src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd create mode 100644 src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx diff --git a/setup.py b/setup.py index 809ad21a..05b6a614 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,12 @@ include_dirs=[np.get_include()], language='c++' ), + Extension( + 'stcal.ramp_fitting.ols_cas22._read_pattern', + ['src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx'], + include_dirs=[np.get_include()], + language='c++' + ), Extension( 'stcal.ramp_fitting.ols_cas22._fit_ramps', ['src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx'], diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pxd b/src/stcal/ramp_fitting/ols_cas22/_core.pxd index e504193f..d5790107 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pxd @@ -15,12 +15,6 @@ cdef struct RampFits: RampQueue index -cdef struct ReadPatternMetadata: - vector[float] t_bar - vector[float] tau - vector[int] n_reads - - cpdef enum Diff: single = 0 double = 1 @@ -42,4 +36,3 @@ cpdef enum RampJumpDQ: cdef float get_power(float s) -cpdef ReadPatternMetadata metadata_from_read_pattern(list[list[int]] read_pattern, float read_time) diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pyx b/src/stcal/ramp_fitting/ols_cas22/_core.pyx index e087e4b3..4b0473b2 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pyx @@ -74,15 +74,10 @@ Functions - cpdef gives a python wrapper, but the python version of this method is considered private, only to be used for testing """ -from libcpp.vector cimport vector -from libc.math cimport log10 - import numpy as np cimport numpy as np cimport cython -from stcal.ramp_fitting.ols_cas22._core cimport ReadPatternMetadata - # Casertano+2022, Table 2 cdef float[2][6] PTABLE = [ @@ -111,43 +106,3 @@ cdef inline float get_power(float signal): return PTABLE[1][i - 1] return PTABLE[1][i] - - -@cython.boundscheck(False) -@cython.wraparound(False) -cpdef ReadPatternMetadata metadata_from_read_pattern(list[list[int]] read_pattern, float read_time): - """ - Derive the input data from the the read pattern - - read pattern is a list of resultant lists, where each resultant list is - a list of the reads in that resultant. - - Parameters - ---------- - read pattern: list[list[int]] - read pattern for the image - read_time : float - Time to perform a readout. - - Returns - ------- - ReadPatternMetadata struct: - vector[float] t_bar: mean time of each resultant - vector[float] tau: variance time of each resultant - vector[int] n_reads: number of reads in each resultant - """ - cdef int n_resultants = len(read_pattern) - cdef ReadPatternMetadata data = ReadPatternMetadata(vector[float](n_resultants), - vector[float](n_resultants), - vector[int](n_resultants)) - - cdef int index, n_reads - cdef list[int] resultant - for index, resultant in enumerate(read_pattern): - n_reads = len(resultant) - - data.n_reads[index] = n_reads - data.t_bar[index] = read_time * np.mean(resultant) - data.tau[index] = np.sum((2 * (n_reads - np.arange(n_reads)) - 1) * resultant) * read_time / n_reads**2 - - return data diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index 1532bb6a..c9593fbe 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -5,13 +5,13 @@ from libcpp.list cimport list as cpp_list cimport cython from stcal.ramp_fitting.ols_cas22._core cimport (RampFits, - metadata_from_read_pattern, Parameter, Variance, RampJumpDQ) from stcal.ramp_fitting.ols_cas22._fixed cimport fixed_values_from_metadata, FixedValues from stcal.ramp_fitting.ols_cas22._pixel cimport make_pixel from stcal.ramp_fitting.ols_cas22._jump cimport Thresh from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps +from stcal.ramp_fitting.ols_cas22._read_pattern cimport from_read_pattern from typing import NamedTuple, Optional @@ -102,7 +102,7 @@ def fit_ramps(np.ndarray[float, ndim=2] resultants, f'match number of resultants {n_resultants}') # Pre-compute data for all pixels - cdef FixedValues fixed = fixed_values_from_metadata(metadata_from_read_pattern(read_pattern, read_time), + cdef FixedValues fixed = fixed_values_from_metadata(from_read_pattern(read_pattern, read_time), Thresh(np.float32(intercept), np.float32(constant)), use_jump) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd index 5dfec4d2..041b08e3 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd @@ -1,12 +1,12 @@ from libcpp cimport bool -from stcal.ramp_fitting.ols_cas22._core cimport ReadPatternMetadata from stcal.ramp_fitting.ols_cas22._jump cimport Thresh +from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern cdef class FixedValues: cdef bool use_jump - cdef ReadPatternMetadata data + cdef ReadPattern data cdef Thresh threshold cdef float[:, :] t_bar_diffs @@ -14,9 +14,5 @@ cdef class FixedValues: cdef float[:, :] read_recip_coeffs cdef float[:, :] var_slope_coeffs - cdef float[:, :] t_bar_diff_vals(FixedValues self) - cdef float[:, :] read_recip_vals(FixedValues self) - cdef float[:, :] var_slope_vals(FixedValues self) - -cpdef FixedValues fixed_values_from_metadata(ReadPatternMetadata data, Thresh threshold, bool use_jump) +cpdef FixedValues fixed_values_from_metadata(ReadPattern data, Thresh threshold, bool use_jump) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx index a84ea815..90a9b122 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx @@ -20,9 +20,10 @@ cimport numpy as np cimport cython from libcpp cimport bool -from stcal.ramp_fitting.ols_cas22._core cimport ReadPatternMetadata, Diff +from stcal.ramp_fitting.ols_cas22._core cimport Diff from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues from stcal.ramp_fitting.ols_cas22._jump cimport Thresh +from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern cdef class FixedValues: """ @@ -84,101 +85,6 @@ cdef class FixedValues: from pre-computing the values and reusing them. """ - @cython.boundscheck(False) - @cython.wraparound(False) - cdef inline float[:, :] t_bar_diff_vals(FixedValues self): - """ - Compute the difference offset of t_bar - - Returns - ------- - [ - , - , - ] - """ - # Cast vector to memory view - # This way of doing it is potentially memory unsafe because the memory - # can outlive the vector. However, this is much faster (no copies) and - # much simpler than creating an intermediate wrapper which can pretend - # to be a memory view. In this case, I make sure that the memory view - # stays local to the function (numpy operations create brand new objects) - cdef float[:] t_bar = self.data.t_bar.data() - cdef int end = len(t_bar) - - cdef np.ndarray[float, ndim=2] t_bar_diff_vals = np.zeros((2, self.data.t_bar.size() - 1), dtype=np.float32) - - t_bar_diff_vals[Diff.single, :] = np.subtract(t_bar[1:], t_bar[:end - 1]) - t_bar_diff_vals[Diff.double, :end - 2] = np.subtract(t_bar[2:], t_bar[:end - 2]) - t_bar_diff_vals[Diff.double, end - 2] = np.nan # last double difference is undefined - - return t_bar_diff_vals - - @cython.boundscheck(False) - @cython.wraparound(False) - cdef inline float[:, :] read_recip_vals(FixedValues self): - """ - Compute the reciprical sum of the number of reads - - Returns - ------- - [ - <(1/n_reads[i+1] + 1/n_reads[i])>, - <(1/n_reads[i+2] + 1/n_reads[i])>, - ] - - """ - # Cast vector to memory view - # This way of doing it is potentially memory unsafe because the memory - # can outlive the vector. However, this is much faster (no copies) and - # much simpler than creating an intermediate wrapper which can pretend - # to be a memory view. In this case, I make sure that the memory view - # stays local to the function (numpy operations create brand new objects) - cdef int[:] n_reads = self.data.n_reads.data() - cdef int end = len(n_reads) - - cdef np.ndarray[float, ndim=2] read_recip_vals = np.zeros((2, self.data.n_reads.size() - 1), dtype=np.float32) - - read_recip_vals[Diff.single, :] = (np.divide(1.0, n_reads[1:], dtype=np.float32) + - np.divide(1.0, n_reads[:end - 1], dtype=np.float32)) - read_recip_vals[Diff.double, :end - 2] = (np.divide(1.0, n_reads[2:], dtype=np.float32) + - np.divide(1.0, n_reads[:end - 2], dtype=np.float32)) - read_recip_vals[Diff.double, end - 2] = np.nan # last double difference is undefined - - return read_recip_vals - - - @cython.boundscheck(False) - @cython.wraparound(False) - cdef inline float[:, :] var_slope_vals(FixedValues self): - """ - Compute slope part of the jump statistic variances - - Returns - ------- - [ - <(tau[i] + tau[i+1] - min(t_bar[i], t_bar[i+1])) * correction(i, i+1)>, - <(tau[i] + tau[i+2] - min(t_bar[i], t_bar[i+2])) * correction(i, i+2)>, - ] - """ - # Cast vectors to memory views - # This way of doing it is potentially memory unsafe because the memory - # can outlive the vector. However, this is much faster (no copies) and - # much simpler than creating an intermediate wrapper which can pretend - # to be a memory view. In this case, I make sure that the memory view - # stays local to the function (numpy operations create brand new objects) - cdef float[:] t_bar = self.data.t_bar.data() - cdef float[:] tau = self.data.tau.data() - cdef int end = len(t_bar) - - cdef np.ndarray[float, ndim=2] var_slope_vals = np.zeros((2, self.data.t_bar.size() - 1), dtype=np.float32) - - var_slope_vals[Diff.single, :] = (np.add(tau[1:], tau[:end - 1]) - 2 * np.minimum(t_bar[1:], t_bar[:end - 1])) - var_slope_vals[Diff.double, :end - 2] = (np.add(tau[2:], tau[:end - 2]) - 2 * np.minimum(t_bar[2:], t_bar[:end - 2])) - var_slope_vals[Diff.double, end - 2] = np.nan # last double difference is undefined - - return var_slope_vals - def _to_dict(FixedValues self): """ This is a private method to convert the FixedValues object to a dictionary, @@ -228,7 +134,7 @@ cdef class FixedValues: else: raise AttributeError("var_slope_coeffs should not exist") - return dict(data=self.data, + return dict(data=self.data._to_dict(), threshold=self.threshold, t_bar_diffs=t_bar_diffs, t_bar_diff_sqrs=t_bar_diff_sqrs, @@ -236,7 +142,81 @@ cdef class FixedValues: var_slope_coeffs=var_slope_coeffs) -cpdef inline FixedValues fixed_values_from_metadata(ReadPatternMetadata data, Thresh threshold, bool use_jump): +@cython.boundscheck(False) +@cython.wraparound(False) +cdef inline float[:, :] t_bar_diff_vals(ReadPattern data): + """ + Compute the difference offset of t_bar + + Returns + ------- + [ + , + , + ] + """ + cdef int end = len(data.t_bar) + + cdef np.ndarray[float, ndim=2] t_bar_diff_vals = np.zeros((2, end - 1), dtype=np.float32) + + t_bar_diff_vals[Diff.single, :] = np.subtract(data.t_bar[1:], data.t_bar[:end - 1]) + t_bar_diff_vals[Diff.double, :end - 2] = np.subtract(data.t_bar[2:], data.t_bar[:end - 2]) + t_bar_diff_vals[Diff.double, end - 2] = np.nan # last double difference is undefined + + return t_bar_diff_vals + +@cython.boundscheck(False) +@cython.wraparound(False) +cdef inline float[:, :] read_recip_vals(ReadPattern data): + """ + Compute the reciprical sum of the number of reads + + Returns + ------- + [ + <(1/n_reads[i+1] + 1/n_reads[i])>, + <(1/n_reads[i+2] + 1/n_reads[i])>, + ] + + """ + cdef int end = len(data.n_reads) + + cdef np.ndarray[float, ndim=2] read_recip_vals = np.zeros((2, end - 1), dtype=np.float32) + + read_recip_vals[Diff.single, :] = (np.divide(1.0, data.n_reads[1:], dtype=np.float32) + + np.divide(1.0, data.n_reads[:end - 1], dtype=np.float32)) + read_recip_vals[Diff.double, :end - 2] = (np.divide(1.0, data.n_reads[2:], dtype=np.float32) + + np.divide(1.0, data.n_reads[:end - 2], dtype=np.float32)) + read_recip_vals[Diff.double, end - 2] = np.nan # last double difference is undefined + + return read_recip_vals + + +@cython.boundscheck(False) +@cython.wraparound(False) +cdef inline float[:, :] var_slope_vals(ReadPattern data): + """ + Compute slope part of the jump statistic variances + + Returns + ------- + [ + <(tau[i] + tau[i+1] - min(t_bar[i], t_bar[i+1])) * correction(i, i+1)>, + <(tau[i] + tau[i+2] - min(t_bar[i], t_bar[i+2])) * correction(i, i+2)>, + ] + """ + cdef int end = len(data.t_bar) + + cdef np.ndarray[float, ndim=2] var_slope_vals = np.zeros((2, end - 1), dtype=np.float32) + + var_slope_vals[Diff.single, :] = (np.add(data.tau[1:], data.tau[:end - 1]) - 2 * np.minimum(data.t_bar[1:], data.t_bar[:end - 1])) + var_slope_vals[Diff.double, :end - 2] = (np.add(data.tau[2:], data.tau[:end - 2]) - 2 * np.minimum(data.t_bar[2:], data.t_bar[:end - 2])) + var_slope_vals[Diff.double, end - 2] = np.nan # last double difference is undefined + + return var_slope_vals + + +cpdef inline FixedValues fixed_values_from_metadata(ReadPattern data, Thresh threshold, bool use_jump): """ Fast constructor for FixedValues class Use this instead of an __init__ because it does not incure the overhead @@ -267,9 +247,9 @@ cpdef inline FixedValues fixed_values_from_metadata(ReadPatternMetadata data, Th # Pre-compute jump detection computations shared by all pixels if use_jump: - fixed.t_bar_diffs = fixed.t_bar_diff_vals() + fixed.t_bar_diffs = t_bar_diff_vals(data) fixed.t_bar_diff_sqrs = np.square(fixed.t_bar_diffs, dtype=np.float32) - fixed.read_recip_coeffs = fixed.read_recip_vals() - fixed.var_slope_coeffs = fixed.var_slope_vals() + fixed.read_recip_coeffs = read_recip_vals(data) + fixed.var_slope_coeffs = var_slope_vals(data) return fixed diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index a1674123..f35cf92a 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -1,9 +1,6 @@ -cimport numpy as cnp - - cdef struct Thresh: - cnp.float32_t intercept - cnp.float32_t constant + float intercept + float constant -cpdef cnp.float32_t threshold(Thresh thresh, cnp.float32_t slope) \ No newline at end of file +cpdef float threshold(Thresh thresh, float slope) \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 32e7d522..19355e3a 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -1,12 +1,11 @@ import numpy as np -cimport numpy as cnp from libc.math cimport log10 from stcal.ramp_fitting.ols_cas22._jump cimport Thresh -cpdef inline cnp.float32_t threshold(Thresh thresh, cnp.float32_t slope): +cpdef inline float threshold(Thresh thresh, float slope): """ Compute jump threshold @@ -21,7 +20,7 @@ cpdef inline cnp.float32_t threshold(Thresh thresh, cnp.float32_t slope): ------- intercept - constant * log10(slope) """ - slope = slope if slope > 1 else np.float32(1) - slope = slope if slope < 1e4 else np.float32(1e4) + slope = slope if slope > 1 else 1 + slope = slope if slope < 1e4 else 1e4 return thresh.intercept - thresh.constant * log10(slope) \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx index 6200ff10..74b575e0 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx @@ -19,7 +19,7 @@ from libc.math cimport sqrt, fabs from libcpp.vector cimport vector import numpy as np -cimport numpy as np +cimport numpy as cnp cimport cython @@ -99,9 +99,9 @@ cdef class Pixel: # Read the t_bar_diffs into a local variable to avoid calling through Python # multiple times - cdef np.ndarray[float, ndim=2] t_bar_diffs = np.array(self.fixed.t_bar_diffs, dtype=np.float32) + cdef cnp.ndarray[float, ndim=2] t_bar_diffs = np.array(self.fixed.t_bar_diffs, dtype=np.float32) - cdef np.ndarray[float, ndim=2] local_slope_vals = np.zeros((2, end - 1), dtype=np.float32) + cdef cnp.ndarray[float, ndim=2] local_slope_vals = np.zeros((2, end - 1), dtype=np.float32) local_slope_vals[Diff.single, :] = (np.subtract(resultants[1:], resultants[:end - 1]) / t_bar_diffs[Diff.single, :]).astype(np.float32) @@ -139,26 +139,15 @@ cdef class Pixel: # Start computing the fit - # Cast vectors to memory views for faster access - # This way of doing it is potentially memory unsafe because the memory - # can outlive the vector. However, this is much faster (no copies) and - # much simpler than creating an intermediate wrapper which can pretend - # to be a memory view. In this case, I make sure that the memory view - # stays local to the function t_bar, tau, n_reads are used only for - # computations whose results are stored in new objects, so they are local - cdef float[:] t_bar_ = self.fixed.data.t_bar.data() - cdef float[:] tau_ = self.fixed.data.tau.data() - cdef int[:] n_reads_ = self.fixed.data.n_reads.data() - # Setup data for fitting (work over subset of data) # Recall that the RampIndex contains the index of the first and last # index of the ramp. Therefore, the Python slice needed to get all the # data within the ramp is: # ramp.start:ramp.end + 1 cdef float[:] resultants = self.resultants[ramp.start:ramp.end + 1] - cdef float[:] t_bar = t_bar_[ramp.start:ramp.end + 1] - cdef float[:] tau = tau_[ramp.start:ramp.end + 1] - cdef int[:] n_reads = n_reads_[ramp.start:ramp.end + 1] + cdef float[:] t_bar = self.fixed.data.t_bar[ramp.start:ramp.end + 1] + cdef float[:] tau = self.fixed.data.tau[ramp.start:ramp.end + 1] + cdef int[:] n_reads = self.fixed.data.n_reads[ramp.start:ramp.end + 1] # Reference read_noise as a local variable to avoid calling through Python # every time it is accessed. @@ -313,7 +302,7 @@ cdef class Pixel: # as the second argument to the `range` is the first index outside of the # range - cdef np.ndarray[float, ndim=1] stats = np.zeros(end - start, dtype=np.float32) + cdef cnp.ndarray[float, ndim=1] stats = np.zeros(end - start, dtype=np.float32) cdef int index, stat for stat, index in enumerate(range(start, end)): @@ -491,10 +480,10 @@ cdef class Pixel: only exists for testing purposes. """ - cdef np.ndarray[float, ndim=1] resultants_ = np.array(self.resultants, dtype=np.float32) + cdef cnp.ndarray[float, ndim=1] resultants_ = np.array(self.resultants, dtype=np.float32) - cdef np.ndarray[float, ndim=2] local_slopes - cdef np.ndarray[float, ndim=2] var_read_noise + cdef cnp.ndarray[float, ndim=2] local_slopes + cdef cnp.ndarray[float, ndim=2] var_read_noise if self.fixed.use_jump: local_slopes = np.array(self.local_slopes, dtype=np.float32) diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index cad21197..661fd6d3 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -1,14 +1,12 @@ -cimport numpy as cnp - from libcpp.vector cimport vector cdef struct RampIndex: - cnp.int32_t start - cnp.int32_t end + int start + int end ctypedef vector[RampIndex] RampQueue -cpdef RampQueue init_ramps(cnp.int32_t[:, :] dq, cnp.int32_t n_resultants, cnp.int32_t index_pixel) +cpdef RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixel) diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index 7eeb29ed..e5032a8e 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -8,7 +8,7 @@ cnp.import_array() @cython.boundscheck(False) @cython.wraparound(False) -cpdef inline RampQueue init_ramps(cnp.int32_t[:, :] dq, cnp.int32_t n_resultants, cnp.int32_t index_pixel): +cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixel): """ Create the initial ramp stack for each pixel if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd new file mode 100644 index 00000000..b4ca0d14 --- /dev/null +++ b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd @@ -0,0 +1,9 @@ +cdef class ReadPattern: + cdef int n_resultants + + cdef float[::1] t_bar + cdef float[::1] tau + cdef int[::1] n_reads + + +cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time) \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx new file mode 100644 index 00000000..a77416d7 --- /dev/null +++ b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx @@ -0,0 +1,78 @@ +import numpy as np +cimport numpy as cnp +cimport cython + +from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern + +cnp.import_array() + +cdef class ReadPattern: + """ + Class to contain the read pattern derived metadata + + Attributes: + ---------- + n_resultants : int + The number of resultants in the read pattern + t_bar : np.ndarray[float_t, ndim=1] + The mean time of each resultant + tau : np.ndarray[float_t, ndim=1] + The variance in time of each resultant + n_reads : np.ndarray[cnp.int32_t, ndim=1] + The number of reads in each resultant + """ + + def _to_dict(ReadPattern self): + """ + This is a private method to convert the ReadPattern object to a dictionary, + so that attributes can be directly accessed in python. Note that this + is needed because class attributes cannot be accessed on cython classes + directly in python. Instead they need to be accessed or set using a + python compatible method. This method is a pure puthon method bound + to to the cython class and should not be used by any cython code, and + only exists for testing purposes. + """ + return dict(n_resultants=self.n_resultants, + t_bar=np.array(self.t_bar, dtype=np.float32), + tau=np.array(self.tau, dtype=np.float32), + n_reads=np.array(self.n_reads, dtype=np.int32)) + + +@cython.boundscheck(False) +@cython.wraparound(False) +cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time): + """ + Derive the input data from the the read pattern + + read pattern is a list of resultant lists, where each resultant list is + a list of the reads in that resultant. + + Parameters + ---------- + read pattern: list[list[int]] + read pattern for the image + read_time : float + Time to perform a readout. + + Returns + ------- + ReadPattern + """ + cdef int n_resultants = len(read_pattern) + + cdef ReadPattern data = ReadPattern() + data.n_resultants = n_resultants + data.t_bar = np.empty(n_resultants, dtype=np.float32) + data.tau = np.empty(n_resultants, dtype=np.float32) + data.n_reads = np.empty(n_resultants, dtype=np.int32) + + cdef int index, n_reads + cdef list[int] resultant + for index, resultant in enumerate(read_pattern): + n_reads = len(resultant) + + data.n_reads[index] = n_reads + data.t_bar[index] = read_time * np.mean(resultant) + data.tau[index] = np.sum((2 * (n_reads - np.arange(n_reads)) - 1) * resultant) * read_time / n_reads**2 + + return data \ No newline at end of file diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index fcbb9ca1..89a5f396 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,11 +2,11 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22._core import metadata_from_read_pattern from stcal.ramp_fitting.ols_cas22._fixed import fixed_values_from_metadata from stcal.ramp_fitting.ols_cas22._jump import threshold from stcal.ramp_fitting.ols_cas22._pixel import make_pixel from stcal.ramp_fitting.ols_cas22._ramp import init_ramps +from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern from stcal.ramp_fitting.ols_cas22 import fit_ramps, Parameter, Variance, Diff, RampJumpDQ @@ -37,7 +37,7 @@ @pytest.fixture(scope="module") -def base_ramp_data(): +def ramp_data(): """ Basic data for simulating ramps for testing (not unpacked) @@ -57,24 +57,31 @@ def base_ramp_data(): [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36] ] - yield read_pattern, metadata_from_read_pattern(read_pattern, READ_TIME) + yield read_pattern, from_read_pattern(read_pattern, READ_TIME) -def test_metadata_from_read_pattern(base_ramp_data): +def test_from_read_pattern(ramp_data): """Test turning read_pattern into the time data""" - _, data = base_ramp_data + read_pattern, data_object = ramp_data + data = data_object._to_dict() # Basic sanity checks (structs become dicts) assert isinstance(data, dict) assert 't_bar' in data assert 'tau' in data assert 'n_reads' in data - assert len(data) == 3 + assert len(data) == 4 # Check that the data is correct + assert data['n_resultants'] == len(read_pattern) assert_allclose(data['t_bar'], [7.6, 15.2, 21.279999, 41.040001, 60.799999, 88.159996]) assert_allclose(data['tau'], [5.7, 15.2, 19.928888, 36.023998, 59.448887, 80.593781]) - assert data['n_reads'] == [4, 1, 3, 10, 3, 15] + assert np.all(data['n_reads'] == [4, 1, 3, 10, 3, 15]) + + # Check datatypes + assert data['t_bar'].dtype == np.float32 + assert data['tau'].dtype == np.float32 + assert data['n_reads'].dtype == np.int32 def test_init_ramps(): @@ -150,43 +157,15 @@ def test_threshold(): assert np.float32(thresh['intercept'] - thresh['constant']) == threshold(thresh, 10.0) -@pytest.fixture(scope="module") -def ramp_data(base_ramp_data): - """ - Unpacked metadata for simulating ramps for testing - - Returns - ------- - read_pattern: - The read pattern used for testing - t_bar: - The t_bar values for the read pattern - tau: - The tau values for the read pattern - n_reads: - The number of reads for the read pattern - """ - read_pattern, read_pattern_metadata = base_ramp_data - t_bar = np.array(read_pattern_metadata['t_bar'], dtype=np.float32) - tau = np.array(read_pattern_metadata['tau'], dtype=np.float32) - n_reads = np.array(read_pattern_metadata['n_reads'], dtype=np.int32) - - yield read_pattern, t_bar, tau, n_reads - - @pytest.mark.parametrize("use_jump", [True, False]) def test_fixed_values_from_metadata(ramp_data, use_jump): """Test computing the fixed data for all pixels""" - _, t_bar, tau, n_reads = ramp_data + _, data = ramp_data - # Create the python analog of the ReadPatternMetadata struct - # Note that structs get mapped to/from python as dictionary objects with - # the keys being the struct members. - data = { - "t_bar": t_bar, - "tau": tau, - "n_reads": n_reads, - } + data_dict = data._to_dict() + t_bar = data_dict['t_bar'] + tau = data_dict['tau'] + n_reads = data_dict['n_reads'] # Create the python analog of the Threshold struct # Note that structs get mapped to/from python as dictionary objects with @@ -308,29 +287,27 @@ def pixel_data(ramp_data): The number of reads for the read pattern used for the resultants """ - read_pattern, t_bar, tau, n_reads = ramp_data + read_pattern, metadata = ramp_data resultants = _generate_resultants(read_pattern) - yield resultants, t_bar, tau, n_reads + yield resultants, metadata @pytest.mark.parametrize("use_jump", [True, False]) def test_make_pixel(pixel_data, use_jump): """Test computing the initial pixel data""" - resultants, t_bar, tau, n_reads = pixel_data - - # Create a fixed object to pass into the constructor - # This requires setting up some structs as dictionaries - data = { - "t_bar": t_bar, - "tau": tau, - "n_reads": n_reads, - } + resultants, metadata = pixel_data + + data = metadata._to_dict() + t_bar = data['t_bar'] + tau = data['tau'] + n_reads = data['n_reads'] + thresh = { 'intercept': np.float32(5.5), 'constant': np.float32(1/3) } - fixed = fixed_values_from_metadata(data, thresh, use_jump) + fixed = fixed_values_from_metadata(metadata, thresh, use_jump) # Note this is converted to a dictionary so we can directly interrogate the # variables in question @@ -389,7 +366,7 @@ def detector_data(ramp_data): read_pattern: The read pattern used for the resultants """ - read_pattern, *_ = ramp_data + read_pattern, _ = ramp_data read_noise = np.ones(N_PIXELS, dtype=np.float32) * READ_NOISE resultants = _generate_resultants(read_pattern, n_pixels=N_PIXELS) From a8c8ebead8ddd11766ee70aac00f45df80e1f41d Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Tue, 7 Nov 2023 16:52:38 -0500 Subject: [PATCH 04/30] Move jump algorithm into a separate module --- src/stcal/ramp_fitting/ols_cas22/_core.pxd | 20 -- src/stcal/ramp_fitting/ols_cas22/_core.pyx | 33 +- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 14 +- src/stcal/ramp_fitting/ols_cas22/_fixed.pxd | 4 +- src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 5 +- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 17 +- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 301 +++++++++++++++++- src/stcal/ramp_fitting/ols_cas22/_pixel.pxd | 10 +- src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 261 +-------------- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 6 + tests/test_jump_cas22.py | 18 +- 11 files changed, 335 insertions(+), 354 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pxd b/src/stcal/ramp_fitting/ols_cas22/_core.pxd index d5790107..1ef4d17d 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pxd @@ -1,20 +1,3 @@ -from libcpp.vector cimport vector -from stcal.ramp_fitting.ols_cas22._ramp cimport RampQueue - - -cdef struct RampFit: - float slope - float read_var - float poisson_var - - -cdef struct RampFits: - RampFit average - vector[int] jumps - vector[RampFit] fits - RampQueue index - - cpdef enum Diff: single = 0 double = 1 @@ -33,6 +16,3 @@ cpdef enum Variance: cpdef enum RampJumpDQ: JUMP_DET = 4 - - -cdef float get_power(float s) diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pyx b/src/stcal/ramp_fitting/ols_cas22/_core.pyx index 4b0473b2..5f401244 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pyx @@ -74,35 +74,4 @@ Functions - cpdef gives a python wrapper, but the python version of this method is considered private, only to be used for testing """ -import numpy as np -cimport numpy as np -cimport cython - - -# Casertano+2022, Table 2 -cdef float[2][6] PTABLE = [ - [-np.inf, 5, 10, 20, 50, 100], - [0, 0.4, 1, 3, 6, 10]] - - -@cython.boundscheck(False) -@cython.wraparound(False) -cdef inline float get_power(float signal): - """ - Return the power from Casertano+22, Table 2 - - Parameters - ---------- - signal: float - signal from the resultants - - Returns - ------- - signal power from Table 2 - """ - cdef int i - for i in range(6): - if signal < PTABLE[0][i]: - return PTABLE[1][i - 1] - - return PTABLE[1][i] +from stcal.ramp_fitting.ols_cas22._core cimport Diff, Parameter, Variance, RampJumpDQ diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index c9593fbe..fa17cb0b 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -4,12 +4,11 @@ from libcpp cimport bool from libcpp.list cimport list as cpp_list cimport cython -from stcal.ramp_fitting.ols_cas22._core cimport (RampFits, - Parameter, Variance, RampJumpDQ) +from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance, RampJumpDQ from stcal.ramp_fitting.ols_cas22._fixed cimport fixed_values_from_metadata, FixedValues from stcal.ramp_fitting.ols_cas22._pixel cimport make_pixel -from stcal.ramp_fitting.ols_cas22._jump cimport Thresh +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, fit_jumps, RampFits from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern cimport from_read_pattern @@ -102,9 +101,8 @@ def fit_ramps(np.ndarray[float, ndim=2] resultants, f'match number of resultants {n_resultants}') # Pre-compute data for all pixels - cdef FixedValues fixed = fixed_values_from_metadata(from_read_pattern(read_pattern, read_time), - Thresh(np.float32(intercept), np.float32(constant)), - use_jump) + cdef FixedValues fixed = fixed_values_from_metadata(from_read_pattern(read_pattern, read_time), use_jump) + cdef Thresh thresh = Thresh(intercept, constant) # Use list because this might grow very large which would require constant # reallocation. We don't need random access, and this gets cast to a python @@ -119,8 +117,8 @@ def fit_ramps(np.ndarray[float, ndim=2] resultants, cdef int index for index in range(n_pixels): # Fit all the ramps for the given pixel - fit = make_pixel(fixed, read_noise[index], - resultants[:, index]).fit_ramps(init_ramps(dq, n_resultants, index), include_diagnostic) + fit = fit_jumps(make_pixel(fixed, read_noise[index], resultants[:, index]), + init_ramps(dq, n_resultants, index), thresh, include_diagnostic) parameters[index, Parameter.slope] = fit.average.slope diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd index 041b08e3..6b624a16 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd @@ -1,13 +1,11 @@ from libcpp cimport bool -from stcal.ramp_fitting.ols_cas22._jump cimport Thresh from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern cdef class FixedValues: cdef bool use_jump cdef ReadPattern data - cdef Thresh threshold cdef float[:, :] t_bar_diffs cdef float[:, :] t_bar_diff_sqrs @@ -15,4 +13,4 @@ cdef class FixedValues: cdef float[:, :] var_slope_coeffs -cpdef FixedValues fixed_values_from_metadata(ReadPattern data, Thresh threshold, bool use_jump) +cpdef FixedValues fixed_values_from_metadata(ReadPattern data, bool use_jump) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx index 90a9b122..c9113b8f 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx @@ -22,7 +22,6 @@ from libcpp cimport bool from stcal.ramp_fitting.ols_cas22._core cimport Diff from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues -from stcal.ramp_fitting.ols_cas22._jump cimport Thresh from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern cdef class FixedValues: @@ -135,7 +134,6 @@ cdef class FixedValues: raise AttributeError("var_slope_coeffs should not exist") return dict(data=self.data._to_dict(), - threshold=self.threshold, t_bar_diffs=t_bar_diffs, t_bar_diff_sqrs=t_bar_diff_sqrs, read_recip_coeffs=read_recip_coeffs, @@ -216,7 +214,7 @@ cdef inline float[:, :] var_slope_vals(ReadPattern data): return var_slope_vals -cpdef inline FixedValues fixed_values_from_metadata(ReadPattern data, Thresh threshold, bool use_jump): +cpdef inline FixedValues fixed_values_from_metadata(ReadPattern data, bool use_jump): """ Fast constructor for FixedValues class Use this instead of an __init__ because it does not incure the overhead @@ -240,7 +238,6 @@ cpdef inline FixedValues fixed_values_from_metadata(ReadPattern data, Thresh thr # Fill in input information for all pixels fixed.use_jump = use_jump - fixed.threshold = threshold # Cast vector to a c array fixed.data = data diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index f35cf92a..7ab1e64b 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -1,6 +1,21 @@ +from libcpp cimport bool +from libcpp.vector cimport vector + +from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampQueue +from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel + cdef struct Thresh: float intercept float constant -cpdef float threshold(Thresh thresh, float slope) \ No newline at end of file +cdef struct RampFits: + RampFit average + vector[int] jumps + vector[RampFit] fits + RampQueue index + + +cpdef float threshold(Thresh thresh, float slope) +cdef float get_power(float s) +cdef RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool include_diagnostic) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 19355e3a..72e8049a 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -1,9 +1,17 @@ import numpy as np +cimport numpy as cnp +cimport cython -from libc.math cimport log10 +from libcpp cimport bool +from libc.math cimport sqrt, log10 -from stcal.ramp_fitting.ols_cas22._jump cimport Thresh +from stcal.ramp_fitting.ols_cas22._core cimport Diff +from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, RampFits +from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel +from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue +from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern cpdef inline float threshold(Thresh thresh, float slope): """ @@ -23,4 +31,291 @@ cpdef inline float threshold(Thresh thresh, float slope): slope = slope if slope > 1 else 1 slope = slope if slope < 1e4 else 1e4 - return thresh.intercept - thresh.constant * log10(slope) \ No newline at end of file + return thresh.intercept - thresh.constant * log10(slope) + + +@cython.boundscheck(False) +@cython.wraparound(False) +cdef inline float get_power(float signal): + """ + Return the power from Casertano+22, Table 2 + + Parameters + ---------- + signal: float + signal from the resultants + + Returns + ------- + signal power from Table 2 + """ + # Casertano+2022, Table 2 + cdef float[2][6] PTABLE = [ + [-np.inf, 5, 10, 20, 50, 100], + [0, 0.4, 1, 3, 6, 10]] + + cdef int i + for i in range(6): + if signal < PTABLE[0][i]: + return PTABLE[1][i - 1] + + return PTABLE[1][i] + + +@cython.boundscheck(False) +@cython.wraparound(False) +cdef inline float correction(ReadPattern data, RampIndex ramp, float slope): + """ + Compute the correction factor for the variance used by a statistic + + - slope / (t_bar[end] - t_bar[start]) + + Parameters + ---------- + ramp : RampIndex + Struct for start and end indices resultants for the ramp + slope : float + The computed slope for the ramp + """ + + cdef float diff = (data.t_bar[ramp.end] - data.t_bar[ramp.start]) + + return - slope / diff + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +cdef inline float statstic(Pixel pixel, float slope, RampIndex ramp, int index, int diff): + """ + Compute a single set of fit statistics + (delta / sqrt(var)) + correction + where + delta = ((R[j] - R[i]) / (t_bar[j] - t_bar[i]) - slope) + * (t_bar[j] - t_bar[i]) + var = sigma * (1/N[j] + 1/N[i]) + + slope * (tau[j] + tau[i] - min(t_bar[j], t_bar[i])) + + Parameters + ---------- + slope : float + The computed slope for the ramp + ramp : RampIndex + Struct for start and end indices resultants for the ramp + index : int + The main index for the resultant to compute the statistic for + diff : int + The offset to use for the delta and sigma values, this should be + a value from the Diff enum. + + Returns + ------- + Create a single instance of the stastic for the given parameters + """ + cdef FixedValues fixed = pixel.fixed + + cdef float delta = (pixel.local_slopes[diff, index] - slope) + cdef float var = ((pixel.var_read_noise[diff, index] + + slope * fixed.var_slope_coeffs[diff, index]) + / fixed.t_bar_diff_sqrs[diff, index]) + cdef float correct = correction(fixed.data, ramp, slope) + + return (delta / sqrt(var + correct)) + + +@cython.boundscheck(False) +@cython.wraparound(False) +cdef inline float[:] statistics(Pixel pixel, float slope, RampIndex ramp): + """ + Compute fit statistics for jump detection on a single ramp + stats[i] = max(stat(i, 0), stat(i, 1)) + Note for i == end - 1, no stat(i, 1) exists, so its just stat(i, 0) + + Parameters + ---------- + slope : float + The computed slope for the ramp + ramp : RampIndex + Struct for start and end of ramp to fit + + Returns + ------- + list of statistics for each resultant + """ + cdef int start = ramp.start # index of first resultant for ramp + cdef int end = ramp.end # index of last resultant for ramp + + # Observe that the length of the ramp's sub array of the resultant would + # be `end - start + 1`. However, we are computing single and double + # "differences" which means we need to reference at least two points in + # this subarray at a time. For the single case, the maximum index allowed + # would be `end - 1`. Observe that `range(start, end)` will iterate over + # `start, start+1, start+1, ..., end-2, end-1` + # as the second argument to the `range` is the first index outside of the + # range + + cdef cnp.ndarray[float, ndim=1] stats = np.zeros(end - start, dtype=np.float32) + + cdef int index, stat + for stat, index in enumerate(range(start, end)): + if index == end - 1: + # It is not possible to compute double differences for the second + # to last resultant in the ramp. Therefore, we just compute the + # single difference for this resultant. + stats[stat] = statstic(pixel, slope, ramp, index, Diff.single) + else: + stats[stat] = max(statstic(pixel, slope, ramp, index, Diff.single), + statstic(pixel, slope, ramp, index, Diff.double)) + + return stats + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool include_diagnostic): + """ + Compute all the ramps for a single pixel using the Casertano+22 algorithm + with jump detection. + + Parameters + ---------- + ramps : vector[RampIndex] + Vector of initial ramps to fit for a single pixel + multiple ramps are possible due to dq flags + + Returns + ------- + RampFits struct of all the fits for a single pixel + """ + # Setup algorithm + cdef RampFits ramp_fits + cdef RampIndex ramp + cdef RampFit ramp_fit + + ramp_fits.average.slope = 0 + ramp_fits.average.read_var = 0 + ramp_fits.average.poisson_var = 0 + + cdef float [:] stats + cdef int jump0, jump1 + cdef float weight, total_weight = 0 + + # Run while the stack is non-empty + while not ramps.empty(): + # Remove top ramp of the stack to use + ramp = ramps.back() + ramps.pop_back() + + # Compute fit + ramp_fit = pixel.fit_ramp(ramp) + + # Run jump detection if enabled + if pixel.fixed.use_jump: + stats = statistics(pixel, ramp_fit.slope, ramp) + + # We have to protect against the case where the passed "ramp" is + # only a single point. In that case, stats will be empty. This + # will create an error in the max() call. + if len(stats) > 0 and max(stats) > threshold(thresh, ramp_fit.slope): + # Compute jump point to create two new ramps + # This jump point corresponds to the index of the largest + # statistic: + # argmax(stats) + # These statistics are indexed relative to the + # ramp's range. Therefore, we need to add the start index + # of the ramp to the result. + # + # Note that because the resultants are averages of reads, but + # jumps occur in individual reads, it is possible that the + # jump is averaged down by the resultant with the actual jump + # causing the computed jump to be off by one index. + # In the idealized case this is when the jump occurs near + # the start of the resultant with the jump. In this case, + # the statistic for the resultant will be maximized at + # index - 1 rather than index. This means that we have to + # remove argmax(stats) + 1 as it is also a possible jump. + # This case is difficult to distinguish from the case where + # argmax(stats) does correspond to the jump resultant. + # Therefore, we just remove both possible resultants from + # consideration. + jump0 = np.argmax(stats) + ramp.start + jump1 = jump0 + 1 + if include_diagnostic: + ramp_fits.jumps.push_back(jump0) + ramp_fits.jumps.push_back(jump1) + + # The two resultant indicies need to be skipped, therefore + # the two + # possible new ramps are: + # RampIndex(ramp.start, jump0 - 1) + # RampIndex(jump1 + 1, ramp.end) + # This is because the RampIndex contains the index of the + # first and last resulants in the sub-ramp it describes. + # Note: The algorithm works via working over the sub-ramps + # backward in time. Therefore, since we are using a stack, + # we need to add the ramps in the time order they were + # observed in. This results in the last observation ramp + # being the top of the stack; meaning that, + # it will be the next ramp handeled. + + if jump0 > ramp.start: + # Note that when jump0 == ramp.start, we have detected a + # jump in the first resultant of the ramp. This means + # there is no sub-ramp before jump0. + # Also, note that this will produce bad results as + # the ramp indexing will go out of bounds. So it is + # important that we exclude it. + # Note that jump0 < ramp.start is not possible because + # the argmax is always >= 0 + ramps.push_back(RampIndex(ramp.start, jump0 - 1)) + + if jump1 < ramp.end: + # Note that if jump1 == ramp.end, we have detected a + # jump in the last resultant of the ramp. This means + # there is no sub-ramp after jump1. + # Also, note that this will produce bad results as + # the ramp indexing will go out of bounds. So it is + # important that we exclude it. + # Note that jump1 > ramp.end is technically possible + # however in those potential cases it will draw on + # resultants which are not considered part of the ramp + # under consideration. Therefore, we have to exlude all + # of those values. + ramps.push_back(RampIndex(jump1 + 1, ramp.end)) + + continue + + # Add ramp_fit to ramp_fits if no jump detection or stats are less + # than threshold + # Note that ramps are computed backward in time meaning we need to + # reverse the order of the fits at the end + if include_diagnostic: + ramp_fits.fits.push_back(ramp_fit) + ramp_fits.index.push_back(ramp) + + # Start computing the averages + # Note we do not do anything in the NaN case for degenerate ramps + if not np.isnan(ramp_fit.slope): + # protect weight against the extremely unlikely case of a zero + # variance + weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var + total_weight += weight + + ramp_fits.average.slope += weight * ramp_fit.slope + ramp_fits.average.read_var += weight**2 * ramp_fit.read_var + ramp_fits.average.poisson_var += weight**2 * ramp_fit.poisson_var + + # Reverse to order in time + if include_diagnostic: + ramp_fits.fits = ramp_fits.fits[::-1] + ramp_fits.index = ramp_fits.index[::-1] + + # Finish computing averages + ramp_fits.average.slope /= total_weight if total_weight != 0 else 1 + ramp_fits.average.read_var /= total_weight**2 if total_weight != 0 else 1 + ramp_fits.average.poisson_var /= total_weight**2 if total_weight != 0 else 1 + + # Multiply poisson term by flux, (no negative fluxes) + ramp_fits.average.poisson_var *= max(ramp_fits.average.slope, 0) + + return ramp_fits \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd index 219e2759..ea4850b9 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd @@ -1,13 +1,12 @@ from libcpp cimport bool -from stcal.ramp_fitting.ols_cas22._core cimport RampFit, RampFits from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues -from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue +from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampFit cdef class Pixel: cdef FixedValues fixed cdef float read_noise - cdef float [:] resultants + cdef float[:] resultants cdef float[:, :] local_slopes cdef float[:, :] var_read_noise @@ -15,10 +14,5 @@ cdef class Pixel: cdef float[:, :] local_slope_vals(Pixel self) cdef RampFit fit_ramp(Pixel self, RampIndex ramp) - cdef float correction(Pixel self, RampIndex ramp, float slope) - cdef float stat(Pixel self, float slope, RampIndex ramp, int index, int diff) - cdef float[:] stats(Pixel self, float slope, RampIndex ramp) - cdef RampFits fit_ramps(Pixel self, RampQueue ramps, bool include_diagnostic) - cpdef Pixel make_pixel(FixedValues fixed, float read_noise, float [:] resultants) diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx index 74b575e0..269ab26d 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx @@ -14,7 +14,6 @@ Functions - cpdef gives a python wrapper, but the python version of this method is considered private, only to be used for testing """ -from libcpp cimport bool from libc.math cimport sqrt, fabs from libcpp.vector cimport vector @@ -23,9 +22,9 @@ cimport numpy as cnp cimport cython -from stcal.ramp_fitting.ols_cas22._core cimport get_power, RampFit, RampFits, Diff +from stcal.ramp_fitting.ols_cas22._core cimport Diff from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues -from stcal.ramp_fitting.ols_cas22._jump cimport threshold +from stcal.ramp_fitting.ols_cas22._jump cimport get_power from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue @@ -213,262 +212,6 @@ cdef class Pixel: return ramp_fit - @cython.boundscheck(False) - @cython.wraparound(False) - cdef inline float correction(Pixel self, RampIndex ramp, float slope): - """ - Compute the correction factor for the variance used by a statistic - - - slope / (t_bar[end] - t_bar[start]) - - Parameters - ---------- - ramp : RampIndex - Struct for start and end indices resultants for the ramp - slope : float - The computed slope for the ramp - """ - - cdef float diff = (self.fixed.data.t_bar[ramp.end] - self.fixed.data.t_bar[ramp.start]) - - return - slope / diff - - @cython.boundscheck(False) - @cython.wraparound(False) - @cython.cdivision(True) - cdef inline float stat(Pixel self, float slope, RampIndex ramp, int index, int diff): - """ - Compute a single set of fit statistics - (delta / sqrt(var)) + correction - where - delta = ((R[j] - R[i]) / (t_bar[j] - t_bar[i]) - slope) - * (t_bar[j] - t_bar[i]) - var = sigma * (1/N[j] + 1/N[i]) - + slope * (tau[j] + tau[i] - min(t_bar[j], t_bar[i])) - - Parameters - ---------- - slope : float - The computed slope for the ramp - ramp : RampIndex - Struct for start and end indices resultants for the ramp - index : int - The main index for the resultant to compute the statistic for - diff : int - The offset to use for the delta and sigma values, this should be - a value from the Diff enum. - - Returns - ------- - Create a single instance of the stastic for the given parameters - """ - cdef float delta = (self.local_slopes[diff, index] - slope) - cdef float var = ((self.var_read_noise[diff, index] + - slope * self.fixed.var_slope_coeffs[diff, index]) - / self.fixed.t_bar_diff_sqrs[diff, index]) - cdef float correct = self.correction(ramp, slope) - - return (delta / sqrt(var + correct)) - - - @cython.boundscheck(False) - @cython.wraparound(False) - cdef inline float[:] stats(Pixel self, float slope, RampIndex ramp): - """ - Compute fit statistics for jump detection on a single ramp - stats[i] = max(stat(i, 0), stat(i, 1)) - Note for i == end - 1, no stat(i, 1) exists, so its just stat(i, 0) - - Parameters - ---------- - slope : float - The computed slope for the ramp - ramp : RampIndex - Struct for start and end of ramp to fit - - Returns - ------- - list of statistics for each resultant - """ - cdef int start = ramp.start # index of first resultant for ramp - cdef int end = ramp.end # index of last resultant for ramp - - # Observe that the length of the ramp's sub array of the resultant would - # be `end - start + 1`. However, we are computing single and double - # "differences" which means we need to reference at least two points in - # this subarray at a time. For the single case, the maximum index allowed - # would be `end - 1`. Observe that `range(start, end)` will iterate over - # `start, start+1, start+1, ..., end-2, end-1` - # as the second argument to the `range` is the first index outside of the - # range - - cdef cnp.ndarray[float, ndim=1] stats = np.zeros(end - start, dtype=np.float32) - - cdef int index, stat - for stat, index in enumerate(range(start, end)): - if index == end - 1: - # It is not possible to compute double differences for the second - # to last resultant in the ramp. Therefore, we just compute the - # single difference for this resultant. - stats[stat] = self.stat(slope, ramp, index, Diff.single) - else: - stats[stat] = max(self.stat(slope, ramp, index, Diff.single), - self.stat(slope, ramp, index, Diff.double)) - - return stats - - - @cython.boundscheck(False) - @cython.wraparound(False) - @cython.cdivision(True) - cdef inline RampFits fit_ramps(Pixel self, RampQueue ramps, bool include_diagnostic): - """ - Compute all the ramps for a single pixel using the Casertano+22 algorithm - with jump detection. - - Parameters - ---------- - ramps : vector[RampIndex] - Vector of initial ramps to fit for a single pixel - multiple ramps are possible due to dq flags - - Returns - ------- - RampFits struct of all the fits for a single pixel - """ - # Setup algorithm - cdef RampFits ramp_fits - cdef RampIndex ramp - cdef RampFit ramp_fit - - ramp_fits.average.slope = 0 - ramp_fits.average.read_var = 0 - ramp_fits.average.poisson_var = 0 - - cdef float [:] stats - cdef int jump0, jump1 - cdef float weight, total_weight = 0 - - # Run while the stack is non-empty - while not ramps.empty(): - # Remove top ramp of the stack to use - ramp = ramps.back() - ramps.pop_back() - - # Compute fit - ramp_fit = self.fit_ramp(ramp) - - # Run jump detection if enabled - if self.fixed.use_jump: - stats = self.stats(ramp_fit.slope, ramp) - - # We have to protect against the case where the passed "ramp" is - # only a single point. In that case, stats will be empty. This - # will create an error in the max() call. - if len(stats) > 0 and max(stats) > threshold(self.fixed.threshold, ramp_fit.slope): - # Compute jump point to create two new ramps - # This jump point corresponds to the index of the largest - # statistic: - # argmax(stats) - # These statistics are indexed relative to the - # ramp's range. Therefore, we need to add the start index - # of the ramp to the result. - # - # Note that because the resultants are averages of reads, but - # jumps occur in individual reads, it is possible that the - # jump is averaged down by the resultant with the actual jump - # causing the computed jump to be off by one index. - # In the idealized case this is when the jump occurs near - # the start of the resultant with the jump. In this case, - # the statistic for the resultant will be maximized at - # index - 1 rather than index. This means that we have to - # remove argmax(stats) + 1 as it is also a possible jump. - # This case is difficult to distinguish from the case where - # argmax(stats) does correspond to the jump resultant. - # Therefore, we just remove both possible resultants from - # consideration. - jump0 = np.argmax(stats) + ramp.start - jump1 = jump0 + 1 - if include_diagnostic: - ramp_fits.jumps.push_back(jump0) - ramp_fits.jumps.push_back(jump1) - - # The two resultant indicies need to be skipped, therefore - # the two - # possible new ramps are: - # RampIndex(ramp.start, jump0 - 1) - # RampIndex(jump1 + 1, ramp.end) - # This is because the RampIndex contains the index of the - # first and last resulants in the sub-ramp it describes. - # Note: The algorithm works via working over the sub-ramps - # backward in time. Therefore, since we are using a stack, - # we need to add the ramps in the time order they were - # observed in. This results in the last observation ramp - # being the top of the stack; meaning that, - # it will be the next ramp handeled. - - if jump0 > ramp.start: - # Note that when jump0 == ramp.start, we have detected a - # jump in the first resultant of the ramp. This means - # there is no sub-ramp before jump0. - # Also, note that this will produce bad results as - # the ramp indexing will go out of bounds. So it is - # important that we exclude it. - # Note that jump0 < ramp.start is not possible because - # the argmax is always >= 0 - ramps.push_back(RampIndex(ramp.start, jump0 - 1)) - - if jump1 < ramp.end: - # Note that if jump1 == ramp.end, we have detected a - # jump in the last resultant of the ramp. This means - # there is no sub-ramp after jump1. - # Also, note that this will produce bad results as - # the ramp indexing will go out of bounds. So it is - # important that we exclude it. - # Note that jump1 > ramp.end is technically possible - # however in those potential cases it will draw on - # resultants which are not considered part of the ramp - # under consideration. Therefore, we have to exlude all - # of those values. - ramps.push_back(RampIndex(jump1 + 1, ramp.end)) - - continue - - # Add ramp_fit to ramp_fits if no jump detection or stats are less - # than threshold - # Note that ramps are computed backward in time meaning we need to - # reverse the order of the fits at the end - if include_diagnostic: - ramp_fits.fits.push_back(ramp_fit) - ramp_fits.index.push_back(ramp) - - # Start computing the averages - # Note we do not do anything in the NaN case for degenerate ramps - if not np.isnan(ramp_fit.slope): - # protect weight against the extremely unlikely case of a zero - # variance - weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var - total_weight += weight - - ramp_fits.average.slope += weight * ramp_fit.slope - ramp_fits.average.read_var += weight**2 * ramp_fit.read_var - ramp_fits.average.poisson_var += weight**2 * ramp_fit.poisson_var - - # Reverse to order in time - if include_diagnostic: - ramp_fits.fits = ramp_fits.fits[::-1] - ramp_fits.index = ramp_fits.index[::-1] - - # Finish computing averages - ramp_fits.average.slope /= total_weight if total_weight != 0 else 1 - ramp_fits.average.read_var /= total_weight**2 if total_weight != 0 else 1 - ramp_fits.average.poisson_var /= total_weight**2 if total_weight != 0 else 1 - - # Multiply poisson term by flux, (no negative fluxes) - ramp_fits.average.poisson_var *= max(ramp_fits.average.slope, 0) - - return ramp_fits - def _to_dict(Pixel self): """ This is a private method to convert the Pixel object to a dictionary, so diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index 661fd6d3..51a1d0f8 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -6,6 +6,12 @@ cdef struct RampIndex: int end +cdef struct RampFit: + float slope + float read_var + float poisson_var + + ctypedef vector[RampIndex] RampQueue diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 89a5f396..75ffb5c1 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -167,24 +167,14 @@ def test_fixed_values_from_metadata(ramp_data, use_jump): tau = data_dict['tau'] n_reads = data_dict['n_reads'] - # Create the python analog of the Threshold struct - # Note that structs get mapped to/from python as dictionary objects with - # the keys being the struct members. - thresh = { - 'intercept': np.float32(5.5), - 'constant': np.float32(1/3) - } - # Note this is converted to a dictionary so we can directly interrogate the # variables in question - fixed = fixed_values_from_metadata(data, thresh, use_jump)._to_dict() + fixed = fixed_values_from_metadata(data, use_jump)._to_dict() # Basic sanity checks that data passed in survives assert (fixed['data']['t_bar'] == t_bar).all() assert (fixed['data']['tau'] == tau).all() assert (fixed['data']['n_reads'] == n_reads).all() - assert fixed['threshold']["intercept"] == thresh['intercept'] - assert fixed['threshold']["constant"] == thresh['constant'] # Check the computed data # These are computed via vectorized operations in the main code, here we @@ -303,11 +293,7 @@ def test_make_pixel(pixel_data, use_jump): tau = data['tau'] n_reads = data['n_reads'] - thresh = { - 'intercept': np.float32(5.5), - 'constant': np.float32(1/3) - } - fixed = fixed_values_from_metadata(metadata, thresh, use_jump) + fixed = fixed_values_from_metadata(metadata, use_jump) # Note this is converted to a dictionary so we can directly interrogate the # variables in question From 64cf05860746bd88f8d075b62101ffb18902dc82 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Tue, 7 Nov 2023 19:57:22 -0500 Subject: [PATCH 05/30] Move ramp fitting into the ramp module --- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 1 - src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 30 +--- src/stcal/ramp_fitting/ols_cas22/_pixel.pxd | 1 - src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 107 --------------- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 3 + src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 143 +++++++++++++++++++- 6 files changed, 146 insertions(+), 139 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 7ab1e64b..364b0892 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -17,5 +17,4 @@ cdef struct RampFits: cpdef float threshold(Thresh thresh, float slope) -cdef float get_power(float s) cdef RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool include_diagnostic) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 72e8049a..32853471 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -10,7 +10,7 @@ from stcal.ramp_fitting.ols_cas22._core cimport Diff from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, RampFits from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel -from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue +from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, fit_ramp from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern cpdef inline float threshold(Thresh thresh, float slope): @@ -34,32 +34,6 @@ cpdef inline float threshold(Thresh thresh, float slope): return thresh.intercept - thresh.constant * log10(slope) -@cython.boundscheck(False) -@cython.wraparound(False) -cdef inline float get_power(float signal): - """ - Return the power from Casertano+22, Table 2 - - Parameters - ---------- - signal: float - signal from the resultants - - Returns - ------- - signal power from Table 2 - """ - # Casertano+2022, Table 2 - cdef float[2][6] PTABLE = [ - [-np.inf, 5, 10, 20, 50, 100], - [0, 0.4, 1, 3, 6, 10]] - - cdef int i - for i in range(6): - if signal < PTABLE[0][i]: - return PTABLE[1][i - 1] - - return PTABLE[1][i] @cython.boundscheck(False) @@ -207,7 +181,7 @@ cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool ramps.pop_back() # Compute fit - ramp_fit = pixel.fit_ramp(ramp) + ramp_fit = fit_ramp(pixel, ramp) # Run jump detection if enabled if pixel.fixed.use_jump: diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd index ea4850b9..901c8902 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd @@ -12,7 +12,6 @@ cdef class Pixel: cdef float[:, :] var_read_noise cdef float[:, :] local_slope_vals(Pixel self) - cdef RampFit fit_ramp(Pixel self, RampIndex ramp) cpdef Pixel make_pixel(FixedValues fixed, float read_noise, float [:] resultants) diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx index 269ab26d..e967dbc7 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx @@ -14,9 +14,6 @@ Functions - cpdef gives a python wrapper, but the python version of this method is considered private, only to be used for testing """ -from libc.math cimport sqrt, fabs -from libcpp.vector cimport vector - import numpy as np cimport numpy as cnp cimport cython @@ -24,9 +21,7 @@ cimport cython from stcal.ramp_fitting.ols_cas22._core cimport Diff from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues -from stcal.ramp_fitting.ols_cas22._jump cimport get_power from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel -from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue cdef class Pixel: @@ -110,108 +105,6 @@ cdef class Pixel: return local_slope_vals - @cython.boundscheck(False) - @cython.wraparound(False) - @cython.cdivision(True) - cdef inline RampFit fit_ramp(Pixel self, RampIndex ramp): - """ - Fit a single ramp using Casertano+22 algorithm. - - Parameters - ---------- - ramp : RampIndex - Struct for start and end of ramp to fit - - Returns - ------- - RampFit struct of slope, read_var, poisson_var - """ - cdef int n_resultants = ramp.end - ramp.start + 1 - - # Special case where there is no or one resultant, there is no fit and - # we bail out before any computations. - # Note that in this case, we cannot compute the slope or the variances - # because these computations require at least two resultants. Therefore, - # this case is degernate and we return NaNs for the values. - if n_resultants <= 1: - return RampFit(np.nan, np.nan, np.nan) - - # Start computing the fit - - # Setup data for fitting (work over subset of data) - # Recall that the RampIndex contains the index of the first and last - # index of the ramp. Therefore, the Python slice needed to get all the - # data within the ramp is: - # ramp.start:ramp.end + 1 - cdef float[:] resultants = self.resultants[ramp.start:ramp.end + 1] - cdef float[:] t_bar = self.fixed.data.t_bar[ramp.start:ramp.end + 1] - cdef float[:] tau = self.fixed.data.tau[ramp.start:ramp.end + 1] - cdef int[:] n_reads = self.fixed.data.n_reads[ramp.start:ramp.end + 1] - - # Reference read_noise as a local variable to avoid calling through Python - # every time it is accessed. - cdef float read_noise = self.read_noise - - # Compute mid point time - cdef int end = len(resultants) - 1 - cdef float t_bar_mid = (t_bar[0] + t_bar[end]) / 2 - - # Casertano+2022 Eq. 44 - # Note we've departed from Casertano+22 slightly; - # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., - # a CR in the first resultant has boosted the whole ramp high but there - # is no actual signal. - cdef float s = max(resultants[end] - resultants[0], 0) - s = s / sqrt(read_noise**2 + s) - cdef float power = get_power(s) - - # It's easy to use up a lot of dynamic range on something like - # (tbar - tbarmid) ** 10. Rescale these. - cdef float t_scale = (t_bar[end] - t_bar[0]) / 2 - t_scale = 1 if t_scale == 0 else t_scale - - # Initalize the fit loop - cdef int i = 0, j = 0 - cdef vector[float] weights = vector[float](n_resultants) - cdef vector[float] coeffs = vector[float](n_resultants) - cdef RampFit ramp_fit = RampFit(0, 0, 0) - cdef float f0 = 0, f1 = 0, f2 = 0 - - # Issue when tbar[] == tbarmid causes exception otherwise - with cython.cpow(True): - for i in range(n_resultants): - # Casertano+22, Eq. 45 - weights[i] = ((((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * - fabs((t_bar[i] - t_bar_mid) / t_scale) ** power) - - # Casertano+22 Eq. 35 - f0 += weights[i] - f1 += weights[i] * t_bar[i] - f2 += weights[i] * t_bar[i]**2 - - # Casertano+22 Eq. 36 - cdef float det = f2 * f0 - f1 ** 2 - if det == 0: - return ramp_fit - - for i in range(n_resultants): - # Casertano+22 Eq. 37 - coeffs[i] = (f0 * t_bar[i] - f1) * weights[i] / det - - for i in range(n_resultants): - # Casertano+22 Eq. 38 - ramp_fit.slope += coeffs[i] * resultants[i] - - # Casertano+22 Eq. 39 - ramp_fit.read_var += (coeffs[i] ** 2 * read_noise ** 2 / n_reads[i]) - - # Casertano+22 Eq 40 - ramp_fit.poisson_var += coeffs[i] ** 2 * tau[i] - for j in range(i + 1, n_resultants): - ramp_fit.poisson_var += (2 * coeffs[i] * coeffs[j] * t_bar[i]) - - return ramp_fit - def _to_dict(Pixel self): """ This is a private method to convert the Pixel object to a dictionary, so diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index 51a1d0f8..ee16fb2d 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -1,5 +1,7 @@ from libcpp.vector cimport vector +from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel + cdef struct RampIndex: int start @@ -16,3 +18,4 @@ ctypedef vector[RampIndex] RampQueue cpdef RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixel) +cdef RampFit fit_ramp(Pixel pixel, RampIndex ramp) diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index e5032a8e..2050a5a9 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -1,7 +1,14 @@ +import numpy as np + cimport cython cimport numpy as cnp -from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue +from libc.math cimport sqrt, fabs +from libcpp.vector cimport vector + +from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit +from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues +from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel cnp.import_array() @@ -65,4 +72,136 @@ cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixe ramp.end = n_resultants - 1 ramps.push_back(ramp) - return ramps \ No newline at end of file + return ramps + + +@cython.boundscheck(False) +@cython.wraparound(False) +cdef inline float get_power(float signal): + """ + Return the power from Casertano+22, Table 2 + + Parameters + ---------- + signal: float + signal from the resultants + + Returns + ------- + signal power from Table 2 + """ + # Casertano+2022, Table 2 + cdef float[2][6] PTABLE = [ + [-np.inf, 5, 10, 20, 50, 100], + [0, 0.4, 1, 3, 6, 10]] + + cdef int i + for i in range(6): + if signal < PTABLE[0][i]: + return PTABLE[1][i - 1] + + return PTABLE[1][i] + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +cdef inline RampFit fit_ramp(Pixel pixel, RampIndex ramp): + """ + Fit a single ramp using Casertano+22 algorithm. + + Parameters + ---------- + ramp : RampIndex + Struct for start and end of ramp to fit + + Returns + ------- + RampFit struct of slope, read_var, poisson_var + """ + cdef int n_resultants = ramp.end - ramp.start + 1 + + # Special case where there is no or one resultant, there is no fit and + # we bail out before any computations. + # Note that in this case, we cannot compute the slope or the variances + # because these computations require at least two resultants. Therefore, + # this case is degernate and we return NaNs for the values. + if n_resultants <= 1: + return RampFit(np.nan, np.nan, np.nan) + + # Start computing the fit + cdef FixedValues fixed = pixel.fixed + + # Setup data for fitting (work over subset of data) + # Recall that the RampIndex contains the index of the first and last + # index of the ramp. Therefore, the Python slice needed to get all the + # data within the ramp is: + # ramp.start:ramp.end + 1 + cdef float[:] resultants = pixel.resultants[ramp.start:ramp.end + 1] + cdef float[:] t_bar = fixed.data.t_bar[ramp.start:ramp.end + 1] + cdef float[:] tau = fixed.data.tau[ramp.start:ramp.end + 1] + cdef int[:] n_reads = fixed.data.n_reads[ramp.start:ramp.end + 1] + + # Reference read_noise as a local variable to avoid calling through Python + # every time it is accessed. + cdef float read_noise = pixel.read_noise + + # Compute mid point time + cdef int end = len(resultants) - 1 + cdef float t_bar_mid = (t_bar[0] + t_bar[end]) / 2 + + # Casertano+2022 Eq. 44 + # Note we've departed from Casertano+22 slightly; + # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., + # a CR in the first resultant has boosted the whole ramp high but there + # is no actual signal. + cdef float s = max(resultants[end] - resultants[0], 0) + s = s / sqrt(read_noise**2 + s) + cdef float power = get_power(s) + + # It's easy to use up a lot of dynamic range on something like + # (tbar - tbarmid) ** 10. Rescale these. + cdef float t_scale = (t_bar[end] - t_bar[0]) / 2 + t_scale = 1 if t_scale == 0 else t_scale + + # Initalize the fit loop + cdef int i = 0, j = 0 + cdef vector[float] weights = vector[float](n_resultants) + cdef vector[float] coeffs = vector[float](n_resultants) + cdef RampFit ramp_fit = RampFit(0, 0, 0) + cdef float f0 = 0, f1 = 0, f2 = 0 + + # Issue when tbar[] == tbarmid causes exception otherwise + with cython.cpow(True): + for i in range(n_resultants): + # Casertano+22, Eq. 45 + weights[i] = ((((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * + fabs((t_bar[i] - t_bar_mid) / t_scale) ** power) + + # Casertano+22 Eq. 35 + f0 += weights[i] + f1 += weights[i] * t_bar[i] + f2 += weights[i] * t_bar[i]**2 + + # Casertano+22 Eq. 36 + cdef float det = f2 * f0 - f1 ** 2 + if det == 0: + return ramp_fit + + for i in range(n_resultants): + # Casertano+22 Eq. 37 + coeffs[i] = (f0 * t_bar[i] - f1) * weights[i] / det + + for i in range(n_resultants): + # Casertano+22 Eq. 38 + ramp_fit.slope += coeffs[i] * resultants[i] + + # Casertano+22 Eq. 39 + ramp_fit.read_var += (coeffs[i] ** 2 * read_noise ** 2 / n_reads[i]) + + # Casertano+22 Eq 40 + ramp_fit.poisson_var += coeffs[i] ** 2 * tau[i] + for j in range(i + 1, n_resultants): + ramp_fit.poisson_var += (2 * coeffs[i] * coeffs[j] * t_bar[i]) + + return ramp_fit From 6e0811e3544c75e4b4c0a4726031700d0081715d Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Tue, 7 Nov 2023 20:01:04 -0500 Subject: [PATCH 06/30] Refactor pixel --- src/stcal/ramp_fitting/ols_cas22/_pixel.pxd | 2 - src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 59 ++++++++++----------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd index 901c8902..c9c358fb 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd @@ -11,7 +11,5 @@ cdef class Pixel: cdef float[:, :] local_slopes cdef float[:, :] var_read_noise - cdef float[:, :] local_slope_vals(Pixel self) - cpdef Pixel make_pixel(FixedValues fixed, float read_noise, float [:] resultants) diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx index e967dbc7..e0dd99d7 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx @@ -75,35 +75,6 @@ cdef class Pixel: with jump detection. """ - @cython.boundscheck(False) - @cython.wraparound(False) - cdef inline float[:, :] local_slope_vals(Pixel self): - """ - Compute the local slopes between resultants for the pixel - - Returns - ------- - [ - <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, - <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, - ] - """ - cdef float[:] resultants = self.resultants - cdef int end = len(resultants) - - # Read the t_bar_diffs into a local variable to avoid calling through Python - # multiple times - cdef cnp.ndarray[float, ndim=2] t_bar_diffs = np.array(self.fixed.t_bar_diffs, dtype=np.float32) - - cdef cnp.ndarray[float, ndim=2] local_slope_vals = np.zeros((2, end - 1), dtype=np.float32) - - local_slope_vals[Diff.single, :] = (np.subtract(resultants[1:], resultants[:end - 1]) - / t_bar_diffs[Diff.single, :]).astype(np.float32) - local_slope_vals[Diff.double, :end - 2] = (np.subtract(resultants[2:], resultants[:end - 2]) - / t_bar_diffs[Diff.double, :end-2]).astype(np.float32) - local_slope_vals[Diff.double, end - 2] = np.nan # last double difference is undefined - - return local_slope_vals def _to_dict(Pixel self): """ @@ -145,6 +116,34 @@ cdef class Pixel: local_slopes=local_slopes, var_read_noise=var_read_noise) +@cython.boundscheck(False) +@cython.wraparound(False) +cdef inline float[:, :] local_slope_vals(float[:] resultants, FixedValues fixed): + """ + Compute the local slopes between resultants for the pixel + + Returns + ------- + [ + <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, + <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, + ] + """ + cdef int end = len(resultants) + + # Read the t_bar_diffs into a local variable to avoid calling through Python + # multiple times + cdef cnp.ndarray[float, ndim=2] t_bar_diffs = np.array(fixed.t_bar_diffs, dtype=np.float32) + cdef cnp.ndarray[float, ndim=2] local_slope_vals = np.zeros((2, end - 1), dtype=np.float32) + + local_slope_vals[Diff.single, :] = (np.subtract(resultants[1:], resultants[:end - 1]) + / t_bar_diffs[Diff.single, :]).astype(np.float32) + local_slope_vals[Diff.double, :end - 2] = (np.subtract(resultants[2:], resultants[:end - 2]) + / t_bar_diffs[Diff.double, :end-2]).astype(np.float32) + local_slope_vals[Diff.double, end - 2] = np.nan # last double difference is undefined + + return local_slope_vals + @cython.boundscheck(False) @cython.wraparound(False) @@ -180,7 +179,7 @@ cpdef inline Pixel make_pixel(FixedValues fixed, float read_noise, float [:] res # Pre-compute values for jump detection shared by all pixels for this pixel if fixed.use_jump: - pixel.local_slopes = pixel.local_slope_vals() + pixel.local_slopes = local_slope_vals(resultants, fixed) pixel.var_read_noise = (read_noise ** 2) * np.array(fixed.read_recip_coeffs) return pixel From 0a861b827a300b8ce37ed728806cdb1d90a4b929 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 8 Nov 2023 13:06:29 -0500 Subject: [PATCH 07/30] Optimize jump detection module --- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 2 - src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 146 ++++++++++++++------- tests/test_jump_cas22.py | 26 ---- 3 files changed, 96 insertions(+), 78 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 364b0892..054d1409 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -15,6 +15,4 @@ cdef struct RampFits: vector[RampFit] fits RampQueue index - -cpdef float threshold(Thresh thresh, float slope) cdef RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool include_diagnostic) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 32853471..d48910ba 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -1,19 +1,15 @@ -import numpy as np -cimport numpy as cnp -cimport cython +from cython cimport boundscheck, wraparound, cdivision from libcpp cimport bool -from libc.math cimport sqrt, log10 +from libc.math cimport sqrt, log10, fmaxf, NAN, isnan from stcal.ramp_fitting.ols_cas22._core cimport Diff -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, RampFits from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, fit_ramp -from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern -cpdef inline float threshold(Thresh thresh, float slope): +cdef inline float threshold(Thresh thresh, float slope): """ Compute jump threshold @@ -36,9 +32,10 @@ cpdef inline float threshold(Thresh thresh, float slope): -@cython.boundscheck(False) -@cython.wraparound(False) -cdef inline float correction(ReadPattern data, RampIndex ramp, float slope): +@boundscheck(False) +@wraparound(False) +@cdivision(True) +cdef inline float correction(float[:] t_bar, RampIndex ramp, float slope): """ Compute the correction factor for the variance used by a statistic @@ -52,14 +49,19 @@ cdef inline float correction(ReadPattern data, RampIndex ramp, float slope): The computed slope for the ramp """ - cdef float diff = (data.t_bar[ramp.end] - data.t_bar[ramp.start]) + cdef float diff = t_bar[ramp.end] - t_bar[ramp.start] return - slope / diff -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -cdef inline float statstic(Pixel pixel, float slope, RampIndex ramp, int index, int diff): +@boundscheck(False) +@wraparound(False) +@cdivision(True) +cdef inline float statstic(float local_slope, + float var_read_noise, + float var_slope_coeff, + float t_bar_diff_sqr, + float slope, + float correct): """ Compute a single set of fit statistics (delta / sqrt(var)) + correction @@ -85,20 +87,23 @@ cdef inline float statstic(Pixel pixel, float slope, RampIndex ramp, int index, ------- Create a single instance of the stastic for the given parameters """ - cdef FixedValues fixed = pixel.fixed - cdef float delta = (pixel.local_slopes[diff, index] - slope) - cdef float var = ((pixel.var_read_noise[diff, index] + - slope * fixed.var_slope_coeffs[diff, index]) - / fixed.t_bar_diff_sqrs[diff, index]) - cdef float correct = correction(fixed.data, ramp, slope) + cdef float delta = (local_slope - slope) + cdef float var = ((var_read_noise + + slope * var_slope_coeff) + / t_bar_diff_sqr) return (delta / sqrt(var + correct)) -@cython.boundscheck(False) -@cython.wraparound(False) -cdef inline float[:] statistics(Pixel pixel, float slope, RampIndex ramp): +@boundscheck(False) +@wraparound(False) +cdef inline (int, float) statistics(float[:, :] local_slopes, + float[:, :] var_read_noise, + float[:, :] var_slope_coeffs, + float[:, :] t_bar_diff_sqrs, + float[:] t_bar, + float slope, RampIndex ramp): """ Compute fit statistics for jump detection on a single ramp stats[i] = max(stat(i, 0), stat(i, 1)) @@ -115,9 +120,6 @@ cdef inline float[:] statistics(Pixel pixel, float slope, RampIndex ramp): ------- list of statistics for each resultant """ - cdef int start = ramp.start # index of first resultant for ramp - cdef int end = ramp.end # index of last resultant for ramp - # Observe that the length of the ramp's sub array of the resultant would # be `end - start + 1`. However, we are computing single and double # "differences" which means we need to reference at least two points in @@ -126,26 +128,51 @@ cdef inline float[:] statistics(Pixel pixel, float slope, RampIndex ramp): # `start, start+1, start+1, ..., end-2, end-1` # as the second argument to the `range` is the first index outside of the # range + cdef int start = ramp.start # index of first resultant for ramp + cdef int end = ramp.end # index of last resultant for ramp - cdef cnp.ndarray[float, ndim=1] stats = np.zeros(end - start, dtype=np.float32) - - cdef int index, stat - for stat, index in enumerate(range(start, end)): - if index == end - 1: - # It is not possible to compute double differences for the second - # to last resultant in the ramp. Therefore, we just compute the - # single difference for this resultant. - stats[stat] = statstic(pixel, slope, ramp, index, Diff.single) - else: - stats[stat] = max(statstic(pixel, slope, ramp, index, Diff.single), - statstic(pixel, slope, ramp, index, Diff.double)) - - return stats + # Case the enum values into integers for indexing + cdef int single = Diff.single + cdef int double = Diff.double + + cdef float correct = correction(t_bar, ramp, slope) + + cdef float stat, double_stat + + cdef int argmax = 0 + cdef float max_stat = NAN + + cdef int index, stat_index + for stat_index, index in enumerate(range(start, end)): + stat = statstic(local_slopes[single, index], + var_read_noise[single, index], + var_slope_coeffs[single, index], + t_bar_diff_sqrs[single, index], + slope, + correct) + + # It is not possible to compute double differences for the second + # to last resultant in the ramp. Therefore, we include the double + # differences for every stat except the last one. + if index != end - 1: + double_stat = statstic(local_slopes[double, index], + var_read_noise[double, index], + var_slope_coeffs[double, index], + t_bar_diff_sqrs[double, index], + slope, + correct) + stat = fmaxf(stat, double_stat) + + if isnan(max_stat) or stat > max_stat: + max_stat = stat + argmax = stat_index + + return argmax, max_stat -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) +@boundscheck(False) +@wraparound(False) +@cdivision(True) cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool include_diagnostic): """ Compute all the ramps for a single pixel using the Casertano+22 algorithm @@ -170,10 +197,23 @@ cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool ramp_fits.average.read_var = 0 ramp_fits.average.poisson_var = 0 - cdef float [:] stats - cdef int jump0, jump1 + cdef int argmax, jump0, jump1 + cdef float max_stat cdef float weight, total_weight = 0 + cdef float[:, :] local_slopes + cdef float[:, :] var_read_noise + cdef float[:, :] var_slope_coeffs + cdef float[:, :] t_bar_diff_sqrs + cdef float[:] t_bar + + if pixel.fixed.use_jump: + local_slopes = pixel.local_slopes + var_read_noise = pixel.var_read_noise + var_slope_coeffs = pixel.fixed.var_slope_coeffs + t_bar_diff_sqrs = pixel.fixed.t_bar_diff_sqrs + t_bar = pixel.fixed.data.t_bar + # Run while the stack is non-empty while not ramps.empty(): # Remove top ramp of the stack to use @@ -185,12 +225,18 @@ cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool # Run jump detection if enabled if pixel.fixed.use_jump: - stats = statistics(pixel, ramp_fit.slope, ramp) + argmax, max_stat = statistics(local_slopes, + var_read_noise, + var_slope_coeffs, + t_bar_diff_sqrs, + t_bar, + ramp_fit.slope, + ramp) # We have to protect against the case where the passed "ramp" is # only a single point. In that case, stats will be empty. This # will create an error in the max() call. - if len(stats) > 0 and max(stats) > threshold(thresh, ramp_fit.slope): + if not isnan(max_stat) and max_stat > threshold(thresh, ramp_fit.slope): # Compute jump point to create two new ramps # This jump point corresponds to the index of the largest # statistic: @@ -212,7 +258,7 @@ cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool # argmax(stats) does correspond to the jump resultant. # Therefore, we just remove both possible resultants from # consideration. - jump0 = np.argmax(stats) + ramp.start + jump0 = argmax + ramp.start jump1 = jump0 + 1 if include_diagnostic: ramp_fits.jumps.push_back(jump0) @@ -269,7 +315,7 @@ cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool # Start computing the averages # Note we do not do anything in the NaN case for degenerate ramps - if not np.isnan(ramp_fit.slope): + if not isnan(ramp_fit.slope): # protect weight against the extremely unlikely case of a zero # variance weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 75ffb5c1..e8ade35f 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -3,7 +3,6 @@ from numpy.testing import assert_allclose from stcal.ramp_fitting.ols_cas22._fixed import fixed_values_from_metadata -from stcal.ramp_fitting.ols_cas22._jump import threshold from stcal.ramp_fitting.ols_cas22._pixel import make_pixel from stcal.ramp_fitting.ols_cas22._ramp import init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern @@ -132,31 +131,6 @@ def test_init_ramps(): assert ramps[15] == [] -def test_threshold(): - """ - Test the threshold object/fucnction - intercept - constant * log10(slope) = threshold - """ - - # Create the python analog of the Threshold struct - # Note that structs get mapped to/from python as dictionary objects with - # the keys being the struct members. - thresh = { - 'intercept': np.float32(5.5), - 'constant': np.float32(1/3) - } - - # Check the 'intercept' is correctly interpreted. - # Since the log of the input slope is taken, log10(1) = 0, meaning that - # we should directly recover the intercept value in that case. - assert thresh['intercept'] == threshold(thresh, 1.0) - - # Check the 'constant' is correctly interpreted. - # Since we know that the intercept is correctly identified and that `log10(10) = 1`, - # we can use that to check that the constant is correctly interpreted. - assert np.float32(thresh['intercept'] - thresh['constant']) == threshold(thresh, 10.0) - - @pytest.mark.parametrize("use_jump", [True, False]) def test_fixed_values_from_metadata(ramp_data, use_jump): """Test computing the fixed data for all pixels""" From f1552012a120352a1a07bf98642b842550167a27 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 8 Nov 2023 15:38:28 -0500 Subject: [PATCH 08/30] Clean up the ramp fit with some optimization --- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 15 ++++++- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 7 +++- src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 49 +++++++++++----------- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index d48910ba..35cac8d3 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -201,11 +201,17 @@ cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool cdef float max_stat cdef float weight, total_weight = 0 + cdef float[:] resultants = pixel.resultants + cdef float read_noise = pixel.read_noise + + cdef float[:] t_bar = pixel.fixed.data.t_bar + cdef float[:] tau = pixel.fixed.data.tau + cdef int[:] n_reads = pixel.fixed.data.n_reads + cdef float[:, :] local_slopes cdef float[:, :] var_read_noise cdef float[:, :] var_slope_coeffs cdef float[:, :] t_bar_diff_sqrs - cdef float[:] t_bar if pixel.fixed.use_jump: local_slopes = pixel.local_slopes @@ -221,7 +227,12 @@ cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool ramps.pop_back() # Compute fit - ramp_fit = fit_ramp(pixel, ramp) + ramp_fit = fit_ramp(resultants, + t_bar, + tau, + n_reads, + read_noise, + ramp) # Run jump detection if enabled if pixel.fixed.use_jump: diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index ee16fb2d..0ad62b1f 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -18,4 +18,9 @@ ctypedef vector[RampIndex] RampQueue cpdef RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixel) -cdef RampFit fit_ramp(Pixel pixel, RampIndex ramp) +cdef RampFit fit_ramp(float[:] resultants_, + float[:] t_bar_, + float[:] tau_, + int[:] n_reads, + float read_noise, + RampIndex ramp) diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index 2050a5a9..0d6c9b04 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -3,7 +3,7 @@ import numpy as np cimport cython cimport numpy as cnp -from libc.math cimport sqrt, fabs +from libc.math cimport sqrt, fabs, INFINITY, NAN, fmaxf from libcpp.vector cimport vector from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit @@ -74,6 +74,13 @@ cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixe return ramps +# Keeps the static type checker/highligher happy this has no actual effect +ctypedef float[6] _row + +# Casertano+2022, Table 2 +cdef _row[2] PTABLE = [[-INFINITY, 5, 10, 20, 50, 100], + [ 0, 0.4, 1, 3, 6, 10 ]] + @cython.boundscheck(False) @cython.wraparound(False) @@ -90,11 +97,6 @@ cdef inline float get_power(float signal): ------- signal power from Table 2 """ - # Casertano+2022, Table 2 - cdef float[2][6] PTABLE = [ - [-np.inf, 5, 10, 20, 50, 100], - [0, 0.4, 1, 3, 6, 10]] - cdef int i for i in range(6): if signal < PTABLE[0][i]: @@ -102,11 +104,15 @@ cdef inline float get_power(float signal): return PTABLE[1][i] - @cython.boundscheck(False) @cython.wraparound(False) @cython.cdivision(True) -cdef inline RampFit fit_ramp(Pixel pixel, RampIndex ramp): +cdef inline RampFit fit_ramp(float[:] resultants_, + float[:] t_bar_, + float[:] tau_, + int[:] n_reads_, + float read_noise, + RampIndex ramp): """ Fit a single ramp using Casertano+22 algorithm. @@ -127,27 +133,23 @@ cdef inline RampFit fit_ramp(Pixel pixel, RampIndex ramp): # because these computations require at least two resultants. Therefore, # this case is degernate and we return NaNs for the values. if n_resultants <= 1: - return RampFit(np.nan, np.nan, np.nan) + return RampFit(NAN, NAN, NAN) - # Start computing the fit - cdef FixedValues fixed = pixel.fixed + # Compute the fit + cdef int i = 0, j = 0 # Setup data for fitting (work over subset of data) # Recall that the RampIndex contains the index of the first and last # index of the ramp. Therefore, the Python slice needed to get all the # data within the ramp is: # ramp.start:ramp.end + 1 - cdef float[:] resultants = pixel.resultants[ramp.start:ramp.end + 1] - cdef float[:] t_bar = fixed.data.t_bar[ramp.start:ramp.end + 1] - cdef float[:] tau = fixed.data.tau[ramp.start:ramp.end + 1] - cdef int[:] n_reads = fixed.data.n_reads[ramp.start:ramp.end + 1] - - # Reference read_noise as a local variable to avoid calling through Python - # every time it is accessed. - cdef float read_noise = pixel.read_noise + cdef float[:] resultants = resultants_[ramp.start:ramp.end + 1] + cdef float[:] t_bar = t_bar_[ramp.start:ramp.end + 1] + cdef float[:] tau = tau_[ramp.start:ramp.end + 1] + cdef int[:] n_reads = n_reads_[ramp.start:ramp.end + 1] # Compute mid point time - cdef int end = len(resultants) - 1 + cdef int end = n_resultants - 1 cdef float t_bar_mid = (t_bar[0] + t_bar[end]) / 2 # Casertano+2022 Eq. 44 @@ -155,9 +157,9 @@ cdef inline RampFit fit_ramp(Pixel pixel, RampIndex ramp): # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., # a CR in the first resultant has boosted the whole ramp high but there # is no actual signal. - cdef float s = max(resultants[end] - resultants[0], 0) - s = s / sqrt(read_noise**2 + s) - cdef float power = get_power(s) + cdef float power = fmaxf(resultants[end] - resultants[0], 0) + power = power / sqrt(read_noise**2 + power) + power = get_power(power) # It's easy to use up a lot of dynamic range on something like # (tbar - tbarmid) ** 10. Rescale these. @@ -165,7 +167,6 @@ cdef inline RampFit fit_ramp(Pixel pixel, RampIndex ramp): t_scale = 1 if t_scale == 0 else t_scale # Initalize the fit loop - cdef int i = 0, j = 0 cdef vector[float] weights = vector[float](n_resultants) cdef vector[float] coeffs = vector[float](n_resultants) cdef RampFit ramp_fit = RampFit(0, 0, 0) From 64a8720ffe493f73676119d209f7330d99a543aa Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 8 Nov 2023 17:35:39 -0500 Subject: [PATCH 09/30] Drop an unnecessary loop --- src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index 0d6c9b04..d908fedf 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -171,6 +171,7 @@ cdef inline RampFit fit_ramp(float[:] resultants_, cdef vector[float] coeffs = vector[float](n_resultants) cdef RampFit ramp_fit = RampFit(0, 0, 0) cdef float f0 = 0, f1 = 0, f2 = 0 + cdef float coeff # Issue when tbar[] == tbarmid causes exception otherwise with cython.cpow(True): @@ -191,18 +192,18 @@ cdef inline RampFit fit_ramp(float[:] resultants_, for i in range(n_resultants): # Casertano+22 Eq. 37 - coeffs[i] = (f0 * t_bar[i] - f1) * weights[i] / det + coeff = (f0 * t_bar[i] - f1) * weights[i] / det + coeffs[i] = coeff - for i in range(n_resultants): # Casertano+22 Eq. 38 - ramp_fit.slope += coeffs[i] * resultants[i] + ramp_fit.slope += coeff * resultants[i] # Casertano+22 Eq. 39 - ramp_fit.read_var += (coeffs[i] ** 2 * read_noise ** 2 / n_reads[i]) + ramp_fit.read_var += (coeff ** 2 * read_noise ** 2 / n_reads[i]) # Casertano+22 Eq 40 - ramp_fit.poisson_var += coeffs[i] ** 2 * tau[i] - for j in range(i + 1, n_resultants): - ramp_fit.poisson_var += (2 * coeffs[i] * coeffs[j] * t_bar[i]) + ramp_fit.poisson_var += coeff ** 2 * tau[i] + for j in range(i): + ramp_fit.poisson_var += (2 * coeff * coeffs[j] * t_bar[j]) return ramp_fit From 7ac2ad44760369579847c9ba0df85cebc8f83386 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 8 Nov 2023 18:26:33 -0500 Subject: [PATCH 10/30] Clean up imports --- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 22 +++++++----- src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 36 +++++++++++-------- src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 14 +++++--- src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 23 +++++------- .../ramp_fitting/ols_cas22/_read_pattern.pyx | 6 ++-- 5 files changed, 55 insertions(+), 46 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index fa17cb0b..96ae7f8f 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -1,8 +1,9 @@ import numpy as np -cimport numpy as np +cimport numpy as cnp + +from cython cimport boundscheck, wraparound from libcpp cimport bool from libcpp.list cimport list as cpp_list -cimport cython from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance, RampJumpDQ from stcal.ramp_fitting.ols_cas22._fixed cimport fixed_values_from_metadata, FixedValues @@ -15,6 +16,9 @@ from stcal.ramp_fitting.ols_cas22._read_pattern cimport from_read_pattern from typing import NamedTuple, Optional +cnp.import_array() + + # Fix the default Threshold values at compile time these values cannot be overridden # dynamically at runtime. DEF DefaultIntercept = 5.5 @@ -48,11 +52,11 @@ class RampFitOutputs(NamedTuple): fits: Optional[list] = None -@cython.boundscheck(False) -@cython.wraparound(False) -def fit_ramps(np.ndarray[float, ndim=2] resultants, - np.ndarray[int, ndim=2] dq, - np.ndarray[float, ndim=1] read_noise, +@boundscheck(False) +@wraparound(False) +def fit_ramps(cnp.ndarray[float, ndim=2] resultants, + cnp.ndarray[int, ndim=2] dq, + cnp.ndarray[float, ndim=1] read_noise, float read_time, list[list[int]] read_pattern, bool use_jump=False, @@ -109,8 +113,8 @@ def fit_ramps(np.ndarray[float, ndim=2] resultants, # list in the end. cdef cpp_list[RampFits] ramp_fits - cdef np.ndarray[float, ndim=2] parameters = np.zeros((n_pixels, 2), dtype=np.float32) - cdef np.ndarray[float, ndim=2] variances = np.zeros((n_pixels, 3), dtype=np.float32) + cdef cnp.ndarray[float, ndim=2] parameters = np.zeros((n_pixels, 2), dtype=np.float32) + cdef cnp.ndarray[float, ndim=2] variances = np.zeros((n_pixels, 3), dtype=np.float32) # Perform all of the fits cdef RampFits fit diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx index c9113b8f..4b8edfa3 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx @@ -16,14 +16,18 @@ Functions is considered private, only to be used for testing """ import numpy as np -cimport numpy as np -cimport cython +cimport numpy as cnp + +from cython cimport boundscheck, wraparound from libcpp cimport bool from stcal.ramp_fitting.ols_cas22._core cimport Diff from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern +cnp.import_array() + + cdef class FixedValues: """ Class to contain all the values which are fixed for all pixels for a given @@ -94,10 +98,10 @@ cdef class FixedValues: to to the cython class and should not be used by any cython code, and only exists for testing purposes. """ - cdef np.ndarray[float, ndim=2] t_bar_diffs - cdef np.ndarray[float, ndim=2] t_bar_diff_sqrs - cdef np.ndarray[float, ndim=2] read_recip_coeffs - cdef np.ndarray[float, ndim=2] var_slope_coeffs + cdef cnp.ndarray[float, ndim=2] t_bar_diffs + cdef cnp.ndarray[float, ndim=2] t_bar_diff_sqrs + cdef cnp.ndarray[float, ndim=2] read_recip_coeffs + cdef cnp.ndarray[float, ndim=2] var_slope_coeffs if self.use_jump: t_bar_diffs = np.array(self.t_bar_diffs, dtype=np.float32) @@ -140,8 +144,8 @@ cdef class FixedValues: var_slope_coeffs=var_slope_coeffs) -@cython.boundscheck(False) -@cython.wraparound(False) +@boundscheck(False) +@wraparound(False) cdef inline float[:, :] t_bar_diff_vals(ReadPattern data): """ Compute the difference offset of t_bar @@ -155,7 +159,7 @@ cdef inline float[:, :] t_bar_diff_vals(ReadPattern data): """ cdef int end = len(data.t_bar) - cdef np.ndarray[float, ndim=2] t_bar_diff_vals = np.zeros((2, end - 1), dtype=np.float32) + cdef cnp.ndarray[float, ndim=2] t_bar_diff_vals = np.zeros((2, end - 1), dtype=np.float32) t_bar_diff_vals[Diff.single, :] = np.subtract(data.t_bar[1:], data.t_bar[:end - 1]) t_bar_diff_vals[Diff.double, :end - 2] = np.subtract(data.t_bar[2:], data.t_bar[:end - 2]) @@ -163,8 +167,8 @@ cdef inline float[:, :] t_bar_diff_vals(ReadPattern data): return t_bar_diff_vals -@cython.boundscheck(False) -@cython.wraparound(False) +@boundscheck(False) +@wraparound(False) cdef inline float[:, :] read_recip_vals(ReadPattern data): """ Compute the reciprical sum of the number of reads @@ -179,7 +183,7 @@ cdef inline float[:, :] read_recip_vals(ReadPattern data): """ cdef int end = len(data.n_reads) - cdef np.ndarray[float, ndim=2] read_recip_vals = np.zeros((2, end - 1), dtype=np.float32) + cdef cnp.ndarray[float, ndim=2] read_recip_vals = np.zeros((2, end - 1), dtype=np.float32) read_recip_vals[Diff.single, :] = (np.divide(1.0, data.n_reads[1:], dtype=np.float32) + np.divide(1.0, data.n_reads[:end - 1], dtype=np.float32)) @@ -190,8 +194,8 @@ cdef inline float[:, :] read_recip_vals(ReadPattern data): return read_recip_vals -@cython.boundscheck(False) -@cython.wraparound(False) +@boundscheck(False) +@wraparound(False) cdef inline float[:, :] var_slope_vals(ReadPattern data): """ Compute slope part of the jump statistic variances @@ -205,7 +209,7 @@ cdef inline float[:, :] var_slope_vals(ReadPattern data): """ cdef int end = len(data.t_bar) - cdef np.ndarray[float, ndim=2] var_slope_vals = np.zeros((2, end - 1), dtype=np.float32) + cdef cnp.ndarray[float, ndim=2] var_slope_vals = np.zeros((2, end - 1), dtype=np.float32) var_slope_vals[Diff.single, :] = (np.add(data.tau[1:], data.tau[:end - 1]) - 2 * np.minimum(data.t_bar[1:], data.t_bar[:end - 1])) var_slope_vals[Diff.double, :end - 2] = (np.add(data.tau[2:], data.tau[:end - 2]) - 2 * np.minimum(data.t_bar[2:], data.t_bar[:end - 2])) @@ -214,6 +218,8 @@ cdef inline float[:, :] var_slope_vals(ReadPattern data): return var_slope_vals +@boundscheck(False) +@wraparound(False) cpdef inline FixedValues fixed_values_from_metadata(ReadPattern data, bool use_jump): """ Fast constructor for FixedValues class diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx index e0dd99d7..0462b633 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx @@ -16,7 +16,8 @@ Functions """ import numpy as np cimport numpy as cnp -cimport cython + +from cython cimport boundscheck, wraparound from stcal.ramp_fitting.ols_cas22._core cimport Diff @@ -24,6 +25,9 @@ from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel +cnp.import_array() + + cdef class Pixel: """ Class to contain the data to fit ramps for a single pixel. @@ -116,8 +120,8 @@ cdef class Pixel: local_slopes=local_slopes, var_read_noise=var_read_noise) -@cython.boundscheck(False) -@cython.wraparound(False) +@boundscheck(False) +@wraparound(False) cdef inline float[:, :] local_slope_vals(float[:] resultants, FixedValues fixed): """ Compute the local slopes between resultants for the pixel @@ -145,8 +149,8 @@ cdef inline float[:, :] local_slope_vals(float[:] resultants, FixedValues fixed) return local_slope_vals -@cython.boundscheck(False) -@cython.wraparound(False) +@boundscheck(False) +@wraparound(False) cpdef inline Pixel make_pixel(FixedValues fixed, float read_noise, float [:] resultants): """ Fast constructor for the Pixel C class. diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index d908fedf..498b4a22 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -1,7 +1,4 @@ -import numpy as np - -cimport cython -cimport numpy as cnp +from cython cimport boundscheck, wraparound, cdivision, cpow from libc.math cimport sqrt, fabs, INFINITY, NAN, fmaxf from libcpp.vector cimport vector @@ -11,10 +8,8 @@ from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel -cnp.import_array() - -@cython.boundscheck(False) -@cython.wraparound(False) +@boundscheck(False) +@wraparound(False) cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixel): """ Create the initial ramp stack for each pixel @@ -82,8 +77,8 @@ cdef _row[2] PTABLE = [[-INFINITY, 5, 10, 20, 50, 100], [ 0, 0.4, 1, 3, 6, 10 ]] -@cython.boundscheck(False) -@cython.wraparound(False) +@boundscheck(False) +@wraparound(False) cdef inline float get_power(float signal): """ Return the power from Casertano+22, Table 2 @@ -104,9 +99,9 @@ cdef inline float get_power(float signal): return PTABLE[1][i] -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) +@boundscheck(False) +@wraparound(False) +@cdivision(True) cdef inline RampFit fit_ramp(float[:] resultants_, float[:] t_bar_, float[:] tau_, @@ -174,7 +169,7 @@ cdef inline RampFit fit_ramp(float[:] resultants_, cdef float coeff # Issue when tbar[] == tbarmid causes exception otherwise - with cython.cpow(True): + with cpow(True): for i in range(n_resultants): # Casertano+22, Eq. 45 weights[i] = ((((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx index a77416d7..3abf4c63 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx @@ -1,6 +1,6 @@ import numpy as np cimport numpy as cnp -cimport cython +from cython cimport boundscheck, wraparound from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern @@ -38,8 +38,8 @@ cdef class ReadPattern: n_reads=np.array(self.n_reads, dtype=np.int32)) -@cython.boundscheck(False) -@cython.wraparound(False) +@boundscheck(False) +@wraparound(False) cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time): """ Derive the input data from the the read pattern From 9e34653394470df9a0c85afe8edc645fcb1a02d4 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 8 Nov 2023 20:12:14 -0500 Subject: [PATCH 11/30] Speed up pixel pre-compute --- src/stcal/ramp_fitting/ols_cas22/_core.pxd | 5 +- src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 62 +++++++++++++++------ 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pxd b/src/stcal/ramp_fitting/ols_cas22/_core.pxd index 1ef4d17d..2b3f7dda 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pxd @@ -1,6 +1,7 @@ cpdef enum Diff: - single = 0 - double = 1 + single + double + n_diff cpdef enum Parameter: diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx index 0462b633..9a4cae87 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx @@ -17,14 +17,16 @@ Functions import numpy as np cimport numpy as cnp -from cython cimport boundscheck, wraparound +from libc.math cimport NAN +from cython cimport boundscheck, wraparound, cdivision - -from stcal.ramp_fitting.ols_cas22._core cimport Diff +from stcal.ramp_fitting.ols_cas22._core cimport Diff, n_diff from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel + + cnp.import_array() @@ -120,9 +122,22 @@ cdef class Pixel: local_slopes=local_slopes, var_read_noise=var_read_noise) +cdef enum Offsets: + single_slope + double_slope + single_var + double_var + n_offsets + + @boundscheck(False) @wraparound(False) -cdef inline float[:, :] local_slope_vals(float[:] resultants, FixedValues fixed): +@cdivision(True) +cdef inline float[:, :] local_slope_vals(float[:] resultants, + float[:, :] t_bar_diffs, + float[:, :] read_recip_coeffs, + float read_noise, + int end): """ Compute the local slopes between resultants for the pixel @@ -133,20 +148,30 @@ cdef inline float[:, :] local_slope_vals(float[:] resultants, FixedValues fixed) <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, ] """ - cdef int end = len(resultants) + cdef int single = Diff.single + cdef int double = Diff.double - # Read the t_bar_diffs into a local variable to avoid calling through Python - # multiple times - cdef cnp.ndarray[float, ndim=2] t_bar_diffs = np.array(fixed.t_bar_diffs, dtype=np.float32) - cdef cnp.ndarray[float, ndim=2] local_slope_vals = np.zeros((2, end - 1), dtype=np.float32) + cdef int single_slope = Offsets.single_slope + cdef int double_slope = Offsets.double_slope + cdef int single_var = Offsets.single_var + cdef int double_var = Offsets.double_var + + cdef float[:, :] pre_compute = np.empty((n_offsets, end - 1), dtype=np.float32) + cdef float read_noise_sqr = read_noise ** 2 + + cdef int i + for i in range(end - 1): + pre_compute[single_slope, i] = (resultants[i + 1] - resultants[i]) / t_bar_diffs[single, i] + + if i < end - 2: + pre_compute[double_slope, i] = (resultants[i + 2] - resultants[i]) / t_bar_diffs[double, i] + else: + pre_compute[double_slope, i] = NAN # last double difference is undefined - local_slope_vals[Diff.single, :] = (np.subtract(resultants[1:], resultants[:end - 1]) - / t_bar_diffs[Diff.single, :]).astype(np.float32) - local_slope_vals[Diff.double, :end - 2] = (np.subtract(resultants[2:], resultants[:end - 2]) - / t_bar_diffs[Diff.double, :end-2]).astype(np.float32) - local_slope_vals[Diff.double, end - 2] = np.nan # last double difference is undefined + pre_compute[single_var, i] = read_noise_sqr * read_recip_coeffs[single, i] + pre_compute[double_var, i] = read_noise_sqr * read_recip_coeffs[double, i] - return local_slope_vals + return pre_compute @boundscheck(False) @@ -182,8 +207,11 @@ cpdef inline Pixel make_pixel(FixedValues fixed, float read_noise, float [:] res pixel.resultants = resultants # Pre-compute values for jump detection shared by all pixels for this pixel + cdef float[:, :] pre_compute + cdef int n_resultants = len(resultants) if fixed.use_jump: - pixel.local_slopes = local_slope_vals(resultants, fixed) - pixel.var_read_noise = (read_noise ** 2) * np.array(fixed.read_recip_coeffs) + pre_compute = local_slope_vals(resultants, fixed.t_bar_diffs, fixed.read_recip_coeffs, read_noise, n_resultants) + pixel.local_slopes = pre_compute[:n_diff, :] + pixel.var_read_noise = pre_compute[n_diff:n_offsets, :] return pixel From 2f80e057f28b93ab38ad7242855c9afae9bf34a7 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 09:15:46 -0500 Subject: [PATCH 12/30] Eliminate the Pixel class object --- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 15 +- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 10 +- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 64 +++--- src/stcal/ramp_fitting/ols_cas22/_pixel.pxd | 28 ++- src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 191 ++---------------- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 2 - src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 1 - tests/test_jump_cas22.py | 109 +++++----- 8 files changed, 142 insertions(+), 278 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index 96ae7f8f..2cfa1541 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -7,7 +7,7 @@ from libcpp.list cimport list as cpp_list from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance, RampJumpDQ from stcal.ramp_fitting.ols_cas22._fixed cimport fixed_values_from_metadata, FixedValues -from stcal.ramp_fitting.ols_cas22._pixel cimport make_pixel +from stcal.ramp_fitting.ols_cas22._pixel cimport n_pixel_offsets from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, fit_jumps, RampFits from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps @@ -116,13 +116,22 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, cdef cnp.ndarray[float, ndim=2] parameters = np.zeros((n_pixels, 2), dtype=np.float32) cdef cnp.ndarray[float, ndim=2] variances = np.zeros((n_pixels, 3), dtype=np.float32) + cdef float[:, :] pixel = np.empty((n_pixel_offsets, fixed.data.n_resultants - 1), dtype=np.float32) + # Perform all of the fits cdef RampFits fit cdef int index for index in range(n_pixels): # Fit all the ramps for the given pixel - fit = fit_jumps(make_pixel(fixed, read_noise[index], resultants[:, index]), - init_ramps(dq, n_resultants, index), thresh, include_diagnostic) + fit = fit_jumps(resultants[:, index], + read_noise[index], + init_ramps(dq, n_resultants, index), + fixed, + pixel, + thresh, + include_diagnostic) + # fit = fit_jumps(make_pixel(fixed, read_noise[index], resultants[:, index]), + # init_ramps(dq, n_resultants, index), thresh, include_diagnostic) parameters[index, Parameter.slope] = fit.average.slope diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 054d1409..579f6ea8 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -2,7 +2,7 @@ from libcpp cimport bool from libcpp.vector cimport vector from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampQueue -from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel +from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues cdef struct Thresh: float intercept @@ -15,4 +15,10 @@ cdef struct RampFits: vector[RampFit] fits RampQueue index -cdef RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool include_diagnostic) +cdef RampFits fit_jumps(float[:] resultants, + float read_noise, + RampQueue ramps, + FixedValues fixed, + float[:, :] pixel, + Thresh thresh, + bool include_diagnostic) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 35cac8d3..f6f2c191 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -6,8 +6,9 @@ from libc.math cimport sqrt, log10, fmaxf, NAN, isnan from stcal.ramp_fitting.ols_cas22._core cimport Diff from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, RampFits -from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel -from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, fit_ramp +from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues +from stcal.ramp_fitting.ols_cas22._pixel cimport PixelOffsets, fill_pixel_values +from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, fit_ramp cdef inline float threshold(Thresh thresh, float slope): """ @@ -98,12 +99,11 @@ cdef inline float statstic(float local_slope, @boundscheck(False) @wraparound(False) -cdef inline (int, float) statistics(float[:, :] local_slopes, - float[:, :] var_read_noise, - float[:, :] var_slope_coeffs, - float[:, :] t_bar_diff_sqrs, - float[:] t_bar, - float slope, RampIndex ramp): +cdef inline (int, float) statistics(float[:, :] pixel, + float[:, :] var_slope_coeffs, + float[:, :] t_bar_diff_sqrs, + float[:] t_bar, + float slope, RampIndex ramp): """ Compute fit statistics for jump detection on a single ramp stats[i] = max(stat(i, 0), stat(i, 1)) @@ -135,6 +135,11 @@ cdef inline (int, float) statistics(float[:, :] local_slopes, cdef int single = Diff.single cdef int double = Diff.double + cdef int single_local_slope = PixelOffsets.single_local_slope + cdef int double_local_slope = PixelOffsets.double_local_slope + cdef int single_var_read_noise = PixelOffsets.single_var_read_noise + cdef int double_var_read_noise = PixelOffsets.double_var_read_noise + cdef float correct = correction(t_bar, ramp, slope) cdef float stat, double_stat @@ -144,8 +149,8 @@ cdef inline (int, float) statistics(float[:, :] local_slopes, cdef int index, stat_index for stat_index, index in enumerate(range(start, end)): - stat = statstic(local_slopes[single, index], - var_read_noise[single, index], + stat = statstic(pixel[single_local_slope, index], + pixel[single_var_read_noise, index], var_slope_coeffs[single, index], t_bar_diff_sqrs[single, index], slope, @@ -155,8 +160,8 @@ cdef inline (int, float) statistics(float[:, :] local_slopes, # to last resultant in the ramp. Therefore, we include the double # differences for every stat except the last one. if index != end - 1: - double_stat = statstic(local_slopes[double, index], - var_read_noise[double, index], + double_stat = statstic(pixel[double_local_slope, index], + pixel[double_var_read_noise, index], var_slope_coeffs[double, index], t_bar_diff_sqrs[double, index], slope, @@ -173,7 +178,13 @@ cdef inline (int, float) statistics(float[:, :] local_slopes, @boundscheck(False) @wraparound(False) @cdivision(True) -cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool include_diagnostic): +cdef inline RampFits fit_jumps(float[:] resultants, + float read_noise, + RampQueue ramps, + FixedValues fixed, + float[:, :] pixel, + Thresh thresh, + bool include_diagnostic): """ Compute all the ramps for a single pixel using the Casertano+22 algorithm with jump detection. @@ -201,24 +212,18 @@ cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool cdef float max_stat cdef float weight, total_weight = 0 - cdef float[:] resultants = pixel.resultants - cdef float read_noise = pixel.read_noise - - cdef float[:] t_bar = pixel.fixed.data.t_bar - cdef float[:] tau = pixel.fixed.data.tau - cdef int[:] n_reads = pixel.fixed.data.n_reads + cdef float[:] t_bar = fixed.data.t_bar + cdef float[:] tau = fixed.data.tau + cdef int[:] n_reads = fixed.data.n_reads - cdef float[:, :] local_slopes - cdef float[:, :] var_read_noise cdef float[:, :] var_slope_coeffs cdef float[:, :] t_bar_diff_sqrs - if pixel.fixed.use_jump: - local_slopes = pixel.local_slopes - var_read_noise = pixel.var_read_noise - var_slope_coeffs = pixel.fixed.var_slope_coeffs - t_bar_diff_sqrs = pixel.fixed.t_bar_diff_sqrs - t_bar = pixel.fixed.data.t_bar + if fixed.use_jump: + pixel = fill_pixel_values(pixel, resultants, fixed.t_bar_diffs, fixed.read_recip_coeffs, read_noise, fixed.data.n_resultants) + var_slope_coeffs = fixed.var_slope_coeffs + t_bar_diff_sqrs = fixed.t_bar_diff_sqrs + t_bar = fixed.data.t_bar # Run while the stack is non-empty while not ramps.empty(): @@ -235,9 +240,8 @@ cdef inline RampFits fit_jumps(Pixel pixel, RampQueue ramps, Thresh thresh, bool ramp) # Run jump detection if enabled - if pixel.fixed.use_jump: - argmax, max_stat = statistics(local_slopes, - var_read_noise, + if fixed.use_jump: + argmax, max_stat = statistics(pixel, var_slope_coeffs, t_bar_diff_sqrs, t_bar, diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd index c9c358fb..a3a30f6f 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd @@ -1,15 +1,13 @@ -from libcpp cimport bool - -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues -from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampFit - -cdef class Pixel: - cdef FixedValues fixed - cdef float read_noise - cdef float[:] resultants - - cdef float[:, :] local_slopes - cdef float[:, :] var_read_noise - - -cpdef Pixel make_pixel(FixedValues fixed, float read_noise, float [:] resultants) +cpdef enum PixelOffsets: + single_local_slope + double_local_slope + single_var_read_noise + double_var_read_noise + n_pixel_offsets + +cpdef float[:, :] fill_pixel_values(float[:, :] pixel, + float[:] resultants, + float[:, :] t_bar_diffs, + float[:, :] read_recip_coeffs, + float read_noise, + int n_resultants) \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx index 9a4cae87..a78d4ff7 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx @@ -14,130 +14,22 @@ Functions - cpdef gives a python wrapper, but the python version of this method is considered private, only to be used for testing """ -import numpy as np -cimport numpy as cnp - from libc.math cimport NAN from cython cimport boundscheck, wraparound, cdivision -from stcal.ramp_fitting.ols_cas22._core cimport Diff, n_diff -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues -from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel - - - - -cnp.import_array() - - -cdef class Pixel: - """ - Class to contain the data to fit ramps for a single pixel. - This data is drawn from for all ramps for a single pixel. - This class pre-computes jump detection values shared by all ramps - for a given pixel. - - Parameters - ---------- - fixed : FixedValues - The object containing all the values and metadata which is fixed for a - given read pattern> - read_noise : float - The read noise for the given pixel - resultants : float [:] - Resultants input for the given pixel - - local_slopes : float [:, :] - These are the local slopes between the resultants for the pixel. - single difference local slope: - local_slopes[Diff.single, :] = (resultants[i+1] - resultants[i]) - / (t_bar[i+1] - t_bar[i]) - double difference local slope: - local_slopes[Diff.double, :] = (resultants[i+2] - resultants[i]) - / (t_bar[i+2] - t_bar[i]) - var_read_noise : float [:, :] - The read noise variance term of the jump statistics - single difference read noise variance: - var_read_noise[Diff.single, :] = read_noise * ((1/n_reads[i+1]) + (1/n_reads[i])) - double difference read_noise variance: - var_read_noise[Diff.doule, :] = read_noise * ((1/n_reads[i+2]) + (1/n_reads[i])) - - Notes - ----- - - local_slopes and var_read_noise are only computed if use_jump is True. - These values represent reused computations for jump detection which are - used by every ramp for the given pixel for jump detection. They are - computed once and stored for reuse by all ramp computations for the pixel. - - The computations are done using vectorized operations for some performance - increases. However, this is marginal compaired with the performance increase - from pre-computing the values and reusing them. - - Methods - ------- - fit_ramp (ramp_index) : method - Compute the ramp fit for a single ramp defined by an inputed RampIndex - fit_ramps (ramp_stack) : method - Compute all the ramps for a single pixel using the Casertano+22 algorithm - with jump detection. - """ - - - def _to_dict(Pixel self): - """ - This is a private method to convert the Pixel object to a dictionary, so - that attributes can be directly accessed in python. Note that this is - needed because class attributes cannot be accessed on cython classes - directly in python. Instead they need to be accessed or set using a - python compatible method. This method is a pure puthon method bound - to to the cython class and should not be used by any cython code, and - only exists for testing purposes. - """ - - cdef cnp.ndarray[float, ndim=1] resultants_ = np.array(self.resultants, dtype=np.float32) - - cdef cnp.ndarray[float, ndim=2] local_slopes - cdef cnp.ndarray[float, ndim=2] var_read_noise - - if self.fixed.use_jump: - local_slopes = np.array(self.local_slopes, dtype=np.float32) - var_read_noise = np.array(self.var_read_noise, dtype=np.float32) - else: - try: - self.local_slopes - except AttributeError: - local_slopes = np.array([[np.nan],[np.nan]], dtype=np.float32) - else: - raise AttributeError("local_slopes should not exist") - - try: - self.var_read_noise - except AttributeError: - var_read_noise = np.array([[np.nan],[np.nan]], dtype=np.float32) - else: - raise AttributeError("var_read_noise should not exist") - - return dict(fixed=self.fixed._to_dict(), - resultants=resultants_, - read_noise=self.read_noise, - local_slopes=local_slopes, - var_read_noise=var_read_noise) - -cdef enum Offsets: - single_slope - double_slope - single_var - double_var - n_offsets +from stcal.ramp_fitting.ols_cas22._core cimport Diff +from stcal.ramp_fitting.ols_cas22._pixel cimport PixelOffsets @boundscheck(False) @wraparound(False) @cdivision(True) -cdef inline float[:, :] local_slope_vals(float[:] resultants, - float[:, :] t_bar_diffs, - float[:, :] read_recip_coeffs, - float read_noise, - int end): +cpdef inline float[:, :] fill_pixel_values(float[:, :] pixel, + float[:] resultants, + float[:, :] t_bar_diffs, + float[:, :] read_recip_coeffs, + float read_noise, + int n_resultants): """ Compute the local slopes between resultants for the pixel @@ -151,67 +43,24 @@ cdef inline float[:, :] local_slope_vals(float[:] resultants, cdef int single = Diff.single cdef int double = Diff.double - cdef int single_slope = Offsets.single_slope - cdef int double_slope = Offsets.double_slope - cdef int single_var = Offsets.single_var - cdef int double_var = Offsets.double_var + cdef int single_slope = PixelOffsets.single_local_slope + cdef int double_slope = PixelOffsets.double_local_slope + cdef int single_var = PixelOffsets.single_var_read_noise + cdef int double_var = PixelOffsets.double_var_read_noise - cdef float[:, :] pre_compute = np.empty((n_offsets, end - 1), dtype=np.float32) + # cdef float[:, :] pixel = np.empty((n_offsets, n_resultants - 1), dtype=np.float32) cdef float read_noise_sqr = read_noise ** 2 cdef int i - for i in range(end - 1): - pre_compute[single_slope, i] = (resultants[i + 1] - resultants[i]) / t_bar_diffs[single, i] + for i in range(n_resultants - 1): + pixel[single_slope, i] = (resultants[i + 1] - resultants[i]) / t_bar_diffs[single, i] - if i < end - 2: - pre_compute[double_slope, i] = (resultants[i + 2] - resultants[i]) / t_bar_diffs[double, i] + if i < n_resultants - 2: + pixel[double_slope, i] = (resultants[i + 2] - resultants[i]) / t_bar_diffs[double, i] else: - pre_compute[double_slope, i] = NAN # last double difference is undefined - - pre_compute[single_var, i] = read_noise_sqr * read_recip_coeffs[single, i] - pre_compute[double_var, i] = read_noise_sqr * read_recip_coeffs[double, i] - - return pre_compute - - -@boundscheck(False) -@wraparound(False) -cpdef inline Pixel make_pixel(FixedValues fixed, float read_noise, float [:] resultants): - """ - Fast constructor for the Pixel C class. - This creates a Pixel object for a single pixel from the input data. - - This is signifantly faster than using the `__init__` or `__cinit__` - this is because this does not have to pass through the Python as part - of the construction. - - Parameters - ---------- - fixed : FixedValues - Fixed values for all pixels - read_noise : float - read noise for the single pixel - resultants : float [:] - array of resultants for the single pixel - - memoryview of a numpy array to avoid passing through Python - - Return - ------ - Pixel C-class object (with pre-computed values if use_jump is True) - """ - cdef Pixel pixel = Pixel() - - # Fill in input information for pixel - pixel.fixed = fixed - pixel.read_noise = read_noise - pixel.resultants = resultants + pixel[double_slope, i] = NAN # last double difference is undefined - # Pre-compute values for jump detection shared by all pixels for this pixel - cdef float[:, :] pre_compute - cdef int n_resultants = len(resultants) - if fixed.use_jump: - pre_compute = local_slope_vals(resultants, fixed.t_bar_diffs, fixed.read_recip_coeffs, read_noise, n_resultants) - pixel.local_slopes = pre_compute[:n_diff, :] - pixel.var_read_noise = pre_compute[n_diff:n_offsets, :] + pixel[single_var, i] = read_noise_sqr * read_recip_coeffs[single, i] + pixel[double_var, i] = read_noise_sqr * read_recip_coeffs[double, i] return pixel diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index 0ad62b1f..3f2ffca4 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -1,7 +1,5 @@ from libcpp.vector cimport vector -from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel - cdef struct RampIndex: int start diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index 498b4a22..e61014e1 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -5,7 +5,6 @@ from libcpp.vector cimport vector from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues -from stcal.ramp_fitting.ols_cas22._pixel cimport Pixel @boundscheck(False) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index e8ade35f..b61e9227 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -3,7 +3,7 @@ from numpy.testing import assert_allclose from stcal.ramp_fitting.ols_cas22._fixed import fixed_values_from_metadata -from stcal.ramp_fitting.ols_cas22._pixel import make_pixel +from stcal.ramp_fitting.ols_cas22._pixel import fill_pixel_values, PixelOffsets from stcal.ramp_fitting.ols_cas22._ramp import init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern @@ -257,59 +257,60 @@ def pixel_data(ramp_data): yield resultants, metadata -@pytest.mark.parametrize("use_jump", [True, False]) -def test_make_pixel(pixel_data, use_jump): - """Test computing the initial pixel data""" - resultants, metadata = pixel_data - - data = metadata._to_dict() - t_bar = data['t_bar'] - tau = data['tau'] - n_reads = data['n_reads'] - - fixed = fixed_values_from_metadata(metadata, use_jump) - - # Note this is converted to a dictionary so we can directly interrogate the - # variables in question - pixel = make_pixel(fixed, READ_NOISE, resultants)._to_dict() - - # Basic sanity checks that data passed in survives - assert (pixel['resultants'] == resultants).all() - assert READ_NOISE == pixel['read_noise'] - - # the "fixed" data is not checked as this is already done above - - # Check the computed data - # These are computed via vectorized operations in the main code, here we - # check using item-by-item operations - if use_jump: - single_gen = zip(pixel['local_slopes'][Diff.single], pixel['var_read_noise'][Diff.single]) - double_gen = zip(pixel['local_slopes'][Diff.double], pixel['var_read_noise'][Diff.double]) - - for index, (local_slope_1, var_read_noise_1) in enumerate(single_gen): - assert local_slope_1 == ( - (resultants[index + 1] - resultants[index]) / (t_bar[index + 1] - t_bar[index])) - assert var_read_noise_1 == np.float32(READ_NOISE ** 2)* ( - np.float32(1 / n_reads[index + 1]) + np.float32(1 / n_reads[index]) - ) - - for index, (local_slope_2, var_read_noise_2) in enumerate(double_gen): - if index == len(pixel['local_slopes'][1]) - 1: - # Last value must be NaN - assert np.isnan(local_slope_2) - assert np.isnan(var_read_noise_2) - else: - assert local_slope_2 == ( - (resultants[index + 2] - resultants[index]) / (t_bar[index + 2] - t_bar[index]) - ) - assert var_read_noise_2 == np.float32(READ_NOISE ** 2) * ( - np.float32(1 / n_reads[index + 2]) + np.float32(1 / n_reads[index]) - ) - else: - # If not using jumps, these values should not even exist. However, for wrapping - # purposes, they are checked to be non-existent and then set to NaN - assert np.isnan(pixel['local_slopes']).all() - assert np.isnan(pixel['var_read_noise']).all() +# @pytest.mark.parametrize("use_jump", [True, False]) +# def test_fill_pixel_values(pixel_data, use_jump): +# """Test computing the initial pixel data""" +# resultants, metadata = pixel_data + +# data = metadata._to_dict() +# t_bar = data['t_bar'] +# tau = data['tau'] +# n_reads = data['n_reads'] + +# fixed = fixed_values_from_metadata(metadata, use_jump) +# pixel = np.empty((PixelOffsets.n_pixel_offsets, fixed.data.n_resultants - 1), dtype=np.float32) + +# # Note this is converted to a dictionary so we can directly interrogate the +# # variables in question +# pixel = fill_pixel_values(pixel, resultants, fixed.t_bar_diffs, fixed.read_recip_coeffs, READ_NOISE, fixed.data.n_resultants) + +# # Basic sanity checks that data passed in survives +# assert (pixel['resultants'] == resultants).all() +# assert READ_NOISE == pixel['read_noise'] + +# # the "fixed" data is not checked as this is already done above + +# # Check the computed data +# # These are computed via vectorized operations in the main code, here we +# # check using item-by-item operations +# if use_jump: +# single_gen = zip(pixel['local_slopes'][Diff.single], pixel['var_read_noise'][Diff.single]) +# double_gen = zip(pixel['local_slopes'][Diff.double], pixel['var_read_noise'][Diff.double]) + +# for index, (local_slope_1, var_read_noise_1) in enumerate(single_gen): +# assert local_slope_1 == ( +# (resultants[index + 1] - resultants[index]) / (t_bar[index + 1] - t_bar[index])) +# assert var_read_noise_1 == np.float32(READ_NOISE ** 2)* ( +# np.float32(1 / n_reads[index + 1]) + np.float32(1 / n_reads[index]) +# ) + +# for index, (local_slope_2, var_read_noise_2) in enumerate(double_gen): +# if index == len(pixel['local_slopes'][1]) - 1: +# # Last value must be NaN +# assert np.isnan(local_slope_2) +# assert np.isnan(var_read_noise_2) +# else: +# assert local_slope_2 == ( +# (resultants[index + 2] - resultants[index]) / (t_bar[index + 2] - t_bar[index]) +# ) +# assert var_read_noise_2 == np.float32(READ_NOISE ** 2) * ( +# np.float32(1 / n_reads[index + 2]) + np.float32(1 / n_reads[index]) +# ) +# else: +# # If not using jumps, these values should not even exist. However, for wrapping +# # purposes, they are checked to be non-existent and then set to NaN +# assert np.isnan(pixel['local_slopes']).all() +# assert np.isnan(pixel['var_read_noise']).all() @pytest.fixture(scope="module") From 8101ca116233c79a4c5b397d8ac2f5069c48587f Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 09:58:41 -0500 Subject: [PATCH 13/30] Speed up fixed pre-compute --- src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 103 ++++++++++++-------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx index 4b8edfa3..f34f2016 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx @@ -19,6 +19,8 @@ import numpy as np cimport numpy as cnp from cython cimport boundscheck, wraparound + +from libc.math cimport NAN from libcpp cimport bool from stcal.ramp_fitting.ols_cas22._core cimport Diff @@ -144,9 +146,24 @@ cdef class FixedValues: var_slope_coeffs=var_slope_coeffs) +cpdef enum FixedOffsets: + single_t_bar_diff + double_t_bar_diff + single_t_bar_diff_sqr + double_t_bar_diff_sqr + single_read_recip + double_read_recip + single_var_slope_val + double_var_slope_val + n_fixed_offsets + + @boundscheck(False) @wraparound(False) -cdef inline float[:, :] t_bar_diff_vals(ReadPattern data): +cdef inline float[:, :] fill_fixed_values(float[:] t_bar, + float[:] tau, + int[:] n_reads, + int end): """ Compute the difference offset of t_bar @@ -156,42 +173,54 @@ cdef inline float[:, :] t_bar_diff_vals(ReadPattern data): , , ] - """ - cdef int end = len(data.t_bar) - - cdef cnp.ndarray[float, ndim=2] t_bar_diff_vals = np.zeros((2, end - 1), dtype=np.float32) - - t_bar_diff_vals[Diff.single, :] = np.subtract(data.t_bar[1:], data.t_bar[:end - 1]) - t_bar_diff_vals[Diff.double, :end - 2] = np.subtract(data.t_bar[2:], data.t_bar[:end - 2]) - t_bar_diff_vals[Diff.double, end - 2] = np.nan # last double difference is undefined - - return t_bar_diff_vals - -@boundscheck(False) -@wraparound(False) -cdef inline float[:, :] read_recip_vals(ReadPattern data): - """ - Compute the reciprical sum of the number of reads - - Returns - ------- + [ + ** 2, + ** 2, + ] [ <(1/n_reads[i+1] + 1/n_reads[i])>, <(1/n_reads[i+2] + 1/n_reads[i])>, ] - + [ + <(tau[i] + tau[i+1] - 2 * min(t_bar[i], t_bar[i+1]))>, + <(tau[i] + tau[i+2] - 2 * min(t_bar[i], t_bar[i+2]))>, + ] """ - cdef int end = len(data.n_reads) - - cdef cnp.ndarray[float, ndim=2] read_recip_vals = np.zeros((2, end - 1), dtype=np.float32) - read_recip_vals[Diff.single, :] = (np.divide(1.0, data.n_reads[1:], dtype=np.float32) + - np.divide(1.0, data.n_reads[:end - 1], dtype=np.float32)) - read_recip_vals[Diff.double, :end - 2] = (np.divide(1.0, data.n_reads[2:], dtype=np.float32) + - np.divide(1.0, data.n_reads[:end - 2], dtype=np.float32)) - read_recip_vals[Diff.double, end - 2] = np.nan # last double difference is undefined + cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff + cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff + cdef int single_t_bar_diff_sqr = FixedOffsets.single_t_bar_diff_sqr + cdef int double_t_bar_diff_sqr = FixedOffsets.double_t_bar_diff_sqr + cdef int single_read_recip = FixedOffsets.single_read_recip + cdef int double_read_recip = FixedOffsets.double_read_recip + cdef int single_var_slope_val = FixedOffsets.single_var_slope_val + cdef int double_var_slope_val = FixedOffsets.double_var_slope_val + + cdef float[:, :] pre_compute = np.empty((n_fixed_offsets, end - 1), dtype=np.float32) + + # Coerce division to be using floats + cdef float num = 1 + + cdef int i + for i in range(end - 1): + pre_compute[single_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] + pre_compute[single_t_bar_diff_sqr, i] = pre_compute[single_t_bar_diff, i] ** 2 + pre_compute[single_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) + pre_compute[single_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) + + if i < end - 2: + pre_compute[double_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] + pre_compute[double_t_bar_diff_sqr, i] = pre_compute[double_t_bar_diff, i] ** 2 + pre_compute[double_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) + pre_compute[double_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) + else: + # Last double difference is undefined + pre_compute[double_t_bar_diff, i] = NAN + pre_compute[double_t_bar_diff_sqr, i] = NAN + pre_compute[double_read_recip, i] = NAN + pre_compute[double_var_slope_val, i] = NAN - return read_recip_vals + return pre_compute @boundscheck(False) @@ -202,10 +231,6 @@ cdef inline float[:, :] var_slope_vals(ReadPattern data): Returns ------- - [ - <(tau[i] + tau[i+1] - min(t_bar[i], t_bar[i+1])) * correction(i, i+1)>, - <(tau[i] + tau[i+2] - min(t_bar[i], t_bar[i+2])) * correction(i, i+2)>, - ] """ cdef int end = len(data.t_bar) @@ -249,10 +274,12 @@ cpdef inline FixedValues fixed_values_from_metadata(ReadPattern data, bool use_j fixed.data = data # Pre-compute jump detection computations shared by all pixels + cdef float[:, :] pre_compute if use_jump: - fixed.t_bar_diffs = t_bar_diff_vals(data) - fixed.t_bar_diff_sqrs = np.square(fixed.t_bar_diffs, dtype=np.float32) - fixed.read_recip_coeffs = read_recip_vals(data) - fixed.var_slope_coeffs = var_slope_vals(data) + pre_compute = fill_fixed_values(data.t_bar, data.tau, data.n_reads, data.n_resultants) + fixed.t_bar_diffs = pre_compute[:FixedOffsets.double_t_bar_diff + 1, :] + fixed.t_bar_diff_sqrs = pre_compute[FixedOffsets.single_t_bar_diff_sqr:FixedOffsets.double_t_bar_diff_sqr + 1, :] + fixed.read_recip_coeffs = pre_compute[FixedOffsets.single_read_recip:FixedOffsets.double_read_recip + 1, :] + fixed.var_slope_coeffs = pre_compute[FixedOffsets.single_var_slope_val:FixedOffsets.double_var_slope_val + 1, :] return fixed From 58ba1d067e28261b1c20bb01aede625fd3d0248f Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 10:53:39 -0500 Subject: [PATCH 14/30] Eliminate the Fixed class object --- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 23 +- src/stcal/ramp_fitting/ols_cas22/_fixed.pxd | 29 ++- src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 243 ++---------------- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 9 +- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 54 ++-- src/stcal/ramp_fitting/ols_cas22/_pixel.pxd | 3 +- src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 29 ++- src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 1 - .../ramp_fitting/ols_cas22/_read_pattern.pxd | 2 - .../ramp_fitting/ols_cas22/_read_pattern.pyx | 4 +- tests/test_jump_cas22.py | 123 +++++---- 11 files changed, 162 insertions(+), 358 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index 2cfa1541..67af4819 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -6,12 +6,12 @@ from libcpp cimport bool from libcpp.list cimport list as cpp_list from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance, RampJumpDQ -from stcal.ramp_fitting.ols_cas22._fixed cimport fixed_values_from_metadata, FixedValues +from stcal.ramp_fitting.ols_cas22._fixed cimport fill_fixed_values, n_fixed_offsets from stcal.ramp_fitting.ols_cas22._pixel cimport n_pixel_offsets from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, fit_jumps, RampFits from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps -from stcal.ramp_fitting.ols_cas22._read_pattern cimport from_read_pattern +from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern, from_read_pattern from typing import NamedTuple, Optional @@ -105,7 +105,16 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, f'match number of resultants {n_resultants}') # Pre-compute data for all pixels - cdef FixedValues fixed = fixed_values_from_metadata(from_read_pattern(read_pattern, read_time), use_jump) + cdef ReadPattern metadata = from_read_pattern(read_pattern, read_time) + cdef float[:] t_bar = metadata.t_bar + cdef float[:] tau = metadata.tau + cdef int[:] n_reads = metadata.n_reads + + cdef float[:, :] fixed = np.empty((n_fixed_offsets, n_resultants - 1), dtype=np.float32) + cdef float[:, :] pixel = np.empty((n_pixel_offsets, n_resultants - 1), dtype=np.float32) + if use_jump: + fixed = fill_fixed_values(fixed, t_bar, tau, n_reads, n_resultants) + cdef Thresh thresh = Thresh(intercept, constant) # Use list because this might grow very large which would require constant @@ -116,7 +125,6 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, cdef cnp.ndarray[float, ndim=2] parameters = np.zeros((n_pixels, 2), dtype=np.float32) cdef cnp.ndarray[float, ndim=2] variances = np.zeros((n_pixels, 3), dtype=np.float32) - cdef float[:, :] pixel = np.empty((n_pixel_offsets, fixed.data.n_resultants - 1), dtype=np.float32) # Perform all of the fits cdef RampFits fit @@ -126,12 +134,15 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, fit = fit_jumps(resultants[:, index], read_noise[index], init_ramps(dq, n_resultants, index), + t_bar, + tau, + n_reads, + n_resultants, fixed, pixel, thresh, + use_jump, include_diagnostic) - # fit = fit_jumps(make_pixel(fixed, read_noise[index], resultants[:, index]), - # init_ramps(dq, n_resultants, index), thresh, include_diagnostic) parameters[index, Parameter.slope] = fit.average.slope diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd index 6b624a16..290e8b3b 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd @@ -1,16 +1,17 @@ -from libcpp cimport bool +cpdef enum FixedOffsets: + single_t_bar_diff + double_t_bar_diff + single_t_bar_diff_sqr + double_t_bar_diff_sqr + single_read_recip + double_read_recip + single_var_slope_val + double_var_slope_val + n_fixed_offsets -from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern - -cdef class FixedValues: - cdef bool use_jump - cdef ReadPattern data - - cdef float[:, :] t_bar_diffs - cdef float[:, :] t_bar_diff_sqrs - cdef float[:, :] read_recip_coeffs - cdef float[:, :] var_slope_coeffs - - -cpdef FixedValues fixed_values_from_metadata(ReadPattern data, bool use_jump) +cpdef float[:, :] fill_fixed_values(float[:, :] fixed, + float[:] t_bar, + float[:] tau, + int[:] n_reads, + int n_resultants) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx index f34f2016..61cd3af4 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx @@ -21,149 +21,18 @@ cimport numpy as cnp from cython cimport boundscheck, wraparound from libc.math cimport NAN -from libcpp cimport bool -from stcal.ramp_fitting.ols_cas22._core cimport Diff -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues -from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern +from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets -cnp.import_array() - - -cdef class FixedValues: - """ - Class to contain all the values which are fixed for all pixels for a given - read pattern. - This class is used to pre-compute these values once so that they maybe - reused for all pixels. This is done for performance reasons. - - Parameters - ---------- - use_jump : bool - flag to indicate whether to use jump detection (user input) - - data : ReadPatternMetadata - Metadata struct created from a read pattern - - threshold : Thresh - Parameterization struct for threshold function - - t_bar_diffs : float[:, :] - These are the differences of t_bar used for jump detection. - single differences of t_bar: - t_bar_diffs[Diff.single, :] = (t_bar[i+1] - t_bar[i]) - double differences of t_bar: - t_bar_diffs[Diff.double, :] = (t_bar[i+2] - t_bar[i]) - t_bar_diff_sqrs : float[:, :] - These are the squared differnences of t_bar used for jump detection. - single differences of t_bar: - t_bar_diff_sqrs[Diff.single, :] = (t_bar[i+1] - t_bar[i])**2 - double differences of t_bar: - t_bar_diff_sqrs[Diff.double, :] = (t_bar[i+2] - t_bar[i])**2 - read_recip_coeffs : float[:, :] - Coefficients for the read noise portion of the variance used to compute - the jump detection statistics. These are formed from the reciprocal sum - of the number of reads. - single sum of reciprocal n_reads: - read_recip_coeffs[Diff.single, :] = ((1/n_reads[i+1]) + (1/n_reads[i])) - double sum of reciprocal n_reads: - read_recip_coeffs[Diff.double, :] = ((1/n_reads[i+2]) + (1/n_reads[i])) - var_slope_coeffs : float[:, :] - Coefficients for the slope portion of the variance used to compute the - jump detection statistics, which happend to be fixed for any given ramp - fit. - single of slope variance term: - var_slope_coeffs[Diff.single, :] = (tau[i] + tau[i+1] - - 2 * min(t_bar[i], t_bar[i+1])) - double of slope variance term: - var_slope_coeffs[Diff.double, :] = (tau[i] + tau[i+2] - - 2 * min(t_bar[i], t_bar[i+2])) - - Notes - ----- - - t_bar_diffs, t_bar_diff_sqrs, read_recip_coeffs, var_slope_coeffs are only - computed if use_jump is True. These values represent reused computations - for jump detection which are used by every pixel for jump detection. They - are computed once and stored in the FixedValues for reuse by all pixels. - - The computations are done using vectorized operations for some performance - increases. However, this is marginal compaired with the performance increase - from pre-computing the values and reusing them. - """ - - def _to_dict(FixedValues self): - """ - This is a private method to convert the FixedValues object to a dictionary, - so that attributes can be directly accessed in python. Note that this - is needed because class attributes cannot be accessed on cython classes - directly in python. Instead they need to be accessed or set using a - python compatible method. This method is a pure puthon method bound - to to the cython class and should not be used by any cython code, and - only exists for testing purposes. - """ - cdef cnp.ndarray[float, ndim=2] t_bar_diffs - cdef cnp.ndarray[float, ndim=2] t_bar_diff_sqrs - cdef cnp.ndarray[float, ndim=2] read_recip_coeffs - cdef cnp.ndarray[float, ndim=2] var_slope_coeffs - - if self.use_jump: - t_bar_diffs = np.array(self.t_bar_diffs, dtype=np.float32) - t_bar_diff_sqrs = np.array(self.t_bar_diff_sqrs, dtype=np.float32) - read_recip_coeffs = np.array(self.read_recip_coeffs, dtype=np.float32) - var_slope_coeffs = np.array(self.var_slope_coeffs, dtype=np.float32) - else: - try: - self.t_bar_diffs - except AttributeError: - t_bar_diffs = np.array([[np.nan],[np.nan]], dtype=np.float32) - else: - raise AttributeError("t_bar_diffs should not exist") - - try: - self.t_bar_diff_sqrs - except AttributeError: - t_bar_diff_sqrs = np.array([[np.nan],[np.nan]], dtype=np.float32) - else: - raise AttributeError("t_bar_diff_sqrs should not exist") - - try: - self.read_recip_coeffs - except AttributeError: - read_recip_coeffs = np.array([[np.nan],[np.nan]], dtype=np.float32) - else: - raise AttributeError("read_recip_coeffs should not exist") - - try: - self.var_slope_coeffs - except AttributeError: - var_slope_coeffs = np.array([[np.nan],[np.nan]], dtype=np.float32) - else: - raise AttributeError("var_slope_coeffs should not exist") - - return dict(data=self.data._to_dict(), - t_bar_diffs=t_bar_diffs, - t_bar_diff_sqrs=t_bar_diff_sqrs, - read_recip_coeffs=read_recip_coeffs, - var_slope_coeffs=var_slope_coeffs) - - -cpdef enum FixedOffsets: - single_t_bar_diff - double_t_bar_diff - single_t_bar_diff_sqr - double_t_bar_diff_sqr - single_read_recip - double_read_recip - single_var_slope_val - double_var_slope_val - n_fixed_offsets @boundscheck(False) @wraparound(False) -cdef inline float[:, :] fill_fixed_values(float[:] t_bar, - float[:] tau, - int[:] n_reads, - int end): +cpdef inline float[:, :] fill_fixed_values(float[:, :] fixed, + float[:] t_bar, + float[:] tau, + int[:] n_reads, + int n_resultants): """ Compute the difference offset of t_bar @@ -172,16 +41,10 @@ cdef inline float[:, :] fill_fixed_values(float[:] t_bar, [ , , - ] - [ ** 2, ** 2, - ] - [ <(1/n_reads[i+1] + 1/n_reads[i])>, <(1/n_reads[i+2] + 1/n_reads[i])>, - ] - [ <(tau[i] + tau[i+1] - 2 * min(t_bar[i], t_bar[i+1]))>, <(tau[i] + tau[i+2] - 2 * min(t_bar[i], t_bar[i+2]))>, ] @@ -196,90 +59,26 @@ cdef inline float[:, :] fill_fixed_values(float[:] t_bar, cdef int single_var_slope_val = FixedOffsets.single_var_slope_val cdef int double_var_slope_val = FixedOffsets.double_var_slope_val - cdef float[:, :] pre_compute = np.empty((n_fixed_offsets, end - 1), dtype=np.float32) - # Coerce division to be using floats cdef float num = 1 cdef int i - for i in range(end - 1): - pre_compute[single_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] - pre_compute[single_t_bar_diff_sqr, i] = pre_compute[single_t_bar_diff, i] ** 2 - pre_compute[single_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) - pre_compute[single_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) - - if i < end - 2: - pre_compute[double_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] - pre_compute[double_t_bar_diff_sqr, i] = pre_compute[double_t_bar_diff, i] ** 2 - pre_compute[double_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) - pre_compute[double_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) + for i in range(n_resultants - 1): + fixed[single_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] + fixed[single_t_bar_diff_sqr, i] = fixed[single_t_bar_diff, i] ** 2 + fixed[single_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) + fixed[single_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) + + if i < n_resultants - 2: + fixed[double_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] + fixed[double_t_bar_diff_sqr, i] = fixed[double_t_bar_diff, i] ** 2 + fixed[double_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) + fixed[double_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) else: # Last double difference is undefined - pre_compute[double_t_bar_diff, i] = NAN - pre_compute[double_t_bar_diff_sqr, i] = NAN - pre_compute[double_read_recip, i] = NAN - pre_compute[double_var_slope_val, i] = NAN - - return pre_compute - - -@boundscheck(False) -@wraparound(False) -cdef inline float[:, :] var_slope_vals(ReadPattern data): - """ - Compute slope part of the jump statistic variances - - Returns - ------- - """ - cdef int end = len(data.t_bar) - - cdef cnp.ndarray[float, ndim=2] var_slope_vals = np.zeros((2, end - 1), dtype=np.float32) - - var_slope_vals[Diff.single, :] = (np.add(data.tau[1:], data.tau[:end - 1]) - 2 * np.minimum(data.t_bar[1:], data.t_bar[:end - 1])) - var_slope_vals[Diff.double, :end - 2] = (np.add(data.tau[2:], data.tau[:end - 2]) - 2 * np.minimum(data.t_bar[2:], data.t_bar[:end - 2])) - var_slope_vals[Diff.double, end - 2] = np.nan # last double difference is undefined - - return var_slope_vals - - -@boundscheck(False) -@wraparound(False) -cpdef inline FixedValues fixed_values_from_metadata(ReadPattern data, bool use_jump): - """ - Fast constructor for FixedValues class - Use this instead of an __init__ because it does not incure the overhead - of switching back and forth to python - - Parameters - ---------- - data : ReadPatternMetadata - metadata object created from the read pattern (user input) - threshold : Thresh - threshold object (user input) - use_jump : bool - flag to indicate whether to use jump detection (user input) - - Returns - ------- - FixedValues object (with pre-computed values for jump detection if use_jump - is True) - """ - cdef FixedValues fixed = FixedValues() - - # Fill in input information for all pixels - fixed.use_jump = use_jump - - # Cast vector to a c array - fixed.data = data - - # Pre-compute jump detection computations shared by all pixels - cdef float[:, :] pre_compute - if use_jump: - pre_compute = fill_fixed_values(data.t_bar, data.tau, data.n_reads, data.n_resultants) - fixed.t_bar_diffs = pre_compute[:FixedOffsets.double_t_bar_diff + 1, :] - fixed.t_bar_diff_sqrs = pre_compute[FixedOffsets.single_t_bar_diff_sqr:FixedOffsets.double_t_bar_diff_sqr + 1, :] - fixed.read_recip_coeffs = pre_compute[FixedOffsets.single_read_recip:FixedOffsets.double_read_recip + 1, :] - fixed.var_slope_coeffs = pre_compute[FixedOffsets.single_var_slope_val:FixedOffsets.double_var_slope_val + 1, :] + fixed[double_t_bar_diff, i] = NAN + fixed[double_t_bar_diff_sqr, i] = NAN + fixed[double_read_recip, i] = NAN + fixed[double_var_slope_val, i] = NAN return fixed diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 579f6ea8..533aa8d4 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -2,7 +2,6 @@ from libcpp cimport bool from libcpp.vector cimport vector from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampQueue -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues cdef struct Thresh: float intercept @@ -15,10 +14,16 @@ cdef struct RampFits: vector[RampFit] fits RampQueue index + cdef RampFits fit_jumps(float[:] resultants, float read_noise, RampQueue ramps, - FixedValues fixed, + float[:] t_bar, + float[:] tau, + int[:] n_reads, + int n_resultants, + float[:, :] fixed, float[:, :] pixel, Thresh thresh, + bool use_jump, bool include_diagnostic) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index f6f2c191..ef59915e 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -4,9 +4,8 @@ from libcpp cimport bool from libc.math cimport sqrt, log10, fmaxf, NAN, isnan -from stcal.ramp_fitting.ols_cas22._core cimport Diff from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, RampFits -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues +from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets from stcal.ramp_fitting.ols_cas22._pixel cimport PixelOffsets, fill_pixel_values from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, fit_ramp @@ -31,8 +30,6 @@ cdef inline float threshold(Thresh thresh, float slope): return thresh.intercept - thresh.constant * log10(slope) - - @boundscheck(False) @wraparound(False) @cdivision(True) @@ -59,8 +56,8 @@ cdef inline float correction(float[:] t_bar, RampIndex ramp, float slope): @cdivision(True) cdef inline float statstic(float local_slope, float var_read_noise, - float var_slope_coeff, float t_bar_diff_sqr, + float var_slope_coeff, float slope, float correct): """ @@ -100,8 +97,7 @@ cdef inline float statstic(float local_slope, @boundscheck(False) @wraparound(False) cdef inline (int, float) statistics(float[:, :] pixel, - float[:, :] var_slope_coeffs, - float[:, :] t_bar_diff_sqrs, + float[:, :] fixed, float[:] t_bar, float slope, RampIndex ramp): """ @@ -131,15 +127,17 @@ cdef inline (int, float) statistics(float[:, :] pixel, cdef int start = ramp.start # index of first resultant for ramp cdef int end = ramp.end # index of last resultant for ramp - # Case the enum values into integers for indexing - cdef int single = Diff.single - cdef int double = Diff.double - + # Cast the enum values into integers for indexing cdef int single_local_slope = PixelOffsets.single_local_slope cdef int double_local_slope = PixelOffsets.double_local_slope cdef int single_var_read_noise = PixelOffsets.single_var_read_noise cdef int double_var_read_noise = PixelOffsets.double_var_read_noise + cdef int single_t_bar_diff_sqr = FixedOffsets.single_t_bar_diff_sqr + cdef int double_t_bar_diff_sqr = FixedOffsets.double_t_bar_diff_sqr + cdef int single_var_slope_val = FixedOffsets.single_var_slope_val + cdef int double_var_slope_val = FixedOffsets.double_var_slope_val + cdef float correct = correction(t_bar, ramp, slope) cdef float stat, double_stat @@ -151,8 +149,8 @@ cdef inline (int, float) statistics(float[:, :] pixel, for stat_index, index in enumerate(range(start, end)): stat = statstic(pixel[single_local_slope, index], pixel[single_var_read_noise, index], - var_slope_coeffs[single, index], - t_bar_diff_sqrs[single, index], + fixed[single_t_bar_diff_sqr, index], + fixed[single_var_slope_val, index], slope, correct) @@ -162,8 +160,8 @@ cdef inline (int, float) statistics(float[:, :] pixel, if index != end - 1: double_stat = statstic(pixel[double_local_slope, index], pixel[double_var_read_noise, index], - var_slope_coeffs[double, index], - t_bar_diff_sqrs[double, index], + fixed[double_t_bar_diff_sqr, index], + fixed[double_var_slope_val, index], slope, correct) stat = fmaxf(stat, double_stat) @@ -181,9 +179,14 @@ cdef inline (int, float) statistics(float[:, :] pixel, cdef inline RampFits fit_jumps(float[:] resultants, float read_noise, RampQueue ramps, - FixedValues fixed, + float[:] t_bar, + float[:] tau, + int[:] n_reads, + int n_resultants, + float[:, :] fixed, float[:, :] pixel, Thresh thresh, + bool use_jump, bool include_diagnostic): """ Compute all the ramps for a single pixel using the Casertano+22 algorithm @@ -212,18 +215,8 @@ cdef inline RampFits fit_jumps(float[:] resultants, cdef float max_stat cdef float weight, total_weight = 0 - cdef float[:] t_bar = fixed.data.t_bar - cdef float[:] tau = fixed.data.tau - cdef int[:] n_reads = fixed.data.n_reads - - cdef float[:, :] var_slope_coeffs - cdef float[:, :] t_bar_diff_sqrs - - if fixed.use_jump: - pixel = fill_pixel_values(pixel, resultants, fixed.t_bar_diffs, fixed.read_recip_coeffs, read_noise, fixed.data.n_resultants) - var_slope_coeffs = fixed.var_slope_coeffs - t_bar_diff_sqrs = fixed.t_bar_diff_sqrs - t_bar = fixed.data.t_bar + if use_jump: + pixel = fill_pixel_values(pixel, resultants, fixed, read_noise, n_resultants) # Run while the stack is non-empty while not ramps.empty(): @@ -240,10 +233,9 @@ cdef inline RampFits fit_jumps(float[:] resultants, ramp) # Run jump detection if enabled - if fixed.use_jump: + if use_jump: argmax, max_stat = statistics(pixel, - var_slope_coeffs, - t_bar_diff_sqrs, + fixed, t_bar, ramp_fit.slope, ramp) diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd index a3a30f6f..50f7a041 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd @@ -7,7 +7,6 @@ cpdef enum PixelOffsets: cpdef float[:, :] fill_pixel_values(float[:, :] pixel, float[:] resultants, - float[:, :] t_bar_diffs, - float[:, :] read_recip_coeffs, + float[:, :] fixed, float read_noise, int n_resultants) \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx index a78d4ff7..a6795da0 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx @@ -17,7 +17,7 @@ Functions from libc.math cimport NAN from cython cimport boundscheck, wraparound, cdivision -from stcal.ramp_fitting.ols_cas22._core cimport Diff +from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets from stcal.ramp_fitting.ols_cas22._pixel cimport PixelOffsets @@ -26,8 +26,7 @@ from stcal.ramp_fitting.ols_cas22._pixel cimport PixelOffsets @cdivision(True) cpdef inline float[:, :] fill_pixel_values(float[:, :] pixel, float[:] resultants, - float[:, :] t_bar_diffs, - float[:, :] read_recip_coeffs, + float[:, :] fixed, float read_noise, int n_resultants): """ @@ -38,29 +37,33 @@ cpdef inline float[:, :] fill_pixel_values(float[:, :] pixel, [ <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, + read_noise ** 2 / <(t_bar[i+1] - t_bar[i])>, + read_noise ** 2 / <(t_bar[i+2] - t_bar[i])>, ] """ - cdef int single = Diff.single - cdef int double = Diff.double - cdef int single_slope = PixelOffsets.single_local_slope cdef int double_slope = PixelOffsets.double_local_slope cdef int single_var = PixelOffsets.single_var_read_noise cdef int double_var = PixelOffsets.double_var_read_noise - # cdef float[:, :] pixel = np.empty((n_offsets, n_resultants - 1), dtype=np.float32) + cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff + cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff + cdef int single_read_recip = FixedOffsets.single_read_recip + cdef int double_read_recip = FixedOffsets.double_read_recip + cdef float read_noise_sqr = read_noise ** 2 cdef int i for i in range(n_resultants - 1): - pixel[single_slope, i] = (resultants[i + 1] - resultants[i]) / t_bar_diffs[single, i] + pixel[single_slope, i] = (resultants[i + 1] - resultants[i]) / fixed[single_t_bar_diff, i] + pixel[single_var, i] = read_noise_sqr * fixed[single_read_recip, i] if i < n_resultants - 2: - pixel[double_slope, i] = (resultants[i + 2] - resultants[i]) / t_bar_diffs[double, i] + pixel[double_slope, i] = (resultants[i + 2] - resultants[i]) / fixed[double_t_bar_diff, i] + pixel[double_var, i] = read_noise_sqr * fixed[double_read_recip, i] else: - pixel[double_slope, i] = NAN # last double difference is undefined - - pixel[single_var, i] = read_noise_sqr * read_recip_coeffs[single, i] - pixel[double_var, i] = read_noise_sqr * read_recip_coeffs[double, i] + # The last double difference is undefined + pixel[double_slope, i] = NAN + pixel[double_var, i] = NAN return pixel diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index e61014e1..992a4c4c 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -4,7 +4,6 @@ from libc.math cimport sqrt, fabs, INFINITY, NAN, fmaxf from libcpp.vector cimport vector from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedValues @boundscheck(False) diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd index b4ca0d14..0c5bb4b0 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd @@ -1,6 +1,4 @@ cdef class ReadPattern: - cdef int n_resultants - cdef float[::1] t_bar cdef float[::1] tau cdef int[::1] n_reads diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx index 3abf4c63..0168d67f 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx @@ -32,8 +32,7 @@ cdef class ReadPattern: to to the cython class and should not be used by any cython code, and only exists for testing purposes. """ - return dict(n_resultants=self.n_resultants, - t_bar=np.array(self.t_bar, dtype=np.float32), + return dict(t_bar=np.array(self.t_bar, dtype=np.float32), tau=np.array(self.tau, dtype=np.float32), n_reads=np.array(self.n_reads, dtype=np.int32)) @@ -61,7 +60,6 @@ cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_tim cdef int n_resultants = len(read_pattern) cdef ReadPattern data = ReadPattern() - data.n_resultants = n_resultants data.t_bar = np.empty(n_resultants, dtype=np.float32) data.tau = np.empty(n_resultants, dtype=np.float32) data.n_reads = np.empty(n_resultants, dtype=np.int32) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index b61e9227..0eeb9c6d 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,7 +2,7 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22._fixed import fixed_values_from_metadata +from stcal.ramp_fitting.ols_cas22._fixed import fill_fixed_values from stcal.ramp_fitting.ols_cas22._pixel import fill_pixel_values, PixelOffsets from stcal.ramp_fitting.ols_cas22._ramp import init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern @@ -69,10 +69,9 @@ def test_from_read_pattern(ramp_data): assert 't_bar' in data assert 'tau' in data assert 'n_reads' in data - assert len(data) == 4 + assert len(data) == 3 # Check that the data is correct - assert data['n_resultants'] == len(read_pattern) assert_allclose(data['t_bar'], [7.6, 15.2, 21.279999, 41.040001, 60.799999, 88.159996]) assert_allclose(data['tau'], [5.7, 15.2, 19.928888, 36.023998, 59.448887, 80.593781]) assert np.all(data['n_reads'] == [4, 1, 3, 10, 3, 15]) @@ -131,66 +130,66 @@ def test_init_ramps(): assert ramps[15] == [] -@pytest.mark.parametrize("use_jump", [True, False]) -def test_fixed_values_from_metadata(ramp_data, use_jump): - """Test computing the fixed data for all pixels""" - _, data = ramp_data - - data_dict = data._to_dict() - t_bar = data_dict['t_bar'] - tau = data_dict['tau'] - n_reads = data_dict['n_reads'] - - # Note this is converted to a dictionary so we can directly interrogate the - # variables in question - fixed = fixed_values_from_metadata(data, use_jump)._to_dict() - - # Basic sanity checks that data passed in survives - assert (fixed['data']['t_bar'] == t_bar).all() - assert (fixed['data']['tau'] == tau).all() - assert (fixed['data']['n_reads'] == n_reads).all() - - # Check the computed data - # These are computed via vectorized operations in the main code, here we - # check using item-by-item operations - if use_jump: - single_gen = zip( - fixed['t_bar_diffs'][Diff.single], - fixed['t_bar_diff_sqrs'][Diff.single], - fixed['read_recip_coeffs'][Diff.single], - fixed['var_slope_coeffs'][Diff.single] - ) - double_gen = zip( - fixed['t_bar_diffs'][Diff.double], - fixed['t_bar_diff_sqrs'][Diff.double], - fixed['read_recip_coeffs'][Diff.double], - fixed['var_slope_coeffs'][Diff.double] - ) +# @pytest.mark.parametrize("use_jump", [True, False]) +# def test_fixed_values_from_metadata(ramp_data, use_jump): +# """Test computing the fixed data for all pixels""" +# _, data = ramp_data + +# data_dict = data._to_dict() +# t_bar = data_dict['t_bar'] +# tau = data_dict['tau'] +# n_reads = data_dict['n_reads'] + +# # Note this is converted to a dictionary so we can directly interrogate the +# # variables in question +# fixed = fixed_values_from_metadata(data, use_jump)._to_dict() + +# # Basic sanity checks that data passed in survives +# assert (fixed['data']['t_bar'] == t_bar).all() +# assert (fixed['data']['tau'] == tau).all() +# assert (fixed['data']['n_reads'] == n_reads).all() - for index, (t_bar_diff_1, t_bar_diff_sqr_1, read_recip_1, var_slope_1) in enumerate(single_gen): - assert t_bar_diff_1 == t_bar[index + 1] - t_bar[index] - assert t_bar_diff_sqr_1 == np.float32((t_bar[index + 1] - t_bar[index]) ** 2) - assert read_recip_1 == np.float32(1 / n_reads[index + 1]) + np.float32(1 / n_reads[index]) - assert var_slope_1 == (tau[index + 1] + tau[index] - 2 * min(t_bar[index], t_bar[index + 1])) - - for index, (t_bar_diff_2, t_bar_diff_sqr_2, read_recip_2, var_slope_2) in enumerate(double_gen): - if index == len(fixed['t_bar_diffs'][1]) - 1: - # Last value must be NaN - assert np.isnan(t_bar_diff_2) - assert np.isnan(read_recip_2) - assert np.isnan(var_slope_2) - else: - assert t_bar_diff_2 == t_bar[index + 2] - t_bar[index] - assert t_bar_diff_sqr_2 == np.float32((t_bar[index + 2] - t_bar[index])**2) - assert read_recip_2 == np.float32(1 / n_reads[index + 2]) + np.float32(1 / n_reads[index]) - assert var_slope_2 == (tau[index + 2] + tau[index] - 2 * min(t_bar[index], t_bar[index + 2])) - else: - # If not using jumps, these values should not even exist. However, for wrapping - # purposes, they are checked to be non-existent and then set to NaN - assert np.isnan(fixed['t_bar_diffs']).all() - assert np.isnan(fixed['t_bar_diff_sqrs']).all() - assert np.isnan(fixed['read_recip_coeffs']).all() - assert np.isnan(fixed['var_slope_coeffs']).all() +# # Check the computed data +# # These are computed via vectorized operations in the main code, here we +# # check using item-by-item operations +# if use_jump: +# single_gen = zip( +# fixed['t_bar_diffs'][Diff.single], +# fixed['t_bar_diff_sqrs'][Diff.single], +# fixed['read_recip_coeffs'][Diff.single], +# fixed['var_slope_coeffs'][Diff.single] +# ) +# double_gen = zip( +# fixed['t_bar_diffs'][Diff.double], +# fixed['t_bar_diff_sqrs'][Diff.double], +# fixed['read_recip_coeffs'][Diff.double], +# fixed['var_slope_coeffs'][Diff.double] +# ) + +# for index, (t_bar_diff_1, t_bar_diff_sqr_1, read_recip_1, var_slope_1) in enumerate(single_gen): +# assert t_bar_diff_1 == t_bar[index + 1] - t_bar[index] +# assert t_bar_diff_sqr_1 == np.float32((t_bar[index + 1] - t_bar[index]) ** 2) +# assert read_recip_1 == np.float32(1 / n_reads[index + 1]) + np.float32(1 / n_reads[index]) +# assert var_slope_1 == (tau[index + 1] + tau[index] - 2 * min(t_bar[index], t_bar[index + 1])) + +# for index, (t_bar_diff_2, t_bar_diff_sqr_2, read_recip_2, var_slope_2) in enumerate(double_gen): +# if index == len(fixed['t_bar_diffs'][1]) - 1: +# # Last value must be NaN +# assert np.isnan(t_bar_diff_2) +# assert np.isnan(read_recip_2) +# assert np.isnan(var_slope_2) +# else: +# assert t_bar_diff_2 == t_bar[index + 2] - t_bar[index] +# assert t_bar_diff_sqr_2 == np.float32((t_bar[index + 2] - t_bar[index])**2) +# assert read_recip_2 == np.float32(1 / n_reads[index + 2]) + np.float32(1 / n_reads[index]) +# assert var_slope_2 == (tau[index + 2] + tau[index] - 2 * min(t_bar[index], t_bar[index + 2])) +# else: +# # If not using jumps, these values should not even exist. However, for wrapping +# # purposes, they are checked to be non-existent and then set to NaN +# assert np.isnan(fixed['t_bar_diffs']).all() +# assert np.isnan(fixed['t_bar_diff_sqrs']).all() +# assert np.isnan(fixed['read_recip_coeffs']).all() +# assert np.isnan(fixed['var_slope_coeffs']).all() def _generate_resultants(read_pattern, n_pixels=1): From bac42a828cb5c96b9618c203e30ac19a725576d2 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 11:01:30 -0500 Subject: [PATCH 15/30] Combine _pixel into _fixed module --- setup.py | 6 -- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 3 +- src/stcal/ramp_fitting/ols_cas22/_fixed.pxd | 15 ++++ src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 55 +++++++++++++-- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 3 +- src/stcal/ramp_fitting/ols_cas22/_pixel.pxd | 12 ---- src/stcal/ramp_fitting/ols_cas22/_pixel.pyx | 69 ------------------- tests/test_jump_cas22.py | 3 +- 8 files changed, 68 insertions(+), 98 deletions(-) delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_pixel.pxd delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_pixel.pyx diff --git a/setup.py b/setup.py index 05b6a614..1a9276d6 100644 --- a/setup.py +++ b/setup.py @@ -19,12 +19,6 @@ include_dirs=[np.get_include()], language='c++' ), - Extension( - 'stcal.ramp_fitting.ols_cas22._pixel', - ['src/stcal/ramp_fitting/ols_cas22/_pixel.pyx'], - include_dirs=[np.get_include()], - language='c++' - ), Extension( 'stcal.ramp_fitting.ols_cas22._jump', ['src/stcal/ramp_fitting/ols_cas22/_jump.pyx'], diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index 67af4819..c40315aa 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -6,8 +6,7 @@ from libcpp cimport bool from libcpp.list cimport list as cpp_list from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance, RampJumpDQ -from stcal.ramp_fitting.ols_cas22._fixed cimport fill_fixed_values, n_fixed_offsets -from stcal.ramp_fitting.ols_cas22._pixel cimport n_pixel_offsets +from stcal.ramp_fitting.ols_cas22._fixed cimport fill_fixed_values, n_fixed_offsets, n_pixel_offsets from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, fit_jumps, RampFits from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd index 290e8b3b..98bee6a6 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd @@ -10,8 +10,23 @@ cpdef enum FixedOffsets: n_fixed_offsets +cpdef enum PixelOffsets: + single_local_slope + double_local_slope + single_var_read_noise + double_var_read_noise + n_pixel_offsets + + cpdef float[:, :] fill_fixed_values(float[:, :] fixed, float[:] t_bar, float[:] tau, int[:] n_reads, int n_resultants) + + +cpdef float[:, :] fill_pixel_values(float[:, :] pixel, + float[:] resultants, + float[:, :] fixed, + float read_noise, + int n_resultants) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx index 61cd3af4..a48bb625 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx @@ -15,14 +15,11 @@ Functions - cpdef gives a python wrapper, but the python version of this method is considered private, only to be used for testing """ -import numpy as np -cimport numpy as cnp - -from cython cimport boundscheck, wraparound +from cython cimport boundscheck, wraparound, cdivision from libc.math cimport NAN -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets +from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets, PixelOffsets @@ -82,3 +79,51 @@ cpdef inline float[:, :] fill_fixed_values(float[:, :] fixed, fixed[double_var_slope_val, i] = NAN return fixed + + +@boundscheck(False) +@wraparound(False) +@cdivision(True) +cpdef inline float[:, :] fill_pixel_values(float[:, :] pixel, + float[:] resultants, + float[:, :] fixed, + float read_noise, + int n_resultants): + """ + Compute the local slopes between resultants for the pixel + + Returns + ------- + [ + <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, + <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, + read_noise ** 2 / <(t_bar[i+1] - t_bar[i])>, + read_noise ** 2 / <(t_bar[i+2] - t_bar[i])>, + ] + """ + cdef int single_slope = PixelOffsets.single_local_slope + cdef int double_slope = PixelOffsets.double_local_slope + cdef int single_var = PixelOffsets.single_var_read_noise + cdef int double_var = PixelOffsets.double_var_read_noise + + cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff + cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff + cdef int single_read_recip = FixedOffsets.single_read_recip + cdef int double_read_recip = FixedOffsets.double_read_recip + + cdef float read_noise_sqr = read_noise ** 2 + + cdef int i + for i in range(n_resultants - 1): + pixel[single_slope, i] = (resultants[i + 1] - resultants[i]) / fixed[single_t_bar_diff, i] + pixel[single_var, i] = read_noise_sqr * fixed[single_read_recip, i] + + if i < n_resultants - 2: + pixel[double_slope, i] = (resultants[i + 2] - resultants[i]) / fixed[double_t_bar_diff, i] + pixel[double_var, i] = read_noise_sqr * fixed[double_read_recip, i] + else: + # The last double difference is undefined + pixel[double_slope, i] = NAN + pixel[double_var, i] = NAN + + return pixel diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index ef59915e..d8cd43ed 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -5,8 +5,7 @@ from libc.math cimport sqrt, log10, fmaxf, NAN, isnan from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, RampFits -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets -from stcal.ramp_fitting.ols_cas22._pixel cimport PixelOffsets, fill_pixel_values +from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets, PixelOffsets, fill_pixel_values from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, fit_ramp cdef inline float threshold(Thresh thresh, float slope): diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd b/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd deleted file mode 100644 index 50f7a041..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pxd +++ /dev/null @@ -1,12 +0,0 @@ -cpdef enum PixelOffsets: - single_local_slope - double_local_slope - single_var_read_noise - double_var_read_noise - n_pixel_offsets - -cpdef float[:, :] fill_pixel_values(float[:, :] pixel, - float[:] resultants, - float[:, :] fixed, - float read_noise, - int n_resultants) \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx b/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx deleted file mode 100644 index a6795da0..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_pixel.pyx +++ /dev/null @@ -1,69 +0,0 @@ -""" -Define the C class for the Cassertano22 algorithm for fitting ramps with jump detection - -Objects -------- -Pixel : class - Class to handle ramp fit with jump detection for a single pixel - Provides fits method which fits all the ramps for a single pixel - -Functions ---------- - make_pixel : function - Fast constructor for a Pixel class from input data. - - cpdef gives a python wrapper, but the python version of this method - is considered private, only to be used for testing -""" -from libc.math cimport NAN -from cython cimport boundscheck, wraparound, cdivision - -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets -from stcal.ramp_fitting.ols_cas22._pixel cimport PixelOffsets - - -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cpdef inline float[:, :] fill_pixel_values(float[:, :] pixel, - float[:] resultants, - float[:, :] fixed, - float read_noise, - int n_resultants): - """ - Compute the local slopes between resultants for the pixel - - Returns - ------- - [ - <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, - <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, - read_noise ** 2 / <(t_bar[i+1] - t_bar[i])>, - read_noise ** 2 / <(t_bar[i+2] - t_bar[i])>, - ] - """ - cdef int single_slope = PixelOffsets.single_local_slope - cdef int double_slope = PixelOffsets.double_local_slope - cdef int single_var = PixelOffsets.single_var_read_noise - cdef int double_var = PixelOffsets.double_var_read_noise - - cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff - cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff - cdef int single_read_recip = FixedOffsets.single_read_recip - cdef int double_read_recip = FixedOffsets.double_read_recip - - cdef float read_noise_sqr = read_noise ** 2 - - cdef int i - for i in range(n_resultants - 1): - pixel[single_slope, i] = (resultants[i + 1] - resultants[i]) / fixed[single_t_bar_diff, i] - pixel[single_var, i] = read_noise_sqr * fixed[single_read_recip, i] - - if i < n_resultants - 2: - pixel[double_slope, i] = (resultants[i + 2] - resultants[i]) / fixed[double_t_bar_diff, i] - pixel[double_var, i] = read_noise_sqr * fixed[double_read_recip, i] - else: - # The last double difference is undefined - pixel[double_slope, i] = NAN - pixel[double_var, i] = NAN - - return pixel diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 0eeb9c6d..24eb04ca 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,8 +2,7 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22._fixed import fill_fixed_values -from stcal.ramp_fitting.ols_cas22._pixel import fill_pixel_values, PixelOffsets +from stcal.ramp_fitting.ols_cas22._fixed import fill_fixed_values, fill_pixel_values, PixelOffsets from stcal.ramp_fitting.ols_cas22._ramp import init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern From 5e3d23e5ab41ba3880a7d3fa3defb46635900b1e Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 11:52:22 -0500 Subject: [PATCH 16/30] Clean up and fix tests --- tests/test_jump_cas22.py | 314 ++++++++++++++++++--------------------- 1 file changed, 146 insertions(+), 168 deletions(-) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 24eb04ca..645d5f42 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,7 +2,7 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22._fixed import fill_fixed_values, fill_pixel_values, PixelOffsets +from stcal.ramp_fitting.ols_cas22._fixed import fill_fixed_values, fill_pixel_values, FixedOffsets, PixelOffsets from stcal.ramp_fitting.ols_cas22._ramp import init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern @@ -34,53 +34,6 @@ GOOD_PROB = 0.7 -@pytest.fixture(scope="module") -def ramp_data(): - """ - Basic data for simulating ramps for testing (not unpacked) - - Returns - ------- - read_pattern : list[list[int]] - The example read pattern - metadata : dict - The metadata computed from the read pattern - """ - read_pattern = [ - [1, 2, 3, 4], - [5], - [6, 7, 8], - [9, 10, 11, 12, 13, 14, 15, 16, 17, 18], - [19, 20, 21], - [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36] - ] - - yield read_pattern, from_read_pattern(read_pattern, READ_TIME) - - -def test_from_read_pattern(ramp_data): - """Test turning read_pattern into the time data""" - read_pattern, data_object = ramp_data - data = data_object._to_dict() - - # Basic sanity checks (structs become dicts) - assert isinstance(data, dict) - assert 't_bar' in data - assert 'tau' in data - assert 'n_reads' in data - assert len(data) == 3 - - # Check that the data is correct - assert_allclose(data['t_bar'], [7.6, 15.2, 21.279999, 41.040001, 60.799999, 88.159996]) - assert_allclose(data['tau'], [5.7, 15.2, 19.928888, 36.023998, 59.448887, 80.593781]) - assert np.all(data['n_reads'] == [4, 1, 3, 10, 3, 15]) - - # Check datatypes - assert data['t_bar'].dtype == np.float32 - assert data['tau'].dtype == np.float32 - assert data['n_reads'].dtype == np.int32 - - def test_init_ramps(): """ Test turning dq flags into initial ramp splits @@ -129,66 +82,106 @@ def test_init_ramps(): assert ramps[15] == [] -# @pytest.mark.parametrize("use_jump", [True, False]) -# def test_fixed_values_from_metadata(ramp_data, use_jump): -# """Test computing the fixed data for all pixels""" -# _, data = ramp_data - -# data_dict = data._to_dict() -# t_bar = data_dict['t_bar'] -# tau = data_dict['tau'] -# n_reads = data_dict['n_reads'] - -# # Note this is converted to a dictionary so we can directly interrogate the -# # variables in question -# fixed = fixed_values_from_metadata(data, use_jump)._to_dict() - -# # Basic sanity checks that data passed in survives -# assert (fixed['data']['t_bar'] == t_bar).all() -# assert (fixed['data']['tau'] == tau).all() -# assert (fixed['data']['n_reads'] == n_reads).all() - -# # Check the computed data -# # These are computed via vectorized operations in the main code, here we -# # check using item-by-item operations -# if use_jump: -# single_gen = zip( -# fixed['t_bar_diffs'][Diff.single], -# fixed['t_bar_diff_sqrs'][Diff.single], -# fixed['read_recip_coeffs'][Diff.single], -# fixed['var_slope_coeffs'][Diff.single] -# ) -# double_gen = zip( -# fixed['t_bar_diffs'][Diff.double], -# fixed['t_bar_diff_sqrs'][Diff.double], -# fixed['read_recip_coeffs'][Diff.double], -# fixed['var_slope_coeffs'][Diff.double] -# ) - -# for index, (t_bar_diff_1, t_bar_diff_sqr_1, read_recip_1, var_slope_1) in enumerate(single_gen): -# assert t_bar_diff_1 == t_bar[index + 1] - t_bar[index] -# assert t_bar_diff_sqr_1 == np.float32((t_bar[index + 1] - t_bar[index]) ** 2) -# assert read_recip_1 == np.float32(1 / n_reads[index + 1]) + np.float32(1 / n_reads[index]) -# assert var_slope_1 == (tau[index + 1] + tau[index] - 2 * min(t_bar[index], t_bar[index + 1])) - -# for index, (t_bar_diff_2, t_bar_diff_sqr_2, read_recip_2, var_slope_2) in enumerate(double_gen): -# if index == len(fixed['t_bar_diffs'][1]) - 1: -# # Last value must be NaN -# assert np.isnan(t_bar_diff_2) -# assert np.isnan(read_recip_2) -# assert np.isnan(var_slope_2) -# else: -# assert t_bar_diff_2 == t_bar[index + 2] - t_bar[index] -# assert t_bar_diff_sqr_2 == np.float32((t_bar[index + 2] - t_bar[index])**2) -# assert read_recip_2 == np.float32(1 / n_reads[index + 2]) + np.float32(1 / n_reads[index]) -# assert var_slope_2 == (tau[index + 2] + tau[index] - 2 * min(t_bar[index], t_bar[index + 2])) -# else: -# # If not using jumps, these values should not even exist. However, for wrapping -# # purposes, they are checked to be non-existent and then set to NaN -# assert np.isnan(fixed['t_bar_diffs']).all() -# assert np.isnan(fixed['t_bar_diff_sqrs']).all() -# assert np.isnan(fixed['read_recip_coeffs']).all() -# assert np.isnan(fixed['var_slope_coeffs']).all() +@pytest.fixture(scope="module") +def read_pattern(): + """ + Basic data for simulating ramps for testing (not unpacked) + + Returns + ------- + read_pattern : list[list[int]] + The example read pattern + metadata : dict + The metadata computed from the read pattern + """ + yield [ + [1, 2, 3, 4], + [5], + [6, 7, 8], + [9, 10, 11, 12, 13, 14, 15, 16, 17, 18], + [19, 20, 21], + [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36] + ] + + +def test_from_read_pattern(read_pattern): + """Test turning read_pattern into the time data""" + metadata = from_read_pattern(read_pattern, READ_TIME)._to_dict() + + t_bar = metadata['t_bar'] + tau = metadata['tau'] + n_reads = metadata['n_reads'] + + # Check that the data is correct + assert_allclose(t_bar, [7.6, 15.2, 21.279999, 41.040001, 60.799999, 88.159996]) + assert_allclose(tau, [5.7, 15.2, 19.928888, 36.023998, 59.448887, 80.593781]) + assert np.all(n_reads == [4, 1, 3, 10, 3, 15]) + + # Check datatypes + assert t_bar.dtype == np.float32 + assert tau.dtype == np.float32 + assert n_reads.dtype == np.int32 + + +@pytest.fixture(scope="module") +def ramp_data(read_pattern): + """ + Basic data for simulating ramps for testing (not unpacked) + + Returns + ------- + read_pattern : list[list[int]] + The example read pattern + metadata : dict + The metadata computed from the read pattern + """ + data = from_read_pattern(read_pattern, READ_TIME)._to_dict() + + yield data['t_bar'], data['tau'], data['n_reads'], read_pattern + + +def test_fill_fixed_values(ramp_data): + """Test computing the fixed data for all pixels""" + t_bar, tau, n_reads, _ = ramp_data + + n_resultants = len(t_bar) + fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + fixed = fill_fixed_values(fixed, t_bar, tau, n_reads, n_resultants) + + # Sanity check that the shape of fixed is correct + assert fixed.shape == (2 * 4, n_resultants - 1) + + # Split into the different types of data + t_bar_diffs = fixed[FixedOffsets.single_t_bar_diff:FixedOffsets.double_t_bar_diff + 1, :] + t_bar_diff_sqrs = fixed[FixedOffsets.single_t_bar_diff_sqr:FixedOffsets.double_t_bar_diff_sqr + 1, :] + read_recip = fixed[FixedOffsets.single_read_recip:FixedOffsets.double_read_recip + 1, :] + var_slope_vals = fixed[FixedOffsets.single_var_slope_val:FixedOffsets.double_var_slope_val + 1, :] + + # Sanity check that these are all the right shape + assert t_bar_diffs.shape == (2, n_resultants - 1) + assert t_bar_diff_sqrs.shape == (2, n_resultants - 1) + assert read_recip.shape == (2, n_resultants - 1) + assert var_slope_vals.shape == (2, n_resultants - 1) + + # Check the computed data + # These are computed using loop in cython, here we check against numpy + # Single diffs + assert np.all(t_bar_diffs[0] == t_bar[1:] - t_bar[:-1]) + assert np.all(t_bar_diff_sqrs[0] == (t_bar[1:] - t_bar[:-1])**2) + assert np.all(read_recip[0] == np.float32(1 / n_reads[1:]) + np.float32(1 / n_reads[:-1])) + assert np.all(var_slope_vals[0] == (tau[1:] + tau[:-1] - 2 * np.minimum(t_bar[1:], t_bar[:-1]))) + + # Double diffs + assert np.all(t_bar_diffs[1, :-1] == t_bar[2:] - t_bar[:-2]) + assert np.all(t_bar_diff_sqrs[1, :-1] == (t_bar[2:] - t_bar[:-2])**2) + assert np.all(read_recip[1, :-1] == np.float32(1 / n_reads[2:]) + np.float32(1 / n_reads[:-2])) + assert np.all(var_slope_vals[1, :-1] == (tau[2:] + tau[:-2] - 2 * np.minimum(t_bar[2:], t_bar[:-2]))) + + # Last double diff should be NaN + assert np.isnan(t_bar_diffs[1, -1]) + assert np.isnan(t_bar_diff_sqrs[1, -1]) + assert np.isnan(read_recip[1, -1]) + assert np.isnan(var_slope_vals[1, -1]) def _generate_resultants(read_pattern, n_pixels=1): @@ -248,71 +241,57 @@ def pixel_data(ramp_data): n_reads: The number of reads for the read pattern used for the resultants """ + t_bar, tau, n_reads, read_pattern = ramp_data + + n_resultants = len(t_bar) + fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + fixed = fill_fixed_values(fixed, t_bar, tau, n_reads, n_resultants) - read_pattern, metadata = ramp_data resultants = _generate_resultants(read_pattern) - yield resultants, metadata - - -# @pytest.mark.parametrize("use_jump", [True, False]) -# def test_fill_pixel_values(pixel_data, use_jump): -# """Test computing the initial pixel data""" -# resultants, metadata = pixel_data - -# data = metadata._to_dict() -# t_bar = data['t_bar'] -# tau = data['tau'] -# n_reads = data['n_reads'] - -# fixed = fixed_values_from_metadata(metadata, use_jump) -# pixel = np.empty((PixelOffsets.n_pixel_offsets, fixed.data.n_resultants - 1), dtype=np.float32) - -# # Note this is converted to a dictionary so we can directly interrogate the -# # variables in question -# pixel = fill_pixel_values(pixel, resultants, fixed.t_bar_diffs, fixed.read_recip_coeffs, READ_NOISE, fixed.data.n_resultants) - -# # Basic sanity checks that data passed in survives -# assert (pixel['resultants'] == resultants).all() -# assert READ_NOISE == pixel['read_noise'] - -# # the "fixed" data is not checked as this is already done above - -# # Check the computed data -# # These are computed via vectorized operations in the main code, here we -# # check using item-by-item operations -# if use_jump: -# single_gen = zip(pixel['local_slopes'][Diff.single], pixel['var_read_noise'][Diff.single]) -# double_gen = zip(pixel['local_slopes'][Diff.double], pixel['var_read_noise'][Diff.double]) - -# for index, (local_slope_1, var_read_noise_1) in enumerate(single_gen): -# assert local_slope_1 == ( -# (resultants[index + 1] - resultants[index]) / (t_bar[index + 1] - t_bar[index])) -# assert var_read_noise_1 == np.float32(READ_NOISE ** 2)* ( -# np.float32(1 / n_reads[index + 1]) + np.float32(1 / n_reads[index]) -# ) - -# for index, (local_slope_2, var_read_noise_2) in enumerate(double_gen): -# if index == len(pixel['local_slopes'][1]) - 1: -# # Last value must be NaN -# assert np.isnan(local_slope_2) -# assert np.isnan(var_read_noise_2) -# else: -# assert local_slope_2 == ( -# (resultants[index + 2] - resultants[index]) / (t_bar[index + 2] - t_bar[index]) -# ) -# assert var_read_noise_2 == np.float32(READ_NOISE ** 2) * ( -# np.float32(1 / n_reads[index + 2]) + np.float32(1 / n_reads[index]) -# ) -# else: -# # If not using jumps, these values should not even exist. However, for wrapping -# # purposes, they are checked to be non-existent and then set to NaN -# assert np.isnan(pixel['local_slopes']).all() -# assert np.isnan(pixel['var_read_noise']).all() + yield resultants, t_bar, tau, n_reads, fixed + + +def test_fill_pixel_values(pixel_data): + """Test computing the initial pixel data""" + resultants, t_bar, tau, n_reads, fixed = pixel_data + + n_resultants = len(t_bar) + pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) + pixel = fill_pixel_values(pixel, resultants, fixed, READ_NOISE, n_resultants) + + # Sanity check that the shape of pixel is correct + assert pixel.shape == (2 * 2, n_resultants - 1) + + # Split into the different types of data + local_slopes = pixel[PixelOffsets.single_local_slope:PixelOffsets.double_local_slope + 1, :] + var_read_noise = pixel[PixelOffsets.single_var_read_noise:PixelOffsets.double_var_read_noise + 1, :] + + # Sanity check that these are all the right shape + assert local_slopes.shape == (2, n_resultants - 1) + assert var_read_noise.shape == (2, n_resultants - 1) + + # Check the computed data + # These are computed using loop in cython, here we check against numpy + # Single diffs + assert np.all(local_slopes[0] == (resultants[1:] - resultants[:-1]) / (t_bar[1:] - t_bar[:-1])) + assert np.all(var_read_noise[0] == np.float32(READ_NOISE ** 2) * ( + np.float32(1 / n_reads[1:]) + np.float32(1 / n_reads[:-1])) + ) + + # Double diffs + assert np.all(local_slopes[1, :-1] == (resultants[2:] - resultants[:-2]) / (t_bar[2:] - t_bar[:-2])) + assert np.all(var_read_noise[1, :-1] == np.float32(READ_NOISE ** 2) * ( + np.float32(1 / n_reads[2:]) + np.float32(1 / n_reads[:-2])) + ) + + # Last double diff should be NaN + assert np.isnan(local_slopes[1, -1]) + assert np.isnan(var_read_noise[1, -1]) @pytest.fixture(scope="module") -def detector_data(ramp_data): +def detector_data(read_pattern): """ Generate a set of with no jumps data as if for a single detector as it would be passed in by the supporting code. @@ -325,7 +304,6 @@ def detector_data(ramp_data): read_pattern: The read pattern used for the resultants """ - read_pattern, _ = ramp_data read_noise = np.ones(N_PIXELS, dtype=np.float32) * READ_NOISE resultants = _generate_resultants(read_pattern, n_pixels=N_PIXELS) From 3b991bc1d18411be127f792e09d75337eb88b44f Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 12:23:22 -0500 Subject: [PATCH 17/30] Speed up driver function --- src/stcal/ramp_fitting/ols_cas22/__init__.py | 2 +- src/stcal/ramp_fitting/ols_cas22/_core.pxd | 18 +++++++--------- src/stcal/ramp_fitting/ols_cas22/_core.pyx | 2 +- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 21 ++++++++++++------- src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 1 + src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 5 ----- .../ramp_fitting/ols_cas22/_read_pattern.pxd | 2 +- .../ramp_fitting/ols_cas22/_read_pattern.pyx | 4 +--- tests/test_jump_cas22.py | 6 +++--- 9 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/ols_cas22/__init__.py index d07f4ddf..c1fb36ec 100644 --- a/src/stcal/ramp_fitting/ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/ols_cas22/__init__.py @@ -1,4 +1,4 @@ from ._fit_ramps import fit_ramps, RampFitOutputs -from ._core import Parameter, Variance, Diff, RampJumpDQ +from ._core import Parameter, Variance, RampJumpDQ __all__ = ['fit_ramps', 'RampFitOutputs', 'Parameter', 'Variance', 'Diff', 'RampJumpDQ'] diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pxd b/src/stcal/ramp_fitting/ols_cas22/_core.pxd index 2b3f7dda..3d9ec9c4 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pxd @@ -1,18 +1,14 @@ -cpdef enum Diff: - single - double - n_diff - - cpdef enum Parameter: - intercept = 0 - slope = 1 + intercept + slope + n_param cpdef enum Variance: - read_var = 0 - poisson_var = 1 - total_var = 2 + read_var + poisson_var + total_var + n_var cpdef enum RampJumpDQ: diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pyx b/src/stcal/ramp_fitting/ols_cas22/_core.pyx index 5f401244..81d186f3 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_core.pyx @@ -74,4 +74,4 @@ Functions - cpdef gives a python wrapper, but the python version of this method is considered private, only to be used for testing """ -from stcal.ramp_fitting.ols_cas22._core cimport Diff, Parameter, Variance, RampJumpDQ +from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance, RampJumpDQ diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index c40315aa..7114e4c7 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -104,7 +104,7 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, f'match number of resultants {n_resultants}') # Pre-compute data for all pixels - cdef ReadPattern metadata = from_read_pattern(read_pattern, read_time) + cdef ReadPattern metadata = from_read_pattern(read_pattern, read_time, n_resultants) cdef float[:] t_bar = metadata.t_bar cdef float[:] tau = metadata.tau cdef int[:] n_reads = metadata.n_reads @@ -121,9 +121,14 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, # list in the end. cdef cpp_list[RampFits] ramp_fits - cdef cnp.ndarray[float, ndim=2] parameters = np.zeros((n_pixels, 2), dtype=np.float32) - cdef cnp.ndarray[float, ndim=2] variances = np.zeros((n_pixels, 3), dtype=np.float32) + # intercept is currently always zero, where as every variance is calculated and set + cdef float[:, :] parameters = np.zeros((n_pixels, Parameter.n_param), dtype=np.float32) + cdef float[:, :] variances = np.empty((n_pixels, Variance.n_var), dtype=np.float32) + cdef int slope = Parameter.slope + cdef int read_var = Variance.read_var + cdef int poisson_var = Variance.poisson_var + cdef int total_var = Variance.total_var # Perform all of the fits cdef RampFits fit @@ -143,11 +148,11 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, use_jump, include_diagnostic) - parameters[index, Parameter.slope] = fit.average.slope + parameters[index, slope] = fit.average.slope - variances[index, Variance.read_var] = fit.average.read_var - variances[index, Variance.poisson_var] = fit.average.poisson_var - variances[index, Variance.total_var] = fit.average.read_var + fit.average.poisson_var + variances[index, read_var] = fit.average.read_var + variances[index, poisson_var] = fit.average.poisson_var + variances[index, total_var] = fit.average.read_var + fit.average.poisson_var for jump in fit.jumps: dq[jump, index] = RampJumpDQ.JUMP_DET @@ -156,4 +161,4 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, ramp_fits.push_back(fit) # return RampFitOutputs(ramp_fits, parameters, variances, dq) - return RampFitOutputs(parameters, variances, dq, ramp_fits if include_diagnostic else None) + return RampFitOutputs(np.array(parameters, dtype=np.float32), np.array(variances, dtype=np.float32), dq, ramp_fits if include_diagnostic else None) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx index a48bb625..dba1e624 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx @@ -25,6 +25,7 @@ from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets, PixelOffsets @boundscheck(False) @wraparound(False) +@cdivision(True) cpdef inline float[:, :] fill_fixed_values(float[:, :] fixed, float[:] t_bar, float[:] tau, diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index d8cd43ed..dae91d9b 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -331,11 +331,6 @@ cdef inline RampFits fit_jumps(float[:] resultants, ramp_fits.average.read_var += weight**2 * ramp_fit.read_var ramp_fits.average.poisson_var += weight**2 * ramp_fit.poisson_var - # Reverse to order in time - if include_diagnostic: - ramp_fits.fits = ramp_fits.fits[::-1] - ramp_fits.index = ramp_fits.index[::-1] - # Finish computing averages ramp_fits.average.slope /= total_weight if total_weight != 0 else 1 ramp_fits.average.read_var /= total_weight**2 if total_weight != 0 else 1 diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd index 0c5bb4b0..a1e00533 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd @@ -4,4 +4,4 @@ cdef class ReadPattern: cdef int[::1] n_reads -cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time) \ No newline at end of file +cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time, int n_resultants) \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx index 0168d67f..ae321221 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx @@ -39,7 +39,7 @@ cdef class ReadPattern: @boundscheck(False) @wraparound(False) -cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time): +cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time, int n_resultants): """ Derive the input data from the the read pattern @@ -57,8 +57,6 @@ cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_tim ------- ReadPattern """ - cdef int n_resultants = len(read_pattern) - cdef ReadPattern data = ReadPattern() data.t_bar = np.empty(n_resultants, dtype=np.float32) data.tau = np.empty(n_resultants, dtype=np.float32) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 645d5f42..58f8356d 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -6,7 +6,7 @@ from stcal.ramp_fitting.ols_cas22._ramp import init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern -from stcal.ramp_fitting.ols_cas22 import fit_ramps, Parameter, Variance, Diff, RampJumpDQ +from stcal.ramp_fitting.ols_cas22 import fit_ramps, Parameter, Variance, RampJumpDQ # Purposefully set a fixed seed so that the tests in this module are deterministic @@ -106,7 +106,7 @@ def read_pattern(): def test_from_read_pattern(read_pattern): """Test turning read_pattern into the time data""" - metadata = from_read_pattern(read_pattern, READ_TIME)._to_dict() + metadata = from_read_pattern(read_pattern, READ_TIME, len(read_pattern))._to_dict() t_bar = metadata['t_bar'] tau = metadata['tau'] @@ -135,7 +135,7 @@ def ramp_data(read_pattern): metadata : dict The metadata computed from the read pattern """ - data = from_read_pattern(read_pattern, READ_TIME)._to_dict() + data = from_read_pattern(read_pattern, READ_TIME, len(read_pattern))._to_dict() yield data['t_bar'], data['tau'], data['n_reads'], read_pattern From 631405cad30dea0348a6dadb05a193ff5f04da86 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 12:33:14 -0500 Subject: [PATCH 18/30] Move jump dq flag marking into jump detection function itself --- src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx | 4 +--- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 1 + src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 5 +++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index 7114e4c7..b84e3dd2 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -136,6 +136,7 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, for index in range(n_pixels): # Fit all the ramps for the given pixel fit = fit_jumps(resultants[:, index], + dq[:, index], read_noise[index], init_ramps(dq, n_resultants, index), t_bar, @@ -154,9 +155,6 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, variances[index, poisson_var] = fit.average.poisson_var variances[index, total_var] = fit.average.read_var + fit.average.poisson_var - for jump in fit.jumps: - dq[jump, index] = RampJumpDQ.JUMP_DET - if include_diagnostic: ramp_fits.push_back(fit) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 533aa8d4..fedf646d 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -16,6 +16,7 @@ cdef struct RampFits: cdef RampFits fit_jumps(float[:] resultants, + int[:] dq, float read_noise, RampQueue ramps, float[:] t_bar, diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index dae91d9b..765d19fb 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -4,6 +4,7 @@ from libcpp cimport bool from libc.math cimport sqrt, log10, fmaxf, NAN, isnan +from stcal.ramp_fitting.ols_cas22._core cimport JUMP_DET from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, RampFits from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets, PixelOffsets, fill_pixel_values from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, fit_ramp @@ -176,6 +177,7 @@ cdef inline (int, float) statistics(float[:, :] pixel, @wraparound(False) @cdivision(True) cdef inline RampFits fit_jumps(float[:] resultants, + int[:] dq, float read_noise, RampQueue ramps, float[:] t_bar, @@ -266,6 +268,9 @@ cdef inline RampFits fit_jumps(float[:] resultants, # consideration. jump0 = argmax + ramp.start jump1 = jump0 + 1 + + dq[jump0] = JUMP_DET + dq[jump1] = JUMP_DET if include_diagnostic: ramp_fits.jumps.push_back(jump0) ramp_fits.jumps.push_back(jump1) From 26a5a5eb904ccbe351e85ae93ba211199ccbd327 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 13:07:04 -0500 Subject: [PATCH 19/30] Fix compiler warnings --- setup.py | 6 -- src/stcal/ramp_fitting/ols_cas22/__init__.py | 7 +- src/stcal/ramp_fitting/ols_cas22/_core.pxd | 15 ---- src/stcal/ramp_fitting/ols_cas22/_core.pyx | 77 ------------------- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 24 ++++-- src/stcal/ramp_fitting/ols_cas22/_fixed.pxd | 2 + src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 2 + src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 6 ++ src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 6 +- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 2 + src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 2 + .../ramp_fitting/ols_cas22/_read_pattern.pxd | 2 + .../ramp_fitting/ols_cas22/_read_pattern.pyx | 2 + tests/test_jump_cas22.py | 8 +- 14 files changed, 46 insertions(+), 115 deletions(-) delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_core.pxd delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_core.pyx diff --git a/setup.py b/setup.py index 1a9276d6..007cf450 100644 --- a/setup.py +++ b/setup.py @@ -7,12 +7,6 @@ Options.annotate = False extensions = [ - Extension( - 'stcal.ramp_fitting.ols_cas22._core', - ['src/stcal/ramp_fitting/ols_cas22/_core.pyx'], - include_dirs=[np.get_include()], - language='c++' - ), Extension( 'stcal.ramp_fitting.ols_cas22._fixed', ['src/stcal/ramp_fitting/ols_cas22/_fixed.pyx'], diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/ols_cas22/__init__.py index c1fb36ec..b1482c67 100644 --- a/src/stcal/ramp_fitting/ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/ols_cas22/__init__.py @@ -1,4 +1,5 @@ -from ._fit_ramps import fit_ramps, RampFitOutputs -from ._core import Parameter, Variance, RampJumpDQ +from ._fit_ramps import fit_ramps, RampFitOutputs, Parameter, Variance +# from ._core import Parameter, Variance +from ._jump import JUMP_DET -__all__ = ['fit_ramps', 'RampFitOutputs', 'Parameter', 'Variance', 'Diff', 'RampJumpDQ'] +__all__ = ['fit_ramps', 'RampFitOutputs', 'Parameter', 'Variance', 'Diff', 'JUMP_DET'] diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pxd b/src/stcal/ramp_fitting/ols_cas22/_core.pxd deleted file mode 100644 index 3d9ec9c4..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pxd +++ /dev/null @@ -1,15 +0,0 @@ -cpdef enum Parameter: - intercept - slope - n_param - - -cpdef enum Variance: - read_var - poisson_var - total_var - n_var - - -cpdef enum RampJumpDQ: - JUMP_DET = 4 diff --git a/src/stcal/ramp_fitting/ols_cas22/_core.pyx b/src/stcal/ramp_fitting/ols_cas22/_core.pyx deleted file mode 100644 index 81d186f3..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_core.pyx +++ /dev/null @@ -1,77 +0,0 @@ -""" -Define the basic types and functions for the CAS22 algorithm with jump detection - -Structs -------- - RampIndex - int start: starting index of the ramp in the resultants - int end: ending index of the ramp in the resultants - - Note that the Python range would be [start:end+1] for any ramp index. - RampFit - float slope: slope of a single ramp - float read_var: read noise variance of a single ramp - float poisson_var: poisson noise variance of single ramp - RampFits - vector[RampFit] fits: ramp fits (in time order) for a single pixel - vector[RampIndex] index: ramp indices (in time order) for a single pixel - RampFit average: average ramp fit for a single pixel - ReadPatternMetata - vector[float] t_bar: mean time of each resultant - vector[float] tau: variance time of each resultant - vector[int] n_reads: number of reads in each resultant - - Note that these are entirely computed from the read_pattern and - read_time (which should be constant for a given telescope) for the - given observation. - Thresh - float intercept: intercept of the threshold - float constant: constant of the threshold - -Enums ------ - Diff - This is the enum to track the index for single vs double difference related - computations. - - single: single difference - double: double difference - - Parameter - This is the enum to track the index of the computed fit parameters for - the ramp fit. - - intercept: the intercept of the ramp fit - slope: the slope of the ramp fit - - Variance - This is the enum to track the index of the computed variance values for - the ramp fit. - - read_var: read variance computed - poisson_var: poisson variance computed - total_var: total variance computed (read_var + poisson_var) - - RampJumpDQ - This enum is to specify the DQ flags for Ramp/Jump detection - - JUMP_DET: jump detected - -Functions ---------- - get_power - Return the power from Casertano+22, Table 2 - threshold - Compute jump threshold - - cpdef gives a python wrapper, but the python version of this method - is considered private, only to be used for testing - init_ramps - Find initial ramps for each pixel, accounts for DQ flags - - A python wrapper, _init_ramps_list, that adjusts types so they can - be directly inspected in python exists for testing purposes only. - metadata_from_read_pattern - Read the read pattern and derive the baseline metadata parameters needed - - cpdef gives a python wrapper, but the python version of this method - is considered private, only to be used for testing -""" -from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance, RampJumpDQ diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index b84e3dd2..c21da1d1 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -1,3 +1,5 @@ +# cython: language_level=3str + import numpy as np cimport numpy as cnp @@ -5,7 +7,7 @@ from cython cimport boundscheck, wraparound from libcpp cimport bool from libcpp.list cimport list as cpp_list -from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance, RampJumpDQ +# from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance from stcal.ramp_fitting.ols_cas22._fixed cimport fill_fixed_values, n_fixed_offsets, n_pixel_offsets from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, fit_jumps, RampFits @@ -18,10 +20,18 @@ from typing import NamedTuple, Optional cnp.import_array() -# Fix the default Threshold values at compile time these values cannot be overridden -# dynamically at runtime. -DEF DefaultIntercept = 5.5 -DEF DefaultConstant = 1/3.0 +cpdef enum Parameter: + intercept + slope + n_param + + +cpdef enum Variance: + read_var + poisson_var + total_var + n_var + class RampFitOutputs(NamedTuple): """ @@ -59,8 +69,8 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, float read_time, list[list[int]] read_pattern, bool use_jump=False, - float intercept=DefaultIntercept, - float constant=DefaultConstant, + float intercept=5.5, + float constant=1/3, bool include_diagnostic=False): """Fit ramps using the Casertano+22 algorithm. This implementation uses the Cas22 algorithm to fit ramps, where diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd index 98bee6a6..2ebdf9ce 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd @@ -1,3 +1,5 @@ +# cython: language_level=3str + cpdef enum FixedOffsets: single_t_bar_diff double_t_bar_diff diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx index dba1e624..393fe0ef 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx @@ -1,3 +1,5 @@ +# cython: language_level=3str + """ Define the data which is fixed for all pixels to compute the CAS22 algorithm with jump detection diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index fedf646d..040be56a 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -1,8 +1,14 @@ +# cython: language_level=3str + from libcpp cimport bool from libcpp.vector cimport vector from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampQueue + +cpdef enum: + JUMP_DET = 4 + cdef struct Thresh: float intercept float constant diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 765d19fb..e43064b5 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -1,11 +1,11 @@ +# cython: language_level=3str + from cython cimport boundscheck, wraparound, cdivision from libcpp cimport bool from libc.math cimport sqrt, log10, fmaxf, NAN, isnan - -from stcal.ramp_fitting.ols_cas22._core cimport JUMP_DET -from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, RampFits +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, RampFits, JUMP_DET from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets, PixelOffsets, fill_pixel_values from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, fit_ramp diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index 3f2ffca4..9472d2fe 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -1,3 +1,5 @@ +# cython: language_level=3str + from libcpp.vector cimport vector diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index 992a4c4c..bde3c086 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -1,3 +1,5 @@ +# cython: language_level=3str + from cython cimport boundscheck, wraparound, cdivision, cpow from libc.math cimport sqrt, fabs, INFINITY, NAN, fmaxf diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd index a1e00533..89d4419c 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd @@ -1,3 +1,5 @@ +# cython: language_level=3str + cdef class ReadPattern: cdef float[::1] t_bar cdef float[::1] tau diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx index ae321221..ce8a99c4 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx @@ -1,3 +1,5 @@ +# cython: language_level=3str + import numpy as np cimport numpy as cnp from cython cimport boundscheck, wraparound diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 58f8356d..769398f9 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -6,7 +6,7 @@ from stcal.ramp_fitting.ols_cas22._ramp import init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern -from stcal.ramp_fitting.ols_cas22 import fit_ramps, Parameter, Variance, RampJumpDQ +from stcal.ramp_fitting.ols_cas22 import fit_ramps, Parameter, Variance, JUMP_DET # Purposefully set a fixed seed so that the tests in this module are deterministic @@ -539,7 +539,7 @@ def test_override_default_threshold(jump_data): def test_jump_dq_set(jump_data): # Check the DQ flag value to start - assert RampJumpDQ.JUMP_DET == 2**2 + assert JUMP_DET == 2**2 resultants, read_noise, read_pattern, jump_reads, jump_resultants = jump_data dq = np.zeros(resultants.shape, dtype=np.int32) @@ -549,7 +549,7 @@ def test_jump_dq_set(jump_data): for fit, pixel_dq in zip(output.fits, output.dq.transpose()): # Check that all jumps found get marked - assert (pixel_dq[fit['jumps']] == RampJumpDQ.JUMP_DET).all() + assert (pixel_dq[fit['jumps']] == JUMP_DET).all() # Check that dq flags for jumps are only set if the jump is marked - assert set(np.where(pixel_dq == RampJumpDQ.JUMP_DET)[0]) == set(fit['jumps']) + assert set(np.where(pixel_dq == JUMP_DET)[0]) == set(fit['jumps']) From ddb15b5af33dc1fee51ef0afa77ed783067c8d90 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 13:26:23 -0500 Subject: [PATCH 20/30] Simplify statistic loop --- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 50 +++++++++++----------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index e43064b5..3a70651d 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -139,34 +139,36 @@ cdef inline (int, float) statistics(float[:, :] pixel, cdef int double_var_slope_val = FixedOffsets.double_var_slope_val cdef float correct = correction(t_bar, ramp, slope) + cdef float stat - cdef float stat, double_stat - - cdef int argmax = 0 - cdef float max_stat = NAN - - cdef int index, stat_index - for stat_index, index in enumerate(range(start, end)): - stat = statstic(pixel[single_local_slope, index], - pixel[single_var_read_noise, index], - fixed[single_t_bar_diff_sqr, index], - fixed[single_var_slope_val, index], - slope, - correct) - - # It is not possible to compute double differences for the second - # to last resultant in the ramp. Therefore, we include the double - # differences for every stat except the last one. - if index != end - 1: - double_stat = statstic(pixel[double_local_slope, index], - pixel[double_var_read_noise, index], - fixed[double_t_bar_diff_sqr, index], - fixed[double_var_slope_val, index], + if start == end: + return 0, NAN + + cdef int index = end - 1 + cdef int argmax = end - start - 1 + cdef float max_stat = statstic(pixel[single_local_slope, index], + pixel[single_var_read_noise, index], + fixed[single_t_bar_diff_sqr, index], + fixed[single_var_slope_val, index], slope, correct) - stat = fmaxf(stat, double_stat) - if isnan(max_stat) or stat > max_stat: + cdef int stat_index + for stat_index, index in enumerate(range(start, end - 1)): + stat = fmaxf(statstic(pixel[single_local_slope, index], + pixel[single_var_read_noise, index], + fixed[single_t_bar_diff_sqr, index], + fixed[single_var_slope_val, index], + slope, + correct), + statstic(pixel[double_local_slope, index], + pixel[double_var_read_noise, index], + fixed[double_t_bar_diff_sqr, index], + fixed[double_var_slope_val, index], + slope, + correct)) + + if stat > max_stat: max_stat = stat argmax = stat_index From c769056883df22e1536aa9d25add8c5c2ad49d6a Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 14:58:42 -0500 Subject: [PATCH 21/30] Update some documentation --- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 6 +- src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 14 +- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 4 +- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 285 ++++++++++++------ src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 92 ++++-- .../ramp_fitting/ols_cas22/_read_pattern.pyx | 24 +- 6 files changed, 292 insertions(+), 133 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index c21da1d1..dd430a91 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -10,7 +10,7 @@ from libcpp.list cimport list as cpp_list # from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance from stcal.ramp_fitting.ols_cas22._fixed cimport fill_fixed_values, n_fixed_offsets, n_pixel_offsets -from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, fit_jumps, RampFits +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, fit_jumps, JumpFits from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern, from_read_pattern @@ -129,7 +129,7 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, # Use list because this might grow very large which would require constant # reallocation. We don't need random access, and this gets cast to a python # list in the end. - cdef cpp_list[RampFits] ramp_fits + cdef cpp_list[JumpFits] ramp_fits # intercept is currently always zero, where as every variance is calculated and set cdef float[:, :] parameters = np.zeros((n_pixels, Parameter.n_param), dtype=np.float32) @@ -141,7 +141,7 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, cdef int total_var = Variance.total_var # Perform all of the fits - cdef RampFits fit + cdef JumpFits fit cdef int index for index in range(n_pixels): # Fit all the ramps for the given pixel diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx index 393fe0ef..a9a8957b 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx @@ -100,20 +100,20 @@ cpdef inline float[:, :] fill_pixel_values(float[:, :] pixel, [ <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, - read_noise ** 2 / <(t_bar[i+1] - t_bar[i])>, - read_noise ** 2 / <(t_bar[i+2] - t_bar[i])>, + read_noise**2 * <(1/n_reads[i+1] + 1/n_reads[i])>, + read_noise**2 * <(1/n_reads[i+2] + 1/n_reads[i])>, ] """ - cdef int single_slope = PixelOffsets.single_local_slope - cdef int double_slope = PixelOffsets.double_local_slope - cdef int single_var = PixelOffsets.single_var_read_noise - cdef int double_var = PixelOffsets.double_var_read_noise - cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff cdef int single_read_recip = FixedOffsets.single_read_recip cdef int double_read_recip = FixedOffsets.double_read_recip + cdef int single_slope = PixelOffsets.single_local_slope + cdef int double_slope = PixelOffsets.double_local_slope + cdef int single_var = PixelOffsets.single_var_read_noise + cdef int double_var = PixelOffsets.double_var_read_noise + cdef float read_noise_sqr = read_noise ** 2 cdef int i diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 040be56a..1f9316d6 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -14,14 +14,14 @@ cdef struct Thresh: float constant -cdef struct RampFits: +cdef struct JumpFits: RampFit average vector[int] jumps vector[RampFit] fits RampQueue index -cdef RampFits fit_jumps(float[:] resultants, +cdef JumpFits fit_jumps(float[:] resultants, int[:] dq, float read_noise, RampQueue ramps, diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 3a70651d..22bf9f2e 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -1,15 +1,53 @@ # cython: language_level=3str + +""" +This module contains all the functions needed to execute jump detection for the + Castentano+22 ramp fitting algorithm + + The _ramp module contains the actual ramp fitting algorithm, this module + contains a driver for the algoritm and detection of jumps/splitting ramps. + +Structs +------- +Thresh : struct + intercept - constant * log10(slope) + - intercept : float + The intercept of the jump threshold + - constant : float + The constant of the jump threshold + +JumpFits : struct + All the data on a given pixel's ramp fit with (or without) jump detection + - average : RampFit + The average of all the ramps fit for the pixel + - jumps : vector[int] + The indices of the resultants which were detected as jumps + - fits : vector[RampFit] + All of the fits for each ramp fit for the pixel + - index : RampQueue + The RampIndex representations correspoinding to each fit in fits + +(Public) Functions +------------------ + +fit_jumps : function + Compute all the ramps for a single pixel using the Casertano+22 algorithm + with jump detection. This is a driver for the ramp fit algorithm in general + meaning it automatically handles splitting ramps across dq flags in addition + to splitting across detected jumps (if jump detection is turned on). +""" + from cython cimport boundscheck, wraparound, cdivision from libcpp cimport bool from libc.math cimport sqrt, log10, fmaxf, NAN, isnan -from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, RampFits, JUMP_DET +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, JumpFits, JUMP_DET from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets, PixelOffsets, fill_pixel_values from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, fit_ramp -cdef inline float threshold(Thresh thresh, float slope): +cdef inline float _threshold(Thresh thresh, float slope): """ Compute jump threshold @@ -33,7 +71,7 @@ cdef inline float threshold(Thresh thresh, float slope): @boundscheck(False) @wraparound(False) @cdivision(True) -cdef inline float correction(float[:] t_bar, RampIndex ramp, float slope): +cdef inline float _correction(float[:] t_bar, RampIndex ramp, float slope): """ Compute the correction factor for the variance used by a statistic @@ -41,6 +79,8 @@ cdef inline float correction(float[:] t_bar, RampIndex ramp, float slope): Parameters ---------- + t_bar : float[:] + The computed t_bar values for the ramp ramp : RampIndex Struct for start and end indices resultants for the ramp slope : float @@ -54,59 +94,73 @@ cdef inline float correction(float[:] t_bar, RampIndex ramp, float slope): @boundscheck(False) @wraparound(False) @cdivision(True) -cdef inline float statstic(float local_slope, - float var_read_noise, - float t_bar_diff_sqr, - float var_slope_coeff, - float slope, - float correct): +cdef inline float _statstic(float local_slope, + float var_read_noise, + float t_bar_diff_sqr, + float var_slope_coeff, + float slope, + float correct): """ - Compute a single set of fit statistics - (delta / sqrt(var)) + correction - where - delta = ((R[j] - R[i]) / (t_bar[j] - t_bar[i]) - slope) - * (t_bar[j] - t_bar[i]) - var = sigma * (1/N[j] + 1/N[i]) - + slope * (tau[j] + tau[i] - min(t_bar[j], t_bar[i])) + Compute a single fit statistic + delta / sqrt(var + correct) + + where: + delta = local_slope - slope + var = (var_read_noise + slope * var_slope_coeff) / t_bar_diff_sqr + + pre-computed: + local_slope = (resultant[i + j] - resultant[i]) / (t_bar[i + j] - t_bar[i]) + var_read_noise = read_noise ** 2 * (1/n_reads[i + j] + 1/n_reads[i]) + var_slope_coeff = tau[i + j] + tau[i] - 2 * min(t_bar[i + j], t_bar[i]) + t_bar_diff_sqr = (t_bar[i + j] - t_bar[i]) ** 2 Parameters ---------- + local_slope : float + The local slope the statistic is computed for + float : var_read_noise + The read noise variance for local_slope + t_bar_diff_sqr : float + The square difference for the t_bar corresponding to local_slope + var_slope_coeff : float + The slope variance coefficient for local_slope slope : float The computed slope for the ramp - ramp : RampIndex - Struct for start and end indices resultants for the ramp - index : int - The main index for the resultant to compute the statistic for - diff : int - The offset to use for the delta and sigma values, this should be - a value from the Diff enum. + correct : float + The correction factor needed Returns ------- Create a single instance of the stastic for the given parameters """ - cdef float delta = (local_slope - slope) - cdef float var = ((var_read_noise + - slope * var_slope_coeff) - / t_bar_diff_sqr) + cdef float delta = local_slope - slope + cdef float var = (var_read_noise + slope * var_slope_coeff) / t_bar_diff_sqr - return (delta / sqrt(var + correct)) + return delta / sqrt(var + correct) @boundscheck(False) @wraparound(False) -cdef inline (int, float) statistics(float[:, :] pixel, - float[:, :] fixed, - float[:] t_bar, - float slope, RampIndex ramp): +cdef inline (int, float) _fit_statistic(float[:, :] pixel, + float[:, :] fixed, + float[:] t_bar, + float slope, + RampIndex ramp): """ - Compute fit statistics for jump detection on a single ramp - stats[i] = max(stat(i, 0), stat(i, 1)) - Note for i == end - 1, no stat(i, 1) exists, so its just stat(i, 0) + Compute the maximum index and its value over all fit statistics for a given + ramp. Each index's stat is the max of the single and double difference + statistics: + all_stats = Parameters ---------- + pixel : float[:, :] + The pre-computed fixed values for a given pixel + fixed : float[:, :] + The pre-computed fixed values for a given read_pattern + t_bar : float[:, :] + The average time for each resultant slope : float The computed slope for the ramp ramp : RampIndex @@ -114,20 +168,10 @@ cdef inline (int, float) statistics(float[:, :] pixel, Returns ------- - list of statistics for each resultant + argmax(all_stats), max(all_stats) """ - # Observe that the length of the ramp's sub array of the resultant would - # be `end - start + 1`. However, we are computing single and double - # "differences" which means we need to reference at least two points in - # this subarray at a time. For the single case, the maximum index allowed - # would be `end - 1`. Observe that `range(start, end)` will iterate over - # `start, start+1, start+1, ..., end-2, end-1` - # as the second argument to the `range` is the first index outside of the - # range - cdef int start = ramp.start # index of first resultant for ramp - cdef int end = ramp.end # index of last resultant for ramp - - # Cast the enum values into integers for indexing + # Cast the enum values into integers for indexing (otherwise compiler complains) + # These will be optimized out cdef int single_local_slope = PixelOffsets.single_local_slope cdef int double_local_slope = PixelOffsets.double_local_slope cdef int single_var_read_noise = PixelOffsets.single_var_read_noise @@ -138,36 +182,49 @@ cdef inline (int, float) statistics(float[:, :] pixel, cdef int single_var_slope_val = FixedOffsets.single_var_slope_val cdef int double_var_slope_val = FixedOffsets.double_var_slope_val - cdef float correct = correction(t_bar, ramp, slope) - cdef float stat - - if start == end: + # Note that a ramp consisting of a single point is degenerate and has no + # fit statistic so we bail out here + if ramp.start == ramp.end: return 0, NAN - cdef int index = end - 1 - cdef int argmax = end - start - 1 - cdef float max_stat = statstic(pixel[single_local_slope, index], - pixel[single_var_read_noise, index], - fixed[single_t_bar_diff_sqr, index], - fixed[single_var_slope_val, index], - slope, - correct) - + # Start computing fit statistics + cdef float correct = _correction(t_bar, ramp, slope) + + # We are computing single and double differences of using the ramp's resultants. + # Each of these computations requires two points meaning that there are + # start - end - 1 possible differences. However, we cannot compute a double + # difference for the last point as there is no point after it. Therefore, + # We use this point's single difference as our initial guess for the fit + # statistic. Note that the fit statistic can technically be negative so + # this makes it much easier to compute a "lazy" max. + cdef int index = ramp.end - 1 + cdef int argmax = ramp.end - ramp.start - 1 + cdef float max_stat = _statstic(pixel[single_local_slope, index], + pixel[single_var_read_noise, index], + fixed[single_t_bar_diff_sqr, index], + fixed[single_var_slope_val, index], + slope, + correct) + + # Compute the rest of the fit statistics + cdef float stat cdef int stat_index - for stat_index, index in enumerate(range(start, end - 1)): - stat = fmaxf(statstic(pixel[single_local_slope, index], - pixel[single_var_read_noise, index], - fixed[single_t_bar_diff_sqr, index], - fixed[single_var_slope_val, index], + for stat_index, index in enumerate(range(ramp.start, ramp.end - 1)): + # Compute max of single and double difference statistics + stat = fmaxf(_statstic(pixel[single_local_slope, index], + pixel[single_var_read_noise, index], + fixed[single_t_bar_diff_sqr, index], + fixed[single_var_slope_val, index], + slope, + correct), + _statstic(pixel[double_local_slope, index], + pixel[double_var_read_noise, index], + fixed[double_t_bar_diff_sqr, index], + fixed[double_var_slope_val, index], slope, - correct), - statstic(pixel[double_local_slope, index], - pixel[double_var_read_noise, index], - fixed[double_t_bar_diff_sqr, index], - fixed[double_var_slope_val, index], - slope, - correct)) + correct)) + # If this is larger than the current max, update the max if stat > max_stat: max_stat = stat argmax = stat_index @@ -178,7 +235,7 @@ cdef inline (int, float) statistics(float[:, :] pixel, @boundscheck(False) @wraparound(False) @cdivision(True) -cdef inline RampFits fit_jumps(float[:] resultants, +cdef inline JumpFits fit_jumps(float[:] resultants, int[:] dq, float read_noise, RampQueue ramps, @@ -197,16 +254,43 @@ cdef inline RampFits fit_jumps(float[:] resultants, Parameters ---------- - ramps : vector[RampIndex] - Vector of initial ramps to fit for a single pixel + resultants : float[:] + The resultants for the pixel + dq : int[:] + The dq flags for the pixel. This is modified in place, so the external + dq flag array will be modified as a side-effect. + read_noise : float + The read noise for the pixel. + ramps : RampQueue + RampQueue for initial ramps to fit for the pixel multiple ramps are possible due to dq flags + t_bar : float[:] + The average time for each resultant + tau : float[:] + The time variance for each resultant + n_reads : int[:] + The number of reads for each resultant + n_resultants : int + The number of resultants for the pixel + fixed : float[:, :] + The jump detection pre-computed values for a given read_pattern + pixel : float[:, :] + A pre-allocated array for the jump detection fixed values for the + given pixel. This will be modified in place, it is passed in to avoid + re-allocating it for each pixel. + thresh : Thresh + The threshold parameter struct for jump detection + use_jump : bool + Turn on or off jump detection. + include_diagnostic : bool + Turn on or off recording all the diaganostic information on the fit Returns ------- RampFits struct of all the fits for a single pixel """ - # Setup algorithm - cdef RampFits ramp_fits + # Initialize algorithm + cdef JumpFits ramp_fits cdef RampIndex ramp cdef RampFit ramp_fit @@ -218,16 +302,17 @@ cdef inline RampFits fit_jumps(float[:] resultants, cdef float max_stat cdef float weight, total_weight = 0 + # Fill in the jump detection pre-compute values for a single pixel if use_jump: pixel = fill_pixel_values(pixel, resultants, fixed, read_noise, n_resultants) - # Run while the stack is non-empty + # Run while the Queue is non-empty while not ramps.empty(): # Remove top ramp of the stack to use ramp = ramps.back() ramps.pop_back() - # Compute fit + # Compute fit using the Casertano+22 algorithm ramp_fit = fit_ramp(resultants, t_bar, tau, @@ -237,24 +322,26 @@ cdef inline RampFits fit_jumps(float[:] resultants, # Run jump detection if enabled if use_jump: - argmax, max_stat = statistics(pixel, - fixed, - t_bar, - ramp_fit.slope, - ramp) - - # We have to protect against the case where the passed "ramp" is - # only a single point. In that case, stats will be empty. This - # will create an error in the max() call. - if not isnan(max_stat) and max_stat > threshold(thresh, ramp_fit.slope): + argmax, max_stat = _fit_statistic(pixel, + fixed, + t_bar, + ramp_fit.slope, + ramp) + + # Note that when a "ramp" is a single point, _fit_statistic returns + # a NaN for max_stat. Note that NaN > anything is always false so the + # result drops through as desired. + if max_stat > _threshold(thresh, ramp_fit.slope): # Compute jump point to create two new ramps # This jump point corresponds to the index of the largest # statistic: - # argmax(stats) + # argmax = argmax(stats) # These statistics are indexed relative to the # ramp's range. Therefore, we need to add the start index # of the ramp to the result. # + jump0 = argmax + ramp.start + # Note that because the resultants are averages of reads, but # jumps occur in individual reads, it is possible that the # jump is averaged down by the resultant with the actual jump @@ -268,11 +355,13 @@ cdef inline RampFits fit_jumps(float[:] resultants, # argmax(stats) does correspond to the jump resultant. # Therefore, we just remove both possible resultants from # consideration. - jump0 = argmax + ramp.start jump1 = jump0 + 1 + # Update the dq flags dq[jump0] = JUMP_DET dq[jump1] = JUMP_DET + + # Record jump diagnotics if include_diagnostic: ramp_fits.jumps.push_back(jump0) ramp_fits.jumps.push_back(jump1) @@ -316,17 +405,17 @@ cdef inline RampFits fit_jumps(float[:] resultants, # of those values. ramps.push_back(RampIndex(jump1 + 1, ramp.end)) + # Skip recording the ramp as it has a detected jump continue - # Add ramp_fit to ramp_fits if no jump detection or stats are less - # than threshold - # Note that ramps are computed backward in time meaning we need to - # reverse the order of the fits at the end + # Start recording the the fit (no jump detected) + + # Record the diagnositcs if include_diagnostic: ramp_fits.fits.push_back(ramp_fit) ramp_fits.index.push_back(ramp) - # Start computing the averages + # Start computing the averages using a lazy process # Note we do not do anything in the NaN case for degenerate ramps if not isnan(ramp_fit.slope): # protect weight against the extremely unlikely case of a zero @@ -338,7 +427,7 @@ cdef inline RampFits fit_jumps(float[:] resultants, ramp_fits.average.read_var += weight**2 * ramp_fit.read_var ramp_fits.average.poisson_var += weight**2 * ramp_fit.poisson_var - # Finish computing averages + # Finish computing averages using the lazy proces ramp_fits.average.slope /= total_weight if total_weight != 0 else 1 ramp_fits.average.read_var /= total_weight**2 if total_weight != 0 else 1 ramp_fits.average.poisson_var /= total_weight**2 if total_weight != 0 else 1 diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index bde3c086..a9087e00 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -1,5 +1,43 @@ # cython: language_level=3str +""" +This module contains all the functions needed to execute the Casertano+22 ramp + fitting algorithm on its own without jump detection. + + The _jump module contains a driver function which calls the `fit_ramp` function + from this module iteratively. This evvetively handles dq flags and detected + jumps simultaneously. + +Structs +------- +RampIndex : struct + - start : int + Index of the first resultant in the ramp + - end : int + Index of the last resultant in the ramp (so indexing of ramp requires end + 1) + +RampFit : struct + - slope : float + The slope fit to the ramp + - read_var : float + The read noise variance for the fit + - poisson_var : float + The poisson variance for the fit + +RampQueue : vector[RampIndex] + Vector of RampIndex objects (convienience typedef) + +(Public) Functions +------------------ +init_ramps : function + Create the initial ramp "queue" for each pixel in order to handle any initial + "dq" flags passed in from outside. + +fit_ramps : function + Implementation of running the Casertano+22 algorithm on a (sub)set of resultants + listed for a single pixel +""" + from cython cimport boundscheck, wraparound, cdivision, cpow from libc.math cimport sqrt, fabs, INFINITY, NAN, fmaxf @@ -12,7 +50,7 @@ from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit @wraparound(False) cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixel): """ - Create the initial ramp stack for each pixel + Create the initial ramp "queue" for each pixel if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp otherwise, the resultant is not in a ramp @@ -27,9 +65,10 @@ cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixe Returns ------- - vector of RampIndex objects - - vector with entry for each ramp found (last entry is last ramp found) - - RampIndex with start and end indices of the ramp in the resultants + RampQueue + vector of RampIndex objects + - vector with entry for each ramp found (last entry is last ramp found) + - RampIndex with start and end indices of the ramp in the resultants """ cdef RampQueue ramps = RampQueue() @@ -73,13 +112,13 @@ cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixe ctypedef float[6] _row # Casertano+2022, Table 2 -cdef _row[2] PTABLE = [[-INFINITY, 5, 10, 20, 50, 100], - [ 0, 0.4, 1, 3, 6, 10 ]] +cdef _row[2] _PTABLE = [[-INFINITY, 5, 10, 20, 50, 100], + [ 0, 0.4, 1, 3, 6, 10 ]] @boundscheck(False) @wraparound(False) -cdef inline float get_power(float signal): +cdef inline float _get_power(float signal): """ Return the power from Casertano+22, Table 2 @@ -94,10 +133,11 @@ cdef inline float get_power(float signal): """ cdef int i for i in range(6): - if signal < PTABLE[0][i]: - return PTABLE[1][i - 1] + if signal < _PTABLE[0][i]: + return _PTABLE[1][i - 1] + + return _PTABLE[1][i] - return PTABLE[1][i] @boundscheck(False) @wraparound(False) @@ -113,12 +153,26 @@ cdef inline RampFit fit_ramp(float[:] resultants_, Parameters ---------- + resultants_ : float[:] + All of the resultants for the pixel + t_bar_ : float[:] + All the t_bar values + tau_ : float[:] + All the tau values + n_reads_ : int[:] + All the n_reads values + read_noise : float + The read noise for the pixel ramp : RampIndex Struct for start and end of ramp to fit Returns ------- - RampFit struct of slope, read_var, poisson_var + RampFit + struct containing + - slope + - read_var + - poisson_var """ cdef int n_resultants = ramp.end - ramp.start + 1 @@ -133,7 +187,7 @@ cdef inline RampFit fit_ramp(float[:] resultants_, # Compute the fit cdef int i = 0, j = 0 - # Setup data for fitting (work over subset of data) + # Setup data for fitting (work over subset of data) to make things cleaner # Recall that the RampIndex contains the index of the first and last # index of the ramp. Therefore, the Python slice needed to get all the # data within the ramp is: @@ -148,13 +202,13 @@ cdef inline RampFit fit_ramp(float[:] resultants_, cdef float t_bar_mid = (t_bar[0] + t_bar[end]) / 2 # Casertano+2022 Eq. 44 - # Note we've departed from Casertano+22 slightly; - # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., - # a CR in the first resultant has boosted the whole ramp high but there - # is no actual signal. + # Note we've departed from Casertano+22 slightly; + # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., + # a CR in the first resultant has boosted the whole ramp high but there + # is no actual signal. cdef float power = fmaxf(resultants[end] - resultants[0], 0) power = power / sqrt(read_noise**2 + power) - power = get_power(power) + power = _get_power(power) # It's easy to use up a lot of dynamic range on something like # (tbar - tbarmid) ** 10. Rescale these. @@ -162,6 +216,7 @@ cdef inline RampFit fit_ramp(float[:] resultants_, t_scale = 1 if t_scale == 0 else t_scale # Initalize the fit loop + # it is faster to generate a c++ vector than a numpy array cdef vector[float] weights = vector[float](n_resultants) cdef vector[float] coeffs = vector[float](n_resultants) cdef RampFit ramp_fit = RampFit(0, 0, 0) @@ -197,6 +252,9 @@ cdef inline RampFit fit_ramp(float[:] resultants_, ramp_fit.read_var += (coeff ** 2 * read_noise ** 2 / n_reads[i]) # Casertano+22 Eq 40 + # Note that this is an inversion of the indexing from the equation; + # however, commutivity of addition results in the same answer. This + # makes it so that we don't have to loop over all the resultants twice. ramp_fit.poisson_var += coeff ** 2 * tau[i] for j in range(i): ramp_fit.poisson_var += (2 * coeff * coeffs[j] * t_bar[j]) diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx index ce8a99c4..75f6a23d 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx @@ -2,7 +2,7 @@ import numpy as np cimport numpy as cnp -from cython cimport boundscheck, wraparound +from cython cimport boundscheck, cdivision, wraparound from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern @@ -11,11 +11,16 @@ cnp.import_array() cdef class ReadPattern: """ Class to contain the read pattern derived metadata + This exists only to allow us to output multiple memory views at the same time + from the same cython function. This is needed because neither structs nor unions + can contain memory views. + + In the case of this code memory views are the fastest "safe" array data structure. + This class will immediately be unpacked into raw memory views, so that we avoid + any further overhead of swithcing between python and cython. Attributes: ---------- - n_resultants : int - The number of resultants in the read pattern t_bar : np.ndarray[float_t, ndim=1] The mean time of each resultant tau : np.ndarray[float_t, ndim=1] @@ -41,12 +46,12 @@ cdef class ReadPattern: @boundscheck(False) @wraparound(False) +@cdivision(True) cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time, int n_resultants): """ Derive the input data from the the read pattern - - read pattern is a list of resultant lists, where each resultant list is - a list of the reads in that resultant. + This is faster than using __init__ or __cinit__ to construct the object with + these calls. Parameters ---------- @@ -54,11 +59,18 @@ cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_tim read pattern for the image read_time : float Time to perform a readout. + n_resultants : int + Number of resultants in the image Returns ------- ReadPattern + Contains: + - t_bar + - tau + - n_reads """ + cdef ReadPattern data = ReadPattern() data.t_bar = np.empty(n_resultants, dtype=np.float32) data.tau = np.empty(n_resultants, dtype=np.float32) From e02a686c8ddb5304acc6e1feecad24a5f58c1088 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 15:07:56 -0500 Subject: [PATCH 22/30] Merge fixed module into jump module --- setup.py | 6 - src/stcal/ramp_fitting/ols_cas22/__init__.py | 1 - .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 5 +- src/stcal/ramp_fitting/ols_cas22/_fixed.pxd | 34 ----- src/stcal/ramp_fitting/ols_cas22/_fixed.pyx | 132 ------------------ src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 34 +++++ src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 112 ++++++++++++++- tests/test_jump_cas22.py | 2 +- 8 files changed, 146 insertions(+), 180 deletions(-) delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_fixed.pxd delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_fixed.pyx diff --git a/setup.py b/setup.py index 007cf450..c9c27d6d 100644 --- a/setup.py +++ b/setup.py @@ -7,12 +7,6 @@ Options.annotate = False extensions = [ - Extension( - 'stcal.ramp_fitting.ols_cas22._fixed', - ['src/stcal/ramp_fitting/ols_cas22/_fixed.pyx'], - include_dirs=[np.get_include()], - language='c++' - ), Extension( 'stcal.ramp_fitting.ols_cas22._jump', ['src/stcal/ramp_fitting/ols_cas22/_jump.pyx'], diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/ols_cas22/__init__.py index b1482c67..c46ab4dd 100644 --- a/src/stcal/ramp_fitting/ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/ols_cas22/__init__.py @@ -1,5 +1,4 @@ from ._fit_ramps import fit_ramps, RampFitOutputs, Parameter, Variance -# from ._core import Parameter, Variance from ._jump import JUMP_DET __all__ = ['fit_ramps', 'RampFitOutputs', 'Parameter', 'Variance', 'Diff', 'JUMP_DET'] diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index dd430a91..79352f20 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -7,10 +7,7 @@ from cython cimport boundscheck, wraparound from libcpp cimport bool from libcpp.list cimport list as cpp_list -# from stcal.ramp_fitting.ols_cas22._core cimport Parameter, Variance -from stcal.ramp_fitting.ols_cas22._fixed cimport fill_fixed_values, n_fixed_offsets, n_pixel_offsets - -from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, fit_jumps, JumpFits +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, JumpFits, fill_fixed_values, fit_jumps, n_fixed_offsets, n_pixel_offsets from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern, from_read_pattern diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd b/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd deleted file mode 100644 index 2ebdf9ce..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pxd +++ /dev/null @@ -1,34 +0,0 @@ -# cython: language_level=3str - -cpdef enum FixedOffsets: - single_t_bar_diff - double_t_bar_diff - single_t_bar_diff_sqr - double_t_bar_diff_sqr - single_read_recip - double_read_recip - single_var_slope_val - double_var_slope_val - n_fixed_offsets - - -cpdef enum PixelOffsets: - single_local_slope - double_local_slope - single_var_read_noise - double_var_read_noise - n_pixel_offsets - - -cpdef float[:, :] fill_fixed_values(float[:, :] fixed, - float[:] t_bar, - float[:] tau, - int[:] n_reads, - int n_resultants) - - -cpdef float[:, :] fill_pixel_values(float[:, :] pixel, - float[:] resultants, - float[:, :] fixed, - float read_noise, - int n_resultants) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx b/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx deleted file mode 100644 index a9a8957b..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_fixed.pyx +++ /dev/null @@ -1,132 +0,0 @@ -# cython: language_level=3str - -""" -Define the data which is fixed for all pixels to compute the CAS22 algorithm with - jump detection - -Objects -------- -FixedValues : class - Class to contain the data fixed for all pixels and commonly referenced - universal values for jump detection - -Functions ---------- - fixed_values_from_metadata : function - Fast constructor for FixedValues from the read pattern metadata - - cpdef gives a python wrapper, but the python version of this method - is considered private, only to be used for testing -""" -from cython cimport boundscheck, wraparound, cdivision - -from libc.math cimport NAN - -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets, PixelOffsets - - - -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cpdef inline float[:, :] fill_fixed_values(float[:, :] fixed, - float[:] t_bar, - float[:] tau, - int[:] n_reads, - int n_resultants): - """ - Compute the difference offset of t_bar - - Returns - ------- - [ - , - , - ** 2, - ** 2, - <(1/n_reads[i+1] + 1/n_reads[i])>, - <(1/n_reads[i+2] + 1/n_reads[i])>, - <(tau[i] + tau[i+1] - 2 * min(t_bar[i], t_bar[i+1]))>, - <(tau[i] + tau[i+2] - 2 * min(t_bar[i], t_bar[i+2]))>, - ] - """ - - cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff - cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff - cdef int single_t_bar_diff_sqr = FixedOffsets.single_t_bar_diff_sqr - cdef int double_t_bar_diff_sqr = FixedOffsets.double_t_bar_diff_sqr - cdef int single_read_recip = FixedOffsets.single_read_recip - cdef int double_read_recip = FixedOffsets.double_read_recip - cdef int single_var_slope_val = FixedOffsets.single_var_slope_val - cdef int double_var_slope_val = FixedOffsets.double_var_slope_val - - # Coerce division to be using floats - cdef float num = 1 - - cdef int i - for i in range(n_resultants - 1): - fixed[single_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] - fixed[single_t_bar_diff_sqr, i] = fixed[single_t_bar_diff, i] ** 2 - fixed[single_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) - fixed[single_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) - - if i < n_resultants - 2: - fixed[double_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] - fixed[double_t_bar_diff_sqr, i] = fixed[double_t_bar_diff, i] ** 2 - fixed[double_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) - fixed[double_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) - else: - # Last double difference is undefined - fixed[double_t_bar_diff, i] = NAN - fixed[double_t_bar_diff_sqr, i] = NAN - fixed[double_read_recip, i] = NAN - fixed[double_var_slope_val, i] = NAN - - return fixed - - -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cpdef inline float[:, :] fill_pixel_values(float[:, :] pixel, - float[:] resultants, - float[:, :] fixed, - float read_noise, - int n_resultants): - """ - Compute the local slopes between resultants for the pixel - - Returns - ------- - [ - <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, - <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, - read_noise**2 * <(1/n_reads[i+1] + 1/n_reads[i])>, - read_noise**2 * <(1/n_reads[i+2] + 1/n_reads[i])>, - ] - """ - cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff - cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff - cdef int single_read_recip = FixedOffsets.single_read_recip - cdef int double_read_recip = FixedOffsets.double_read_recip - - cdef int single_slope = PixelOffsets.single_local_slope - cdef int double_slope = PixelOffsets.double_local_slope - cdef int single_var = PixelOffsets.single_var_read_noise - cdef int double_var = PixelOffsets.double_var_read_noise - - cdef float read_noise_sqr = read_noise ** 2 - - cdef int i - for i in range(n_resultants - 1): - pixel[single_slope, i] = (resultants[i + 1] - resultants[i]) / fixed[single_t_bar_diff, i] - pixel[single_var, i] = read_noise_sqr * fixed[single_read_recip, i] - - if i < n_resultants - 2: - pixel[double_slope, i] = (resultants[i + 2] - resultants[i]) / fixed[double_t_bar_diff, i] - pixel[double_var, i] = read_noise_sqr * fixed[double_read_recip, i] - else: - # The last double difference is undefined - pixel[double_slope, i] = NAN - pixel[double_var, i] = NAN - - return pixel diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 1f9316d6..a7e011e1 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -6,6 +6,26 @@ from libcpp.vector cimport vector from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampQueue +cpdef enum FixedOffsets: + single_t_bar_diff + double_t_bar_diff + single_t_bar_diff_sqr + double_t_bar_diff_sqr + single_read_recip + double_read_recip + single_var_slope_val + double_var_slope_val + n_fixed_offsets + + +cpdef enum PixelOffsets: + single_local_slope + double_local_slope + single_var_read_noise + double_var_read_noise + n_pixel_offsets + + cpdef enum: JUMP_DET = 4 @@ -21,6 +41,20 @@ cdef struct JumpFits: RampQueue index +cpdef float[:, :] fill_fixed_values(float[:, :] fixed, + float[:] t_bar, + float[:] tau, + int[:] n_reads, + int n_resultants) + + +cpdef float[:, :] fill_pixel_values(float[:, :] pixel, + float[:] resultants, + float[:, :] fixed, + float read_noise, + int n_resultants) + + cdef JumpFits fit_jumps(float[:] resultants, int[:] dq, float read_noise, diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 22bf9f2e..0934a3ab 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -43,10 +43,118 @@ from cython cimport boundscheck, wraparound, cdivision from libcpp cimport bool from libc.math cimport sqrt, log10, fmaxf, NAN, isnan -from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, JumpFits, JUMP_DET -from stcal.ramp_fitting.ols_cas22._fixed cimport FixedOffsets, PixelOffsets, fill_pixel_values +from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, JumpFits, JUMP_DET, FixedOffsets, PixelOffsets from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, fit_ramp + + +@boundscheck(False) +@wraparound(False) +@cdivision(True) +cpdef inline float[:, :] fill_fixed_values(float[:, :] fixed, + float[:] t_bar, + float[:] tau, + int[:] n_reads, + int n_resultants): + """ + Compute the difference offset of t_bar + + Returns + ------- + [ + , + , + ** 2, + ** 2, + <(1/n_reads[i+1] + 1/n_reads[i])>, + <(1/n_reads[i+2] + 1/n_reads[i])>, + <(tau[i] + tau[i+1] - 2 * min(t_bar[i], t_bar[i+1]))>, + <(tau[i] + tau[i+2] - 2 * min(t_bar[i], t_bar[i+2]))>, + ] + """ + + cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff + cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff + cdef int single_t_bar_diff_sqr = FixedOffsets.single_t_bar_diff_sqr + cdef int double_t_bar_diff_sqr = FixedOffsets.double_t_bar_diff_sqr + cdef int single_read_recip = FixedOffsets.single_read_recip + cdef int double_read_recip = FixedOffsets.double_read_recip + cdef int single_var_slope_val = FixedOffsets.single_var_slope_val + cdef int double_var_slope_val = FixedOffsets.double_var_slope_val + + # Coerce division to be using floats + cdef float num = 1 + + cdef int i + for i in range(n_resultants - 1): + fixed[single_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] + fixed[single_t_bar_diff_sqr, i] = fixed[single_t_bar_diff, i] ** 2 + fixed[single_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) + fixed[single_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) + + if i < n_resultants - 2: + fixed[double_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] + fixed[double_t_bar_diff_sqr, i] = fixed[double_t_bar_diff, i] ** 2 + fixed[double_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) + fixed[double_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) + else: + # Last double difference is undefined + fixed[double_t_bar_diff, i] = NAN + fixed[double_t_bar_diff_sqr, i] = NAN + fixed[double_read_recip, i] = NAN + fixed[double_var_slope_val, i] = NAN + + return fixed + + +@boundscheck(False) +@wraparound(False) +@cdivision(True) +cpdef inline float[:, :] fill_pixel_values(float[:, :] pixel, + float[:] resultants, + float[:, :] fixed, + float read_noise, + int n_resultants): + """ + Compute the local slopes between resultants for the pixel + + Returns + ------- + [ + <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, + <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, + read_noise**2 * <(1/n_reads[i+1] + 1/n_reads[i])>, + read_noise**2 * <(1/n_reads[i+2] + 1/n_reads[i])>, + ] + """ + cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff + cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff + cdef int single_read_recip = FixedOffsets.single_read_recip + cdef int double_read_recip = FixedOffsets.double_read_recip + + cdef int single_slope = PixelOffsets.single_local_slope + cdef int double_slope = PixelOffsets.double_local_slope + cdef int single_var = PixelOffsets.single_var_read_noise + cdef int double_var = PixelOffsets.double_var_read_noise + + cdef float read_noise_sqr = read_noise ** 2 + + cdef int i + for i in range(n_resultants - 1): + pixel[single_slope, i] = (resultants[i + 1] - resultants[i]) / fixed[single_t_bar_diff, i] + pixel[single_var, i] = read_noise_sqr * fixed[single_read_recip, i] + + if i < n_resultants - 2: + pixel[double_slope, i] = (resultants[i + 2] - resultants[i]) / fixed[double_t_bar_diff, i] + pixel[double_var, i] = read_noise_sqr * fixed[double_read_recip, i] + else: + # The last double difference is undefined + pixel[double_slope, i] = NAN + pixel[double_var, i] = NAN + + return pixel + + cdef inline float _threshold(Thresh thresh, float slope): """ Compute jump threshold diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 769398f9..1833563d 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,7 +2,7 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22._fixed import fill_fixed_values, fill_pixel_values, FixedOffsets, PixelOffsets +from stcal.ramp_fitting.ols_cas22._jump import fill_fixed_values, fill_pixel_values, FixedOffsets, PixelOffsets from stcal.ramp_fitting.ols_cas22._ramp import init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern From 804718dba8e2d39296b6e87d0fd4d8450bcef847 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 15:24:32 -0500 Subject: [PATCH 23/30] Finish documenting the jump detection algorithm --- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 7 --- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 65 +++++++++++++++++++--- tests/test_jump_cas22.py | 6 +- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index a7e011e1..004758c5 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -48,13 +48,6 @@ cpdef float[:, :] fill_fixed_values(float[:, :] fixed, int n_resultants) -cpdef float[:, :] fill_pixel_values(float[:, :] pixel, - float[:] resultants, - float[:, :] fixed, - float read_noise, - int n_resultants) - - cdef JumpFits fit_jumps(float[:] resultants, int[:] dq, float read_noise, diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 0934a3ab..7312f5af 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -28,8 +28,24 @@ JumpFits : struct - index : RampQueue The RampIndex representations correspoinding to each fit in fits +Enums +----- + FixedOffsets : enum + Enumerate the different pieces of information computed for jump detection + which only depend on the read pattern. + + PixelOffsets : enum + Enumerate the different pieces of information computed for jump detection + which only depend on the given pixel (independent of specific ramp). + + JUMP_DET : value + A the fixed value for the jump detection dq flag. + (Public) Functions ------------------ +fill_fixed_values : function + Pre-compute all the values needed for jump detection for a given read_pattern, + this is independent of the pixel involved. fit_jumps : function Compute all the ramps for a single pixel using the Casertano+22 algorithm @@ -57,7 +73,22 @@ cpdef inline float[:, :] fill_fixed_values(float[:, :] fixed, int[:] n_reads, int n_resultants): """ - Compute the difference offset of t_bar + Pre-compute all the values needed for jump detection which only depend on + the read pattern. + + Parameters + ---------- + fixed : float[:, :] + A pre-allocated memoryview to store the pre-computed values in, its faster + to allocate outside this function. + t_bar : float[:] + The average time for each resultant + tau : float[:] + The time variance for each resultant + n_reads : int[:] + The number of reads for each resultant + n_resultants : int + The number of resultants for the read pattern Returns ------- @@ -72,7 +103,8 @@ cpdef inline float[:, :] fill_fixed_values(float[:, :] fixed, <(tau[i] + tau[i+2] - 2 * min(t_bar[i], t_bar[i+2]))>, ] """ - + # Cast the enum values into integers for indexing (otherwise compiler complains) + # These will be optimized out cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff cdef int single_t_bar_diff_sqr = FixedOffsets.single_t_bar_diff_sqr @@ -110,13 +142,28 @@ cpdef inline float[:, :] fill_fixed_values(float[:, :] fixed, @boundscheck(False) @wraparound(False) @cdivision(True) -cpdef inline float[:, :] fill_pixel_values(float[:, :] pixel, - float[:] resultants, - float[:, :] fixed, - float read_noise, - int n_resultants): +cpdef inline float[:, :] _fill_pixel_values(float[:, :] pixel, + float[:] resultants, + float[:, :] fixed, + float read_noise, + int n_resultants): """ - Compute the local slopes between resultants for the pixel + Pre-compute all the values needed for jump detection which only depend on + the a specific pixel (independent of the given ramp for a pixel). + + Parameters + ---------- + pixel : float[:, :] + A pre-allocated memoryview to store the pre-computed values in, its faster + to allocate outside this function. + resultants : float[:] + The resultants for the pixel in question. + fixed : float[:, :] + The pre-computed fixed values for the read_pattern + read_noise : float + The read noise for the pixel + n_resultants : int + The number of resultants for the read_pattern Returns ------- @@ -412,7 +459,7 @@ cdef inline JumpFits fit_jumps(float[:] resultants, # Fill in the jump detection pre-compute values for a single pixel if use_jump: - pixel = fill_pixel_values(pixel, resultants, fixed, read_noise, n_resultants) + pixel = _fill_pixel_values(pixel, resultants, fixed, read_noise, n_resultants) # Run while the Queue is non-empty while not ramps.empty(): diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 1833563d..8b044af6 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,7 +2,7 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22._jump import fill_fixed_values, fill_pixel_values, FixedOffsets, PixelOffsets +from stcal.ramp_fitting.ols_cas22._jump import fill_fixed_values, _fill_pixel_values, FixedOffsets, PixelOffsets from stcal.ramp_fitting.ols_cas22._ramp import init_ramps from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern @@ -252,13 +252,13 @@ def pixel_data(ramp_data): yield resultants, t_bar, tau, n_reads, fixed -def test_fill_pixel_values(pixel_data): +def test__fill_pixel_values(pixel_data): """Test computing the initial pixel data""" resultants, t_bar, tau, n_reads, fixed = pixel_data n_resultants = len(t_bar) pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) - pixel = fill_pixel_values(pixel, resultants, fixed, READ_NOISE, n_resultants) + pixel = _fill_pixel_values(pixel, resultants, fixed, READ_NOISE, n_resultants) # Sanity check that the shape of pixel is correct assert pixel.shape == (2 * 2, n_resultants - 1) From 21a0c916a25f21dc676da3fd0f493e96fd1feaf0 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 15:34:53 -0500 Subject: [PATCH 24/30] Merge read_pattern module into ramp fitting module --- setup.py | 6 -- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 10 +- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 17 +++- src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 99 ++++++++++++++++++- .../ramp_fitting/ols_cas22/_read_pattern.pxd | 9 -- .../ramp_fitting/ols_cas22/_read_pattern.pyx | 88 ----------------- tests/test_jump_cas22.py | 3 +- 7 files changed, 122 insertions(+), 110 deletions(-) delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx diff --git a/setup.py b/setup.py index c9c27d6d..5ba62d24 100644 --- a/setup.py +++ b/setup.py @@ -19,12 +19,6 @@ include_dirs=[np.get_include()], language='c++' ), - Extension( - 'stcal.ramp_fitting.ols_cas22._read_pattern', - ['src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx'], - include_dirs=[np.get_include()], - language='c++' - ), Extension( 'stcal.ramp_fitting.ols_cas22._fit_ramps', ['src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx'], diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index 79352f20..f72c2367 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -7,9 +7,13 @@ from cython cimport boundscheck, wraparound from libcpp cimport bool from libcpp.list cimport list as cpp_list -from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, JumpFits, fill_fixed_values, fit_jumps, n_fixed_offsets, n_pixel_offsets -from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps -from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern, from_read_pattern +from stcal.ramp_fitting.ols_cas22._jump cimport (Thresh, + JumpFits, + fill_fixed_values, + fit_jumps, + n_fixed_offsets, + n_pixel_offsets) +from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps, ReadPattern, from_read_pattern from typing import NamedTuple, Optional diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index 9472d2fe..b5b3f163 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -17,7 +17,22 @@ cdef struct RampFit: ctypedef vector[RampIndex] RampQueue -cpdef RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixel) +cdef class ReadPattern: + cdef float[::1] t_bar + cdef float[::1] tau + cdef int[::1] n_reads + + +cpdef RampQueue init_ramps(int[:, :] dq, + int n_resultants, + int index_pixel) + + +cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, + float read_time, + int n_resultants) + + cdef RampFit fit_ramp(float[:] resultants_, float[:] t_bar_, float[:] tau_, diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index a9087e00..38ee584e 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -27,23 +27,120 @@ RampFit : struct RampQueue : vector[RampIndex] Vector of RampIndex objects (convienience typedef) +Classes +------- + ReadPattern : + Container class for all the metadata derived from the read pattern, this + is just a temporary object to allow us to return multiple memory views from + a single function. + (Public) Functions ------------------ init_ramps : function Create the initial ramp "queue" for each pixel in order to handle any initial "dq" flags passed in from outside. +from_read_pattern : function + Derive the input data from the the read pattern + This is faster than using __init__ or __cinit__ to construct the object with + these calls. + fit_ramps : function Implementation of running the Casertano+22 algorithm on a (sub)set of resultants listed for a single pixel """ +import numpy as np +cimport numpy as cnp from cython cimport boundscheck, wraparound, cdivision, cpow from libc.math cimport sqrt, fabs, INFINITY, NAN, fmaxf from libcpp.vector cimport vector -from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit +from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, ReadPattern + + +cnp.import_array() + + +cdef class ReadPattern: + """ + Class to contain the read pattern derived metadata + This exists only to allow us to output multiple memory views at the same time + from the same cython function. This is needed because neither structs nor unions + can contain memory views. + + In the case of this code memory views are the fastest "safe" array data structure. + This class will immediately be unpacked into raw memory views, so that we avoid + any further overhead of swithcing between python and cython. + + Attributes: + ---------- + t_bar : np.ndarray[float_t, ndim=1] + The mean time of each resultant + tau : np.ndarray[float_t, ndim=1] + The variance in time of each resultant + n_reads : np.ndarray[cnp.int32_t, ndim=1] + The number of reads in each resultant + """ + + def _to_dict(ReadPattern self): + """ + This is a private method to convert the ReadPattern object to a dictionary, + so that attributes can be directly accessed in python. Note that this + is needed because class attributes cannot be accessed on cython classes + directly in python. Instead they need to be accessed or set using a + python compatible method. This method is a pure puthon method bound + to to the cython class and should not be used by any cython code, and + only exists for testing purposes. + """ + return dict(t_bar=np.array(self.t_bar, dtype=np.float32), + tau=np.array(self.tau, dtype=np.float32), + n_reads=np.array(self.n_reads, dtype=np.int32)) + + +@boundscheck(False) +@wraparound(False) +@cdivision(True) +cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time, int n_resultants): + """ + Derive the input data from the the read pattern + This is faster than using __init__ or __cinit__ to construct the object with + these calls. + + Parameters + ---------- + read pattern: list[list[int]] + read pattern for the image + read_time : float + Time to perform a readout. + n_resultants : int + Number of resultants in the image + + Returns + ------- + ReadPattern + Contains: + - t_bar + - tau + - n_reads + """ + + cdef ReadPattern data = ReadPattern() + data.t_bar = np.empty(n_resultants, dtype=np.float32) + data.tau = np.empty(n_resultants, dtype=np.float32) + data.n_reads = np.empty(n_resultants, dtype=np.int32) + + cdef int index, n_reads + cdef list[int] resultant + for index, resultant in enumerate(read_pattern): + n_reads = len(resultant) + + data.n_reads[index] = n_reads + data.t_bar[index] = read_time * np.mean(resultant) + data.tau[index] = np.sum((2 * (n_reads - np.arange(n_reads)) - 1) * resultant) * read_time / n_reads**2 + + return data @boundscheck(False) diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd deleted file mode 100644 index 89d4419c..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pxd +++ /dev/null @@ -1,9 +0,0 @@ -# cython: language_level=3str - -cdef class ReadPattern: - cdef float[::1] t_bar - cdef float[::1] tau - cdef int[::1] n_reads - - -cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time, int n_resultants) \ No newline at end of file diff --git a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx b/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx deleted file mode 100644 index 75f6a23d..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_read_pattern.pyx +++ /dev/null @@ -1,88 +0,0 @@ -# cython: language_level=3str - -import numpy as np -cimport numpy as cnp -from cython cimport boundscheck, cdivision, wraparound - -from stcal.ramp_fitting.ols_cas22._read_pattern cimport ReadPattern - -cnp.import_array() - -cdef class ReadPattern: - """ - Class to contain the read pattern derived metadata - This exists only to allow us to output multiple memory views at the same time - from the same cython function. This is needed because neither structs nor unions - can contain memory views. - - In the case of this code memory views are the fastest "safe" array data structure. - This class will immediately be unpacked into raw memory views, so that we avoid - any further overhead of swithcing between python and cython. - - Attributes: - ---------- - t_bar : np.ndarray[float_t, ndim=1] - The mean time of each resultant - tau : np.ndarray[float_t, ndim=1] - The variance in time of each resultant - n_reads : np.ndarray[cnp.int32_t, ndim=1] - The number of reads in each resultant - """ - - def _to_dict(ReadPattern self): - """ - This is a private method to convert the ReadPattern object to a dictionary, - so that attributes can be directly accessed in python. Note that this - is needed because class attributes cannot be accessed on cython classes - directly in python. Instead they need to be accessed or set using a - python compatible method. This method is a pure puthon method bound - to to the cython class and should not be used by any cython code, and - only exists for testing purposes. - """ - return dict(t_bar=np.array(self.t_bar, dtype=np.float32), - tau=np.array(self.tau, dtype=np.float32), - n_reads=np.array(self.n_reads, dtype=np.int32)) - - -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time, int n_resultants): - """ - Derive the input data from the the read pattern - This is faster than using __init__ or __cinit__ to construct the object with - these calls. - - Parameters - ---------- - read pattern: list[list[int]] - read pattern for the image - read_time : float - Time to perform a readout. - n_resultants : int - Number of resultants in the image - - Returns - ------- - ReadPattern - Contains: - - t_bar - - tau - - n_reads - """ - - cdef ReadPattern data = ReadPattern() - data.t_bar = np.empty(n_resultants, dtype=np.float32) - data.tau = np.empty(n_resultants, dtype=np.float32) - data.n_reads = np.empty(n_resultants, dtype=np.int32) - - cdef int index, n_reads - cdef list[int] resultant - for index, resultant in enumerate(read_pattern): - n_reads = len(resultant) - - data.n_reads[index] = n_reads - data.t_bar[index] = read_time * np.mean(resultant) - data.tau[index] = np.sum((2 * (n_reads - np.arange(n_reads)) - 1) * resultant) * read_time / n_reads**2 - - return data \ No newline at end of file diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 8b044af6..906e5f2c 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -3,8 +3,7 @@ from numpy.testing import assert_allclose from stcal.ramp_fitting.ols_cas22._jump import fill_fixed_values, _fill_pixel_values, FixedOffsets, PixelOffsets -from stcal.ramp_fitting.ols_cas22._ramp import init_ramps -from stcal.ramp_fitting.ols_cas22._read_pattern import from_read_pattern +from stcal.ramp_fitting.ols_cas22._ramp import from_read_pattern, init_ramps from stcal.ramp_fitting.ols_cas22 import fit_ramps, Parameter, Variance, JUMP_DET From 26679849c7bd6aff9671fc71d46fd6e3758341c2 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 15:43:06 -0500 Subject: [PATCH 25/30] Move where ramp initialization takes place --- src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx | 3 +-- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 1 - src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 6 ++++-- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 5 ++--- src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 10 ++++------ tests/test_jump_cas22.py | 2 +- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index f72c2367..3f2329ec 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -13,7 +13,7 @@ from stcal.ramp_fitting.ols_cas22._jump cimport (Thresh, fit_jumps, n_fixed_offsets, n_pixel_offsets) -from stcal.ramp_fitting.ols_cas22._ramp cimport init_ramps, ReadPattern, from_read_pattern +from stcal.ramp_fitting.ols_cas22._ramp cimport ReadPattern, from_read_pattern from typing import NamedTuple, Optional @@ -149,7 +149,6 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, fit = fit_jumps(resultants[:, index], dq[:, index], read_noise[index], - init_ramps(dq, n_resultants, index), t_bar, tau, n_reads, diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 004758c5..8693e791 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -51,7 +51,6 @@ cpdef float[:, :] fill_fixed_values(float[:, :] fixed, cdef JumpFits fit_jumps(float[:] resultants, int[:] dq, float read_noise, - RampQueue ramps, float[:] t_bar, float[:] tau, int[:] n_reads, diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 7312f5af..6e696d14 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -60,7 +60,7 @@ from libcpp cimport bool from libc.math cimport sqrt, log10, fmaxf, NAN, isnan from stcal.ramp_fitting.ols_cas22._jump cimport Thresh, JumpFits, JUMP_DET, FixedOffsets, PixelOffsets -from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, fit_ramp +from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, fit_ramp, init_ramps @@ -393,7 +393,6 @@ cdef inline (int, float) _fit_statistic(float[:, :] pixel, cdef inline JumpFits fit_jumps(float[:] resultants, int[:] dq, float read_noise, - RampQueue ramps, float[:] t_bar, float[:] tau, int[:] n_reads, @@ -444,6 +443,9 @@ cdef inline JumpFits fit_jumps(float[:] resultants, ------- RampFits struct of all the fits for a single pixel """ + # Find initial set of ramps + cdef RampQueue ramps = init_ramps(dq, n_resultants) + # Initialize algorithm cdef JumpFits ramp_fits cdef RampIndex ramp diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index b5b3f163..de31cd6c 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -23,9 +23,8 @@ cdef class ReadPattern: cdef int[::1] n_reads -cpdef RampQueue init_ramps(int[:, :] dq, - int n_resultants, - int index_pixel) +cpdef RampQueue init_ramps(int[:] dq, + int n_resultants) cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index 38ee584e..2f3c9172 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -145,7 +145,7 @@ cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_tim @boundscheck(False) @wraparound(False) -cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixel): +cpdef inline RampQueue init_ramps(int[:] dq, int n_resultants): """ Create the initial ramp "queue" for each pixel if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp @@ -153,12 +153,10 @@ cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixe Parameters ---------- - dq : int[n_resultants, n_pixel] + dq : int[n_resultants] DQ array n_resultants : int Number of resultants - index_pixel : int - The index of the pixel to find ramps for Returns ------- @@ -177,7 +175,7 @@ cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixe for index_resultant in range(n_resultants): if ramp.start == -1: # Looking for the start of a ramp - if dq[index_resultant, index_pixel] == 0: + if dq[index_resultant] == 0: # We have found the start of a ramp! ramp.start = index_resultant else: @@ -185,7 +183,7 @@ cpdef inline RampQueue init_ramps(int[:, :] dq, int n_resultants, int index_pixe continue else: # Looking for the end of a ramp - if dq[index_resultant, index_pixel] == 0: + if dq[index_resultant] == 0: # This pixel is in the ramp do nothing continue else: diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 906e5f2c..87a701c9 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -48,7 +48,7 @@ def test_init_ramps(): [0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1]], dtype=np.int32) n_resultants, n_pixels = dq.shape - ramps = [init_ramps(dq, n_resultants, index_pixel) for index_pixel in range(n_pixels)] + ramps = [init_ramps(dq[:, index], n_resultants) for index in range(n_pixels)] assert len(ramps) == dq.shape[1] == 16 From 525ff627f7307c7a839337db94c59286cfe30959 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 16:20:05 -0500 Subject: [PATCH 26/30] Document _fit_ramps module --- .../ramp_fitting/ols_cas22/_fit_ramps.pyx | 88 +++++++++++++++---- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 16 ++-- src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 9 +- 3 files changed, 86 insertions(+), 27 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx index 3f2329ec..5764b139 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx @@ -1,5 +1,32 @@ # cython: language_level=3str +""" +External interface module for the Casertano+22 ramp fitting algorithm with jump detection. + This module is intended to contain everything needed by external code. + +Enums +----- +Parameter : + Enumerate the index for the output parameters array. + +Variance : + Enumerate the index for the output variances array. + +Classes +------- +RampFitOutputs : NamedTuple + Simple tuple wrapper for outputs from the ramp fitting algorithm + This clarifies the meaning of the outputs via naming them something + descriptive. + +(Public) Functions +------------------ +fit_ramps : function + Fit ramps using the Castenario+22 algorithm to a set of pixels accounting + for jumps (if use_jump is True) and bad pixels (via the dq array). This + is the primary externally callable function. +""" + import numpy as np cimport numpy as cnp @@ -18,6 +45,7 @@ from stcal.ramp_fitting.ols_cas22._ramp cimport ReadPattern, from_read_pattern from typing import NamedTuple, Optional +# Initialize numpy for cython use in this module cnp.import_array() @@ -64,9 +92,9 @@ class RampFitOutputs(NamedTuple): @boundscheck(False) @wraparound(False) -def fit_ramps(cnp.ndarray[float, ndim=2] resultants, +def fit_ramps(float[:, :] resultants, cnp.ndarray[int, ndim=2] dq, - cnp.ndarray[float, ndim=1] read_noise, + float[:] read_noise, float read_time, list[list[int]] read_pattern, bool use_jump=False, @@ -82,12 +110,16 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, Parameters ---------- - resultants : np.ndarry[n_resultants, n_pixel] - the resultants in electrons + resultants : float[n_resultants, n_pixel] + the resultants in electrons (Note that this can be based as any sort of + array, such as a numpy array. The memmory view is just for efficiency in + cython) dq : np.ndarry[n_resultants, n_pixel] - the dq array. dq != 0 implies bad pixel / CR. - read_noise : np.ndarray[n_pixel] - the read noise in electrons for each pixel + the dq array. dq != 0 implies bad pixel / CR. (Kept as a numpy array + so that it can be passed out without copying into new numpy array, will + be working on memmory views of this array) + read_noise : float[n_pixel] + the read noise in electrons for each pixel (same note as the resultants) read_time : float Time to perform a readout. For Roman data, this is FRAME_TIME. read_pattern : list[list[int]] @@ -110,38 +142,58 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, n_resultants = resultants.shape[0] n_pixels = resultants.shape[1] + # Raise error if input data is inconsistent if n_resultants != len(read_pattern): raise RuntimeError(f'The read pattern length {len(read_pattern)} does not ' f'match number of resultants {n_resultants}') - # Pre-compute data for all pixels + # Compute the main metadata from the read pattern and cast it to memory views cdef ReadPattern metadata = from_read_pattern(read_pattern, read_time, n_resultants) cdef float[:] t_bar = metadata.t_bar cdef float[:] tau = metadata.tau cdef int[:] n_reads = metadata.n_reads - cdef float[:, :] fixed = np.empty((n_fixed_offsets, n_resultants - 1), dtype=np.float32) - cdef float[:, :] pixel = np.empty((n_pixel_offsets, n_resultants - 1), dtype=np.float32) + # Setup pre-compute arrays for jump detection + cdef float[:, :] fixed + cdef float[:, :] pixel if use_jump: - fixed = fill_fixed_values(fixed, t_bar, tau, n_reads, n_resultants) + # Initialize arrays for the jump detection pre-computed values + fixed = np.empty((n_fixed_offsets, n_resultants - 1), dtype=np.float32) + pixel = np.empty((n_pixel_offsets, n_resultants - 1), dtype=np.float32) + # Pre-compute the values from the read pattern + fixed = fill_fixed_values(fixed, t_bar, tau, n_reads, n_resultants) + else: + # "Initialize" the arrays when not using jump detection, they need to be + # initialized because they do get passed around, but they don't need + # to actually have any entries + fixed = np.empty((0, 0), dtype=np.float32) + pixel = np.empty((0, 0), dtype=np.float32) + + # Create a threshold struct cdef Thresh thresh = Thresh(intercept, constant) + # Create variable to old the diagnostic data # Use list because this might grow very large which would require constant # reallocation. We don't need random access, and this gets cast to a python # list in the end. cdef cpp_list[JumpFits] ramp_fits - # intercept is currently always zero, where as every variance is calculated and set + # Initialize the output arrays. Note that the fit intercept is currently always + # zero, where as every variance is calculated and set. This means that the + # parameters need to be filled with zeros, where as the variances can just + # be allocated cdef float[:, :] parameters = np.zeros((n_pixels, Parameter.n_param), dtype=np.float32) cdef float[:, :] variances = np.empty((n_pixels, Variance.n_var), dtype=np.float32) + # Cast the enum values into integers for indexing (otherwise compiler complains) + # These will be optimized out cdef int slope = Parameter.slope cdef int read_var = Variance.read_var cdef int poisson_var = Variance.poisson_var cdef int total_var = Variance.total_var - # Perform all of the fits + # Run the jump fitting algorithm for each pixel cdef JumpFits fit cdef int index for index in range(n_pixels): @@ -159,14 +211,20 @@ def fit_ramps(cnp.ndarray[float, ndim=2] resultants, use_jump, include_diagnostic) + # Extract the output fit's parameters parameters[index, slope] = fit.average.slope + # Extract the output fit's variances variances[index, read_var] = fit.average.read_var variances[index, poisson_var] = fit.average.poisson_var variances[index, total_var] = fit.average.read_var + fit.average.poisson_var + # Store diagnostic data if requested if include_diagnostic: ramp_fits.push_back(fit) - # return RampFitOutputs(ramp_fits, parameters, variances, dq) - return RampFitOutputs(np.array(parameters, dtype=np.float32), np.array(variances, dtype=np.float32), dq, ramp_fits if include_diagnostic else None) + # Cast memory views into numpy arrays for ease of use in python. + return RampFitOutputs(np.array(parameters, dtype=np.float32), + np.array(variances, dtype=np.float32), + dq, + ramp_fits if include_diagnostic else None) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 6e696d14..4ace423b 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -30,16 +30,16 @@ JumpFits : struct Enums ----- - FixedOffsets : enum - Enumerate the different pieces of information computed for jump detection - which only depend on the read pattern. +FixedOffsets : enum + Enumerate the different pieces of information computed for jump detection + which only depend on the read pattern. - PixelOffsets : enum - Enumerate the different pieces of information computed for jump detection - which only depend on the given pixel (independent of specific ramp). +PixelOffsets : enum + Enumerate the different pieces of information computed for jump detection + which only depend on the given pixel (independent of specific ramp). - JUMP_DET : value - A the fixed value for the jump detection dq flag. +JUMP_DET : value + A the fixed value for the jump detection dq flag. (Public) Functions ------------------ diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index 2f3c9172..bd60e0fc 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -29,10 +29,10 @@ RampQueue : vector[RampIndex] Classes ------- - ReadPattern : - Container class for all the metadata derived from the read pattern, this - is just a temporary object to allow us to return multiple memory views from - a single function. +ReadPattern : + Container class for all the metadata derived from the read pattern, this + is just a temporary object to allow us to return multiple memory views from + a single function. (Public) Functions ------------------ @@ -60,6 +60,7 @@ from libcpp.vector cimport vector from stcal.ramp_fitting.ols_cas22._ramp cimport RampIndex, RampQueue, RampFit, ReadPattern +# Initialize numpy for cython use in this module cnp.import_array() From 3627943b04ded1062136d5adfde77edcb7384d12 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 16:22:13 -0500 Subject: [PATCH 27/30] Rename _fit_ramps module --- setup.py | 12 ++++++------ src/stcal/ramp_fitting/ols_cas22/__init__.py | 2 +- .../ols_cas22/{_fit_ramps.pyx => _fit.pyx} | 0 3 files changed, 7 insertions(+), 7 deletions(-) rename src/stcal/ramp_fitting/ols_cas22/{_fit_ramps.pyx => _fit.pyx} (100%) diff --git a/setup.py b/setup.py index 5ba62d24..ffb85902 100644 --- a/setup.py +++ b/setup.py @@ -8,20 +8,20 @@ extensions = [ Extension( - 'stcal.ramp_fitting.ols_cas22._jump', - ['src/stcal/ramp_fitting/ols_cas22/_jump.pyx'], + 'stcal.ramp_fitting.ols_cas22._ramp', + ['src/stcal/ramp_fitting/ols_cas22/_ramp.pyx'], include_dirs=[np.get_include()], language='c++' ), Extension( - 'stcal.ramp_fitting.ols_cas22._ramp', - ['src/stcal/ramp_fitting/ols_cas22/_ramp.pyx'], + 'stcal.ramp_fitting.ols_cas22._jump', + ['src/stcal/ramp_fitting/ols_cas22/_jump.pyx'], include_dirs=[np.get_include()], language='c++' ), Extension( - 'stcal.ramp_fitting.ols_cas22._fit_ramps', - ['src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx'], + 'stcal.ramp_fitting.ols_cas22._fit', + ['src/stcal/ramp_fitting/ols_cas22/_fit.pyx'], include_dirs=[np.get_include()], language='c++' ), diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/ols_cas22/__init__.py index c46ab4dd..9d0e18e6 100644 --- a/src/stcal/ramp_fitting/ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/ols_cas22/__init__.py @@ -1,4 +1,4 @@ -from ._fit_ramps import fit_ramps, RampFitOutputs, Parameter, Variance +from ._fit import fit_ramps, RampFitOutputs, Parameter, Variance from ._jump import JUMP_DET __all__ = ['fit_ramps', 'RampFitOutputs', 'Parameter', 'Variance', 'Diff', 'JUMP_DET'] diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit.pyx similarity index 100% rename from src/stcal/ramp_fitting/ols_cas22/_fit_ramps.pyx rename to src/stcal/ramp_fitting/ols_cas22/_fit.pyx From 2534b6ede67733f81e9a371b3d7d53a446bbbbe7 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 9 Nov 2023 16:24:28 -0500 Subject: [PATCH 28/30] Use future annotations --- src/stcal/ramp_fitting/ols_cas22/_fit.pyx | 5 +++-- tests/test_jump_cas22.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit.pyx index 5764b139..06098a54 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.pyx @@ -26,6 +26,7 @@ fit_ramps : function for jumps (if use_jump is True) and bad pixels (via the dq array). This is the primary externally callable function. """ +from __future__ import annotations import numpy as np cimport numpy as cnp @@ -42,7 +43,7 @@ from stcal.ramp_fitting.ols_cas22._jump cimport (Thresh, n_pixel_offsets) from stcal.ramp_fitting.ols_cas22._ramp cimport ReadPattern, from_read_pattern -from typing import NamedTuple, Optional +from typing import NamedTuple # Initialize numpy for cython use in this module @@ -87,7 +88,7 @@ class RampFitOutputs(NamedTuple): parameters: np.ndarray variances: np.ndarray dq: np.ndarray - fits: Optional[list] = None + fits: list | None = None @boundscheck(False) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 87a701c9..a6966a02 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,7 +2,10 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22._jump import fill_fixed_values, _fill_pixel_values, FixedOffsets, PixelOffsets +from stcal.ramp_fitting.ols_cas22._jump import (fill_fixed_values, + _fill_pixel_values, + FixedOffsets, + PixelOffsets) from stcal.ramp_fitting.ols_cas22._ramp import from_read_pattern, init_ramps from stcal.ramp_fitting.ols_cas22 import fit_ramps, Parameter, Variance, JUMP_DET From 4e1c872de2b7d13ee24472fbe5b9292f4d17c760 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 10 Nov 2023 10:44:00 -0500 Subject: [PATCH 29/30] Further optimization related to dq memory map --- src/stcal/ramp_fitting/ols_cas22/_fit.pyx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit.pyx index 06098a54..87aaf02d 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.pyx @@ -194,13 +194,17 @@ def fit_ramps(float[:, :] resultants, cdef int poisson_var = Variance.poisson_var cdef int total_var = Variance.total_var + # Pull memory view of dq for speed of access later + # changes to this array will backpropagate to the original numpy array + cdef int[:, :] dq_ = dq + # Run the jump fitting algorithm for each pixel cdef JumpFits fit cdef int index for index in range(n_pixels): # Fit all the ramps for the given pixel fit = fit_jumps(resultants[:, index], - dq[:, index], + dq_[:, index], read_noise[index], t_bar, tau, From 7ec3a6b287bd3067329e5c8d373701bafee2acc5 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 10 Nov 2023 13:06:44 -0500 Subject: [PATCH 30/30] Update changes --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 14bf31e0..03572974 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -15,6 +15,8 @@ ramp_fitting - Moving some CI tests from JWST to STCAL. [#228, spacetelescope/jwst#6080] +- Significantly, improve the performance of the uneven ramp fitting algorithm. [#229] + Changes to API --------------