Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JP-3771 and JP-3791: Ramp Fitting Multiprocessing Regression Testing and Read Noise Variance #313

Closed
wants to merge 4 commits into from
Closed

JP-3771 and JP-3791: Ramp Fitting Multiprocessing Regression Testing and Read Noise Variance #313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b7eedf4
Adding 'orig_gdq' handling for multiprocessing.
kmacdonald-stsci Oct 24, 2024
130c63f
Debugging the read noise variance recalculation.
kmacdonald-stsci Oct 24, 2024
2c25e58
Updating the ramp data slice to carry through the CHARGELOSS flag.
kmacdonald-stsci Oct 26, 2024
20388ec
Updating slope_fitter for CHARGELOSS handling during multiprocessing.
kmacdonald-stsci Oct 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/stcal/ramp_fitting/ols_fit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,11 +164,13 @@
ramp_data, buffsize, save_opt, readnoise_2d, gain_2d, weighting, number_slices
)

# warnings.filterwarnings("ignore", ".*leaked semaphore objects.*", UserWarning)
ctx = multiprocessing.get_context("forkserver")
pool = ctx.Pool(processes=number_slices)
pool_results = pool.starmap(ols_ramp_fit_single, slices)
pool.close()
pool.join()
# warnings.resetwarnings()

# Reassemble results
image_info, integ_info, opt_info = assemble_pool_results(
Expand Down Expand Up @@ -564,12 +566,20 @@
drop_frames1=ramp_data.drop_frames1,
)

# For possible CHARGELOSS flagging.
if ramp_data.orig_gdq is not None:
ogdq = ramp_data.orig_gdq[:, :, start_row : start_row + nrows, :].copy()
ramp_data_slice.orig_gdq = ogdq

Check warning on line 572 in src/stcal/ramp_fitting/ols_fit.py

View check run for this annotation

Codecov / codecov/patch

src/stcal/ramp_fitting/ols_fit.py#L571-L572 

Added lines #L571 - L572 were not covered by tests
else:
ramp_data_slice.orig_gdq = None

# Carry over DQ flags.
ramp_data_slice.flags_do_not_use = ramp_data.flags_do_not_use
ramp_data_slice.flags_jump_det = ramp_data.flags_jump_det
ramp_data_slice.flags_saturated = ramp_data.flags_saturated
ramp_data_slice.flags_no_gain_val = ramp_data.flags_no_gain_val
ramp_data_slice.flags_unreliable_slope = ramp_data.flags_unreliable_slope
ramp_data_slice.flags_chargeloss = ramp_data.flags_chargeloss

# Slice info
ramp_data_slice.start_row = start_row
Expand Down
44 changes: 35 additions & 9 deletions src/stcal/ramp_fitting/src/slope_fitter.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,10 @@ struct ramp_data {
real_t one_group_time; /* Time for ramps with only 0th good group */
weight_t weight; /* The weighting for OLS */

/* Multiprocessing Slice Data */
int start_row; /* Slice starts at this row in the unsliced data */
int num_rows; /* The number of rows in this slice */

/* Debug switch */
int debug;
}; /* END: struct ramp_data */
Expand Down Expand Up @@ -738,22 +742,25 @@ print_delim_char(char c, int len) {

/* Used for debugging to determine if a pixel is in a list */
static inline int
is_pix_in_list(struct pixel_ramp * pr)
is_pix_in_list(struct ramp_data * rd, struct pixel_ramp * pr)
{
/* Pixel list */
// JP-3669 - (1804, 173)
// JP-3771 - (5, 1445)
const int len = 1;
npy_intp rows[len];
npy_intp cols[len];
npy_intp row, col;
int k;

return 0; /* XXX Null function */
// return 0; /* XXX Null function */

rows[0] = 1804;
cols[0] = 173;
// TODO put handling in here for slicing.
rows[0] = 5;
cols[0] = 1445;

for (k=0; k<len; ++k) {
if (pr->row==rows[k] && pr->col==cols[k]) {
row = pr->row + rd->start_row;
if (row==rows[k] && pr->col==cols[k]) {
return 1;
}
}
Expand Down Expand Up @@ -1869,6 +1876,23 @@ get_ramp_data_meta(
}
Py_XDECREF(test);

test = PyObject_GetAttrString(Py_ramp_data, "start_row");
if (!test|| (test == Py_None)) {
rd->start_row = 0;
} else {
rd->start_row = py_ramp_data_get_int(Py_ramp_data, "start_row");
}
Py_XDECREF(test);

// int num_rows; /* The number of rows in this slice */
test = PyObject_GetAttrString(Py_ramp_data, "num_rows");
if (!test|| (test == Py_None)) {
rd->num_rows = 0;
} else {
rd->num_rows = py_ramp_data_get_int(Py_ramp_data, "num_rows");
}
Py_XDECREF(test);

rd->invalid = rd->dnu | rd->sat;

/* Debugging switch */
Expand Down Expand Up @@ -2271,12 +2295,12 @@ ols_slope_fit_pixels(
struct rateint_product * rateint_prod) /* The rateints product */
{
npy_intp row, col;
pid_t pid = getpid(); // XXX Debug

for (row = 0; row < rd->nrows; ++row) {
for (col = 0; col < rd->ncols; ++col) {

// dbg_ols_print("Running (%ld, %ld)\r", row, col);

get_pixel_ramp(pr, rd, row, col);

/* Compute ramp fitting */
Expand Down Expand Up @@ -2631,9 +2655,10 @@ ramp_fit_pixel_rnoise_chargeloss(
/* Clean segment list */
clean_segment_list_basic(&segs);
}

if (!is_chargeloss) {
/* No CHARGELOSS flag in pixel */
return 0;
goto END;
}

/* Capture recomputed exposure level read noise variance */
Expand Down Expand Up @@ -2703,11 +2728,12 @@ ramp_fit_pixel_rnoise_chargeloss_remove(

for (group=0; group<pr->ngroups; ++group) {
idx = get_ramp_index(rd, integ, group);

if (rd->chargeloss & pr->orig_gdq[idx]) {
/* It is assumed that DO_NOT_USE also needs to be removed */
pr->orig_gdq[idx] ^= dnu_chg;
}
}
} /* for group */
}

/*
Expand Down
Loading