From 33b3fb3639d9cb88a4edaac49a40386dd56f4685 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Mon, 28 Oct 2024 03:15:14 -0400 Subject: [PATCH 1/6] Adding ramp fit multiprocessing regression test. --- jwst/regtest/test_niriss_image.py | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/jwst/regtest/test_niriss_image.py b/jwst/regtest/test_niriss_image.py index cf07ae16df..7f5a0cf79a 100644 --- a/jwst/regtest/test_niriss_image.py +++ b/jwst/regtest/test_niriss_image.py @@ -9,6 +9,7 @@ from jwst import datamodels from jwst.stpipe import Step from jwst.tweakreg import TweakRegStep +from jwst.ramp_fitting.ramp_fit_step import RampFitStep @pytest.fixture(scope="module") @@ -35,6 +36,30 @@ def run_detector1(rtdata_module): Step.from_cmdline(args) +@pytest.fixture(scope="module") +def run_detector1_multi(rtdata_module): + """Run calwebb_detector1 pipeline on NIRISS imaging data.""" + rtdata = rtdata_module + + rtdata.get_data("niriss/imaging/jw01094001002_02107_00001_nis_uncal.fits") + + # Run detector1 pipeline on an _uncal files + args = ["calwebb_detector1", rtdata.input, + "--steps.persistence.save_trapsfilled=False", + "--steps.dq_init.save_results=True", + "--steps.saturation.save_results=True", + "--steps.superbias.save_results=True", + "--steps.refpix.save_results=True", + "--steps.linearity.save_results=True", + "--steps.dark_current.save_results=True", + "--steps.charge_migration.skip=False", + "--steps.charge_migration.save_results=True", + "--steps.jump.save_results=True", + "--steps.ramp_fit.maximum_cores=2", # Multiprocessing + ] + + Step.from_cmdline(args) + @pytest.fixture(scope="module") def run_detector1_with_clean_flicker_noise(rtdata_module): @@ -67,6 +92,7 @@ def test_niriss_image_detector1(run_detector1, rtdata_module, fitsdiff_default_k _assert_is_same(rtdata_module, fitsdiff_default_kwargs, suffix, truth_dir) + @pytest.mark.bigdata @pytest.mark.parametrize("suffix", ["cfn_clean_flicker_noise", "mask", "flicker_bkg", "flicker_noise", @@ -110,6 +136,17 @@ def test_niriss_tweakreg_no_sources(rtdata, fitsdiff_default_kwargs): result.shelve(model, modify=False) +def test_niriss_image_detector1_multi( + run_detector1_multi, rtdata_module, fitsdiff_default_kwargs): + """ + Runs test_niriss_image_detector1[rate], but with ramp fitting run with multiprocessing + on two processors. + """ + truth_dir = 'test_niriss_image' + # fitsdiff_default_kwargs["numdiffs"] = 10 + _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate", truth_dir) + + def _assert_is_same(rtdata_module, fitsdiff_default_kwargs, suffix, truth_dir): """Assertion helper for the above tests""" rtdata = rtdata_module From 7ed7bfb6cf28a93854422c3cda4d60891b093843 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Mon, 28 Oct 2024 11:37:26 -0400 Subject: [PATCH 2/6] Adding multiprocessing regression testing for ramp fit and jump. --- jwst/regtest/test_miri_image.py | 56 +++++++++++++++++++++++++++++++ jwst/regtest/test_niriss_image.py | 44 ++++++++++++++++++++++-- 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/jwst/regtest/test_miri_image.py b/jwst/regtest/test_miri_image.py index 2d6f46061f..4b79c4b6ab 100644 --- a/jwst/regtest/test_miri_image.py +++ b/jwst/regtest/test_miri_image.py @@ -32,6 +32,52 @@ def run_detector1(rtdata_module): Step.from_cmdline(args) +@pytest.fixture(scope="module") +def run_detector1_multi_rate(rtdata_module): + """Run detector1 pipeline on MIRI imaging data.""" + rtdata = rtdata_module + rtdata.get_data("miri/image/jw01024001001_04101_00001_mirimage_uncal.fits") + + # Run detector1 pipeline only on one of the _uncal files + args = ["jwst.pipeline.Detector1Pipeline", rtdata.input, + "--save_calibrated_ramp=True", + "--steps.dq_init.save_results=True", + "--steps.saturation.save_results=True", + "--steps.firstframe.save_results=True", + "--steps.lastframe.save_results=True", + "--steps.reset.save_results=True", + "--steps.linearity.save_results=True", + "--steps.rscd.save_results=True", + "--steps.dark_current.save_results=True", + "--steps.refpix.save_results=True", + "--steps.ramp_fit.maximum_cores=2", # Multiprocessing + ] + Step.from_cmdline(args) + + +@pytest.fixture(scope="module") +def run_detector1_multi_jump(rtdata_module): + """Run detector1 pipeline on MIRI imaging data.""" + rtdata = rtdata_module + rtdata.get_data("miri/image/jw01024001001_04101_00001_mirimage_uncal.fits") + + # Run detector1 pipeline only on one of the _uncal files + args = ["jwst.pipeline.Detector1Pipeline", rtdata.input, + "--save_calibrated_ramp=True", + "--steps.dq_init.save_results=True", + "--steps.saturation.save_results=True", + "--steps.firstframe.save_results=True", + "--steps.lastframe.save_results=True", + "--steps.reset.save_results=True", + "--steps.linearity.save_results=True", + "--steps.rscd.save_results=True", + "--steps.dark_current.save_results=True", + "--steps.refpix.save_results=True", + "--steps.jump.maximum_cores=2", # Multiprocessing + ] + Step.from_cmdline(args) + + @pytest.fixture(scope="module") def run_detector1_with_average_dark_current(rtdata_module): """Run detector1 pipeline on MIRI imaging data, providing an @@ -115,6 +161,16 @@ def test_miri_image_detector1(run_detector1, rtdata_module, fitsdiff_default_kwa _assert_is_same(rtdata_module, fitsdiff_default_kwargs, suffix) +def test_miri_image_detector1_multi_rate(run_detector1_multi_rate, rtdata_module, fitsdiff_default_kwargs): + """Regression test of detector1 pipeline performed on MIRI imaging data.""" + _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate") + + +def test_miri_image_detector1_multi_jump(run_detector1_multi_jump, rtdata_module, fitsdiff_default_kwargs): + """Regression test of detector1 pipeline performed on MIRI imaging data.""" + _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate") + + @pytest.mark.bigdata def test_detector1_mem_usage(rtdata_module): """Determine the memory usage for Detector 1""" diff --git a/jwst/regtest/test_niriss_image.py b/jwst/regtest/test_niriss_image.py index 7f5a0cf79a..7ccca052bd 100644 --- a/jwst/regtest/test_niriss_image.py +++ b/jwst/regtest/test_niriss_image.py @@ -36,8 +36,9 @@ def run_detector1(rtdata_module): Step.from_cmdline(args) + @pytest.fixture(scope="module") -def run_detector1_multi(rtdata_module): +def run_detector1_multi_rate(rtdata_module): """Run calwebb_detector1 pipeline on NIRISS imaging data.""" rtdata = rtdata_module @@ -61,6 +62,32 @@ def run_detector1_multi(rtdata_module): Step.from_cmdline(args) +@pytest.fixture(scope="module") +def run_detector1_multi_jump(rtdata_module): + """Run calwebb_detector1 pipeline on NIRISS imaging data.""" + rtdata = rtdata_module + + rtdata.get_data("niriss/imaging/jw01094001002_02107_00001_nis_uncal.fits") + + # Run detector1 pipeline on an _uncal files + args = ["calwebb_detector1", rtdata.input, + "--steps.persistence.save_trapsfilled=False", + "--steps.dq_init.save_results=True", + "--steps.saturation.save_results=True", + "--steps.superbias.save_results=True", + "--steps.refpix.save_results=True", + "--steps.linearity.save_results=True", + "--steps.dark_current.save_results=True", + "--steps.charge_migration.skip=False", + "--steps.charge_migration.save_results=True", + "--steps.jump.save_results=True", + "--steps.jump.maximum_cores=2", # Multiprocessing + "--steps.ramp_fit.skip=True", + ] + + Step.from_cmdline(args) + + @pytest.fixture(scope="module") def run_detector1_with_clean_flicker_noise(rtdata_module): """Run detector1 pipeline on NIRISS imaging data with noise cleaning.""" @@ -136,8 +163,8 @@ def test_niriss_tweakreg_no_sources(rtdata, fitsdiff_default_kwargs): result.shelve(model, modify=False) -def test_niriss_image_detector1_multi( - run_detector1_multi, rtdata_module, fitsdiff_default_kwargs): +def test_niriss_image_detector1_multi_rate( + run_detector1_multi_rate, rtdata_module, fitsdiff_default_kwargs): """ Runs test_niriss_image_detector1[rate], but with ramp fitting run with multiprocessing on two processors. @@ -147,6 +174,17 @@ def test_niriss_image_detector1_multi( _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate", truth_dir) +def test_niriss_image_detector1_multi_jump( + run_detector1_multi_jump, rtdata_module, fitsdiff_default_kwargs): + """ + Runs test_niriss_image_detector1[rate], but with ramp fitting run with multiprocessing + on two processors. + """ + truth_dir = 'test_niriss_image' + # fitsdiff_default_kwargs["numdiffs"] = 10 + _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "jump", truth_dir) + + def _assert_is_same(rtdata_module, fitsdiff_default_kwargs, suffix, truth_dir): """Assertion helper for the above tests""" rtdata = rtdata_module From 08e5449140daf93e85211b196879f8017b442fa4 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 7 Nov 2024 14:00:13 -0500 Subject: [PATCH 3/6] Changing names to make it more clear the tests are for multiprocessing. --- jwst/regtest/test_miri_image.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwst/regtest/test_miri_image.py b/jwst/regtest/test_miri_image.py index 4b79c4b6ab..12d8c0b326 100644 --- a/jwst/regtest/test_miri_image.py +++ b/jwst/regtest/test_miri_image.py @@ -33,7 +33,7 @@ def run_detector1(rtdata_module): @pytest.fixture(scope="module") -def run_detector1_multi_rate(rtdata_module): +def run_detector1_multiprocess_rate(rtdata_module): """Run detector1 pipeline on MIRI imaging data.""" rtdata = rtdata_module rtdata.get_data("miri/image/jw01024001001_04101_00001_mirimage_uncal.fits") @@ -56,7 +56,7 @@ def run_detector1_multi_rate(rtdata_module): @pytest.fixture(scope="module") -def run_detector1_multi_jump(rtdata_module): +def run_detector1_multiprocess_jump(rtdata_module): """Run detector1 pipeline on MIRI imaging data.""" rtdata = rtdata_module rtdata.get_data("miri/image/jw01024001001_04101_00001_mirimage_uncal.fits") @@ -161,12 +161,12 @@ def test_miri_image_detector1(run_detector1, rtdata_module, fitsdiff_default_kwa _assert_is_same(rtdata_module, fitsdiff_default_kwargs, suffix) -def test_miri_image_detector1_multi_rate(run_detector1_multi_rate, rtdata_module, fitsdiff_default_kwargs): +def test_miri_image_detector1_multiprocess_rate(run_detector1_multiprocess_rate, rtdata_module, fitsdiff_default_kwargs): """Regression test of detector1 pipeline performed on MIRI imaging data.""" _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate") -def test_miri_image_detector1_multi_jump(run_detector1_multi_jump, rtdata_module, fitsdiff_default_kwargs): +def test_miri_image_detector1_multiprocess_jump(run_detector1_multiprocess_jump, rtdata_module, fitsdiff_default_kwargs): """Regression test of detector1 pipeline performed on MIRI imaging data.""" _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate") From 1b1fde674c9a61b659c96b551828d6868864ee2e Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Thu, 7 Nov 2024 14:01:07 -0500 Subject: [PATCH 4/6] Changing names to clarify the tests are for multiprocessing. --- jwst/regtest/test_niriss_image.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jwst/regtest/test_niriss_image.py b/jwst/regtest/test_niriss_image.py index 7ccca052bd..f2d2c1e629 100644 --- a/jwst/regtest/test_niriss_image.py +++ b/jwst/regtest/test_niriss_image.py @@ -38,7 +38,7 @@ def run_detector1(rtdata_module): @pytest.fixture(scope="module") -def run_detector1_multi_rate(rtdata_module): +def run_detector1_multiprocess_rate(rtdata_module): """Run calwebb_detector1 pipeline on NIRISS imaging data.""" rtdata = rtdata_module @@ -63,7 +63,7 @@ def run_detector1_multi_rate(rtdata_module): @pytest.fixture(scope="module") -def run_detector1_multi_jump(rtdata_module): +def run_detector1_multiprocess_jump(rtdata_module): """Run calwebb_detector1 pipeline on NIRISS imaging data.""" rtdata = rtdata_module @@ -163,8 +163,8 @@ def test_niriss_tweakreg_no_sources(rtdata, fitsdiff_default_kwargs): result.shelve(model, modify=False) -def test_niriss_image_detector1_multi_rate( - run_detector1_multi_rate, rtdata_module, fitsdiff_default_kwargs): +def test_niriss_image_detector1_multiprocess_rate( + run_detector1_multiprocess_rate, rtdata_module, fitsdiff_default_kwargs): """ Runs test_niriss_image_detector1[rate], but with ramp fitting run with multiprocessing on two processors. @@ -174,8 +174,8 @@ def test_niriss_image_detector1_multi_rate( _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate", truth_dir) -def test_niriss_image_detector1_multi_jump( - run_detector1_multi_jump, rtdata_module, fitsdiff_default_kwargs): +def test_niriss_image_detector1_multiprocess_jump( + run_detector1_multiprocess_jump, rtdata_module, fitsdiff_default_kwargs): """ Runs test_niriss_image_detector1[rate], but with ramp fitting run with multiprocessing on two processors. From 48c88f14da9804b5b4c0f10a9a74366bade71647 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Tue, 12 Nov 2024 23:04:41 -0500 Subject: [PATCH 5/6] Adding ramp fit multiprocessing test with the optional results product. --- jwst/regtest/test_niriss_image.py | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/jwst/regtest/test_niriss_image.py b/jwst/regtest/test_niriss_image.py index f2d2c1e629..23ccf8ff9d 100644 --- a/jwst/regtest/test_niriss_image.py +++ b/jwst/regtest/test_niriss_image.py @@ -62,6 +62,33 @@ def run_detector1_multiprocess_rate(rtdata_module): Step.from_cmdline(args) +@pytest.fixture(scope="module") +def run_detector1_multiprocess_rate_save_opt(rtdata_module): + """Run calwebb_detector1 pipeline on NIRISS imaging data.""" + rtdata = rtdata_module + + rtdata.get_data("niriss/imaging/jw01094001002_02107_00001_nis_uncal.fits") + + # Run detector1 pipeline on an _uncal files + args = ["calwebb_detector1", rtdata.input, + "--steps.persistence.save_trapsfilled=False", + "--steps.dq_init.save_results=True", + "--steps.saturation.save_results=True", + "--steps.superbias.save_results=True", + "--steps.refpix.save_results=True", + "--steps.linearity.save_results=True", + "--steps.dark_current.save_results=True", + "--steps.charge_migration.skip=False", + "--steps.charge_migration.save_results=True", + "--steps.jump.save_results=True", + "--steps.ramp_fit.maximum_cores=2", # Multiprocessing + "--steps.ramp_fit.save_opt=True", + "--steps.ramp_fit.opt_name=jw01094001002_02107_00001_nis.fits", + ] + + Step.from_cmdline(args) + + @pytest.fixture(scope="module") def run_detector1_multiprocess_jump(rtdata_module): """Run calwebb_detector1 pipeline on NIRISS imaging data.""" @@ -163,6 +190,7 @@ def test_niriss_tweakreg_no_sources(rtdata, fitsdiff_default_kwargs): result.shelve(model, modify=False) +@pytest.mark.bigdata def test_niriss_image_detector1_multiprocess_rate( run_detector1_multiprocess_rate, rtdata_module, fitsdiff_default_kwargs): """ @@ -174,6 +202,20 @@ def test_niriss_image_detector1_multiprocess_rate( _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate", truth_dir) +@pytest.mark.bigdata +@pytest.mark.parametrize("suffix", ["rate", "fitopt"]) +def test_niriss_image_detector1_multiprocess_rate_save_opt( + run_detector1_multiprocess_rate_save_opt, rtdata_module, fitsdiff_default_kwargs, suffix): + """ + Runs test_niriss_image_detector1[rate], but with ramp fitting run with multiprocessing + on two processors and the optional results product. + """ + truth_dir = 'test_niriss_image' + # fitsdiff_default_kwargs["numdiffs"] = 10 + _assert_is_same(rtdata_module, fitsdiff_default_kwargs, suffix, truth_dir) + + +@pytest.mark.bigdata def test_niriss_image_detector1_multiprocess_jump( run_detector1_multiprocess_jump, rtdata_module, fitsdiff_default_kwargs): """ From 7578fd67f0620dd82f41c81cc95d6574a273f2de Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Fri, 15 Nov 2024 20:09:55 -0500 Subject: [PATCH 6/6] Adding bigdata decorator to new regression test. --- jwst/regtest/test_miri_image.py | 2 ++ jwst/regtest/test_niriss_image.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/jwst/regtest/test_miri_image.py b/jwst/regtest/test_miri_image.py index 12d8c0b326..b6863eccec 100644 --- a/jwst/regtest/test_miri_image.py +++ b/jwst/regtest/test_miri_image.py @@ -161,11 +161,13 @@ def test_miri_image_detector1(run_detector1, rtdata_module, fitsdiff_default_kwa _assert_is_same(rtdata_module, fitsdiff_default_kwargs, suffix) +@pytest.mark.bigdata def test_miri_image_detector1_multiprocess_rate(run_detector1_multiprocess_rate, rtdata_module, fitsdiff_default_kwargs): """Regression test of detector1 pipeline performed on MIRI imaging data.""" _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate") +@pytest.mark.bigdata def test_miri_image_detector1_multiprocess_jump(run_detector1_multiprocess_jump, rtdata_module, fitsdiff_default_kwargs): """Regression test of detector1 pipeline performed on MIRI imaging data.""" _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate") diff --git a/jwst/regtest/test_niriss_image.py b/jwst/regtest/test_niriss_image.py index 23ccf8ff9d..be31fbda67 100644 --- a/jwst/regtest/test_niriss_image.py +++ b/jwst/regtest/test_niriss_image.py @@ -9,7 +9,6 @@ from jwst import datamodels from jwst.stpipe import Step from jwst.tweakreg import TweakRegStep -from jwst.ramp_fitting.ramp_fit_step import RampFitStep @pytest.fixture(scope="module")