JP-2581: allow multiprocessing for jump detection

CHANGES.rst

@@ -24,6 +24,10 @@ extract_1d
 
 - Exit gracefully if data is with F277W filter; avoid masking entire wavemap
   if subarray is SUBSTRIP96 [#6840]
+
+jump
+----
+- Enable multiprocessing in jump detection [#6845]
 
 linearity
 ---------

jwst/jump/jump.py

@@ -37,13 +37,10 @@ def run_detect_jumps(input_model, gain_model, readnoise_model,
         readnoise_2d = reffile_utils.get_subarray_data(input_model,
                                                        readnoise_model)
 
-    # turn off MP for now
-    # max_cores = None
-
     new_gdq, new_pdq = detect_jumps(frames_per_group, data, gdq, pdq, err,
                                     gain_2d, readnoise_2d,
                                     rejection_thresh, three_grp_thresh,
-                                    four_grp_thresh,
+                                    four_grp_thresh, max_cores,
                                     max_jump_to_flag_neighbors,
                                     min_jump_to_flag_neighbors,
                                     flag_4_neighbors, dqflags.pixel)

jwst/jump/tests/test_detect_jumps.py

@@ -31,7 +31,7 @@ def test_exec_time_0_crs(setup_inputs):
 
     tstart = time.time()
     # using dummy variable in next to prevent "F841-variable is assigned to but never used"
-    _ = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
+    _ = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, True)
     tstop = time.time()
 
     t_elapsed = tstop - tstart
@@ -59,7 +59,7 @@ def test_exec_time_many_crs(setup_inputs):
 
     tstart = time.time()
     # using dummy variable in next to prevent "F841-variable is assigned to but never used"
-    _ = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
+    _ = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, True)
     tstop = time.time()
 
     t_elapsed = tstop - tstart
@@ -73,7 +73,7 @@ def test_nocrs_noflux(setup_inputs):
     All pixel values are zero. So slope should be zero
     """
     model, rnoise, gain = setup_inputs(ngroups=5)
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, True)
     assert np.max(out_model.groupdq) == GOOD
 
 
@@ -85,7 +85,7 @@ def test_nocrs_noflux_badgain_pixel(setup_inputs):
     model, rnoise, gain = setup_inputs(ngroups=5, nrows=20, ncols=20)
     gain.data[7, 7] = -10  # bad gain
     gain.data[17, 17] = np.nan  # bad gain
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, True)
 
     # 2 bits are set for each pixel, so use bitwise_and to check is set
     assert np.bitwise_and(out_model.pixeldq[7, 7], NO_GAIN_VALUE) == NO_GAIN_VALUE
@@ -100,7 +100,7 @@ def test_nocrs_noflux_subarray(setup_inputs):
     extracted from the full frame versions.
     """
     model, rnoise, gain = setup_inputs(ngroups=5, subarray=True)
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, True)
     assert np.max(out_model.groupdq) == GOOD
 
 
@@ -126,7 +126,7 @@ def test_onecr_10_groups_neighbors_flagged(setup_inputs):
     model.data[0, 7, 5, 5] = 160.0
     model.data[0, 8, 5, 5] = 170.0
     model.data[0, 9, 5, 5] = 180.0
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, True)
 
     assert np.max(out_model.groupdq[0, 5, 5, 5]) == JUMP_DET
     assert out_model.groupdq[0, 5, 5, 6] == JUMP_DET
@@ -161,7 +161,7 @@ def test_nocr_100_groups_nframes1(setup_inputs):
     model.data[0, 9, 5, 5] = 68.0
     for i in range(10, 100):
         model.data[0, i, 5, 5] = i * 5
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, True)
     assert np.max(out_model.groupdq) == GOOD
 
 
@@ -198,7 +198,7 @@ def test_twoints_onecr_each_10_groups_neighbors_flagged(setup_inputs):
     model.data[1, 7, 15, 5] = 160.0
     model.data[1, 8, 15, 5] = 170.0
     model.data[1, 9, 15, 5] = 180.0
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, True)
 
     assert np.max(out_model.groupdq[0, 5, 5, 5]) == JUMP_DET
     assert out_model.groupdq[0, 5, 5, 6] == JUMP_DET
@@ -276,7 +276,7 @@ def test_multiple_neighbor_jumps_firstlastbad(setup_inputs):
     # run jump detection
     out_model = run_detect_jumps(model, gain, rnoise, rejection_thresh=200.0, three_grp_thresh=200,
                                  four_grp_thresh=200,
-                                 max_cores=None, max_jump_to_flag_neighbors=200,
+                                 max_cores='none', max_jump_to_flag_neighbors=200,
                                  min_jump_to_flag_neighbors=10, flag_4_neighbors=True)
 
     # Verify that the correct groups have been flagged. The entries for pixels
@@ -334,7 +334,7 @@ def test_nirspec_saturated_pix(setup_inputs):
     # run jump detection
     out_model = run_detect_jumps(model, gain, rnoise, rejection_thresh=200.0, three_grp_thresh=200,
                                  four_grp_thresh=200,
-                                 max_cores=None, max_jump_to_flag_neighbors=200,
+                                 max_cores='none', max_jump_to_flag_neighbors=200,
                                  min_jump_to_flag_neighbors=10, flag_4_neighbors=True)
 
     # Check the results. There should not be any pixels with DQ values of 6, which
@@ -374,7 +374,6 @@ def add_crs(model, crs_frac):
     return model
 
 
-@pytest.mark.skip(reason='multiprocessing temporarily disabled')
 def test_flagging_of_CRs_across_slice_boundaries(setup_inputs):
     """"
     A multiprocessing test that has two CRs on the boundary between two slices.
@@ -432,7 +431,6 @@ def test_flagging_of_CRs_across_slice_boundaries(setup_inputs):
         assert out_model.groupdq[1, 7, yincrement - 1, 25] == JUMP_DET
 
 
-@pytest.mark.skip(reason='multiprocessing temporarily disabled')
 def test_twoints_onecr_10_groups_neighbors_flagged_multi(setup_inputs):
     """"
     A multiprocessing test that has two CRs on the boundary between two slices
@@ -574,7 +572,7 @@ def test_crs_on_edge_with_neighbor_flagging(setup_inputs):
     model.data[0, 8, 15, -1] = 170.0
     model.data[0, 9, 15, -1] = 180.0
 
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 10, True)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 10, True)
 
     # flag CR and three neighbors of first row CR
     assert out_model.groupdq[0, 5, 0, 15] == JUMP_DET
@@ -621,7 +619,7 @@ def test_onecr_10_groups(setup_inputs):
     model.data[0, 7, 5, 5] = 160.0
     model.data[0, 8, 5, 5] = 170.0
     model.data[0, 9, 5, 5] = 180.0
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 10, False)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 10, False)
 
     assert out_model.groupdq[0, 5, 5, 5] == JUMP_DET
     assert out_model.groupdq[0, 5, 4, 5] == GOOD
@@ -660,7 +658,7 @@ def test_onecr_10_groups_fullarray(setup_inputs):
     model.data[0, 7, 5, 10] = 400
     model.data[0, 8, 5, 10] = 410
     model.data[0, 9, 5, 10] = 420
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 10, False)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 10, False)
 
     # The jump is in group 5 for columns 0-9
     assert_array_equal(out_model.groupdq[0, 5, 5, 0:10], JUMP_DET)
@@ -694,7 +692,7 @@ def test_onecr_50_groups(setup_inputs):
     model.data[0, 9, 5, 5] = 180.0
     model.data[0, 10:30, 5, 5] = np.arange(190, 290, 5)
     model.data[0, 30:50, 5, 5] = np.arange(500, 600, 5)
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 10, False)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 10, False)
 
     # CR in group 5
     assert out_model.groupdq[0, 5, 5, 5] == JUMP_DET
@@ -732,7 +730,7 @@ def test_single_CR_neighbor_flag(setup_inputs):
     model.data[0, 9, 3, 3] = 180.0
 
     # Flag neighbors
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, True)
 
     assert np.max(out_model.groupdq[0, 5, 3, 3]) == JUMP_DET
     assert out_model.groupdq[0, 5, 3, 4] == JUMP_DET
@@ -741,7 +739,7 @@ def test_single_CR_neighbor_flag(setup_inputs):
     assert out_model.groupdq[0, 5, 4, 3] == JUMP_DET
 
     # Do not flag neighbors
-    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, False)
+    out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, False)
 
     assert np.max(out_model.groupdq[0, 5, 3, 3]) == JUMP_DET
     assert out_model.groupdq[0, 5, 3, 4] == GOOD
@@ -750,7 +748,6 @@ def test_single_CR_neighbor_flag(setup_inputs):
     assert out_model.groupdq[0, 5, 4, 3] == GOOD
 
 
-@pytest.mark.skip(reason='multiprocessing temporarily disabled')
 def test_proc(setup_inputs):
     """"
     A single CR in a 10 group exposure. Verify that the pixels flagged using
@@ -777,7 +774,7 @@ def test_proc(setup_inputs):
     model.data[0, 8, 2, 3] = 170.0
     model.data[0, 9, 2, 3] = 180.0
 
-    out_model_a = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, None, 200, 4, True)
+    out_model_a = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4, True)
     out_model_b = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'half', 200, 4, True)
     assert_array_equal(out_model_a.groupdq, out_model_b.groupdq)
 

requirements-sdp.txt

@@ -76,7 +76,7 @@ scipy==1.8.0
 semantic-version==2.9.0
 six==1.16.0
 spherical-geometry==1.2.22
-stcal==0.7.0
+stcal==0.7.1
 stdatamodels==0.4.2
 stpipe==0.3.3
 stsci.image==2.3.5

setup.cfg

@@ -43,7 +43,7 @@ install_requires =
     scikit-image>=0.17.2
     scipy>=1.6.0
     spherical-geometry>=1.2.22
-    stcal>=0.7
+    stcal>=0.7.1
     stdatamodels>=0.4.2,<1.0
     stpipe>=0.3.3,<1.0
     stsci.image>=2.3.5