JP-2645 - Snowball and shower flagging in Jump

CHANGES.rst

@@ -1,7 +1,11 @@
 1.1.1 (unreleased)
 ==================
 
--
+jump
+----
+
+- Changes to flag both NIR snowballs and MIRI showers
+for  JP-#2645. [#118]
 
 1.1.0 (2022-08-17)
 ==================

setup.cfg

@@ -28,6 +28,7 @@ install_requires =
     numpy>=1.17
     astropy>=5.0.4
     scipy>=1.6.0
+    opencv-python>=4.6.0.66
 
 package_dir =
     =src

src/stcal/jump/jump.py

@@ -1,11 +1,11 @@
 import time
 import logging
 import numpy as np
-
+import cv2 as cv
 from . import twopoint_difference as twopt
 from . import constants
 
-import multiprocessing  # 05/18/21: TODO- commented out now; reinstate later
+import multiprocessing
 
 log = logging.getLogger(__name__)
 log.setLevel(logging.DEBUG)
@@ -18,7 +18,13 @@ def detect_jumps(frames_per_group, data, gdq, pdq, err,
                  after_jump_flag_dn1=0.0,
                  after_jump_flag_n1=0,
                  after_jump_flag_dn2=0.0,
-                 after_jump_flag_n2=0):
+                 after_jump_flag_n2=0,
+                 min_sat_area=1,
+                 min_jump_area=5,
+                 expand_factor=2.0,
+                 use_ellipses=False,
+                 sat_required_snowball=True,
+                 expand_large_events=True):
     """
     This is the high-level controlling routine for the jump detection process.
     It loads and sets the various input data and parameters needed by each of
@@ -108,6 +114,30 @@ def detect_jumps(frames_per_group, data, gdq, pdq, err,
         Gives the number of groups to flag after jumps with DN values above that
         given by after_jump_flag_dn2
 
+    min_sat_area : float
+        The minimum area of saturated pixels at the center of a snowball. Only contours
+        with area above the minimum will create snowballs.
+
+    min_jump_area : float
+        The minimum contour area to trigger the creation of enclosing ellipses or circles.
+
+    expand_factor : float
+        The factor that is used to increase the size of the enclosing circle/ellipse jump
+        flagged pixels.
+
+    use_ellipses : bool
+        If true the minimum enclosing ellipse will be created for jump regions that meet the area
+        requirement. This is best for MIRI showers which are only rarely circular. For the NIR detectors
+        this should set to False to force circles to be used.
+
+    sat_required_snowball : bool
+        If true there must be a saturation circle within the radius of the jump circle to trigger
+        the creation of a snowball. All true snowballs appear to have at least one saturated pixel.
+
+    expand_large_events : bool
+        When True this triggers the flagging of snowballs and showers for NIR and MIRI detectors. If
+        set to False nether type of extended flagging occurrs.
+
     Returns
     -------
     gdq : int, 4D array
@@ -117,7 +147,8 @@ def detect_jumps(frames_per_group, data, gdq, pdq, err,
         updated pixel dq array
     """
     constants.update_dqflags(dqflags)  # populate dq flags
-
+    sat_flag = dqflags["SATURATED"]
+    jump_flag = dqflags["JUMP_DET"]
     # Flag the pixeldq where the gain is <=0 or NaN so they will be ignored
     wh_g = np.where(gain_2d <= 0.)
     if len(wh_g[0] > 0):
@@ -176,7 +207,13 @@ def detect_jumps(frames_per_group, data, gdq, pdq, err,
                            after_jump_flag_e2=after_jump_flag_e2,
                            after_jump_flag_n2=after_jump_flag_n2)
 
-        elapsed = time.time() - start
+        #  This is the flag that controls the flagging of either snowballs or showers.
+        if expand_large_events:
+            flag_large_events(gdq, jump_flag, sat_flag, min_sat_area=min_sat_area,
+                              min_jump_area=min_jump_area,
+                              expand_factor=expand_factor, use_ellipses=use_ellipses,
+                              sat_required_snowball=sat_required_snowball)
+
     else:
         yinc = int(n_rows / n_slices)
         slices = []
@@ -249,7 +286,13 @@ def detect_jumps(frames_per_group, data, gdq, pdq, err,
             # save the neighbors to be flagged that will be in the next slice
             previous_row_above_gdq = row_above_gdq.copy()
             k += 1
-        elapsed = time.time() - start
+
+        #  This is the flag that controls the flagging of either snowballs or showers.
+        if expand_large_events:
+            flag_large_events(gdq, jump_flag, sat_flag, min_sat_area=min_sat_area,
+                              min_jump_area=min_jump_area,
+                              expand_factor=expand_factor, use_ellipses=use_ellipses,
+                              sat_required_snowball=sat_required_snowball)
 
     elapsed = time.time() - start
     log.info('Total elapsed time = %g sec' % elapsed)
@@ -262,3 +305,151 @@ def detect_jumps(frames_per_group, data, gdq, pdq, err,
 
     # Return the updated data quality arrays
     return gdq, pdq
+
+
+def flag_large_events(gdq, jump_flag, sat_flag, min_sat_area=1,
+                      min_jump_area=6,
+                      expand_factor=1.9, use_ellipses=False,
+                      sat_required_snowball=True):
+    """
+    This routine controls the creation of expanded regions that are flagged as jumps. These are called
+    snowballs for the NIR and are almost always circular with a saturated core. For MIRI they are better
+    fit with ellipses.
+
+    :param gdq: The group DQ cube for all integrations
+    :param jump_flag: The bit value that represents jump
+    :param sat_flag:  The bit value that represents saturation
+    :param min_sat_area:  The minimum area of saturated pixels within the jump circle to trigger the
+                        creation of a snowball.
+    :param min_jump_area:  The minimum area of a contour to cause the creation of the minimum
+                        enclosing ellipse or circle
+    :param expand_factor: The factor that increases the size of the snowball or enclosing ellipse.
+    :param use_ellipses:  Use ellipses rather than circles (better for MIRI)
+    :param sat_required_snowball: Require that there is a saturated pixel within the radius of the jump
+                        circle to trigger the formation of a snowball.
+    :return: none
+    """
+
+    log.info('Flagging large events (snowballs, showers).')
+
+    n_showers_grp = []
+    n_showers_grp_ellipse = []
+    for integration in range(gdq.shape[0]):
+        for group in range(1, gdq.shape[1]):
+            if use_ellipses:
+                jump_ellipses = find_ellipses(gdq[integration, group, :, :], jump_flag, min_jump_area)
+                n_showers_grp_ellipse.append(len(jump_ellipses))
+                gdq[integration, group, :, :], num_events = \
+                    extend_ellipses(gdq[integration, group, :, :],
+                                    jump_ellipses, sat_flag, jump_flag,
+                                    expansion=expand_factor)
+            else:
+                #  this line needs to be changed later to just look at saturation flags.
+                new_flagged_pixels = gdq[integration, group, :, :] - gdq[integration, group - 1, :, :]
+
+                sat_circles = find_circles(new_flagged_pixels, sat_flag, min_sat_area)
+                sat_pixels = np.bitwise_and(new_flagged_pixels, sat_flag)
+                saty, satx = np.where(sat_pixels == sat_flag)
+                only_jump = gdq[integration, group, :, :].copy()
+                only_jump[saty, satx] = jump_flag
+                jump_circles = find_circles(only_jump, jump_flag, min_jump_area)
+
+                if sat_required_snowball:
+                    snowballs = make_snowballs(jump_circles, sat_circles)
+                else:
+                    snowballs = jump_circles
+                n_showers_grp.append(len(snowballs))
+                gdq[integration, group, :, :], num_events = extend_snowballs(gdq[integration, group, :, :],
+                                                                             snowballs, sat_flag,
+                                                                             jump_flag,
+                                                                             expansion=expand_factor)
+        if use_ellipses:
+            if np.all(np.array(n_showers_grp_ellipse) == 0):
+                log.info(f'No showers found in integration {integration}.')
+            else:
+                log.info(f' In integration {integration}, number of' +
+                         f'showers in each group = {n_showers_grp_ellipse}')
+        else:
+            if np.all(np.array(n_showers_grp) == 0):
+                log.info(f'No snowballs found in integration {integration}.')
+            else:
+                log.info(f' In integration {integration}, number of snowballs ' +
+                         f'in each group = {n_showers_grp}')
+
+
+def extend_snowballs(plane, snowballs, sat_flag, jump_flag, expansion=1.5):
+    # For a given DQ plane it will use the list of snowballs to create expanded circles of pixels with
+    # the jump flag set.
+    image = np.zeros(shape=(plane.shape[0], plane.shape[1], 3), dtype=np.uint8)
+    num_circles = len(snowballs)
+    sat_pix = np.bitwise_and(plane, 2)
+    for snowball in snowballs:
+        jump_radius = snowball[1]
+        jump_center = snowball[0]
+        cenx = jump_center[1]
+        ceny = jump_center[0]
+        extend_radius = round(jump_radius * expansion)
+        image = cv.circle(image, (round(ceny), round(cenx)), extend_radius, (0, 0, 4), -1)
+        jump_circle = image[:, :, 2]
+        saty, satx = np.where(sat_pix == 2)
+        jump_circle[saty, satx] = 0
+        plane = np.bitwise_or(plane, jump_circle)
+
+    return plane, num_circles
+
+
+def extend_ellipses(plane, ellipses, sat_flag, jump_flag, expansion=1.1):
+    # For a given DQ plane it will use the list of ellipses to create expanded ellipses of pixels with
+    # the jump flag set.
+    image = np.zeros(shape=(plane.shape[0], plane.shape[1], 3), dtype=np.uint8)
+    num_ellipses = len(ellipses)
+    sat_pix = np.bitwise_and(plane, 2)
+    for ellipse in ellipses:
+        ceny = ellipse[0][0]
+        cenx = ellipse[0][1]
+        majaxis = ellipse[1][0] * expansion
+        minaxis = ellipse[1][1] * expansion
+        alpha = ellipse[2]
+        image = cv.ellipse(image, (round(ceny), round(cenx)), (round(majaxis / 2),
+                           round(minaxis / 2)), alpha, 0, 360, (0, 0, 4), -1)
+        jump_ellipse = image[:, :, 2]
+        saty, satx = np.where(sat_pix == 2)
+        jump_ellipse[saty, satx] = 0
+        plane = np.bitwise_or(plane, jump_ellipse)
+    return plane, num_ellipses
+
+
+def find_circles(dqplane, bitmask, min_area):
+    # Using an input DQ plane this routine will find the groups of pixels with at least the minimum
+    # area and return a list of the minimum enclosing circle parameters.
+    pixels = np.bitwise_and(dqplane, bitmask)
+    contours, hierarchy = cv.findContours(pixels, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
+    bigcontours = [con for con in contours if cv.contourArea(con) >= min_area]
+    circles = [cv.minEnclosingCircle(con) for con in bigcontours]
+    return circles
+
+
+def find_ellipses(dqplane, bitmask, min_area):
+    # Using an input DQ plane this routine will find the groups of pixels with at least the minimum
+    # area and return a list of the minimum enclosing ellipse parameters.
+    pixels = np.bitwise_and(dqplane, bitmask)
+    contours, hierarchy = cv.findContours(pixels, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
+    bigcontours = [con for con in contours if cv.contourArea(con) > min_area]
+    # minAreaRect is used becuase fitEllipse requires 5 points and it is possible to have a contour
+    # with just 4 points.
+    ellipses = [cv.minAreaRect(con) for con in bigcontours]
+    return ellipses
+
+
+def make_snowballs(jump_circles, sat_circles):
+    # Ths routine will create a list of snowballs (circles) that have the center of the saturation circle
+    # within the unclosing jump circle.
+    snowballs = []
+    for jump in jump_circles:
+        radius = jump[1]
+        for sat in sat_circles:
+            distance = np.sqrt((jump[0][0] - sat[0][0]) ** 2 + (jump[0][1] - sat[0][1]) ** 2)
+            if distance < radius:  # center of saturation is within the enclosing jump circle
+                if jump not in snowballs:
+                    snowballs.append(jump)
+    return snowballs

src/stcal/jump/twopoint_difference.py

@@ -165,8 +165,6 @@ def find_crs(dataa, group_dq, read_noise, rejection_thresh,
 
         log_str = 'From highest outlier, two-point found {} pixels with at least one CR from {} groups.'
         log.info(log_str.format(len(row4cr), 'five or more'))
-        log.info(log_str.format(len(row3cr), 'four'))
-        log.info(log_str.format(len(row2cr), 'three'))
 
         # get the rows, col pairs for all pixels with at least one CR
         all_crs_row = np.concatenate((row4cr, row3cr, row2cr))
@@ -288,7 +286,6 @@ def find_crs(dataa, group_dq, read_noise, rejection_thresh,
         flag_groups = [after_jump_flag_n1, after_jump_flag_n2]
 
         cr_group, cr_row, cr_col = np.where(np.bitwise_and(gdq[integ], jump_flag))
-
         for cthres, cgroup in zip(flag_e_threshold, flag_groups):
             if cgroup > 0:
                 log.info(f"Flagging {cgroup} groups after detected jumps with e >= {np.mean(cthres)}.")

tests/test_jump.py

@@ -0,0 +1,65 @@
+import pytest
+import numpy as np
+from astropy.io import fits
+import cv2 as cv
+
+from stcal.jump.jump import flag_large_events, make_snowballs, extend_snowballs, extend_ellipses, find_circles, find_ellipses
+
+
+
+DQFLAGS = {'JUMP_DET': 4, 'SATURATED': 2, 'DO_NOT_USE': 1}
+
+
+@pytest.fixture(scope='function')
+def setup_cube():
+
+    def _cube(ngroups, readnoise=10):
+        nints = 1
+        nrows = 204
+        ncols = 204
+        rej_threshold = 3
+        nframes = 1
+        data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
+        read_noise = np.full((nrows, ncols), readnoise, dtype=np.float32)
+        gdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.uint32)
+
+        return data, gdq, nframes, read_noise, rej_threshold
+
+    return _cube
+
+
+def test_find_simple_circle():
+    plane = np.zeros(shape=(5, 5), dtype=np.uint8)
+    plane[2, 2] = DQFLAGS['JUMP_DET']
+    plane[3, 2] = DQFLAGS['JUMP_DET']
+    plane[1, 2] = DQFLAGS['JUMP_DET']
+    plane[2, 3] = DQFLAGS['JUMP_DET']
+    plane[2, 1] = DQFLAGS['JUMP_DET']
+    circle = find_circles(plane, DQFLAGS['JUMP_DET'], 1)
+    assert circle[0][1] == pytest.approx(1.0, 1e-3)
+
+
+def test_find_simple_ellipse():
+    plane = np.zeros(shape=(5, 5), dtype=np.uint8)
+    plane[2, 2] = DQFLAGS['JUMP_DET']
+    plane[3, 2] = DQFLAGS['JUMP_DET']
+    plane[1, 2] = DQFLAGS['JUMP_DET']
+    plane[2, 3] = DQFLAGS['JUMP_DET']
+    plane[2, 1] = DQFLAGS['JUMP_DET']
+    plane[1, 3] = DQFLAGS['JUMP_DET']
+    plane[2, 4] = DQFLAGS['JUMP_DET']
+    plane[3, 3] = DQFLAGS['JUMP_DET']
+    ellipse = find_ellipses(plane, DQFLAGS['JUMP_DET'], 1)
+    assert ellipse[0][2] == pytest.approx(45.0, 1e-3) # 90 degree rotation
+    assert ellipse[0][0] == pytest.approx((2.5, 2.0))  # center
+
+@pytest.mark.skip(reason="only for local testing")
+def test_single_group():
+    inplane = fits.getdata("jumppix.fits")
+    indq = np.zeros(shape=(1, 1, inplane.shape[0], inplane.shape[1]), dtype=np.uint8)
+    indq[0, 0, :, :] = inplane
+    flag_large_events(indq, DQFLAGS['JUMP_DET'], DQFLAGS['SATURATED'], min_sat_area=1,
+                      min_jump_area=15, max_offset=1, expand_factor=1.1, use_ellipses=True,
+                      sat_required_snowball=False)
+    fits.writeto("jumppix_expand.fits", indq, overwrite=True)
+