JP-3683: fix abs_deriv handling of off-edge and nan values (#311)

changes/311.bugfix.rst

@@ -0,0 +1 @@
+Fix abs_deriv handling of off-edge and nan values.

src/stcal/outlier_detection/utils.py

@@ -85,43 +85,33 @@ def compute_weight_threshold(weight, maskpt):
 
 
 def _abs_deriv(array):
-    """
-    Do not use this function.
-
-    Take the absolute derivative of a numpy array.
-
-    This function assumes off-edge pixel values are 0
-    and leads to erroneous derivative values and should
-    likely not be used.
-    """
-    tmp = np.zeros(array.shape, dtype=np.float64)
-    out = np.zeros(array.shape, dtype=np.float64)
-
-    tmp[1:, :] = array[:-1, :]
-    tmp, out = _absolute_subtract(array, tmp, out)
-    tmp[:-1, :] = array[1:, :]
-    tmp, out = _absolute_subtract(array, tmp, out)
-
-    tmp[:, 1:] = array[:, :-1]
-    tmp, out = _absolute_subtract(array, tmp, out)
-    tmp[:, :-1] = array[:, 1:]
-    tmp, out = _absolute_subtract(array, tmp, out)
-
+    """Take the absolute derivate of a numpy array."""
+    out = np.zeros_like(array)  # use same dtype as input
+
+    # make output values nan where input is nan (for floating point input)
+    if np.issubdtype(array.dtype, np.floating):
+        out[np.isnan(array)] = np.nan
+
+    # compute row-wise absolute diffference
+    row_diff = np.abs(np.diff(array, axis=0))
+    np.putmask(out[1:], np.isfinite(row_diff), row_diff)  # no need to do max yet
+
+    # since these are absolute differences |r0-r1| = |r1-r0|
+    # make a view of the target portion of the array
+    row_offset_view = out[:-1]
+    # compute an in-place maximum
+    np.putmask(row_offset_view, row_diff > row_offset_view, row_diff)
+    del row_diff
+
+    # compute col-wise absolute difference
+    col_diff = np.abs(np.diff(array, axis=1))
+    col_offset_view = out[:, 1:]
+    np.putmask(col_offset_view, col_diff > col_offset_view, col_diff)
+    col_offset_view = out[:, :-1]
+    np.putmask(col_offset_view, col_diff > col_offset_view, col_diff)
     return out
 
 
-def _absolute_subtract(array, tmp, out):
-    """
-    Do not use this function.
-
-    A helper function for _abs_deriv.
-    """
-    tmp = np.abs(array - tmp)
-    out = np.maximum(tmp, out)
-    tmp = tmp * 0.
-    return tmp, out
-
-
 def flag_crs(
     sci_data,
     sci_err,

tests/outlier_detection/test_utils.py

@@ -33,14 +33,23 @@ def test_abs_deriv_single_value(shape, diff):
     np.testing.assert_allclose(result, expected)
 
 
-@pytest.mark.skip(reason="_abs_deriv has edge effects due to treating off-edge pixels as 0: see JP-3683")
 @pytest.mark.parametrize("nrows,ncols", [(5, 5), (7, 11), (17, 13)])
 def test_abs_deriv_range(nrows, ncols):
     arr = np.arange(nrows * ncols).reshape(nrows, ncols)
     result = _abs_deriv(arr)
     np.testing.assert_allclose(result, ncols)
 
 
+def test_abs_deriv_nan():
+    arr = np.arange(25, dtype='f4').reshape(5, 5)
+    arr[2, 2] = np.nan
+    expect_nan = np.zeros_like(arr, dtype=bool)
+    expect_nan[2, 2] = True
+    result = _abs_deriv(arr)
+    assert np.isnan(result[expect_nan])
+    assert np.all(np.isfinite(result[~expect_nan]))
+
+
 @pytest.mark.parametrize("shape,mean,maskpt,expected", [
     ([5, 5], 11, 0.5, 5.5),
     ([5, 5], 11, 0.25, 2.75),