use resample many_to_many in outlier detection (#1260)

CHANGES.rst

@@ -6,7 +6,6 @@ Documentation
 
 - Update RTD to include mosaic data (i2d) description [#1262]
 
-
 general
 -------
 
@@ -18,7 +17,6 @@ general
 
 - Rename highlevelpipeline to mosaic pipeline [#1249]
 
-
 source_catalog
 --------------
 - Add PSF photometry capability. [#1243]
@@ -27,6 +25,11 @@ dq_init
 -------
 -  Refactor DQInitStep to use the RampModel method of creating ramps. [#1258]
 
+outlier_detection
+-----------------
+
+- Set ``single=True`` to use ``many_to_many`` when creating median image. [#1260]
+
 
 0.15.1 (2024-05-15)
 ===================

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "tweakwcs >=0.8.6",
     "spherical-geometry >= 1.2.22",
     "stsci.imagestats >= 1.6.3",
-    "drizzle >= 1.13.7",
+    "drizzle >= 1.14.0",
     "webbpsf == 1.2.1",
 ]
 dynamic = [

romancal/outlier_detection/outlier_detection.py

@@ -84,7 +84,7 @@ def do_detection(self):
             # Start by creating resampled/mosaic images for
             # each group of exposures
             resamp = resample.ResampleData(
-                self.input_models, single=False, blendheaders=False, **pars
+                self.input_models, single=True, blendheaders=False, **pars
             )
             drizzled_models = resamp.do_drizzle()
 

romancal/outlier_detection/tests/test_outlier_detection.py

@@ -191,88 +191,59 @@ def test_outlier_do_detection_write_files_to_custom_location(tmp_path, base_imag
     assert all(x.exists() for x in outlier_files_path)
 
 
-def test_outlier_do_detection_find_outliers(tmp_path, base_image):
+def test_find_outliers(tmp_path, base_image):
     """
     Test that OutlierDetection can find outliers.
     """
-    img_1 = base_image()
-    img_1.meta.filename = "img_1.asdf"
-    img_2 = base_image()
-    img_2.meta.filename = "img_2.asdf"
+    cr_value = Quantity(100, "DN / s")
+    source_value = Quantity(10, "DN / s")
+    err_value = Quantity(10, "DN / s")  # snr=1
+
+    imgs = []
+    for i in range(3):
+        img = base_image()
+        img.data[42, 72] = source_value
+        img.err[:] = err_value
+        img.meta.filename = f"img{i}_suffix.asdf"
+        img.meta.observation.exposure = i
+        imgs.append(img)
 
     # add outliers
-    img_1_input_coords = np.array(
-        [(5, 45), (25, 25), (45, 85), (65, 65), (85, 5)], dtype=[("x", int), ("y", int)]
-    )
-    img_2_input_coords = np.array(
-        [(15, 25), (35, 5), (75, 65), (95, 45), (99, 5)], dtype=[("x", int), ("y", int)]
-    )
-    img_1.data[img_1_input_coords["x"], img_1_input_coords["y"]] = Quantity(
-        100000, "DN / s"
+    img_0_input_coords = np.array(
+        [[5, 25, 45, 65, 85], [45, 25, 85, 65, 5]], dtype=np.int64
     )
-    img_2.data[img_2_input_coords["x"], img_2_input_coords["y"]] = Quantity(
-        100000, "DN / s"
+    img_1_input_coords = np.array(
+        [[15, 35, 75, 95, 99], [25, 5, 65, 45, 5]], dtype=np.int64
     )
 
-    input_models = ModelContainer([img_1, img_2])
+    imgs[0].data[img_0_input_coords[0], img_0_input_coords[1]] = cr_value
+    imgs[1].data[img_1_input_coords[0], img_1_input_coords[1]] = cr_value
+
+    input_models = ModelContainer(imgs)
 
     outlier_step = OutlierDetectionStep()
     # set output dir for all files created by the step
     outlier_step.output_dir = tmp_path.as_posix()
     # make sure files are written out to disk
     outlier_step.in_memory = False
 
-    pars = {
-        "weight_type": "exptime",
-        "pixfrac": 1.0,
-        "kernel": "square",
-        "fillval": "INDEF",
-        "nlow": 0,
-        "nhigh": 0,
-        "maskpt": 0.7,
-        "grow": 1,
-        "snr": "4.0 3.0",
-        "scale": "0.5 0.4",
-        "backg": 0.0,
-        "kernel_size": "7 7",
-        "save_intermediate_results": False,
-        "resample_data": False,
-        "good_bits": 0,
-        "allowed_memory": None,
-        "in_memory": outlier_step.in_memory,
-        "make_output_path": outlier_step.make_output_path,
-        "resample_suffix": "i2d",
-    }
+    result = outlier_step(input_models)
 
-    detection_step = outlier_detection.OutlierDetection
-    step = detection_step(input_models, **pars)
+    expected_crs = [img_0_input_coords, img_1_input_coords, None]
+    for cr_coords, flagged_img in zip(expected_crs, result):
+        if cr_coords is None:
+            assert not np.any(flagged_img.dq > 0)
+        else:
+            flagged_coords = np.where(flagged_img.dq > 0)
+            np.testing.assert_equal(cr_coords, flagged_coords)
 
-    step.do_detection()
-
-    # get flagged outliers coordinates from DQ array
-    img_1_outlier_output_coords = np.where(step.input_models[0].dq > 0)
-
-    # reformat output and input coordinates and sort by x coordinate
-    outliers_output_coords = np.array(
-        list(zip(*img_1_outlier_output_coords)), dtype=[("x", int), ("y", int)]
-    )
-    outliers_input_coords = np.concatenate((img_1_input_coords, img_2_input_coords))
-
-    outliers_output_coords.sort(axis=0)
-    outliers_input_coords.sort(axis=0)
 
-    # assert all(outliers_input_coords == outliers_output_coords) doesn't work with python 3.9
-    assert all(o == i for i, o in zip(outliers_input_coords, outliers_output_coords))
-
-
-def test_outlier_do_detection_do_not_find_outliers_in_identical_images(
-    tmp_path, base_image, caplog
-):
+def test_identical_images(tmp_path, base_image, caplog):
     """
     Test that OutlierDetection does not flag any outliers in the DQ array if images are identical.
     """
     img_1 = base_image()
-    img_1.meta.filename = "img_1.asdf"
+    img_1.meta.filename = "img1_suffix.asdf"
     # add outliers
     img_1_input_coords = np.array(
         [(5, 45), (25, 25), (45, 85), (65, 65), (85, 5)], dtype=[("x", int), ("y", int)]
@@ -282,9 +253,9 @@ def test_outlier_do_detection_do_not_find_outliers_in_identical_images(
     )
 
     img_2 = img_1.copy()
-    img_2.meta.filename = "img_2.asdf"
+    img_2.meta.filename = "img2_suffix.asdf"
     img_3 = img_1.copy()
-    img_3.meta.filename = "img_3.asdf"
+    img_3.meta.filename = "img3_suffix.asdf"
 
     input_models = ModelContainer([img_1, img_2, img_3])
 
@@ -294,39 +265,14 @@ def test_outlier_do_detection_do_not_find_outliers_in_identical_images(
     # make sure files are written out to disk
     outlier_step.in_memory = False
 
-    pars = {
-        "weight_type": "exptime",
-        "pixfrac": 1.0,
-        "kernel": "square",
-        "fillval": "INDEF",
-        "nlow": 0,
-        "nhigh": 0,
-        "maskpt": 0.7,
-        "grow": 1,
-        "snr": "4.0 3.0",
-        "scale": "0.5 0.4",
-        "backg": 0.0,
-        "kernel_size": "7 7",
-        "save_intermediate_results": False,
-        "resample_data": False,
-        "good_bits": 0,
-        "allowed_memory": None,
-        "in_memory": outlier_step.in_memory,
-        "make_output_path": outlier_step.make_output_path,
-        "resample_suffix": "i2d",
-    }
-
-    detection_step = outlier_detection.OutlierDetection
-    step = detection_step(input_models, **pars)
-
-    step.do_detection()
+    result = outlier_step(input_models)
 
     # assert that log shows no new outliers detected
     assert "New pixels flagged as outliers: 0 (0.00%)" in {
         x.message for x in caplog.records
     }
     # assert that DQ array has nothing flagged as outliers
-    assert [np.count_nonzero(x.dq) for x in step.input_models] == [0, 0, 0]
+    assert [np.count_nonzero(x.dq) for x in result] == [0, 0, 0]
 
 
 @pytest.mark.parametrize(

romancal/resample/resample.py

@@ -203,6 +203,7 @@ def resample_many_to_many(self):
 
         Used for outlier detection
         """
+        output_list = []
         for exposure in self.input_models.models_grouped:
             output_model = self.blank_output
             output_model.meta["resample"] = maker_utils.mk_resample()
@@ -224,7 +225,6 @@ def resample_many_to_many(self):
             )
 
             log.info(f"{len(exposure)} exposures to drizzle together")
-            output_list = []
             for img in exposure:
                 img = datamodels.open(img)
                 # TODO: should weight_type=None here?
@@ -233,11 +233,12 @@ def resample_many_to_many(self):
                 )
 
                 # apply sky subtraction
-                if not hasattr(img.meta, "background"):
-                    self._create_background_attribute(img)
-                blevel = img.meta.background.level
-                if not img.meta.background.subtracted and blevel is not None:
-                    data = img.data - blevel
+                if (
+                    hasattr(img.meta, "background")
+                    and img.meta.background.subtracted is False
+                    and img.meta.background.level is not None
+                ):
+                    data = img.data - img.meta.background.level
                 else:
                     data = img.data
 
@@ -268,10 +269,11 @@ def resample_many_to_many(self):
             else:
                 output_list.append(output_model.copy())
 
-            self.output_models = ModelContainer(output_list, return_open=self.in_memory)
             output_model.data *= 0.0
             output_model.weight *= 0.0
 
+        self.output_models = ModelContainer(output_list, return_open=self.in_memory)
+
         return self.output_models
 
     def resample_many_to_one(self):
@@ -305,11 +307,12 @@ def resample_many_to_one(self):
                 weight_type=self.weight_type,
                 good_bits=self.good_bits,
             )
-            if not hasattr(img.meta, "background"):
-                self._create_background_attribute(img)
-            blevel = img.meta.background.level
-            if not img.meta.background.subtracted and blevel is not None:
-                data = img.data - blevel
+            if (
+                hasattr(img.meta, "background")
+                and img.meta.background.subtracted is False
+                and img.meta.background.level is not None
+            ):
+                data = img.data - img.meta.background.level
             else:
                 data = img.data
 
@@ -368,11 +371,6 @@ def resample_many_to_one(self):
 
         return self.output_models
 
-    def _create_background_attribute(self, img):
-        img.meta["background"] = {}
-        img.meta.background["level"] = 0
-        img.meta.background["subtracted"] = True
-
     def resample_variance_array(self, name, output_model):
         """Resample variance arrays from ``self.input_models`` to the ``output_model``.