RCAL-776: Change image units. (#1128)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
spacetelescope · Mar 12, 2024 · 22293cc · 22293cc
1 parent 6a7e5d2
commit 22293cc
Show file tree

Hide file tree

Showing 10 changed files with 49 additions and 63 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -22,6 +22,11 @@ dq_init
 - Copy reference pixels during ``dq_init`` to avoid larger files in later
   processing steps [#1121]
 
+ramp_fitting
+------------
+
+- Changed image units from e/s to DN/s (and added support for MJy/sr). Added gain reduction to convert to these units. [#1128]
+
 
 
 0.14.0 (2024-02-12)

diff --git a/docs/roman/data_products/product_types.rst b/docs/roman/data_products/product_types.rst
@@ -22,7 +22,7 @@ Exposure Pipeline Steps And Data Products
 
 The following table contains lists of all data product types for the Exposure pipeline, as given by their file name suffix. The input uncal file and the final cal file
 are the only files that are produced in the standard processing. All the other are optional (opt) files that can be produced when
-the user is running the pipeline. The input for each optional step is the output of the preceeding step.
+the user is running the pipeline. The input for each optional step is the output of the preceding step.
 
 +------------------------------------------------+-----------------+--------------------------+------------------+---------------------+---------------------------------------+
 | Pipeline Step                                  | Input           |  Output suffix           | Data Model       | Units               | Description                           |
@@ -39,19 +39,19 @@ the user is running the pipeline. The input for each optional step is the output
 +------------------------------------------------+-----------------+--------------------------+------------------+---------------------+---------------------------------------+
 | :ref:`dark_current <dark_current_step>`        |                 | darkcurrent (opt)        | RampModel        | DN                  | 3-D dark current subtracted data      |
 +------------------------------------------------+-----------------+--------------------------+------------------+---------------------+---------------------------------------+
-| :ref:`ramp_fitting <ramp_fitting_step>`        |                 | rampfit (opt)            | ImageModel       | electron/s          | 2-D slope corrected data              |
+| :ref:`ramp_fitting <ramp_fitting_step>`        |                 | rampfit (opt)            | ImageModel       | DN/s        	       | 2-D slope corrected data              |
 +------------------------------------------------+-----------------+--------------------------+------------------+---------------------+---------------------------------------+
-| :ref:`assign_wcs <assign_wcs_step>`            |                 | assignwcs (opt)          | ImageModel       | electron/s          | 2-D data with gwcs                    |
+| :ref:`assign_wcs <assign_wcs_step>`            |                 | assignwcs (opt)          | ImageModel       | DN/s                | 2-D data with gwcs                    |
 +------------------------------------------------+-----------------+--------------------------+------------------+---------------------+---------------------------------------+
-| :ref:`flatfield <flatfield_step>`              |                 | flat (opt)               | ImageModel       | electron/s          | 2-D QE corrected data                 |
+| :ref:`flatfield <flatfield_step>`              |                 | flat (opt)               | ImageModel       | DN/s                | 2-D QE corrected data                 |
 +------------------------------------------------+-----------------+--------------------------+------------------+---------------------+---------------------------------------+
-| :ref:`photom <photom_step>`                    |                 | photom (opt)             | ImageModel       | electron/s          | Add phometric data to header          |
+| :ref:`photom <photom_step>`                    |                 | photom (opt)             | ImageModel       | DN/s                | Add phometric data to header          |
 +------------------------------------------------+-----------------+--------------------------+------------------+---------------------+---------------------------------------+
-| :ref:`source_detection <source_detection_step>`|                 | sourcedetectionstep (opt)| ImageModel       | electron/s          | Sources identified in the data        |
+| :ref:`source_detection <source_detection_step>`|                 | sourcedetectionstep (opt)| ImageModel       | DN/s                | Sources identified in the data        |
 +------------------------------------------------+-----------------+--------------------------+------------------+---------------------+---------------------------------------+
-| :ref:`tweakreg <tweakreg_step>`                |                 | tweakregstep (opt)       | ImageModel       | electron/s          | WCS aligned with GAIA                 |
+| :ref:`tweakreg <tweakreg_step>`                |                 | tweakregstep (opt)       | ImageModel       | DN/s                | WCS aligned with GAIA                 |
 +------------------------------------------------+-----------------+--------------------------+------------------+---------------------+---------------------------------------+
-|                                                |                 | cal                      | ImageModel       | electron/s          | 2-D calibrated exposure data          |
+|                                                |                 | cal                      | ImageModel       | DN/s                | 2-D calibrated exposure data          |
 +------------------------------------------------+-----------------+--------------------------+------------------+---------------------+---------------------------------------+
 
 
@@ -62,7 +62,7 @@ High Level Processing Steps And Data Products
 The following table contain lists of all data product types for the HighLevel Processsing (HLP) Pipeline, as given by their file name suffix.
 The input to the HLP is an association file (in JSON format), the output is a combined image.
 All the other are optional (opt) files that can be produced when
-the user is running the pipeline. The input for each optional step is the output of the preceeding step.
+the user is running the pipeline. The input for each optional step is the output of the preceding step.
 
 +---------------------------------------------------+-----------------+------------------------------+------------------+---------------------+---------------------------------------+
 | Pipeline Step                                     | Input           |  Output suffix               | Data Model       | Units               | Description                           |

diff --git a/pyproject.toml b/pyproject.toml
@@ -22,9 +22,9 @@ dependencies = [
     'photutils >=1.10.0',
     'pyparsing >=2.4.7',
     'requests >=2.26',
-    #'rad >=0.19.0',
+    # 'rad >=0.19.0',
     'rad @ git+https://github.com/spacetelescope/rad.git',
-    #'roman_datamodels >=0.19.0',
+    # 'roman_datamodels >=0.19.0',
     'roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git',
     'scipy >=1.11',
     #'stcal >=1.5.2',

diff --git a/romancal/flatfield/flat_field.py b/romancal/flatfield/flat_field.py
@@ -106,7 +106,7 @@ def apply_flat_field(science, flat):
     # Now let's apply the correction to science data and error arrays.  Rely
     # on array broadcasting to handle the cubes
     science.data = u.Quantity(
-        (science.data.value / flat_data), u.electron / u.s, dtype=science.data.dtype
+        (science.data.value / flat_data), u.DN / u.s, dtype=science.data.dtype
     )
 
     # Update the variances using BASELINE algorithm.  For guider data, it has

diff --git a/romancal/flatfield/tests/test_flatfield.py b/romancal/flatfield/tests/test_flatfield.py
@@ -27,20 +27,20 @@ def test_flatfield_step_interface(instrument, exptype):
     wfi_image.meta.instrument.optical_element = "F158"
     wfi_image.meta.exposure.type = exptype
     wfi_image.data = u.Quantity(
-        np.ones(shape, dtype=np.float32), u.electron / u.s, dtype=np.float32
+        np.ones(shape, dtype=np.float32), u.DN / u.s, dtype=np.float32
     )
     wfi_image.dq = np.zeros(shape, dtype=np.uint32)
     wfi_image.err = u.Quantity(
-        np.zeros(shape, dtype=np.float32), u.electron / u.s, dtype=np.float32
+        np.zeros(shape, dtype=np.float32), u.DN / u.s, dtype=np.float32
     )
     wfi_image.var_poisson = u.Quantity(
-        np.zeros(shape, dtype=np.float32), u.electron**2 / u.s**2, dtype=np.float32
+        np.zeros(shape, dtype=np.float32), u.DN**2 / u.s**2, dtype=np.float32
     )
     wfi_image.var_rnoise = u.Quantity(
-        np.zeros(shape, dtype=np.float32), u.electron**2 / u.s**2, dtype=np.float32
+        np.zeros(shape, dtype=np.float32), u.DN**2 / u.s**2, dtype=np.float32
     )
     wfi_image.var_flat = u.Quantity(
-        np.zeros(shape, dtype=np.float32), u.electron**2 / u.s**2, dtype=np.float32
+        np.zeros(shape, dtype=np.float32), u.DN**2 / u.s**2, dtype=np.float32
     )
 
     wfi_image_model = ImageModel(wfi_image)

diff --git a/romancal/lib/tests/test_psf.py b/romancal/lib/tests/test_psf.py
@@ -30,7 +30,7 @@ def setup_inputs(
     """
     wfi_image = testutil.mk_level2_image(shape=shape)
     wfi_image.data = u.Quantity(
-        np.ones(shape, dtype=np.float32), u.electron / u.s, dtype=np.float32
+        np.ones(shape, dtype=np.float32), u.DN / u.s, dtype=np.float32
     )
     wfi_image.meta.filename = "filename"
     wfi_image.meta.instrument["optical_element"] = "F087"
@@ -40,10 +40,10 @@ def setup_inputs(
         setup_rng = np.random.default_rng(seed or 19)
         wfi_image.data = u.Quantity(
             setup_rng.normal(scale=noise, size=shape),
-            u.electron / u.s,
+            u.DN / u.s,
             dtype=np.float32,
         )
-        wfi_image.err = noise * np.ones(shape, dtype=np.float32) * u.electron / u.s
+        wfi_image.err = noise * np.ones(shape, dtype=np.float32) * u.DN / u.s
 
     # add dq array
     wfi_image.dq = np.zeros(shape, dtype=np.uint32)

diff --git a/romancal/ramp_fitting/ramp_fit_step.py b/romancal/ramp_fitting/ramp_fit_step.py
@@ -109,35 +109,13 @@ def ols(self, input_model, readnoise_model, gain_model):
             pixel,
         )
 
-        if image_info is not None:
-            # JWST has a separate GainScale step.  This isn't in Roman.
-            # We can adjust the gains here instead.
-            # data, dq, var_poisson, var_rnoise, err = image_info
-            # This step isn't needed for ols_cas22, which works directly in
-            # electrons.
-            image_info[0][...] *= gain_model.data.value  # data
-            # no dq multiplication!
-            image_info[2][...] *= gain_model.data.value**2  # var_poisson
-            image_info[3][...] *= gain_model.data.value**2  # var_rnoise
-            image_info[4][...] *= gain_model.data.value  # err
-
         readnoise_model.close()
 
         # Save the OLS optional fit product, if it exists
         if opt_info is not None:
-            # We should scale these by the gain, too.
             # opt_info = (slope, sigslope, var_poisson, var_readnoise, yint,
             # sig_yint,
             # ped_int, weights, cr_mag_seg
-            opt_info[0][...] *= gain_model.data.value  # slope
-            opt_info[1][...] *= gain_model.data.value  # sigslope
-            opt_info[2][...] *= gain_model.data.value**2  # var_poisson
-            opt_info[3][...] *= gain_model.data.value**2  # var_readnoise
-            opt_info[4][...] *= gain_model.data.value  # yint
-            opt_info[5][...] *= gain_model.data.value  # sigyint
-            opt_info[6][...] *= gain_model.data.value  # pedestal
-            # no change to weights due to gain
-            opt_info[8][...] *= gain_model.data.value  # crmag
             opt_model = create_optional_results_model(input_model, opt_info)
             self.save_model(opt_model, "fitopt", output_file=self.opt_name)
 
@@ -157,6 +135,7 @@ def ols(self, input_model, readnoise_model, gain_model):
             )
 
         out_model = create_image_model(input_model, image_info)
+
         return out_model
 
     def ols_cas22(self, input_model, readnoise_model, gain_model):
@@ -229,6 +208,12 @@ def ols_cas22(self, input_model, readnoise_model, gain_model):
         image_info = (slopes, ramp_dq, var_poisson, var_rnoise, err)
         image_model = create_image_model(input_model, image_info)
 
+        # Rescale by the gain back to DN/s
+        image_model.data /= gain[4:-4, 4:-4]
+        image_model.err /= gain[4:-4, 4:-4]
+        image_model.var_poisson /= gain[4:-4, 4:-4] ** 2
+        image_model.var_rnoise /= gain[4:-4, 4:-4] ** 2
+
         # That's all folks
         return image_model
 
@@ -255,12 +240,10 @@ def create_image_model(input_model, image_info):
     """
     data, dq, var_poisson, var_rnoise, err = image_info
 
-    data = u.Quantity(data, u.electron / u.s, dtype=data.dtype)
-    var_poisson = u.Quantity(
-        var_poisson, u.electron**2 / u.s**2, dtype=var_poisson.dtype
-    )
-    var_rnoise = u.Quantity(var_rnoise, u.electron**2 / u.s**2, dtype=var_rnoise.dtype)
-    err = u.Quantity(err, u.electron / u.s, dtype=err.dtype)
+    data = u.Quantity(data, u.DN / u.s, dtype=data.dtype)
+    var_poisson = u.Quantity(var_poisson, u.DN**2 / u.s**2, dtype=var_poisson.dtype)
+    var_rnoise = u.Quantity(var_rnoise, u.DN**2 / u.s**2, dtype=var_rnoise.dtype)
+    err = u.Quantity(err, u.DN / u.s, dtype=err.dtype)
     if dq is None:
         dq = np.zeros(data.shape, dtype="u4")
 
@@ -272,15 +255,13 @@ def create_image_model(input_model, image_info):
     meta["photometry"] = maker_utils.mk_photometry()
     inst = {
         "meta": meta,
-        "data": u.Quantity(data, u.electron / u.s, dtype=data.dtype),
+        "data": u.Quantity(data, u.DN / u.s, dtype=data.dtype),
         "dq": dq,
         "var_poisson": u.Quantity(
-            var_poisson, u.electron**2 / u.s**2, dtype=var_poisson.dtype
-        ),
-        "var_rnoise": u.Quantity(
-            var_rnoise, u.electron**2 / u.s**2, dtype=var_rnoise.dtype
+            var_poisson, u.DN**2 / u.s**2, dtype=var_poisson.dtype
         ),
-        "err": u.Quantity(err, u.electron / u.s, dtype=err.dtype),
+        "var_rnoise": u.Quantity(var_rnoise, u.DN**2 / u.s**2, dtype=var_rnoise.dtype),
+        "err": u.Quantity(err, u.DN / u.s, dtype=err.dtype),
         "amp33": input_model.amp33.copy(),
         "border_ref_pix_left": input_model.border_ref_pix_left.copy(),
         "border_ref_pix_right": input_model.border_ref_pix_right.copy(),

diff --git a/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py
@@ -47,13 +47,13 @@
     ),
 }
 SIMPLE_EXPECTED_GAIN = {
-    "data": np.array([[2.631579, 2.631579], [1.151316, 3.50926]], dtype=np.float32),
-    "err": np.array([[0.542564, 0.542564], [0.358915, 0.623119]], dtype=np.float32),
+    "data": np.array([[0.526316, 0.526316], [0.230263, 0.701852]], dtype=np.float32),
+    "err": np.array([[0.108513, 0.108513], [0.071783, 0.124624]], dtype=np.float32),
     "var_poisson": np.array(
-        [[0.294321, 0.294321], [0.128766, 0.388223]], dtype=np.float32
+        [[1.1772858e-02, 1.1772858e-02], [5.150624e-03, 1.55289e-02]], dtype=np.float32
     ),
     "var_rnoise": np.array(
-        [[5.410319e-05, 5.410319e-05], [5.410319e-05, 5.476514e-05]], dtype=np.float32
+        [[2.164127e-06, 2.164127e-06], [2.164127e-06, 2.190606e-06]], dtype=np.float32
     ),
 }
 SIMPLE_EXPECTED_RNOISE = {

diff --git a/romancal/ramp_fitting/tests/test_ramp_fit_ols.py b/romancal/ramp_fitting/tests/test_ramp_fit_ols.py
@@ -107,7 +107,7 @@ def test_ols_one_group_small_buffer_fit(max_cores, make_data):
     data = out_model.data.value
 
     # Index changes due to trimming of reference pixels
-    np.testing.assert_allclose(data[11, 6], 10 * 2, 1e-5)
+    np.testing.assert_allclose(data[11, 6], 10, 1e-5)
 
 
 @pytest.mark.parametrize("max_cores", MAXIMUM_CORES)

diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py
@@ -55,22 +55,22 @@ def create_image(self):
                 },
                 "data": u.Quantity(
                     rng.poisson(2.5, size=self.shape).astype(np.float32),
-                    u.electron / u.s,
+                    u.DN / u.s,
                     dtype=np.float32,
                 ),
                 "var_rnoise": u.Quantity(
                     rng.normal(1, 0.05, size=self.shape).astype(np.float32),
-                    u.electron**2 / u.s**2,
+                    u.DN**2 / u.s**2,
                     dtype=np.float32,
                 ),
                 "var_poisson": u.Quantity(
                     rng.poisson(1, size=self.shape).astype(np.float32),
-                    u.electron**2 / u.s**2,
+                    u.DN**2 / u.s**2,
                     dtype=np.float32,
                 ),
                 "var_flat": u.Quantity(
                     rng.uniform(0, 1, size=self.shape).astype(np.float32),
-                    u.electron**2 / u.s**2,
+                    u.DN**2 / u.s**2,
                     dtype=np.float32,
                 ),
             },