Rcal 406 science array quantities

CHANGES.rst

@@ -6,20 +6,20 @@ general
 - Update the suffix for the stored filename to match the filename [#609]
 
 - DQ step flags science data affected by guide window read [#599]
-  
+
 - Fix deprecation warnings introduced by ``pytest`` ``7.2`` ahead of ``8.0`` [#597]
-  
+
 - Implemented support for quantities in reference files. Updated unit tests for these changes. [#624]
 
 jump
 ----
 
 - Update default input CR thresholds to give reasonable results [#625]
 
+- Added support for Quantities for data arrays. [#616]
 
 
-
-  0.9.0 (2022-11-14)
+0.9.0 (2022-11-14)
 ==================
 
 general
@@ -308,7 +308,7 @@ general
 general
 -------
 
-- Added regressions tests for ``dq_ini``t utilizing ``mask`` file in CRDS. [#290]
+- Added regressions tests for ``dq_init`` utilizing ``mask`` file in CRDS. [#290]
 
 - Updates for requirements & pip changes [#286]
 
@@ -409,9 +409,6 @@ datamodels
 
 - Update output_ext in the base Step class to .asdf from .fits [#127]
 
-
-=======
-
 - Added ``RampModel``, ``GLS_RampFitModel``, ``RampFitOutputModel`` and
   schemas. [#110]
 

pyproject.toml

@@ -12,7 +12,7 @@ classifiers = [
     'Programming Language :: Python :: 3',
 ]
 dependencies = [
-    'asdf >=2.12.1',
+    'asdf >=2.13.0',
     'astropy >=5.0.4',
     'crds >=11.16.16',
     'gwcs >=0.18.1',

romancal/dark_current/dark_current_step.py

@@ -7,6 +7,7 @@
 from roman_datamodels import datamodels as rdd
 from stcal.dark_current import dark_sub
 from roman_datamodels.testing import utils as testutil
+from roman_datamodels import units as ru
 
 
 __all__ = ["DarkCurrentStep"]
@@ -45,7 +46,7 @@ def process(self, input):
                 dark_model.meta.exposure['groupgap'] = input_model.meta.exposure.groupgap
 
             # Reshaping data variables for stcal compatibility
-            input_model.data = input_model.data.astype(np.float32)[np.newaxis, :]
+            input_model.data = input_model.data[np.newaxis, :]
             input_model.groupdq = input_model.groupdq[np.newaxis, :]
             input_model.err = input_model.err[np.newaxis, :]
 
@@ -135,10 +136,10 @@ def dark_output_data_as_ramp_model(out_data, input_model):
     # Removing integration dimension from variables (added for stcal
     # compatibility)
     # Roman 3D
-    out_model.data = u.Quantity(out_data.data[0], out_model.data.unit, dtype=out_model.data.dtype)
+    out_model.data = u.Quantity(out_data.data[0], ru.DN, dtype=out_data.data.dtype)
     out_model.groupdq = out_data.groupdq[0]
     # Roman 2D
     out_model.pixeldq = out_data.pixeldq
-    out_model.err = u.Quantity(out_data.err[0], out_model.err.unit, dtype=out_model.err.dtype)
+    out_model.err = u.Quantity(out_data.err[0], ru.DN, dtype=out_data.err.dtype)
 
     return out_model

romancal/dark_current/tests/test_dark.py

@@ -7,7 +7,6 @@
 import os
 from astropy import units as u
 
-
 from romancal.dark_current import DarkCurrentStep
 
 from roman_datamodels.datamodels import RampModel, DarkRefModel
@@ -81,10 +80,10 @@ def test_dark_step_subtraction(instrument, exptype):
     result = DarkCurrentStep.call(ramp_model, override_dark=darkref_model)
 
     # check that the dark file is subtracted frame by frame from the science data
-    diff = ramp_model.data - darkref_model.data
+    diff = ramp_model.data.value - darkref_model.data.value
 
     # test that the output data file is equal to the difference found when subtracting reffile from sci file
-    np.testing.assert_array_equal(result.data, diff, err_msg='dark file should be subtracted from sci file ')
+    np.testing.assert_array_equal(result.data.value, diff, err_msg='dark file should be subtracted from sci file ')
 
 
 @pytest.mark.parametrize(
@@ -130,7 +129,6 @@ def create_ramp_and_dark(shape, instrument, exptype):
     ramp.meta.instrument.detector = 'WFI01'
     ramp.meta.instrument.optical_element = 'F158'
     ramp.meta.exposure.type = exptype
-
     ramp.data = u.Quantity(np.ones(shape, dtype=np.float32), ru.DN, dtype=np.float32)
     ramp_model = RampModel(ramp)
 

romancal/dq_init/tests/test_dq_init.py

@@ -272,7 +272,7 @@ def test_dqinit_refpix(instrument, exptype):
 
     # check if reference pixels are correct
     assert result.data.shape == (2, 20, 20)  # no pixels should be trimmed
-    assert result.amp33.shape == (2, 4096 ,128)
+    assert result.amp33.value.shape == (2, 4096 ,128)
     assert result.border_ref_pix_right.shape == (2, 20, 4)
     assert result.border_ref_pix_left.shape == (2, 20, 4)
     assert result.border_ref_pix_top.shape == (2, 4, 20)

romancal/flatfield/flat_field.py

@@ -4,6 +4,8 @@
 
 import logging
 import numpy as np
+from astropy import units as u
+from roman_datamodels import units as ru
 
 from romancal.lib import dqflags
 
@@ -109,19 +111,20 @@ def apply_flat_field(science, flat):
     flat_data[np.where(flat_bad)] = 1.0
     # Now let's apply the correction to science data and error arrays.  Rely
     # on array broadcasting to handle the cubes
-    science.data /= flat_data
+    science.data = u.Quantity((science.data.value / flat_data), ru.electron / u.s, dtype=science.data.dtype)
 
     # Update the variances using BASELINE algorithm.  For guider data, it has
     # not gone through ramp fitting so there is no Poisson noise or readnoise
     flat_data_squared = flat_data**2
     science.var_poisson /= flat_data_squared
     science.var_rnoise /= flat_data_squared
     try:
-        science.var_flat = science.data**2 / flat_data_squared * flat_err**2
+        science.var_flat = science.data ** 2 / flat_data_squared * flat_err ** 2
     except AttributeError:
         science['var_flat'] = np.zeros(shape=science.data.shape,
                                        dtype=np.float32)
-        science.var_flat = science.data**2 / flat_data_squared * flat_err**2
+        science.var_flat = science.data ** 2 / flat_data_squared * flat_err ** 2
+
     science.err = np.sqrt(science.var_poisson +
                           science.var_rnoise + science.var_flat)
 

romancal/flatfield/tests/test_flatfield.py

@@ -3,13 +3,13 @@
 import pytest
 import numpy as np
 from astropy import units as u
-
+from astropy.time import Time
 
 from roman_datamodels import stnode
 from roman_datamodels.datamodels import ImageModel, FlatRefModel
 from roman_datamodels.testing import utils as testutil
+from roman_datamodels import units as ru
 from romancal.flatfield import FlatFieldStep
-from astropy.time import Time
 
 
 @pytest.mark.parametrize(
@@ -33,12 +33,18 @@ def test_flatfield_step_interface(instrument, exptype):
     wfi_image.meta.instrument.detector = 'WFI01'
     wfi_image.meta.instrument.optical_element = 'F158'
     wfi_image.meta.exposure.type = exptype
-    wfi_image.data = u.Quantity(np.ones(shape, dtype=np.float32), wfi_image.data.unit, wfi_image.data.dtype)
+    wfi_image.data = u.Quantity(np.ones(shape, dtype=np.float32),
+                                ru.electron / u.s, dtype=np.float32)
     wfi_image.dq = np.zeros(shape, dtype=np.uint32)
-    wfi_image.err = u.Quantity(np.zeros(shape, dtype=np.float32), wfi_image.err.unit, wfi_image.err.dtype)
-    wfi_image.var_poisson = u.Quantity(np.zeros(shape, dtype=np.float32), wfi_image.var_poisson.unit, wfi_image.var_poisson.dtype)
-    wfi_image.var_rnoise = u.Quantity(np.zeros(shape, dtype=np.float32), wfi_image.var_rnoise.unit, wfi_image.var_rnoise.dtype)
-    wfi_image.var_flat = u.Quantity(np.zeros(shape, dtype=np.float32), wfi_image.var_flat.unit, wfi_image.var_flat.dtype)
+    wfi_image.err = u.Quantity(np.zeros(shape, dtype=np.float32),
+                               ru.electron / u.s, dtype=np.float32)
+    wfi_image.var_poisson = u.Quantity(np.zeros(shape, dtype=np.float32),
+                                       ru.electron**2 / u.s**2, dtype=np.float32)
+    wfi_image.var_rnoise = u.Quantity(np.zeros(shape, dtype=np.float32),
+                                      ru.electron**2 / u.s**2, dtype=np.float32)
+    wfi_image.var_flat = u.Quantity(np.zeros(shape, dtype=np.float32),
+                                    ru.electron**2 / u.s**2, dtype=np.float32)
+
     wfi_image_model = ImageModel(wfi_image)
     flatref = stnode.FlatRef()
     meta = {}

romancal/jump/tests/test_jump_step.py

@@ -145,6 +145,7 @@ def test_one_CR(generate_wfi_reffiles, max_cores, setup_inputs):
 
     for i in range(ngroups):
         model1.data[i, :, :] = deltaDN * i * model1.data.unit
+
     first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0]
 
     CR_locs = [x for x in range(xsize * ysize) if x % CR_fraction == 0]

romancal/linearity/linearity_step.py

@@ -9,6 +9,7 @@
 from romancal.stpipe import RomanStep
 from romancal.lib import dqflags
 from stcal.linearity.linearity import linearity_correction
+from astropy import units as u
 
 __all__ = ["LinearityStep"]
 
@@ -56,11 +57,11 @@ def process(self, input):
             # Call linearity correction function in stcal
             # The third return value is the procesed zero frame which
             # Roman does not use.
-            new_data, new_pdq, _ = linearity_correction(output_model.data,
+            new_data, new_pdq, _ = linearity_correction(output_model.data.value,
                                                      gdq, pdq, lin_coeffs,
                                                      lin_dq, dqflags.pixel)
 
-            output_model.data = new_data[0, :, :, :]
+            output_model.data = u.Quantity(new_data[0, :, :, :], u.DN, dtype=new_data.dtype)
             output_model.pixeldq = new_pdq
 
             # Close the reference file and update the step status

romancal/ramp_fitting/ramp_fit_step.py

@@ -44,15 +44,15 @@ def create_optional_results_model(input_model, opt_info):
     crmag.dtype = np.float32
 
     inst = {'meta': meta,
-            'slope': np.squeeze(slope),
-            'sigslope': np.squeeze(sigslope),
-            'var_poisson': np.squeeze(var_poisson),
-            'var_rnoise': np.squeeze(var_rnoise),
-            'yint': np.squeeze(yint),
-            'sigyint': np.squeeze(sigyint),
-            'pedestal': np.squeeze(pedestal),
+            'slope': u.Quantity(np.squeeze(slope), ru.electron / u.s, dtype=slope.dtype),
+            'sigslope': u.Quantity(np.squeeze(sigslope), ru.electron / u.s, dtype=sigslope.dtype),
+            'var_poisson': u.Quantity(np.squeeze(var_poisson), ru.electron**2 / u.s**2, dtype=var_poisson.dtype),
+            'var_rnoise': u.Quantity(np.squeeze(var_rnoise), ru.electron**2 / u.s**2, dtype=var_rnoise.dtype),
+            'yint': u.Quantity(np.squeeze(yint), ru.electron, dtype=yint.dtype),
+            'sigyint': u.Quantity(np.squeeze(sigyint), ru.electron, dtype=sigyint.dtype),
+            'pedestal': u.Quantity(np.squeeze(pedestal), ru.electron, dtype=pedestal.dtype),
             'weights': np.squeeze(weights),
-            'crmag': crmag
+            'crmag': u.Quantity(crmag, ru.electron, dtype=pedestal.dtype),
             }
 
     out_node = rds.RampFitOutput(inst)
@@ -82,6 +82,11 @@ def create_image_model(input_model, image_info):
     """
     data, dq, var_poisson, var_rnoise, err = image_info
 
+    data = u.Quantity(data, ru.electron / u.s, dtype=data.dtype)
+    var_poisson = u.Quantity(var_poisson, ru.electron**2 / u.s**2, dtype=var_poisson.dtype)
+    var_rnoise = u.Quantity(var_rnoise, ru.electron**2 / u.s**2, dtype=var_rnoise.dtype)
+    err = u.Quantity(err, ru.electron / u.s, dtype=err.dtype)
+
     # Create output datamodel
     # ... and add all keys from input
     meta = {}
@@ -144,7 +149,6 @@ def process(self, input):
             readnoise_filename = self.get_reference_file(input_model, 'readnoise')
             gain_filename = self.get_reference_file(input_model, 'gain')
             input_model.data = input_model.data[np.newaxis, :]
-            input_model.data.dtype=np.float32
             input_model.groupdq = input_model.groupdq[np.newaxis, :]
             input_model.err = input_model.err[np.newaxis, :]
 

romancal/ramp_fitting/tests/test_ramp_fit.py

@@ -4,7 +4,6 @@
 from astropy.time import Time
 from astropy import units as u
 
-
 from roman_datamodels.datamodels import RampModel, GainRefModel, ReadnoiseRefModel, ImageModel
 from roman_datamodels.testing import utils as testutil
 from roman_datamodels import units as ru
@@ -32,12 +31,10 @@ def generate_ramp_model(shape, deltatime=1):
     gdq = np.zeros(shape=shape, dtype=np.uint8)
 
     dm_ramp = testutil.mk_ramp(shape)
-    dm_ramp.data = data
+    dm_ramp.data = u.Quantity(data, ru.DN, dtype=np.float32)
     dm_ramp.pixeldq = pixdq
     dm_ramp.groupdq = gdq
-    dm_ramp.err = err
-
-    #dm_ramp.meta['photometry'] = testutil.mk_photometry()
+    dm_ramp.err = u.Quantity(err, ru.DN, dtype=np.float32)
 
     dm_ramp.meta.exposure.frame_time = deltatime
     dm_ramp.meta.exposure.ngroups = shape[0]
@@ -183,10 +180,10 @@ def test_saturated_ramp_fit(max_cores):
                          maximum_cores=max_cores)
 
     # Test data and error arrays are zeroed out
-    np.testing.assert_array_equal(out_model.data, 0)
-    np.testing.assert_array_equal(out_model.err, 0)
-    np.testing.assert_array_equal(out_model.var_poisson, 0)
-    np.testing.assert_array_equal(out_model.var_rnoise, 0)
+    np.testing.assert_array_equal(out_model.data.value, 0)
+    np.testing.assert_array_equal(out_model.err.value, 0)
+    np.testing.assert_array_equal(out_model.var_poisson.value, 0)
+    np.testing.assert_array_equal(out_model.var_rnoise.value, 0)
 
     # Test that all pixels are flagged saturated
     assert np.all(np.bitwise_and(out_model.dq, SATURATED) == SATURATED)

romancal/saturation/saturation.py

@@ -34,7 +34,7 @@ def flag_saturation(input_model, ref_model):
         the GROUPDQ array
     """
 
-    data = input_model.data.value[np.newaxis, :]
+    data = input_model.data[np.newaxis, :].value
 
     # Create the output model as a copy of the input
     output_model = input_model

romancal/saturation/tests/test_saturation.py

@@ -178,14 +178,14 @@ def test_dq_propagation(setup_wfi_datamodels):
     dqval1 = 5
     dqval2 = 10
 
-    data, satmap = setup_wfi_datamodels(ngroups, nrows, ncols)
+    ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols)
 
     # Add DQ values to the data and reference file
-    data.pixeldq[5, 5] = dqval1
+    ramp.pixeldq[5, 5] = dqval1
     satmap.dq[5, 5] = dqval2
 
     # Run the pipeline
-    output = flag_saturation(data, satmap)
+    output = flag_saturation(ramp, satmap)
 
     # Make sure DQ values from data and reference file are added in the output
     assert output.pixeldq[5, 5] == dqval1 + dqval2