Rcal 854 Tweakreg step listed as skipped when run & refactor exposure…

… pipeline (spacetelescope#1271)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

CHANGES.rst

@@ -10,6 +10,8 @@ Documentation
 general
 -------
 
+- refactor exposure level pipeline to use asn's and ModelContainer [#1271]
+
 - Add catalog source step to the mosaic pipeline [#1266]
 
 - Rename highlevelpipeline to mosaic pipeline [#1249]

docs/roman/pipeline/exposure_pipeline.rst

@@ -61,10 +61,15 @@ Inputs
 :Data model: `~romancal.datamodels.RampModel`
 :File suffix: _uncal
 
-The input to the ``ExposurePipeline`` is a single raw exposure,
+The input to the ``ExposurePipeline`` can be a single raw exposure,
 e.g. "r0008308002010007027_06311_0019_WFI01_uncal.asdf", which contains the
 original raw data from all of the detector readouts in the exposure
-( ngroups x ncols x nrows ).
+( ngroups x ncols x nrows ). The raw data may also be input using an association file.
+
+If the ``ExposurePipeline`` is given a single file the final alignment to Gaia will be done
+with the sources found in the exposure. If multiple exposures exist in the association file
+then the final alignment will use all the sources found in the exposures
+(see :ref:`tweakreg <tweakreg_step>`).
 
 Note that in the operational environment, the
 input will be in the form of a `~romancal.datamodels.RawScienceModel`, which only

romancal/pipeline/exposure_pipeline.py

@@ -10,9 +10,7 @@
 
 # step imports
 from romancal.assign_wcs import AssignWcsStep
-
-# from romancal.associations.exceptions import AssociationNotValidError
-# from romancal.associations.load_as_asn import LoadAsLevel2Asn
+from romancal.associations.asn_from_list import asn_from_list
 from romancal.dark_current import DarkCurrentStep
 from romancal.datamodels import ModelContainer
 from romancal.dq_init import dq_init_step
@@ -78,29 +76,28 @@ def process(self, input):
 
         # open the input file
         file_type = filetype.check(input)
-        asn = None
+        if file_type == "asn":
+            asn = ModelContainer.read_asn(input)
+
         if file_type == "asdf":
             try:
-                input = rdm.open(input)
+                # set the product name based on the input filename
+                asn = asn_from_list([input], product_name=input_filename.split(".")[0])
+                file_type = "asn"
             except TypeError:
                 log.debug("Error opening file:")
                 return
 
-        if file_type == "asn":
-            asn = ModelContainer.read_asn(input)
-
         # Build a list of observations to process
         expos_file = []
         n_members = 0
-        if file_type == "asdf":
-            expos_file = [input]
-        elif file_type == "asn":
-            for product in asn["products"]:
-                n_members = len(product["members"])
-                for member in product["members"]:
-                    expos_file.append(member["expname"])
-
-        results = []
+        # extract the members from the asn to run the files through the steps
+        for product in asn["products"]:
+            n_members = len(product["members"])
+            for member in product["members"]:
+                expos_file.append(member["expname"])
+
+        results = ModelContainer()
         tweakreg_input = ModelContainer()
         for in_file in expos_file:
             if isinstance(in_file, str):
@@ -121,6 +118,7 @@ def process(self, input):
 
             # Test for fully saturated data
             if is_fully_saturated(result):
+                # Return fully saturated image file (stopping pipeline)
                 log.info("All pixels are saturated. Returning a zeroed-out image.")
 
                 #    if is_fully_saturated(result):
@@ -142,7 +140,6 @@ def process(self, input):
                 # Set suffix for proper output naming
                 self.suffix = "cal"
                 results.append(result)
-                # Return fully saturated image file (stopping pipeline)
                 return results
 
             result = self.refpix(result)
@@ -155,13 +152,10 @@ def process(self, input):
                 result = self.flatfield(result)
                 result = self.photom(result)
                 result = self.source_detection(result)
-                if file_type == "asn":
-                    result.meta.cal_step.tweakreg = "SKIPPED"
-                    self.suffix = "sourcedetection"
-                    tweakreg_input.append(result)
-                    log.info(
-                        f"Number of models to tweakreg:   {len(tweakreg_input._models), n_members}"
-                    )
+                tweakreg_input.append(result)
+                log.info(
+                    f"Number of models to tweakreg:   {len(tweakreg_input._models), n_members}"
+                )
             else:
                 log.info("Flat Field step is being SKIPPED")
                 log.info("Photom step is being SKIPPED")
@@ -173,10 +167,6 @@ def process(self, input):
                 result.meta.cal_step.tweakreg = "SKIPPED"
                 self.suffix = "cal"
 
-            # setup output_file for saving
-            if result.meta.cal_step.tweakreg == "COMPLETE":
-                self.suffix = "cal"
-
             self.setup_output(result)
 
             self.output_use_model = True
@@ -185,45 +175,15 @@ def process(self, input):
         # Now that all the exposures are collated, run tweakreg
         # Note: this does not cover the case where the asn mixes imaging and spectral
         #          observations. This should not occur on-prem
-        if result.meta.exposure.type == "WFI_IMAGE":
-            if file_type == "asdf":
-                result.meta.cal_step.tweakreg = "SKIPPED"
-                mc_result = ModelContainer(result)
-                if hasattr(ModelContainer(result), "_models") and mc_result._models:
-                    result = mc_result._models.pop()
+        result = self.tweakreg(results)
 
-            if file_type == "asn":
-                result = self.tweakreg(tweakreg_input)
-                for model in result._models:
-                    model.meta.cal_step.tweakreg = "SKIPPED"
-
-            log.info("Roman exposure calibration pipeline ending...")
+        log.info("Roman exposure calibration pipeline ending...")
 
         return results
 
     def setup_output(self, input):
         """Determine the proper file name suffix to use later"""
-        if input.meta.cal_step.tweakreg == "COMPLETE":
-            self.suffix = "cal"
-        # elif input.meta.cal_step.tweakreg == "SKIPPED" and input.meta.exposure.type == "WFI_IMAGE":
-        #    self.suffix = "sourcedetection"
-        elif (
-            input.meta.cal_step.tweakreg == "INCOMPLETE"
-            and input.meta.exposure.type == "WFI_IMAGE"
-        ):
-            self.suffix = "sourcedetection"
-        elif (
-            input.meta.cal_step.tweakreg == "SKIPPED"
-            and input.meta.exposure.type != "WFI_IMAGE"
-        ):
-            self.suffix = "cal"
-
-        if not self.steps["tweakreg"]["skip"]:
-            self.suffix = "cal"
-
-        input.meta.filename = input.meta.filename.replace("uncal", self.suffix)
-        input["output_file"] = input.meta.filename
-        self.output_file = input.meta.filename
+        self.suffix = "cal"
 
     def create_fully_saturated_zeroed_image(self, input_model):
         """