diff --git a/CHANGES.rst b/CHANGES.rst index d92baa68b..c30af399d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,8 @@ Documentation general ------- +- refactor exposure level pipeline to use asn's and ModelContainer [#1271] + - Add catalog source step to the mosaic pipeline [#1266] - Rename highlevelpipeline to mosaic pipeline [#1249] diff --git a/docs/roman/pipeline/exposure_pipeline.rst b/docs/roman/pipeline/exposure_pipeline.rst index 1c412085e..50b354494 100644 --- a/docs/roman/pipeline/exposure_pipeline.rst +++ b/docs/roman/pipeline/exposure_pipeline.rst @@ -61,10 +61,15 @@ Inputs :Data model: `~romancal.datamodels.RampModel` :File suffix: _uncal -The input to the ``ExposurePipeline`` is a single raw exposure, +The input to the ``ExposurePipeline`` can be a single raw exposure, e.g. "r0008308002010007027_06311_0019_WFI01_uncal.asdf", which contains the original raw data from all of the detector readouts in the exposure -( ngroups x ncols x nrows ). +( ngroups x ncols x nrows ). The raw data may also be input using an association file. + +If the ``ExposurePipeline`` is given a single file the final alignment to Gaia will be done +with the sources found in the exposure. If multiple exposures exist in the association file +then the final alignment will use all the sources found in the exposures +(see :ref:`tweakreg `). Note that in the operational environment, the input will be in the form of a `~romancal.datamodels.RawScienceModel`, which only diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py index bb67a7a97..c6a6ecc84 100644 --- a/romancal/pipeline/exposure_pipeline.py +++ b/romancal/pipeline/exposure_pipeline.py @@ -10,9 +10,7 @@ # step imports from romancal.assign_wcs import AssignWcsStep - -# from romancal.associations.exceptions import AssociationNotValidError -# from romancal.associations.load_as_asn import LoadAsLevel2Asn +from romancal.associations.asn_from_list import asn_from_list from romancal.dark_current import DarkCurrentStep from romancal.datamodels import ModelContainer from romancal.dq_init import dq_init_step @@ -78,29 +76,28 @@ def process(self, input): # open the input file file_type = filetype.check(input) - asn = None + if file_type == "asn": + asn = ModelContainer.read_asn(input) + if file_type == "asdf": try: - input = rdm.open(input) + # set the product name based on the input filename + asn = asn_from_list([input], product_name=input_filename.split(".")[0]) + file_type = "asn" except TypeError: log.debug("Error opening file:") return - if file_type == "asn": - asn = ModelContainer.read_asn(input) - # Build a list of observations to process expos_file = [] n_members = 0 - if file_type == "asdf": - expos_file = [input] - elif file_type == "asn": - for product in asn["products"]: - n_members = len(product["members"]) - for member in product["members"]: - expos_file.append(member["expname"]) - - results = [] + # extract the members from the asn to run the files through the steps + for product in asn["products"]: + n_members = len(product["members"]) + for member in product["members"]: + expos_file.append(member["expname"]) + + results = ModelContainer() tweakreg_input = ModelContainer() for in_file in expos_file: if isinstance(in_file, str): @@ -121,6 +118,7 @@ def process(self, input): # Test for fully saturated data if is_fully_saturated(result): + # Return fully saturated image file (stopping pipeline) log.info("All pixels are saturated. Returning a zeroed-out image.") # if is_fully_saturated(result): @@ -142,7 +140,6 @@ def process(self, input): # Set suffix for proper output naming self.suffix = "cal" results.append(result) - # Return fully saturated image file (stopping pipeline) return results result = self.refpix(result) @@ -155,13 +152,10 @@ def process(self, input): result = self.flatfield(result) result = self.photom(result) result = self.source_detection(result) - if file_type == "asn": - result.meta.cal_step.tweakreg = "SKIPPED" - self.suffix = "sourcedetection" - tweakreg_input.append(result) - log.info( - f"Number of models to tweakreg: {len(tweakreg_input._models), n_members}" - ) + tweakreg_input.append(result) + log.info( + f"Number of models to tweakreg: {len(tweakreg_input._models), n_members}" + ) else: log.info("Flat Field step is being SKIPPED") log.info("Photom step is being SKIPPED") @@ -173,10 +167,6 @@ def process(self, input): result.meta.cal_step.tweakreg = "SKIPPED" self.suffix = "cal" - # setup output_file for saving - if result.meta.cal_step.tweakreg == "COMPLETE": - self.suffix = "cal" - self.setup_output(result) self.output_use_model = True @@ -185,45 +175,15 @@ def process(self, input): # Now that all the exposures are collated, run tweakreg # Note: this does not cover the case where the asn mixes imaging and spectral # observations. This should not occur on-prem - if result.meta.exposure.type == "WFI_IMAGE": - if file_type == "asdf": - result.meta.cal_step.tweakreg = "SKIPPED" - mc_result = ModelContainer(result) - if hasattr(ModelContainer(result), "_models") and mc_result._models: - result = mc_result._models.pop() + result = self.tweakreg(results) - if file_type == "asn": - result = self.tweakreg(tweakreg_input) - for model in result._models: - model.meta.cal_step.tweakreg = "SKIPPED" - - log.info("Roman exposure calibration pipeline ending...") + log.info("Roman exposure calibration pipeline ending...") return results def setup_output(self, input): """Determine the proper file name suffix to use later""" - if input.meta.cal_step.tweakreg == "COMPLETE": - self.suffix = "cal" - # elif input.meta.cal_step.tweakreg == "SKIPPED" and input.meta.exposure.type == "WFI_IMAGE": - # self.suffix = "sourcedetection" - elif ( - input.meta.cal_step.tweakreg == "INCOMPLETE" - and input.meta.exposure.type == "WFI_IMAGE" - ): - self.suffix = "sourcedetection" - elif ( - input.meta.cal_step.tweakreg == "SKIPPED" - and input.meta.exposure.type != "WFI_IMAGE" - ): - self.suffix = "cal" - - if not self.steps["tweakreg"]["skip"]: - self.suffix = "cal" - - input.meta.filename = input.meta.filename.replace("uncal", self.suffix) - input["output_file"] = input.meta.filename - self.output_file = input.meta.filename + self.suffix = "cal" def create_fully_saturated_zeroed_image(self, input_model): """