adding source detection step

spacetelescope · Mar 16, 2023 · ccbed39 · ccbed39
1 parent ef8d671
commit ccbed39
Show file tree

Hide file tree

Showing 10 changed files with 576 additions and 4 deletions.
diff --git a/docs/roman/package_index.rst b/docs/roman/package_index.rst
@@ -13,3 +13,4 @@
    references_general/index.rst
    stpipe/index.rst
    refpix/index.rst
+   source_detection/index.rst
diff --git a/docs/roman/source_detection/arguments.rst b/docs/roman/source_detection/arguments.rst
@@ -0,0 +1,55 @@
+Arguments
+=========
+The source detection fitting step has several arguments. These can be specified
+by the user by passing them to the step in a Python session, or setting them
+in a parameter file.
+
+* ``--coverage_mask``: True/False to mask pixels with DO_NOT_USE flags in their
+                       dq array. Default is True.
+* ``--kernel_fwhm``: A parameter for DAOStarFinder: size of Gaussian kernel in
+                     pixels. Default is 2.0.
+* ``--sharplo``: A parameter for DAOStarFinder: lower bound for sharpness.
+                 Default is 0.0.
+* ``--sharphi``: A parameter for DAOStarFinder: upper bound for sharpness.
+                 Default is 1.0.
+* ``--roundlo``: A parameter for DAOStarFinder: lower bound for roundness.
+                 Default is -1.0.
+* ``--roundhi``: A parameter for DAOStarFinder: upper bound for roundness.
+                 Default is 1.0.
+* ``--peakmax``: A parameter for DAOStarFinder: upper limit on brightest pixel
+                 in sources. Default is 1000.0.
+* ``--max_sources``: The maximum number of sources in the output catalog,
+                     choosing brightest. Default is None, which will return all
+                     detected sources.
+* ``--scalar_threshold``: If specified, a single scalar threshold to be used
+                          for source detection with DAOStarFinder across the
+                          entire image. One of `scalar_threshold`,
+                          `calc_scalar_threshold` must be chosen. Default is
+                          None.
+* ``--calc_scalar_threshold``: If specified, a single scalar threshold will be
+                               determined for the entire image. This is done by
+                               calculating a 2D background image, and using that
+                               to determine a single threshold value for the
+                               entire image. One of `scalar_threshold` or
+                               `calc_scalar_threshold` must be chosen.
+                               must be chosen. Default is True.
+* ``--snr_threshold``: If using `calc_threshold_img`, the SNR for the threshold
+                       image. Default is 3.0.
+* ``--bkg_estimator``: If using `calc_threshold_img`, choice of mean, median, or
+                        mode. Default is median.
+* ``--bkg_boxsize``: If using `calc_threshold_img` size of box in pixels for
+                     2D background / threshold images and if using
+                     calc_threshold_2d, the size of the box used when detecting
+                     sources. Default is 3.
+* ``--bkg_sigma``: If using `calc_threshold_img`, n sigma for sigma clipping
+                   for background calculation. Default is 2.0.
+* ``--bkg_filter_size``: If using `calc_threshold_img` or `calc_threshold_2d`,
+                         size of square gaussian kernel for background
+                         calculation. Default is 3.
+* ``--save_catalogs``: A True/False value that specifies whether to write
+                      the optional output catalog. Default is False.
+* ``--output_cat_filetype``: If `save_catalogs` is True, file type of output
+                             catalog from choice of asdf and escv. Catalog
+                             will be saved as a numpy array with four dimensions.
+                             In order, these represent source ID, x centroid
+                             position, y centroid position, and flux.
diff --git a/docs/roman/source_detection/description.rst b/docs/roman/source_detection/description.rst
@@ -0,0 +1,62 @@
+Description
+============
+
+The source detection step produces catalogs of point-like sources for use by the
+Tweakreg step for image alignment. It uses DAOStarFinder to detect point sources
+in the image.
+
+
+Outputs / Returns
+=================
+
+By default, the resulting source catalog will be temporarily attached to the
+output ImageModel in the `meta.source_catalog.tweakreg_catalog` attribute as 4D
+numpy array representing, in order, source ID, x centroid position, y centroid
+positon, and flux. This catalog will then be deleted from the model in the
+Tweakreg step.
+
+Optionally, the catalog can be saved to disk in which case a
+`meta.source_catalog.tweakreg_catalog_name` attribute will be added to the file
+to point Tweakreg to the catalog on disk. To do this, set `save_catalogs` to
+True. Output catalogs will be saved in the same directory as input files, and
+are also 4D numpy arrays representing, in order, source ID, x centroid position,
+y centroid positon, and flux. Output catalogs can be in ASDF or ECSV format.
+
+** NOTE: **The intermediate resulting ImageModel from SourceDetectionStep can
+only be saved if it does not contain an attached catalog - to do this, use the
+`save_catalogs` option to seperate the catalog from the file and save them
+seperately.
+
+Options for Thresholding
+========================
+
+The DAOStarFinder routine detects point-like sources in in image that are above
+a certain, specified floating point threshold. This step provides several options
+for calculating this threshold, either using one value for the entire image,
+or by detecting sources in segments of the image and using a different appropriate
+threshold for each (useful if background varies across the image).
+
+The first option is to set `scalar_threshold` - this will use the specified
+threshold as the detection threshold for the entire image.
+
+The second option is to use `calc_threshold` - this will calculate a single
+threshold value for the entire image based on the sigma-clipped average
+(mean, median, or mode) background level of the whole image.
+
+Other Options
+=============
+
+Masking DO_NOT_USE pixels
+-------------------------
+
+By default, sources that fall on pixels with DO_NOT_USE in their DQ array
+will be excluded from the final output catalog. This can be controlled with
+the `coverage_mask` parameter.
+
+Limiting maximum number of sources
+----------------------------------
+
+By default, all detected sources will be returned in the final output catalog.
+If you wish to limit the number of sources, this can be done with the
+`max_sources` argument, which will sort the output catalog by flux and return
+only the N brightest.
diff --git a/docs/roman/source_detection/index.rst b/docs/roman/source_detection/index.rst
@@ -0,0 +1,13 @@
+.. _source_detection_step:
+
+================
+Source Detection
+================
+
+.. toctree::
+   :maxdepth: 2
+
+   description.rst
+   arguments.rst
+
+.. automodapi:: romancal.source_detection
diff --git a/pyproject.toml b/pyproject.toml
@@ -18,11 +18,12 @@ dependencies = [
     'gwcs >=0.18.1',
     'jsonschema >=3.0.2',
     'numpy >=1.20',
+    'photutils >= 1.6.0',
     'pyparsing >=2.4.7',
     'requests >=2.22',
-    'rad >=0.14.1',
+    'rad @git+https://github.com/cshanahan1/rad.git@source_cat',
     #'roman_datamodels >=0.14.1',
-    'roman_datamodels @git+https://github.com/spacetelescope/roman_datamodels.git@main',
+    'roman_datamodels @git+https://github.com/cshanahan1/roman_datamodels.git@source_detection_new',
     'stcal >=1.3.3',
     'stpipe >=0.4.2',
     'tweakwcs >=0.8.0'
@@ -118,7 +119,7 @@ results_root = 'roman-pipeline-results'
 doctest_plus = 'enabled'
 doctest_rst = 'enabled'
 text_file_format = 'rst'
-addopts = '--show-capture=no --open-files --doctest-ignore-import-errors'
+#addopts = '--show-capture=no --open-files --doctest-ignore-import-errors'
 markers = [
     'soctests: run only the SOC tests in the suite.',
 ]

diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py
@@ -16,6 +16,7 @@
 from romancal.dark_current import DarkCurrentStep
 from romancal.linearity import LinearityStep
 from romancal.photom import PhotomStep
+from romancal.source_detection import SourceDetectionStep
 from romancal.ramp_fitting import ramp_fit_step
 from romancal.saturation import SaturationStep
 
@@ -52,6 +53,7 @@ class ExposurePipeline(RomanPipeline):
                  'assign_wcs': AssignWcsStep,
                  'flatfield': FlatFieldStep,
                  'photom': PhotomStep,
+                 'source_detection': SourceDetectionStep
                  }
 
     # start the actual processing
@@ -109,6 +111,8 @@ def process(self, input):
             result.meta.cal_step.flat_field = 'SKIPPED'
         result = self.photom(result)
 
+        result = self.source_detection(result)
+
         # setup output_file for saving
         self.setup_output(result)
         log.info('Roman exposure calibration pipeline ending...')
@@ -145,7 +149,7 @@ def create_fully_saturated_zeroed_image(self, input_model):
 
         # Set all subsequent steps to skipped
         for step_str in ['linearity', 'dark', 'jump', 'ramp_fit', 'assign_wcs',
-                         'flat_field', 'photom']:
+                         'flat_field', 'photom', 'source_detection']:
             fully_saturated_model.meta.cal_step[step_str] = 'SKIPPED'
 
         # Set suffix for proper output naming

diff --git a/romancal/source_detection/__init__.py b/romancal/source_detection/__init__.py
@@ -0,0 +1,3 @@
+from .source_detection_step import SourceDetectionStep
+
+__all__ = ['SourceDetectionStep']
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		from .source_detection_step import SourceDetectionStep

		__all__ = ['SourceDetectionStep']