From 9200cdacfc71b3b77cc3396f14a92ad7504428f4 Mon Sep 17 00:00:00 2001 From: Clare Shanahan Date: Tue, 6 Dec 2022 15:05:40 -0500 Subject: [PATCH] adding source detection step --- CHANGES.rst | 6 + docs/roman/package_index.rst | 1 + docs/roman/source_detection/arguments.rst | 57 ++++ docs/roman/source_detection/description.rst | 55 ++++ docs/roman/source_detection/index.rst | 13 + pyproject.toml | 7 + romancal/lib/suffix.py | 1 + romancal/pipeline/exposure_pipeline.py | 4 + romancal/source_detection/__init__.py | 3 + .../source_detection/source_detection_step.py | 238 +++++++++++++++++ romancal/source_detection/tests/__init__.py | 0 .../tests/test_source_detection_step.py | 252 ++++++++++++++++++ 12 files changed, 637 insertions(+) create mode 100644 docs/roman/source_detection/arguments.rst create mode 100644 docs/roman/source_detection/description.rst create mode 100644 docs/roman/source_detection/index.rst create mode 100644 romancal/source_detection/__init__.py create mode 100644 romancal/source_detection/source_detection_step.py create mode 100644 romancal/source_detection/tests/__init__.py create mode 100644 romancal/source_detection/tests/test_source_detection_step.py diff --git a/CHANGES.rst b/CHANGES.rst index b913bd8aa..28fbef5db 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,9 +5,15 @@ general ------- - Updated datamodel maker utility imports. [#654] +<<<<<<< HEAD - Update non-VOunits to using ``astropy.units``. [#658] - update minimum version of ``asdf`` to ``2.14.2`` and ``jsonschema`` to ``4.0.1`` and added minimum dependency checks to CI [#664] +======= +source_detection +---------------- +- Added SourceDetection Step to pipeline [#608] +>>>>>>> 05ff1e7 (adding source detection step) 0.10.0 (2023-02-21) =================== diff --git a/docs/roman/package_index.rst b/docs/roman/package_index.rst index 98b4b7d50..86117ac62 100644 --- a/docs/roman/package_index.rst +++ b/docs/roman/package_index.rst @@ -13,3 +13,4 @@ references_general/index.rst stpipe/index.rst refpix/index.rst + source_detection/index.rst diff --git a/docs/roman/source_detection/arguments.rst b/docs/roman/source_detection/arguments.rst new file mode 100644 index 000000000..edffce08e --- /dev/null +++ b/docs/roman/source_detection/arguments.rst @@ -0,0 +1,57 @@ +Arguments +========= +The source detection fitting step has several arguments. These can be specified +by the user by passing them to the step in a Python session, or setting them +in a parameter file. + +* ``--kernel_fwhm``: A parameter for DAOStarFinder: size of Gaussian kernel in + pixels. Default is 2.0. +* ``--sharplo``: A parameter for DAOStarFinder: lower bound for sharpness. + Default is 0.0. +* ``--sharphi``: A parameter for DAOStarFinder: upper bound for sharpness. + Default is 1.0. +* ``--roundlo``: A parameter for DAOStarFinder: lower bound for roundness. + Default is -1.0. A circular source will have a zero roundness. + A source extended in x or y will have a negative or positive + roundness, respectively. +* ``--roundhi``: A parameter for DAOStarFinder: upper bound for roundness. + Default is 1.0. A circular source will have a zero roundness. + A source extended in x or y will have a negative or positive + roundness, respectively. +* ``--peakmax``: A parameter for DAOStarFinder: upper limit on brightest pixel + in sources. Default is 1000.0. +* ``--max_sources``: The maximum number of sources in the output catalog, + choosing brightest. Default is None, which will return all + detected sources. +* ``--scalar_threshold``: If specified, the absolute detection threshold to be + used for the entire image. Units are assumed to be the + same as input data. One of `scalar_threshold`, + `calc_scalar_threshold` must be chosen. Default is + None. +* ``--calc_scalar_threshold``: If specified, a single scalar threshold will be + determined for the entire image. This is done by + calculating a 2D background image, and using that + to determine a single threshold value for the + entire image. One of `scalar_threshold` or + `calc_scalar_threshold` must be chosen. + must be chosen. Default is True. +* ``--snr_threshold``: If using `calc_threshold_img`, the SNR for the threshold + image. Default is 3.0. +* ``--bkg_estimator``: If using `calc_threshold_img`, choice of mean, median, or + mode. Default is median. +* ``--bkg_boxsize``: If using `calc_threshold_img` size of box in pixels for + 2D background / threshold images and if using + calc_threshold_2d, the size of the box used when detecting + sources. Default is 3. +* ``--bkg_sigma``: If using `calc_threshold_img`, n sigma for sigma clipping + for background calculation. Default is 2.0. +* ``--bkg_filter_size``: If using `calc_threshold_img` or `calc_threshold_2d`, + size of square gaussian kernel for background + calculation. Default is 3. +* ``--save_catalogs``: A True/False value that specifies whether to write + the optional output catalog. Default is False. +* ``--output_cat_filetype``: If `save_catalogs` is True, file type of output + catalog from choice of asdf and escv. Catalog + will be saved as a numpy array with four dimensions. + In order, these represent source ID, x centroid + position, y centroid position, and flux. diff --git a/docs/roman/source_detection/description.rst b/docs/roman/source_detection/description.rst new file mode 100644 index 000000000..2f9dfb95a --- /dev/null +++ b/docs/roman/source_detection/description.rst @@ -0,0 +1,55 @@ +Description +============ + +The source detection step produces catalogs of point-like sources for use by the +Tweakreg step for image alignment. It uses DAOStarFinder to detect point sources +in the image. + + +Outputs / Returns +================= + +By default, the resulting source catalog will be temporarily attached to the +output ImageModel in the `meta.source_catalog.tweakreg_catalog` attribute as 4D +numpy array representing, in order, source ID, x centroid position, y centroid +positon, and flux. This catalog will then be deleted from the model in the +Tweakreg step. + +Optionally, the catalog can be saved to disk in which case a +`meta.source_catalog.tweakreg_catalog_name` attribute will be added to the file +to point Tweakreg to the catalog on disk. To do this, set `save_catalogs` to +True. Output catalogs will be saved in the same directory as input files, and +are also 4D numpy arrays representing, in order, source ID, x centroid position, +y centroid positon, and flux. Output catalogs can be in ASDF or ECSV format. + +NOTE: The intermediate resulting ImageModel from SourceDetectionStep can +only be saved if it does not contain an attached catalog - to do this, use the +`save_catalogs` option to seperate the catalog from the file and save them +separately. + +Options for Thresholding +======================== + +The DAOStarFinder routine detects point-like sources in in image that are above +a certain, specified floating point threshold. This step provides several options +for calculating this threshold, either using one value for the entire image, +or by detecting sources in segments of the image and using a different appropriate +threshold for each (useful if background varies across the image). + +The first option is to set `scalar_threshold` - this will use the specified +threshold as the detection threshold for the entire image. + +The second option is to use `calc_threshold` - this will calculate a single +threshold value for the entire image based on the sigma-clipped average +(mean, median, or mode) background level of the whole image. + +Other Options +============= + +Limiting maximum number of sources +---------------------------------- + +By default, all detected sources will be returned in the final output catalog. +If you wish to limit the number of sources, this can be done with the +`max_sources` argument, which will sort the output catalog by flux and return +only the N brightest. diff --git a/docs/roman/source_detection/index.rst b/docs/roman/source_detection/index.rst new file mode 100644 index 000000000..2dc35264f --- /dev/null +++ b/docs/roman/source_detection/index.rst @@ -0,0 +1,13 @@ +.. _source_detection_step: + +================ +Source Detection +================ + +.. toctree:: + :maxdepth: 2 + + description.rst + arguments.rst + +.. automodapi:: romancal.source_detection diff --git a/pyproject.toml b/pyproject.toml index 2db4234dc..7bdc89271 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,12 +19,19 @@ dependencies = [ 'gwcs >=0.18.1', 'jsonschema >=4.0.1', 'numpy >=1.20', + 'photutils >= 1.6.0', 'pyparsing >=2.4.7', 'requests >=2.22', +<<<<<<< HEAD # 'rad >=0.14.1', 'rad @ git+https://github.com/spacetelescope/rad.git@main', #'roman_datamodels >=0.14.1', 'roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git@main', +======= + 'rad @git+https://github.com/cshanahan1/rad.git@source_cat', + #'roman_datamodels >=0.14.1', + 'roman_datamodels @git+https://github.com/cshanahan1/roman_datamodels.git@source_detection_new', +>>>>>>> 05ff1e7 (adding source detection step) 'stcal >=1.3.3', 'stpipe >=0.4.2', 'tweakwcs >=0.8.0' diff --git a/romancal/lib/suffix.py b/romancal/lib/suffix.py index 02ddaefe7..517da0f0d 100644 --- a/romancal/lib/suffix.py +++ b/romancal/lib/suffix.py @@ -82,6 +82,7 @@ "linearitystep", "dark_current", "jump", + "sourcedetectionstep", "tweakregstep", } diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py index 6e0bc2b88..0f487cbd3 100644 --- a/romancal/pipeline/exposure_pipeline.py +++ b/romancal/pipeline/exposure_pipeline.py @@ -17,6 +17,7 @@ from romancal.photom import PhotomStep from romancal.ramp_fitting import ramp_fit_step from romancal.saturation import SaturationStep +from romancal.source_detection import SourceDetectionStep from ..stpipe import RomanPipeline @@ -53,6 +54,7 @@ class ExposurePipeline(RomanPipeline): "assign_wcs": AssignWcsStep, "flatfield": FlatFieldStep, "photom": PhotomStep, + "source_detection": SourceDetectionStep, } # start the actual processing @@ -110,6 +112,8 @@ def process(self, input): result.meta.cal_step.flat_field = "SKIPPED" result = self.photom(result) + result = self.source_detection(result) + # setup output_file for saving self.setup_output(result) log.info("Roman exposure calibration pipeline ending...") diff --git a/romancal/source_detection/__init__.py b/romancal/source_detection/__init__.py new file mode 100644 index 000000000..7b7af9b06 --- /dev/null +++ b/romancal/source_detection/__init__.py @@ -0,0 +1,3 @@ +from .source_detection_step import SourceDetectionStep + +__all__ = ["SourceDetectionStep"] diff --git a/romancal/source_detection/source_detection_step.py b/romancal/source_detection/source_detection_step.py new file mode 100644 index 000000000..12966848a --- /dev/null +++ b/romancal/source_detection/source_detection_step.py @@ -0,0 +1,238 @@ +""" +Create a source catalog for tweakreg +""" + + +import logging + +import numpy as np +from asdf import AsdfFile +from astropy.stats import SigmaClip +from astropy.table import Table +from photutils.background import ( + Background2D, + MeanBackground, + MedianBackground, + ModeEstimatorBackground, +) +from photutils.detection import DAOStarFinder +from roman_datamodels import datamodels as rdd + +from romancal.lib import dqflags +from romancal.stpipe import RomanStep + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +__all__ = ["SourceDetectionStep"] + + +class SourceDetectionStep(RomanStep): + """ + SourceDetectionStep: Detect point-like sources in image to create a catalog + for alignment in tweakreg. + """ + + spec = """ + kernel_fwhm = float(default=2.) # DAOStarFinder:Size of Gaussian kernel, + # in pixels. + sharplo = float(default=0.) # DAOStarFinder: Lower bound for sharpness. + sharphi = float(default=1.0) # DAOStarFinder: Upper bound for sharpness. + roundlo = float(default=-1.0) # DAOStarFinder: Lower bound for roundness. + # A circular source will have a zero roundness. A source extended in x or + # y will have a negative or positive roundness, respectively. + roundhi = float(default=1.0) # DAOStarFinder: Upper bound for roundness. + peakmax = float(default=1000.0) # Upper limit on brightest pixel in sources. + max_sources = float(default=None) # Max number of sources, choosing brightest. + scalar_threshold = float(default=None) # Detection threshold, to + # be used for entire image. Assumed to be in same units as data, and is + # an absolute threshold over background. + calc_threshold = boolean(default=True) # Calculate a single absoulte + # detection threshold from image based on background. + snr_threshold = float(default=3.0) # if calc_threshold_img or + # calc_threshold_2d: the SNR for the threshold image. + bkg_estimator = string(default='median') # if calc_threshold_img or + # calc_threshold_2d: choice of mean, median, or mode. + bkg_boxsize = integer(default=3) # if calc_threshold_img or + # calc_threshold_2d: size of box in pixels for 2D background. + bkg_sigma = float(default=2.0) # if calc_threshold_img or + # calc_threshold_2d, n sigma for sigma clipping bkgrnd. + bkg_filter_size = integer(default=3) # if calc_threshold_img or + # calc_threshold_2d, size of gauss. kernel for background. + save_catalogs = boolean(default=False) # Save source catalog to file? + # Will overwrite an existing catalog of the same name. + output_cat_filetype = option('asdf', 'ecsv', default='asdf') # Used if + #save_catalogs=True - file type of output catalog. + """ + + def process(self, input): + + with rdd.open(input) as input_model: + + # remove units from data in this step. + # DAOStarFinder requires unitless input + if hasattr(input_model.data, "unit"): + self.data = input_model.data.value + else: + self.data = input_model.data + + # mask DO_NOT_USE pixels + + self.coverage_mask = ( + (dqflags.pixel["DO_NOT_USE"]) & input_model.dq + ).astype(bool) + + + # if a pre-determined threshold value for detection for the whole + # image is provided, use this + if self.scalar_threshold is not None: + threshold = float(self.scalar_threshold) + log.info(f"Using a detection threshold of {threshold}.") + + # otherwise, if specified, calculate a scalar threshold from the + # image by calculating a 2D background image, using this to create + # a 2d threshold image, and using the median of the 2d threshold + # image as the scalar detection threshold for the whole image + elif self.calc_threshold is not None: + log.info("Determining detection threshold from image.") + bkg = self._calc_2D_background() + threshold_img = bkg.background + self.snr_threshold * bkg.background_rms + threshold = np.median(threshold_img) + log.info(f"Calculated a detection threshold of {threshold} from image.") + + log.info("Detecting sources with DAOFind, using entire image array.") + daofind = DAOStarFinder( + fwhm=self.kernel_fwhm, + threshold=threshold, + sharplo=self.sharplo, + sharphi=self.sharphi, + roundlo=self.roundlo, + roundhi=self.roundhi, + brightest=self.max_sources, + peakmax=self.peakmax, + ) + + if self.scalar_threshold is not None: + # if absolute threshold is provided + sources = daofind(self.data, mask=self.coverage_mask) + + elif self.calc_threshold is not None: + # subtrack background from data if calculating abs. threshold + sources = daofind(self.data - bkg.background, + mask=self.coverage_mask) + + # reduce table to minimal number of columns, just source ID, + # positions, and fluxes + columns = ["id", "xcentroid", "ycentroid", "flux"] + + if sources: + catalog = sources[columns] + log.info(f"Found {len(catalog)} sources.") + else: + # if no sources were detected, return an empty table + self.log.warning("No sources detected, returning empty catalog.") + catalog = Table( + names=columns, dtype=(int, np.float64, np.float64, np.float64) + ) + + # attach source catalog to output model as array in meta.source_detecion + # the table will be stored as a 1D array with the four columns + # concatenated, in order, with units attached + catalog_as_array = np.array( + [ + catalog["id"].value, + catalog["xcentroid"].value, + catalog["ycentroid"].value, + catalog["flux"].value, + ] + ) + + # create meta.source detection section in file + # if save_catalogs is True, this will be updated with the + # attribute 'tweakreg_catalog_name' to point to the location + # of the catalog on disk. If save_catalogs is false, this section + # will be updated to contain the catalog to pass to TweakReg + + # tweakreg_catalog_name will be saved to the final output file, + # while tweakreg_catalog is intended to be deleted by TweakRegStep + input_model.meta["source_detection"] = {} + + # if 'save_catalogs'= True, also save the output catalog to a file + # (format specified by output_cat_filetype) and add an attribute + # to the file that contains the path to this file + if self.save_catalogs: + cat_filename = input_model.meta.filename.replace(".asdf", "") + cat_filename += f"_tweakreg_catalog.{self.output_cat_filetype}" + log.info(f"Saving catalog to file: {cat_filename}.") + + if self.output_cat_filetype == "asdf": + tree = {"tweakreg_catalog": catalog_as_array} + ff = AsdfFile(tree) + ff.write_to(cat_filename) + else: + catalog.write(cat_filename, format="ascii.ecsv", overwrite=True) + + input_model.meta.source_detection[ + "tweakreg_catalog_name" + ] = cat_filename + + # only attach catalog to file if its being passed to the next step + # and save_catalogs is false, since it is not in the schema + input_model.meta.source_detection["tweakreg_catalog"] = catalog_as_array + + # just pass input model to next step - catalog is stored in meta + return input_model + + def _calc_2D_background(self): + """Calculates a 2D background image. + + Calculates the background value for the input image in boxes specified by + self.bkg_box_size. A mean, median, or mode estimator may be used (set + by `bkg_estimator`). The pixels in each box will be sigma clipped, + using a sigma specified by `bkg_sigma`.""" + + filter_size = ( + self.bkg_filter_size, + self.bkg_filter_size, + ) # square size specified + box_size = np.asarray(self.bkg_boxsize).astype(int) # must be integer + + if self.bkg_estimator == "median": + bkg_estimator = MedianBackground() + elif self.bkg_estimator == "mean": + bkg_estimator = MeanBackground() + elif self.bkg_estimator == "mode": + bkg_estimator = ModeEstimatorBackground() + else: + raise ValueError("bkg_estimator must be one of 'mean', 'median', or 'mode'") + + sigma_clip = SigmaClip(self.bkg_sigma) + + try: + bkg_2D = Background2D( + self.data, + box_size, + filter_size=filter_size, + coverage_mask=self.coverage_mask, + sigma_clip=sigma_clip, + bkg_estimator=bkg_estimator, + ) + except ValueError: + # use the entire unmasked array + log.info( + "Background could not be estimated in meshes. " + "Using the entire unmasked array for background " + f"estimation: bkg_boxsize={self.data.shape}." + ) + + bkg_2D = Background2D( + self.data, + self.data.shape, + filter_size=filter_size, + coverage_mask=self.coverage_mask, + sigma_clip=sigma_clip, + bkg_estimator=bkg_estimator, + exclude_percentile=100.0, + ) + + return bkg_2D diff --git a/romancal/source_detection/tests/__init__.py b/romancal/source_detection/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/romancal/source_detection/tests/test_source_detection_step.py b/romancal/source_detection/tests/test_source_detection_step.py new file mode 100644 index 000000000..236dbd101 --- /dev/null +++ b/romancal/source_detection/tests/test_source_detection_step.py @@ -0,0 +1,252 @@ +""" + Unit tests for the Roman source detection step code +""" + +import os + +import numpy as np +import pytest +from astropy import units as u +from astropy.convolution import Gaussian2DKernel +from roman_datamodels import maker_utils as testutil +from roman_datamodels import units as ru +from roman_datamodels.datamodels import ImageModel + +from romancal.lib import dqflags +from romancal.source_detection import SourceDetectionStep + + +@pytest.fixture +def setup_inputs(): + def _setup(nrows=100, ncols=100, noise=1.0): + + """Return ImageModel of lvl 2 image""" + + shape = (100, 100) # size of test image + wfi_image = testutil.mk_level2_image(shape=shape) + wfi_image.data = u.Quantity( + np.ones(shape, dtype=np.float32), ru.electron / u.s, dtype=np.float32 + ) + wfi_image.meta.filename = "filename" + + # add noise to data + if noise is not None: + wfi_image.data += u.Quantity( + noise * np.random.random(shape), ru.electron / u.s, dtype=np.float32 + ) + + # add dq array + + wfi_image.dq = np.zeros(shape, dtype=np.uint32) + # construct ImageModel + mod = ImageModel(wfi_image) + + return mod + + return _setup + + +@pytest.mark.skipif( + os.environ.get("CI") == "true", + reason="Roman CRDS servers are not currently available outside the internal " + "network", +) +def add_random_gauss(arr, x_positions, y_positions, min_amp=200, max_amp=500): + + """Add random 2D Gaussians to `arr` at specified positions, + with random amplitudes from `min_amp` to `max_amp`. Assumes + units of e-/s.""" + + for i, x in enumerate(x_positions): + y = y_positions[i] + gauss = Gaussian2DKernel(2, x_size=20, y_size=20).array + amp = np.random.randint(200, 700) + arr[y - 10 : y + 10, x - 10 : x + 10] += ( + u.Quantity(gauss, ru.electron / u.s, dtype=np.float32) * amp + ) + + +@pytest.mark.skipif( + os.environ.get("CI") == "true", + reason="Roman CRDS servers are not currently available outside the internal " + "network", +) +def test_source_detection_defaults(setup_inputs): + + """Test SourceDetectionStep with its default parameters. The detection + threshold will be chosen based on the image's background level.""" + + model = setup_inputs() + + # add in 12 sources, roughly evenly distributed + # sort by true_x so they can be matched up to output + + true_x = np.array([20, 25, 30, 35, 40, 45, 50, 55, 60, 70, 80, 82]) + true_y = np.array([26, 80, 44, 19, 66, 39, 29, 72, 54, 29, 80, 62]) + + # at each position, place a 2d gaussian + # randomly vary flux from 100 to 500 for each source + add_random_gauss(model.data, true_x, true_y) + + # call SourceDetectionStep with default parameters + res = SourceDetectionStep.call(model) + + # unpack output catalog array + _, xcentroid, ycentroid, flux = res.meta.source_detection.tweakreg_catalog + + # sort based on x coordinate, like input + ycentroid = [x for y, x in sorted(zip(xcentroid, ycentroid))] + flux = [x for y, x in sorted(zip(xcentroid, flux))] + xcentroid = sorted(xcentroid) + + # check that the number of input and output sources are the same + assert len(flux) == len(true_x) + + # check that their locations agree + # the differences between real/actual are always around 0.5. im assuming + # this is because photutils uses zero-indexed pixels so the centers are + # (0.5, 0.5)? atol=0.2 seems to be the lowest safe value for this right now. + + assert np.allclose(np.abs(xcentroid - true_x), 0.5, atol=0.2) + assert np.allclose(np.abs(ycentroid - true_y), 0.5, atol=0.2) + + +@pytest.mark.skipif( + os.environ.get("CI") == "true", + reason="Roman CRDS servers are not currently available outside the internal " + "network", +) +def test_source_detection_scalar_threshold(setup_inputs): + + """Test SourceDetectionStep using the option to choose a detection + threshold for entire image.""" + + model = setup_inputs() + + # add in 12 sources, roughly evenly distributed + # sort by true_x so they can be matched up to output + + true_x = np.array([20, 25, 30, 35, 40, 45, 50, 55, 60, 70, 80, 82]) + true_y = np.array([26, 80, 44, 19, 66, 39, 29, 72, 54, 29, 80, 62]) + + # at each position, place a 2d gaussian + # randomly vary flux from 100 to 500 for each source + add_random_gauss(model.data, true_x, true_y) + + # call SourceDetectionStep with default parameters + res = SourceDetectionStep.call(model, scalar_threshold=2.0) + + # unpack output catalog array + _, xcentroid, ycentroid, flux = res.meta.source_detection.tweakreg_catalog + + # sort based on x coordinate, like input + ycentroid = [x for y, x in sorted(zip(xcentroid, ycentroid))] + flux = [x for y, x in sorted(zip(xcentroid, flux))] + xcentroid = sorted(xcentroid) + + # check that the number of input and output sources are the same + assert len(flux) == len(true_x) + + # check that their locations agree + # the differences between real/actual are always around 0.5. im assuming + # this is because photutils uses zero-indexed pixels so the centers are + # (0.5, 0.5)? atol=0.2 seems to be the lowest safe value for this right now. + + assert np.allclose(np.abs(xcentroid - true_x), 0.5, atol=0.2) + assert np.allclose(np.abs(ycentroid - true_y), 0.5, atol=0.2) + + +@pytest.mark.skipif( + os.environ.get("CI") == "true", + reason="Roman CRDS servers are not currently available outside the internal " + "network", +) +@pytest.mark.filterwarnings() # how to supress the no sources detected warning? +def test_coverage_mask(setup_inputs): + """Make sure `coverage_mask` works to ignore sources that contain + DO_NOT_USE pixels.""" + + model = setup_inputs() + + # add a single source and make sure its detected + add_random_gauss(model.data, [50], [50]) + res = SourceDetectionStep.call(model) + # unpack output catalog array + _, xcentroid, ycentroid, flux = res.meta.source_detection.tweakreg_catalog + assert len(xcentroid) == 1 # make sure only one source detected + assert np.allclose(np.abs(xcentroid - 50), 0.5, atol=0.2) + assert np.allclose(np.abs(ycentroid - 50), 0.5, atol=0.2) + + # now, add some DO_NOT_USE to dq on source and make sure source is NOT detected + # when coverage_mask=True (default) + + model.dq[40:60, 40:60] += dqflags.pixel["DO_NOT_USE"] + res = SourceDetectionStep.call(model) + # unpack output catalog array + _, xcentroid, ycentroid, flux = res.meta.source_detection.tweakreg_catalog + # make sure catalog is empty + assert len(xcentroid) == 0 + + # finally, make sure when coverage_mask=False and there are DNU pixels, + # that this is applied correctly + res = SourceDetectionStep.call(model, coverage_mask=False) + # unpack output catalog array + _, xcentroid, ycentroid, flux = res.meta.source_detection.tweakreg_catalog + assert len(xcentroid) == 1 # make sure only one source detected + assert np.allclose(np.abs(xcentroid - 50), 0.5, atol=0.2) + assert np.allclose(np.abs(ycentroid - 50), 0.5, atol=0.2) + + +@pytest.mark.skipif( + os.environ.get("CI") == "true", + reason="Roman CRDS servers are not currently available outside the internal " + "network", +) +def test_outputs(setup_inputs): + """Make sure `save_catalogs` and `output_cat_filetype` work correctly.""" + + model = setup_inputs() + + # add a single source to image so a non-empty catalog is produced + add_random_gauss(model.data, [50], [50]) + + # run step and direct it to save catalog. default format should be asdf + SourceDetectionStep.call(model, save_catalogs=True) + # make sure file exists + assert os.path.isfile("filename_tweakreg_catalog.asdf") + + # run again, specifying that catalog should be saved in ecsv format + SourceDetectionStep.call(model, save_catalogs=True, output_cat_filetype="ecsv") + + assert os.path.isfile("filename_tweakreg_catalog.ecsv") + + +@pytest.mark.skipif( + os.environ.get("CI") == "true", + reason="Roman CRDS servers are not currently available outside the internal " + "network", +) +def test_limiting_catalog_size(setup_inputs): + + """Test to make sure setting `max_sources` limits the size of the + output catalog to contain only the N brightest sources""" + + model = setup_inputs() + + amps = [200, 300, 400] # flux + pos = [20, 50, 80] # 3 sources in a line + for i in range(3): + xy = pos[i] + gauss = Gaussian2DKernel(2, x_size=20, y_size=20).array + model.data[xy - 10 : xy + 10, xy - 10 : xy + 10] += ( + u.Quantity(gauss, ru.electron / u.s, dtype=np.float32) * amps[i] + ) + + res = SourceDetectionStep.call(model, max_sources=2) + _, xcentroid, ycentroid, flux = res.meta.source_detection.tweakreg_catalog + + # make sure only 2 of the three sources are returned in output catalog + assert len(flux) == 2 + + # and make sure one of them is not the first, dimmest source + assert np.all(xcentroid > 22) # dimmest position is really 20, give a