diff --git a/msaexp/msa.py b/msaexp/msa.py index de22a2b..55668cc 100644 --- a/msaexp/msa.py +++ b/msaexp/msa.py @@ -43,6 +43,12 @@ def pad_msa_metafile( prefix : str Prefix of new file to create (``prefix + metafile``) + verbose : bool + Print verbose output + + primary_sources : bool + Include only primary sources + Returns ------- @@ -401,7 +407,8 @@ def metadata_id_unique(self): @property def key_pairs(self): """ - List of unique ``msa_metadata_id, dither_point_index`` pairs from the ``shutter_table`` + List of unique ``msa_metadata_id, dither_point_index`` pairs + from the ``shutter_table`` Returns ------- @@ -450,7 +457,7 @@ def query_mast_exposures(self, force=False): Returns ------- - mat : `~astropy.table.Table` + mast : `~astropy.table.Table` Query results from `mastquery.jwst.query_jwst`. """ @@ -511,7 +518,6 @@ def get_transforms( msa_metadata_id=None, fit_degree=2, verbose=False, - min_source_id=0, **kwargs, ): """ @@ -833,6 +839,9 @@ def plot_slitlet( add_labels : bool Add plot labels + set_axis_labels : bool + Set axis labels + Returns ------- fig : `matplotlib.figure.Figure` @@ -1253,7 +1262,26 @@ def get_siaf_transforms( check_rms=True, ): """ - Read shutter (i,j) > (v2,v3) transformations from the files at https://github.com/spacetelescope/pysiaf/tree/master/pysiaf/source_data/NIRSpec/delivery/test_data/apertures_testData + Read shutter (i,j) > (v2,v3) transformations from the files at + https://github.com/spacetelescope/pysiaf/tree/master/pysiaf/source_data/NIRSpec/delivery/test_data/apertures_testData + + Parameters + ---------- + prefix : str, optional + URL prefix for the files containing the transformations. + + check_rms : bool, optional + If True, calculate and print the root mean square (RMS) of the + transformations. + + Returns + ------- + + transforms : dict + + A dictionary containing the (i,j) > (v2,v3) transformations for each + quadrant of the MSA. + """ from astropy.modeling.models import Polynomial2D from astropy.modeling.fitting import LinearLSQFitter @@ -1669,6 +1697,12 @@ def regions_from_metafile_siaf( msa_metadata_id, dither_point_index : int Exposure definition + meta_keys : list of str, optional + List of metadata keys to include in the output + + verbose : bool, optional + Print verbose messages + Returns ------- String or a list of `grizli.utils.SRegion` objects, depending on @@ -1798,7 +1832,8 @@ def regions_from_metafile_siaf( def all_regions_from_metafile_siaf(self, **kwargs): """ - Run `~msaexp.msa.MSAMetafile.regions_from_metafile_siaf` for all exposures + Run `~msaexp.msa.MSAMetafile.regions_from_metafile_siaf` + for all exposures Parameters ---------- @@ -1834,7 +1869,47 @@ def fit_siaf_shutter_transforms( prefix=PYSIAF_GITHUB, degree=3, inverse_degree=2, check_rms=True ): """ - Fit shutter (i,j) > (v2,v3) transformations from the files at https://github.com/spacetelescope/pysiaf/tree/main/pysiaf/source_data/NIRSpec/delivery/test_data/apertures_testData + Fit shutter (i,j) > (v2,v3) transformations from the files at + https://github.com/spacetelescope/pysiaf/tree/main/pysiaf/source_data/NIRSpec/delivery/test_data/apertures_testData + + Parameters + ---------- + prefix : str, optional + The URL prefix for the files containing the transformations. + degree : int, optional + The degree of the polynomial fit for the transformations. Default is 3. + inverse_degree : int, optional + The degree of the polynomial fit for the inverse transformations. + Default is 2. + check_rms : bool, optional + Whether to calculate and print the RMS of the transformations. + Default is True. + + Returns + ------- + transforms : dict + A dictionary containing the fitted transformations. The keys are: + + - "degree": the degree of the polynomial fit + + - "coeffs": a nested dictionary containing the coefficients of the + transformations for each quadrant + + - "irange": a nested dictionary containing the range of valid "i" + values for each quadrant + + - "jrange": a nested dictionary containing the range of valid "j" + values for each quadrant + + - "inverse_degree": the degree of the polynomial fit for the inverse + transformations + + - "inverse": a nested dictionary containing the coefficients of the + inverse transformations for each quadrant + + - "rms" (optional): a nested dictionary containing the RMS of the + transformations for each quadrant + """ from astropy.modeling.models import Polynomial2D from astropy.modeling.fitting import LinearLSQFitter @@ -2034,9 +2109,7 @@ def load_siaf_inverse_shutter_transforms(): return tr -def msa_shutter_catalog( - ra, dec, pointing=None, ap=None, inv=None, verbose=True -): +def msa_shutter_catalog(ra, dec, pointing=None, ap=None, inv=None): """ Compute shutter centering for a list of input coordinates @@ -2056,9 +2129,6 @@ def msa_shutter_catalog( Inverse shutter transformations from `msaexp.msa.load_siaf_inverse_shutter_transforms` - verbose : bool - Messaging - Returns ------- tab : `astropy.table.table` diff --git a/msaexp/pipeline.py b/msaexp/pipeline.py index bdc2f52..54b6f67 100644 --- a/msaexp/pipeline.py +++ b/msaexp/pipeline.py @@ -158,6 +158,21 @@ def query_program( def download_msa_meta_files(files=None, do_download=True): """ Download ``MSAMETFL`` files indicated in header keywords + + Parameters + ---------- + files : list, optional + List of files to consider. If not provided, it will search for all + "*rate.fits" and "*cal.fits" files in the current directory. + + do_download : bool, optional + Flag indicating whether to download the files. Default is True. + + Returns + ------- + msa : list + List of MSA files downloaded from MAST. + """ import mastquery.utils @@ -182,15 +197,12 @@ def download_msa_meta_files(files=None, do_download=True): return msa -def exposure_groups(path="./", files=None, split_groups=True, verbose=True): +def exposure_groups(files=None, split_groups=True, verbose=True): """ Group files by MSAMETFL, grating, filter, detector Parameters ---------- - path : str - Path to ``rate.fits`` files - files : list, None Explicit list of ``rate.fits`` files to consider. Otherwise, `glob` in working directory @@ -313,7 +325,6 @@ def __init__( self, file="jw02756001001_03101_00001_nrs1_rate.fits", step="phot", - verbose=True, read=False, indices=None, targets=None, @@ -329,9 +340,6 @@ def __init__( step : str Calibration pipeline processing step of the output slitlet file - verbose : bool - Print status messages - read : bool Don't just find the filenames but also read the data @@ -386,6 +394,11 @@ def N(self): def read_data(self, verbose=True): """ Read files into SlitModel objects in `slits` attribute + + Parameters + ---------- + verbose : bool, optional + Prints a statement per read file if True (default). """ from jwst.datamodels import SlitModel @@ -475,7 +488,7 @@ def __init__( utils.LOGFILE = self.mode + ".log.txt" if files is None: - groups = exposure_groups(verbose=False) + groups = exposure_groups(verbose=verbose) if mode in groups: self.files = groups[mode] else: @@ -484,12 +497,12 @@ def __init__( self.files = files msg = f"msaexp.NirspecPipeline: Initialize {mode}" - utils.log_comment(utils.LOGFILE, msg, verbose=True, show_date=True) + utils.log_comment(utils.LOGFILE, msg, verbose=verbose, show_date=True) for file in self.files: msg = f"msaexp.NirspecPipeline: {file}" utils.log_comment( - utils.LOGFILE, msg, verbose=True, show_date=False + utils.LOGFILE, msg, verbose=verbose, show_date=False ) self.pipe = OrderedDict() @@ -611,7 +624,19 @@ def targets(self): def slit_index(self, key): """ - Index of ``key`` in ``self.slitlets`` + Index of ``key`` in ``self.slitlets``. + + Parameters + ---------- + key : str + The key to search for in ``self.slitlets``. + + Returns + ------- + int or None + The index of ``key`` in ``self.slitlets`` if it exists, + otherwise None. + """ if key not in self.slitlets: return None @@ -621,6 +646,20 @@ def slit_index(self, key): def initialize_from_cals(self, key="phot", verbose=True): """ Initialize processing object from cal.fits products + + Parameters + ---------- + key : str + The key to identify the calibration product to load. + Default is "phot". + + verbose : bool, optional + If True, print verbose output. Default is True. + + Returns + ------- + None + """ import jwst.datamodels @@ -629,7 +668,9 @@ def initialize_from_cals(self, key="phot", verbose=True): msg = ( f"msaexp.initialize_from_cals : load {file} as MultiSlitModel" ) - utils.log_comment(utils.LOGFILE, msg, verbose=True, show_date=True) + utils.log_comment( + utils.LOGFILE, msg, verbose=verbose, show_date=True + ) self.pipe[key].append(jwst.datamodels.MultiSlitModel(file)) self.last_step = key @@ -662,6 +703,9 @@ def preprocess( Calculate rescaling of the ``VAR_RNOISE`` data extension based on pixel statistics + skip_completed : bool + Skip steps that have already been completed + Returns ------- status : bool @@ -926,6 +970,20 @@ def run_jwst_pipeline( def save_slit_data(self, step="phot", verbose=True): """ Save slit data to FITS + + Parameters + ---------- + step : str + The step of the pipeline from which the slit data is saved. + + verbose : bool, optional + If True, print verbose output. Default is True. + + Returns + ------- + bool + True if the slit data is saved successfully. + """ from jwst.datamodels import SlitModel @@ -952,7 +1010,9 @@ def save_slit_data(self, step="phot", verbose=True): dm = SlitModel(self.pipe[step][j].slits[i].instance) dm.write(slit_file, overwrite=True) except: - utils.log_exception(utils.LOGFILE, traceback, verbose=True) + utils.log_exception( + utils.LOGFILE, traceback, verbose=verbose + ) return True @@ -1117,6 +1177,19 @@ def set_background_slits(self, find_by_id=False): """ Initialize elements in ``self.pipe['bkg']`` for background-subtracted slitlets + + Parameters + ---------- + find_by_id : bool, optional + If True, find background slits by source ID. + If False, find background slits by slitlet ID. + Default is False. + + Returns + ------- + bool + True if the initialization is successful. + """ # Get from slitlet_ids # indices = [self.slitlets[k]['slitlet_id'] for k in self.slitlets] @@ -1167,6 +1240,37 @@ def fit_profile( ): """ Fit for profile width and offset + + Parameters + ---------- + key : str + The key of the slitlet to fit the profile for. + + yoffset : float, optional + The initial guess for the cross-dispersion offset. Default is None. + + prof_sigma : float, optional + The initial guess for the profile width. Default is None. + + bounds : list, optional + The bounds for the fitting parameters. + Default is [(-5, 5), (1.4 / 2.35, 3.5 / 2.35)]. + + min_delta : float, optional + The minimum change in chi-square required to perform the fit. + Default is 100. + + use_huber : bool, optional + If True, use Huber loss function for fitting. Default is True. + + verbose : bool, optional + If True, print verbose output. Default is True. + + Returns + ------- + tuple + A tuple containing the fitted profile width and offset. + """ from photutils.psf import IntegratedGaussianPRF from scipy.optimize import minimize @@ -1205,9 +1309,35 @@ def fit_profile( def _objfun_fit_profile(params, data, ret): """ - Loss function for fitting profile parameters + Loss function for fitting profile parameters. + + Parameters + ---------- + params : array-like + The fitting parameters yoffset and prof_sigma + (see 'fit_profile'). + + data : tuple + A tuple containing the data needed for the fitting. + - xx0 : unused + - yp : y pixel + - ytr : trace + - sh : shape + _clean : cleaned array + _ivar : inverse variance weight + bad : mask + ret : int + The return value (see 'Returns'). + + Returns + ------- + array-like or float + The desired output based on the value of `ret`: + - 0: return the full chi array + - 1: return the chi squared value + - 2: return the loss value + - other: return the fitted profile - params: yoffset, prof_sigma """ from scipy.special import huber @@ -1291,6 +1421,18 @@ def get_background_slits( """ Get background-subtracted slitlets + Parameters + ---------- + key : str + The key identifier of the slitlet + + step : str, optional + The step in the pipeline to get the slitlets from. Default is "bkg" + + check_background : bool, optional + If True, check if the background subtraction has been performed. + Default is True. + Returns ------- slits : list @@ -1320,7 +1462,23 @@ def get_background_slits( def drizzle_2d(self, key, drizzle_params={}, **kwargs): """ - not used... + Not used + + Drizzle the 2D spectra for a given slitlet. + + Parameters + ---------- + key : str + The key of the slitlet to drizzle the spectra for. + + drizzle_params : dict, optional + Additional parameters to pass to the `ResampleSpecData` class. + + Returns + ------- + `jwst.datamodels.ModelContainer` + The drizzled 2D spectra. + """ from jwst.datamodels import ModelContainer from jwst.resample.resample_spec import ResampleSpecData @@ -1340,7 +1498,17 @@ def drizzle_2d(self, key, drizzle_params={}, **kwargs): def get_slit_traces(self, verbose=True): """ - Set center of slit traces in `slitlets` + Set center of slit traces in `slitlets`. + + Parameters + ---------- + verbose : bool, optional + If True, print verbose output. Default is True. + + Returns + ------- + None + """ msg = "msaexp.get_slit_traces: Run" @@ -1407,6 +1575,86 @@ def extract_spectrum( ): """ Main function for extracting 2D/1D spectra from individual slitlets + + Parameters + ---------- + + key: str + The key of the slitlet to extract the spectrum from. + + slit_key: str + The key of the slit to use for extraction. + If None, the last step in the pipeline will be used. + + prof_sigma: float + The sigma parameter for the IntegratedGaussianPRF. + If None, the value from the slitlet will be used. + + fit_profile_params: dict + Additional parameters for fitting the profile. + Default is {"min_delta": 100}. + + pstep: float + The step size for the running median. Default is 1.0. + + show_sn: bool + Whether to show the signal-to-noise ratio. Default is True. + + flux_unit: str + The unit of the flux. Default is FNU_UNIT. + + vmax: float + The maximum value for the plot. Default is 0.2. + + yoffset: float + The y offset for the slit. + If None, the value from the slitlet will be used. + + skip: list + A list of indices to skip. Default is None. + + bad_dq_bits: int + The bad dq bits to mask. Default is (1 | 1024). + + clip_sigma: float + The sigma value for clipping. Default is -4. + + ntrim: int + The number of points to trim. Default is 5. + + get_slit_data: bool + Whether to get the slit data. Default is False. + + verbose: bool + Whether to print verbose output. Default is False. + + center2d: bool + Whether to center the 2D spectrum. Default is False. + + trace_sign: int + The sign of the trace. Default is 1. + + min_dyoffset: float + The minimum value for the y offset. Default is 0.2. + + Returns: + ---------- + If 'key' is not found in 'slitlets', returns None. + + Else: + + slitlet: object + The slitlet object. + + tabs: list + A list of tables containing the extracted spectra. + + full_tab: object + The combined table of all extracted spectra. + + fig: object + The `matplotlib` figure object. + """ from photutils.psf import IntegratedGaussianPRF import eazy.utils @@ -1892,6 +2140,24 @@ def extract_spectrum( def extract_all_slits(self, keys=None, verbose=True, close=True, **kwargs): """ Extract all spectra and make diagnostic figures + + Parameters + ---------- + keys : list, optional + List of keys corresponding to the slitlets to extract spectra from. + If not provided, spectra will be extracted from all slitlets. + verbose : bool, optional + Print status messages. Default is True. + close : bool, optional + Close all figures after saving. Default is True. + **kwargs : dict, optional + Additional keyword arguments to pass to the + `extract_spectrum` method. + + Returns + ------- + None + """ slitlets = self.slitlets @@ -2029,6 +2295,11 @@ def load_slit_data( targets : list, None Optional target names of specific individual sources + Returns + ------- + slit_lists : list + List of `msaexp.pipeline.SlitData` objects containing the loaded + slitlet data. """ slit_lists = [ SlitData( @@ -2049,7 +2320,19 @@ def load_slit_data( def parse_slit_info(self, write=True): """ - Parse information from / to ``{mode}.slits.yaml`` file + Parse information from / to ``{mode}.slits.yaml`` file. + + Parameters + ---------- + write : bool, optional + Whether to write the parsed information back to the YAML file. + Default is True. + + Returns + ------- + info : dict + A dictionary containing the parsed information from the YAML file. + """ import yaml @@ -2108,6 +2391,46 @@ def full_pipeline( ): """ Run all steps through extractions + + Parameters + ---------- + load_saved : str, optional + The calibration pipeline processing step to load saved data from. + If provided, the pipeline will skip the preprocessing + and JWST pipeline steps. + Default is None. + + run_preprocess : bool, optional + Whether to run the preprocessing step. Default is True. + + run_extractions : bool, optional + Whether to run the extraction step. Default is True. + + indices : list, optional + List of slit indices of specific files to process. Default is None. + + targets : list, optional + List of target names of specific individual sources to process. + Default is None. + + initialize_bkg : bool, optional + Whether to initialize the background slits. Default is True. + + make_regions : bool, optional + Whether to create slit source regions. Default is True. + + use_yaml_metadata : bool, optional + Whether to use YAML metadata for initializing slit metadata. + Default is True. + + **kwargs : dict, optional + Additional keyword arguments to pass to the preprocessing and + extraction steps. + + Returns + ------- + None + """ if load_saved is not None: @@ -2163,6 +2486,25 @@ def full_pipeline( def make_summary_tables(root="msaexp", zout=None): """ Make a summary table of all extracted sources + + Parameters: + ----------- + + root : str + The root directory where the data is stored. Default is "msaexp". + + zout : astropy.table.Table + Optional table containing photometric redshift information. Default is None. + + Returns: + -------- + tabs : list + List of astropy tables containing the extracted source information. + + full : astropy.table.Table + + Combined astropy table containing all the extracted source information. + """ import yaml import astropy.table