From 7e1f365d6b6a76af9c0ad8c1a7d23b68d9645b71 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 23 Nov 2020 10:38:48 -0500 Subject: [PATCH 01/13] Intermediate commit. Lost apt_input changes. Need to redo --- mirage/apt/read_apt_xml.py | 2 +- mirage/yaml/generate_observationlist.py | 28 ++- mirage/yaml/yaml_generator.py | 223 +++++++++++++++++++++++- 3 files changed, 245 insertions(+), 8 deletions(-) diff --git a/mirage/apt/read_apt_xml.py b/mirage/apt/read_apt_xml.py index c27263ed6..650f81eaf 100755 --- a/mirage/apt/read_apt_xml.py +++ b/mirage/apt/read_apt_xml.py @@ -83,7 +83,7 @@ def __init__(self): self.observation_info = OrderedDict() def get_tracking_type(self, observation): - """Deetermine whether the observation uses sidereal or non-sidereal + """Determine whether the observation uses sidereal or non-sidereal tracking Parameters diff --git a/mirage/yaml/generate_observationlist.py b/mirage/yaml/generate_observationlist.py index da976bc32..bb45243be 100755 --- a/mirage/yaml/generate_observationlist.py +++ b/mirage/yaml/generate_observationlist.py @@ -459,7 +459,7 @@ def expand_for_dithers(indict, verbose=True): def get_observation_dict(xml_file, yaml_file, catalogs, - parameter_overrides={'cosmic_rays': None, 'background': None, 'roll_angle': None, 'dates': None}, + parameter_overrides={'cosmic_rays': None, 'background': None, 'roll_angle': None, 'dates': None, 'times': None}, verbose=False): """Write observation list file (required mirage input) on the basis of APT files. @@ -518,6 +518,7 @@ def get_observation_dict(xml_file, yaml_file, catalogs, # entry in parameter_defaults default_values = {} default_values['Date'] = '2021-10-04' + default_values['Time'] = '12:34' default_values['PAV3'] = '0.' default_values['PointsourceCatalog'] = 'None' default_values['GalaxyCatalog'] = 'None' @@ -597,9 +598,16 @@ def get_observation_dict(xml_file, yaml_file, catalogs, # Just use dates below when looping over observations pass - #for key in parameter_defaults.keys(): - # if key in default_values.keys(): - # default_values[key] = parameter_defaults[key] + times = parameter_overrides['times'] + if times is not None: + if isinstance(times, str): + default_values['Time'] = times + # Now set times to None so that it won't be used when looping + # over observations below + times = None + else: + # Just use times below when looping over observations + pass # Roll angle, aka PAV3 # pav3 = 34.5 @@ -694,6 +702,17 @@ def get_observation_dict(xml_file, yaml_file, catalogs, "Quitting.\n\n".format(observation_number))) raise KeyError + # Get the proper time value + if times is None: + time_value = default_values['Time'] + else: + try: + time_value = times[observation_number] + except KeyError: + logger.error(("\n\nERROR: No time value specified for Observation {} in time dictionary. " + "Quitting.\n\n".format(observation_number))) + raise KeyError + # Get the proper PAV3 value if pav3 is None: pav3_value = default_values['PAV3'] @@ -752,6 +771,7 @@ def get_observation_dict(xml_file, yaml_file, catalogs, " EntryNumber{}:\n".format(entry_number), " Instrument: {}\n".format(instrument), " Date: {}\n".format(date_value), + " Time: {}\n".format(time_value), " PAV3: {}\n".format(pav3_value), " DitherIndex: {}\n".format(dither_index), " CosmicRayLibrary: {}\n".format(cr_library_value), diff --git a/mirage/yaml/yaml_generator.py b/mirage/yaml/yaml_generator.py index abd359e02..9e0a8f858 100755 --- a/mirage/yaml/yaml_generator.py +++ b/mirage/yaml/yaml_generator.py @@ -116,7 +116,7 @@ from ..constants import NIRISS_PUPIL_WHEEL_ELEMENTS, NIRISS_FILTER_WHEEL_ELEMENTS from ..utils.constants import CRDS_FILE_TYPES, SEGMENTATION_MIN_SIGNAL_RATE, \ LOG_CONFIG_FILENAME, STANDARD_LOGFILE_NAME -from ..utils import utils +from ..utils import siaf_interface, utils ENV_VAR = 'MIRAGE_DATA' @@ -128,7 +128,7 @@ class SimInput: def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffile_defaults='crds', reffile_overrides=None, use_JWST_pipeline=True, catalogs=None, cosmic_rays=None, - background=None, roll_angle=None, dates=None, + background=None, roll_angle=None, dates=None, times=None, observation_list_file=None, verbose=False, output_dir='./', simdata_output_dir='./', dateobs_for_background=False, segmap_flux_limit=None, segmap_flux_limit_units=None, offline=False): @@ -220,6 +220,11 @@ def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffil See Mirage's onling documentation for examples: https://mirage-data-simulator.readthedocs.io/en/latest/yaml_generator.html + times : str or dict + Observation times assocated with the exposures or program. Used only + in the case of non-sidereal observations, where it is used to determine + target location for each exposure. + observation_list_file=None verbose : bool Print more information to the screen while running @@ -256,7 +261,7 @@ def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffil self.logger.info('Original log file name: ./{}'.format(STANDARD_LOGFILE_NAME)) parameter_overrides = {'cosmic_rays': cosmic_rays, 'background': background, 'roll_angle': roll_angle, - 'dates': dates} + 'dates': dates, 'times': times} self.info = {} self.input_xml = input_xml @@ -273,6 +278,7 @@ def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffil raise ValueError("reffile_defaults must be 'crds' or 'crds_full_name'") self.reffile_overrides = reffile_overrides + self.catalogs = catalogs self.table_file = None self.use_nonstsci_names = False self.use_linearized_darks = True @@ -651,6 +657,11 @@ def create_inputs(self): warnings.simplefilter("ignore") self.make_start_times() + # If we have a non-sidereal observation, then we need to + # update the pointing information based on the input + # ephemeris file or target velocity + self.nonsidereal_pointing_updates() + # Add a list of output yaml names to the dictionary self.make_output_names() @@ -1167,6 +1178,212 @@ def make_output_names(self): self.info['outputfits'] = fits_names # Table([self.info['yamlfile']]).pprint() + def nonsidereal_pointing_updates(self): + """The pointing info for non-sidereal observations will be incorrect + because you cannot specify a time in APT. Once observation times/dates + have been calculated here, we can now get the pointing information + correct. + """ + for k in self.info.keys(): + print(k) + + print(self.info['ObservationID']) + print(self.info['Instrument']) + print(self.info['TargetID']) + print(self.info['Tracking']) + print(self.info['Date']) + print(self.info['Dither']) + print(self.info['Exposures']) + print(self.info['entry_number']) + print(self.info['exposure']) + print(self.info['dither']) + stop + + obs = np.array(self.info['ObservationID']) + all_date_obs = np.array(self.info['date_obs']) + all_time_obs = np.array(self.info['time_obs']) + all_exposure = np.array(sel.info['exposre']) + tracking = np.array(self.info['Tracking']) + targs = np.array(self.info['TargetID']) + inst = np.array(self.info['Instrument']) + ra_from_pointing_file = np.array(self.info['ra']) + dec_from_pointing_file = np.array(self.info['dec']) + nonsidereal_index = np.where(np.array(tracking) == 'non-sidereal') + all_nonsidereal_targs = targs[nonsidereal_index] + nonsidereal_targs, u_indexes = np.unique(targs[nonsidereal_index], return_indexes=True) + nonsidereal_instruments = inst[nonsidereal_index][u_indexes] + + # Check that all non-sidereal targets have catalogs associated with them + for targ, inst in zip(nonsidereal_targs, nonsidereal_instruments): + ns_catalog = get_nonsidereal_catalog_name(self.catalogs, targ, inst) + catalog_table, pos_in_xy, vel_in_xy = read_nonsidereal_catalog(ns_catalog) + + # Find the observations that use this target + exp_index_this_target = targs == targ + obs_this_target = np.unique(obs[exp_index_this_target]) + + # Loop over observations + for obs_name in obs_this_target: + + # Get date/time for every exposure + obs_exp_indexes = np.where(obs == obs_name) + obs_dates = all_date_obs[obs_exp_indexes] + obs_times = all_time_obs[obs_exp_indexes] + exposures = all_exposure[obs_exp_indexes] + + start_dates = [] + for date_obs, time_obs in zip(obs_dates, obs_times): + ob_time = '{}T{}'.format(date_obs, time_obs) + try: + start_dates.append(datetime.datetime.strptime(ob_time, '%Y-%m-%dT%H:%M:%S')) + except ValueError: + start_dates.append(datetime.datetime.strptime(ob_time, '%Y-%m-%dT%H:%M:%S.%f')) + + if 'ephemeris_file' in catalog_table.colnames: + ephemeris_file = catalog_table['ephemeris_file'][0] + ra_ephem, dec_ephem = ephemeris_tools.read_ephemeris_file(ephemeris_file) + all_times = [ephemeris_tools.to_timestamp(elem) for elem in start_dates] + + # Create list of positions for all frames + ra_target = ra_eph(all_times) + dec_target = dec_eph(all_times) + else: + if not pos_in_xy: + if not vel_in_xy: + # Here we assume that the source (and aperture reference location) + # is located at the given RA, Dec at the start of the first exposure + base_ra = ra_from_pointing_file[obs_exp_indexes] + base_dec = dec_from_pointing_file[obs_exp_indexes] + ra_target = [base_ra] + dec_target = [base_dec] + for ob_date in start_dates[1:]: + delta_time = ob_date - start_dates[0] + delta_ra = 0.05 * delta_time.total_seconds() / 3600. + delta_dec = 0.02 * delta_time.total_seconds() / 3600. + ra_target.append(base_ra + delta_ra) + dec_target.append(base_dec + delta_dec) + else: + # Velocity is in units of pixels/hr. Need to translate to arcsec/hr + print('To do') + else: + # Input position should be at the aperture reference location + # or else we're not tracking the target, right? + if not vel_in_xy: + print('to do') + else: + # Velocity is in units of pixels/hr. Need to translate to arcsec/hr + print('to do') + + + ra_from_pointing_file[obs_exp_indexes] = ra_target + dec_from_pointing_file[obs_exp_indexes] = dec_target + + self.info['ra'] = ra_from_pointing_file + self.info['dec'] = dec_from_pointing_file + + # Create a pysiaf.Siaf instance for each instrument in the proposal + print('Lines below are untested') + siaf_dictionary = {} + for instrument_name in np.unique(self.info['Instrument']): + siaf_dictionary[instrument_name] = siaf_interface.get_instance(instrument_name) + + print('hopefully this updates pointings base on the new ra dec values. check carefully') + self.info = apt_inputs.ra_dec_update(self.info, siaf_dictionary) + + + """ + read in catalog + determine ephemeris file or velocities + + 1. read in ephemeris file + 2. get ra, dec at observation date+time + + or + + 1. calculate ra, dec from ra,dec,ra_vel,dec_vel and observation date+time + """ + + + + + stop + + + + @staticmethod + def read_nonsidereal_catalog(filename): + """Read in a Mirage formatted non-sidereal source catalog + """ + catalog_table = ascii.read(ns_catalog, comment='#') + + # Check to see whether the position is in x,y or ra,dec + pixelflag = False + try: + if 'position_pixels' in catalog_table.meta['comments'][0:4]: + pixelflag = True + except: + pass + + # If present, check whether the velocity entries are pix/sec + # or arcsec/sec. + pixelvelflag = False + try: + if 'velocity_pixels' in catalog_table.meta['comments'][0:4]: + pixelvelflag = True + except: + pass + return catalog_table, pixelflag, pixelvelflag + + + + @staticmethod + def get_nonsidereal_catalog_name(cat_dict, target_name, instrument_name): + """Given a dictionary or nested dictionary of source catalogs, + return the name of the non-sidereal catalog for the given target + name. + + Parameters + ---------- + cat_dict : dict + Dictionary of source catalogs as input by the user + + target_name : str + Target name to search for + + instrument_name : str + Name of instrument used to observe ``target_name`` + + Returns + ------- + cat_file : str + Name of source catalog + """ + target_cat = cat_dict[target_name] + if 'moving_target_to_track' in target_cat.keys(): + if not os.path.isfile(target_cat['moving_target_to_track']): + raise ValueError(("{} is listed as the non-sidereal target catalog for " + "{}, but it appears this file does not exist.".format(target_cat['moving_target_to_track'], + target_name))) + else: + cat_file = target_cat['moving_target_to_track'] + else: + if instrument_name not in target_cat.keys(): + raise ValueError(("Catalog dictionary does not contain a 'moving_target_to_track' catalog " + "for {}. Unable to proceed.".format(target_name))) + else: + if 'moving_target_to_track' in target_cat[instrument_name].keys(): + if not os.path.isfile(target_cat[instrument_name]['moving_target_to_track']): + raise ValueError(("{} is listed as the non-sidereal target catalog for " + "{}, but it appears this file does not exist." + .format(target_cat['moving_target_to_track'], target_name))) + else: + cat_file = target_cat[instrument_name]['moving_target_to_track'] + else: + raise ValueError(("Catalog dictionary does not contain a 'moving_target_to_track' catalog " + "for {}. Unable to proceed.".format(target_name))) + return cat_file + + def set_global_definitions(self): """Store the subarray definitions of all supported instruments.""" # TODO: Investigate how this could be combined with the creation of From f33fcdd77ecd5d5eae270c0c283beb92256e9cc7 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 23 Nov 2020 21:36:26 -0500 Subject: [PATCH 02/13] Reorganizing order of operations --- mirage/apt/apt_inputs.py | 314 +++++++++++++++++- mirage/catalogs/utils.py | 92 +++++- mirage/seed_image/catalog_seed_image.py | 36 +-- mirage/seed_image/ephemeris_tools.py | 32 ++ mirage/utils/utils.py | 71 +++++ mirage/yaml/yaml_generator.py | 408 ++++++++++++++++++------ 6 files changed, 814 insertions(+), 139 deletions(-) diff --git a/mirage/apt/apt_inputs.py b/mirage/apt/apt_inputs.py index 765639ab3..e440e5fc3 100755 --- a/mirage/apt/apt_inputs.py +++ b/mirage/apt/apt_inputs.py @@ -47,6 +47,7 @@ import pkg_resources from astropy.table import Table, vstack +from astropy.time import Time, TimeDelta from astropy.io import ascii import numpy as np from pysiaf import rotations, Siaf @@ -141,7 +142,7 @@ def add_observation_info(self, intab): with open(self.observation_list_file, 'r') as infile: self.obstab = yaml.safe_load(infile) - OBSERVATION_LIST_FIELDS = 'Date PAV3 Filter PointSourceCatalog GalaxyCatalog ' \ + OBSERVATION_LIST_FIELDS = 'Date Time PAV3 Filter PointSourceCatalog GalaxyCatalog ' \ 'ExtendedCatalog ExtendedScale ExtendedCenter MovingTargetList ' \ 'MovingTargetSersic MovingTargetExtended ' \ 'MovingTargetConvolveExtended MovingTargetToTrack ' \ @@ -185,7 +186,7 @@ def add_observation_info(self, intab): if instrument == 'nircam': # keep the number of entries in the dictionary consistent for key in OBSERVATION_LIST_FIELDS: - if key in ['Date', 'PAV3', 'Instrument', 'CosmicRayLibrary', 'CosmicRayScale']: + if key in ['Date', 'Time', 'PAV3', 'Instrument', 'CosmicRayLibrary', 'CosmicRayScale']: value = str(entry[key]) else: value = str(None) @@ -309,6 +310,15 @@ def create_input_table(self, verbose=False): # Add epoch and catalog information observation_dictionary = self.add_observation_info(observation_dictionary) + + + + make_start_times??? + + + + + # if verbose: # print('Summary of observation dictionary:') # for key in observation_dictionary.keys(): @@ -354,7 +364,7 @@ def create_input_table(self, verbose=False): self.siaf[instrument_name] = siaf_interface.get_instance(instrument_name) # Calculate the correct V2, V3 and RA, Dec for each exposure/detector - self.ra_dec_update() + self.exposure_tab = ra_dec_update(self.exposure_tab, self.siaf) # Output to a csv file. if self.output_csv is None: @@ -950,6 +960,211 @@ def global_alignment_pointing(self, obs_dict): return obs_dict + def make_start_times_test(self, obs_info): + """Create exposure start times for each entry in the observation dictionary.""" + date_obs = [] + time_obs = [] + expstart = [] + nframe = [] + nskip = [] + namp = [] + + + + # choose arbitrary start time for each epoch + epoch_base_time = '16:44:12' + epoch_base_time0 = copy.deepcopy(epoch_base_time) + + if 'epoch_start_date' in obs_info.keys(): + epoch_base_date = obs_info['epoch_start_date'][0] + else: + epoch_base_date = obs_info['Date'][0] + base = Time(epoch_base_date + 'T' + epoch_base_time) + base_date, base_time = base.iso.split() + + # Pick some arbirary overhead values + act_overhead = 90 # seconds. (filter change) + visit_overhead = 600 # seconds. (slew) + + # Get visit, activity_id, dither_id info for first exposure + ditherid = obs_info['dither'][0] + actid = obs_info['act_id'][0] + visit = obs_info['visit_num'][0] + # obsname = obs_info['obs_label'][0] + + # for i in range(len(obs_info['Module'])): + for i, instrument in enumerate(obs_info['Instrument']): + # Get dither/visit + # Files with the same activity_id should have the same start time + # Overhead after a visit break should be large, smaller between + # exposures within a visit + next_actid = obs_info['act_id'][i] + next_visit = obs_info['visit_num'][i] + next_obsname = obs_info['obs_label'][i] + next_ditherid = obs_info['dither'][i] + + # Find the readpattern of the file + readpatt = obs_info['ReadoutPattern'][i] + groups = np.int(obs_info['Groups'][i]) + integrations = np.int(obs_info['Integrations'][i]) + + if instrument.lower() in ['miri', 'nirspec']: + nframe.append(0) + nskip.append(0) + namp.append(0) + date_obs.append(base_date) + time_obs.append(base_time) + expstart.append(base.mjd) + + else: + # Now read in readpattern definitions + #readpatt_def = self.global_readout_patterns[instrument.lower()] + print('XXXXXTry setting up config info using the moved funcn in utils!!!!') + # Read in file containing subarray definitions + #subarray_def = self.global_subarray_definitions[instrument.lower()] + config_information = utils.organize_config_files() + readpatt_def = config_information['global_readout_patterns'][instrument.lower()] + subarray_def = config_information['global_subarray_definitions'][instrument.lower()] + + match2 = readpatt == readpatt_def['name'] + if np.sum(match2) == 0: + raise RuntimeError(("WARNING!! Readout pattern {} not found in definition file." + .format(readpatt))) + + # Now get nframe and nskip so we know how many frames in a group + fpg = np.int(readpatt_def['nframe'][match2][0]) + spg = np.int(readpatt_def['nskip'][match2][0]) + nframe.append(fpg) + nskip.append(spg) + + # Get the aperture name. For non-NIRCam instruments, + # this is simply the obs_info['aperture']. But for NIRCam, + # we need to be careful of entries like NRCBS_FULL, which is used + # for observations using all 4 shortwave B detectors. In that case, + # we need to build the aperture name from the combination of detector + # and subarray name. + aperture = obs_info['aperture'][i] + + # Get the number of amps from the subarray definition file + match = aperture == subarray_def['AperName'] + + # needed for NIRCam case + if np.sum(match) == 0: + aperture = [apername for apername, name in + np.array(subarray_def['AperName', 'Name']) if + (sub in apername) or (sub in name)] + + match = aperture == subarray_def['AperName'] + + if len(aperture) > 1 or len(aperture) == 0 or np.sum(match) == 0: + raise ValueError('Cannot combine detector {} and subarray {}\ + into valid aperture name.'.format(det, sub)) + # We don't want aperture as a list + aperture = aperture[0] + + # For grism tso observations, get the number of + # amplifiers to use from the APT file. + # For other modes, check the subarray def table. + try: + amp = int(obs_info['NumOutputs'][i]) + except ValueError: + amp = subarray_def['num_amps'][match][0] + + # Default to amps=4 for subarrays that can have 1 or 4 + # if the number of amps is not defined. Hopefully we + # should never enter this code block given the lines above. + if amp == 0: + amp = 4 + self.logger.info(('Aperture {} can be used with 1 or 4 readout amplifiers. Defaulting to use 4.' + 'In the future this information should be made a user input.'.format(aperture))) + namp.append(amp) + + # same activity ID + # Remove this for now, since Mirage was not correctly + # specifying activities. At the moment all exposures have + # the same activity ID, which means we must allow the + # the epoch_start_date to change even if the activity ID + # does not. This will change back in the future when we + # figure out more realistic activity ID values. + #if next_actid == actid: + # # in this case, the start time should remain the same + # date_obs.append(base_date) + # time_obs.append(base_time) + # expstart.append(base.mjd) + # continue + + epoch_date = obs_info['epoch_start_date'][i] + epoch_time = copy.deepcopy(epoch_base_time0) + + # new epoch - update the base time + if epoch_date != epoch_base_date: + epoch_base_date = copy.deepcopy(epoch_date) + base = Time(epoch_base_date + 'T' + epoch_base_time) + base_date, base_time = base.iso.split() + basereset = True + date_obs.append(base_date) + time_obs.append(base_time) + expstart.append(base.mjd) + actid = copy.deepcopy(next_actid) + visit = copy.deepcopy(next_visit) + obsname = copy.deepcopy(next_obsname) + continue + + # new visit + if next_visit != visit: + # visit break. Larger overhead + overhead = visit_overhead + + # This block should be updated when we have more realistic + # activity IDs + elif ((next_actid > actid) & (next_visit == visit)): + # same visit, new activity. Smaller overhead + overhead = act_overhead + elif ((next_ditherid != ditherid) & (next_visit == visit)): + # same visit, new dither position. Smaller overhead + overhead = act_overhead + else: + # same observation, activity, dither. Filter changes + # will still fall in here, which is not accurate + overhead = 10. + + # For cases where the base time needs to change + # continue down here + siaf_inst = obs_info['Instrument'][i].upper() + if siaf_inst == 'NIRCAM': + siaf_inst = "NIRCam" + siaf_obj = pysiaf.Siaf(siaf_inst)[aperture] + + # Calculate the readout time for a single frame + frametime = utils.calc_frame_time(siaf_inst, aperture, + siaf_obj.XSciSize, siaf_obj.YSciSize, amp) + + # Estimate total exposure time + exptime = ((fpg + spg) * groups + fpg) * integrations * frametime + + # Delta should include the exposure time, plus overhead + delta = TimeDelta(exptime + overhead, format='sec') + base += delta + base_date, base_time = base.iso.split() + + # Add updated dates and times to the list + date_obs.append(base_date) + time_obs.append(base_time) + expstart.append(base.mjd) + + # increment the activity ID and visit + actid = copy.deepcopy(next_actid) + visit = copy.deepcopy(next_visit) + obsname = copy.deepcopy(next_obsname) + ditherid = copy.deepcopy(next_ditherid) + + obs_info['date_obs'] = date_obs + obs_info['time_obs'] = time_obs + # obs_info['expstart'] = expstart + obs_info['nframe'] = nframe + obs_info['nskip'] = nskip + obs_info['namp'] = namp + return obs_info def tight_dithers(self, input_dict): """ @@ -1147,6 +1362,99 @@ def get_filters(pointing_info): return filters +def ra_dec_update(exposure_dict, siaf_instances, verbose=False): + """Given the V2, V3 values for the reference locations associated + with detector apertures, calculate corresponding RA, Dec. + + Parameters + ---------- + exposure_dict : dict + Dictionary of exposure parameters, like self.exposure_tab, after expanding for + detectors + + siaf_instances : dict + Dictionary of instrument level SIAF instances. Instrument names are the + keys and the SIAF instances are the values + + Returns + ------- + exposure_dict : dict + Modified exposure dictionary with updated RA, Dec values for the pointing + """ + sw_grismts_apertures = ['NRCA1_GRISMTS256', 'NRCA1_GRISMTS128', 'NRCA1_GRISMTS64', + 'NRCA3_GRISMTS256', 'NRCA3_GRISMTS128', 'NRCA3_GRISMTS64'] + + lw_grismts_apertures = ['NRCA5_GRISM256_F322W2', 'NRCA5_GRISM128_F322W2', 'NRCA5_GRISM64_F322W2', + 'NRCA5_GRISM256_F444W', 'NRCA5_GRISM128_F444W', 'NRCA5_GRISM64_F444W'] + + intermediate_lw_grismts_apertures = ['NRCA5_TAGRISMTS_SCI_F444W', 'NRCA5_TAGRISMTS_SCI_F322W2'] + + aperture_ra = [] + aperture_dec = [] + + lw_grismts_aperture = None + for i in range(len(exposure_dict['Module'])): + siaf_instrument = exposure_dict["Instrument"][i] + aperture_name = exposure_dict['aperture'][i] + pointing_ra = np.float(exposure_dict['ra'][i]) + pointing_dec = np.float(exposure_dict['dec'][i]) + pointing_v2 = np.float(exposure_dict['v2'][i]) + pointing_v3 = np.float(exposure_dict['v3'][i]) + + # When we run across a LW grism TS aperture, save + # the aperture name, because we'll need it when looking + # at the accompanying SW apertuers to follow. THIS + # RELIES ON THE LW ENTRY COMING BEFORE THE SW ENTRIES. + if aperture_name in lw_grismts_apertures: + lw_grismts_aperture = copy.deepcopy(aperture_name) + lw_filter = lw_grismts_aperture.split('_')[2] + lw_intermediate_aperture = [ap for ap in intermediate_lw_grismts_apertures if lw_filter in ap][0] + + if 'pav3' in exposure_dict.keys(): + pav3 = np.float(exposure_dict['pav3'][i]) + else: + pav3 = np.float(exposure_dict['PAV3'][i]) + + telescope_roll = pav3 + + aperture = siaf_instances[siaf_instrument][aperture_name] + + if 'NRCA5_GRISM' in aperture_name and 'WFSS' not in aperture_name: + ra = pointing_ra + dec = pointing_dec + else: + if aperture_name in sw_grismts_apertures: + # Special case. When looking at grism time series observation + # we force the pointing to be at the reference location of the + # LW *intermediate* aperture, rather than paying attention to + # the V2, V3 in the pointing file. V2, V3 from the intermediate + # aperture is where the source would land on the detector if + # the grism were not in the beam. This is exactly what we want + # for the SW detectors, where this is no grism. + + # Generate an attitude matrix from this and + # use to get the RA, Dec in the SW apertures + lw_gts = siaf_instances[siaf_instrument][lw_intermediate_aperture] + pointing_v2 = lw_gts.V2Ref + pointing_v3 = lw_gts.V3Ref + + local_roll, attitude_matrix, fullframesize, subarray_boundaries = \ + siaf_interface.get_siaf_information(siaf_instances[siaf_instrument], aperture_name, + pointing_ra, pointing_dec, telescope_roll, + v2_arcsec=pointing_v2, v3_arcsec=pointing_v3) + + # Calculate RA, Dec of reference location for the detector + # Add in any offsets from the pointing file in the BaseX, BaseY columns + ra, dec = rotations.pointing(attitude_matrix, aperture.V2Ref, aperture.V3Ref) + aperture_ra.append(ra) + aperture_dec.append(dec) + + exposure_dict['ra_ref'] = aperture_ra + exposure_dict['dec_ref'] = aperture_dec + return exposure_dict + + + # if __name__ == '__main__': # # usagestring = 'USAGE: apt_inputs.py NIRCam_obs.xml NIRCam_obs.pointing' diff --git a/mirage/catalogs/utils.py b/mirage/catalogs/utils.py index b74ff1965..08ac1426e 100644 --- a/mirage/catalogs/utils.py +++ b/mirage/catalogs/utils.py @@ -2,13 +2,14 @@ """This module contains utility functions related to source catalogs """ +import os + from astropy.io import ascii import numpy as np from mirage.utils.constants import IMAGING_ALLOWED_CATALOGS, WFSS_ALLOWED_CATALOGS, \ TS_IMAGING_ALLOWED_CATALOGS, TS_GRISM_ALLOWED_CATALOGS - def catalog_index_check(catalogs): """Check to see if there are any overlaps in the index values of the provided catalogs. Search only by looking @@ -91,3 +92,92 @@ def determine_used_cats(obs_mode, cat_dict): # Remove any catalogs that are set to None cats = [ele for ele in possible_cats if str(ele).lower() != 'none'] return cats + + +def get_nonsidereal_catalog_name(cat_dict, target_name, instrument_name): + """Given a dictionary or nested dictionary of source catalogs, + return the name of the non-sidereal catalog for the given target + name. + + Parameters + ---------- + cat_dict : dict + Dictionary of source catalogs as input by the user + + target_name : str + Target name to search for + + instrument_name : str + Name of instrument used to observe ``target_name`` + + Returns + ------- + cat_file : str + Name of source catalog + """ + target_cat = cat_dict[target_name] + if 'moving_target_to_track' in target_cat.keys(): + if not os.path.isfile(target_cat['moving_target_to_track']): + raise ValueError(("{} is listed as the non-sidereal target catalog for " + "{}, but it appears this file does not exist.".format(target_cat['moving_target_to_track'], + target_name))) + else: + cat_file = target_cat['moving_target_to_track'] + else: + if instrument_name not in target_cat.keys(): + raise ValueError(("Catalog dictionary does not contain a 'moving_target_to_track' catalog " + "for {}. Unable to proceed.".format(target_name))) + else: + if 'moving_target_to_track' in target_cat[instrument_name].keys(): + if not os.path.isfile(target_cat[instrument_name]['moving_target_to_track']): + raise ValueError(("{} is listed as the non-sidereal target catalog for " + "{}, but it appears this file does not exist." + .format(target_cat['moving_target_to_track'], target_name))) + else: + cat_file = target_cat[instrument_name]['moving_target_to_track'] + else: + raise ValueError(("Catalog dictionary does not contain a 'moving_target_to_track' catalog " + "for {}. Unable to proceed.".format(target_name))) + return cat_file + + +def read_nonsidereal_catalog(filename): + """Read in a Mirage formatted non-sidereal source catalog + + Paramters + --------- + filename : str + Name of ascii catalog + + Returns + ------- + catalog_table : astropy.table.Table + Catalog contents + + pixelflag : bool + True if the source position is in units of detector (x, y) + False if RA, Dec + + pixelvelflag : bool + True if the source velocity is given in pixels/hour. + False if arcsec/hour + """ + catalog_table = ascii.read(filename, comment='#') + + # Check to see whether the position is in x,y or ra,dec + pixelflag = False + try: + if 'position_pixels' in catalog_table.meta['comments'][0:4]: + pixelflag = True + except: + pass + + # If present, check whether the velocity entries are pix/sec + # or arcsec/sec. + pixelvelflag = False + try: + if 'velocity_pixels' in catalog_table.meta['comments'][0:4]: + pixelvelflag = True + except: + pass + return catalog_table, pixelflag, pixelvelflag diff --git a/mirage/seed_image/catalog_seed_image.py b/mirage/seed_image/catalog_seed_image.py index a19eccb5e..9945d9e95 100755 --- a/mirage/seed_image/catalog_seed_image.py +++ b/mirage/seed_image/catalog_seed_image.py @@ -1475,7 +1475,7 @@ def movingTargetInputs(self, filename, input_type, MT_tracking=False, if entry['ephemeris_file'].lower() != 'none': self.logger.info(("Using ephemeris file {} to find the location of source #{} in {}." .format(entry['ephemeris_file'], index, filename))) - ra_eph, dec_eph = self.get_ephemeris(entry['ephemeris_file']) + ra_eph, dec_eph = ephemeris_tools.get_ephemeris(entry['ephemeris_file']) # Create list of positions for all frames ra_frames = ra_eph(all_times) @@ -1770,38 +1770,6 @@ def xy_list_to_radec_list(self, x_list, y_list): dec_list = np.array(dec_list) return ra_list, dec_list - def get_ephemeris(self, method): - """Wrapper function to simplify the creation of an ephemeris - - Parameters - ---------- - method : str - Method to use to create the ephemeris. Can be one of two - options: - 1) Name of an ascii file containing an ephemeris. - 2) 'create' - Horizons is queried in order to produce an ephemeris - - Returns - ------- - ephemeris : tup - Tuple of interpolation functions for (RA, Dec). Interpolation - functions are for RA (or Dec) in degrees as a function of - calendar timestamp - """ - if method.lower() != 'create': - ephem = ephemeris_tools.read_ephemeris_file(method) - else: - start_date = datetime.datetime.strptime(self.params['Output']['date_obs'], '%Y-%m-%d') - earlier = start_date - datetime.timedelta(days=1) - later = start_date + datetime.timedelta(days=1) - step_size = 0.1 # days - ephem = ephemeris_tools.query_horizons(self.params['Output']['target_name'], earlier, later, step_size) - raise NotImplementedError('Horizons query not yet working') - - ephemeris = ephemeris_tools.create_interpol_function(ephem) - return ephemeris - - def on_detector(self, xloc, yloc, stampdim, finaldim): """Given a set of x, y locations, stamp image dimensions, and final image dimensions, determine whether the stamp @@ -2146,7 +2114,7 @@ def ephemeris_radec_value_at_obstime(self, src_catalog): self.logger.info(('Calculating target RA, Dec at the observation time using ephemeris file: {}' .format(src_catalog['ephemeris_file'][0]))) - ra_eph, dec_eph = self.get_ephemeris(src_catalog['ephemeris_file'][0]) + ra_eph, dec_eph = ephemeris_tools.get_ephemeris(src_catalog['ephemeris_file'][0]) # If the input x_or_RA and y_or_Dec columns have values of 'none', then # populating them with the values calculated here results in truncated diff --git a/mirage/seed_image/ephemeris_tools.py b/mirage/seed_image/ephemeris_tools.py index c72768463..176b9c706 100644 --- a/mirage/seed_image/ephemeris_tools.py +++ b/mirage/seed_image/ephemeris_tools.py @@ -34,6 +34,38 @@ def create_interpol_function(ephemeris): return ra_interp, dec_interp +def get_ephemeris(method): + """Wrapper function to simplify the creation of an ephemeris + + Parameters + ---------- + method : str + Method to use to create the ephemeris. Can be one of two + options: + 1) Name of an ascii file containing an ephemeris. + 2) 'create' - Horizons is queried in order to produce an ephemeris + + Returns + ------- + ephemeris : tup + Tuple of interpolation functions for (RA, Dec). Interpolation + functions are for RA (or Dec) in degrees as a function of + calendar timestamp + """ + if method.lower() != 'create': + ephem = read_ephemeris_file(method) + else: + raise NotImplementedError('Horizons query not yet working') + start_date = datetime.datetime.strptime(starting_date, '%Y-%m-%d') + earlier = start_date - datetime.timedelta(days=1) + later = start_date + datetime.timedelta(days=1) + step_size = 0.1 # days + ephem = query_horizons(target_name, earlier, later, step_size) + + ephemeris = create_interpol_function(ephem) + return ephemeris + + def to_timestamp(date): """Convert a datetime object into a calendar timestamp object diff --git a/mirage/utils/utils.py b/mirage/utils/utils.py index 5be330be3..1a52960d3 100755 --- a/mirage/utils/utils.py +++ b/mirage/utils/utils.py @@ -883,6 +883,77 @@ def sigma_clipped_mean_value_of_image(array, sigma_value): return meanval +def organize_config_files(): + """Organize the names of the various config files for each instrument + """ + data_dir = expand_environment_variable('MIRAGE_DATA') + modpath = pkg_resources.resource_filename('mirage', '') + + config_info = {} + + config_info['global_subarray_definitions'] = {} + config_info['global_readout_patterns'] = {} + config_info['global_subarray_definition_files'] = {} + config_info['global_readout_pattern_files'] = {} + + config_info['global_crosstalk_files'] = {} + config_info['global_filtpupilcombo_files'] = {} + config_info['global_filter_position_files'] = {} + config_info['global_flux_cal_files'] = {} + config_info['global_psf_wing_threshold_file'] = {} + config_info['global_psfpath'] = {} + + for instrument in 'niriss fgs nircam miri nirspec'.split(): + if instrument.lower() == 'niriss': + readout_pattern_file = 'niriss_readout_pattern.txt' + subarray_def_file = 'niriss_subarrays.list' + crosstalk_file = 'niriss_xtalk_zeros.txt' + filtpupilcombo_file = 'niriss_dual_wheel_list.txt' + filter_position_file = 'niriss_filter_and_pupil_wheel_positions.txt' + flux_cal_file = 'niriss_zeropoints.list' + psf_wing_threshold_file = 'niriss_psf_wing_rate_thresholds.txt' + psfpath = os.path.join(data_dir, 'niriss/gridded_psf_library') + elif instrument.lower() == 'fgs': + readout_pattern_file = 'guider_readout_pattern.txt' + subarray_def_file = 'guider_subarrays.list' + crosstalk_file = 'guider_xtalk_zeros.txt' + filtpupilcombo_file = 'guider_filter_dummy.list' + filter_position_file = 'dummy.txt' + flux_cal_file = 'guider_zeropoints.list' + psf_wing_threshold_file = 'fgs_psf_wing_rate_thresholds.txt' + psfpath = os.path.join(data_dir, 'fgs/gridded_psf_library') + elif instrument.lower() == 'nircam': + readout_pattern_file = 'nircam_read_pattern_definitions.list' + subarray_def_file = 'NIRCam_subarray_definitions.list' + crosstalk_file = 'xtalk20150303g0.errorcut.txt' + filtpupilcombo_file = 'nircam_filter_pupil_pairings.list' + filter_position_file = 'nircam_filter_and_pupil_wheel_positions.txt' + flux_cal_file = 'NIRCam_zeropoints.list' + psf_wing_threshold_file = 'nircam_psf_wing_rate_thresholds.txt' + psfpath = os.path.join(data_dir, 'nircam/gridded_psf_library') + else: + readout_pattern_file = 'N/A' + subarray_def_file = 'N/A' + crosstalk_file = 'N/A' + filtpupilcombo_file = 'N/A' + filter_position_file = 'N/A' + flux_cal_file = 'N/A' + psf_wing_threshold_file = 'N/A' + psfpath = 'N/A' + if instrument in 'niriss fgs nircam'.split(): + config_info['global_subarray_definitions'][instrument] = ascii.read(filename=os.path.join(modpath, 'config', subarray_def_file)) + config_info['global_readout_patterns'][instrument] = ascii.read(filename=os.path.join(modpath, 'config', readout_pattern_file)) + config_info['global_subarray_definition_files'][instrument] = os.path.join(modpath, 'config', subarray_def_file) + config_info['global_readout_pattern_files'][instrument] = os.path.join(modpath, 'config', readout_pattern_file) + config_info['global_crosstalk_files'][instrument] = os.path.join(modpath, 'config', crosstalk_file) + config_info['global_filtpupilcombo_files'][instrument] = os.path.join(modpath, 'config', filtpupilcombo_file) + config_info['global_filter_position_files'][instrument] = os.path.join(modpath, 'config', filter_position_file) + config_info['global_flux_cal_files'][instrument] = os.path.join(modpath, 'config', flux_cal_file) + config_info['global_psf_wing_threshold_file'][instrument] = os.path.join(modpath, 'config', psf_wing_threshold_file) + config_info['global_psfpath'][instrument] = psfpath + return config_info + + def parse_RA_Dec(ra_string, dec_string): """Convert input RA and Dec strings to floats diff --git a/mirage/yaml/yaml_generator.py b/mirage/yaml/yaml_generator.py index 9e0a8f858..a1ac3c0ca 100755 --- a/mirage/yaml/yaml_generator.py +++ b/mirage/yaml/yaml_generator.py @@ -107,9 +107,11 @@ import pysiaf from ..apt import apt_inputs +from ..catalogs.utils import get_nonsidereal_catalog_name, read_nonsidereal_catalog from ..logging import logging_functions from ..reference_files import crds_tools from ..reference_files.utils import get_transmission_file +from ..seed_image import ephemeris_tools from ..utils.constants import FGS1_DARK_SEARCH_STRING, FGS2_DARK_SEARCH_STRING from ..utils.utils import calc_frame_time, ensure_dir_exists, expand_environment_variable from .generate_observationlist import get_observation_dict @@ -214,16 +216,19 @@ def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffil https://mirage-data-simulator.readthedocs.io/en/latest/yaml_generator.html dates : str or dict - Observation dates assocated with the exposures or program. If + Observation dates. Can be a single value, which will be treated as the + start date for the first observation, with all other observations following + immediately after, or a dictionary with one date per observation. If ``dateobs_for_background`` is True, the background for each exposure will be calculated based on these dates. - See Mirage's onling documentation for examples: + See Mirage's online documentation for examples: https://mirage-data-simulator.readthedocs.io/en/latest/yaml_generator.html times : str or dict - Observation times assocated with the exposures or program. Used only - in the case of non-sidereal observations, where it is used to determine - target location for each exposure. + Observation start time. Can be a single value, which will be treated as the + start time for the first observation, with all other observations following + immediately after, or a dictionary with one time per observation. Used in + conjunction with ``dates``. observation_list_file=None verbose : bool @@ -308,7 +313,10 @@ def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffil # Get the path to the 'MIRAGE' package self.modpath = pkg_resources.resource_filename('mirage', '') - self.set_global_definitions() + update the line below to use utils.organize_config_files instead + #self.set_global_definitions() + self.config_information = utils.organize_config_files() + self.path_defs() if (input_xml is not None): @@ -322,6 +330,16 @@ def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffil self.reffile_setup() + + (print('****At end of yaml_generator __init__:')) + for key in self.apt_xml_dict: + print(key) + + + + + + def add_catalogs(self): """ Add list(s) of source catalogs to the table containing the @@ -637,6 +655,25 @@ def create_inputs(self): 'Observation_table_for_' + infile + '_with_yaml_parameters.csv') + + # Add start time info to each element. + # Ignore warnings as astropy.time.Time will give a warning + # related to unknown leap seconds if the date is too far in + # the future. + + + print('***At beginning of create_inputs:') + for key in self.apt_xml_dict: + print(key) + + + #with warnings.catch_warnings(): + # warnings.simplefilter("ignore") + # self.make_start_times_test() + + + + # Read XML file and make observation table apt = apt_inputs.AptInput(input_xml=self.input_xml, pointing_file=self.pointing_file, output_dir=self.output_dir) @@ -649,13 +686,27 @@ def create_inputs(self): apt.create_input_table() self.info = apt.exposure_tab + + + + # Add start time info to each element. # Ignore warnings as astropy.time.Time will give a warning # related to unknown leap seconds if the date is too far in # the future. - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - self.make_start_times() + #with warnings.catch_warnings(): + # warnings.simplefilter("ignore") + # self.make_start_times() + + + print('*****After (the later) make_start_times:') + for key in self.info: + print(key) + stop + + + + # If we have a non-sidereal observation, then we need to # update the pointing information based on the input @@ -816,13 +867,14 @@ def create_inputs(self): file_dict['primary_dither_num'] = np.int(primary_dither) subpix_dither = (tot_dith-1) % subpixtot file_dict['subpix_dither_num'] = subpix_dither + 1 - file_dict['subarray_def_file'] = self.global_subarray_definition_files[instrument] - file_dict['readpatt_def_file'] = self.global_readout_pattern_files[instrument] - file_dict['crosstalk_file'] = self.global_crosstalk_files[instrument] - file_dict['filtpupilcombo_file'] = self.global_filtpupilcombo_files[instrument] - file_dict['filter_position_file'] = self.global_filter_position_files[instrument] - file_dict['flux_cal_file'] = self.global_flux_cal_files[instrument] - file_dict['psf_wing_threshold_file'] = self.global_psf_wing_threshold_file[instrument] + + file_dict['subarray_def_file'] = self.config_information['global_subarray_definition_files'][instrument] + file_dict['readpatt_def_file'] = self.config_information['global_readout_pattern_files'][instrument] + file_dict['crosstalk_file'] = self.config_information['global_crosstalk_files'][instrument] + file_dict['filtpupilcombo_file'] = self.config_information['global_filtpupilcombo_files'][instrument] + file_dict['filter_position_file'] = self.config_information['global_filter_position_files'][instrument] + file_dict['flux_cal_file'] = self.config_information['global_flux_cal_files'][instrument] + file_dict['psf_wing_threshold_file'] = self.config_information['global_psf_wing_threshold_file'][instrument] fname = self.write_yaml(file_dict) yamls.append(fname) self.yaml_files = yamls @@ -1197,12 +1249,17 @@ def nonsidereal_pointing_updates(self): print(self.info['entry_number']) print(self.info['exposure']) print(self.info['dither']) + + for t_ele, a_ele, act_ele, v_ele, obsl_ele, d_ele in zip(self.info['time_obs'], self.info['aperture'], + self.info['act_id'], self.info['visit_num'], + self.info['obs_label'], self.info['dither']): + print(t_ele, a_ele, act_ele, v_ele, obsl_ele, d_ele) stop obs = np.array(self.info['ObservationID']) all_date_obs = np.array(self.info['date_obs']) all_time_obs = np.array(self.info['time_obs']) - all_exposure = np.array(sel.info['exposre']) + all_exposure = np.array(self.info['exposure']) tracking = np.array(self.info['Tracking']) targs = np.array(self.info['TargetID']) inst = np.array(self.info['Instrument']) @@ -1210,7 +1267,7 @@ def nonsidereal_pointing_updates(self): dec_from_pointing_file = np.array(self.info['dec']) nonsidereal_index = np.where(np.array(tracking) == 'non-sidereal') all_nonsidereal_targs = targs[nonsidereal_index] - nonsidereal_targs, u_indexes = np.unique(targs[nonsidereal_index], return_indexes=True) + nonsidereal_targs, u_indexes = np.unique(targs[nonsidereal_index], return_index=True) nonsidereal_instruments = inst[nonsidereal_index][u_indexes] # Check that all non-sidereal targets have catalogs associated with them @@ -1241,12 +1298,12 @@ def nonsidereal_pointing_updates(self): if 'ephemeris_file' in catalog_table.colnames: ephemeris_file = catalog_table['ephemeris_file'][0] - ra_ephem, dec_ephem = ephemeris_tools.read_ephemeris_file(ephemeris_file) + ra_ephem, dec_ephem = ephemeris_tools.get_ephemeris(ephemeris_file) all_times = [ephemeris_tools.to_timestamp(elem) for elem in start_dates] # Create list of positions for all frames - ra_target = ra_eph(all_times) - dec_target = dec_eph(all_times) + ra_target = ra_ephem(all_times) + dec_target = dec_ephem(all_times) else: if not pos_in_xy: if not vel_in_xy: @@ -1278,9 +1335,19 @@ def nonsidereal_pointing_updates(self): ra_from_pointing_file[obs_exp_indexes] = ra_target dec_from_pointing_file[obs_exp_indexes] = dec_target + self.info['TargetRA'] = ra_from_pointing_file + self.info['TargetDec'] = dec_from_pointing_file self.info['ra'] = ra_from_pointing_file self.info['dec'] = dec_from_pointing_file + + + for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): + print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) + + + + # Create a pysiaf.Siaf instance for each instrument in the proposal print('Lines below are untested') siaf_dictionary = {} @@ -1291,97 +1358,27 @@ def nonsidereal_pointing_updates(self): self.info = apt_inputs.ra_dec_update(self.info, siaf_dictionary) - """ - read in catalog - determine ephemeris file or velocities - - 1. read in ephemeris file - 2. get ra, dec at observation date+time - - or - - 1. calculate ra, dec from ra,dec,ra_vel,dec_vel and observation date+time - """ + for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): + print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) - - stop - - - - @staticmethod - def read_nonsidereal_catalog(filename): - """Read in a Mirage formatted non-sidereal source catalog """ - catalog_table = ascii.read(ns_catalog, comment='#') + read in catalog + determine ephemeris file or velocities - # Check to see whether the position is in x,y or ra,dec - pixelflag = False - try: - if 'position_pixels' in catalog_table.meta['comments'][0:4]: - pixelflag = True - except: - pass - - # If present, check whether the velocity entries are pix/sec - # or arcsec/sec. - pixelvelflag = False - try: - if 'velocity_pixels' in catalog_table.meta['comments'][0:4]: - pixelvelflag = True - except: - pass - return catalog_table, pixelflag, pixelvelflag + 1. read in ephemeris file + 2. get ra, dec at observation date+time + or + 1. calculate ra, dec from ra,dec,ra_vel,dec_vel and observation date+time + """ - @staticmethod - def get_nonsidereal_catalog_name(cat_dict, target_name, instrument_name): - """Given a dictionary or nested dictionary of source catalogs, - return the name of the non-sidereal catalog for the given target - name. - Parameters - ---------- - cat_dict : dict - Dictionary of source catalogs as input by the user - target_name : str - Target name to search for - instrument_name : str - Name of instrument used to observe ``target_name`` - - Returns - ------- - cat_file : str - Name of source catalog - """ - target_cat = cat_dict[target_name] - if 'moving_target_to_track' in target_cat.keys(): - if not os.path.isfile(target_cat['moving_target_to_track']): - raise ValueError(("{} is listed as the non-sidereal target catalog for " - "{}, but it appears this file does not exist.".format(target_cat['moving_target_to_track'], - target_name))) - else: - cat_file = target_cat['moving_target_to_track'] - else: - if instrument_name not in target_cat.keys(): - raise ValueError(("Catalog dictionary does not contain a 'moving_target_to_track' catalog " - "for {}. Unable to proceed.".format(target_name))) - else: - if 'moving_target_to_track' in target_cat[instrument_name].keys(): - if not os.path.isfile(target_cat[instrument_name]['moving_target_to_track']): - raise ValueError(("{} is listed as the non-sidereal target catalog for " - "{}, but it appears this file does not exist." - .format(target_cat['moving_target_to_track'], target_name))) - else: - cat_file = target_cat[instrument_name]['moving_target_to_track'] - else: - raise ValueError(("Catalog dictionary does not contain a 'moving_target_to_track' catalog " - "for {}. Unable to proceed.".format(target_name))) - return cat_file + stop def set_global_definitions(self): @@ -1698,6 +1695,215 @@ def make_start_times(self): self.info['nskip'] = nskip self.info['namp'] = namp + + + + def make_start_times_test(self): + """Create exposure start times for each entry in the observation dictionary.""" + date_obs = [] + time_obs = [] + expstart = [] + nframe = [] + nskip = [] + namp = [] + + # choose arbitrary start time for each epoch + epoch_base_time = '16:44:12' + epoch_base_time0 = deepcopy(epoch_base_time) + + if 'epoch_start_date' in self.apt_xml_dict.keys(): + epoch_base_date = self.apt_xml_dict['epoch_start_date'][0] + else: + epoch_base_date = self.apt_xml_dict['Date'][0] + base = Time(epoch_base_date + 'T' + epoch_base_time) + base_date, base_time = base.iso.split() + + # Pick some arbirary overhead values + act_overhead = 90 # seconds. (filter change) + visit_overhead = 600 # seconds. (slew) + + # Get visit, activity_id, dither_id info for first exposure + ditherid = self.apt_xml_dict['dither'][0] + actid = self.apt_xml_dict['act_id'][0] + visit = self.apt_xml_dict['visit_num'][0] + # obsname = self.apt_xml_dict['obs_label'][0] + + # for i in range(len(self.apt_xml_dict['Module'])): + for i, instrument in enumerate(self.apt_xml_dict['Instrument']): + # Get dither/visit + # Files with the same activity_id should have the same start time + # Overhead after a visit break should be large, smaller between + # exposures within a visit + next_actid = self.apt_xml_dict['act_id'][i] + next_visit = self.apt_xml_dict['visit_num'][i] + next_obsname = self.apt_xml_dict['obs_label'][i] + next_ditherid = self.apt_xml_dict['dither'][i] + + # Find the readpattern of the file + readpatt = self.apt_xml_dict['ReadoutPattern'][i] + groups = np.int(self.apt_xml_dict['Groups'][i]) + integrations = np.int(self.apt_xml_dict['Integrations'][i]) + + if instrument.lower() in ['miri', 'nirspec']: + nframe.append(0) + nskip.append(0) + namp.append(0) + date_obs.append(base_date) + time_obs.append(base_time) + expstart.append(base.mjd) + + else: + # Now read in readpattern definitions + readpatt_def = self.global_readout_patterns[instrument.lower()] + + # Read in file containing subarray definitions + subarray_def = self.global_subarray_definitions[instrument.lower()] + + match2 = readpatt == readpatt_def['name'] + if np.sum(match2) == 0: + raise RuntimeError(("WARNING!! Readout pattern {} not found in definition file." + .format(readpatt))) + + # Now get nframe and nskip so we know how many frames in a group + fpg = np.int(readpatt_def['nframe'][match2][0]) + spg = np.int(readpatt_def['nskip'][match2][0]) + nframe.append(fpg) + nskip.append(spg) + + # Get the aperture name. For non-NIRCam instruments, + # this is simply the self.apt_xml_dict['aperture']. But for NIRCam, + # we need to be careful of entries like NRCBS_FULL, which is used + # for observations using all 4 shortwave B detectors. In that case, + # we need to build the aperture name from the combination of detector + # and subarray name. + aperture = self.apt_xml_dict['aperture'][i] + + # Get the number of amps from the subarray definition file + match = aperture == subarray_def['AperName'] + + # needed for NIRCam case + if np.sum(match) == 0: + aperture = [apername for apername, name in + np.array(subarray_def['AperName', 'Name']) if + (sub in apername) or (sub in name)] + + match = aperture == subarray_def['AperName'] + + if len(aperture) > 1 or len(aperture) == 0 or np.sum(match) == 0: + raise ValueError('Cannot combine detector {} and subarray {}\ + into valid aperture name.'.format(det, sub)) + # We don't want aperture as a list + aperture = aperture[0] + + # For grism tso observations, get the number of + # amplifiers to use from the APT file. + # For other modes, check the subarray def table. + try: + amp = int(self.apt_xml_dict['NumOutputs'][i]) + except ValueError: + amp = subarray_def['num_amps'][match][0] + + # Default to amps=4 for subarrays that can have 1 or 4 + # if the number of amps is not defined. Hopefully we + # should never enter this code block given the lines above. + if amp == 0: + amp = 4 + self.logger.info(('Aperture {} can be used with 1 or 4 readout amplifiers. Defaulting to use 4.' + 'In the future this information should be made a user input.'.format(aperture))) + namp.append(amp) + + # same activity ID + # Remove this for now, since Mirage was not correctly + # specifying activities. At the moment all exposures have + # the same activity ID, which means we must allow the + # the epoch_start_date to change even if the activity ID + # does not. This will change back in the future when we + # figure out more realistic activity ID values. + #if next_actid == actid: + # # in this case, the start time should remain the same + # date_obs.append(base_date) + # time_obs.append(base_time) + # expstart.append(base.mjd) + # continue + + epoch_date = self.apt_xml_dict['epoch_start_date'][i] + epoch_time = deepcopy(epoch_base_time0) + + # new epoch - update the base time + if epoch_date != epoch_base_date: + epoch_base_date = deepcopy(epoch_date) + base = Time(epoch_base_date + 'T' + epoch_base_time) + base_date, base_time = base.iso.split() + basereset = True + date_obs.append(base_date) + time_obs.append(base_time) + expstart.append(base.mjd) + actid = deepcopy(next_actid) + visit = deepcopy(next_visit) + obsname = deepcopy(next_obsname) + continue + + # new visit + if next_visit != visit: + # visit break. Larger overhead + overhead = visit_overhead + + # This block should be updated when we have more realistic + # activity IDs + elif ((next_actid > actid) & (next_visit == visit)): + # same visit, new activity. Smaller overhead + overhead = act_overhead + elif ((next_ditherid != ditherid) & (next_visit == visit)): + # same visit, new dither position. Smaller overhead + overhead = act_overhead + else: + # same observation, activity, dither. Filter changes + # will still fall in here, which is not accurate + overhead = 10. + + # For cases where the base time needs to change + # continue down here + siaf_inst = self.apt_xml_dict['Instrument'][i].upper() + if siaf_inst == 'NIRCAM': + siaf_inst = "NIRCam" + siaf_obj = pysiaf.Siaf(siaf_inst)[aperture] + + # Calculate the readout time for a single frame + frametime = calc_frame_time(siaf_inst, aperture, + siaf_obj.XSciSize, siaf_obj.YSciSize, amp) + + # Estimate total exposure time + exptime = ((fpg + spg) * groups + fpg) * integrations * frametime + + # Delta should include the exposure time, plus overhead + delta = TimeDelta(exptime + overhead, format='sec') + base += delta + base_date, base_time = base.iso.split() + + # Add updated dates and times to the list + date_obs.append(base_date) + time_obs.append(base_time) + expstart.append(base.mjd) + + # increment the activity ID and visit + actid = deepcopy(next_actid) + visit = deepcopy(next_visit) + obsname = deepcopy(next_obsname) + ditherid = deepcopy(next_ditherid) + + self.apt_xml_dict['date_obs'] = date_obs + self.apt_xml_dict['time_obs'] = time_obs + # self.apt_xml_dict['expstart'] = expstart + self.apt_xml_dict['nframe'] = nframe + self.apt_xml_dict['nskip'] = nskip + self.apt_xml_dict['namp'] = namp + + + + + + + def multiple_catalog_match(self, filter, cattype, matchlist): """ Alert the user if more than one catalog matches the filter/pupil From f70cbc66c71b2bc9d2f6456b42f7de3abafd72ca Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 25 Nov 2020 15:23:58 -0500 Subject: [PATCH 03/13] Functions for nonsidereal. Need ra_dec_update for sidereal. --- mirage/apt/apt_inputs.py | 463 +++++++++++++----------- mirage/utils/utils.py | 12 +- mirage/yaml/generate_observationlist.py | 44 +-- mirage/yaml/yaml_generator.py | 72 ++-- 4 files changed, 321 insertions(+), 270 deletions(-) diff --git a/mirage/apt/apt_inputs.py b/mirage/apt/apt_inputs.py index e440e5fc3..926a9914b 100755 --- a/mirage/apt/apt_inputs.py +++ b/mirage/apt/apt_inputs.py @@ -142,7 +142,7 @@ def add_observation_info(self, intab): with open(self.observation_list_file, 'r') as infile: self.obstab = yaml.safe_load(infile) - OBSERVATION_LIST_FIELDS = 'Date Time PAV3 Filter PointSourceCatalog GalaxyCatalog ' \ + OBSERVATION_LIST_FIELDS = 'Date PAV3 Filter PointSourceCatalog GalaxyCatalog ' \ 'ExtendedCatalog ExtendedScale ExtendedCenter MovingTargetList ' \ 'MovingTargetSersic MovingTargetExtended ' \ 'MovingTargetConvolveExtended MovingTargetToTrack ' \ @@ -186,7 +186,7 @@ def add_observation_info(self, intab): if instrument == 'nircam': # keep the number of entries in the dictionary consistent for key in OBSERVATION_LIST_FIELDS: - if key in ['Date', 'Time', 'PAV3', 'Instrument', 'CosmicRayLibrary', 'CosmicRayScale']: + if key in ['Date', 'PAV3', 'Instrument', 'CosmicRayLibrary', 'CosmicRayScale']: value = str(entry[key]) else: value = str(None) @@ -313,9 +313,6 @@ def create_input_table(self, verbose=False): - make_start_times??? - - @@ -331,6 +328,19 @@ def create_input_table(self, verbose=False): self.exposure_tab = self.expand_for_detectors(observation_dictionary) + + + # Add start times for each exposure + self.exposure_tab = make_start_times(self.exposure_tab) + + + #for ob, ap, date, time, act, vis in zip(self.exposure_tab['ObservationID'], self.exposure_tab['aperture'], self.exposure_tab['date_obs'], self.exposure_tab['time_obs'], self.exposure_tab['act_id'], self.exposure_tab['visit_num']): + # print(ob, vis, act, ap, date, time) + #stop + + + + # fix data for filename generation # set parallel seq id for j, isparallel in enumerate(self.exposure_tab['ParallelInstrument']): @@ -960,212 +970,6 @@ def global_alignment_pointing(self, obs_dict): return obs_dict - def make_start_times_test(self, obs_info): - """Create exposure start times for each entry in the observation dictionary.""" - date_obs = [] - time_obs = [] - expstart = [] - nframe = [] - nskip = [] - namp = [] - - - - # choose arbitrary start time for each epoch - epoch_base_time = '16:44:12' - epoch_base_time0 = copy.deepcopy(epoch_base_time) - - if 'epoch_start_date' in obs_info.keys(): - epoch_base_date = obs_info['epoch_start_date'][0] - else: - epoch_base_date = obs_info['Date'][0] - base = Time(epoch_base_date + 'T' + epoch_base_time) - base_date, base_time = base.iso.split() - - # Pick some arbirary overhead values - act_overhead = 90 # seconds. (filter change) - visit_overhead = 600 # seconds. (slew) - - # Get visit, activity_id, dither_id info for first exposure - ditherid = obs_info['dither'][0] - actid = obs_info['act_id'][0] - visit = obs_info['visit_num'][0] - # obsname = obs_info['obs_label'][0] - - # for i in range(len(obs_info['Module'])): - for i, instrument in enumerate(obs_info['Instrument']): - # Get dither/visit - # Files with the same activity_id should have the same start time - # Overhead after a visit break should be large, smaller between - # exposures within a visit - next_actid = obs_info['act_id'][i] - next_visit = obs_info['visit_num'][i] - next_obsname = obs_info['obs_label'][i] - next_ditherid = obs_info['dither'][i] - - # Find the readpattern of the file - readpatt = obs_info['ReadoutPattern'][i] - groups = np.int(obs_info['Groups'][i]) - integrations = np.int(obs_info['Integrations'][i]) - - if instrument.lower() in ['miri', 'nirspec']: - nframe.append(0) - nskip.append(0) - namp.append(0) - date_obs.append(base_date) - time_obs.append(base_time) - expstart.append(base.mjd) - - else: - # Now read in readpattern definitions - #readpatt_def = self.global_readout_patterns[instrument.lower()] - print('XXXXXTry setting up config info using the moved funcn in utils!!!!') - # Read in file containing subarray definitions - #subarray_def = self.global_subarray_definitions[instrument.lower()] - config_information = utils.organize_config_files() - readpatt_def = config_information['global_readout_patterns'][instrument.lower()] - subarray_def = config_information['global_subarray_definitions'][instrument.lower()] - - match2 = readpatt == readpatt_def['name'] - if np.sum(match2) == 0: - raise RuntimeError(("WARNING!! Readout pattern {} not found in definition file." - .format(readpatt))) - - # Now get nframe and nskip so we know how many frames in a group - fpg = np.int(readpatt_def['nframe'][match2][0]) - spg = np.int(readpatt_def['nskip'][match2][0]) - nframe.append(fpg) - nskip.append(spg) - - # Get the aperture name. For non-NIRCam instruments, - # this is simply the obs_info['aperture']. But for NIRCam, - # we need to be careful of entries like NRCBS_FULL, which is used - # for observations using all 4 shortwave B detectors. In that case, - # we need to build the aperture name from the combination of detector - # and subarray name. - aperture = obs_info['aperture'][i] - - # Get the number of amps from the subarray definition file - match = aperture == subarray_def['AperName'] - - # needed for NIRCam case - if np.sum(match) == 0: - aperture = [apername for apername, name in - np.array(subarray_def['AperName', 'Name']) if - (sub in apername) or (sub in name)] - - match = aperture == subarray_def['AperName'] - - if len(aperture) > 1 or len(aperture) == 0 or np.sum(match) == 0: - raise ValueError('Cannot combine detector {} and subarray {}\ - into valid aperture name.'.format(det, sub)) - # We don't want aperture as a list - aperture = aperture[0] - - # For grism tso observations, get the number of - # amplifiers to use from the APT file. - # For other modes, check the subarray def table. - try: - amp = int(obs_info['NumOutputs'][i]) - except ValueError: - amp = subarray_def['num_amps'][match][0] - - # Default to amps=4 for subarrays that can have 1 or 4 - # if the number of amps is not defined. Hopefully we - # should never enter this code block given the lines above. - if amp == 0: - amp = 4 - self.logger.info(('Aperture {} can be used with 1 or 4 readout amplifiers. Defaulting to use 4.' - 'In the future this information should be made a user input.'.format(aperture))) - namp.append(amp) - - # same activity ID - # Remove this for now, since Mirage was not correctly - # specifying activities. At the moment all exposures have - # the same activity ID, which means we must allow the - # the epoch_start_date to change even if the activity ID - # does not. This will change back in the future when we - # figure out more realistic activity ID values. - #if next_actid == actid: - # # in this case, the start time should remain the same - # date_obs.append(base_date) - # time_obs.append(base_time) - # expstart.append(base.mjd) - # continue - - epoch_date = obs_info['epoch_start_date'][i] - epoch_time = copy.deepcopy(epoch_base_time0) - - # new epoch - update the base time - if epoch_date != epoch_base_date: - epoch_base_date = copy.deepcopy(epoch_date) - base = Time(epoch_base_date + 'T' + epoch_base_time) - base_date, base_time = base.iso.split() - basereset = True - date_obs.append(base_date) - time_obs.append(base_time) - expstart.append(base.mjd) - actid = copy.deepcopy(next_actid) - visit = copy.deepcopy(next_visit) - obsname = copy.deepcopy(next_obsname) - continue - - # new visit - if next_visit != visit: - # visit break. Larger overhead - overhead = visit_overhead - - # This block should be updated when we have more realistic - # activity IDs - elif ((next_actid > actid) & (next_visit == visit)): - # same visit, new activity. Smaller overhead - overhead = act_overhead - elif ((next_ditherid != ditherid) & (next_visit == visit)): - # same visit, new dither position. Smaller overhead - overhead = act_overhead - else: - # same observation, activity, dither. Filter changes - # will still fall in here, which is not accurate - overhead = 10. - - # For cases where the base time needs to change - # continue down here - siaf_inst = obs_info['Instrument'][i].upper() - if siaf_inst == 'NIRCAM': - siaf_inst = "NIRCam" - siaf_obj = pysiaf.Siaf(siaf_inst)[aperture] - - # Calculate the readout time for a single frame - frametime = utils.calc_frame_time(siaf_inst, aperture, - siaf_obj.XSciSize, siaf_obj.YSciSize, amp) - - # Estimate total exposure time - exptime = ((fpg + spg) * groups + fpg) * integrations * frametime - - # Delta should include the exposure time, plus overhead - delta = TimeDelta(exptime + overhead, format='sec') - base += delta - base_date, base_time = base.iso.split() - - # Add updated dates and times to the list - date_obs.append(base_date) - time_obs.append(base_time) - expstart.append(base.mjd) - - # increment the activity ID and visit - actid = copy.deepcopy(next_actid) - visit = copy.deepcopy(next_visit) - obsname = copy.deepcopy(next_obsname) - ditherid = copy.deepcopy(next_ditherid) - - obs_info['date_obs'] = date_obs - obs_info['time_obs'] = time_obs - # obs_info['expstart'] = expstart - obs_info['nframe'] = nframe - obs_info['nskip'] = nskip - obs_info['namp'] = namp - return obs_info - def tight_dithers(self, input_dict): """ In NIRCam, when the 'FULL' dither pattern is @@ -1362,6 +1166,243 @@ def get_filters(pointing_info): return filters +def make_start_times(obs_info): + """Create exposure start times for each entry in the observation dictionary. + + Parameters + ---------- + obs_info : dict + Dictionary of exposures. Development was around a dictionary containing + APT xml-derived properties as well as pointing file properties. Should + be before expanding to have one entry for each detector in each exposure. + + Returns + ------- + obs_info : dict + Modified dictionary with observation dates and times added + """ + date_obs = [] + time_obs = [] + expstart = [] + nframe = [] + nskip = [] + namp = [] + + # Read in file containing subarray definitions + config_information = utils.organize_config_files() + + # choose arbitrary start time for each epoch + #epoch_base_time = '16:44:12' + #epoch_base_time0 = copy.deepcopy(epoch_base_time) + + if 'epoch_start_date' in obs_info.keys(): + epoch_base_date = obs_info['epoch_start_date'][0] + else: + epoch_base_date = obs_info['Date'][0] + + base = Time(obs_info['epoch_start_date'][0]) + #base = Time(epoch_base_date + 'T' + epoch_base_time) + base_date, base_time = base.iso.split() + + # Pick some arbirary overhead values + act_overhead = 90 # seconds. (filter change) + visit_overhead = 600 # seconds. (slew) + + # Get visit, activity_id, dither_id info for first exposure + ditherid = obs_info['dither'][0] + actid = obs_info['act_id'][0] + visit = obs_info['visit_num'][0] + obsid = obs_info['ObservationID'][0] + + for i, instrument in enumerate(obs_info['Instrument']): + # Get dither/visit + # Files with the same activity_id should have the same start time + # Overhead after a visit break should be large, smaller between + # exposures within a visit + next_actid = obs_info['act_id'][i] + next_visit = obs_info['visit_num'][i] + next_obsname = obs_info['obs_label'][i] + next_ditherid = obs_info['dither'][i] + next_obsid = obs_info['ObservationID'][i] + + # Find the readpattern of the file + readpatt = obs_info['ReadoutPattern'][i] + groups = np.int(obs_info['Groups'][i]) + integrations = np.int(obs_info['Integrations'][i]) + + if instrument.lower() in ['miri', 'nirspec']: + nframe.append(0) + nskip.append(0) + namp.append(0) + date_obs.append(base_date) + time_obs.append(base_time) + expstart.append(base.mjd) + + else: + readpatt_def = config_information['global_readout_patterns'][instrument.lower()] + subarray_def = config_information['global_subarray_definitions'][instrument.lower()] + + match2 = readpatt == readpatt_def['name'] + if np.sum(match2) == 0: + raise RuntimeError(("WARNING!! Readout pattern {} not found in definition file." + .format(readpatt))) + + # Now get nframe and nskip so we know how many frames in a group + fpg = np.int(readpatt_def['nframe'][match2][0]) + spg = np.int(readpatt_def['nskip'][match2][0]) + nframe.append(fpg) + nskip.append(spg) + + # Get the aperture name. For non-NIRCam instruments, + # this is simply the obs_info['aperture']. But for NIRCam, + # we need to be careful of entries like NRCBS_FULL, which is used + # for observations using all 4 shortwave B detectors. In that case, + # we need to build the aperture name from the combination of detector + # and subarray name. + aperture = obs_info['aperture'][i] + + # Get the number of amps from the subarray definition file + match = aperture == subarray_def['AperName'] + + + + + + # needed for NIRCam case + if np.sum(match) == 0: + self.logger.info(('Aperture: {} does not match any entries in the subarray definition file. Guessing at the ' + 'aperture for the purpose of calculating the exposure time and number of amps.'.format(aperture))) + sub = aperture.split('_')[1] + aperture = [apername for apername, name in + np.array(subarray_def['AperName', 'Name']) if + (sub in apername) or (sub in name)] + + match = aperture == subarray_def['AperName'] + + if len(aperture) > 1 or len(aperture) == 0 or np.sum(match) == 0: + raise ValueError('Cannot combine detector {} and subarray {}\ + into valid aperture name.'.format(det, sub)) + # We don't want aperture as a list + aperture = aperture[0] + + # For grism tso observations, get the number of + # amplifiers to use from the APT file. + # For other modes, check the subarray def table. + try: + amp = int(obs_info['NumOutputs'][i]) + except ValueError: + amp = subarray_def['num_amps'][match][0] + + # Default to amps=4 for subarrays that can have 1 or 4 + # if the number of amps is not defined. Hopefully we + # should never enter this code block given the lines above. + if amp == 0: + amp = 4 + self.logger.info(('Aperture {} can be used with 1 or 4 readout amplifiers. Defaulting to use 4.' + 'In the future this information should be made a user input.'.format(aperture))) + namp.append(amp) + + # same activity ID + # Remove this for now, since Mirage was not correctly + # specifying activities. At the moment all exposures have + # the same activity ID, which means we must allow the + # the epoch_start_date to change even if the activity ID + # does not. This will change back in the future when we + # figure out more realistic activity ID values. + #if next_actid == actid: + # # in this case, the start time should remain the same + # date_obs.append(base_date) + # time_obs.append(base_time) + # expstart.append(base.mjd) + # continue + + epoch_date = obs_info['epoch_start_date'][i] + #epoch_time = copy.deepcopy(epoch_base_time0) + + # new epoch - update the base time + if epoch_date != epoch_base_date: + epoch_base_date = copy.deepcopy(epoch_date) + #base = Time(epoch_base_date + 'T' + epoch_base_time) + base = Time(obs_info['epoch_start_date'][i]) + base_date, base_time = base.iso.split() + basereset = True + date_obs.append(base_date) + time_obs.append(base_time) + expstart.append(base.mjd) + actid = copy.deepcopy(next_actid) + visit = copy.deepcopy(next_visit) + obsid = copy.deepcopy(next_obsid) + obsname = copy.deepcopy(next_obsname) + ditherid = copy.deepcopy(next_ditherid) + continue + + # new observation or visit (if a different epoch time has + # not been provided) + if ((next_obsid != obsid) | (next_visit != visit)): + # visit break. Larger overhead + overhead = visit_overhead + print("new visit/obs: {} {}".format(next_obsid, next_visit)) + elif ((next_actid > actid) & (next_visit == visit)): + # This block should be updated when we have more realistic + # activity IDs + # same visit, new activity. Smaller overhead + overhead = act_overhead + print('Next activity: {}'.format(next_actid)) + elif ((next_ditherid != ditherid) & (next_visit == visit)): + # same visit, new dither position. Smaller overhead + overhead = act_overhead + print('next dither: {}'.format(next_ditherid)) + else: + # same observation, activity, dither. Filter changes + # will still fall in here, which is not accurate + overhead = 0. # Reset frame captured in exptime below + print('same activity') + + # For cases where the base time needs to change + # continue down here + siaf_inst = obs_info['Instrument'][i].upper() + siaf_obj = Siaf(siaf_inst)[aperture] + + # Calculate the readout time for a single frame + frametime = utils.calc_frame_time(siaf_inst, aperture, + siaf_obj.XSciSize, siaf_obj.YSciSize, amp) + + # Estimate total exposure time + exptime = ((fpg + spg) * groups + fpg) * integrations * frametime + + if ((next_obsid == obsid) & (next_visit == visit) & (next_actid == actid) & (next_ditherid == ditherid)): + # If we are in the same exposure (but with a different detector), + # then we should keep the start time the same + delta = TimeDelta(0., format='sec') + else: + # If we are moving on to the next exposure, activity, or visit + # then move the start time by the expoure time of the current + # exposure, plus the overhead + delta = TimeDelta(exptime + overhead, format='sec') + + base += delta + base_date, base_time = base.iso.split() + + # Add updated dates and times to the list + date_obs.append(base_date) + time_obs.append(base_time) + expstart.append(base.mjd) + + # increment the activity ID and visit + actid = copy.deepcopy(next_actid) + visit = copy.deepcopy(next_visit) + obsname = copy.deepcopy(next_obsname) + ditherid = copy.deepcopy(next_ditherid) + obsid = copy.deepcopy(next_obsid) + + obs_info['date_obs'] = date_obs + obs_info['time_obs'] = time_obs + obs_info['nframe'] = nframe + obs_info['nskip'] = nskip + obs_info['namp'] = namp + return obs_info + + def ra_dec_update(exposure_dict, siaf_instances, verbose=False): """Given the V2, V3 values for the reference locations associated with detector apertures, calculate corresponding RA, Dec. diff --git a/mirage/utils/utils.py b/mirage/utils/utils.py index 1a52960d3..e9282a1cc 100755 --- a/mirage/utils/utils.py +++ b/mirage/utils/utils.py @@ -941,8 +941,8 @@ def organize_config_files(): psf_wing_threshold_file = 'N/A' psfpath = 'N/A' if instrument in 'niriss fgs nircam'.split(): - config_info['global_subarray_definitions'][instrument] = ascii.read(filename=os.path.join(modpath, 'config', subarray_def_file)) - config_info['global_readout_patterns'][instrument] = ascii.read(filename=os.path.join(modpath, 'config', readout_pattern_file)) + config_info['global_subarray_definitions'][instrument] = asc.read(os.path.join(modpath, 'config', subarray_def_file)) + config_info['global_readout_patterns'][instrument] = asc.read(os.path.join(modpath, 'config', readout_pattern_file)) config_info['global_subarray_definition_files'][instrument] = os.path.join(modpath, 'config', subarray_def_file) config_info['global_readout_pattern_files'][instrument] = os.path.join(modpath, 'config', readout_pattern_file) config_info['global_crosstalk_files'][instrument] = os.path.join(modpath, 'config', crosstalk_file) @@ -977,6 +977,14 @@ def parse_RA_Dec(ra_string, dec_string): dec_degrees : float Declination value in degrees """ + # First, a quick check to see if the inputs are in + # decimal degrees already. + try: + return np.float(ra_string), np.float(dec_string) + except ValueError: + pass + + # Convert from HMS or ::: to decimal degrees try: ra_string = ra_string.lower() ra_string = ra_string.replace("h", ":") diff --git a/mirage/yaml/generate_observationlist.py b/mirage/yaml/generate_observationlist.py index bb45243be..88269bcd3 100755 --- a/mirage/yaml/generate_observationlist.py +++ b/mirage/yaml/generate_observationlist.py @@ -459,7 +459,7 @@ def expand_for_dithers(indict, verbose=True): def get_observation_dict(xml_file, yaml_file, catalogs, - parameter_overrides={'cosmic_rays': None, 'background': None, 'roll_angle': None, 'dates': None, 'times': None}, + parameter_overrides={'cosmic_rays': None, 'background': None, 'roll_angle': None, 'dates': None}, verbose=False): """Write observation list file (required mirage input) on the basis of APT files. @@ -516,9 +516,9 @@ def get_observation_dict(xml_file, yaml_file, catalogs, # Set default values. These are overwritten if there is an appropriate # entry in parameter_defaults + default_time = '00:00:00' default_values = {} - default_values['Date'] = '2021-10-04' - default_values['Time'] = '12:34' + default_values['Date'] = '2022-10-04T00:00:00' default_values['PAV3'] = '0.' default_values['PointsourceCatalog'] = 'None' default_values['GalaxyCatalog'] = 'None' @@ -586,11 +586,16 @@ def get_observation_dict(xml_file, yaml_file, catalogs, # Dates # dates = '2019-5-25' - # dates = {'001': '2019-05-25', '002': '2019-11-15'} + # dates = {'001': '2019-05-25', '002': '2019-11-15T12:13:14'} dates = parameter_overrides['dates'] if dates is not None: if isinstance(dates, str): - default_values['Date'] = dates + if 'T' in dates: + # In the end we need dates in the format of YYYY-MM-DDTHH:MM:SS + default_values['Date'] = dates + else: + # If the time part is not present in the input, then add it. + default_values['Date'] = '{}T{}'.format(dates, default_time) # Now set dates to None so that it won't be used when looping # over observations below dates = None @@ -598,17 +603,6 @@ def get_observation_dict(xml_file, yaml_file, catalogs, # Just use dates below when looping over observations pass - times = parameter_overrides['times'] - if times is not None: - if isinstance(times, str): - default_values['Time'] = times - # Now set times to None so that it won't be used when looping - # over observations below - times = None - else: - # Just use times below when looping over observations - pass - # Roll angle, aka PAV3 # pav3 = 34.5 # pav3 = {'001': 34.5, '002': 154.5} @@ -696,23 +690,16 @@ def get_observation_dict(xml_file, yaml_file, catalogs, date_value = default_values['Date'] else: try: - date_value = dates[observation_number] + value = dates[observation_number] + if 'T' in value: + date_value = dates[observation_number] + else: + date_value = '{}T{}'.format(dates[observation_number], default_time) except KeyError: logger.error(("\n\nERROR: No date value specified for Observation {} in date dictionary. " "Quitting.\n\n".format(observation_number))) raise KeyError - # Get the proper time value - if times is None: - time_value = default_values['Time'] - else: - try: - time_value = times[observation_number] - except KeyError: - logger.error(("\n\nERROR: No time value specified for Observation {} in time dictionary. " - "Quitting.\n\n".format(observation_number))) - raise KeyError - # Get the proper PAV3 value if pav3 is None: pav3_value = default_values['PAV3'] @@ -771,7 +758,6 @@ def get_observation_dict(xml_file, yaml_file, catalogs, " EntryNumber{}:\n".format(entry_number), " Instrument: {}\n".format(instrument), " Date: {}\n".format(date_value), - " Time: {}\n".format(time_value), " PAV3: {}\n".format(pav3_value), " DitherIndex: {}\n".format(dither_index), " CosmicRayLibrary: {}\n".format(cr_library_value), diff --git a/mirage/yaml/yaml_generator.py b/mirage/yaml/yaml_generator.py index a1ac3c0ca..7786e32ed 100755 --- a/mirage/yaml/yaml_generator.py +++ b/mirage/yaml/yaml_generator.py @@ -130,7 +130,7 @@ class SimInput: def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffile_defaults='crds', reffile_overrides=None, use_JWST_pipeline=True, catalogs=None, cosmic_rays=None, - background=None, roll_angle=None, dates=None, times=None, + background=None, roll_angle=None, dates=None, observation_list_file=None, verbose=False, output_dir='./', simdata_output_dir='./', dateobs_for_background=False, segmap_flux_limit=None, segmap_flux_limit_units=None, offline=False): @@ -266,7 +266,7 @@ def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffil self.logger.info('Original log file name: ./{}'.format(STANDARD_LOGFILE_NAME)) parameter_overrides = {'cosmic_rays': cosmic_rays, 'background': background, 'roll_angle': roll_angle, - 'dates': dates, 'times': times} + 'dates': dates} self.info = {} self.input_xml = input_xml @@ -313,8 +313,6 @@ def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffil # Get the path to the 'MIRAGE' package self.modpath = pkg_resources.resource_filename('mirage', '') - update the line below to use utils.organize_config_files instead - #self.set_global_definitions() self.config_information = utils.organize_config_files() self.path_defs() @@ -699,10 +697,13 @@ def create_inputs(self): # self.make_start_times() - print('*****After (the later) make_start_times:') - for key in self.info: - print(key) - stop + #print('*****After (the later) make_start_times:') + #for key in self.info: + # print(key) + + #for ob, ap, date, time, act, vis in zip(self.info['ObservationID'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'], self.info['act_id'], self.info['visit_num']): + # print(ob, vis, act, ap, date, time) + #stop @@ -1250,11 +1251,10 @@ def nonsidereal_pointing_updates(self): print(self.info['exposure']) print(self.info['dither']) - for t_ele, a_ele, act_ele, v_ele, obsl_ele, d_ele in zip(self.info['time_obs'], self.info['aperture'], - self.info['act_id'], self.info['visit_num'], - self.info['obs_label'], self.info['dither']): - print(t_ele, a_ele, act_ele, v_ele, obsl_ele, d_ele) - stop + #for t_ele, a_ele, act_ele, v_ele, obsl_ele, d_ele in zip(self.info['time_obs'], self.info['aperture'], + # self.info['act_id'], self.info['visit_num'], + # self.info['obs_label'], self.info['dither']): + # print(t_ele, a_ele, act_ele, v_ele, obsl_ele, d_ele) obs = np.array(self.info['ObservationID']) all_date_obs = np.array(self.info['date_obs']) @@ -1302,8 +1302,13 @@ def nonsidereal_pointing_updates(self): all_times = [ephemeris_tools.to_timestamp(elem) for elem in start_dates] # Create list of positions for all frames - ra_target = ra_ephem(all_times) - dec_target = dec_ephem(all_times) + try: + ra_target = ra_ephem(all_times) + dec_target = dec_ephem(all_times) + + except ValueError: + raise ValueError(("Observation dates ({} - {}) are not present within the ephemeris file {}" + .format(start_dates[0], start_dates[-1], ephemeris_file))) else: if not pos_in_xy: if not vel_in_xy: @@ -1335,10 +1340,22 @@ def nonsidereal_pointing_updates(self): ra_from_pointing_file[obs_exp_indexes] = ra_target dec_from_pointing_file[obs_exp_indexes] = dec_target + + print('Before modifying:') + print(type(self.info['TargetRA'][0])) + print(self.info['TargetRA'][0]) + print(type(self.info['ra'][0])) + print(self.info['ra'][0]) + + self.info['TargetRA'] = ra_from_pointing_file self.info['TargetDec'] = dec_from_pointing_file - self.info['ra'] = ra_from_pointing_file - self.info['dec'] = dec_from_pointing_file + + # Need to update these values (which come from the pointing file) + # so that below we can adjust them for the different detectors/apertures + self.info['ra'] = [np.float64(ele) for ele in ra_from_pointing_file] + self.info['dec'] = [np.float64(ele) for ele in dec_from_pointing_file] + @@ -1361,7 +1378,9 @@ def nonsidereal_pointing_updates(self): for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) - + # These go into the pointing in the yaml file + self.info['ra_ref'] = ra_from_pointing_file + self.info['dec_ref'] = dec_from_pointing_file """ read in catalog @@ -1377,10 +1396,6 @@ def nonsidereal_pointing_updates(self): - - stop - - def set_global_definitions(self): """Store the subarray definitions of all supported instruments.""" # TODO: Investigate how this could be combined with the creation of @@ -1551,10 +1566,10 @@ def make_start_times(self): else: # Now read in readpattern definitions - readpatt_def = self.global_readout_patterns[instrument.lower()] + readpatt_def = self.config_information['global_readout_patterns'][instrument.lower()] # Read in file containing subarray definitions - subarray_def = self.global_subarray_definitions[instrument.lower()] + subarray_def = self.config_information['global_subarray_definitions'][instrument.lower()] match2 = readpatt == readpatt_def['name'] if np.sum(match2) == 0: @@ -1754,10 +1769,10 @@ def make_start_times_test(self): else: # Now read in readpattern definitions - readpatt_def = self.global_readout_patterns[instrument.lower()] + readpatt_def = self.config_information['global_readout_patterns'][instrument.lower()] # Read in file containing subarray definitions - subarray_def = self.global_subarray_definitions[instrument.lower()] + subarray_def = self.config_information['global_subarray_definitions'][instrument.lower()] match2 = readpatt == readpatt_def['name'] if np.sum(match2) == 0: @@ -2130,7 +2145,7 @@ def get_psf_path(self): self.logger.info('No PSF path provided. Using default path as PSF path for all yamls.') paths_out = [] for instrument in self.info['Instrument']: - default_path = self.global_psfpath[instrument.lower()] + default_path = self.config_information['global_psfpath'][instrument.lower()] paths_out.append(default_path) return paths_out @@ -2404,7 +2419,7 @@ def write_yaml(self, input): if instrument.lower() in ['niriss', 'fgs']: full_ap = input['aperture'] - subarray_definitions = self.global_subarray_definitions[instrument.lower()] + subarray_definitions = self.config_information['global_subarray_definitions'][instrument.lower()] if full_ap not in subarray_definitions['AperName']: @@ -2600,6 +2615,7 @@ def write_yaml(self, input): # For now, skip populating the target RA and Dec in WFSC data. # The read_xxxxx funtions for these observation types will have # to be updated to make use of the proposal_parameter_dictionary + print('writing out, TargetRA: ', input['TargetRA'], type(input['TargetRA'])) if np.isreal(input['TargetRA']): input['TargetRA'] = str(input['TargetRA']) input['TargetDec'] = str(input['TargetDec']) From 36434707066a91c0e7864ce41635d8c1dd9ecccd Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 1 Dec 2020 11:35:06 -0500 Subject: [PATCH 04/13] Functional for sidereal and nonsidereal --- mirage/apt/apt_inputs.py | 3 ++- mirage/yaml/yaml_generator.py | 39 +++++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/mirage/apt/apt_inputs.py b/mirage/apt/apt_inputs.py index 926a9914b..9b8f639dd 100755 --- a/mirage/apt/apt_inputs.py +++ b/mirage/apt/apt_inputs.py @@ -374,7 +374,8 @@ def create_input_table(self, verbose=False): self.siaf[instrument_name] = siaf_interface.get_instance(instrument_name) # Calculate the correct V2, V3 and RA, Dec for each exposure/detector - self.exposure_tab = ra_dec_update(self.exposure_tab, self.siaf) + print('moving call to ra_dec_update to later in yaml_generator') + #self.exposure_tab = ra_dec_update(self.exposure_tab, self.siaf) # Output to a csv file. if self.output_csv is None: diff --git a/mirage/yaml/yaml_generator.py b/mirage/yaml/yaml_generator.py index 7786e32ed..4df6bccfa 100755 --- a/mirage/yaml/yaml_generator.py +++ b/mirage/yaml/yaml_generator.py @@ -714,6 +714,23 @@ def create_inputs(self): # ephemeris file or target velocity self.nonsidereal_pointing_updates() + # Get the correct pointing for each aperture + print('calling ra_dec_update now.....') + siaf_dictionary = {} + for instrument_name in np.unique(self.info['Instrument']): + siaf_dictionary[instrument_name] = siaf_interface.get_instance(instrument_name) + + self.info = apt_inputs.ra_dec_update(self.info, siaf_dictionary) + + + here we should update target_ra, dec to match pointing ra, dec for nonsidereal targs only + + + for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): + print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) + + + # Add a list of output yaml names to the dictionary self.make_output_names() @@ -1359,29 +1376,33 @@ def nonsidereal_pointing_updates(self): - for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): - print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) # Create a pysiaf.Siaf instance for each instrument in the proposal print('Lines below are untested') - siaf_dictionary = {} - for instrument_name in np.unique(self.info['Instrument']): - siaf_dictionary[instrument_name] = siaf_interface.get_instance(instrument_name) + #siaf_dictionary = {} + #for instrument_name in np.unique(self.info['Instrument']): + # siaf_dictionary[instrument_name] = siaf_interface.get_instance(instrument_name) - print('hopefully this updates pointings base on the new ra dec values. check carefully') - self.info = apt_inputs.ra_dec_update(self.info, siaf_dictionary) + print('move ra_dec_update to yaml_generator. check carefully') + #self.info = apt_inputs.ra_dec_update(self.info, siaf_dictionary) - for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): - print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) + #for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): + # print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) # These go into the pointing in the yaml file self.info['ra_ref'] = ra_from_pointing_file self.info['dec_ref'] = dec_from_pointing_file + print('At the end of nonsidereal_pointing_updates:\n\n') + for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): + print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) + + + """ read in catalog determine ephemeris file or velocities From 12dc5cea9c726df0debaa67be163b89c87a55d1f Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 1 Dec 2020 13:37:43 -0500 Subject: [PATCH 05/13] Clean up dev statements --- mirage/apt/apt_inputs.py | 69 ++--- mirage/yaml/yaml_generator.py | 532 +--------------------------------- 2 files changed, 22 insertions(+), 579 deletions(-) diff --git a/mirage/apt/apt_inputs.py b/mirage/apt/apt_inputs.py index 9b8f639dd..7c9f3fdaa 100755 --- a/mirage/apt/apt_inputs.py +++ b/mirage/apt/apt_inputs.py @@ -45,6 +45,7 @@ import re import argparse import pkg_resources +import warnings from astropy.table import Table, vstack from astropy.time import Time, TimeDelta @@ -268,16 +269,13 @@ def combine_dicts(self, dict1, dict2): def create_input_table(self, verbose=False): """ - - Expansion for dithers is done upstream. + Main function for creating a table of parameters for each + exposure Parameters ---------- - verbose - - Returns - ------- - + verbose : bool + If True, extra information is printed to the log """ # Expand paths to full paths # self.input_xml = os.path.abspath(self.input_xml) @@ -287,11 +285,8 @@ def create_input_table(self, verbose=False): if self.observation_list_file is not None: self.observation_list_file = os.path.abspath(self.observation_list_file) - # main_dir = os.path.split(self.input_xml)[0] - # if APT.xml content has already been generated during observation list creation # (generate_observationlist.py) load it here - if self.apt_xml_dict is None: raise RuntimeError('self.apt_xml_dict is not defined') @@ -310,39 +305,29 @@ def create_input_table(self, verbose=False): # Add epoch and catalog information observation_dictionary = self.add_observation_info(observation_dictionary) - - - - - - - # if verbose: - # print('Summary of observation dictionary:') - # for key in observation_dictionary.keys(): - # print('{:<25}: number of elements is {:>5}'.format(key, len(observation_dictionary[key]))) + if verbose: + self.logger.info('Summary of observation dictionary:') + for key in observation_dictionary.keys(): + self.logger.info('{:<25}: number of elements is {:>5}'.format(key, len(observation_dictionary[key]))) # Global Alignment observations need to have the pointing information for the # FGS exposures updated if 'WfscGlobalAlignment' in observation_dictionary['APTTemplate']: observation_dictionary = self.global_alignment_pointing(observation_dictionary) + # Expand the dictionary to have one entry for each detector in each exposure self.exposure_tab = self.expand_for_detectors(observation_dictionary) - - # Add start times for each exposure - self.exposure_tab = make_start_times(self.exposure_tab) - - - #for ob, ap, date, time, act, vis in zip(self.exposure_tab['ObservationID'], self.exposure_tab['aperture'], self.exposure_tab['date_obs'], self.exposure_tab['time_obs'], self.exposure_tab['act_id'], self.exposure_tab['visit_num']): - # print(ob, vis, act, ap, date, time) - #stop - - - - - # fix data for filename generation - # set parallel seq id + # Ignore warnings as astropy.time.Time will give a warning + # related to unknown leap seconds if the date is too far in + # the future. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self.exposure_tab = make_start_times(self.exposure_tab) + + # Fix data for filename generation + # Set parallel seq id for j, isparallel in enumerate(self.exposure_tab['ParallelInstrument']): if isparallel: self.exposure_tab['sequence_id'][j] = '2' @@ -373,10 +358,6 @@ def create_input_table(self, verbose=False): for instrument_name in np.unique(observation_dictionary['Instrument']): self.siaf[instrument_name] = siaf_interface.get_instance(instrument_name) - # Calculate the correct V2, V3 and RA, Dec for each exposure/detector - print('moving call to ra_dec_update to later in yaml_generator') - #self.exposure_tab = ra_dec_update(self.exposure_tab, self.siaf) - # Output to a csv file. if self.output_csv is None: indir, infile = os.path.split(self.input_xml) @@ -1192,10 +1173,6 @@ def make_start_times(obs_info): # Read in file containing subarray definitions config_information = utils.organize_config_files() - # choose arbitrary start time for each epoch - #epoch_base_time = '16:44:12' - #epoch_base_time0 = copy.deepcopy(epoch_base_time) - if 'epoch_start_date' in obs_info.keys(): epoch_base_date = obs_info['epoch_start_date'][0] else: @@ -1265,10 +1242,6 @@ def make_start_times(obs_info): # Get the number of amps from the subarray definition file match = aperture == subarray_def['AperName'] - - - - # needed for NIRCam case if np.sum(match) == 0: self.logger.info(('Aperture: {} does not match any entries in the subarray definition file. Guessing at the ' @@ -1342,22 +1315,18 @@ def make_start_times(obs_info): if ((next_obsid != obsid) | (next_visit != visit)): # visit break. Larger overhead overhead = visit_overhead - print("new visit/obs: {} {}".format(next_obsid, next_visit)) elif ((next_actid > actid) & (next_visit == visit)): # This block should be updated when we have more realistic # activity IDs # same visit, new activity. Smaller overhead overhead = act_overhead - print('Next activity: {}'.format(next_actid)) elif ((next_ditherid != ditherid) & (next_visit == visit)): # same visit, new dither position. Smaller overhead overhead = act_overhead - print('next dither: {}'.format(next_ditherid)) else: # same observation, activity, dither. Filter changes # will still fall in here, which is not accurate overhead = 0. # Reset frame captured in exptime below - print('same activity') # For cases where the base time needs to change # continue down here diff --git a/mirage/yaml/yaml_generator.py b/mirage/yaml/yaml_generator.py index 4df6bccfa..6b5c43e91 100755 --- a/mirage/yaml/yaml_generator.py +++ b/mirage/yaml/yaml_generator.py @@ -97,7 +97,6 @@ from copy import deepcopy from glob import glob import datetime -import warnings from astropy.time import Time, TimeDelta from astropy.table import Table @@ -653,25 +652,6 @@ def create_inputs(self): 'Observation_table_for_' + infile + '_with_yaml_parameters.csv') - - # Add start time info to each element. - # Ignore warnings as astropy.time.Time will give a warning - # related to unknown leap seconds if the date is too far in - # the future. - - - print('***At beginning of create_inputs:') - for key in self.apt_xml_dict: - print(key) - - - #with warnings.catch_warnings(): - # warnings.simplefilter("ignore") - # self.make_start_times_test() - - - - # Read XML file and make observation table apt = apt_inputs.AptInput(input_xml=self.input_xml, pointing_file=self.pointing_file, output_dir=self.output_dir) @@ -684,38 +664,12 @@ def create_inputs(self): apt.create_input_table() self.info = apt.exposure_tab - - - - - # Add start time info to each element. - # Ignore warnings as astropy.time.Time will give a warning - # related to unknown leap seconds if the date is too far in - # the future. - #with warnings.catch_warnings(): - # warnings.simplefilter("ignore") - # self.make_start_times() - - - #print('*****After (the later) make_start_times:') - #for key in self.info: - # print(key) - - #for ob, ap, date, time, act, vis in zip(self.info['ObservationID'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'], self.info['act_id'], self.info['visit_num']): - # print(ob, vis, act, ap, date, time) - #stop - - - - - # If we have a non-sidereal observation, then we need to # update the pointing information based on the input # ephemeris file or target velocity self.nonsidereal_pointing_updates() # Get the correct pointing for each aperture - print('calling ra_dec_update now.....') siaf_dictionary = {} for instrument_name in np.unique(self.info['Instrument']): siaf_dictionary[instrument_name] = siaf_interface.get_instance(instrument_name) @@ -723,9 +677,7 @@ def create_inputs(self): self.info = apt_inputs.ra_dec_update(self.info, siaf_dictionary) - here we should update target_ra, dec to match pointing ra, dec for nonsidereal targs only - - + print('Final RA, Dec values:') for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) @@ -1249,30 +1201,9 @@ def make_output_names(self): # Table([self.info['yamlfile']]).pprint() def nonsidereal_pointing_updates(self): - """The pointing info for non-sidereal observations will be incorrect - because you cannot specify a time in APT. Once observation times/dates - have been calculated here, we can now get the pointing information - correct. + """Update the pointing info for non-sidereal observations using the + requested observation date/time. """ - for k in self.info.keys(): - print(k) - - print(self.info['ObservationID']) - print(self.info['Instrument']) - print(self.info['TargetID']) - print(self.info['Tracking']) - print(self.info['Date']) - print(self.info['Dither']) - print(self.info['Exposures']) - print(self.info['entry_number']) - print(self.info['exposure']) - print(self.info['dither']) - - #for t_ele, a_ele, act_ele, v_ele, obsl_ele, d_ele in zip(self.info['time_obs'], self.info['aperture'], - # self.info['act_id'], self.info['visit_num'], - # self.info['obs_label'], self.info['dither']): - # print(t_ele, a_ele, act_ele, v_ele, obsl_ele, d_ele) - obs = np.array(self.info['ObservationID']) all_date_obs = np.array(self.info['date_obs']) all_time_obs = np.array(self.info['time_obs']) @@ -1357,14 +1288,6 @@ def nonsidereal_pointing_updates(self): ra_from_pointing_file[obs_exp_indexes] = ra_target dec_from_pointing_file[obs_exp_indexes] = dec_target - - print('Before modifying:') - print(type(self.info['TargetRA'][0])) - print(self.info['TargetRA'][0]) - print(type(self.info['ra'][0])) - print(self.info['ra'][0]) - - self.info['TargetRA'] = ra_from_pointing_file self.info['TargetDec'] = dec_from_pointing_file @@ -1373,50 +1296,10 @@ def nonsidereal_pointing_updates(self): self.info['ra'] = [np.float64(ele) for ele in ra_from_pointing_file] self.info['dec'] = [np.float64(ele) for ele in dec_from_pointing_file] - - - - - - - - # Create a pysiaf.Siaf instance for each instrument in the proposal - print('Lines below are untested') - #siaf_dictionary = {} - #for instrument_name in np.unique(self.info['Instrument']): - # siaf_dictionary[instrument_name] = siaf_interface.get_instance(instrument_name) - - print('move ra_dec_update to yaml_generator. check carefully') - #self.info = apt_inputs.ra_dec_update(self.info, siaf_dictionary) - - - #for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): - # print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) - # These go into the pointing in the yaml file self.info['ra_ref'] = ra_from_pointing_file self.info['dec_ref'] = dec_from_pointing_file - print('At the end of nonsidereal_pointing_updates:\n\n') - for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): - print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) - - - - """ - read in catalog - determine ephemeris file or velocities - - 1. read in ephemeris file - 2. get ra, dec at observation date+time - - or - - 1. calculate ra, dec from ra,dec,ra_vel,dec_vel and observation date+time - """ - - - def set_global_definitions(self): """Store the subarray definitions of all supported instruments.""" # TODO: Investigate how this could be combined with the creation of @@ -1531,415 +1414,6 @@ def lowercase_dict_keys(self): else: newkey6 = key6 - def make_start_times(self): - """Create exposure start times for each entry in the observation dictionary.""" - date_obs = [] - time_obs = [] - expstart = [] - nframe = [] - nskip = [] - namp = [] - - # choose arbitrary start time for each epoch - epoch_base_time = '16:44:12' - epoch_base_time0 = deepcopy(epoch_base_time) - - if 'epoch_start_date' in self.info.keys(): - epoch_base_date = self.info['epoch_start_date'][0] - else: - epoch_base_date = self.info['Date'][0] - base = Time(epoch_base_date + 'T' + epoch_base_time) - base_date, base_time = base.iso.split() - - # Pick some arbirary overhead values - act_overhead = 90 # seconds. (filter change) - visit_overhead = 600 # seconds. (slew) - - # Get visit, activity_id, dither_id info for first exposure - ditherid = self.info['dither'][0] - actid = self.info['act_id'][0] - visit = self.info['visit_num'][0] - # obsname = self.info['obs_label'][0] - - # for i in range(len(self.info['Module'])): - for i, instrument in enumerate(self.info['Instrument']): - # Get dither/visit - # Files with the same activity_id should have the same start time - # Overhead after a visit break should be large, smaller between - # exposures within a visit - next_actid = self.info['act_id'][i] - next_visit = self.info['visit_num'][i] - next_obsname = self.info['obs_label'][i] - next_ditherid = self.info['dither'][i] - - # Find the readpattern of the file - readpatt = self.info['ReadoutPattern'][i] - groups = np.int(self.info['Groups'][i]) - integrations = np.int(self.info['Integrations'][i]) - - if instrument.lower() in ['miri', 'nirspec']: - nframe.append(0) - nskip.append(0) - namp.append(0) - date_obs.append(base_date) - time_obs.append(base_time) - expstart.append(base.mjd) - - else: - # Now read in readpattern definitions - readpatt_def = self.config_information['global_readout_patterns'][instrument.lower()] - - # Read in file containing subarray definitions - subarray_def = self.config_information['global_subarray_definitions'][instrument.lower()] - - match2 = readpatt == readpatt_def['name'] - if np.sum(match2) == 0: - raise RuntimeError(("WARNING!! Readout pattern {} not found in definition file." - .format(readpatt))) - - # Now get nframe and nskip so we know how many frames in a group - fpg = np.int(readpatt_def['nframe'][match2][0]) - spg = np.int(readpatt_def['nskip'][match2][0]) - nframe.append(fpg) - nskip.append(spg) - - # Get the aperture name. For non-NIRCam instruments, - # this is simply the self.info['aperture']. But for NIRCam, - # we need to be careful of entries like NRCBS_FULL, which is used - # for observations using all 4 shortwave B detectors. In that case, - # we need to build the aperture name from the combination of detector - # and subarray name. - aperture = self.info['aperture'][i] - - # Get the number of amps from the subarray definition file - match = aperture == subarray_def['AperName'] - - # needed for NIRCam case - if np.sum(match) == 0: - aperture = [apername for apername, name in - np.array(subarray_def['AperName', 'Name']) if - (sub in apername) or (sub in name)] - - match = aperture == subarray_def['AperName'] - - if len(aperture) > 1 or len(aperture) == 0 or np.sum(match) == 0: - raise ValueError('Cannot combine detector {} and subarray {}\ - into valid aperture name.'.format(det, sub)) - # We don't want aperture as a list - aperture = aperture[0] - - # For grism tso observations, get the number of - # amplifiers to use from the APT file. - # For other modes, check the subarray def table. - try: - amp = int(self.info['NumOutputs'][i]) - except ValueError: - amp = subarray_def['num_amps'][match][0] - - # Default to amps=4 for subarrays that can have 1 or 4 - # if the number of amps is not defined. Hopefully we - # should never enter this code block given the lines above. - if amp == 0: - amp = 4 - self.logger.info(('Aperture {} can be used with 1 or 4 readout amplifiers. Defaulting to use 4.' - 'In the future this information should be made a user input.'.format(aperture))) - namp.append(amp) - - # same activity ID - # Remove this for now, since Mirage was not correctly - # specifying activities. At the moment all exposures have - # the same activity ID, which means we must allow the - # the epoch_start_date to change even if the activity ID - # does not. This will change back in the future when we - # figure out more realistic activity ID values. - #if next_actid == actid: - # # in this case, the start time should remain the same - # date_obs.append(base_date) - # time_obs.append(base_time) - # expstart.append(base.mjd) - # continue - - epoch_date = self.info['epoch_start_date'][i] - epoch_time = deepcopy(epoch_base_time0) - - # new epoch - update the base time - if epoch_date != epoch_base_date: - epoch_base_date = deepcopy(epoch_date) - base = Time(epoch_base_date + 'T' + epoch_base_time) - base_date, base_time = base.iso.split() - basereset = True - date_obs.append(base_date) - time_obs.append(base_time) - expstart.append(base.mjd) - actid = deepcopy(next_actid) - visit = deepcopy(next_visit) - obsname = deepcopy(next_obsname) - continue - - # new visit - if next_visit != visit: - # visit break. Larger overhead - overhead = visit_overhead - - # This block should be updated when we have more realistic - # activity IDs - elif ((next_actid > actid) & (next_visit == visit)): - # same visit, new activity. Smaller overhead - overhead = act_overhead - elif ((next_ditherid != ditherid) & (next_visit == visit)): - # same visit, new dither position. Smaller overhead - overhead = act_overhead - else: - # same observation, activity, dither. Filter changes - # will still fall in here, which is not accurate - overhead = 10. - - # For cases where the base time needs to change - # continue down here - siaf_inst = self.info['Instrument'][i].upper() - if siaf_inst == 'NIRCAM': - siaf_inst = "NIRCam" - siaf_obj = pysiaf.Siaf(siaf_inst)[aperture] - - # Calculate the readout time for a single frame - frametime = calc_frame_time(siaf_inst, aperture, - siaf_obj.XSciSize, siaf_obj.YSciSize, amp) - - # Estimate total exposure time - exptime = ((fpg + spg) * groups + fpg) * integrations * frametime - - # Delta should include the exposure time, plus overhead - delta = TimeDelta(exptime + overhead, format='sec') - base += delta - base_date, base_time = base.iso.split() - - # Add updated dates and times to the list - date_obs.append(base_date) - time_obs.append(base_time) - expstart.append(base.mjd) - - # increment the activity ID and visit - actid = deepcopy(next_actid) - visit = deepcopy(next_visit) - obsname = deepcopy(next_obsname) - ditherid = deepcopy(next_ditherid) - - self.info['date_obs'] = date_obs - self.info['time_obs'] = time_obs - # self.info['expstart'] = expstart - self.info['nframe'] = nframe - self.info['nskip'] = nskip - self.info['namp'] = namp - - - - - def make_start_times_test(self): - """Create exposure start times for each entry in the observation dictionary.""" - date_obs = [] - time_obs = [] - expstart = [] - nframe = [] - nskip = [] - namp = [] - - # choose arbitrary start time for each epoch - epoch_base_time = '16:44:12' - epoch_base_time0 = deepcopy(epoch_base_time) - - if 'epoch_start_date' in self.apt_xml_dict.keys(): - epoch_base_date = self.apt_xml_dict['epoch_start_date'][0] - else: - epoch_base_date = self.apt_xml_dict['Date'][0] - base = Time(epoch_base_date + 'T' + epoch_base_time) - base_date, base_time = base.iso.split() - - # Pick some arbirary overhead values - act_overhead = 90 # seconds. (filter change) - visit_overhead = 600 # seconds. (slew) - - # Get visit, activity_id, dither_id info for first exposure - ditherid = self.apt_xml_dict['dither'][0] - actid = self.apt_xml_dict['act_id'][0] - visit = self.apt_xml_dict['visit_num'][0] - # obsname = self.apt_xml_dict['obs_label'][0] - - # for i in range(len(self.apt_xml_dict['Module'])): - for i, instrument in enumerate(self.apt_xml_dict['Instrument']): - # Get dither/visit - # Files with the same activity_id should have the same start time - # Overhead after a visit break should be large, smaller between - # exposures within a visit - next_actid = self.apt_xml_dict['act_id'][i] - next_visit = self.apt_xml_dict['visit_num'][i] - next_obsname = self.apt_xml_dict['obs_label'][i] - next_ditherid = self.apt_xml_dict['dither'][i] - - # Find the readpattern of the file - readpatt = self.apt_xml_dict['ReadoutPattern'][i] - groups = np.int(self.apt_xml_dict['Groups'][i]) - integrations = np.int(self.apt_xml_dict['Integrations'][i]) - - if instrument.lower() in ['miri', 'nirspec']: - nframe.append(0) - nskip.append(0) - namp.append(0) - date_obs.append(base_date) - time_obs.append(base_time) - expstart.append(base.mjd) - - else: - # Now read in readpattern definitions - readpatt_def = self.config_information['global_readout_patterns'][instrument.lower()] - - # Read in file containing subarray definitions - subarray_def = self.config_information['global_subarray_definitions'][instrument.lower()] - - match2 = readpatt == readpatt_def['name'] - if np.sum(match2) == 0: - raise RuntimeError(("WARNING!! Readout pattern {} not found in definition file." - .format(readpatt))) - - # Now get nframe and nskip so we know how many frames in a group - fpg = np.int(readpatt_def['nframe'][match2][0]) - spg = np.int(readpatt_def['nskip'][match2][0]) - nframe.append(fpg) - nskip.append(spg) - - # Get the aperture name. For non-NIRCam instruments, - # this is simply the self.apt_xml_dict['aperture']. But for NIRCam, - # we need to be careful of entries like NRCBS_FULL, which is used - # for observations using all 4 shortwave B detectors. In that case, - # we need to build the aperture name from the combination of detector - # and subarray name. - aperture = self.apt_xml_dict['aperture'][i] - - # Get the number of amps from the subarray definition file - match = aperture == subarray_def['AperName'] - - # needed for NIRCam case - if np.sum(match) == 0: - aperture = [apername for apername, name in - np.array(subarray_def['AperName', 'Name']) if - (sub in apername) or (sub in name)] - - match = aperture == subarray_def['AperName'] - - if len(aperture) > 1 or len(aperture) == 0 or np.sum(match) == 0: - raise ValueError('Cannot combine detector {} and subarray {}\ - into valid aperture name.'.format(det, sub)) - # We don't want aperture as a list - aperture = aperture[0] - - # For grism tso observations, get the number of - # amplifiers to use from the APT file. - # For other modes, check the subarray def table. - try: - amp = int(self.apt_xml_dict['NumOutputs'][i]) - except ValueError: - amp = subarray_def['num_amps'][match][0] - - # Default to amps=4 for subarrays that can have 1 or 4 - # if the number of amps is not defined. Hopefully we - # should never enter this code block given the lines above. - if amp == 0: - amp = 4 - self.logger.info(('Aperture {} can be used with 1 or 4 readout amplifiers. Defaulting to use 4.' - 'In the future this information should be made a user input.'.format(aperture))) - namp.append(amp) - - # same activity ID - # Remove this for now, since Mirage was not correctly - # specifying activities. At the moment all exposures have - # the same activity ID, which means we must allow the - # the epoch_start_date to change even if the activity ID - # does not. This will change back in the future when we - # figure out more realistic activity ID values. - #if next_actid == actid: - # # in this case, the start time should remain the same - # date_obs.append(base_date) - # time_obs.append(base_time) - # expstart.append(base.mjd) - # continue - - epoch_date = self.apt_xml_dict['epoch_start_date'][i] - epoch_time = deepcopy(epoch_base_time0) - - # new epoch - update the base time - if epoch_date != epoch_base_date: - epoch_base_date = deepcopy(epoch_date) - base = Time(epoch_base_date + 'T' + epoch_base_time) - base_date, base_time = base.iso.split() - basereset = True - date_obs.append(base_date) - time_obs.append(base_time) - expstart.append(base.mjd) - actid = deepcopy(next_actid) - visit = deepcopy(next_visit) - obsname = deepcopy(next_obsname) - continue - - # new visit - if next_visit != visit: - # visit break. Larger overhead - overhead = visit_overhead - - # This block should be updated when we have more realistic - # activity IDs - elif ((next_actid > actid) & (next_visit == visit)): - # same visit, new activity. Smaller overhead - overhead = act_overhead - elif ((next_ditherid != ditherid) & (next_visit == visit)): - # same visit, new dither position. Smaller overhead - overhead = act_overhead - else: - # same observation, activity, dither. Filter changes - # will still fall in here, which is not accurate - overhead = 10. - - # For cases where the base time needs to change - # continue down here - siaf_inst = self.apt_xml_dict['Instrument'][i].upper() - if siaf_inst == 'NIRCAM': - siaf_inst = "NIRCam" - siaf_obj = pysiaf.Siaf(siaf_inst)[aperture] - - # Calculate the readout time for a single frame - frametime = calc_frame_time(siaf_inst, aperture, - siaf_obj.XSciSize, siaf_obj.YSciSize, amp) - - # Estimate total exposure time - exptime = ((fpg + spg) * groups + fpg) * integrations * frametime - - # Delta should include the exposure time, plus overhead - delta = TimeDelta(exptime + overhead, format='sec') - base += delta - base_date, base_time = base.iso.split() - - # Add updated dates and times to the list - date_obs.append(base_date) - time_obs.append(base_time) - expstart.append(base.mjd) - - # increment the activity ID and visit - actid = deepcopy(next_actid) - visit = deepcopy(next_visit) - obsname = deepcopy(next_obsname) - ditherid = deepcopy(next_ditherid) - - self.apt_xml_dict['date_obs'] = date_obs - self.apt_xml_dict['time_obs'] = time_obs - # self.apt_xml_dict['expstart'] = expstart - self.apt_xml_dict['nframe'] = nframe - self.apt_xml_dict['nskip'] = nskip - self.apt_xml_dict['namp'] = namp - - - - - - - def multiple_catalog_match(self, filter, cattype, matchlist): """ Alert the user if more than one catalog matches the filter/pupil From bc2986d3390b3b8713c1664582aef81f068e8ece Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 1 Dec 2020 20:55:18 -0500 Subject: [PATCH 06/13] Updates from testing --- mirage/apt/apt_inputs.py | 15 +++- mirage/utils/siaf_interface.py | 47 ++++++++++++ mirage/yaml/yaml_generator.py | 132 +++++++++++++++++++++++---------- 3 files changed, 154 insertions(+), 40 deletions(-) diff --git a/mirage/apt/apt_inputs.py b/mirage/apt/apt_inputs.py index 7c9f3fdaa..27754c0d9 100755 --- a/mirage/apt/apt_inputs.py +++ b/mirage/apt/apt_inputs.py @@ -1191,6 +1191,8 @@ def make_start_times(obs_info): actid = obs_info['act_id'][0] visit = obs_info['visit_num'][0] obsid = obs_info['ObservationID'][0] + exp = obs_info['exposure'][0] + entry_num = obs_info['entry_number'][0] for i, instrument in enumerate(obs_info['Instrument']): # Get dither/visit @@ -1202,6 +1204,8 @@ def make_start_times(obs_info): next_obsname = obs_info['obs_label'][i] next_ditherid = obs_info['dither'][i] next_obsid = obs_info['ObservationID'][i] + next_exp = obs_info['exposure'][i] + next_entry_num = obs_info['entry_number'][i] # Find the readpattern of the file readpatt = obs_info['ReadoutPattern'][i] @@ -1308,6 +1312,8 @@ def make_start_times(obs_info): obsid = copy.deepcopy(next_obsid) obsname = copy.deepcopy(next_obsname) ditherid = copy.deepcopy(next_ditherid) + exp = copy.deepcopy(next_exp) + entry_num = copy.deepcopy(next_entry_num) continue # new observation or visit (if a different epoch time has @@ -1340,7 +1346,7 @@ def make_start_times(obs_info): # Estimate total exposure time exptime = ((fpg + spg) * groups + fpg) * integrations * frametime - if ((next_obsid == obsid) & (next_visit == visit) & (next_actid == actid) & (next_ditherid == ditherid)): + if ((next_obsid == obsid) & (next_visit == visit) & (next_actid == actid) & (next_ditherid == ditherid) & (next_entry_num == entry_num)): # If we are in the same exposure (but with a different detector), # then we should keep the start time the same delta = TimeDelta(0., format='sec') @@ -1364,6 +1370,8 @@ def make_start_times(obs_info): obsname = copy.deepcopy(next_obsname) ditherid = copy.deepcopy(next_ditherid) obsid = copy.deepcopy(next_obsid) + exp = copy.deepcopy(next_exp) + entry_num = copy.deepcopy(next_entry_num) obs_info['date_obs'] = date_obs obs_info['time_obs'] = time_obs @@ -1454,6 +1462,11 @@ def ra_dec_update(exposure_dict, siaf_instances, verbose=False): pointing_ra, pointing_dec, telescope_roll, v2_arcsec=pointing_v2, v3_arcsec=pointing_v3) + + + print(siaf_instrument, aperture_name, local_roll, pointing_ra, pointing_dec, pointing_v2, pointing_v3, telescope_roll, aperture.V2Ref, aperture.V3Ref) + + # Calculate RA, Dec of reference location for the detector # Add in any offsets from the pointing file in the BaseX, BaseY columns ra, dec = rotations.pointing(attitude_matrix, aperture.V2Ref, aperture.V3Ref) diff --git a/mirage/utils/siaf_interface.py b/mirage/utils/siaf_interface.py index 4ef188a15..08ec82bd7 100644 --- a/mirage/utils/siaf_interface.py +++ b/mirage/utils/siaf_interface.py @@ -79,6 +79,53 @@ def aperture_ra_dec(siaf_instance, aperture_name, ra, dec, telescope_roll, outpu return aperture_pointing +def aperture_xy_to_radec(x, y, instrument, aperture, fiducial_ra, fiducial_dec, pav3): + """For a given aperture and roll angle, translate a given detector + (x, y) location to RA, Dec + + Parameters + ---------- + x : float + X-coordinate within ```aperture``` + + y : float + Y-coordinate within ```aperture``` + + instrument : str + Name of JWST instrument (e.g. 'nircam') + + aperture : str + Name of aperture (e.g. 'NRCA1_FULL') + + fiducial_ra : float + Right ascention value at the reference location of the aperture, + in decimal degrees + + fiducial_dec : float + Declination value at the reference location of the aperture, + in decimal degrees + + pav3 : float + Telescope roll angle, in degrees + + Returns + ------- + ra : float + RA corresponding to (x, y) + + dec : float + Dec corresponding to (x, y) + """ + instrument_siaf = siaf_interface.get_instance(instrument) + siaf = instrument_siaf[aperture] + local_roll, attitude_matrix, ffsize, \ + subarray_bounds = get_siaf_information(instrument, aperture, fiducial_ra, + fiducial_dec, pav3) + loc_v2, loc_v3 = siaf.sci_to_tel(x + 1, y + 1) + ra, dec = pysiaf.utils.rotations.pointing(attitude_matrix, loc_v2, loc_v3) + return ra, dec + + def get_instance(instrument): """Return an instance of a pysiaf.Siaf object for the given instrument diff --git a/mirage/yaml/yaml_generator.py b/mirage/yaml/yaml_generator.py index 6b5c43e91..f8d74dc34 100755 --- a/mirage/yaml/yaml_generator.py +++ b/mirage/yaml/yaml_generator.py @@ -93,6 +93,7 @@ import sys import os import argparse +from collections import Counter import logging from copy import deepcopy from glob import glob @@ -112,7 +113,8 @@ from ..reference_files.utils import get_transmission_file from ..seed_image import ephemeris_tools from ..utils.constants import FGS1_DARK_SEARCH_STRING, FGS2_DARK_SEARCH_STRING -from ..utils.utils import calc_frame_time, ensure_dir_exists, expand_environment_variable +from ..utils.siaf_interface import aperture_xy_to_radec +from ..utils.utils import calc_frame_time, ensure_dir_exists, expand_environment_variable, parse_RA_Dec from .generate_observationlist import get_observation_dict from ..constants import NIRISS_PUPIL_WHEEL_ELEMENTS, NIRISS_FILTER_WHEEL_ELEMENTS from ..utils.constants import CRDS_FILE_TYPES, SEGMENTATION_MIN_SIGNAL_RATE, \ @@ -327,16 +329,6 @@ def __init__(self, input_xml=None, pointing_file=None, datatype='linear', reffil self.reffile_setup() - - (print('****At end of yaml_generator __init__:')) - for key in self.apt_xml_dict: - print(key) - - - - - - def add_catalogs(self): """ Add list(s) of source catalogs to the table containing the @@ -674,9 +666,17 @@ def create_inputs(self): for instrument_name in np.unique(self.info['Instrument']): siaf_dictionary[instrument_name] = siaf_interface.get_instance(instrument_name) + + print(self.info['ra_ref'], self.info['dec_ref']) + + self.info = apt_inputs.ra_dec_update(self.info, siaf_dictionary) + print(self.info['ra_ref'], self.info['dec_ref']) + print('') + + print('Final RA, Dec values:') for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) @@ -1208,6 +1208,7 @@ def nonsidereal_pointing_updates(self): all_date_obs = np.array(self.info['date_obs']) all_time_obs = np.array(self.info['time_obs']) all_exposure = np.array(self.info['exposure']) + all_apertures = np.array(self.info['aperture']) tracking = np.array(self.info['Tracking']) targs = np.array(self.info['TargetID']) inst = np.array(self.info['Instrument']) @@ -1215,14 +1216,31 @@ def nonsidereal_pointing_updates(self): dec_from_pointing_file = np.array(self.info['dec']) nonsidereal_index = np.where(np.array(tracking) == 'non-sidereal') all_nonsidereal_targs = targs[nonsidereal_index] - nonsidereal_targs, u_indexes = np.unique(targs[nonsidereal_index], return_index=True) - nonsidereal_instruments = inst[nonsidereal_index][u_indexes] + all_nonsidereal_instruments = inst[nonsidereal_index] + + # Get a list of the unique (target, instrument) combinations for + # non-sidereal observations + inst_targs = [[t, i] for t, i in zip(all_nonsidereal_targs, all_nonsidereal_instruments)] + ctr = Counter(tuple(x) for x in inst_targs) + unique = [list(key) for key in ctr.keys()] # Check that all non-sidereal targets have catalogs associated with them - for targ, inst in zip(nonsidereal_targs, nonsidereal_instruments): + #for targ, inst in zip(nonsidereal_targs, nonsidereal_instruments): + for element in unique: + targ, inst = element ns_catalog = get_nonsidereal_catalog_name(self.catalogs, targ, inst) catalog_table, pos_in_xy, vel_in_xy = read_nonsidereal_catalog(ns_catalog) + # If the ephemeris_file column is present but equal to 'none', then + # remove the column + if 'ephemeris_file' in catalog_table.colnames: + if catalog_table['ephemeris_file'][0].lower() == 'none': + catalog_table.remove_column('ephemeris_file') + + if 'ephemeris_file' in catalog_table.colnames: + ephemeris_file = catalog_table['ephemeris_file'][0] + ra_ephem, dec_ephem = ephemeris_tools.get_ephemeris(ephemeris_file) + # Find the observations that use this target exp_index_this_target = targs == targ obs_this_target = np.unique(obs[exp_index_this_target]) @@ -1235,6 +1253,8 @@ def nonsidereal_pointing_updates(self): obs_dates = all_date_obs[obs_exp_indexes] obs_times = all_time_obs[obs_exp_indexes] exposures = all_exposure[obs_exp_indexes] + apertures = all_apertures[obs_exp_indexes] + unique_apertures = np.unique(apertures) start_dates = [] for date_obs, time_obs in zip(obs_dates, obs_times): @@ -1245,8 +1265,6 @@ def nonsidereal_pointing_updates(self): start_dates.append(datetime.datetime.strptime(ob_time, '%Y-%m-%dT%H:%M:%S.%f')) if 'ephemeris_file' in catalog_table.colnames: - ephemeris_file = catalog_table['ephemeris_file'][0] - ra_ephem, dec_ephem = ephemeris_tools.get_ephemeris(ephemeris_file) all_times = [ephemeris_tools.to_timestamp(elem) for elem in start_dates] # Create list of positions for all frames @@ -1259,30 +1277,66 @@ def nonsidereal_pointing_updates(self): .format(start_dates[0], start_dates[-1], ephemeris_file))) else: if not pos_in_xy: - if not vel_in_xy: - # Here we assume that the source (and aperture reference location) - # is located at the given RA, Dec at the start of the first exposure - base_ra = ra_from_pointing_file[obs_exp_indexes] - base_dec = dec_from_pointing_file[obs_exp_indexes] - ra_target = [base_ra] - dec_target = [base_dec] - for ob_date in start_dates[1:]: - delta_time = ob_date - start_dates[0] - delta_ra = 0.05 * delta_time.total_seconds() / 3600. - delta_dec = 0.02 * delta_time.total_seconds() / 3600. - ra_target.append(base_ra + delta_ra) - dec_target.append(base_dec + delta_dec) - else: - # Velocity is in units of pixels/hr. Need to translate to arcsec/hr - print('To do') + # Here we assume that the source (and aperture reference location) + # is located at the given RA, Dec at the start of the first exposure + base_ra, base_dec = parse_RA_Dec(catalog_table['x_or_RA'].data[0], catalog_table['y_or_Dec'].data[0]) + ra_target = [base_ra] + dec_target = [base_dec] + + ra_vel = catalog_table['x_or_RA_velocity'].data[0] + dec_vel = catalog_table['y_or_Dec_velocity'].data[0] + + # If the source velocity is given in units of pixels/hour, then we need + # to multiply this by the appropriate pixel scale. + if vel_in_xy: + print('UNIQUE APERTURES: ') + print(unique_apertures, '\n') + if len(unique_apertures) > 1: + if inst.lower() == 'nircam': + det_ints = [int(ele.split('_')[0][-1]) for ele in unique_apertures] + # If the observation contains NIRCam exposures in both the LW and + # SW channels, then the source velocity is ambiguous due to the + # different pixel scales. In that case, raise an exception. + if np.min(det_ints) < 5 and np.max(det_ints) == 5: + raise ValueError(('Non-sidereal source {} has no ephemeris file, and a velocity that ' + 'is specified in units of pixels/hour in the source catalog. ' + 'Since observation {} contains NIRCam apertures within both the ' + 'SW and LW channels (which have different pixel scales), ' + 'Mirage does not know which pixel scale to use ' + 'when placing the source.'.format(targ, obs_name))) + + # In this case, there is a well-defined pixel scale, so we can translate + # velocities to units of arcsec/hour + siaf = pysiaf.Siaf(inst)[unique_apertures[0]] + ra_vel *= siaf.XSciScale + dec_vel *= siaf.XSciScale + + # Calculate RA, Dec for each exposure given the velocities + for ob_date in start_dates[1:]: + delta_time = ob_date - start_dates[0] + delta_ra = ra_vel * delta_time.total_seconds() / 3600. + delta_dec = dec_vel * delta_time.total_seconds() / 3600. + ra_target.append(base_ra + delta_ra) + dec_target.append(base_dec + delta_dec) + else: - # Input position should be at the aperture reference location - # or else we're not tracking the target, right? - if not vel_in_xy: - print('to do') - else: - # Velocity is in units of pixels/hr. Need to translate to arcsec/hr - print('to do') + # Source location comes from the source catalog and is in units of pixels. + # This can't really be supported, since we don't know which detector the + # location is for. We could proceed, but Mirage would then put the source + # pixel (x, y) in every aperture/detector. + if len(unique_apertures) > 1: + raise ValueError(('Non-sidereal source {} has no ephemeris file, and a location that ' + 'is specified in units of detector pixels in the source catalog. ' + 'Since observation {} contains multiple apertures (implying different ' + 'coordinate systems), Mirage does not know which coordinate system ' + 'to use when placing the source.'.format(targ, obs_name))) + + # If there is only a single aperture associated with the observation, + # then we can proceed. We first need to translate the given x, y position + # to RA, Dec + base_ra, base_dec = aperture_xy_to_radec(catalog_table['x_or_RA'].data[0], + catalog_table['y_or_Dec'].data[0], + inst, aperture, fiducial_ra, fiducial_dec, pav3) ra_from_pointing_file[obs_exp_indexes] = ra_target From ca26b767014caf4396fc58b229223e2febe8de07 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 2 Dec 2020 09:42:31 -0500 Subject: [PATCH 07/13] Remove old copy of ra_dec_update --- mirage/apt/apt_inputs.py | 86 +--------------------------------------- 1 file changed, 1 insertion(+), 85 deletions(-) diff --git a/mirage/apt/apt_inputs.py b/mirage/apt/apt_inputs.py index 27754c0d9..83017ce02 100755 --- a/mirage/apt/apt_inputs.py +++ b/mirage/apt/apt_inputs.py @@ -981,86 +981,6 @@ def tight_dithers(self, input_dict): input_dict['PrimaryDithers'] = modlist return input_dict - def ra_dec_update(self, verbose=False): - """Given the V2, V3 values for the reference pixels associated - with detector apertures, calculate corresponding RA, Dec. - """ - sw_grismts_apertures = ['NRCA1_GRISMTS256', 'NRCA1_GRISMTS128', 'NRCA1_GRISMTS64', - 'NRCA3_GRISMTS256', 'NRCA3_GRISMTS128', 'NRCA3_GRISMTS64'] - - lw_grismts_apertures = ['NRCA5_GRISM256_F322W2', 'NRCA5_GRISM128_F322W2', 'NRCA5_GRISM64_F322W2', - 'NRCA5_GRISM256_F444W', 'NRCA5_GRISM128_F444W', 'NRCA5_GRISM64_F444W'] - - intermediate_lw_grismts_apertures = ['NRCA5_TAGRISMTS_SCI_F444W', 'NRCA5_TAGRISMTS_SCI_F322W2'] - - aperture_ra = [] - aperture_dec = [] - - lw_grismts_aperture = None - for i in range(len(self.exposure_tab['Module'])): - siaf_instrument = self.exposure_tab["Instrument"][i] - aperture_name = self.exposure_tab['aperture'][i] - pointing_ra = np.float(self.exposure_tab['ra'][i]) - pointing_dec = np.float(self.exposure_tab['dec'][i]) - pointing_v2 = np.float(self.exposure_tab['v2'][i]) - pointing_v3 = np.float(self.exposure_tab['v3'][i]) - - # When we run across a LW grism TS aperture, save - # the aperture name, because we'll need it when looking - # at the accompanying SW apertuers to follow. THIS - # RELIES ON THE LW ENTRY COMING BEFORE THE SW ENTRIES. - if aperture_name in lw_grismts_apertures: - lw_grismts_aperture = copy.deepcopy(aperture_name) - lw_filter = lw_grismts_aperture.split('_')[2] - lw_intermediate_aperture = [ap for ap in intermediate_lw_grismts_apertures if lw_filter in ap][0] - - if 'pav3' in self.exposure_tab.keys(): - pav3 = np.float(self.exposure_tab['pav3'][i]) - else: - pav3 = np.float(self.exposure_tab['PAV3'][i]) - - telescope_roll = pav3 - - aperture = self.siaf[siaf_instrument][aperture_name] - - if 'NRCA5_GRISM' in aperture_name and 'WFSS' not in aperture_name: - # This puts the source in row 29, but faster just to grab the ra, dec directly - #local_roll, attitude_matrix, fullframesize, subarray_boundaries = \ - # siaf_interface.get_siaf_information(self.siaf[siaf_instrument], aperture_name, - # pointing_ra, pointing_dec, telescope_roll, - # v2_arcsec=aperture.V2Ref, v3_arcsec=aperture.V3Ref) - ra = pointing_ra - dec = pointing_dec - else: - if aperture_name in sw_grismts_apertures: - # Special case. When looking at grism time series observation - # we force the pointing to be at the reference location of the - # LW *intermediate* aperture, rather than paying attention to - # the V2, V3 in the pointing file. V2, V3 from the intermediate - # aperture is where the source would land on the detector if - # the grism were not in the beam. This is exactly what we want - # for the SW detectors, where this is no grism. - - # Generate an attitude matrix from this and - # use to get the RA, Dec in the SW apertures - lw_gts = self.siaf[siaf_instrument][lw_intermediate_aperture] - pointing_v2 = lw_gts.V2Ref - pointing_v3 = lw_gts.V3Ref - - local_roll, attitude_matrix, fullframesize, subarray_boundaries = \ - siaf_interface.get_siaf_information(self.siaf[siaf_instrument], aperture_name, - pointing_ra, pointing_dec, telescope_roll, - v2_arcsec=pointing_v2, v3_arcsec=pointing_v3) - - # Calculate RA, Dec of reference location for the detector - # Add in any offsets from the pointing file in the BaseX, BaseY columns - ra, dec = rotations.pointing(attitude_matrix, aperture.V2Ref, aperture.V3Ref) - aperture_ra.append(ra) - aperture_dec.append(dec) - - self.exposure_tab['ra_ref'] = aperture_ra - self.exposure_tab['dec_ref'] = aperture_dec - def add_options(self, parser=None, usage=None): if parser is None: parser = argparse.ArgumentParser(usage=usage, description='Simulate JWST ramp') @@ -1462,14 +1382,10 @@ def ra_dec_update(exposure_dict, siaf_instances, verbose=False): pointing_ra, pointing_dec, telescope_roll, v2_arcsec=pointing_v2, v3_arcsec=pointing_v3) - - - print(siaf_instrument, aperture_name, local_roll, pointing_ra, pointing_dec, pointing_v2, pointing_v3, telescope_roll, aperture.V2Ref, aperture.V3Ref) - - # Calculate RA, Dec of reference location for the detector # Add in any offsets from the pointing file in the BaseX, BaseY columns ra, dec = rotations.pointing(attitude_matrix, aperture.V2Ref, aperture.V3Ref) + aperture_ra.append(ra) aperture_dec.append(dec) From d2ccba70a14acecdada18c3d92941abe1b64ae68 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 2 Dec 2020 13:27:03 -0500 Subject: [PATCH 08/13] Tweak docs to show input datetime string --- docs/yaml_generator.rst | 9 +++++++-- mirage/yaml/yaml_generator.py | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/yaml_generator.rst b/docs/yaml_generator.rst index a74dc6dfd..ec83046aa 100644 --- a/docs/yaml_generator.rst +++ b/docs/yaml_generator.rst @@ -245,10 +245,10 @@ Observation Dates +++++++++++++++++ You may also specify the observation date for each observation in your APT file. This may be used along with roll angle to help define epochs in your observations, or simply -to associate a given dataset with a date. **Note that Mirage does not pay attention to dates in any way** other than to save them into the *date-obs* header keyword in the output +to associate a given dataset with a date. Mirage saves the input dates into the *date-obs* header keyword in the output files. Mirage does not check that a given roll angle and pointing are physically realizable on a given date. It is up to you to provide realistic values for these paramters if they are important to you. The `JWST Target Visibility Tools `_ (TVT) are -useful for this. Note that in all cases below, Mirage will use the entered date (along with a default time) as the starting time of the first exposure in the observation. +useful for this. Note that in all cases below, Mirage will use the entered date (along with a default time if necessary) as the starting time of the first exposure in the observation. Mirage keeps track of exposure times and makes some guesses about overheads, and increments the observation time and date for each exposure. To use the Mirage default for observation date (arbitrarily set to 2021-10-04), you can either not supply any date information, or explicitly use None. @@ -270,6 +270,11 @@ the date strings for each. dates = {'001': '2022-06-25', '002': '2022-11-15', '003': '2023-03-14'} +For observations where you want to be more specific about the observation time (such as for non-sidereal targets), datetime strings can be provided: + +:: + + dates = {'001': '2022-06-25T12:00:00', '002': '2022-11-15T12:34:56', '003': '2023-03-14T18:35:42'} .. _yam_gen_cr_inputs: diff --git a/mirage/yaml/yaml_generator.py b/mirage/yaml/yaml_generator.py index f8d74dc34..f445de25f 100755 --- a/mirage/yaml/yaml_generator.py +++ b/mirage/yaml/yaml_generator.py @@ -2164,7 +2164,6 @@ def write_yaml(self, input): # For now, skip populating the target RA and Dec in WFSC data. # The read_xxxxx funtions for these observation types will have # to be updated to make use of the proposal_parameter_dictionary - print('writing out, TargetRA: ', input['TargetRA'], type(input['TargetRA'])) if np.isreal(input['TargetRA']): input['TargetRA'] = str(input['TargetRA']) input['TargetDec'] = str(input['TargetDec']) From dc98cbafdb6f43a2784051eb29b90f0d0f33962b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 3 Dec 2020 16:14:33 -0500 Subject: [PATCH 09/13] Testing --- mirage/apt/apt_inputs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mirage/apt/apt_inputs.py b/mirage/apt/apt_inputs.py index 83017ce02..d1b5a3106 100755 --- a/mirage/apt/apt_inputs.py +++ b/mirage/apt/apt_inputs.py @@ -1377,6 +1377,7 @@ def ra_dec_update(exposure_dict, siaf_instances, verbose=False): pointing_v2 = lw_gts.V2Ref pointing_v3 = lw_gts.V3Ref + local_roll, attitude_matrix, fullframesize, subarray_boundaries = \ siaf_interface.get_siaf_information(siaf_instances[siaf_instrument], aperture_name, pointing_ra, pointing_dec, telescope_roll, @@ -1384,7 +1385,18 @@ def ra_dec_update(exposure_dict, siaf_instances, verbose=False): # Calculate RA, Dec of reference location for the detector # Add in any offsets from the pointing file in the BaseX, BaseY columns + #if siaf_instrument.lower() != 'niriss': ra, dec = rotations.pointing(attitude_matrix, aperture.V2Ref, aperture.V3Ref) + #else: + # ra, dec = exposure_dict['ra_ref'][i], exposure_dict['dec_ref'][i] + + + + #print(exposure_dict['aperture'], exposure_dict['ra_ref'], exposure_dict['dec_ref']) + #print(ra, dec) + #stop + + aperture_ra.append(ra) aperture_dec.append(dec) From ac9060c677c09aaf1e7d350667358eabc1c04b72 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 4 Dec 2020 13:45:28 -0500 Subject: [PATCH 10/13] Fix failing tests --- mirage/apt/apt_inputs.py | 10 ++++++---- tests/test_yaml_generator.py | 26 +++++++++++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/mirage/apt/apt_inputs.py b/mirage/apt/apt_inputs.py index d1b5a3106..5d586f04f 100755 --- a/mirage/apt/apt_inputs.py +++ b/mirage/apt/apt_inputs.py @@ -1083,6 +1083,8 @@ def make_start_times(obs_info): obs_info : dict Modified dictionary with observation dates and times added """ + logger = logging.getLogger('mirage.apt.apt_inputs') + date_obs = [] time_obs = [] expstart = [] @@ -1168,8 +1170,8 @@ def make_start_times(obs_info): # needed for NIRCam case if np.sum(match) == 0: - self.logger.info(('Aperture: {} does not match any entries in the subarray definition file. Guessing at the ' - 'aperture for the purpose of calculating the exposure time and number of amps.'.format(aperture))) + logger.info(('Aperture: {} does not match any entries in the subarray definition file. Guessing at the ' + 'aperture for the purpose of calculating the exposure time and number of amps.'.format(aperture))) sub = aperture.split('_')[1] aperture = [apername for apername, name in np.array(subarray_def['AperName', 'Name']) if @@ -1196,8 +1198,8 @@ def make_start_times(obs_info): # should never enter this code block given the lines above. if amp == 0: amp = 4 - self.logger.info(('Aperture {} can be used with 1 or 4 readout amplifiers. Defaulting to use 4.' - 'In the future this information should be made a user input.'.format(aperture))) + logger.info(('Aperture {} can be used with 1 or 4 readout amplifiers. Defaulting to use 4.' + 'In the future this information should be made a user input.'.format(aperture))) namp.append(amp) # same activity ID diff --git a/tests/test_yaml_generator.py b/tests/test_yaml_generator.py index d0755f7eb..51f6fd071 100644 --- a/tests/test_yaml_generator.py +++ b/tests/test_yaml_generator.py @@ -72,7 +72,7 @@ def test_user_inputs_basic(): } } cr = {'library': 'FLARE', 'scale': 44.0} - date = '2019-5-25' + date = '2019-05-25' background = {'001': 'high', '002': 'medium', '003': 22.3} roll_angle = 34.5 @@ -88,7 +88,18 @@ def test_user_inputs_basic(): nis_obs = np.vstack([nis_obs1, nis_obs2, nis_obs3]) # Check dates - date_match = [entry == date for entry in tab['Date']] + date_to_match = '{} 00:00:00'.format(date) + date_match = [str(entry) == date_to_match for entry in tab['Date']] + + #for entry in tab['Date']: + # print(date_to_match, str(entry), type(date_to_match), type(str(entry)), date_to_match == str(entry)) + + + + + + + assert all(date_match) is True # Check roll angle @@ -197,7 +208,7 @@ def test_user_inputs_complex(): '002': {'library': 'SUNMIN', 'scale': 5.5}, '003': {'library': 'SUNMAX', 'scale': 4.4}} - date = {'001': '2019-05-25', '002': '2019-11-15', '003': '2020-10-14'} + date = {'001': '2019-05-25', '002': '2019-11-15T12:12:12.120000', '003': '2020-10-14T19:20:21'} background = {'001': {'nircam': {'sw': 0.2, 'lw': 0.3}, 'niriss': 0.4}, '002': {'nircam': {'sw': 'medium', 'lw': 'high'}, 'niriss': 'low'}, @@ -218,9 +229,14 @@ def test_user_inputs_complex(): # Check dates for i, key in enumerate(date): - date_match = [entry == date[key] for entry in np.array(tab['Date'])[nis_obs[i, :]]] + if 'T' not in date[key]: + date_to_match = '{} 00:00:00'.format(date[key]) + else: + date_to_match = date[key].replace('T', ' ') + + date_match = [str(entry) == date_to_match for entry in np.array(tab['Date'])[nis_obs[i, :]]] assert all(date_match) is True - date_match = [entry == date[key] for entry in np.array(tab['Date'])[nrc_obs[i, :]]] + date_match = [str(entry) == date_to_match for entry in np.array(tab['Date'])[nrc_obs[i, :]]] assert all(date_match) is True # Check roll angle From 604d80a44e3a8f82020330e7a8879e198d49d16d Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 4 Dec 2020 17:33:33 -0500 Subject: [PATCH 11/13] Update example notebook. Can make NS catalog without RA Dec --- .../MovingTarget_simulator_use_examples.ipynb | 812 +++++++++--------- mirage/catalogs/catalog_generator.py | 10 + 2 files changed, 397 insertions(+), 425 deletions(-) diff --git a/examples/MovingTarget_simulator_use_examples.ipynb b/examples/MovingTarget_simulator_use_examples.ipynb index 3d368f40e..291b69ce9 100755 --- a/examples/MovingTarget_simulator_use_examples.ipynb +++ b/examples/MovingTarget_simulator_use_examples.ipynb @@ -12,9 +12,22 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The `mirage` simulator is broken up into three basic stages:\n", + "This notebook shows an example of how to simulate observations of a non-sidereal target. In this case, JWST tracks the non-sidereal target during the exposure, causing sidereal targets to move over the course of the exposure." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `mirage` simulator is broken up into four basic stages:\n", "\n", - "1. **Creation of a \"seed image\".**
\n", + "1. **Creation of yaml-formatted input files**.
\n", + " Calls to Mirage generally require one [yaml input](https://mirage-data-simulator.readthedocs.io/en/latest/example_yaml.html) file. This file specifies\n", + " details about the instrument set-up, and source catalogs and reference files\n", + " to use. Each yaml file specifies exposure details for a single exposure\n", + " in a single detector.

\n", + "\n", + "2. **Creation of a \"seed image\".**
\n", " This is generally a noiseless countrate image that contains signal\n", " only from the astronomical sources to be simulated. Currently, the \n", " mirage package contains code to produce a seed image starting\n", @@ -23,32 +36,19 @@ " seed image containing moving targets means that this step will be significantly\n", " slower than when generating a simple seed image for a sidereal observation.

\n", " \n", - "2. **Dark current preparation.**
\n", + "3. **Dark current preparation.**
\n", " The simulated data will be created by adding the simulated sources\n", " in the seed image to a real dark current exposure. This step\n", " converts the dark current exposure to the requested readout pattern\n", " and subarray size requested by the user.

\n", " \n", - "3. **Observation generation.**
\n", + "4. **Observation generation.**
\n", " This step converts the seed image into an exposure of the requested\n", " readout pattern and subarray size. It also adds cosmic rays and \n", " Poisson noise, as well as other detector effects (IPC, crosstalk, etc).\n", " This exposure is then added to the dark current exposure from step 2.

" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "*Table of Contents:*\n", - "* Single image simulation\n", - " * [Running simulator steps independently](#run_steps_independently)\n", - " * [Running simulator steps together](#run_steps_together)\n", - "* [Running multiple simulations](#mult_sims)\n", - "* [Generating `yaml` files](#make_yaml)\n", - "* [Example `yaml` file](#yaml_example)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -72,11 +72,11 @@ "source": [ "*Table of Contents:*\n", "* [Imports](#imports)\n", + "* [Create Source Catalogs](#make_catalogs)\n", "* [Generating `yaml` files](#make_yaml)\n", "* [Create Simulated Data](#run_steps_together)\n", - "* [Simulating Multiple Exposures](#mult_sims)\n", "* [Running Simulation Steps Independently](#run_steps_independently)\n", - "* [Example `yaml` file](#yaml_example)" + "* [Simulating Multiple Exposures](#mult_sims)" ] }, { @@ -116,12 +116,12 @@ "metadata": {}, "outputs": [], "source": [ - "# For examining outputs\n", - "from glob import glob\n", - "from scipy.stats import sigmaclip\n", - "import numpy as np\n", "from astropy.io import fits\n", + "from glob import glob\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pkg_resources\n", + "from scipy.stats import sigmaclip\n", "%matplotlib inline" ] }, @@ -133,10 +133,238 @@ "source": [ "# mirage imports\n", "from mirage import imaging_simulator\n", + "from mirage.catalogs import catalog_generator\n", "from mirage.seed_image import catalog_seed_image\n", "from mirage.dark import dark_prep\n", "from mirage.ramp_generator import obs_generator\n", - "from mirage.yaml import yaml_generator" + "from mirage.yaml.yaml_generator import SimInput" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "TEST_DATA_DIRECTORY = os.path.normpath(os.path.join(pkg_resources.resource_filename('mirage', ''),\n", + " '../examples/movingtarget_example_data'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if not os.path.isdir(TEST_DATA_DIRECTORY):\n", + " print(\"WARNING: test data directory does not exist!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "output_dir = './'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if not os.path.isdir(output_dir):\n", + " print(\"WARNING: output directory does not exist!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "# Create Source Catalogs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first task to prepare for the creation of simulated data is to create source catalogs. Mirage supports several different types of catalogs, with a different catalog for each type of source (e.g. point sources, galaxies, etc. See the [catalog documentation](https://mirage-data-simulator.readthedocs.io/en/stable/catalogs.html) for details.)\n", + "\n", + "For this example, for our target we use the ephemeris for Mars (in order to maximize velocity and make the motion easy to see in a short exposure). However, for simplicity we will use a point source in place of Mars' disk. We will also include background stars in order to show the motion of background sources." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create non-sidereal catalog" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, create the source catalog containing our target. For this we will use Mirage's non-sidereal catalog type. By using the non-sidereal catalog, we will be telling Mirage that we wish to have JWST track this source during the exposure. The motion of the non-sidereal source can be captured via either manually entered velocites, or by providing a [JPL Horizons](https://ssd.jpl.nasa.gov/horizons.cgi) formatted ephemeris file. In this example, we will use an ephemeris file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ephemeris_file = os.path.join(TEST_DATA_DIRECTORY, 'mars_ephemeris.txt')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "non_sidereal_catalog = os.path.join(output_dir, 'mars_nonsidereal.cat')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create the catalog. Since we are using an ephemeris, there is no need to\n", + "# specify the RA, Dec, nor velocity of the source. All will be retrieved from\n", + "# the ephemeris file.\n", + "ns = catalog_generator.NonSiderealCatalog(object_type=['pointSource'], ephemeris_file=[ephemeris_file])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Add the source magnitudes to the catalog. Note that the magnitude values required by Mirage are magnitudes in the NIRCam/NIRISS filters of interest, so we cannot get these from the ephemeris file. Also, Mirage does not yet support source magnitudes that change with time. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mag = 14." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Be sure to add magnitude columns for all filters you wish to simulate. In this case\n", + "# the APT file uses filters F150W and F356W\n", + "ns.add_magnitude_column([mag], magnitude_system='abmag', instrument='nircam', filter_name='f150w')\n", + "ns.add_magnitude_column([mag], magnitude_system='abmag', instrument='nircam', filter_name='f356w')\n", + "ns.save(non_sidereal_catalog)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ns.table" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create catalog of background stars" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "See the [Catalog Generation Tools](https://github.com/spacetelescope/mirage/blob/master/examples/Catalog_Generation_Tools.ipynb) example notebook for more details on creating source catalogs, including the use of 2MASS/GAIA/WISE/Besancon queries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "point_source_catalog = os.path.join(output_dir, 'background_point_sources.cat')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "base_ra = 25.7442083 # degrees\n", + "base_dec = 6.4404722 # degrees\n", + "cat_width = 93. # arcseconds\n", + "cat_width /= 3600." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Let's just randomly scatter some stars in the area\n", + "ra_val = np.random.uniform(base_ra - cat_width, base_ra + cat_width, 50)\n", + "dec_val = np.random.uniform(base_dec - cat_width, base_dec + cat_width, 50)\n", + "mags = np.random.uniform(17, 20, 50)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Set the first background source to be ~140 pixels from our non-sidereal target\n", + "# This will make it easier to see the difference between the two in the \n", + "# resulting simulated data\n", + "ra_val[0] = 25.74248611078\n", + "dec_val[0] = 6.438749978" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that all Mirage source catalogs must have an \"index\" column that assigns a number to each source. You cannot have multiple sources with the same index number, even across catalogs (because these index numbers will be used to populate the segmentation map). Since the non-sidereal catalog contains one source (with an index number of 1), we start the index numbers in this catalog at 2." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ptsrc = catalog_generator.PointSourceCatalog(ra=ra_val, dec=dec_val, starting_index=2)\n", + "ptsrc.add_magnitude_column(mags, magnitude_system='abmag', instrument='nircam', filter_name='f150w')\n", + "ptsrc.add_magnitude_column(mags, magnitude_system='abmag', instrument='nircam', filter_name='f356w')\n", + "ptsrc.save(point_source_catalog)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ptsrc.table" ] }, { @@ -152,7 +380,91 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Note that Mirage does not yet support the automated creation of yaml files from an APT file for solar system targets. For the time being, the easiest work-around for this is to start with an existing yaml file (such at that on the [example yaml page of the documentation](https://mirage-data-simulator.readthedocs.io/en/latest/example_yaml.html), and manually edit the input fields. Remember to set the `Telescope: tracking` entry to `non-sidereal`, so that targets in the `movingTargetToTrack` catalog will remain at a fixed location in the output data, while background targets in the `pointsource`, `galaxyListFile`, and `extended` catalogs will trail across the field of view over the course of each exposure." + "The easiest way to construct input yaml files is to begin with a proposal in [APT](https://jwst-docs.stsci.edu/jwst-astronomers-proposal-tool-overview). In this example, we use an APT file in the examples/movingtarget_example_data directory. Mirage does not use the apt file dirctly, but instead the exported xml and pointing files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xml_file = os.path.join(TEST_DATA_DIRECTORY, 'mars_example.xml')\n", + "pointing_file = xml_file.replace('.xml', '.pointing')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Due to the large number of ancillary files output by Mirage, it is often helpful to store the yaml files in their own directory, separate from the outputs of the simulator itself." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "yaml_output_dir = './'\n", + "simdata_output_dir = './'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Inputs into the yaml generator function include the source catalogs, as well as a number of other options detailed on the [yaml_generator documentation page](https://mirage-data-simulator.readthedocs.io/en/stable/yaml_generator.html). See that page for more information." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Catalogs must be put in a nested dictionary with target names (from the APT file) as\n", + "# the top level keys, and catalog types as the second level keys. \n", + "cats = {'MARS': {'moving_target_to_track': non_sidereal_catalog,\n", + " 'point_source': point_source_catalog},\n", + " 'ONE': {'point_source': point_source_catalog}}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Dates can be specified using a date-only or a datetime string for each observation in\n", + "# the proposal. In this case, with a fast-moving target, we will use datetime strings. Keys\n", + "# for this dictionary are the observation numbers from the proposal.\n", + "dates = {'001': '2020-09-25T00:00:00.0'}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# Now run the yaml_generator and create the yaml files\n", + "y = SimInput(input_xml=xml_file, pointing_file=pointing_file, catalogs=cats,\n", + " dates=dates, dateobs_for_background=False, datatype='raw',\n", + " output_dir=yaml_output_dir, simdata_output_dir=simdata_output_dir)\n", + "y.create_inputs()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# List the newly-created yaml files\n", + "y.yaml_files" ] }, { @@ -175,7 +487,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The imaging_simulator.ImgSim class is a wrapper around the three main steps of the simulator (detailed in the [Running simulator steps independently](#run_steps_independently) section below). This convenience function is useful when creating simulated imaging mode data. WFSS data will need to be run in a slightly different way. See the WFSS example notebook for details." + "The imaging_simulator.ImgSim class is a wrapper around the three main steps of the simulator (detailed in the [Running simulator steps independently](#run_steps_independently) section below). This convenience function is useful when creating simulated imaging mode data." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For this example we'll simulate the first exposure in the NRCB5 detector. This should have our target relatively close to the center of the detector." ] }, { @@ -185,7 +504,7 @@ "outputs": [], "source": [ "# Specify the yaml input file to use\n", - "yamlfile = 'movingtarget_example_data/moving_target_test.yaml'" + "yamlfile = os.path.join(yaml_output_dir, 'jw00042001001_01101_00001_nrcb5.yaml')" ] }, { @@ -196,7 +515,7 @@ }, "outputs": [], "source": [ - "# Run all steps of the imaging simulator for yaml file #1\n", + "# Run all steps of the imaging simulator for the yaml file\n", "m = imaging_simulator.ImgSim()\n", "m.paramfile = yamlfile\n", "m.create()" @@ -217,7 +536,7 @@ "source": [ "def show(array, title, min=0, max=1000):\n", " plt.figure(figsize=(12, 12))\n", - " plt.imshow(array, clim=(min, max))\n", + " plt.imshow(array, clim=(min, max), origin='lower')\n", " plt.title(title)\n", " plt.colorbar().set_label('DN$^{-}$/s')" ] @@ -231,12 +550,12 @@ "def show_mult(array1, array2, array3, title, min=0, max=1000):\n", " fig = plt.figure(figsize=(18, 18))\n", " a = fig.add_subplot(131)\n", - " aplt = plt.imshow(array1, clim=(min, max))\n", + " aplt = plt.imshow(array1, clim=(min, max), origin='lower')\n", " b = fig.add_subplot(132)\n", - " bplt = plt.imshow(array2, clim=(min, max))\n", + " bplt = plt.imshow(array2, clim=(min, max), origin='lower')\n", " plt.title(title)\n", " c = fig.add_subplot(133)\n", - " cplt = plt.imshow(array3, clim=(min, max))" + " cplt = plt.imshow(array3, clim=(min, max), origin='lower')" ] }, { @@ -253,37 +572,13 @@ "This image is an intermediate product. It contains only the signal from the astronomical sources and background. There are no detector effects, nor cosmic rays added to this count rate image." ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# First, look at the noiseless seed image\n", - "\n", - "# In this case, the seed image is 4D rather than the\n", - "# 2D that it is for sidereal targets.\n", - "# So let's look at just the final frame of the seed image\n", - "\n", - "# The non-sidereal target is in the center of the frame and appears\n", - "# as a normal PSF (although hard to see in this view). All of the \n", - "# background stars and galaxies are\n", - "# smeared, since the telescope was not tracking at the sidereal rate.\n", - "show(m.seedimage[0,-1,:,:], 'Seed Image', max=250)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Final Output Product" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Next examine the final output product. The `datatype` parameter in the yaml file specifies that Mirage should save both the raw and linearized versions of the output. Let's look first at the linearized version." + "In this case, the seed image has 4 dimensions rather than the 2 dimensions that it is for sidereal targets. This is because the moving sources lead to a seed image that is different in each group of each integration. So let's look at just the final frame of one integration of the seed image.\n", + "\n", + "We'll also zoom in, to make the motion of the background targets more visible. The non-sidereal target is in the upper left corner and appears as a normal PSF. The background star whose coordinates we specified manually when creating the point source catalog is smeared, since the telescope was not tracking at the sidereal rate." ] }, { @@ -292,26 +587,23 @@ "metadata": {}, "outputs": [], "source": [ - "lin_file = 'movingtarget_example_data/jw12345024002_01101_00001_ncrb5_linear.fits'\n", - "with fits.open(lin_file) as hdulist:\n", - " linear_data = hdulist['SCI'].data\n", - "print(linear_data.shape)" + "# First, look at the noiseless seed image. Zoom in to make the smeared\n", + "# background sources obvious. \n", + "show(m.seedimage[0,-1,850:1100,750:1000], 'Seed Image', max=25000)" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "show(linear_data[0, -1, :, :], \"Final Group\", max=250)" + "#### Final Output Product" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Examine the raw output. First a single group, which is dominated by noise and detector artifacts." + "Examine the raw output. First a single group, which contains noise and detector artifacts. By zooming in we can minimize the appearance of these effects." ] }, { @@ -320,7 +612,9 @@ "metadata": {}, "outputs": [], "source": [ - "raw_file = 'movingtarget_example_data/jw12345024002_01101_00001_ncrb5_uncal.fits'\n", + "y_base = os.path.basename(yamlfile)\n", + "raw_base = y_base.replace('.yaml', '_uncal.fits')\n", + "raw_file = os.path.join(simdata_output_dir, raw_base)\n", "with fits.open(raw_file) as hdulist:\n", " raw_data = hdulist['SCI'].data\n", "print(raw_data.shape)" @@ -332,7 +626,7 @@ "metadata": {}, "outputs": [], "source": [ - "show(raw_data[0, -1, :, :], \"Final Group\", max=15000)" + "show(raw_data[0, -1, 850:1100,750:1000], \"Final Group\", max=15000)" ] }, { @@ -348,14 +642,15 @@ "metadata": {}, "outputs": [], "source": [ - "show(1. * raw_data[0, -1, :, :] - 1. * raw_data[0, 0, :, :], \"Last Minus First Group\", max=200)" + "show(1. * raw_data[0, -1, 850:1100,750:1000] - 1. * raw_data[0, 0, 850:1100,750:1000],\n", + " \"Last Minus First Group\", max=20000)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "This raw data file is now ready to be run through the [JWST calibration pipeline](https://jwst-pipeline.readthedocs.io/en/stable/) from the beginning. If dark current subtraction is not important for you, you can use Mirage's linear output, skip some of the initial steps of the pipeline, and begin by running the [Jump detection](https://jwst-pipeline.readthedocs.io/en/stable/jwst/jump/index.html?highlight=jump) and [ramp fitting](https://jwst-pipeline.readthedocs.io/en/stable/jwst/ramp_fitting/index.html) steps." + "This raw data file is now ready to be run through the [JWST calibration pipeline](https://jwst-pipeline.readthedocs.io/en/stable/) from the beginning." ] }, { @@ -385,16 +680,14 @@ "source": [ "# yaml file that contains the parameters of the\n", "# data to be simulated\n", - "# Example yaml file shown at the bottom of this\n", - "# notebook\n", - "yamlfile = 'movingtarget_example_data/moving_target_test.yaml'" + "yamlfile = os.path.join(yaml_output_dir, 'jw00042001001_01101_00001_nrcb5.yaml')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": false + "scrolled": true }, "outputs": [], "source": [ @@ -410,30 +703,6 @@ "### Look at the seed image" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def show(array,title,min=0,max=1000):\n", - " plt.figure(figsize=(12,12))\n", - " plt.imshow(array,clim=(min,max))\n", - " plt.title(title)\n", - " plt.colorbar().set_label('DN/s')\n", - " \n", - "def show_mult(array1,array2,array3,title,min=0,max=1000):\n", - " fig = plt.figure(figsize=(18,18))\n", - " a = fig.add_subplot(131)\n", - " aplt = plt.imshow(array1,clim=(min,max))\n", - " b = fig.add_subplot(132)\n", - " bplt = plt.imshow(array2,clim=(min,max))\n", - " plt.title(title)\n", - " c = fig.add_subplot(133)\n", - " cplt = plt.imshow(array3,clim=(min,max))\n", - " #plt.colorbar().set_label('DN/s')" - ] - }, { "cell_type": "code", "execution_count": null, @@ -448,7 +717,7 @@ "# as a normal PSF (although hard to see in this view). All of the \n", "# background stars and galaxies are\n", "# smeared, since the telescope was not tracking at the sidereal rate.\n", - "show(cat.seedimage[0,-1,:,:],'Seed Image',max=250)" + "show(cat.seedimage[0, -1, 850:1100, 750:1000],'Seed Image',max=25000)" ] }, { @@ -458,8 +727,12 @@ "outputs": [], "source": [ "# Look at the first, middle, and last frames of the seed image\n", - "# so we can see the background sources moving relative to the target\n", - "show_mult(cat.seedimage[0,0,:,:],cat.seedimage[0,3,:,:],cat.seedimage[0,-1,:,:],'Seed Images',max=250)" + "# so we can see the background sources moving relative to the target,\n", + "# and the stationary non-sidereal source getting brighter as exposure\n", + "# time increases.\n", + "show_mult(cat.seedimage[0, 0, 850:1100, 750:1000],\n", + " cat.seedimage[0, 3,850:1100, 750:1000],\n", + " cat.seedimage[0, -1, 850:1100, 750:1000], 'Seed Images',max=25000)" ] }, { @@ -476,7 +749,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "d = dark_prep.DarkPrep()\n", @@ -500,7 +775,7 @@ "outputs": [], "source": [ "exptime = d.linDark.header['NGROUPS'] * cat.frametime\n", - "diff = (d.linDark.data[0,-1,:,:] - d.linDark.data[0,0,:,:]) / exptime\n", + "diff = (d.linDark.data[0, -1, 850:1100, 750:1000] - d.linDark.data[0, 0, 850:1100,750:1000]) / exptime\n", "show(diff,'Dark Current Countrate',max=0.1)" ] }, @@ -525,7 +800,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "obs = obs_generator.Observation()\n", @@ -551,7 +828,7 @@ "metadata": {}, "outputs": [], "source": [ - "with fits.open(obs.linear_output) as h:\n", + "with fits.open(obs.raw_output) as h:\n", " lindata = h[1].data\n", " header = h[0].header" ] @@ -564,19 +841,8 @@ "source": [ "# The central target is difficult to see in this full field view\n", "exptime = header['EFFINTTM']\n", - "diffdata = (lindata[0,-1,:,:] - lindata[0,0,:,:]) / exptime\n", - "show(diffdata,'Simulated Data',min=0,max=20)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Zoom in on the center of the field of view, where the target of\n", - "# interest lies.\n", - "show(diffdata[800:1200,800:1200],'Center of FOV',min=0,max=20)" + "diffdata = (lindata[0, -1, 850:1100, 750:1000] - lindata[0, 0, 850:1100, 750:1000]) / exptime\n", + "show(diffdata, 'Simulated Data', min=0, max=200)" ] }, { @@ -591,82 +857,11 @@ "# everything is positive and the noise is visible\n", "offset = 2.\n", "plt.figure(figsize=(12,12))\n", - "plt.imshow(np.log10(diffdata[800:1200,800:1200]+offset),clim=(0.001,np.log10(80)))\n", + "plt.imshow(np.log10(diffdata + offset), clim=(0.001,np.log10(80)), origin='lower')\n", "plt.title('Simulated Data')\n", "plt.colorbar().set_label('DN/s')" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "# Running simulation steps together" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## For convenience, combine the three steps into a single function." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "By having modular steps, the steps can be combined in various ways. For imaging data, including data with non-sidereal or moving targets, we will most likely want to run the three steps above in order for each target. For convenience, the imaging_simulator.py function wraps the three steps together." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from mirage import imaging_simulator" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# First, run all steps of the imaging simulator for yaml file #1\n", - "m = imaging_simulator.ImgSim()\n", - "m.paramfile = 'movingtarget_example_data/moving_target_test.yaml'\n", - "m.create()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If you have multiple exposures that will use the same dark current image (with the same readout pattern, subarray size, and number of groups), you can feed the output from the initial run of `dark_prep` into future runs of the `obs_generator`, to save time. This can be accomplished with the `imaging_simulator.py` code, as shown below.\n", - "(Note that time savings are minimal in this case, where the readout pattern is RAPID and there are only a handful of groups. This means that no averaging/skipping of frames has to be done within `dark_prep.py`)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Now that the linearized dark product has been created, if you want to use it\n", - "# when running the simulator with a different yaml file (or repeating the run\n", - "# with the same yaml file) you can provide the filename of the dark product, and the\n", - "# dark_prep step will be skipped. \n", - "# NOTE: if you use the same dark product for multiple exposures, those exposures\n", - "# will contain exactly the same dark signal. This may or may not be advisable, depending\n", - "# on your goals for the simulated data.\n", - "m = imaging_simulator.ImgSim()\n", - "m.paramfile = 'movingtarget_example_data/moving_target_test.yaml'\n", - "m.override_dark = 'movingtarget_example_data/V12345024002P000000000112o_B5_F250M_movingtarget_uncal_linear_dark_prep_object.fits'\n", - "m.create()" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -732,239 +927,6 @@ " pool.map(make_sim, paramlist)\n", "```" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Generating input yaml files" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For convenience, observing programs with multiple pointings \n", - "and detectors can be simulated starting with the program's \n", - "APT file. The xml and pointings files must be exported from \n", - "APT, and are then used as input into a tool that will\n", - "generate a series of yaml input files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from mirage.apt import apt_inputs\n", - "from mirage.yaml import yaml_generator" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "# ## Only works for normal imaging, right? Not yet modified for moving targets\n", - "\n", - "# # Create a series of data simluator input yaml files\n", - "# # from APT files\n", - "# yam = yaml_generator.SimInput()\n", - "# yam.input_xml = 'example_imaging_program.xml'\n", - "# yam.pointing_file = 'example_imaging_program.pointing'\n", - "# yam.siaf = '$MIRAGE_DATA/nircam/reference_files/SIAF/NIRCam_SIAF_2018-01-08.csv'\n", - "# yam.output_dir = './'\n", - "# yam.simdata_output_dir = './'\n", - "# yam.observation_table = 'observation_list.yaml'\n", - "# yam.use_JWST_pipeline = True\n", - "# yam.use_linearized_darks = False\n", - "# yam.datatype = 'linear'\n", - "# yam.reffile_setup(instrument='nircam')\n", - "# yam.create_inputs()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# yfiles = glob(os.path.join(yam.output_dir,'V*yaml'))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "# m = imaging_simulator.ImgSim()\n", - "# m.paramfile = yfiles[0]\n", - "# m.create()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "--- \n", - "\n", - "## Example yaml input file\n", - "\n", - "Entries listed as 'config' have default files that are present in the \n", - "config directory of the repository. The scripts are set up to \n", - "automatically find and use these files. The user can replace 'config'\n", - "with a filename if they wish to override the default.\n", - "\n", - "In general, if 'None' is placed in a field, then the step that uses\n", - "that particular file will be skipped.\n", - "\n", - "Note that the linearized_darkfile entry overrides the dark entry, unless\n", - "linearized_darkfile is set to None, in which case the dark entry will be\n", - "used.\n", - "\n", - "Use of a valid readout pattern in the readpatt entry will cause the \n", - "simulator to look up the values of nframe and nskip and ignore the \n", - "values given in the yaml file." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```yaml\n", - "Inst:\n", - " instrument: NIRCam #Instrument name\n", - " mode: imaging #Observation mode (e.g. imaging, WFSS, moving_target)\n", - " use_JWST_pipeline: False #Use pipeline in data transformations\n", - "\n", - "Readout:\n", - " readpatt: RAPID #Readout pattern (RAPID, BRIGHT2, etc) overrides nframe,nskip unless it is not recognized\n", - " nframe: 1 #Number of frames per group\n", - " nint: 1 #Number of integrations per exposure\n", - " resets_bet_ints: 1 #Number of detector resets between integrations\n", - " array_name: NRCB5_FULL #Name of array (FULL, SUB160, SUB64P, etc)\n", - " filter: F250M #Filter of simulated data (F090W, F322W2, etc)\n", - " pupil: CLEAR #Pupil element for simulated data (CLEAR, GRISMC, etc)\n", - "\n", - "Reffiles: #Set to None or leave blank if you wish to skip that step\n", - " dark: None #Dark current integration used as the base\n", - " linearized_darkfile: $MIRAGE_DATA/nircam/darks/linearized/B5/Linearized_Dark_and_SBRefpix_NRCNRCBLONG-DARK-60090141241_1_490_SE_2016-01-09T02h46m50_uncal.fits # Linearized dark ramp to use as input. Supercedes dark above\n", - " badpixmask: $MIRAGE_DATA/nircam/reference_files/badpix/NRCB5_17161_BPM_ISIMCV3_2016-01-21_ssbspmask_DMSorient.fits # If linearized dark is used, populate output DQ extensions using this file\n", - " superbias: $MIRAGE_DATA/nircam/reference_files/superbias/NRCB5_superbias_from_list_of_biasfiles.list.fits #Superbias file. Set to None or leave blank if not using\n", - " linearity: $MIRAGE_DATA/nircam/reference_files/linearity/NRCBLONG_17161_LinearityCoeff_ADU0_2016-05-22_ssblinearity_v2_DMSorient.fits #linearity correction coefficients\n", - " saturation: $MIRAGE_DATA/nircam/reference_files/saturation/NRCB5_17161_WellDepthADU_2016-03-10_ssbsaturation_wfact_DMSorient.fits #well depth reference files\n", - " gain: $MIRAGE_DATA/nircam/reference_files/gain/NRCB5_17161_Gain_ISIMCV3_2016-02-25_ssbgain_DMSorient.fits #Gain map\n", - " pixelflat: None \n", - " illumflat: None #Illumination flat field file\n", - " astrometric: $MIRAGE_DATA/nircam/reference_files/distortion/NRCB5_FULL_distortion.asdf #Astrometric distortion file (asdf)\n", - " distortion_coeffs: $MIRAGE_DATA/nircam/reference_files/SIAF/NIRCam_SIAF_2017-03-28.csv #CSV file containing distortion coefficients\n", - " ipc: $MIRAGE_DATA/nircam/reference_files/ipc/NRCB5_17161_IPCDeconvolutionKernel_2016-03-18_ssbipc_DMSorient.fits #File containing IPC kernel to apply\n", - " invertIPC: True #Invert the IPC kernel before the convolution. True or False. Use True if the kernel is designed for the removal of IPC effects, like the JWST reference files are.\n", - " occult: None #Occulting spots correction image\n", - " pixelAreaMap: $MIRAGE_DATA/nircam/reference_files/pam/NIRCam_B5_PAM_imaging.fits #Pixel area map for the detector. Used to introduce distortion into the output ramp.\n", - " subarray_defs: config #File that contains a list of all possible subarray names and coordinates\n", - " readpattdefs: config #File that contains a list of all possible readout pattern names and associated NFRAME/NSKIP values\n", - " crosstalk: config #File containing crosstalk coefficients\n", - " filtpupilcombo: config #File that lists the filter wheel element / pupil wheel element combinations. Used only in writing output file\n", - " flux_cal: config #File that lists flux conversion factor and pivot wavelength for each filter. Only used when making direct image outputs to be fed into the grism disperser code.\n", - " \n", - "nonlin:\n", - " limit: 60000.0 #Upper singal limit to which nonlinearity is applied (ADU)\n", - " accuracy: 0.000001 #Non-linearity accuracy threshold\n", - " maxiter: 10 #Maximum number of iterations to use when applying non-linearity\n", - " robberto: False #Use Massimo Robberto type non-linearity coefficients\n", - "\n", - "cosmicRay:\n", - " path: $MIRAGE_DATA/nircam/cosmic_ray_library/ #Path to CR library\n", - " library: SUNMIN #Type of cosmic rayenvironment (SUNMAX, SUNMIN, FLARE)\n", - " scale: 1.5 #Cosmic ray scaling factor\n", - " suffix: IPC_NIRCam_B5 #Suffix of library file names\n", - " seed: 2956411739 #Seed for random number generator\n", - "\n", - "simSignals:\n", - " pointsource: my_ptsrc_catalog.list #File containing a list of point sources to add (x,y locations and magnitudes)\n", - " psfpath: $MIRAGE_DATA/nircam/psf_data/ #Path to PSF library\n", - " psfbasename: nircam #Basename of the files in the psf library\n", - " psfpixfrac: 0.25 #Fraction of a pixel between entries in PSF library (e.g. 0.1 = files for PSF centered at 0.25 pixel intervals within pixel)\n", - " psfwfe: predicted #PSF WFE value (predicted, requirements)\n", - " psfwfegroup: 0 #WFE realization group (0 to 9)\n", - " galaxyListFile: my_galaxies_catalog.list\n", - " extended: None #Extended emission count rate image file name\n", - " extendedscale: 1.0 #Scaling factor for extended emission image\n", - " extendedCenter: 1024,1024 #x,y pixel location at which to place the extended image if it is smaller than the output array size\n", - " PSFConvolveExtended: True #Convolve the extended image with the PSF before adding to the output image (True or False)\n", - " movingTargetList: None #Name of file containing a list of point source moving targets (e.g. KBOs, asteroids) to add.\n", - " movingTargetSersic: None #ascii file containing a list of 2D sersic profiles to have moving through the field\n", - " movingTargetExtended: None #ascii file containing a list of stamp images to add as moving targets (planets, moons, etc)\n", - " movingTargetConvolveExtended: True #convolve the extended moving targets with PSF before adding.\n", - " movingTargetToTrack: my_nonsidereal_target.cat #File containing a single moving target which JWST will track during observation (e.g. a planet, moon, KBO, asteroid)\tThis file will only be used if mode is set to \"moving_target\" \n", - " zodiacal: None #Zodiacal light count rate image file \n", - " zodiscale: 1.0 #Zodi scaling factor\n", - " scattered: None #Scattered light count rate image file\n", - " scatteredscale: 1.0 #Scattered light scaling factor\n", - " bkgdrate: 0.0 #Constant background count rate (electrons/sec/pixel)\n", - " poissonseed: 2012872553 #Random number generator seed for Poisson simulation)\n", - " photonyield: True #Apply photon yield in simulation\n", - " pymethod: True #Use double Poisson simulation for photon yield\n", - "\n", - "Telescope:\n", - " ra: 53.1 #RA of simulated pointing\n", - " dec: -27.8 #Dec of simulated pointing\n", - " rotation: 0.0 #y axis rotation (degrees E of N)\n", - " tracking: non-sidereal #sidereal or non-sidereal\n", - " \n", - "newRamp:\n", - " dq_configfile: config #config file used by JWST pipeline\n", - " sat_configfile: config #config file used by JWST pipeline\n", - " superbias_configfile: config #config file used by JWST pipeline\n", - " refpix_configfile: config #config file used by JWST pipeline \n", - " linear_configfile: config #config file used by JWST pipeline\n", - "\n", - "Output:\n", - " file: V42424024002P000000000112o_B5_F250M_uncal.fits #Output filename\n", - " directory: ./ # Directory in which to place output files\n", - " datatype: linear,raw # Type of data to save. 'linear' for linearized ramp. 'raw' for raw ramp. 'linear,raw' for both\n", - " format: DMS #Output file format Options: DMS, SSR(not yet implemented)\n", - " save_intermediates: False #Save intermediate products separately (point source image, etc)\n", - " grism_source_image: False # Create an image to be dispersed?\n", - " unsigned: True #Output unsigned integers? (0-65535 if true. -32768 to 32768 if false)\n", - " dmsOrient: True #Output in DMS orientation (vs. fitswriter orientation).\n", - " program_number: 42424 #Program Number\n", - " title: Supernovae and Black Holes Near Hyperspatial Bypasses #Program title\n", - " PI_Name: Doug Adams #Proposal PI Name\n", - " Proposal_category: GO #Proposal category\n", - " Science_category: Cosmology #Science category\n", - " observation_number: '002' #Observation Number\n", - " observation_label: Obs2 #User-generated observation Label\n", - " visit_number: '024' #Visit Number\n", - " visit_group: '01' #Visit Group\n", - " visit_id: '42424024002' #Visit ID\n", - " sequence_id: '2' #Sequence ID\n", - " activity_id: '2o' #Activity ID. Increment with each exposure.\n", - " exposure_number: '00001' #Exposure Number\n", - " obs_id: 'V42424024002P000000000112o' #Observation ID number\n", - " date_obs: '2019-10-15' #Date of observation\n", - " time_obs: '06:29:11.852' #Time of observation\n", - " obs_template: 'NIRCam Imaging' #Observation template\n", - " primary_dither_type: NONE #Primary dither pattern name\n", - " total_primary_dither_positions: 1 #Total number of primary dither positions\n", - " primary_dither_position: 1 #Primary dither position number\n", - " subpix_dither_type: 2-POINT-MEDIUM-WITH-NIRISS #Subpixel dither pattern name\n", - " total_subpix_dither_positions: 2 #Total number of subpixel dither positions\n", - " subpix_dither_position: 2 #Subpixel dither position number\n", - " xoffset: 344.284 #Dither pointing offset in x (arcsec)\n", - " yoffset: 466.768 #Dither pointing offset in y (arcsec)\n", - "```" - ] } ], "metadata": { diff --git a/mirage/catalogs/catalog_generator.py b/mirage/catalogs/catalog_generator.py index 2a4a17013..ef884d2b8 100644 --- a/mirage/catalogs/catalog_generator.py +++ b/mirage/catalogs/catalog_generator.py @@ -411,6 +411,16 @@ class MovingPointSourceCatalog(PointSourceCatalog): def __init__(self, ra=[], dec=[], x=[], y=[], ra_velocity=[], dec_velocity=[], x_velocity=[], y_velocity=[], ephemeris_file=[], starting_index=1): + # If the user provides ephemeris files, they shouldn't need to provide any + # other position or velocity information. + if len(ephemeris_file) > 0: + if len(ra) == 0 and len(x) == 0: + ra = [None] * len(ephemeris_file) + dec = [None] * len(ephemeris_file) + if len(ra_velocity) == 0 and len(x_velocity) == 0: + ra_velocity = [None] * len(ephemeris_file) + dec_velocity = [None] * len(ephemeris_file) + # Add location information PointSourceCatalog.__init__(self, ra=ra, dec=dec, x=x, y=y, starting_index=starting_index) From 6f68834335348128e4a75e85ceca9fa65bafb4ca Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 7 Dec 2020 13:44:56 -0500 Subject: [PATCH 12/13] Add new files needed for example notebook --- .../mars_ephemeris.txt | 1 + .../mars_example.pointing | 16 +++ .../mars_example.xml | 127 ++++++++++++++++++ mirage/apt/apt_inputs.py | 11 -- 4 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 examples/movingtarget_example_data/mars_ephemeris.txt create mode 100644 examples/movingtarget_example_data/mars_example.pointing create mode 100644 examples/movingtarget_example_data/mars_example.xml diff --git a/examples/movingtarget_example_data/mars_ephemeris.txt b/examples/movingtarget_example_data/mars_ephemeris.txt new file mode 100644 index 000000000..878622a00 --- /dev/null +++ b/examples/movingtarget_example_data/mars_ephemeris.txt @@ -0,0 +1 @@ +******************************************************************************* Revised: June 21, 2016 Mars 499 / 4 PHYSICAL DATA (updated 2019-Oct-29): Vol. mean radius (km) = 3389.92+-0.04 Density (g/cm^3) = 3.933(5+-4) Mass x10^23 (kg) = 6.4171 Flattening, f = 1/169.779 Volume (x10^10 km^3) = 16.318 Equatorial radius (km)= 3396.19 Sidereal rot. period = 24.622962 hr Sid. rot. rate, rad/s = 0.0000708822 Mean solar day (sol) = 88775.24415 s Polar gravity m/s^2 = 3.758 Core radius (km) = ~1700 Equ. gravity m/s^2 = 3.71 Geometric Albedo = 0.150 GM (km^3/s^2) = 42828.375214 Mass ratio (Sun/Mars) = 3098703.59 GM 1-sigma (km^3/s^2) = +- 0.00028 Mass of atmosphere, kg= ~ 2.5 x 10^16 Mean temperature (K) = 210 Atmos. pressure (bar) = 0.0056 Obliquity to orbit = 25.19 deg Max. angular diam. = 17.9" Mean sidereal orb per = 1.88081578 y Visual mag. V(1,0) = -1.52 Mean sidereal orb per = 686.98 d Orbital speed, km/s = 24.13 Hill's sphere rad. Rp = 319.8 Escape speed, km/s = 5.027 Perihelion Aphelion Mean Solar Constant (W/m^2) 717 493 589 Maximum Planetary IR (W/m^2) 470 315 390 Minimum Planetary IR (W/m^2) 30 30 30 ******************************************************************************* ******************************************************************************* Ephemeris / WWW_USER Tue Sep 8 07:58:04 2020 Pasadena, USA / Horizons ******************************************************************************* Target body name: Mars (499) {source: mar097} Center body name: Earth (399) {source: mar097} Center-site name: GEOCENTRIC ******************************************************************************* Start time : A.D. 2020-Sep-08 00:00:00.0000 UT Stop time : A.D. 2020-Oct-08 00:00:00.0000 UT Step-size : 1440 minutes ******************************************************************************* Target pole/equ : IAU_MARS {West-longitude positive} Target radii : 3396.2 x 3396.2 x 3376.2 km {Equator, meridian, pole} Center geodetic : 0.00000000,0.00000000,0.0000000 {E-lon(deg),Lat(deg),Alt(km)} Center cylindric: 0.00000000,0.00000000,0.0000000 {E-lon(deg),Dxy(km),Dz(km)} Center pole/equ : High-precision EOP model {East-longitude positive} Center radii : 6378.1 x 6378.1 x 6356.8 km {Equator, meridian, pole} Target primary : Sun Vis. interferer : MOON (R_eq= 1737.400) km {source: mar097} Rel. light bend : Sun, EARTH {source: mar097} Rel. lght bnd GM: 1.3271E+11, 3.9860E+05 km^3/s^2 Atmos refraction: NO (AIRLESS) RA format : HMS Time format : CAL EOP file : eop.200904.p201126 EOP coverage : DATA-BASED 1962-JAN-20 TO 2020-SEP-04. PREDICTS-> 2020-NOV-25 Units conversion: 1 au= 149597870.700 km, c= 299792.458 km/s, 1 day= 86400.0 s Table cut-offs 1: Elevation (-90.0deg=NO ),Airmass (>38.000=NO), Daylight (NO ) Table cut-offs 2: Solar elongation ( 0.0,180.0=NO ),Local Hour Angle( 0.0=NO ) Table cut-offs 3: RA/DEC angular rate ( 0.0=NO ) **************************************************************************************************************** Date__(UT)__HR:MN R.A._____(ICRF)_____DEC APmag S-brt delta deldot S-O-T /r S-T-O **************************************************************************************************************** $$SOE 2020-Sep-08 00:00 01 49 28.77 +06 43 43.8 -1.971 4.194 0.46952793693765 -5.9857519 137.5163 /L 29.3023 2020-Sep-09 00:00 01 49 32.98 +06 44 29.8 -2.002 4.182 0.46611159837859 -5.8444871 138.4610 /L 28.7015 2020-Sep-10 00:00 01 49 33.78 +06 45 01.7 -2.040 4.162 0.46277820312601 -5.6984724 139.4207 /L 28.0866 2020-Sep-11 00:00 01 49 31.14 +06 45 19.4 -2.072 4.148 0.45953055063547 -5.5475133 140.3956 /L 27.4577 2020-Sep-12 00:00 01 49 25.05 +06 45 23.1 -2.099 4.140 0.45637155219742 -5.3914172 141.3856 /L 26.8146 2020-Sep-13 00:00 01 49 15.49 +06 45 12.8 -2.112 4.144 0.45330422893377 -5.2299976 142.3908 /L 26.1573 2020-Sep-14 00:00 01 49 02.46 +06 44 48.6 -2.133 4.140 0.45033170583641 -5.0630836 143.4113 /L 25.4859 2020-Sep-15 00:00 01 48 45.96 +06 44 10.7 -2.144 4.145 0.44745719813726 -4.8905370 144.4468 /L 24.8004 2020-Sep-16 00:00 01 48 26.02 +06 43 19.2 -2.166 4.140 0.44468398611775 -4.7122749 145.4974 /L 24.1009 2020-Sep-17 00:00 01 48 02.64 +06 42 14.6 -2.184 4.138 0.44201537626493 -4.5282930 146.5628 /L 23.3875 2020-Sep-18 00:00 01 47 35.87 +06 40 57.0 -2.201 4.137 0.43945465112138 -4.3386795 147.6429 /L 22.6605 2020-Sep-19 00:00 01 47 05.75 +06 39 26.7 -2.219 4.133 0.43700501610956 -4.1436105 148.7374 /L 21.9201 2020-Sep-20 00:00 01 46 32.35 +06 37 44.3 -2.236 4.131 0.43466955556327 -3.9433235 149.8459 /L 21.1666 2020-Sep-21 00:00 01 45 55.72 +06 35 50.1 -2.245 4.136 0.43245120856057 -3.7380771 150.9682 /L 20.4003 2020-Sep-22 00:00 01 45 15.95 +06 33 44.7 -2.252 4.141 0.43035276792797 -3.5281138 152.1036 /L 19.6217 2020-Sep-23 00:00 01 44 33.12 +06 31 28.5 -2.272 4.134 0.42837689742046 -3.3136358 153.2520 /L 18.8312 2020-Sep-24 00:00 01 43 47.30 +06 29 01.9 -2.296 4.122 0.42652615755415 -3.0947986 154.4127 /L 18.0291 2020-Sep-25 00:00 01 42 58.61 +06 26 25.7 -2.318 4.111 0.42480303123358 -2.8717172 155.5852 /L 17.2161 2020-Sep-26 00:00 01 42 07.13 +06 23 40.2 -2.343 4.097 0.42320994395582 -2.6444776 156.7690 /L 16.3927 2020-Sep-27 00:00 01 41 12.97 +06 20 46.0 -2.361 4.088 0.42174927709785 -2.4131484 157.9634 /L 15.5594 2020-Sep-28 00:00 01 40 16.26 +06 17 43.8 -2.374 4.085 0.42042337508501 -2.1777902 159.1678 /L 14.7169 2020-Sep-29 00:00 01 39 17.09 +06 14 34.2 -2.366 4.100 0.41923454802078 -1.9384622 160.3814 /L 13.8658 2020-Sep-30 00:00 01 38 15.61 +06 11 17.6 -2.388 4.085 0.41818507119621 -1.6952265 161.6033 /L 13.0071 2020-Oct-01 00:00 01 37 11.95 +06 07 54.9 -2.413 4.067 0.41727718237934 -1.4481517 162.8326 /L 12.1416 2020-Oct-02 00:00 01 36 06.24 +06 04 26.6 -2.423 4.063 0.41651307729612 -1.1973152 164.0681 /L 11.2702 2020-Oct-03 00:00 01 34 58.64 +06 00 53.4 -2.419 4.072 0.41589490343749 -0.9428071 165.3084 /L 10.3943 2020-Oct-04 00:00 01 33 49.29 +05 57 16.1 -2.436 4.059 0.41542475229821 -0.6847317 166.5517 /L 9.5151 2020-Oct-05 00:00 01 32 38.35 +05 53 35.3 -2.468 4.030 0.41510465030449 -0.4232102 167.7960 /L 8.6344 2020-Oct-06 00:00 01 31 26.00 +05 49 51.9 -2.491 4.008 0.41493654889816 -0.1583813 169.0384 /L 7.7546 2020-Oct-07 00:00 01 30 12.41 +05 46 06.6 -2.518 3.983 0.41492231438136 0.1095989 170.2748 /L 6.8785 2020-Oct-08 00:00 01 28 57.75 +05 42 20.3 -2.544 3.957 0.41506371808352 0.3805578 171.4994 /L 6.0107 $$EOE **************************************************************************************************************** Column meaning: TIME Times PRIOR to 1962 are UT1, a mean-solar time closely related to the prior but now-deprecated GMT. Times AFTER 1962 are in UTC, the current civil or "wall-clock" time-scale. UTC is kept within 0.9 seconds of UT1 using integer leap-seconds for 1972 and later years. Conversion from the internal Barycentric Dynamical Time (TDB) of solar system dynamics to the non-uniform civil UT time-scale requested for output has not been determined for UTC times after the next July or January 1st. Therefore, the last known leap-second is used as a constant over future intervals. Time tags refer to the UT time-scale conversion from TDB on Earth regardless of observer location within the solar system, although clock rates may differ due to the local gravity field and no analog to "UT" may be defined for that location. Any 'b' symbol in the 1st-column denotes a B.C. date. First-column blank (" ") denotes an A.D. date. Calendar dates prior to 1582-Oct-15 are in the Julian calendar system. Later calendar dates are in the Gregorian system. NOTE: "n.a." in output means quantity "not available" at the print-time. 'R.A._____(ICRF)_____DEC' = Astrometric right ascension and declination of the target center with respect to the observing site (coordinate origin) in the reference frame of the planetary ephemeris (ICRF). Compensated for down-leg light-time delay aberration. Units: RA in hours-minutes-seconds of time, HH MM SS.ff{ffff} DEC in degrees-minutes-seconds of arc, sDD MN SC.f{ffff} 'APmag S-brt' = The targets' approximate apparent visual magnitude and surface brightness. For planets and natural satellites, output is restricted to solar phase angles covered by observational data. Outside the observed phase angle range, "n.a." may be output to avoid extrapolation beyond the limit of model validity. For Earth-based observers, the estimated dimming due to atmospheric absorption (extinction) is available as a separate, requestable quantity. Surface brightness is the average airless visual magnitude of a square-arcsecond of the illuminated portion of the apparent disk. It is computed only if the target radius is known. Units: MAGNITUDES & MAGNITUDES PER SQUARE ARCSECOND 'delta deldot' = Apparent range ("delta", light-time aberrated) and range-rate ("delta-dot") of the target center relative to the observer. A positive "deldot" means the target center is moving away from the observer, negative indicates movement toward the observer. Units: AU and KM/S 'S-O-T /r' = Sun-Observer-Target apparent SOLAR ELONGATION ANGLE seen from the observers' location at print-time. The '/r' column provides a code indicating the targets' apparent position relative to the Sun in the observers' sky, as described below: Case A: For an observing location on the surface of a rotating body, that body rotational sense is considered: /T indicates target TRAILS Sun (evening sky: rises and sets AFTER Sun) /L indicates target LEADS Sun (morning sky: rises and sets BEFORE Sun) Case B: For an observing point that does not have a rotational model (such as a spacecraft), the "leading" and "trailing" condition is defined by the observers' heliocentric ORBITAL motion: * If continuing in the observers' current direction of heliocentric motion would encounter the targets' apparent longitude first, followed by the Sun's, the target LEADS the Sun as seen by the observer. * If the Sun's apparent longitude would be encountered first, followed by the targets', the target TRAILS the Sun. Two other codes can be output: /* indicates observer is Sun-centered (undefined) /? Target is aligned with Sun center (no lead or trail) The S-O-T solar elongation angle is numerically the minimum separation angle of the Sun and target in the sky in any direction. It does NOT indicate the amount of separation in the leading or trailing directions, which would be defined along the equator of a spherical coordinate system. Units: DEGREES 'S-T-O' = The Sun-Target-Observer angle; the interior vertex angle at target center formed by a vector from the target to the apparent center of the Sun (at reflection time on the target) and the apparent vector from target to the observer at print-time. Slightly different from true PHASE ANGLE (requestable separately) at the few arcsecond level in that it includes stellar aberration on the down-leg from target to observer. Units: DEGREES Computations by ... Solar System Dynamics Group, Horizons On-Line Ephemeris System 4800 Oak Grove Drive, Jet Propulsion Laboratory Pasadena, CA 91109 USA Information : https://ssd.jpl.nasa.gov/ Documentation: https://ssd.jpl.nasa.gov/?horizons_doc Connect : https://ssd.jpl.nasa.gov/?horizons (browser) telnet ssd.jpl.nasa.gov 6775 (command-line) e-mail command interface available Script and CGI interfaces available Author : Jon.D.Giorgini@jpl.nasa.gov **************************************************************************************************************** \ No newline at end of file diff --git a/examples/movingtarget_example_data/mars_example.pointing b/examples/movingtarget_example_data/mars_example.pointing new file mode 100644 index 000000000..79d3e0ef7 --- /dev/null +++ b/examples/movingtarget_example_data/mars_example.pointing @@ -0,0 +1,16 @@ +# APT Output Product +# +# APT Version: Version 2020.4.1 JWST PRD: PRDOPSSOC-030 +# Date: Fri Dec 04 20:45:00 GMT 2020 + +JWST Pointing Report for JWST Draft Proposal (mars_example.aptx) + +======================================== +* Observation 1 (Obs 1) +** Visit 1:1 +Tar Tile Exp Dith Aperture Name Target RA Dec BaseX BaseY DithX DithY V2 V3 IdlX IdlY Level Type ExPar DkPar dDist + 1 1 1 1 NRCBS_FULL 3 MARS +0.00000 +0.00000 +0.000 +0.000 -3.750 +3.750 -78.369 -492.291 -3.750 +3.750 TARGET SCIENCE 0 0 0.000 (base) + 1 1 1 2 NRCBS_FULL 3 MARS +0.00000 +0.00000 +0.000 +0.000 +3.750 -3.750 -85.873 -499.787 +3.750 -3.750 DITHER SCIENCE 0 0 10.607 + 1 1 1 3 NRCBS_FULL 3 MARS +0.00000 +0.00000 +0.000 +0.000 +11.250 -11.250 -93.377 -507.283 +11.250 -11.250 DITHER SCIENCE 0 0 10.607 + + diff --git a/examples/movingtarget_example_data/mars_example.xml b/examples/movingtarget_example_data/mars_example.xml new file mode 100644 index 000000000..1cec84996 --- /dev/null +++ b/examples/movingtarget_example_data/mars_example.xml @@ -0,0 +1,127 @@ + + + + + + + Draft + 0 + GO + NIRCAM + SMALL + O[0 Months] + 1 + true + false + false + false + + + false + false + false + + true + + + + false + + false + false + false + + false + false + + + + + 1 + ONE + ONE + + + + + Unknown + Calibration + Astrometric + Detector linearity test + + + + Arcsec + + + + Arcsec + + + false + false + + + 3 + MARS + MARS + MARS + YES + + false + + + + + + + 1 + 3 MARS + + NIRCAM + + 324 + false + + + 1 + 1 + 10.0 + 10.0 + 0.0 + 0.0 + + Tile Included + + DEFAULT + + + + + + false + + + + + + diff --git a/mirage/apt/apt_inputs.py b/mirage/apt/apt_inputs.py index 5d586f04f..7af05a392 100755 --- a/mirage/apt/apt_inputs.py +++ b/mirage/apt/apt_inputs.py @@ -1387,18 +1387,7 @@ def ra_dec_update(exposure_dict, siaf_instances, verbose=False): # Calculate RA, Dec of reference location for the detector # Add in any offsets from the pointing file in the BaseX, BaseY columns - #if siaf_instrument.lower() != 'niriss': ra, dec = rotations.pointing(attitude_matrix, aperture.V2Ref, aperture.V3Ref) - #else: - # ra, dec = exposure_dict['ra_ref'][i], exposure_dict['dec_ref'][i] - - - - #print(exposure_dict['aperture'], exposure_dict['ra_ref'], exposure_dict['dec_ref']) - #print(ra, dec) - #stop - - aperture_ra.append(ra) aperture_dec.append(dec) From 18d03f964323fb5db4cdc8183472ed90d1303f13 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 7 Dec 2020 13:50:14 -0500 Subject: [PATCH 13/13] Remove print statements left from development --- mirage/apt/apt_inputs.py | 7 +------ mirage/yaml/yaml_generator.py | 18 ------------------ 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/mirage/apt/apt_inputs.py b/mirage/apt/apt_inputs.py index 7af05a392..60b1d614f 100755 --- a/mirage/apt/apt_inputs.py +++ b/mirage/apt/apt_inputs.py @@ -855,12 +855,7 @@ def get_pointing_info(self, file, propid=0, verbose=False): type_str.append(elements[18]) expar.append(np.int(elements[19])) dkpar.append(np.int(elements[20])) - #if elements[18] == 'PARALLEL': - # ddist.append(None) - #else: - #print('line is: {}'.format(elements)) - #print(ddist) - #ddist.append(np.float(elements[21])) + # For the moment we assume that the instrument being simulated is not being # run in parallel, so the parallel proposal number will be all zeros, # as seen in the line below. diff --git a/mirage/yaml/yaml_generator.py b/mirage/yaml/yaml_generator.py index f445de25f..a6f7de8a7 100755 --- a/mirage/yaml/yaml_generator.py +++ b/mirage/yaml/yaml_generator.py @@ -665,24 +665,8 @@ def create_inputs(self): siaf_dictionary = {} for instrument_name in np.unique(self.info['Instrument']): siaf_dictionary[instrument_name] = siaf_interface.get_instance(instrument_name) - - - print(self.info['ra_ref'], self.info['dec_ref']) - - self.info = apt_inputs.ra_dec_update(self.info, siaf_dictionary) - - print(self.info['ra_ref'], self.info['dec_ref']) - print('') - - - print('Final RA, Dec values:') - for ele_ra, ele_raref, ele_ap, ele_date, ele_time in zip(self.info['ra'], self.info['ra_ref'], self.info['aperture'], self.info['date_obs'], self.info['time_obs'],): - print(ele_ap, ele_date, ele_time, ele_ra, ele_raref) - - - # Add a list of output yaml names to the dictionary self.make_output_names() @@ -1289,8 +1273,6 @@ def nonsidereal_pointing_updates(self): # If the source velocity is given in units of pixels/hour, then we need # to multiply this by the appropriate pixel scale. if vel_in_xy: - print('UNIQUE APERTURES: ') - print(unique_apertures, '\n') if len(unique_apertures) > 1: if inst.lower() == 'nircam': det_ints = [int(ele.split('_')[0][-1]) for ele in unique_apertures]