Merge pull request #101 from sot/rename-observation_id

Rename observation_id

agasc/scripts/update_mag_supplement.py

@@ -108,10 +108,10 @@ def main():
         format="%(asctime)s %(message)s"
     )
 
-    if (((args.obsid or args.observation_id) and not args.status)
-            or (not (args.obsid or args.observation_id) and args.status)):
+    if (((args.obsid or args.mp_starcat_time) and not args.status)
+            or (not (args.obsid or args.mp_starcat_time) and args.status)):
         logger.error(
-            'To override OBS status, both --obs/observation-id and --status options are needed.')
+            'To override OBS status, both --obs/mp-starcat-time and --status options are needed.')
         the_parser.exit(1)
 
     star_obs_catalogs.load(args.stop)

agasc/scripts/update_supplement.py

@@ -61,14 +61,14 @@ def _sanitize_args(status):
     for value in status['obs']:
         if cat.STARS_OBS is None:
             raise RuntimeError('Observation catalog is not initialized')
-        if 'observation_id' in value:
+        if 'mp_starcat_time' in value:
             key = 'mp_starcat_time'
-            rows = cat.STARS_OBS[cat.STARS_OBS['mp_starcat_time'] == value['observation_id']]
+            rows = cat.STARS_OBS[cat.STARS_OBS['mp_starcat_time'] == value['mp_starcat_time']]
         elif 'obsid' in value:
             key = 'obsid'
             rows = cat.STARS_OBS[cat.STARS_OBS['obsid'] == value['obsid']]
         else:
-            raise Exception('Need to specify observation_id or OBSID')
+            raise Exception('Need to specify mp_starcat_time or OBSID')
         if len(rows) == 0:
             raise Exception(
                 f'Observation catalog has no observation with {key}={value[key]}'
@@ -80,8 +80,8 @@ def _sanitize_args(status):
             value['agasc_id'] = (list(np.atleast_1d(value['agasc_id'])))
         if 'comments' not in value:
             value['comments'] = ''
-        if 'observation_id' not in value:
-            value['observation_id'] = rows['mp_starcat_time'][0]
+        if 'mp_starcat_time' not in value:
+            value['mp_starcat_time'] = rows['mp_starcat_time'][0]
         if 'obsid' not in value:
             value['obsid'] = rows['obsid'][0]
 
@@ -91,7 +91,7 @@ def _sanitize_args(status):
         # matching mp_starcat_time and OBSID
         if rows['obsid'][0] != value['obsid']:
             raise Exception(f'inconsistent observation spec {value}')
-        if rows['mp_starcat_time'][0] != value['observation_id']:
+        if rows['mp_starcat_time'][0] != value['mp_starcat_time']:
             raise Exception(f'inconsistent observation spec {value}')
 
     rows = []
@@ -105,7 +105,7 @@ def _sanitize_args(status):
 
 
 def parse_args(filename=None, bad_star_id=None, bad_star_source=None,
-               obsid=None, status=None, comments='', agasc_id=None, observation_id=None,
+               obsid=None, status=None, comments='', agasc_id=None, mp_starcat_time=None,
                **_
                ):
     """
@@ -122,7 +122,7 @@ def parse_args(filename=None, bad_star_id=None, bad_star_source=None,
     :param status: int
     :param comments: str
     :param agasc_id: int
-    :param observation_id: str
+    :param mp_starcat_time: str
     :return:
     """
     obs_status_override = []
@@ -139,15 +139,15 @@ def parse_args(filename=None, bad_star_id=None, bad_star_source=None,
         obs_status_override = status_file['obs']
         mags = status_file['mags']
 
-    if (obsid is not None or observation_id is not None) and status is not None:
+    if (obsid is not None or mp_starcat_time is not None) and status is not None:
         row = {
-            'observation_id': observation_id,
+            'mp_starcat_time': mp_starcat_time,
             'agasc_id': agasc_id,
             'obsid': obsid,
             'status': status,
             'comments': comments
         }
-        optional = ['obsid', 'observation_id', 'agasc_id', 'comments']
+        optional = ['obsid', 'mp_starcat_time', 'agasc_id', 'comments']
         obs_status_override.append(
             {key: row[key] for key in row if key not in optional or row[key] is not None}
         )
@@ -179,8 +179,8 @@ def get_obs_status_parser():
     status.add_argument('--obs-status-file',
                         help='YAML file with star/observation status. '
                              'More info at https://sot.github.io/agasc/supplement.html')
-    status.add_argument('--observation-id',
-                        help='Observation ID for status override. '
+    status.add_argument('--mp-starcat-time',
+                        help='Observation starcat time for status override. '
                              'Usually the mission planning catalog time')
     status.add_argument('--obsid', help='OBSID for status override.')
     status.add_argument('--agasc-id', help='AGASC ID for status override.')

agasc/supplement/magnitudes/mag_estimate.py

@@ -58,30 +58,30 @@
 
 
 class MagStatsException(Exception):
-    def __init__(self, msg='', agasc_id=None, obsid=None, timeline_id=None, observation_id=None,
+    def __init__(self, msg='', agasc_id=None, obsid=None, timeline_id=None, mp_starcat_time=None,
                  **kwargs):
         super().__init__(msg)
         self.error_code = EXCEPTION_CODES[msg]
         self.msg = msg
         self.agasc_id = agasc_id
         self.obsid = obsid[0] if type(obsid) is list and len(obsid) == 1 else obsid
         self.timeline_id = timeline_id
-        self.observation_id = (observation_id[0] if type(observation_id) is list
-                               and len(observation_id) == 1 else observation_id)
+        self.mp_starcat_time = (mp_starcat_time[0] if type(mp_starcat_time) is list
+                                and len(mp_starcat_time) == 1 else mp_starcat_time)
         for k in kwargs:
             setattr(self, k, kwargs[k])
 
     def __str__(self):
         return f'MagStatsException: {self.msg} (agasc_id: {self.agasc_id}, ' \
-               f'obsid: {self.obsid}, observation_id: {self.observation_id})'
+               f'obsid: {self.obsid}, mp_starcat_time: {self.mp_starcat_time})'
 
     def __iter__(self):
         yield 'error_code', self.error_code
         yield 'msg', self.msg
         yield 'agasc_id', self.agasc_id
         yield 'obsid', self.obsid
         yield 'timeline_id', self.timeline_id
-        yield 'observation_id', self.observation_id
+        yield 'mp_starcat_time', self.mp_starcat_time
 
 
 def _magnitude_correction(time, mag_aca):
@@ -299,7 +299,7 @@ def get_telemetry(obs):
         raise MagStatsException('No level 0 data',
                                 agasc_id=obs["agasc_id"],
                                 obsid=obs["obsid"],
-                                observation_id=obs["mp_starcat_time"],
+                                mp_starcat_time=obs["mp_starcat_time"],
                                 time_range=[start, stop],
                                 slot=obs['slot'])
     tmin = np.min([np.min(slot_data['END_INTEG_TIME']), np.min(msid.times)])
@@ -316,7 +316,7 @@ def get_telemetry(obs):
         raise MagStatsException('Time mismatch between cheta and level0',
                                 agasc_id=obs["agasc_id"],
                                 obsid=obs["obsid"],
-                                observation_id=obs["mp_starcat_time"])
+                                mp_starcat_time=obs["mp_starcat_time"])
 
     # get the normal sun and safe sun mode intervals, which will be removed
     excluded_ranges = []
@@ -355,7 +355,7 @@ def get_telemetry(obs):
     if len(telem['AOACASEQ']) != len(telem['IMGSIZE']):
         raise MagStatsException(
             "Mismatch in telemetry between aca_l0 and cheta",
-            agasc_id=obs['agasc_id'], obsid=obs['obsid'], observation_id=obs["mp_starcat_time"]
+            agasc_id=obs['agasc_id'], obsid=obs['obsid'], mp_starcat_time=obs["mp_starcat_time"]
         )
     for name in ['AOACIIR', 'AOACISP', 'AOACYAN', 'AOACZAN', 'AOACMAG', 'AOACFCT']:
         telem[name] = telem[f'{name}{slot}']
@@ -583,7 +583,7 @@ def get_obs_stats(obs, telem=None):
 
     stats = {k: obs[k] for k in
              ['agasc_id', 'obsid', 'slot', 'type', 'mp_starcat_time', 'timeline_id']}
-    stats['observation_id'] = stats['mp_starcat_time']
+    stats['mp_starcat_time'] = stats['mp_starcat_time']
     droop_shift = get_droop_systematic_shift(star['MAG_ACA'])
     responsivity = get_responsivity(start)
     stats.update({'tstart': start,
@@ -849,7 +849,7 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None):
         comment = ''
         if (oi, ai) in obs_status_override:
             status = obs_status_override[(oi, ai)]
-            logger.debug(f'  overriding status for (AGASC ID {ai}, observation ID {oi}): '
+            logger.debug(f'  overriding status for (AGASC ID {ai}, starcat time {oi}): '
                          f'{status["status"]}, {status["comments"]}')
             comment = status['comments']
         try:
@@ -876,7 +876,7 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None):
                     dict(MagStatsException(msg='Suspect observation',
                                            agasc_id=obs['agasc_id'],
                                            obsid=obs['obsid'],
-                                           observation_id=obs["mp_starcat_time"],)))
+                                           mp_starcat_time=obs["mp_starcat_time"],)))
         except MagStatsException as e:
             # this except branch deals with exceptions thrown by get_telemetry
             all_telem.append(None)

agasc/supplement/magnitudes/update_mag_supplement.py

@@ -290,25 +290,25 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None):
     obs = []
     if obs_stats and len(obs_stats):
         obs_stats = obs_stats[~obs_stats['obs_ok']]
-        observation_ids = np.unique(obs_stats['observation_id'])
-        for observation_id in observation_ids:
-            rows = obs_stats[obs_stats['observation_id'] == observation_id]
+        mp_starcat_times = np.unique(obs_stats['mp_starcat_time'])
+        for mp_starcat_time in mp_starcat_times:
+            rows = obs_stats[obs_stats['mp_starcat_time'] == mp_starcat_time]
             rows.sort(keys='agasc_id')
             obs.append({
-                'observation_id': observation_id,
+                'mp_starcat_time': mp_starcat_time,
                 'agasc_id': list(rows['agasc_id']),
                 'status': 1,
                 'comments': obs_stats['comment']
             })
     for fail in fails:
-        if fail['agasc_id'] is None or fail['observation_id'] is None:
+        if fail['agasc_id'] is None or fail['mp_starcat_time'] is None:
             continue
-        observation_ids = fail['observation_id'] if type(fail['observation_id']) is list \
-            else [fail['observation_id']]
+        mp_starcat_times = fail['mp_starcat_time'] if type(fail['mp_starcat_time']) is list \
+            else [fail['mp_starcat_time']]
         agasc_id = fail['agasc_id']
-        for observation_id in observation_ids:
+        for mp_starcat_time in mp_starcat_times:
             obs.append({
-                'observation_id': observation_id,
+                'mp_starcat_time': mp_starcat_time,
                 'agasc_id': [agasc_id],
                 'status': 1,
                 'comments': fail['msg']
@@ -318,7 +318,7 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None):
 
     yaml_template = """obs:
   {%- for obs in observations %}
-  - observation_id: {{ obs.observation_id }}
+  - mp_starcat_time: {{ obs.mp_starcat_time }}
     status: {{ obs.status }}
     agasc_id: [{% for agasc_id in obs.agasc_id -%}
                   {{ agasc_id }}{%- if not loop.last %}, {% endif -%}
@@ -416,7 +416,7 @@ def do(start,
                 obs_status_override = table.Table(h5.root.obs[:])
                 obs_status_override.convert_bytestring_to_unicode()
                 obs_status_override = {
-                    (r['observation_id'], r['agasc_id']):
+                    (r['mp_starcat_time'], r['agasc_id']):
                         {'status': r['status'], 'comments': r['comments']}
                     for r in obs_status_override
                 }

agasc/supplement/utils.py

@@ -35,7 +35,7 @@
 ])
 
 OBS_DTYPE = np.dtype([
-    ('observation_id', '<U21'),
+    ('mp_starcat_time', '<U21'),
     ('agasc_id', np.int32),
     ('obsid', np.int32),
     ('status', np.int32),
@@ -51,7 +51,7 @@ def get_supplement_table(name, agasc_dir=None, as_dict=False):
 
     - ``bad``: Bad stars (agasc_id, source)
     - ``mags``: Estimated mags (agasc_id, mag_aca mag_aca_err)
-    - ``obs``: Star-observation status for mag estimation (observation_id, agasc_id, obsid, status,
+    - ``obs``: Star-observation status for mag estimation (mp_starcat_time, agasc_id, obsid, status,
       comments)
 
     This function is cached with a timeout of an hour, so you can call it
@@ -62,7 +62,7 @@ def get_supplement_table(name, agasc_dir=None, as_dict=False):
 
     If ``as_dict=True`` then the table is returned as a dict of {key: value}
     pairs. For ``mags`` and ``bad``, the key is ``agasc_id``. For ``obs`` the
-    key is the ``(agasc_id, observation_id)`` tuple. In all cases the value is a dict
+    key is the ``(agasc_id, mp_starcat_time)`` tuple. In all cases the value is a dict
     of the remaining columns.
 
     :param name: Table name within the AGASC supplement HDF5 file
@@ -91,7 +91,7 @@ def get_supplement_table(name, agasc_dir=None, as_dict=False):
         keys_names = {
             'mags': ['agasc_id'],
             'bad': ['agasc_id'],
-            'obs': ['agasc_id', 'observation_id']}
+            'obs': ['agasc_id', 'mp_starcat_time']}
         key_names = keys_names[name]
         for row in dat:
             # Make the key, coercing the values from numpy to native Python
@@ -311,7 +311,7 @@ def update_obs_table(filename, obs, dry_run=False, create=False):
         list entries are dictionaries like::
 
             {'agasc_id': 1,
-             'observation_id': '2009:310:17:26:44.706',
+             'mp_starcat_time': '2009:310:17:26:44.706',
              'obsid': 0,
              'status': 0,
              'comments': 'some comment'}
@@ -325,7 +325,7 @@ def update_obs_table(filename, obs, dry_run=False, create=False):
     """
 
     update_table(filename, obs, 'obs', OBS_DTYPE,
-                 keys=['agasc_id', 'observation_id'],
+                 keys=['agasc_id', 'mp_starcat_time'],
                  dry_run=dry_run,
                  create=create)