Bugfixes

.gitignore

@@ -2,10 +2,16 @@ mp_get_agasc_bug.png
 miniagasc.h5
 ra_dec.npy
 agasc1p6.h5
-doc/_*
+docs/_*
 build
 data/
 dist/
 MANIFEST
 parsetab.py
-__pycache__
+__pycache__
+.vscode
+.DS_store
+_*
+*~
+supplement_reports
+test

agasc/scripts/update_mag_supplement.py

@@ -77,6 +77,13 @@ def main():
     else:
         args.reports_dir = Path(os.path.expandvars(args.reports_dir))
 
+    if args.whole_history:
+        if args.start or args.stop:
+            logger.error('--whole-history argument is incompatible with --start/--stop arguments')
+            the_parser.exit(1)
+        args.start = None
+        args.stop = None
+
     pyyaks.logger.get_logger(
         name='agasc.supplement',
         level=args.log_level.upper(),

agasc/scripts/update_obs_status.py

@@ -115,16 +115,21 @@ def update_obs_table(filename, obs_status_override, dry_run=False):
     if not update:
         return
 
+    obs_dtype = np.dtype([
+        ('obsid', np.int32),
+        ('agasc_id', np.int32),
+        ('ok', np.int32),
+        ('comments', '<U80')])
     if obs_status:
         t = list(zip(
             *[[oi, ai, np.uint(obs_status[(oi, ai)]['ok']), obs_status[(oi, ai)]['comments']]
               for oi, ai in obs_status]
         ))
-        obs_status = table.Table(t, names=['obsid', 'agasc_id', 'ok', 'comments'])
+        obs_status = table.Table(t, names=obs_dtype.names,
+                                 dtype=[obs_dtype[name] for name in obs_dtype.names])
     else:
         logger.info('creating empty obs table')
-        dtype = [('obsid', int), ('agasc_id', int), ('ok', np.uint), ('comments', '<U80')]
-        obs_status = table.Table(dtype=dtype)
+        obs_status = table.Table(dtype=obs_dtype)
 
     if not dry_run:
         obs_status.write(str(filename), format='hdf5', path='obs', append=True, overwrite=True)

agasc/supplement/magnitudes/update_mag_supplement.py

@@ -209,6 +209,11 @@ def update_supplement(agasc_stats, filename, include_all=True):
         if False, only OK entries marked 'selected_*'
     :return:
     """
+    mags_dtype = np.dtype([('agasc_id', np.int32),
+                           ('mag_aca', np.float32),
+                           ('mag_aca_err', np.float32),
+                           ('last_obs_time', np.float64)])
+
     if include_all:
         outliers_new = agasc_stats[
             (agasc_stats['n_obsids_ok'] > 0)
@@ -223,8 +228,10 @@ def update_supplement(agasc_stats, filename, include_all=True):
         ]
     outliers_new['mag_aca'] = outliers_new['mag_obs']
     outliers_new['mag_aca_err'] = outliers_new['mag_obs_err']
-    names = ['agasc_id', 'mag_aca', 'mag_aca_err', 'last_obs_time']
-    outliers_new = outliers_new[names].as_array()
+
+    outliers_new = outliers_new[mags_dtype.names].as_array()
+    if outliers_new.dtype != mags_dtype:
+        outliers_new = outliers_new.astype(mags_dtype)
 
     outliers = None
     new_stars = None
@@ -246,9 +253,7 @@ def update_supplement(agasc_stats, filename, include_all=True):
                 i_new = i_new[current['last_obs_time'] != new['last_obs_time']]
                 # overwrite current values with new values (and calculate diff to return)
                 updated_stars = np.zeros(len(outliers_new[i_new]),
-                                         dtype=[('agasc_id', np.int64),
-                                                ('mag_aca', np.float64),
-                                                ('mag_aca_err', np.float64)])
+                                         dtype=mags_dtype)
                 updated_stars['mag_aca'] = (outliers_new[i_new]['mag_aca']
                                             - outliers_current[i_cur]['mag_aca'])
                 updated_stars['mag_aca_err'] = (outliers_new[i_new]['mag_aca_err']
@@ -267,8 +272,7 @@ def update_supplement(agasc_stats, filename, include_all=True):
     if outliers is None:
         outliers = outliers_new
         new_stars = outliers_new['agasc_id']
-        updated_stars = np.array([], dtype=[('agasc_id', np.int64), ('mag_aca', np.float64),
-                                            ('mag_aca_err', np.float64)])
+        updated_stars = np.array([], dtype=mags_dtype)
 
     mode = 'r+' if filename.exists() else 'w'
     with tables.File(filename, mode) as h5:
@@ -308,11 +312,6 @@ def do(output_dir,
     # as ascii. It displays a warning which I want to avoid:
     warnings.filterwarnings("ignore", category=tables.exceptions.FlavorWarning)
 
-    if start:
-        start = CxoTime(start)
-    if stop:
-        stop = CxoTime(stop)
-
     filename = output_dir / 'agasc_supplement.h5'
 
     if multi_process:
@@ -324,28 +323,19 @@ def do(output_dir,
         agasc_ids = np.intersect1d(agasc_ids, star_obs_catalogs.STARS_OBS['agasc_id'])
 
     # set start/stop times and agasc_ids
-    if whole_history:
-        if start:
-            logger.warning('Ignoring --start argument from commant line (--whole-history)')
-        if stop:
-            logger.warning('Ignoring --stop argument from commant line (--whole-history)')
+    if whole_history or agasc_ids is not None:
+        if start or stop:
+            raise ValueError('incompatible arguments: whole_history and start/stop')
         start = CxoTime(star_obs_catalogs.STARS_OBS['mp_starcat_time']).min().date
         stop = CxoTime(star_obs_catalogs.STARS_OBS['mp_starcat_time']).max().date
         if agasc_ids is None:
             agasc_ids = sorted(star_obs_catalogs.STARS_OBS['agasc_id'])
-    elif agasc_ids is None:
-        if not start:
-            start = CxoTime(star_obs_catalogs.STARS_OBS['mp_starcat_time']).min().date
-        if not stop:
-            stop = CxoTime(star_obs_catalogs.STARS_OBS['mp_starcat_time']).max().date
+    else:
+        stop = CxoTime(stop).date if stop else CxoTime.now().date
+        start = CxoTime(start).date if start else (CxoTime(stop) - 14 * u.day).date
         obs_in_time = ((star_obs_catalogs.STARS_OBS['mp_starcat_time'] >= start)
                        & (star_obs_catalogs.STARS_OBS['mp_starcat_time'] <= stop))
         agasc_ids = sorted(star_obs_catalogs.STARS_OBS[obs_in_time]['agasc_id'])
-    else:
-        if not stop:
-            stop = CxoTime.now().date
-        if not start:
-            start = CxoTime(stop) - 14 * u.day
 
     agasc_ids = np.unique(agasc_ids)
     stars_obs = star_obs_catalogs.STARS_OBS[
@@ -402,6 +392,7 @@ def do(output_dir,
 
     # do the processing
     logger.info(f'Will process {len(agasc_ids)} stars on {len(stars_obs)} observations')
+    logger.info(f'from {start} to {stop}')
     if dry_run:
         return
 
@@ -478,9 +469,9 @@ def do(output_dir,
                 report = msr.MagEstimateReport(agasc_stats, obs_stats, directory=directory)
                 report.multi_star_html(**multi_star_html_args)
                 latest = reports_dir / 'latest'
-                if latest.exists:
+                if os.path.lexists(latest):
                     latest.unlink()
-                latest.symlink_to(directory)
+                latest.symlink_to(directory.absolute())
             except Exception as e:
                 logger.error(f'Exception when creating report: {e}')
                 multi_star_html_args['directory'] = directory

agasc/tests/test_obs_status.py

@@ -196,22 +196,22 @@ def test_parse_obs_status_args_bad(monkeypatch):
     #######################
 
     status = update_obs_status._parse_obs_status_args(
-        bad_star=1, bad_star_source=2
+        bad_star_id=1, bad_star_source=2
     )
     assert status['obs'] == {}
     assert status['bad'] == {1: 2}
 
     # bad star can be a list
     status = update_obs_status._parse_obs_status_args(
-        bad_star=[1, 2], bad_star_source=3
+        bad_star_id=[1, 2], bad_star_source=3
     )
     assert status['obs'] == {}
     assert status['bad'] == {1: 3, 2: 3}
 
     # if you specify bad_star, you must specify bad_star_source
     with pytest.raises(RuntimeError, match=r"specify bad_star_source"):
         update_obs_status._parse_obs_status_args(
-            bad_star=1
+            bad_star_id=1
         )
 
 
@@ -297,14 +297,14 @@ def test_parse_obs_status_args(monkeypatch):
     with pytest.raises(RuntimeError, match=r"name collision"):
         _ = update_obs_status._parse_obs_status_args(
             filename=filename,
-            bad_star=23434,
+            bad_star_id=23434,
             bad_star_source=12
         )
 
     # can specify bad star in the file and in args if the source is the same.
     status = update_obs_status._parse_obs_status_args(
         filename=filename,
-        bad_star=23434,
+        bad_star_id=23434,
         bad_star_source=10
     )
     ref = copy.deepcopy(TEST_DATA[filename])
@@ -316,7 +316,7 @@ def test_parse_obs_status_args(monkeypatch):
         obs=56309,
         agasc_id=[762184312, 762184768, 762185584, 762186016],
         status=False,
-        bad_star=[1, 2],
+        bad_star_id=[1, 2],
         bad_star_source=1000
     )
     ref = update_obs_status._parse_obs_status_args(
@@ -326,7 +326,7 @@ def test_parse_obs_status_args(monkeypatch):
         obs=56309,
         agasc_id=[762184312, 762184768, 762185584, 762186016],
         status=False,
-        bad_star=[1, 2],
+        bad_star_id=[1, 2],
         bad_star_source=1000
     )
     ref['obs'].update(ref_2['obs'])