Merge pull request #212 from sot/report-methods

Improve report generation

proseco/acq.py

@@ -190,6 +190,19 @@ def thumbs_up(self):
             out = int(self.get_log_p_2_or_fewer() <= np.log10(ACQ.acq_prob))
         return out
 
+    def make_report(self, rootdir='.'):
+        """
+        Make summary HTML report for acq selection process and outputs.
+
+        Output is in ``<rootdir>/obs<obsid>/acq/index.html`` plus related images
+        in that directory.
+
+        :param rootdir: root directory for outputs
+
+        """
+        from .report_acq import make_report
+        make_report(self, rootdir=rootdir)
+
     def update_p_acq_column(self):
         """
         Update (in-place) the marginalized acquisition probability column

proseco/catalog.py

@@ -153,6 +153,19 @@ def thumbs_up(self):
                    self.fids.thumbs_up &
                    self.guides.thumbs_up)
 
+    def make_report(self, rootdir='.'):
+        """
+        Make summary HTML report for acq and guide selection process and outputs.
+
+        Outputs are in ``<rootdir>/obs<obsid>/{acq,guide}/index.html`` plus related
+        images in that directory.
+
+        :param rootdir: root directory for outputs
+
+        """
+        self.acqs.make_report(rootdir=rootdir)
+        self.guides.make_report(rootdir=rootdir)
+
     def optimize_acqs_fids(self):
         """
         Concurrently optimize acqs and fids in the case where there is not

proseco/guide.py

@@ -106,6 +106,19 @@ def thumbs_up(self):
             out = int(count >= GUIDE.min_guide_count)
         return out
 
+    def make_report(self, rootdir='.'):
+        """
+        Make summary HTML report for guide selection process and outputs.
+
+        Output is in ``<rootdir>/obs<obsid>/guide/index.html`` plus related images
+        in that directory.
+
+        :param rootdir: root directory for outputs
+
+        """
+        from .report_guide import make_report
+        make_report(self, rootdir=rootdir)
+
     def run_search_stages(self):
         """
         Run through search stages to select stars with priority given to "better"

proseco/index_template_acq.html

@@ -40,12 +40,16 @@
 <h1> Obsid {{obsid}} at {{date}}</h1>
 
 <table class="table-striped">
-  <tr>  <td> <b>RA</b> </td> <td> {{ att[0] }} </td> </tr>
-  <tr>  <td> <b>Dec</b> </td> <td> {{ att[1] }} </td> </tr>
-  <tr>  <td> <b>Roll</b> </td> <td> {{ att[2] }} </td> </tr>
-  <tr>  <td> <b>T_ccd</b> </td> <td> {{ t_ccd }} </td> </tr>
-  <tr>  <td> <b>Man angle</b> </td> <td> {{ man_angle }} </td> </tr>
-  <tr>  <td> <b>Dither</b> </td> <td> {{ dither }} </td> </tr>
+  <tr>  <td> <b>ra</b> </td> <td> {{ att[0] }} </td> </tr>
+  <tr>  <td> <b>dec</b> </td> <td> {{ att[1] }} </td> </tr>
+  <tr>  <td> <b>roll</b> </td> <td> {{ att[2] }} </td> </tr>
+  <tr>  <td> <b>t_ccd</b> </td> <td> {{ t_ccd_acq }} C</td> </tr>
+  <tr>  <td> <b>man angle</b> </td> <td> {{ man_angle }} deg</td> </tr>
+  <tr>  <td> <b>dither (y, z)</b> </td> <td> {{ dither_acq.y }}, {{ dither_acq.z }} arcsec</td> </tr>
+  <tr>  <td> <b>n_acq</b> </td> <td> {{ n_acq }} </td> </tr>
+  {% if include_ids %} <tr>  <td> <b>include_ids</b> </td> <td> {{ include_ids }} </td> </tr> {% endif %}
+  {% if include_halfws %} <tr>  <td> <b>include_halfws</b> </td> <td> {{ include_halfws }} arcsec </td> </tr> {% endif %}
+  {% if exclude_ids %} <tr>  <td> <b>exclude_ids</b> </td> <td> {{ exclude_ids }} </td> </tr> {% endif %}
 </table>
 
 <table>

proseco/index_template_guide.html

@@ -40,15 +40,14 @@
 <h1> Obsid {{obsid}} at {{date}}</h1>
 
 <table class="table-striped">
-  <tr>  <td> <b>RA</b> </td> <td> {{ att[0] }} </td> </tr>
-  <tr>  <td> <b>Dec</b> </td> <td> {{ att[1] }} </td> </tr>
-  <tr>  <td> <b>Roll</b> </td> <td> {{ att[2] }} </td> </tr>
-  <tr>  <td> <b>t_ccd</b> </td> <td> {{ t_ccd_guide }} </td> </tr>
-  <tr>  <td> <b>dither_y</b> </td> <td> {{ dither_guide.y }} </td> </tr>
-  <tr>  <td> <b>dither_z</b> </td> <td> {{ dither_guide.z }} </td> </tr>
+  <tr>  <td> <b>ra</b> </td> <td> {{ att[0] }} </td> </tr>
+  <tr>  <td> <b>dec</b> </td> <td> {{ att[1] }} </td> </tr>
+  <tr>  <td> <b>roll</b> </td> <td> {{ att[2] }} </td> </tr>
+  <tr>  <td> <b>t_ccd</b> </td> <td> {{ t_ccd_guide }} C</td> </tr>
+  <tr>  <td> <b>dither (y, z)</b> </td> <td> {{ dither_guide.y }}, {{ dither_guide.z }} arcsec</td> </tr>
   <tr>  <td> <b>n_guide</b> </td> <td> {{ n_guide }} </td> </tr>
-  <tr>  <td> <b>include_ids</b> </td> <td> {{ str_include_ids }} </td> </tr>
-  <tr>  <td> <b>exclude_ids</b> </td> <td> {{ str_exclude_ids }} </td> </tr>
+  {% if include_ids %} <tr>  <td> <b>include_ids</b> </td> <td> {{ include_ids }} </td> </tr> {% endif %}
+  {% if exclude_ids %} <tr>  <td> <b>exclude_ids</b> </td> <td> {{ exclude_ids }} </td> </tr> {% endif %}
 </table>
 
 

proseco/report_acq.py

@@ -2,7 +2,6 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 
 from copy import copy, deepcopy
-import re
 from pathlib import Path
 
 import matplotlib
@@ -149,7 +148,6 @@ def make_cand_acqs_report(acqs, cand_acqs, events, context, obsdir):
     # Now plot figure
     filename = obsdir / 'candidate_stars.png'
     if not filename.exists():
-        print(f'Making candidate stars plot {filename}')
 
         # Pull a fast-one and mark the final selected ACQ stars as BOT so they
         # get a circle in the plot.  This might be confusing and need fixing
@@ -184,7 +182,6 @@ def make_acq_star_details_report(acqs, cand_acqs, events, context, obsdir):
     context['cand_acqs'] = []
 
     for ii, acq in enumerate(cand_acqs):
-        print('Doing detail for star {}'.format(acq['id']))
         # Local context dict for each cand_acq star
         cca = {'id': acq['id'],
                'selected': 'SELECTED' if acq['id'] in acqs['id'] else 'not selected'}
@@ -279,7 +276,7 @@ def make_obsid_summary(acqs, events, context, obsdir):
     context['acqs_table'] = table_to_html(acqs_table)
 
     basename = 'acq_stars.png'
-    filename = obsdir / basename
+    filename = obsdir / 'acq' / basename
     context['acq_stars_plot'] = basename
     if not filename.exists():
         fig = plt.figure(figsize=(4, 4))
@@ -290,18 +287,41 @@ def make_obsid_summary(acqs, events, context, obsdir):
                             bad_stars=acqs.bad_stars, ax=ax)
         # When Ska3 has matplotlib 2.2+ then just use `filename`
         plt.savefig(str(filename))
-        plt.close()
+        plt.close(fig)
 
 
 def make_report(obsid, rootdir='.'):
+    """
+    Make summary HTML report for acq star selection.
+
+    The first arg ``obsid`` can be an obsid (int) or a ``AcqTable`` object.
+    For ``int`` the acq table is read from ``<rootdir>/obs<obsid>/acq.pkl``.
+
+    Output is in ``<rootdir>/obs<obsid>/acq/index.html`` plus related images
+    in that directory.
+
+    :param obsid: int obsid or AcqTable instance
+    :param rootdir: root directory for outputs
+
+    :returns: AcqTable object (mostly for testing)
+    """
     rootdir = Path(rootdir)
-    print(f'Processing obsid {obsid}')
+    if isinstance(obsid, AcqTable):
+        acqs = obsid
+        obsid = acqs.obsid
+    else:
+        acqs = AcqTable.from_pickle(obsid, rootdir)
+    cand_acqs = acqs.cand_acqs
 
+    # Define and make directories as needed
     obsdir = rootdir / f'obs{obsid:05}'
-    acqs = AcqTable.from_pickle(obsid, rootdir)
-    cand_acqs = acqs.cand_acqs
+    outdir = obsdir / 'acq'
+    outdir.mkdir(exist_ok=True, parents=True)
 
     context = copy(acqs.meta)
+    context['include_ids'] = ", ".join([str(val) for val in acqs.include_ids])
+    context['include_halfws'] = ", ".join([str(val) for val in acqs.include_halfws])
+    context['exclude_ids'] = ", ".join([str(val) for val in acqs.exclude_ids])
 
     # Get information that is not stored in the acqs pickle for space reasons
     acqs.stars = StarsTable.from_agasc(acqs.att, date=acqs.date)
@@ -312,15 +332,15 @@ def make_report(obsid, rootdir='.'):
 
     make_obsid_summary(acqs, events, context, obsdir)
     make_p_man_errs_report(context)
-    make_cand_acqs_report(acqs, cand_acqs, events, context, obsdir)
+    make_cand_acqs_report(acqs, cand_acqs, events, context, outdir)
     make_initial_cat_report(events, context)
-    make_acq_star_details_report(acqs, cand_acqs, events, context, obsdir)
+    make_acq_star_details_report(acqs, cand_acqs, events, context, outdir)
     make_optimize_catalog_report(events, context)
 
     template_file = FILEDIR / ACQ.index_template_file
     template = Template(open(template_file, 'r').read())
     out_html = template.render(context)
-    out_filename = obsdir / 'index.html'
+    out_filename = outdir / 'index.html'
     with open(out_filename, 'w') as fh:
         fh.write(out_html)
 
@@ -472,5 +492,6 @@ def plot_imposters(acq, dark, dither, vmin=100, vmax=2000,
     if filename is not None:
         # When Ska3 has matplotlib 2.2+ then just use `filename`
         plt.savefig(str(filename), pad_inches=0.0)
+    plt.close(fig)
 
     return img, ax

proseco/report_guide.py

@@ -30,11 +30,32 @@
 
 
 def make_report(obsid, rootdir='.'):
+    """
+    Make summary HTML report for guide star selection.
+
+    The first arg ``obsid`` can be an obsid (int) or a ``GuideTable`` object.
+    For ``int`` the guide table is read from ``<rootdir>/obs<obsid>/guide.pkl``.
+
+    Output is in ``<rootdir>/obs<obsid>/guide/index.html`` plus related images
+    in that directory.
+
+    :param obsid: int obsid or GuideTable instance
+    :param rootdir: root directory for outputs
+
+    :returns: GuideTable object (mostly for testing)
+    """
     rootdir = Path(rootdir)
-    print(f'Processing obsid {obsid}')
 
+    if isinstance(obsid, GuideTable):
+        guides = obsid
+        obsid = guides.obsid
+    else:
+        guides = GuideTable.from_pickle(obsid, rootdir)
+
+    # Define and make directories as needed
     obsdir = rootdir / f'obs{obsid:05}'
-    guides = GuideTable.from_pickle(obsid, rootdir)
+    outdir = obsdir / 'guide'
+    outdir.mkdir(exist_ok=True, parents=True)
 
     cand_guides = guides.cand_guides.copy()
     cand_guides['sort_stage'] = cand_guides['stage']
@@ -44,8 +65,8 @@ def make_report(obsid, rootdir='.'):
         cand_guides = cand_guides[0:MAX_CAND]
 
     context = copy(guides.meta)
-    context['str_include_ids'] = ",".join([str(sid) for sid in guides.include_ids_guide])
-    context['str_exclude_ids'] = ",".join([str(sid) for sid in guides.exclude_ids_guide])
+    context['include_ids'] = ", ".join([str(val) for val in guides.include_ids])
+    context['exclude_ids'] = ", ".join([str(val) for val in guides.exclude_ids])
 
     # Get information that is not stored in the acqs pickle for space reasons
     guides.stars = StarsTable.from_agasc(guides.att, date=guides.date)
@@ -70,7 +91,7 @@ def make_report(obsid, rootdir='.'):
         cand_guides['stage'][-1] = -1
         cand_guides['forced'][-1] = True
 
-    make_cand_report(guides, cand_guides, context, obsdir)
+    make_cand_report(guides, cand_guides, context, outdir)
 
     # Guide star table
     cols = COLS.copy()
@@ -91,7 +112,7 @@ def make_report(obsid, rootdir='.'):
     template_file = FILEDIR / GUIDE.index_template_file
     template = Template(open(template_file, 'r').read())
     out_html = template.render(context)
-    out_filename = obsdir / 'guide_index.html'
+    out_filename = outdir / 'index.html'
     with open(out_filename, 'w') as fh:
         fh.write(out_html)
 
@@ -103,7 +124,6 @@ def make_cand_report(guides, cand_guides, context, obsdir):
     n_stages = np.max(cand_guides['stage'])
     context['cand_guides'] = []
     for ii, guide in enumerate(cand_guides):
-        print('Doing detail for star {}'.format(guide['id']))
         select = 'SELECTED' if guide['id'] in guides['id'] else 'not selected'
 
         # Add debugging information if the star was in include/exclude lists

proseco/tests/test_acq.py

@@ -521,13 +521,16 @@ def test_make_report(tmpdir):
 
     tmpdir = Path(tmpdir)
     obsdir = tmpdir / f'obs{obsid:05}'
+    outdir = obsdir / 'acq'
 
     acqs.to_pickle(rootdir=tmpdir)
 
     acqs2 = make_report(obsid, rootdir=tmpdir)
 
-    assert (obsdir / 'index.html').exists()
-    assert len(list(obsdir.glob('*.png'))) > 0
+    assert (outdir / 'index.html').exists()
+    assert (outdir / 'acq_stars.png').exists()
+    assert (outdir / 'candidate_stars.png').exists()
+    assert len(list(outdir.glob('*.png'))) > 0
 
     assert repr(acqs) == repr(acqs2)
     assert repr(acqs.cand_acqs) == repr(acqs2.cand_acqs)

proseco/tests/test_catalog.py

@@ -7,7 +7,7 @@
 import numpy as np
 import mica.starcheck
 
-from .test_common import STD_INFO, mod_std_info, DARK40
+from .test_common import STD_INFO, mod_std_info, DARK40, OBS_INFO
 from ..core import StarsTable, ACACatalogTable
 from ..catalog import get_aca_catalog
 from ..fid import get_fid_catalog
@@ -431,3 +431,25 @@ def test_aca_acqs_include_exclude():
     guides = aca.guides
     assert guides.include_ids == include_ids
     assert guides.exclude_ids == exclude_ids
+
+
+def test_report_from_objects(tmpdir):
+    """
+    Test making guide and acq reports without the intermediate pickle.
+
+    This is just a sanity check that appropriate files are created.  More
+    detailed testing (including pickle round trip) is tested in test_guide
+    and test_acq.
+    """
+    rootdir = Path(tmpdir)
+
+    obsid = 20603
+    aca = get_aca_catalog(**OBS_INFO[obsid])
+    aca.guides.make_report(rootdir=rootdir)
+    aca.acqs.make_report(rootdir=rootdir)
+
+    obsdir = rootdir / f'obs{obsid:05}'
+    for subdir in 'acq', 'guide':
+        outdir = obsdir / subdir
+        assert(outdir / 'index.html').exists()
+        assert len(list(outdir.glob('*.png'))) > 0

proseco/tests/test_guide.py

@@ -419,13 +419,14 @@ def test_make_report_guide(tmpdir):
 
     tmpdir = Path(tmpdir)
     obsdir = tmpdir / f'obs{obsid:05}'
+    outdir = obsdir / 'guide'
 
     guides.to_pickle(rootdir=tmpdir)
 
     guides2 = make_report(obsid, rootdir=tmpdir)
 
-    assert (obsdir / 'guide_index.html').exists()
-    assert len(list(obsdir.glob('*.png'))) > 0
+    assert (outdir / 'index.html').exists()
+    assert len(list(outdir.glob('*.png'))) > 0
 
     assert repr(guides) == repr(guides2)
     assert repr(guides.cand_guides) == repr(guides2.cand_guides)