From 6f0d552ec1e1334a85213e00ee50d79ce013323a Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Fri, 27 Sep 2019 14:34:03 -0400 Subject: [PATCH 1/4] Py3, flake8, make a package, and remove characteristics --- LICENSE | 29 ++ Makefile | 36 --- characteristics.py | 51 ---- get_perigee_telem.pl | 202 ------------- install_plots.pl | 25 -- install_plots.yaml | 142 ---------- perigee_health_plots/__init__.py | 11 + .../month_index_template.html | 0 .../pass_index_template.html | 0 .../pass_plots.py | 267 +++++++++--------- .../top_index_template.html | 0 setup.py | 39 +++ 12 files changed, 218 insertions(+), 584 deletions(-) create mode 100644 LICENSE delete mode 100644 Makefile delete mode 100644 characteristics.py delete mode 100755 get_perigee_telem.pl delete mode 100755 install_plots.pl delete mode 100644 install_plots.yaml create mode 100644 perigee_health_plots/__init__.py rename {templates => perigee_health_plots}/month_index_template.html (100%) rename {templates => perigee_health_plots}/pass_index_template.html (100%) rename pass_plots.py => perigee_health_plots/pass_plots.py (82%) rename {templates => perigee_health_plots}/top_index_template.html (100%) create mode 100644 setup.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0647bf0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2019, sot +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile deleted file mode 100644 index 2088044..0000000 --- a/Makefile +++ /dev/null @@ -1,36 +0,0 @@ - -TASK = perigee_health_plots -VERSION = 2.1 - -include /proj/sot/ska/include/Makefile.FLIGHT - -SHARE = get_perigee_telem.pl pass_plots.py characteristics.py -DATA = task_schedule.cfg -TEMPLATES = templates/top_index_template.html templates/month_index_template.html templates/pass_index_template.html - - -install: -ifdef TEMPLATES - mkdir -p $(INSTALL_SHARE)/templates/ - rsync --times --cvs-exclude $(TEMPLATES) $(INSTALL_SHARE)/templates/ -endif -ifdef DATA - mkdir -p $(INSTALL_DATA) - rsync --times --cvs-exclude $(DATA) $(INSTALL_DATA)/ -endif -ifdef SHARE - mkdir -p $(INSTALL_SHARE) - rsync --times --cvs-exclude $(SHARE) $(INSTALL_SHARE)/ -endif -ifdef LIB - mkdir -p $(INSTALL_PERLLIB)/Ska/ - rsync --times --cvs-exclude $(LIB) $(INSTALL_PERLLIB)/Ska/ -endif - - -dist: - mkdir $(TASK)-$(VERSION) - rsync -aruvz --cvs-exclude --exclude $(TASK)-$(VERSION) * $(TASK)-$(VERSION) - tar cvf $(TASK)-$(VERSION).tar $(TASK)-$(VERSION) - gzip --best $(TASK)-$(VERSION).tar - rm -rf $(TASK)-$(VERSION)/ diff --git a/characteristics.py b/characteristics.py deleted file mode 100644 index 843373a..0000000 --- a/characteristics.py +++ /dev/null @@ -1,51 +0,0 @@ - - -# red, green, blue, magenta, cyan, orange, purple... maybe -plot_colors = ['#ff0000', '#00ff00', '#0000ff', - '#ff00ff', '#00ffff', '#ff6600', - '#6600ff' ] - -# break up telem in chunks of time_interval seconds -time_interval = 20.0 - -# and require min_samples per time_interval -min_samples = 5 - -# if telem values exceed these limits, cut the values -telem_chomp_limits = { 'dac' : { 'max': 550 }, - 'ccd_temp' : {'min' : -35, - 'max' : 50 }, - 'aca_temp' : {'max' : 50, - 'min' : 5 }, - } - -# If telem values exceed these limits send a warning -# This is intended to be planning limit +1C -telem_limits = {'ccd_temp': {'max': -6.1}} - -# plot stuff, just ranges now -dac_plot = { 'ylim' : (460,515) } -dacvsdtemp_plot = { 'ylim' : (460,515), - 'xlim' : (37,40) } -aca_temp_plot = {'ylim': (8, 32)} -ccd_temp_plot = {'ylim': (-15, -5)} - - - -# at some point, create routine to do this more cleanly from characteristics - -#products['dac'] = aca0[7]['HD3TLM76'][ok[7]] * 256. + aca0[7]['HD3TLM77'][ok[7]] -#products['time'] = aca0[7]['TIME'][ok[7]] -##products['h066'] = aca0[0]['HD3TLM66'] * 256 + aca0[0]['HD3TLM67'] -##products['h072'] = aca0[0]['HD3TLM72'] * 256 + aca0[0]['HD3TLM73'] -##products['h074'] = aca0[0]['HD3TLM74'] * 256 + aca0[0]['HD3TLM75'] -##products['h174'] = aca0[1]['HD3TLM74'] * 256 + aca0[1]['HD3TLM75'] -##products['h176'] = aca0[1]['HD3TLM76'] * 256 + aca0[1]['HD3TLM77'] -##products['h262'] = aca0[2]['HD3TLM62'] * 256 + aca0[2]['HD3TLM63'] -##products['h264'] = aca0[2]['HD3TLM64'] * 256 + aca0[2]['HD3TLM65'] -##products['h266'] = aca0[2]['HD3TLM66'] * 256 + aca0[2]['HD3TLM67'] -##products['h272'] = aca0[2]['HD3TLM72'] * 256 + aca0[2]['HD3TLM73'] -##products['h274'] = aca0[2]['HD3TLM74'] * 256 + aca0[2]['HD3TLM75'] -##products['h276'] = aca0[2]['HD3TLM76'] * 256 + aca0[2]['HD3TLM77'] -#products['aca_temp'] = aca0[7]['HD3TLM73'][ok[7]] * (1/256.) + aca0[7]['HD3TLM72'][ok[7]] -#products['ccd_temp'] = ( (aca0[6]['HD3TLM76'][ok[6]] * 256.) + (aca0[6]['HD3TLM77'][ok[6]] - 65536. )) / 100. diff --git a/get_perigee_telem.pl b/get_perigee_telem.pl deleted file mode 100755 index 04f520f..0000000 --- a/get_perigee_telem.pl +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use Cwd; -use Getopt::Long; -use Pod::Help qw( --help ); -use Carp; -use Chandra::Time; -use Ska::Run qw( run ); -use File::chdir; - -my %opt = (); - -eval { main () }; -if ( $@ ) { - print STDERR "# $0: $_\n" foreach split /\n/, $@; - exit 1; -}; -exit 0; - - -sub main{ - - - GetOptions (\%opt, - 'tstart=s', - 'tstop=s', - 'dir=s', - 'verbose!', - ); - - - my $tstart = Chandra::Time->new($opt{tstart})->secs() - or croak("Must define time --tstart"); - my $tstop = Chandra::Time->new($opt{tstop})->secs() - or croak("Must define time --tstop"); - my $dir = $opt{dir} - or croak("Must define destination --dir"); - - my %telems = ( 'ccdm0{ccdm10eng}' => 'ccdm*10_eng0.fits*', - 'aca0{acaimg}' => 'acaf*_img0.fits*' ); - - for my $telem_type (keys %telems){ - my $glob = $telems{$telem_type}; - my $files = list_archive_files( $tstart, $tstop, $telem_type ); - my $have_files = check_for_files( $dir, $files ); - unless ($have_files){ - fetch_archive_files( $dir, $tstart, $tstop, $telem_type, $glob ); - $have_files = check_for_files( $dir, $files ); - } - unless ($have_files){ - croak("unable to retrieve $telem_type files") - } - - } - - -} - - -sub check_for_files{ - - my ($dir, $files ) = @_; - - my $have_files = 1; - for my $file (@{$files}){ - my $filename = $file->{filename}; - print "checking ${dir}/${filename}" if $opt{verbose}; - if (-e "${dir}/${filename}"){ - my $filesize = -s "${dir}/${filename}"; - if ($filesize == $file->{size}){ - print " ... found \n" if $opt{verbose}; - next; - } - } - else{ - print " ... missing \n" if $opt{verbose}; - $have_files = 0; - } - } - return $have_files; -} - - -sub list_archive_files{ - my ($tstart, $tstop, $product) = @_; - - my $browse = < 60); - - my @files; - my $matching = 0; - for my $line (@lines){ - chomp $line; - if ($line =~ /.*Filename.*/){ - $matching = 1; - next; - } - if (($matching) and ($line =~ /(\w*\.fits(\.gz)?)\s+(\d+).*/)){ - push @files, {filename => $1, - size => $3}; - - } - } - return \@files; -} - -sub fetch_archive_files{ - my ($dir, $tstart, $tstop, $product, $glob) = @_; - - local $CWD = $dir; - # let's just chuck 'em first - unlink(glob($glob)); - - my $get = < 300); - - - my @gzfiles = glob("${dir}/${glob}*.gz"); - map { run("gunzip $_")} @gzfiles; - - return 0; - -} - - -=pod - -=head1 NAME - -get_perigee_telem.pl - Retrieve the aca0 and ccdm0 telemetry used by the camera -health plotting system. - -=head1 SYNOPSIS - -get_perigee_telem.pl --tstart --tstop --dir - -=head1 OPTIONS - -=over 12 - -=item B<--tstart> - -Beginning of time range for arc5gl retrieval of telemetry - -=item B<--tstop> - -End of time range for arc5gl retrieval of telemetry - -=item B<--dir> - -Directory in which to store retrieved telemetry - -=item B<--verbose> - -Be verbose about file checks and fetching - -=back - -=head1 DESCRIPTION - -The camera health plots require 8x8 image telemetry (which is mostly found during perigee -passes). This script accepts options for a perigee pass time range (or just an ER time range), -checks to see if the desired telemetry files (ccdm0 and aca0) already exist in the directory -specified with the --dir option, and retrieves any needed files. - -=head1 EXAMPLES - -get_perigee_telem.pl --tstart 2009:273:09:44:30.137 \ - --tstop 2009:273:10:34:33.387 \ - --dir "2009:273:09:44:30.137" - -=head1 AUTHOR - -Jean Connelly, Ejeanconn@localdomain - -=head1 COPYRIGHT & LICENSE - -Copyright (C) 2009 by Smithsonian Astropysical Observatory - -This library is free software; you can redistribute it and/or modify -it under the same terms as Perl itself, either Perl version 5.8.0 or, -at your option, any later version of Perl 5 you may have available. - -=cut diff --git a/install_plots.pl b/install_plots.pl deleted file mode 100755 index 493c612..0000000 --- a/install_plots.pl +++ /dev/null @@ -1,25 +0,0 @@ -#! /usr/bin/env /proj/sot/ska/bin/perlska - -use strict; -use warnings; - -use Getopt::Long; - - -my %opt = (); - -GetOptions (\%opt, - 'help!', - 'shared_config=s', - 'config=s', - 'redo!', - 'verbose!', - 'dryrun!', -# 'dir=s', -# 'web_dir=s', - ); - -use Ska::Perigee::Data; - -my $status = Ska::Perigee::Data::install_plots(\%opt); - diff --git a/install_plots.yaml b/install_plots.yaml deleted file mode 100644 index 4a0d8e6..0000000 --- a/install_plots.yaml +++ /dev/null @@ -1,142 +0,0 @@ -# Default config file for install_plots.pl - ---- task ---- - -#pg_to_html_colors: -# red: Red -# green: Green -# blue: Cornflower Blue -# magenta: Magenta3 -# cyan: Cyan2 -# orange: Sienna2 -# purple: Violet - -pg_to_html_colors: - red: #FF0000 - green: #00FF00 - blue: #0000FF - magenta: #FF00CC - cyan: #00FFFF - orange: #E66C2C - purple: #8D38C9 - -# source plot ext -source_plot_ext: ps - -#pass num (default number to go back and make gifs etcs -passnum: 15 - -loadconfig: - - /proj/sot/ska/share/perigee_health_plots/plot_summary.yaml - -stylesheet: /mta/ASPECT/aspect.css - - -# source plots -source_plots: - - aca_temp.ps - - ccd_temp.ps - - dac.ps - - dacvsdtemp.ps - - legend.ps - - -# destination plots -dest_plots: - - aca_temp.gif - - ccd_temp.gif - - dac.gif - - dacvsdtemp.gif - - legend.gif - -# plot_folder index.html -pass_dir_index: index.html - -# master index -master_index: index.html - -month_map: - 01: January - 02: February - 03: March - 04: April - 05: May - 06: June - 07: July - 08: August - 09: September - 10: October - 11: November - 12: December - -# For pass text, available variables include -# $tstart, $tstop -# 5 element array of images in @dest_plots -# $nav_links -pass_text: > - $nav_links -

ACA Perigee Pass Health Indicators

-

Perigee Pass Interval

- - - -
TSTARTTSTOP
${tstart}${tstop}
-

ACA Housing Temperature, CCD Temperature, and DAC Control - Level from Header 3 ACA telemetry gathered during a perigee pass. - The following are data from the perigee pass that occurred from - the TSTART to TSTOP indicated. Each engineering obsid is plotted in a - different color. -

Health Plots

- - - - - - - - - - - -
Obsid Colors
- - - -# available variables -# $month_name, $year -# array of 4 images at @dest_plots -# string for current url: $summ_string -month_text: > - $nav_links -

$month_name $year, ACA Health Summary Plots

-

Summary - plots of ACA housing temperature, CCD temperature, and DAC control - level. Data is only available during perigee passes, so each point on - these summary plots represents the mean of the aforementioned - indicators during a perigee pass. Plots for single passes are - available from the links below. -
- - - -
-
-

In the plot of TEC DAC vs delta Temperature, the linear fit to - the current data is a dotted line (fit not calculated if data range - is insufficient). The dashed line is the fit line - from a range of -15 to -19 data. - -

Plots for each perigee pass summarized above are available from the links below. - $pass_table - - -main_text: > -

ACA Perigee Health Plots

-

Plots for trending ACA Housing Temperature, CCD Temperature, and DAC Control Level as - determined from Header 3 Telemetry. The table - below contains links to the summary pages/plots for each month. - From these pages, links to pages with individual pass plots are available. -
- $nav_table - diff --git a/perigee_health_plots/__init__.py b/perigee_health_plots/__init__.py new file mode 100644 index 0000000..ecdb7dd --- /dev/null +++ b/perigee_health_plots/__init__.py @@ -0,0 +1,11 @@ +import ska_helpers + +__version__ = ska_helpers.get_version(__package__) + + +def test(*args, **kwargs): + """ + Run py.test unit tests. + """ + import testr + return testr.test(*args, **kwargs) diff --git a/templates/month_index_template.html b/perigee_health_plots/month_index_template.html similarity index 100% rename from templates/month_index_template.html rename to perigee_health_plots/month_index_template.html diff --git a/templates/pass_index_template.html b/perigee_health_plots/pass_index_template.html similarity index 100% rename from templates/pass_index_template.html rename to perigee_health_plots/pass_index_template.html diff --git a/pass_plots.py b/perigee_health_plots/pass_plots.py similarity index 82% rename from pass_plots.py rename to perigee_health_plots/pass_plots.py index aceaa5d..3587e41 100755 --- a/pass_plots.py +++ b/perigee_health_plots/pass_plots.py @@ -1,38 +1,65 @@ -#!/usr/bin/env python import glob +from pathlib import Path from optparse import OptionParser import os -import time -import cPickle import re import numpy as np import numpy.ma as ma import logging from logging.handlers import SMTPHandler -from itertools import izip, cycle -import jinja2 +from itertools import cycle +from jinja2 import Template import json import matplotlib + # Matplotlib setup # Use Agg backend for command-line (non-interactive) operation if __name__ == '__main__': matplotlib.use('Agg') import matplotlib.pyplot as plt plt.rcParams['lines.markeredgewidth'] = 0 +from astropy.table import Table +from kadi import events +import kadi.commands.states as kadi_states from Chandra.Time import DateTime -from Ska.DBI import DBI -import Ska.Table import Ska.Shell import Ska.report_ranges from Ska.engarchive import fetch from mica.archive import aca_hdr3 from Ska.Matplotlib import plot_cxctime -from Chandra.cmd_states import get_cmd_states -import chandra_models import xija -import characteristics + +# Colors for plots: red, green, blue, magenta, cyan, orange, purple... maybe +PLOT_COLORS = ['#ff0000', '#00ff00', '#0000ff', + '#ff00ff', '#00ffff', '#ff6600', + '#6600ff'] + +# Break up telem in chunks of time_interval seconds +TIME_INTERVAL = 20.0 + +# Require min_samples per time_interval +MIN_SAMPLES = 5 + +# if telem values exceed these limits, cut the values +TELEM_CHOMP_LIMITS = {'dac': {'max': 550}, + 'ccd_temp': {'min': -35, + 'max': 50}, + 'aca_temp': {'max': 50, + 'min': 5}} + +# If telem values exceed these limits send a warning +# This is intended to be planning limit +1C +TELEM_LIMITS = {'ccd_temp': {'max': -6.1}} + +# Plot ranges +DAC_PLOT = {'ylim': (460, 515)} +DACVSDTEMP_PLOT = {'ylim': (460, 515), + 'xlim': (37, 40)} +ACA_TEMP_PLOT = {'ylim': (8, 32)} +CCD_TEMP_PLOT = {'ylim': (-15, -5)} + log = logging.getLogger() log.setLevel(logging.DEBUG) @@ -40,7 +67,7 @@ # emails... smtp_handler = SMTPHandler('localhost', 'aca@head.cfa.harvard.edu', - 'aca_alert@cfa.harvard.edu', + 'jconnelly@cfa.harvard.edu', 'perigee health mon') smtp_handler.setLevel(logging.WARN) @@ -51,33 +78,14 @@ if not has_smtp: log.addHandler(smtp_handler) -colors = characteristics.plot_colors -pass_color_maker = cycle(colors) -obsid_color_maker = cycle(colors) +pass_color_maker = cycle(PLOT_COLORS) +obsid_color_maker = cycle(PLOT_COLORS) task = 'perigee_health_plots' SKA = os.environ['SKA'] TASK_SHARE = os.path.join(os.environ['SKA'], 'share', 'perigee_health_plots') -#TASK_DIR = '/proj/sot/ska/www/ASPECT/perigee_health_plots' -#URL = 'http://cxc.harvard.edu/mta/ASPECT/perigee_health_plots' -#PASS_DATA = os.path.join(TASK_DIR, 'PASS_DATA') -#SUMMARY_DATA = os.path.join(TASK_DIR, 'SUMMARY_DATA') - -## Django setup for template rendering -#import django.template -#import django.conf -#if not django.conf.settings._target: -# try: -# django.conf.settings.configure() -# except RuntimeError, msg: -# print msg - -jinja_env = jinja2.Environment( - loader=jinja2.FileSystemLoader(os.path.join(TASK_SHARE, 'templates'))) - - def get_options(): parser = OptionParser(usage='pass_plots.py [options]') parser.set_defaults() @@ -107,15 +115,18 @@ def get_options(): def aca_ccd_model(tstart, tstop, init_temp): - cmd_states = get_cmd_states.fetch_states(tstart, - tstop, - vals=['obsid', - 'pitch', - 'q1', 'q2', 'q3', 'q4']) - model_spec = chandra_models.get_xija_model_file('aca') - model = xija.ThermalModel('aca', start=tstart, stop=tstop, model_spec=model_spec) - times = np.array([cmd_states['tstart'], cmd_states['tstop']]) - model.comp['pitch'].set_data(cmd_states['pitch'], times) + state_keys = ['obsid', 'pitch', 'q1', 'q2', 'q3', 'q4', 'eclipse'] + states = kadi_states.get_states(start=tstart, stop=tstop, + state_keys=state_keys, merge_identical=True) + + # Cheat and get model spec from starcheck for now until xija is updated + import starcheck + model_spec = str(Path(starcheck.__file__).parent / 'data' / 'aca_spec.json') + model = xija.ThermalModel('aca', start=tstart, stop=tstop, + model_spec=model_spec) + times = np.array([states['tstart'], states['tstop']]) + model.comp['pitch'].set_data(states['pitch'], times) + model.comp['eclipse'].set_data(states['eclipse'] != 'DAY', times) model.comp['aca0'].set_data(init_temp, tstart) model.comp['aacccdpt'].set_data(init_temp, tstart) model.make() @@ -141,42 +152,39 @@ def retrieve_perigee_telem(start='2009:100:00:00:00.000', """ tstart = DateTime(start) - # default tstop should be now + + # Default tstop should be now if stop is None: - tstop = DateTime(time.time(), format='unix') + tstop = DateTime() log.info("retrieve_perigee_telem(): Checking for current telemetry from %s" % tstart.date) pass_time_file = 'pass_times.txt' - aca_db = DBI(dbi='sybase', server='sybase', - user='aca_read', database='aca') - obsids = aca_db.fetchall("""SELECT obsid,obsid_datestart,obsid_datestop - from observations - where obsid_datestart > '%s' - and obsid_datestart < '%s' order by obsid_datestart""" - % (tstart.date, tstop.date)) + obsid_events = events.obsids.filter(tstart, tstop) # Get contiguous ER chunks, which are largely perigee passes chunks = [] chunk = {'start': None, 'stop': None} - for obsid in obsids: + for obs in obsid_events: + # If a OR, end a "chunk" of ER unless undefined # (this should only append on the first OR after one or more ERs) - if obsid['obsid'] < 40000: + if obs.obsid < 40000: if chunk['start'] is not None and chunk['stop'] is not None: chunks.append(chunk.copy()) chunk = {'start': None, 'stop': None} else: if chunk['start'] is None: - chunk['start'] = obsid['obsid_datestart'] - chunk['stop'] = obsid['obsid_datestop'] + chunk['start'] = obs.start + chunk['stop'] = obs.stop pass_dirs = [] - # for each ER chunk get telemetry + + # For each ER chunk get telemetry for chunk in chunks: er_start = chunk['start'] er_stop = chunk['stop'] @@ -188,15 +196,11 @@ def retrieve_perigee_telem(start='2009:100:00:00:00.000', pass_dir = os.path.join(pass_data_dir, "%s" % er_year, er_start) if not os.access(pass_dir, os.R_OK): os.mkdir(pass_dir) - if (DateTime(er_stop).secs - DateTime(er_start).secs > 86400 * 2): - if not os.path.exists(os.path.join(pass_dir, 'warned.txt')): - log.warn("Skipping %s pass, more than 48 hours long" % er_start) - continue pass_dirs.append(pass_dir) made_timefile = os.path.exists(os.path.join(pass_dir, pass_time_file)) if made_timefile: - pass_done = Ska.Table.read_ascii_table( - os.path.join(pass_dir, pass_time_file)) + pass_done = Table.read( + os.path.join(pass_dir, pass_time_file), format='ascii') if ((pass_done['obsid_datestart'] == er_start) & (pass_done['obsid_datestop'] == er_stop)): log.debug("%s times match" % pass_dir) @@ -243,10 +247,10 @@ def perigee_parse(pass_dir, min_samples=5, time_interval=20): pass_time_file = 'pass_times.txt' if not os.path.exists(os.path.join(pass_dir, pass_time_file)): raise MissingDataError("Missing telem for pass %s" % pass_dir) - pass_times = Ska.Table.read_ascii_table(os.path.join(pass_dir, - pass_time_file)) - mintime = pass_times[0].obsid_datestart - maxtime = pass_times[0].obsid_datestop + pass_times = Table.read(os.path.join(pass_dir, + pass_time_file), format='ascii') + mintime = pass_times[0]['obsid_datestart'] + maxtime = pass_times[0]['obsid_datestop'] hdr3 = aca_hdr3.MSIDset(['dac', 'ccd_temp', 'aca_temp'], mintime, maxtime) @@ -257,16 +261,21 @@ def perigee_parse(pass_dir, min_samples=5, time_interval=20): 'ccd_temp': hdr3['ccd_temp']} if not len(parsed_telem['ccd_temp'].vals): - raise MissingDataError("No HDR3 data for pass {}".format(pass_times[0].obsid_datestart)) + if DateTime(maxtime) - DateTime(mintime) < 1.5: + raise MissingDataError("No HDR3 data for pass {}".format( + pass_times[0]['obsid_datestart'])) + else: + raise ValueError("Long pass with no HDR3") return parsed_telem def get_telem_range(telem): + # find the 5th and 95th percentiles sorted = np.sort(telem) ninety_five = sorted[int(len(sorted) * .95)] five = sorted[int(len(sorted) * .05)] - mean = telem.mean() + # mean = telem.mean() # 10% = (ninety_five - five)/9. # min = five - 10% # max = ninety_five + 10% @@ -312,7 +321,7 @@ def plot_pass(telem, pass_dir, url, redo=False): # in time order for obsid in np.array(uniq_obs)[np.argsort(obs_times)]: obsmatch = np.flatnonzero(telem['obsid'].vals == obsid) - curr_color = obsid_color_maker.next() + curr_color = next(obsid_color_maker) tstart = telem['obsid'].times[obsmatch][0] tstop = telem['obsid'].times[obsmatch][-1] time_idx = (telem['dac'].times >= tstart) & (telem['dac'].times <= tstop) @@ -327,33 +336,32 @@ def plot_pass(telem, pass_dir, url, redo=False): DateTime(tstop).date, )) - - rand_obs_dac = (telem['dac'].vals[time_idx] + np.random.random(len(telem['dac'].vals[time_idx])) - .5) dtemp = telem['aca_temp'].vals[time_idx] - telem['ccd_temp'].vals[time_idx] + # Only plot if there's at least 1 unmasked sample if len(np.flatnonzero(~dtemp.mask)): plt.figure(tfig['dacvsdtemp'].number) plt.plot(dtemp, rand_obs_dac, color=curr_color, - marker='.', markersize=1) + marker='.', linestyle='None') if len(np.flatnonzero(~telem['dac'].vals[time_idx].mask)): plt.figure(tfig['dac'].number) plot_cxctime(telem['dac'].times[time_idx], rand_obs_dac, - color=curr_color, marker='.') + color=curr_color, marker='.', linestyle='None') if len(np.flatnonzero(~telem['aca_temp'].vals[time_idx].mask)): plt.figure(tfig['aca_temp'].number) plot_cxctime(telem['aca_temp'].times[time_idx], telem['aca_temp'].vals[time_idx], - color=curr_color, marker='.') + color=curr_color, marker='.', linestyle='None') if len(np.flatnonzero(~telem['ccd_temp'].vals[time_idx].mask)): plt.figure(tfig['ccd_temp'].number) plot_cxctime(telem['ccd_temp'].times[time_idx], telem['ccd_temp'].vals[time_idx], - color=curr_color, marker='.') + color=curr_color, marker='.', linestyle='None') obslist.write("\n") obslist.close() @@ -362,11 +370,11 @@ def plot_pass(telem, pass_dir, url, redo=False): ccd_temp_lims = get_telem_range(telem['ccd_temp'].vals) h = plt.figure(tfig['dacvsdtemp'].number) - plt.ylim(characteristics.dacvsdtemp_plot['ylim']) + plt.ylim(DACVSDTEMP_PLOT['ylim']) plt.ylabel('TEC DAC Control Level') - plt.xlim(min(characteristics.dacvsdtemp_plot['xlim'][0], + plt.xlim(min(DACVSDTEMP_PLOT['xlim'][0], np.min(dtemp) - .5), - max(characteristics.dacvsdtemp_plot['xlim'][1], + max(DACVSDTEMP_PLOT['xlim'][1], np.max(dtemp) + .5)) plt.xlabel("ACA temp - CCD temp (C)\n\n") h.subplots_adjust(bottom=0.2, left=.2) @@ -375,7 +383,7 @@ def plot_pass(telem, pass_dir, url, redo=False): plt.close(h) h = plt.figure(tfig['dac'].number) - plt.ylim(characteristics.dac_plot['ylim']) + plt.ylim(DAC_PLOT['ylim']) plt.ylabel('TEC DAC Control Level') h.subplots_adjust(left=.2) plt.grid(True) @@ -384,9 +392,9 @@ def plot_pass(telem, pass_dir, url, redo=False): h = plt.figure(tfig['aca_temp'].number) plt.ylabel('ACA temp (C)') - plt.ylim(min(characteristics.aca_temp_plot['ylim'][0], + plt.ylim(min(ACA_TEMP_PLOT['ylim'][0], aca_temp_lims[0]), - max(characteristics.aca_temp_plot['ylim'][1], + max(ACA_TEMP_PLOT['ylim'][1], aca_temp_lims[1])) h.subplots_adjust(left=0.2) plt.grid(True) @@ -395,16 +403,18 @@ def plot_pass(telem, pass_dir, url, redo=False): h = plt.figure(tfig['ccd_temp'].number) h.subplots_adjust(left=0.2) - plt.ylim(min(characteristics.ccd_temp_plot['ylim'][0], + plt.ylim(min(CCD_TEMP_PLOT['ylim'][0], ccd_temp_lims[0]), - max(characteristics.ccd_temp_plot['ylim'][1], + max(CCD_TEMP_PLOT['ylim'][1], ccd_temp_lims[1])) plt.ylabel('CCD temp (C)') plt.grid(True) plt.savefig(os.path.join(pass_dir, 'ccd_temp.png')) plt.close(h) - pass_index_template = jinja_env.get_template('pass_index_template.html') + file_dir = Path(__file__).parent + pass_index_template = Template(open(file_dir / 'pass_index_template.html', + 'r').read()) pass_index_page = pass_index_template.render(task={'url': url}) pass_fh = open(os.path.join(pass_dir, 'index.html'), 'w') pass_fh.write(pass_index_page) @@ -413,13 +423,11 @@ def plot_pass(telem, pass_dir, url, redo=False): def per_pass_tasks(pass_tail_dir, opt): - tfile = 'telem.pickle' pass_data_dir = os.path.join(opt.data_dir, 'PASS_DATA', pass_tail_dir) if not os.path.exists(pass_data_dir): os.makedirs(pass_data_dir) reduced_data = perigee_parse(pass_data_dir) - telem_time_file = 'telem_time.htm' pass_web_dir = os.path.join(opt.web_dir, 'PASS_DATA', pass_tail_dir) if not os.path.exists(pass_web_dir): @@ -434,12 +442,11 @@ def per_pass_tasks(pass_tail_dir, opt): tf.write("\n") tf.close() - # cut out nonsense bad data + # Cut out nonsense bad data types = ['aca_temp', 'ccd_temp', 'dac'] - filters = characteristics.telem_chomp_limits + filters = TELEM_CHOMP_LIMITS for type in filters.keys(): if 'max' in filters[type]: - goods = np.flatnonzero(reduced_data[type].vals <= filters[type]['max']) maxbads = np.flatnonzero(reduced_data[type].vals > filters[type]['max']) for bad in maxbads: log.info("filtering %s,%s,%6.2f" % @@ -450,7 +457,6 @@ def per_pass_tasks(pass_tail_dir, opt): for ttype in types: reduced_data[ttype].vals.mask[maxbads] = ma.masked if 'min' in filters[type]: - goods = np.flatnonzero(reduced_data[type].vals >= filters[type]['min']) minbads = np.flatnonzero(reduced_data[type].vals < filters[type]['min']) for bad in minbads: log.info("filtering %s,%s,%6.2f" % @@ -460,8 +466,8 @@ def per_pass_tasks(pass_tail_dir, opt): for ttype in types: reduced_data[ttype].vals.mask[maxbads] = ma.masked - # limit checks - limits = characteristics.telem_limits + # Run limit checks + limits = TELEM_LIMITS pass_url = opt.web_server + os.path.join( opt.url_dir, 'PASS_DATA', pass_tail_dir) for type in limits.keys(): @@ -475,7 +481,7 @@ def per_pass_tasks(pass_tail_dir, opt): max(reduced_data[type].vals), limits[type]['max'], pass_url)) - log.warn(warn_text) + log.warning(warn_text) warn_file = open( os.path.join(pass_data_dir, 'warned.txt'), 'w') warn_file.write(warn_text) @@ -491,15 +497,13 @@ def per_pass_tasks(pass_tail_dir, opt): min(reduced_data[type].vals), limits[type]['min'], pass_url)) - log.warn(warn_text) + log.warning(warn_text) warn_file = open( os.path.join(pass_data_dir, 'warned.txt'), 'w') warn_file.write(warn_text) warn_file.close() plot_pass(reduced_data, pass_web_dir, url=opt.url_dir) - #save_pass(reduced_data, pass_data_dir) - return reduced_data @@ -514,7 +518,6 @@ def save_pass(telem, pass_data_dir): rep_file.close() - def month_stats_and_plots(start, opt, redo=False): """ Make summary plots and statistics reports for months @@ -531,14 +534,14 @@ def month_stats_and_plots(start, opt, redo=False): year_dirs = glob.glob(os.path.join(opt.data_dir, 'PASS_DATA', '*')) year_dirs.sort() for year_dir in year_dirs: - match_year = re.match(".*(\d{4})$", year_dir) + match_year = re.match(r".*(\d{4})$", year_dir) toptable.write("%d" % int(match_year.group(1))) pass_dirs = glob.glob(os.path.join(year_dir, '*')) months = {} pass_dirs.sort() for pass_dir in pass_dirs: match_date = re.search( - "(\d{4}:\d{3}:\d{2}:\d{2}:\d{2}\.\d{3})", pass_dir) + r"(\d{4}:\d{3}:\d{2}:\d{2}:\d{2}\.\d{3})", pass_dir) obsdate = DateTime(match_date.group(1)) month = "%04d-M%02d" % (obsdate.year, obsdate.mon) try: @@ -587,7 +590,7 @@ def month_stats_and_plots(start, opt, redo=False): for pass_dir in months[month]: match_date = re.search( - "(\d{4}:\d{3}:\d{2}:\d{2}:\d{2}\.\d{3})", pass_dir) + r"(\d{4}:\d{3}:\d{2}:\d{2}:\d{2}\.\d{3})", pass_dir) passdate = match_date.group(1) passdates.append(passdate) ctpassdate = DateTime(passdate) @@ -597,7 +600,7 @@ def month_stats_and_plots(start, opt, redo=False): pass_tail_dir = re.sub( "%s/" % PASS_DATA, '', pass_dir) telem = per_pass_tasks(pass_tail_dir, opt) - curr_color = pass_color_maker.next() + curr_color = next(pass_color_maker) passlist.write( "%s" " \n" @@ -607,7 +610,8 @@ def month_stats_and_plots(start, opt, redo=False): DateTime(telem['dac'].times[0]).date, curr_color)) plt.figure(tfig['dacvsdtemp'].number) - # add randomization to dac + + # Add randomization to dac rand_dac = (telem['dac'].vals + np.random.random(len(telem['dac'].vals)) - .5) @@ -616,11 +620,11 @@ def month_stats_and_plots(start, opt, redo=False): color=curr_color, marker='.', markersize=.5) dtemp = telem['aca_temp'].vals - telem['ccd_temp'].vals if (temp_range['dtemp']['min'] is None - or (temp_range['dtemp']['min'] > np.min(dtemp))): - temp_range['dtemp']['min'] = np.min(dtemp) + or (temp_range['dtemp']['min'] > np.min(dtemp))): + temp_range['dtemp']['min'] = np.min(dtemp) if (temp_range['dtemp']['max'] is None - or (temp_range['dtemp']['max'] < np.max(dtemp))): - temp_range['dtemp']['max'] = np.max(dtemp) + or (temp_range['dtemp']['max'] < np.max(dtemp))): + temp_range['dtemp']['max'] = np.max(dtemp) for ttype in ('aca_temp', 'ccd_temp', 'dac'): if telem[ttype].vals.mean() is ma.masked: raise MissingDataError("Data for pass %s is masked" % pass_dir) @@ -652,16 +656,16 @@ def month_stats_and_plots(start, opt, redo=False): temp_range[ttype]['max'] = telem[ttype].vals.max() except MissingDataError: - print "skipping %s" % pass_dir + log.info("skipping %s" % pass_dir) passlist.write("\n") passlist.close() f = plt.figure(tfig['dacvsdtemp'].number) - plt.ylim(characteristics.dacvsdtemp_plot['ylim']) - plt.xlim(min(characteristics.dacvsdtemp_plot['xlim'][0], + plt.ylim(DACVSDTEMP_PLOT['ylim']) + plt.xlim(min(DACVSDTEMP_PLOT['xlim'][0], -0.5 + temp_range['dtemp']['min']), - max(characteristics.dacvsdtemp_plot['xlim'][1], + max(DACVSDTEMP_PLOT['xlim'][1], +0.5 + temp_range['dtemp']['max'])) plt.ylabel('TEC DAC Control Level') plt.xlabel('ACA temp - CCD temp (C)') @@ -674,9 +678,9 @@ def month_stats_and_plots(start, opt, redo=False): f = plt.figure(tfig['aca_temp'].number) plt.ylabel('ACA temp (C)') f.subplots_adjust(left=0.2) - plt.ylim(min(characteristics.aca_temp_plot['ylim'][0], + plt.ylim(min(ACA_TEMP_PLOT['ylim'][0], temp_range['aca_temp']['min']), - max(characteristics.aca_temp_plot['ylim'][1], + max(ACA_TEMP_PLOT['ylim'][1], temp_range['aca_temp']['max'])) curr_xlims = plt.xlim() dxlim = curr_xlims[1] - curr_xlims[0] @@ -688,9 +692,9 @@ def month_stats_and_plots(start, opt, redo=False): f = plt.figure(tfig['ccd_temp'].number) plt.ylabel('CCD temp (C)') - plt.ylim(min(characteristics.ccd_temp_plot['ylim'][0], + plt.ylim(min(CCD_TEMP_PLOT['ylim'][0], temp_range['ccd_temp']['min']), - max(characteristics.ccd_temp_plot['ylim'][1], + max(CCD_TEMP_PLOT['ylim'][1], temp_range['ccd_temp']['max'])) f.subplots_adjust(left=0.2) curr_xlims = plt.xlim() @@ -702,7 +706,7 @@ def month_stats_and_plots(start, opt, redo=False): plt.close(f) f = plt.figure(tfig['dac'].number) - plt.ylim(characteristics.dac_plot['ylim']) + plt.ylim(DAC_PLOT['ylim']) plt.ylabel('TEC DAC Control Level') curr_xlims = plt.xlim() dxlim = curr_xlims[1] - curr_xlims[0] @@ -719,22 +723,24 @@ def month_stats_and_plots(start, opt, redo=False): DateTime(passdates[0]).secs - 86400, DateTime(passdates[-1]).secs + 86400) ccd_temps = telem.vals - 273.15 - # cut out nonsense bad data using the ccd_temp filter - # in characteristics - filters = characteristics.telem_chomp_limits + + # Cut out nonsense bad data using the ccd_temp filter + filters = TELEM_CHOMP_LIMITS type = 'ccd_temp' goodfilt = ((ccd_temps <= filters[type]['max']) & (ccd_temps >= filters[type]['min'])) goods = np.flatnonzero(goodfilt) - bads = np.flatnonzero(goodfilt == False) + bads = np.flatnonzero(~goodfilt) for bad in bads: log.info("filtering %s,%s,%6.2f" % (DateTime(telem.times[bad]).date, 'AACCCDPT', ccd_temps[bad])) + # Filter in place ccd_temps = ccd_temps[goods] ccd_times = telem.times[goods] + # Save some trouble and only run the model when telem is available eclipse_data_range = fetch.get_time_range('AOECLIPS') model_end_time = np.min([DateTime(passdates[-1]).secs + 86400, @@ -746,10 +752,10 @@ def month_stats_and_plots(start, opt, redo=False): plot_cxctime(model_ccd_temp.times, model_ccd_temp.comp['aacccdpt'].mvals, 'b.', markersize=2) - plt.ylim(min(characteristics.ccd_temp_plot['ylim'][0], + plt.ylim(min(CCD_TEMP_PLOT['ylim'][0], temp_range['ccd_temp']['min'], model_ccd_temp.comp['aacccdpt'].mvals.min()), - max(characteristics.ccd_temp_plot['ylim'][1], + max(CCD_TEMP_PLOT['ylim'][1], temp_range['ccd_temp']['max'], model_ccd_temp.comp['aacccdpt'].mvals.max())) plt.ylabel('CCD Temp (C)') @@ -766,8 +772,11 @@ def month_stats_and_plots(start, opt, redo=False): month_index = os.path.join(month_web_dir, 'index.html') log.info("making %s" % month_index) - month_template = jinja_env.get_template( - 'month_index_template.html') + + file_dir = Path(__file__).parent + month_template = Template( + open(file_dir / 'month_index_template.html', + 'r').read()) page = month_template.render(task={'url': opt.url_dir, 'next': next_string, 'prev': prev_string}, @@ -781,9 +790,11 @@ def month_stats_and_plots(start, opt, redo=False): toptable.write("\n") toptable.close() - topindex_template_file = os.path.join( - TASK_SHARE, 'templates', 'top_index_template.html') - topindex_template = open(topindex_template_file).read() + file_dir = Path(__file__).parent + + # This one isn't jinja2, it uses a virtual include + topindex_template = open(file_dir / 'top_index_template.html', + 'r').read() topindex = open(os.path.join(opt.web_dir, 'index.html'), 'w') topindex.writelines(topindex_template) topindex.close() @@ -817,7 +828,7 @@ def main(): nowminus = DateTime(opt.start_time) last_month_start = DateTime("{}-{:02d}-01 00:00:00.000".format( - nowminus.year, nowminus.mon)) + nowminus.year, nowminus.mon)) log.info("---------- Perigee Pass Plots ran at %s ----------" % nowdate.date) log.info("Processing %s to %s" % (last_month_start.date, nowdate.date)) @@ -826,9 +837,9 @@ def main(): for pass_dir in pass_dirs: try: pass_tail_dir = re.sub("%s/" % PASS_DATA, '', pass_dir) - telem = per_pass_tasks(pass_tail_dir, opt) + per_pass_tasks(pass_tail_dir, opt) except MissingDataError: - print "skipping %s" % pass_dir + log.info("skipping %s" % pass_dir) month_stats_and_plots(start=last_month_start, opt=opt) diff --git a/templates/top_index_template.html b/perigee_health_plots/top_index_template.html similarity index 100% rename from templates/top_index_template.html rename to perigee_health_plots/top_index_template.html diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..0f74808 --- /dev/null +++ b/setup.py @@ -0,0 +1,39 @@ +# Licensed under a 3-clause BSD style license - see LICENSE +import os +import sys +from setuptools import setup + +try: + from testr.setup_helper import cmdclass +except ImportError: + cmdclass = {} + + +entry_points = {'console_scripts': [ + 'perigee_health_plots_update=perigee_health_plots.pass_plots:main']} + + +if "--user" not in sys.argv: + share_path = os.path.join(sys.prefix, "share", "perigee_health_plots") + data_files = [(share_path, ['task_schedule.cfg'])] +else: + data_files = None + + +setup(name='aca_hi_bgd', + author='Jean Connelly, Tom Aldcroft', + description='ACA perigee health plots', + author_email='jconnelly@cfa.harvard.edu', + use_scm_version=True, + zip_safe=False, + packages=['perigee_health_plots'], + package_data={'perigee_health_plots': + ['top_index_template.html', + 'pass_idex_template.html', + 'month_index_template.html']}, + tests_require=['pytest'], + cmdclass=cmdclass, + data_files=data_files, + entry_points=entry_points, + include_package_data=True, + ) From 48d1a40d1bc9d51d8ef505f296efb4422a310582 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Mon, 12 Oct 2020 13:52:31 -0400 Subject: [PATCH 2/4] Update task schedule --- task_schedule.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/task_schedule.cfg b/task_schedule.cfg index 8e4f838..2000538 100755 --- a/task_schedule.cfg +++ b/task_schedule.cfg @@ -9,9 +9,9 @@ loud 0 # Run loudly # Data files and directories. The *_dir vars can have $ENV{} vars which # get interpolated. (Note lack of task name after TST_DATA because this is just for test). -data_dir $ENV{SKA_DATA}/perigee_health_plots/ # Data file directory -log_dir $ENV{SKA_DATA}/perigee_health_plots/Logs # Log file directory -bin_dir $ENV{SKA_SHARE}/perigee_health_plots/ # Bin dir (optional, see task def'n) +data_dir $ENV{SKA}/data/perigee_health_plots/ # Data file directory +log_dir $ENV{SKA}/data/perigee_health_plots/Logs # Log file directory +bin_dir $ENV{SKA}/bin master_log run.log # Composite master log (created in log_dir) @@ -42,7 +42,7 @@ disable_alerts 0 cron * * * * * check_cron * * * * * - exec 1: pass_plots.py --verbose 2 + exec 1: perigee_health_plots_update --verbose 2 context 1 From d59f29ea3156927c06c6af0022502f3fa04d6e7f Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Wed, 14 Oct 2020 12:28:44 -0400 Subject: [PATCH 3/4] Add .git scm files --- .git_archival.txt | 1 + .gitattributes | 1 + 2 files changed, 2 insertions(+) create mode 100644 .git_archival.txt create mode 100644 .gitattributes diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 0000000..95cb3ee --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1 @@ +ref-names: $Format:%D$ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..00a7b00 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.git_archival.txt export-subst From e27d34aaf5c862652abf1530b2afd2c2c25c816b Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Wed, 14 Oct 2020 12:54:53 -0400 Subject: [PATCH 4/4] Fix trailing whitespace --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0f74808..2ab5487 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ use_scm_version=True, zip_safe=False, packages=['perigee_health_plots'], - package_data={'perigee_health_plots': + package_data={'perigee_health_plots': ['top_index_template.html', 'pass_idex_template.html', 'month_index_template.html']},