From a86f95ddd75a581512ba9261ed860242dfd0a1f4 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 31 Dec 2020 11:11:05 -0500 Subject: [PATCH 01/11] Make into a package --- Makefile | 22 ----------- jobwatch/__init__.py | 12 ++++++ .../hourly_template.html | 0 hourly_watch.py => jobwatch/hourly_watch.py | 0 .../index_template.html | 0 jobwatch.py => jobwatch/jobwatch.py | 0 .../log_template.html | 0 skawatch.py => jobwatch/skawatch.py | 0 .../stale.log => jobwatch/tests/__init__.py | 0 .../tests}/logs/eng_archive.log | 0 {tests => jobwatch/tests}/logs/errors.log | 0 jobwatch/tests/logs/stale.log | 0 {tests => jobwatch/tests}/test_jobwatch.py | 0 setup.py | 38 +++++++++++++++++++ 14 files changed, 50 insertions(+), 22 deletions(-) delete mode 100644 Makefile create mode 100644 jobwatch/__init__.py rename hourly_template.html => jobwatch/hourly_template.html (100%) rename hourly_watch.py => jobwatch/hourly_watch.py (100%) rename index_template.html => jobwatch/index_template.html (100%) rename jobwatch.py => jobwatch/jobwatch.py (100%) rename log_template.html => jobwatch/log_template.html (100%) rename skawatch.py => jobwatch/skawatch.py (100%) rename tests/logs/stale.log => jobwatch/tests/__init__.py (100%) rename {tests => jobwatch/tests}/logs/eng_archive.log (100%) rename {tests => jobwatch/tests}/logs/errors.log (100%) create mode 100644 jobwatch/tests/logs/stale.log rename {tests => jobwatch/tests}/test_jobwatch.py (100%) create mode 100755 setup.py diff --git a/Makefile b/Makefile deleted file mode 100644 index 0d48e7a..0000000 --- a/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -# Set the task name -TASK = skawatch3 - -SHARE = jobwatch.py skawatch.py hourly_watch.py \ - log_template.html index_template.html hourly_template.html -DATA = task_schedule_daily.cfg task_schedule_hourly.cfg -WWW = overlib.js - -SKA3 = /proj/sot/ska3/flight -INSTALL_WWW = /proj/sot/ska/www/ASPECT/$(TASK) -INSTALL_DATA = $(SKA3)/data/$(TASK) -INSTALL_SHARE = $(SKA3)/share/$(TASK) - -install: - mkdir -p $(INSTALL_DATA) - mkdir -p $(INSTALL_SHARE) - mkdir -p $(INSTALL_WWW) - mkdir -p $(INSTALL_WWW)/hourly - - rsync --times --cvs-exclude $(DATA) $(INSTALL_DATA)/ - rsync --times --cvs-exclude $(SHARE) $(INSTALL_SHARE)/ - rsync --times --cvs-exclude $(WWW) $(INSTALL_WWW)/ diff --git a/jobwatch/__init__.py b/jobwatch/__init__.py new file mode 100644 index 0000000..2a80ac8 --- /dev/null +++ b/jobwatch/__init__.py @@ -0,0 +1,12 @@ +import ska_helpers +__version__ = ska_helpers.get_version(__package__) + +from .jobwatch import * + +def test(*args, **kwargs): + ''' + Run py.test unit tests. + ''' + import testr + return testr.test(*args, **kwargs) + diff --git a/hourly_template.html b/jobwatch/hourly_template.html similarity index 100% rename from hourly_template.html rename to jobwatch/hourly_template.html diff --git a/hourly_watch.py b/jobwatch/hourly_watch.py similarity index 100% rename from hourly_watch.py rename to jobwatch/hourly_watch.py diff --git a/index_template.html b/jobwatch/index_template.html similarity index 100% rename from index_template.html rename to jobwatch/index_template.html diff --git a/jobwatch.py b/jobwatch/jobwatch.py similarity index 100% rename from jobwatch.py rename to jobwatch/jobwatch.py diff --git a/log_template.html b/jobwatch/log_template.html similarity index 100% rename from log_template.html rename to jobwatch/log_template.html diff --git a/skawatch.py b/jobwatch/skawatch.py similarity index 100% rename from skawatch.py rename to jobwatch/skawatch.py diff --git a/tests/logs/stale.log b/jobwatch/tests/__init__.py similarity index 100% rename from tests/logs/stale.log rename to jobwatch/tests/__init__.py diff --git a/tests/logs/eng_archive.log b/jobwatch/tests/logs/eng_archive.log similarity index 100% rename from tests/logs/eng_archive.log rename to jobwatch/tests/logs/eng_archive.log diff --git a/tests/logs/errors.log b/jobwatch/tests/logs/errors.log similarity index 100% rename from tests/logs/errors.log rename to jobwatch/tests/logs/errors.log diff --git a/jobwatch/tests/logs/stale.log b/jobwatch/tests/logs/stale.log new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_jobwatch.py b/jobwatch/tests/test_jobwatch.py similarity index 100% rename from tests/test_jobwatch.py rename to jobwatch/tests/test_jobwatch.py diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..6c9453f --- /dev/null +++ b/setup.py @@ -0,0 +1,38 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import os +import sys +from setuptools import setup + +try: + from testr.setup_helper import cmdclass +except ImportError: + cmdclass = {} + +if "--user" not in sys.argv: + share_path = os.path.join(sys.prefix, "share", "jobwatch") + web_path = os.path.join(sys.prefix, "www", "jobwatch") + data_files = [(share_path, ['task_schedule_hourly.cfg', 'task_schedule_daily.cfg']), + (web_path, ['overlib.js'])] +else: + data_files = None + +entry_points = {'console_scripts': ['skawatch_daily=jobwatch.skawatch:main', + 'skawatch_hourly=jobwatch.hourly_watch:main']} + +setup(name='jobwatch', + author='Tom Aldcroft', + description='Watch ska jobs', + author_email='taldcroft@cfa.harvard.edu', + packages=['jobwatch'], + package_data={'jobwatch': ['*html']}, + include_package_data=True, + data_files=data_files, + license=("New BSD/3-clause BSD License\nCopyright (c) 2019" + " Smithsonian Astrophysical Observatory\nAll rights reserved."), + use_scm_version=True, + setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'], + entry_points=entry_points, + zip_safe=False, + tests_require=['pytest'], + cmdclass=cmdclass, + ) From b47757543d976d4ce34adabc92ae488ab6be9b4a Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 31 Dec 2020 18:29:35 -0500 Subject: [PATCH 02/11] Fix previously-broken tests --- jobwatch/jobwatch.py | 4 ++-- jobwatch/tests/test_jobwatch.py | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/jobwatch/jobwatch.py b/jobwatch/jobwatch.py index 74a5b68..266eebd 100644 --- a/jobwatch/jobwatch.py +++ b/jobwatch/jobwatch.py @@ -35,7 +35,7 @@ def __init__(self, task, filename, self.maxage = maxage self.filetime = None self.filedate = None - + self.type = 'Job' self.check() @property @@ -224,7 +224,7 @@ def make_html_report(jobwatches, rootdir, datenow=None, nextdir = (DateTime(datenow) + 1).greta[:7] outdir = os.path.join(rootdir, currdir) if not os.path.exists(outdir): - os.mkdir(outdir) + os.makedirs(outdir) log_template = jinja2.Template(open(LOG_TEMPLATE, 'r').read()) root_prefix = '../{}/' diff --git a/jobwatch/tests/test_jobwatch.py b/jobwatch/tests/test_jobwatch.py index 913a4f0..e033b72 100644 --- a/jobwatch/tests/test_jobwatch.py +++ b/jobwatch/tests/test_jobwatch.py @@ -1,6 +1,7 @@ import sys import os import time +import pytest sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) import jobwatch @@ -95,16 +96,16 @@ def test_missing_requires(): assert jw.missing_requires == set(['hello world']) -def test_filewatch(): +def test_filewatch(tmpdir): jobfile = ('/proj/sot/ska/www/ASPECT/obc_rate_noise/' 'trending/pitch_hist_recent.png') Watch = jobwatch.FileWatch jws = [Watch(task='obc_rate_noise', filename=jobfile, maxage=50)] jobwatch.set_report_attrs(jws) - jobwatch.make_html_report(jws, rootdir='out_file') + jobwatch.make_html_report(jws, rootdir=os.path.join(tmpdir, 'out_file')) -def test_dbwatch(): +def test_dbwatch(tmpdir): Watch = SkaDbWatch jws = [Watch('DB acq_stats_data', timekey='tstart', table='acq_stats_data', maxage=4), @@ -114,10 +115,10 @@ def test_dbwatch(): Watch('DB aiprops', timekey='tstart', table='aiprops', maxage=4), ] jobwatch.set_report_attrs(jws) - jobwatch.make_html_report(jws, rootdir='out_db') + jobwatch.make_html_report(jws, rootdir=os.path.join(tmpdir, 'out_db')) -def test_make_html_report(): +def test_make_html_report(tmpdir): perlidl_errs = ('Use of uninitialized value', '(? Date: Thu, 31 Dec 2020 18:41:42 -0500 Subject: [PATCH 03/11] Remove commented-out aca_bgd_mon --- jobwatch/skawatch.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jobwatch/skawatch.py b/jobwatch/skawatch.py index a027329..0e408fb 100755 --- a/jobwatch/skawatch.py +++ b/jobwatch/skawatch.py @@ -102,11 +102,6 @@ def __init__(self, task, maxage=1, dbfile=None, table=None, timekey='tstart'): jws = [] jws.extend([ - # Commented out 2015-Jan-25. Hopefully this will be resurrected. - # SkaJobWatch('aca_bgd_mon', 400, errors=perl_errs, - # requires=('Copying plots and log file ' - # 'to /proj/sot/ska/www/ASPECT',)), - SkaJobWatch('aca_lts_eval', 2, errors=py_errs), SkaJobWatch('aca_hi_bgd_mon', 2, errors=py_errs, filename='/proj/sot/ska/data/aca_hi_bgd_mon/logs/daily.0/aca_hi_bgd.log'), From 6a13fc7c3416e7ddf99cdf1512a5e3fefab50811 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 31 Dec 2020 18:48:01 -0500 Subject: [PATCH 04/11] Move code into mains --- jobwatch/hourly_watch.py | 150 +++++++++++------------ jobwatch/skawatch.py | 251 ++++++++++++++++++++------------------- 2 files changed, 208 insertions(+), 193 deletions(-) diff --git a/jobwatch/hourly_watch.py b/jobwatch/hourly_watch.py index 6b6206c..1965df6 100755 --- a/jobwatch/hourly_watch.py +++ b/jobwatch/hourly_watch.py @@ -20,6 +20,22 @@ HOURS = 1 / 24. FILEDIR = os.path.dirname(__file__) +def get_options(): + parser = argparse.ArgumentParser(description='Replan Central monitor') + parser.add_argument('--date-now', + help='Processing date') + parser.add_argument('--rootdir', + default='.', + help='Output root directory') + parser.add_argument('--email', + action='store_true', + help='Send email report') + parser.add_argument('--loud', + action='store_true', + help='Send email report') + args = parser.parse_args() + return args + # Ska-specific watchers class SkaURLWatch(JobWatch): @@ -136,76 +152,64 @@ def age(self): return self._age -parser = argparse.ArgumentParser(description='Replan Central monitor') -parser.add_argument('--date-now', - help='Processing date') -parser.add_argument('--rootdir', - default='.', - help='Output root directory') -parser.add_argument('--email', - action='store_true', - help='Send email report') -parser.add_argument('--loud', - action='store_true', - help='Send email report') -args = parser.parse_args() - -jobwatch.LOUD = args.loud - - -jws = [] -jws.extend( - [ - SkaURLWatch('kadi', 1, 'http://kadi.cfa.harvard.edu'), - SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/index.html'), - SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/timeline.png'), - SkaURLWatch('arc', 20, 'http://cxc.harvard.edu/mta/ASPECT/arc/ACE_5min.gif'), - #SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_5min.gif'), - #SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_xray.gif'), - #SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/hrc_shield.png'), - H5Watch('arc', 1, 'ACE.h5'), - #H5Watch('arc', 1, 'hrc_shield.h5'), - #H5Watch('arc', 1, 'GOES_X.h5'), - IfotFileWatch('arc', 1, 'comm'), - IfotFileWatch('arc', 1, 'eclipse'), - IfotFileWatch('arc', 1, 'grating'), - IfotFileWatch('arc', 1, 'grating'), - IfotFileWatch('arc', 1, 'load_segment'), - IfotFileWatch('arc', 1, 'maneuver'), - IfotFileWatch('arc', 1, 'momentum_mon'), - IfotFileWatch('arc', 1, 'or_er'), - IfotFileWatch('arc', 1, 'radmon'), - IfotFileWatch('arc', 1, 'safe'), - IfotFileWatch('arc', 1, 'sim'), - IfotFileWatch('arc', 1, 'sun_pos_mon'), - NonSkaFileWatch('mta snapshot', 1, '/data/mta4/www/Snapshot/chandra.snapshot'), - SkaWebWatch('arc', 1, 'index.html'), - SkaWebWatch('arc', 1, 'chandra.snapshot'), - #SkaWebWatch('arc', 1, 'hrc_shield.png'), - #SkaWebWatch('arc', 2, 'GOES_xray.gif'), - #SkaWebWatch('arc', 2, 'GOES_5min.gif'), - SkaWebWatch('arc', 20, 'ACE_5min.gif')]) - -set_report_attrs(jws) -# Are all the reports OK? -report_ok = all([j.ok for j in jws]) -errors = [job.basename for job in jws if not job.ok] -# Set the age strings manually to display in hours -for jw in jws: - jw.age_str = '{:.2f}'.format(jw.age / HOURS) if jw.exists else 'None' -index_html = make_html_report(jws, args.rootdir, - index_template=os.path.join(FILEDIR, - 'hourly_template.html'), - just_status=True - ) -recipients = ['aca@head.cfa.harvard.edu', - 'msobolewska@cfa.harvard.edu', 'tisobe@cfa.harvard.edu', 'swolk@cfa.harvard.edu', - 'lina.pulgarin-duque@cfa.harvard.edu'] - -if args.email and not report_ok: - jobwatch.sendmail( - recipients, index_html, args.date_now, - subject="{} Week {} errors: {}".format( - time.strftime("%Y", time.localtime()), - time.strftime("%W", time.localtime()), - ", ".join(errors))) +def main(): + + args = get_options() + jobwatch.LOUD = args.loud + + jws = [] + jws.extend( + [ + SkaURLWatch('kadi', 1, 'http://kadi.cfa.harvard.edu'), + SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/index.html'), + SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/timeline.png'), + SkaURLWatch('arc', 20, 'http://cxc.harvard.edu/mta/ASPECT/arc/ACE_5min.gif'), + #SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_5min.gif'), + #SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_xray.gif'), + #SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/hrc_shield.png'), + H5Watch('arc', 1, 'ACE.h5'), + #H5Watch('arc', 1, 'hrc_shield.h5'), + #H5Watch('arc', 1, 'GOES_X.h5'), + IfotFileWatch('arc', 1, 'comm'), + IfotFileWatch('arc', 1, 'eclipse'), + IfotFileWatch('arc', 1, 'grating'), + IfotFileWatch('arc', 1, 'grating'), + IfotFileWatch('arc', 1, 'load_segment'), + IfotFileWatch('arc', 1, 'maneuver'), + IfotFileWatch('arc', 1, 'momentum_mon'), + IfotFileWatch('arc', 1, 'or_er'), + IfotFileWatch('arc', 1, 'radmon'), + IfotFileWatch('arc', 1, 'safe'), + IfotFileWatch('arc', 1, 'sim'), + IfotFileWatch('arc', 1, 'sun_pos_mon'), + NonSkaFileWatch('mta snapshot', 1, '/data/mta4/www/Snapshot/chandra.snapshot'), + SkaWebWatch('arc', 1, 'index.html'), + SkaWebWatch('arc', 1, 'chandra.snapshot'), + #SkaWebWatch('arc', 1, 'hrc_shield.png'), + #SkaWebWatch('arc', 2, 'GOES_xray.gif'), + #SkaWebWatch('arc', 2, 'GOES_5min.gif'), + SkaWebWatch('arc', 20, 'ACE_5min.gif')]) + + set_report_attrs(jws) + # Are all the reports OK? + report_ok = all([j.ok for j in jws]) + errors = [job.basename for job in jws if not job.ok] + # Set the age strings manually to display in hours + for jw in jws: + jw.age_str = '{:.2f}'.format(jw.age / HOURS) if jw.exists else 'None' + index_html = make_html_report(jws, args.rootdir, + index_template=os.path.join(FILEDIR, + 'hourly_template.html'), + just_status=True + ) + recipients = ['aca@head.cfa.harvard.edu', + 'msobolewska@cfa.harvard.edu', 'tisobe@cfa.harvard.edu', 'swolk@cfa.harvard.edu', + 'lina.pulgarin-duque@cfa.harvard.edu'] + + if args.email and not report_ok: + jobwatch.sendmail( + recipients, index_html, args.date_now, + subject="{} Week {} errors: {}".format( + time.strftime("%Y", time.localtime()), + time.strftime("%W", time.localtime()), + ", ".join(errors))) diff --git a/jobwatch/skawatch.py b/jobwatch/skawatch.py index 0e408fb..48f52f6 100755 --- a/jobwatch/skawatch.py +++ b/jobwatch/skawatch.py @@ -8,6 +8,27 @@ set_report_attrs) +def get_options(): + parser = argparse.ArgumentParser(description='Ska processing monitor') + parser.add_argument('--date-now', + help='Processing date') + parser.add_argument('--rootdir', + default='.', + help='Output root directory') + parser.add_argument('--email', + action='store_true', + help='Send email report') + parser.add_argument('--loud', + action='store_true', + help='Run loudly') + parser.add_argument('--max-age', + type=int, + default=30, + help='Maximum age of watch reports in days') + args = parser.parse_args() + return args + + # Ska-specific watchers class SkaWebWatch(FileWatch): def __init__(self, task, maxage, basename, @@ -48,25 +69,8 @@ def __init__(self, task, maxage=1, dbfile=None, table=None, timekey='tstart'): dbi='sqlite', server=dbfile) -parser = argparse.ArgumentParser(description='Ska processing monitor') -parser.add_argument('--date-now', - help='Processing date') -parser.add_argument('--rootdir', - default='.', - help='Output root directory') -parser.add_argument('--email', - action='store_true', - help='Send email report') -parser.add_argument('--loud', - action='store_true', - help='Run loudly') -parser.add_argument('--max-age', - type=int, - default=30, - help='Maximum age of watch reports in days') -args = parser.parse_args() - -jobwatch.LOUD = args.loud + + # Customized errors and paths py_errs = set(('error', 'warn', 'fail', 'fatal', 'exception', 'traceback')) @@ -100,115 +104,122 @@ def __init__(self, task, maxage=1, dbfile=None, table=None, timekey='tstart'): last_chimchim_log = sorted(glob("/home/kadi/occ_ska_sync_logs/chimchim-old/*.log"))[-1] last_cheru_log = sorted(glob("/home/kadi/occ_ska_sync_logs/cheru/*.log"))[-1] -jws = [] -jws.extend([ - SkaJobWatch('aca_lts_eval', 2, errors=py_errs), - SkaJobWatch('aca_hi_bgd_mon', 2, errors=py_errs, - filename='/proj/sot/ska/data/aca_hi_bgd_mon/logs/daily.0/aca_hi_bgd.log'), - SkaJobWatch('acdc', 2, errors=py_errs), - SkaJobWatch('aimpoint_mon', 2, errors=py_errs), - SkaJobWatch('arc', 2, errors=perl_errs, - exclude_errors=arc_exclude_errors, logdir='Logs'), - SkaJobWatch('astromon', 8, errors=astromon_errs), - SkaJobWatch('attitude_error_mon', 2, errors=att_mon_errs), - SkaJobWatch('dsn_summary', 2, errors=perl_errs, - logtask='dsn_summary_master'), - SkaJobWatch('eng_archive', 1, errors=engarchive_errs, - requires=('Checking dp_pcad32 content',)), - SkaJobWatch('fid_drift_mon', 2, errors=py_errs.union(perl_errs)), - - SkaJobWatch('kadi', 1, logtask='kadi_events', errors=py_errs, - exclude_errors=['InsecureRequestWarning']), - - SkaJobWatch('kadi', 1, logtask='kadi_cmds', errors=py_errs), - SkaJobWatch('star_stats', 2, filename=star_stat, - exclude_errors=['Cannot determine guide transition time']), - SkaJobWatch('timelines', 2, logtask='timelines_cmd_states', logdir='Logs'), - SkaJobWatch('mica', 2, errors=trace_plus_errs), - SkaJobWatch('acis_taco', 8, filename='/proj/sot/ska/data/acis_taco/logs/daily.0/taco.log'), - SkaJobWatch('acq_database', 2, filename=jean_db), - SkaJobWatch('guide_database', 2, filename=jean_db), - SkaJobWatch('guide_stat_db', 2, filename=jean_db), - SkaJobWatch('load_database', 2, filename=jean_db), - SkaJobWatch('obsid_load_database', 2, filename=jean_db), - SkaJobWatch('occ ska sync chimchim-old', 1, - filename=last_chimchim_log, - requires='total size is', - exclude_errors=['Welcome! Warning', - '5OHWFAIL.h5']), - SkaJobWatch('occ ska sync cheru', 1, - filename=last_cheru_log, - requires='total size is', - exclude_errors=['Welcome! Warning', - '5OHWFAIL.h5']), - SkaJobWatch('star_database', 2, filename=jean_db), - SkaJobWatch('starcheck_database', 2, filename=jean_db), - SkaJobWatch('vv_database', 2, filename=jean_db), - SkaJobWatch('validate_states', 2, errors=trace_plus_errs), - SkaJobWatch('scs107', 2, logdir='Logs', logtask='scs107_check'), - SkaJobWatch('telem_archive', 2, errors=telem_archive_errs), - SkaJobWatch('perigee_health_plots', 2, logdir='Logs', - errors=perigee_errs), - SkaJobWatch('vv_trend', 10, errors=py_errs), +def main(): + + args = get_options() + jobwatch.LOUD = args.loud + + jws = [] + jws.extend([ + SkaJobWatch('aca_lts_eval', 2, errors=py_errs), + SkaJobWatch('aca_hi_bgd_mon', 2, errors=py_errs, + filename='/proj/sot/ska/data/aca_hi_bgd_mon/logs/daily.0/aca_hi_bgd.log'), + SkaJobWatch('acdc', 2, errors=py_errs), + SkaJobWatch('aimpoint_mon', 2, errors=py_errs), + SkaJobWatch('arc', 2, errors=perl_errs, + exclude_errors=arc_exclude_errors, logdir='Logs'), + SkaJobWatch('astromon', 8, errors=astromon_errs), + SkaJobWatch('attitude_error_mon', 2, errors=att_mon_errs), + SkaJobWatch('dsn_summary', 2, errors=perl_errs, + logtask='dsn_summary_master'), + SkaJobWatch('eng_archive', 1, errors=engarchive_errs, + requires=('Checking dp_pcad32 content',)), + SkaJobWatch('fid_drift_mon', 2, errors=py_errs.union(perl_errs)), + + SkaJobWatch('kadi', 1, logtask='kadi_events', errors=py_errs, + exclude_errors=['InsecureRequestWarning']), + + SkaJobWatch('kadi', 1, logtask='kadi_cmds', errors=py_errs), + SkaJobWatch('star_stats', 2, filename=star_stat, + exclude_errors=['Cannot determine guide transition time']), + SkaJobWatch('timelines', 2, logtask='timelines_cmd_states', logdir='Logs'), + SkaJobWatch('mica', 2, errors=trace_plus_errs), + SkaJobWatch('acis_taco', 8, filename='/proj/sot/ska/data/acis_taco/logs/daily.0/taco.log'), + SkaJobWatch('acq_database', 2, filename=jean_db), + SkaJobWatch('guide_database', 2, filename=jean_db), + SkaJobWatch('guide_stat_db', 2, filename=jean_db), + SkaJobWatch('load_database', 2, filename=jean_db), + SkaJobWatch('obsid_load_database', 2, filename=jean_db), + SkaJobWatch('occ ska sync chimchim-old', 1, + filename=last_chimchim_log, + requires='total size is', + exclude_errors=['Welcome! Warning', + '5OHWFAIL.h5']), + SkaJobWatch('occ ska sync cheru', 1, + filename=last_cheru_log, + requires='total size is', + exclude_errors=['Welcome! Warning', + '5OHWFAIL.h5']), + SkaJobWatch('star_database', 2, filename=jean_db), + SkaJobWatch('starcheck_database', 2, filename=jean_db), + SkaJobWatch('vv_database', 2, filename=jean_db), + SkaJobWatch('validate_states', 2, errors=trace_plus_errs), + SkaJobWatch('scs107', 2, logdir='Logs', logtask='scs107_check'), + SkaJobWatch('telem_archive', 2, errors=telem_archive_errs), + SkaJobWatch('perigee_health_plots', 2, logdir='Logs', + errors=perigee_errs), + SkaJobWatch('vv_trend', 10, errors=py_errs), ]) -jws.extend([ - SkaWebWatch('aca_lts_eval', 12, 'index.html'), - SkaWebWatch('acq_stat_reports', 10, 'index.html'), - SkaWebWatch('aimpoint_mon', 1, 'index.html'), - SkaWebWatch('aimpoint_mon', 1, 'info.json'), - SkaWebWatch('attitude_error_mon', 2, 'one_shot_vs_angle.png'), - FileWatch('attitude_error_mon', 2, '/proj/sot/ska/data/attitude_error_mon/data.dat'), - SkaWebWatch('arc', 1, 'index.html'), - SkaWebWatch('arc', 1, 'chandra.snapshot'), - #SkaWebWatch('arc', 1, 'hrc_shield.png'), - #SkaWebWatch('arc', 2, 'GOES_xray.gif'), - #SkaWebWatch('arc', 2, 'GOES_5min.gif'), - SkaWebWatch('arc', 24, 'solar_wind.gif'), - SkaWebWatch('arc', 24, 'solar_flare_monitor.png'), - SkaWebWatch('arc', 2, 'ACE_5min.gif'), - SkaWebWatch('celmon', 30, 'offsets-ACIS-S-hist.gif'), - FileWatch('dsn_summary', 1, - '/proj/sot/ska/data/dsn_summary/dsn_summary.dat'), - FileWatch('dsn_summary', 1, - '/data/mta4/proj/rac/ops/ephem/dsn_summary.dat'), - SkaWebWatch('gui_stat_reports', 10, 'index.html'), - SkaWebWatch('fid_drift', 2, 'drift_acis_s.png'), - SkaWebWatch('eng_archive', 1, '', filename='/proj/sot/ska/data/eng_archive/data/dp_pcad32/TIME.h5'), - SkaWebWatch('kadi', 1, '', filename='/proj/sot/ska/data/kadi/events.db3'), - SkaWebWatch('obc_rate_noise', 50, 'trending/pitch_hist_recent.png'), - SkaWebWatch('perigee_health_plots', 5, 'index.html'), - SkaWebWatch('vv_rms', 10, 'hist2d_fig.png'), + jws.extend([ + SkaWebWatch('aca_lts_eval', 12, 'index.html'), + SkaWebWatch('acq_stat_reports', 10, 'index.html'), + SkaWebWatch('aimpoint_mon', 1, 'index.html'), + SkaWebWatch('aimpoint_mon', 1, 'info.json'), + SkaWebWatch('attitude_error_mon', 2, 'one_shot_vs_angle.png'), + FileWatch('attitude_error_mon', 2, '/proj/sot/ska/data/attitude_error_mon/data.dat'), + SkaWebWatch('arc', 1, 'index.html'), + SkaWebWatch('arc', 1, 'chandra.snapshot'), + #SkaWebWatch('arc', 1, 'hrc_shield.png'), + #SkaWebWatch('arc', 2, 'GOES_xray.gif'), + #SkaWebWatch('arc', 2, 'GOES_5min.gif'), + SkaWebWatch('arc', 24, 'solar_wind.gif'), + SkaWebWatch('arc', 24, 'solar_flare_monitor.png'), + SkaWebWatch('arc', 2, 'ACE_5min.gif'), + SkaWebWatch('celmon', 30, 'offsets-ACIS-S-hist.gif'), + FileWatch('dsn_summary', 1, + '/proj/sot/ska/data/dsn_summary/dsn_summary.dat'), + FileWatch('dsn_summary', 1, + '/data/mta4/proj/rac/ops/ephem/dsn_summary.dat'), + SkaWebWatch('gui_stat_reports', 10, 'index.html'), + SkaWebWatch('fid_drift', 2, 'drift_acis_s.png'), + SkaWebWatch('eng_archive', 1, '', filename='/proj/sot/ska/data/eng_archive/data/dp_pcad32/TIME.h5'), + SkaWebWatch('kadi', 1, '', filename='/proj/sot/ska/data/kadi/events.db3'), + SkaWebWatch('obc_rate_noise', 50, 'trending/pitch_hist_recent.png'), + SkaWebWatch('perigee_health_plots', 5, 'index.html'), + SkaWebWatch('vv_rms', 10, 'hist2d_fig.png'), ]) -jws.extend([ - SkaDbWatch('acq_stats_data', 4), - SkaDbWatch('aiprops', 4), - SkaDbWatch('cmds', -1, timekey='date'), - SkaDbWatch('cmd_states', -1, timekey='datestart'), - SkaDbWatch('load_segments', -1, timekey='datestop'), - SkaDbWatch('obspar', 4), - SkaDbWatch('starcheck_obs', 4, timekey='mp_starcat_time'), - SkaDbWatch('trak_stats_data', 4, timekey='kalman_tstart'), + jws.extend([ + SkaDbWatch('acq_stats_data', 4), + SkaDbWatch('aiprops', 4), + SkaDbWatch('cmds', -1, timekey='date'), + SkaDbWatch('cmd_states', -1, timekey='datestart'), + SkaDbWatch('load_segments', -1, timekey='datestop'), + SkaDbWatch('obspar', 4), + SkaDbWatch('starcheck_obs', 4, timekey='mp_starcat_time'), + SkaDbWatch('trak_stats_data', 4, timekey='kalman_tstart'), ]) -jws.extend([ - SkaSqliteDbWatch('cmds', -1, timekey='date', - dbfile='/proj/sot/ska/data/cmd_states/cmd_states.db3'), - SkaSqliteDbWatch('cmd_states', -1, timekey='datestart', - dbfile='/proj/sot/ska/data/cmd_states/cmd_states.db3'), - SkaSqliteDbWatch('load_segments', -1, timekey='datestop', - dbfile='/proj/sot/ska/data/cmd_states/cmd_states.db3'), - SkaSqliteDbWatch('timeline_loads', -1, timekey='datestop', - dbfile='/proj/sot/ska/data/cmd_states/cmd_states.db3'), + jws.extend([ + SkaSqliteDbWatch('cmds', -1, timekey='date', + dbfile='/proj/sot/ska/data/cmd_states/cmd_states.db3'), + SkaSqliteDbWatch('cmd_states', -1, timekey='datestart', + dbfile='/proj/sot/ska/data/cmd_states/cmd_states.db3'), + SkaSqliteDbWatch('load_segments', -1, timekey='datestop', + dbfile='/proj/sot/ska/data/cmd_states/cmd_states.db3'), + SkaSqliteDbWatch('timeline_loads', -1, timekey='datestop', + dbfile='/proj/sot/ska/data/cmd_states/cmd_states.db3'), ]) -set_report_attrs(jws) -index_html = make_html_report(jws, args.rootdir, args.date_now) -recipients = ['aca@head.cfa.harvard.edu'] + set_report_attrs(jws) + index_html = make_html_report(jws, args.rootdir, args.date_now) + recipients = ['aca@head.cfa.harvard.edu'] + + if args.email: + jobwatch.sendmail(recipients, index_html, args.date_now) + + jobwatch.remove_old_reports(args.rootdir, args.date_now, args.max_age) -if args.email: - jobwatch.sendmail(recipients, index_html, args.date_now) -jobwatch.remove_old_reports(args.rootdir, args.date_now, args.max_age) From d9c72346dcabe453f7b4a3ef00bdb1b344399087 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 31 Dec 2020 19:09:04 -0500 Subject: [PATCH 05/11] flake8 --- jobwatch/hourly_watch.py | 22 ++++++++++++---------- jobwatch/jobwatch.py | 2 +- jobwatch/skawatch.py | 20 ++++++++------------ 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/jobwatch/hourly_watch.py b/jobwatch/hourly_watch.py index 1965df6..9e49ffe 100755 --- a/jobwatch/hourly_watch.py +++ b/jobwatch/hourly_watch.py @@ -20,6 +20,7 @@ HOURS = 1 / 24. FILEDIR = os.path.dirname(__file__) + def get_options(): parser = argparse.ArgumentParser(description='Replan Central monitor') parser.add_argument('--date-now', @@ -50,7 +51,7 @@ def headers(self): try: response = requests.get(self.basename) - except: + except Exception: self._exists = False self._headers = None else: @@ -77,7 +78,8 @@ def age(self): # zone. Finally subtract the two to get an age. tm = time.strptime(self.headers[time_header], "%a, %d %b %Y %H:%M:%S %Z") - dtm_url = datetime(tm.tm_year, tm.tm_mon, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, + dtm_url = datetime(tm.tm_year, tm.tm_mon, tm.tm_mday, tm.tm_hour, + tm.tm_min, tm.tm_sec, tzinfo=pytz.timezone(tm.tm_zone)) # Current local time; .astimezone() makes this explicitly tz-aware dtm_now_local = datetime.now().astimezone() @@ -164,12 +166,12 @@ def main(): SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/index.html'), SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/timeline.png'), SkaURLWatch('arc', 20, 'http://cxc.harvard.edu/mta/ASPECT/arc/ACE_5min.gif'), - #SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_5min.gif'), - #SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_xray.gif'), - #SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/hrc_shield.png'), + # SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_5min.gif'), + # SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_xray.gif'), + # SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/hrc_shield.png'), H5Watch('arc', 1, 'ACE.h5'), - #H5Watch('arc', 1, 'hrc_shield.h5'), - #H5Watch('arc', 1, 'GOES_X.h5'), + # H5Watch('arc', 1, 'hrc_shield.h5'), + # H5Watch('arc', 1, 'GOES_X.h5'), IfotFileWatch('arc', 1, 'comm'), IfotFileWatch('arc', 1, 'eclipse'), IfotFileWatch('arc', 1, 'grating'), @@ -185,9 +187,9 @@ def main(): NonSkaFileWatch('mta snapshot', 1, '/data/mta4/www/Snapshot/chandra.snapshot'), SkaWebWatch('arc', 1, 'index.html'), SkaWebWatch('arc', 1, 'chandra.snapshot'), - #SkaWebWatch('arc', 1, 'hrc_shield.png'), - #SkaWebWatch('arc', 2, 'GOES_xray.gif'), - #SkaWebWatch('arc', 2, 'GOES_5min.gif'), + # SkaWebWatch('arc', 1, 'hrc_shield.png'), + # SkaWebWatch('arc', 2, 'GOES_xray.gif'), + # SkaWebWatch('arc', 2, 'GOES_5min.gif'), SkaWebWatch('arc', 20, 'ACE_5min.gif')]) set_report_attrs(jws) diff --git a/jobwatch/jobwatch.py b/jobwatch/jobwatch.py index 266eebd..df3f4b8 100644 --- a/jobwatch/jobwatch.py +++ b/jobwatch/jobwatch.py @@ -183,7 +183,7 @@ def set_report_attrs(jobwatches): for _, line, _ in jw.found_errors[:maxerrs]] if len(jw.found_errors) > maxerrs: popups.append('AND {} MORE'.format( - len(jw.found_errors) - maxerrs)) + len(jw.found_errors) - maxerrs)) popup = '
'.join(popups) jw.overlib = ('ONMOUSEOVER="return overlib (\'{}\', WIDTH, 600);" ' 'ONMOUSEOUT="return nd();"'.format(popup)) diff --git a/jobwatch/skawatch.py b/jobwatch/skawatch.py index 48f52f6..d089c89 100755 --- a/jobwatch/skawatch.py +++ b/jobwatch/skawatch.py @@ -69,15 +69,12 @@ def __init__(self, task, maxage=1, dbfile=None, table=None, timekey='tstart'): dbi='sqlite', server=dbfile) - - - # Customized errors and paths py_errs = set(('error', 'warn', 'fail', 'fatal', 'exception', 'traceback')) perl_errs = set(('uninitialized value', '(? Date: Mon, 4 Jan 2021 12:35:36 -0500 Subject: [PATCH 06/11] Remove hack to install overlib.js in www --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6c9453f..6652413 100755 --- a/setup.py +++ b/setup.py @@ -10,9 +10,7 @@ if "--user" not in sys.argv: share_path = os.path.join(sys.prefix, "share", "jobwatch") - web_path = os.path.join(sys.prefix, "www", "jobwatch") - data_files = [(share_path, ['task_schedule_hourly.cfg', 'task_schedule_daily.cfg']), - (web_path, ['overlib.js'])] + data_files = [(share_path, ['task_schedule_hourly.cfg', 'task_schedule_daily.cfg'])] else: data_files = None From 0ee4f330ee09cf8031226bd52dd98e21a157d28e Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Mon, 4 Jan 2021 13:02:57 -0500 Subject: [PATCH 07/11] Put overlib.js in package data and copy when reporting --- jobwatch/hourly_template.html | 2 +- jobwatch/index_template.html | 2 +- jobwatch/jobwatch.py | 4 ++++ overlib.js => jobwatch/overlib.js | 0 setup.py | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) rename overlib.js => jobwatch/overlib.js (100%) diff --git a/jobwatch/hourly_template.html b/jobwatch/hourly_template.html index 2856147..c7558ed 100644 --- a/jobwatch/hourly_template.html +++ b/jobwatch/hourly_template.html @@ -1,7 +1,7 @@ - + Hourly Job Status: {{runtime_long}} diff --git a/jobwatch/index_template.html b/jobwatch/index_template.html index 5732e11..b8b41b7 100644 --- a/jobwatch/index_template.html +++ b/jobwatch/index_template.html @@ -1,7 +1,7 @@ - + Ska Job Status: {{rundate}} diff --git a/jobwatch/jobwatch.py b/jobwatch/jobwatch.py index df3f4b8..78234b7 100644 --- a/jobwatch/jobwatch.py +++ b/jobwatch/jobwatch.py @@ -260,6 +260,10 @@ def make_html_report(jobwatches, rootdir, datenow=None, outfile.write(index_html) outfile.close() + # Copy the overlib.js into outdir (needed by the current templates) + shutil.copy(os.path.join(FILEDIR, 'overlib.js'), + outdir) + if just_status: return index_html diff --git a/overlib.js b/jobwatch/overlib.js similarity index 100% rename from overlib.js rename to jobwatch/overlib.js diff --git a/setup.py b/setup.py index 6652413..641c6f0 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ description='Watch ska jobs', author_email='taldcroft@cfa.harvard.edu', packages=['jobwatch'], - package_data={'jobwatch': ['*html']}, + package_data={'jobwatch': ['*html', '*js']}, include_package_data=True, data_files=data_files, license=("New BSD/3-clause BSD License\nCopyright (c) 2019" From b58a63d488044bad417942dc5b6579c8ccd77436 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Mon, 4 Jan 2021 13:05:02 -0500 Subject: [PATCH 08/11] Make copying overlib.js conditional (per Tom's comment) --- jobwatch/jobwatch.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jobwatch/jobwatch.py b/jobwatch/jobwatch.py index 78234b7..c1f3ed0 100644 --- a/jobwatch/jobwatch.py +++ b/jobwatch/jobwatch.py @@ -260,9 +260,11 @@ def make_html_report(jobwatches, rootdir, datenow=None, outfile.write(index_html) outfile.close() - # Copy the overlib.js into outdir (needed by the current templates) - shutil.copy(os.path.join(FILEDIR, 'overlib.js'), - outdir) + # Copy the overlib.js into outdir if not there. This is hardcoded + # in the common templates. + if not os.path.exists(os.path.join(outdir, 'overlib.js')): + shutil.copy(os.path.join(FILEDIR, 'overlib.js'), + outdir) if just_status: return index_html From d740351451773a9009145cfaa5a41c684321f386 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Mon, 4 Jan 2021 21:16:43 -0500 Subject: [PATCH 09/11] Remove default JobWatch type --- jobwatch/jobwatch.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jobwatch/jobwatch.py b/jobwatch/jobwatch.py index c1f3ed0..33cbda4 100644 --- a/jobwatch/jobwatch.py +++ b/jobwatch/jobwatch.py @@ -35,7 +35,6 @@ def __init__(self, task, filename, self.maxage = maxage self.filetime = None self.filedate = None - self.type = 'Job' self.check() @property @@ -95,7 +94,7 @@ def check(self): self.found_errors = found_errors def __repr__(self): - return ''.format(self.type, self.task) + return ''.format(getattr(self, 'type', None), self.task) class FileWatch(JobWatch): @@ -174,8 +173,11 @@ def set_report_attrs(jobwatches): if jw.stale: jw.age_str = '{}'.format( jw.age_str) - if i_jw == 0 or jw.type != jobwatches[i_jw - 1].type: - jw.span_cols_text = jw.type + + this_type = getattr(jw, 'type', 'Job') + last_type = getattr(jobwatches[i_jw -1], 'type', 'Job') + if i_jw == 0 or this_type != last_type: + jw.span_cols_text = this_type maxerrs = 10 if not jw.ok and jw.found_errors: From 275618bd25946b45547ac7623fe81b97ae5cc919 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Wed, 6 Jan 2021 10:50:46 -0500 Subject: [PATCH 10/11] Fix old pre-existing copy/paste error --- jobwatch/hourly_watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobwatch/hourly_watch.py b/jobwatch/hourly_watch.py index 9e49ffe..3a779ea 100755 --- a/jobwatch/hourly_watch.py +++ b/jobwatch/hourly_watch.py @@ -33,7 +33,7 @@ def get_options(): help='Send email report') parser.add_argument('--loud', action='store_true', - help='Send email report') + help='Run loudly') args = parser.parse_args() return args From a61039b319fc055d9ffc93dc0b9d13ae3b8c89cb Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Wed, 6 Jan 2021 10:52:34 -0500 Subject: [PATCH 11/11] Update description of hourly_watch --- jobwatch/hourly_watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobwatch/hourly_watch.py b/jobwatch/hourly_watch.py index 3a779ea..410bc3a 100755 --- a/jobwatch/hourly_watch.py +++ b/jobwatch/hourly_watch.py @@ -22,7 +22,7 @@ def get_options(): - parser = argparse.ArgumentParser(description='Replan Central monitor') + parser = argparse.ArgumentParser(description='Hourly status monitor') parser.add_argument('--date-now', help='Processing date') parser.add_argument('--rootdir',