diff --git a/Makefile b/Makefile
deleted file mode 100644
index 0d48e7a..0000000
--- a/Makefile
+++ /dev/null
@@ -1,22 +0,0 @@
-# Set the task name
-TASK = skawatch3
-
-SHARE = jobwatch.py skawatch.py hourly_watch.py \
- log_template.html index_template.html hourly_template.html
-DATA = task_schedule_daily.cfg task_schedule_hourly.cfg
-WWW = overlib.js
-
-SKA3 = /proj/sot/ska3/flight
-INSTALL_WWW = /proj/sot/ska/www/ASPECT/$(TASK)
-INSTALL_DATA = $(SKA3)/data/$(TASK)
-INSTALL_SHARE = $(SKA3)/share/$(TASK)
-
-install:
- mkdir -p $(INSTALL_DATA)
- mkdir -p $(INSTALL_SHARE)
- mkdir -p $(INSTALL_WWW)
- mkdir -p $(INSTALL_WWW)/hourly
-
- rsync --times --cvs-exclude $(DATA) $(INSTALL_DATA)/
- rsync --times --cvs-exclude $(SHARE) $(INSTALL_SHARE)/
- rsync --times --cvs-exclude $(WWW) $(INSTALL_WWW)/
diff --git a/jobwatch/__init__.py b/jobwatch/__init__.py
new file mode 100644
index 0000000..2a80ac8
--- /dev/null
+++ b/jobwatch/__init__.py
@@ -0,0 +1,12 @@
+import ska_helpers
+__version__ = ska_helpers.get_version(__package__)
+
+from .jobwatch import *
+
+def test(*args, **kwargs):
+ '''
+ Run py.test unit tests.
+ '''
+ import testr
+ return testr.test(*args, **kwargs)
+
diff --git a/hourly_template.html b/jobwatch/hourly_template.html
similarity index 95%
rename from hourly_template.html
rename to jobwatch/hourly_template.html
index 2856147..c7558ed 100644
--- a/hourly_template.html
+++ b/jobwatch/hourly_template.html
@@ -1,7 +1,7 @@
-
+
Hourly Job Status: {{runtime_long}}
diff --git a/hourly_watch.py b/jobwatch/hourly_watch.py
similarity index 55%
rename from hourly_watch.py
rename to jobwatch/hourly_watch.py
index 6b6206c..410bc3a 100755
--- a/hourly_watch.py
+++ b/jobwatch/hourly_watch.py
@@ -21,6 +21,23 @@
FILEDIR = os.path.dirname(__file__)
+def get_options():
+ parser = argparse.ArgumentParser(description='Hourly status monitor')
+ parser.add_argument('--date-now',
+ help='Processing date')
+ parser.add_argument('--rootdir',
+ default='.',
+ help='Output root directory')
+ parser.add_argument('--email',
+ action='store_true',
+ help='Send email report')
+ parser.add_argument('--loud',
+ action='store_true',
+ help='Run loudly')
+ args = parser.parse_args()
+ return args
+
+
# Ska-specific watchers
class SkaURLWatch(JobWatch):
def __init__(self, task, maxage_hours, url=None,):
@@ -34,7 +51,7 @@ def headers(self):
try:
response = requests.get(self.basename)
- except:
+ except Exception:
self._exists = False
self._headers = None
else:
@@ -61,7 +78,8 @@ def age(self):
# zone. Finally subtract the two to get an age.
tm = time.strptime(self.headers[time_header],
"%a, %d %b %Y %H:%M:%S %Z")
- dtm_url = datetime(tm.tm_year, tm.tm_mon, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec,
+ dtm_url = datetime(tm.tm_year, tm.tm_mon, tm.tm_mday, tm.tm_hour,
+ tm.tm_min, tm.tm_sec,
tzinfo=pytz.timezone(tm.tm_zone))
# Current local time; .astimezone() makes this explicitly tz-aware
dtm_now_local = datetime.now().astimezone()
@@ -136,76 +154,64 @@ def age(self):
return self._age
-parser = argparse.ArgumentParser(description='Replan Central monitor')
-parser.add_argument('--date-now',
- help='Processing date')
-parser.add_argument('--rootdir',
- default='.',
- help='Output root directory')
-parser.add_argument('--email',
- action='store_true',
- help='Send email report')
-parser.add_argument('--loud',
- action='store_true',
- help='Send email report')
-args = parser.parse_args()
-
-jobwatch.LOUD = args.loud
-
-
-jws = []
-jws.extend(
- [
- SkaURLWatch('kadi', 1, 'http://kadi.cfa.harvard.edu'),
- SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/index.html'),
- SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/timeline.png'),
- SkaURLWatch('arc', 20, 'http://cxc.harvard.edu/mta/ASPECT/arc/ACE_5min.gif'),
- #SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_5min.gif'),
- #SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_xray.gif'),
- #SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/hrc_shield.png'),
- H5Watch('arc', 1, 'ACE.h5'),
- #H5Watch('arc', 1, 'hrc_shield.h5'),
- #H5Watch('arc', 1, 'GOES_X.h5'),
- IfotFileWatch('arc', 1, 'comm'),
- IfotFileWatch('arc', 1, 'eclipse'),
- IfotFileWatch('arc', 1, 'grating'),
- IfotFileWatch('arc', 1, 'grating'),
- IfotFileWatch('arc', 1, 'load_segment'),
- IfotFileWatch('arc', 1, 'maneuver'),
- IfotFileWatch('arc', 1, 'momentum_mon'),
- IfotFileWatch('arc', 1, 'or_er'),
- IfotFileWatch('arc', 1, 'radmon'),
- IfotFileWatch('arc', 1, 'safe'),
- IfotFileWatch('arc', 1, 'sim'),
- IfotFileWatch('arc', 1, 'sun_pos_mon'),
- NonSkaFileWatch('mta snapshot', 1, '/data/mta4/www/Snapshot/chandra.snapshot'),
- SkaWebWatch('arc', 1, 'index.html'),
- SkaWebWatch('arc', 1, 'chandra.snapshot'),
- #SkaWebWatch('arc', 1, 'hrc_shield.png'),
- #SkaWebWatch('arc', 2, 'GOES_xray.gif'),
- #SkaWebWatch('arc', 2, 'GOES_5min.gif'),
- SkaWebWatch('arc', 20, 'ACE_5min.gif')])
-
-set_report_attrs(jws)
-# Are all the reports OK?
-report_ok = all([j.ok for j in jws])
-errors = [job.basename for job in jws if not job.ok]
-# Set the age strings manually to display in hours
-for jw in jws:
- jw.age_str = '{:.2f}'.format(jw.age / HOURS) if jw.exists else 'None'
-index_html = make_html_report(jws, args.rootdir,
- index_template=os.path.join(FILEDIR,
- 'hourly_template.html'),
- just_status=True
- )
-recipients = ['aca@head.cfa.harvard.edu',
- 'msobolewska@cfa.harvard.edu', 'tisobe@cfa.harvard.edu', 'swolk@cfa.harvard.edu',
- 'lina.pulgarin-duque@cfa.harvard.edu']
-
-if args.email and not report_ok:
- jobwatch.sendmail(
- recipients, index_html, args.date_now,
- subject="{} Week {} errors: {}".format(
- time.strftime("%Y", time.localtime()),
- time.strftime("%W", time.localtime()),
- ", ".join(errors)))
+def main():
+
+ args = get_options()
+ jobwatch.LOUD = args.loud
+
+ jws = []
+ jws.extend(
+ [
+ SkaURLWatch('kadi', 1, 'http://kadi.cfa.harvard.edu'),
+ SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/index.html'),
+ SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/timeline.png'),
+ SkaURLWatch('arc', 20, 'http://cxc.harvard.edu/mta/ASPECT/arc/ACE_5min.gif'),
+ # SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_5min.gif'),
+ # SkaURLWatch('arc', 2, 'http://cxc.harvard.edu/mta/ASPECT/arc/GOES_xray.gif'),
+ # SkaURLWatch('arc', 1, 'http://cxc.harvard.edu/mta/ASPECT/arc/hrc_shield.png'),
+ H5Watch('arc', 1, 'ACE.h5'),
+ # H5Watch('arc', 1, 'hrc_shield.h5'),
+ # H5Watch('arc', 1, 'GOES_X.h5'),
+ IfotFileWatch('arc', 1, 'comm'),
+ IfotFileWatch('arc', 1, 'eclipse'),
+ IfotFileWatch('arc', 1, 'grating'),
+ IfotFileWatch('arc', 1, 'grating'),
+ IfotFileWatch('arc', 1, 'load_segment'),
+ IfotFileWatch('arc', 1, 'maneuver'),
+ IfotFileWatch('arc', 1, 'momentum_mon'),
+ IfotFileWatch('arc', 1, 'or_er'),
+ IfotFileWatch('arc', 1, 'radmon'),
+ IfotFileWatch('arc', 1, 'safe'),
+ IfotFileWatch('arc', 1, 'sim'),
+ IfotFileWatch('arc', 1, 'sun_pos_mon'),
+ NonSkaFileWatch('mta snapshot', 1, '/data/mta4/www/Snapshot/chandra.snapshot'),
+ SkaWebWatch('arc', 1, 'index.html'),
+ SkaWebWatch('arc', 1, 'chandra.snapshot'),
+ # SkaWebWatch('arc', 1, 'hrc_shield.png'),
+ # SkaWebWatch('arc', 2, 'GOES_xray.gif'),
+ # SkaWebWatch('arc', 2, 'GOES_5min.gif'),
+ SkaWebWatch('arc', 20, 'ACE_5min.gif')])
+
+ set_report_attrs(jws)
+ # Are all the reports OK?
+ report_ok = all([j.ok for j in jws])
+ errors = [job.basename for job in jws if not job.ok]
+ # Set the age strings manually to display in hours
+ for jw in jws:
+ jw.age_str = '{:.2f}'.format(jw.age / HOURS) if jw.exists else 'None'
+ index_html = make_html_report(jws, args.rootdir,
+ index_template=os.path.join(FILEDIR,
+ 'hourly_template.html'),
+ just_status=True
+ )
+ recipients = ['aca@head.cfa.harvard.edu',
+ 'msobolewska@cfa.harvard.edu', 'tisobe@cfa.harvard.edu', 'swolk@cfa.harvard.edu',
+ 'lina.pulgarin-duque@cfa.harvard.edu']
+
+ if args.email and not report_ok:
+ jobwatch.sendmail(
+ recipients, index_html, args.date_now,
+ subject="{} Week {} errors: {}".format(
+ time.strftime("%Y", time.localtime()),
+ time.strftime("%W", time.localtime()),
+ ", ".join(errors)))
diff --git a/index_template.html b/jobwatch/index_template.html
similarity index 96%
rename from index_template.html
rename to jobwatch/index_template.html
index 5732e11..b8b41b7 100644
--- a/index_template.html
+++ b/jobwatch/index_template.html
@@ -1,7 +1,7 @@
-
+
Ska Job Status: {{rundate}}
diff --git a/jobwatch.py b/jobwatch/jobwatch.py
similarity index 94%
rename from jobwatch.py
rename to jobwatch/jobwatch.py
index 74a5b68..33cbda4 100644
--- a/jobwatch.py
+++ b/jobwatch/jobwatch.py
@@ -35,7 +35,6 @@ def __init__(self, task, filename,
self.maxage = maxage
self.filetime = None
self.filedate = None
-
self.check()
@property
@@ -95,7 +94,7 @@ def check(self):
self.found_errors = found_errors
def __repr__(self):
- return ''.format(self.type, self.task)
+ return ''.format(getattr(self, 'type', None), self.task)
class FileWatch(JobWatch):
@@ -174,8 +173,11 @@ def set_report_attrs(jobwatches):
if jw.stale:
jw.age_str = '{}'.format(
jw.age_str)
- if i_jw == 0 or jw.type != jobwatches[i_jw - 1].type:
- jw.span_cols_text = jw.type
+
+ this_type = getattr(jw, 'type', 'Job')
+ last_type = getattr(jobwatches[i_jw -1], 'type', 'Job')
+ if i_jw == 0 or this_type != last_type:
+ jw.span_cols_text = this_type
maxerrs = 10
if not jw.ok and jw.found_errors:
@@ -183,7 +185,7 @@ def set_report_attrs(jobwatches):
for _, line, _ in jw.found_errors[:maxerrs]]
if len(jw.found_errors) > maxerrs:
popups.append('AND {} MORE'.format(
- len(jw.found_errors) - maxerrs))
+ len(jw.found_errors) - maxerrs))
popup = '
'.join(popups)
jw.overlib = ('ONMOUSEOVER="return overlib (\'{}\', WIDTH, 600);" '
'ONMOUSEOUT="return nd();"'.format(popup))
@@ -224,7 +226,7 @@ def make_html_report(jobwatches, rootdir, datenow=None,
nextdir = (DateTime(datenow) + 1).greta[:7]
outdir = os.path.join(rootdir, currdir)
if not os.path.exists(outdir):
- os.mkdir(outdir)
+ os.makedirs(outdir)
log_template = jinja2.Template(open(LOG_TEMPLATE, 'r').read())
root_prefix = '../{}/'
@@ -260,6 +262,12 @@ def make_html_report(jobwatches, rootdir, datenow=None,
outfile.write(index_html)
outfile.close()
+ # Copy the overlib.js into outdir if not there. This is hardcoded
+ # in the common templates.
+ if not os.path.exists(os.path.join(outdir, 'overlib.js')):
+ shutil.copy(os.path.join(FILEDIR, 'overlib.js'),
+ outdir)
+
if just_status:
return index_html
diff --git a/log_template.html b/jobwatch/log_template.html
similarity index 100%
rename from log_template.html
rename to jobwatch/log_template.html
diff --git a/overlib.js b/jobwatch/overlib.js
similarity index 100%
rename from overlib.js
rename to jobwatch/overlib.js
diff --git a/jobwatch/skawatch.py b/jobwatch/skawatch.py
new file mode 100755
index 0000000..d089c89
--- /dev/null
+++ b/jobwatch/skawatch.py
@@ -0,0 +1,221 @@
+#!/usr/bin/env python
+
+import argparse
+import jobwatch
+from glob import glob
+from jobwatch import (FileWatch, JobWatch, DbWatch,
+ make_html_report, copy_errs,
+ set_report_attrs)
+
+
+def get_options():
+ parser = argparse.ArgumentParser(description='Ska processing monitor')
+ parser.add_argument('--date-now',
+ help='Processing date')
+ parser.add_argument('--rootdir',
+ default='.',
+ help='Output root directory')
+ parser.add_argument('--email',
+ action='store_true',
+ help='Send email report')
+ parser.add_argument('--loud',
+ action='store_true',
+ help='Run loudly')
+ parser.add_argument('--max-age',
+ type=int,
+ default=30,
+ help='Maximum age of watch reports in days')
+ args = parser.parse_args()
+ return args
+
+
+# Ska-specific watchers
+class SkaWebWatch(FileWatch):
+ def __init__(self, task, maxage, basename,
+ filename='/proj/sot/ska/www/ASPECT/{task}/{basename}'):
+ self.basename = basename
+ super(SkaWebWatch, self).__init__(task, maxage, filename)
+
+
+class SkaJobWatch(JobWatch):
+ def __init__(self, task, maxage=1, errors=jobwatch.ERRORS, requires=(),
+ logdir='logs', logtask=None,
+ exclude_errors=(),
+ filename=('/proj/sot/ska/data/{task}/'
+ '{logdir}/daily.0/{logtask}.log')):
+ self.type = 'Log'
+ self.task = task
+ self.logtask = logtask or task
+ self.logdir = logdir
+ super(SkaJobWatch, self).__init__(task, filename, errors=errors,
+ requires=requires,
+ exclude_errors=exclude_errors,
+ maxage=maxage)
+
+
+class SkaDbWatch(DbWatch):
+ def __init__(self, task, maxage=1, table=None, timekey='tstart'):
+ super(SkaDbWatch, self).__init__(
+ task, maxage=maxage, table=table, timekey=timekey,
+ query='SELECT MAX({timekey}) AS maxtime FROM {table}',
+ dbi='sybase', server='sybase', user='aca_read', database='aca')
+
+
+class SkaSqliteDbWatch(DbWatch):
+ def __init__(self, task, maxage=1, dbfile=None, table=None, timekey='tstart'):
+ super(SkaSqliteDbWatch, self).__init__(
+ task, maxage=maxage, table=table, timekey=timekey,
+ query='SELECT MAX({timekey}) AS maxtime FROM {table}',
+ dbi='sqlite', server=dbfile)
+
+
+# Customized errors and paths
+py_errs = set(('error', 'warn', 'fail', 'fatal', 'exception', 'traceback'))
+perl_errs = set(('uninitialized value',
+ '(?