Added supplement diff script

agasc/scripts/supplement_diff.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+"""
+Generate diff between to supplement files and output in HTML format.
+"""
+
+import os
+import tables
+from pathlib import Path
+import difflib
+import pygments.lexers
+import pygments.formatters
+import argparse
+import datetime
+
+from io import StringIO
+
+from astropy.io import ascii
+from astropy import table
+
+
+def read_file(filename, exclude=[]):
+    formats = {
+        'agasc_versions': {k: '{:>21s}' for k in ['mags', 'obs', 'bad', 'supplement']},
+        'last_updated': {k: '{:>21s}' for k in ['mags', 'obs', 'bad', 'supplement']},
+        'obs': {
+            'comments': '{:>80s}',
+            'agasc_id': '{:10d}'
+        },
+    }
+
+    node_names = ['agasc_versions', 'last_updated', 'obs', 'bad']
+    with tables.open_file(filename) as h5:
+        for node in h5.root:
+            if node.name not in node_names:
+                node_names.append(node.name)
+        all_lines = []
+        for name in node_names:
+            if name in exclude:
+                continue
+            node = h5.get_node(f'/{name}')
+            t = table.Table(node[:])
+            t.convert_bytestring_to_unicode
+            s = StringIO()
+            ascii.write(t, s, format='fixed_width', formats=formats.get(name, {}))
+            s.seek(0)
+            lines = s.readlines()
+            dashes = "-" * (len(lines[0]) - 1) + "\n"
+            all_lines += ([dashes, f'| {node.name}\n', dashes] + lines + [dashes])
+    return all_lines
+
+
+def diff_files(fromfile, tofile, exclude_mags=False):
+    exclude = ['mags'] if exclude_mags else []
+    fromlines = read_file(fromfile, exclude)
+    tolines = read_file(tofile, exclude)
+    diff = difflib.unified_diff(fromlines, tolines, str(fromfile), str(tofile))
+    diff = ''.join(diff)
+    return diff
+
+
+def diff_to_html(diff):
+    date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
+    lexer = pygments.lexers.DiffLexer()
+    formatter = pygments.formatters.HtmlFormatter(
+        full=True, linenos='table',
+        title=f'AGASC supplement diff - {date}')
+    return pygments.highlight(diff, lexer, formatter)
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--from', dest='fromfile', type=Path, help='The original supplement file.')
+    parser.add_argument('--to', dest='tofile', type=Path, help='The modified supplement file.')
+    parser.add_argument(
+        '-o',
+        help='Output HTML file',
+        type=Path,
+        default='agasc_supplement_diff.html'
+    )
+    parser.add_argument(
+        '--exclude-mags', default=False, action='store_true',
+        help='Exclude changes in the "mags" table. There can be many of these.',
+    )
+    return parser
+
+
+def main():
+    args = get_parser().parse_args()
+    if args.fromfile is None and 'SKA' in os.environ:
+        args.fromfile = Path(os.environ['SKA']) / 'data' / 'agasc' / 'agasc_supplement.h5'
+    if args.tofile is None and 'SKA' in os.environ:
+        args.tofile = Path(os.environ['SKA']) / 'data' / 'agasc' / 'rc' / 'agasc_supplement.h5'
+    assert args.tofile, 'Option "--to" was not given and SKA is not defined'
+    assert args.fromfile, 'Option "--from" was not given and SKA is not defined'
+    assert args.fromfile.exists(), f'File {args.fromfile} does not exist'
+    assert args.tofile.exists(), f'File {args.tofile} does not exist'
+
+    if not args.o.parent.exists():
+        args.o.parent.mkdir()
+
+    with open(args.o, 'w') as fh:
+        diff = diff_to_html(diff_files(args.fromfile, args.tofile, exclude_mags=args.exclude_mags))
+        fh.write(diff)
+
+
+if __name__ == "__main__":
+    main()

docs/supplement.rst

@@ -287,6 +287,14 @@ Scripts
    :ref: agasc.scripts.supplement_tasks.get_parser
    :prog: agasc-supplement-tasks
 
+.. _`agasc-supplement-diff`:
+
+:ref:`agasc-supplement-diff`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. argparse::
+   :ref: agasc.scripts.supplement_diff.get_parser
+   :prog: agasc-supplement-diff
 
 Data products
 -------------

setup.py

@@ -14,6 +14,7 @@
         'agasc-update-magnitudes=agasc.scripts.update_mag_supplement:main',
         'agasc-update-supplement=agasc.scripts.update_supplement:main',
         'agasc-supplement-tasks=agasc.scripts.supplement_tasks:main',
+        'agasc-supplement-diff=agasc.scripts.supplement_diff:main',
     ]
 }
 

task_schedules/task_schedule_supplement_dispositions.cfg

@@ -52,6 +52,7 @@ notify       aca@cfa.harvard.edu
   exec 1: agasc-update-magnitudes --log-level debug --output-dir $ENV{SKA}/data/agasc/rc --report --obs-status-file $ENV{SKA}/data/agasc/rc/obs_status.yml --args-file $ENV{SKA}/data/agasc/rc/supplement_reports/weekly/latest/call_args.yml
   exec 1: agasc-magnitudes-report --output-dir $ENV{SKA}/data/agasc/rc/supplement_reports/suspect --input-dir $ENV{SKA}/data/agasc/rc
   exec 1: cp -fr $ENV{SKA}/data/agasc/rc/supplement_reports/weekly/latest/stars/* $ENV{SKA}/data/agasc/rc/supplement_reports/stars
+  exec 1: agasc-supplement-diff -o $ENV{SKA}/www/ASPECT/agasc/supplement/agasc_supplement_diff.html
   <check>
     <error>
       #    File                       Expression

task_schedules/task_schedule_update_supplement_rc.cfg

@@ -55,6 +55,7 @@ NOTIFY
   exec 1: agasc-update-magnitudes --log-level debug --output-dir $ENV{SKA}/data/agasc/rc --report
   exec 1: agasc-magnitudes-report --output-dir $ENV{SKA}/data/agasc/rc/supplement_reports/suspect --input-dir $ENV{SKA}/data/agasc/rc
   exec 1: cp -fr $ENV{SKA}/data/agasc/rc/supplement_reports/weekly/latest/stars/* $ENV{SKA}/data/agasc/rc/supplement_reports/stars
+  exec 1: agasc-supplement-diff -o $ENV{SKA}/www/ASPECT/agasc/supplement/agasc_supplement_diff.html
   <check>
     <error>
       #    File                  Expression