Merge pull request #4 from sot/unbundle-redux

Unbundle TDB data and add testing

.gitignore

@@ -3,6 +3,5 @@ MANIFEST
 dist
 doc/_build
 TDB
-Ska/tdb/data/
+data/
 *.pyc
-

Makefile

@@ -14,6 +14,9 @@ DOC = doc/_build/html/*
 doc:
 	cd doc; make html
 
+install:
+	rsync -av data/ $(INSTALL_DATA)/
+
 install_doc:
 	mkdir -p $(INSTALL_DOC)
 	rsync --archive --times $(DOC)   $(INSTALL_DOC)/

NOTES.data

@@ -2,8 +2,13 @@ To create the package data files which store the TDB, do:
 
 % python make_tdb.py
 
-This creates files in Ska/tdb/data/p0<VERSION>/.
+This creates files in data/p0<VERSION>/.
 
-These files are not version controlled so that the repo can be on github.
+These files are not version controlled so that the repo can be on public github.
 DO NOT add to repo.
 
+To install do:
+
+% make install
+
+

NOTES.test

@@ -0,0 +1,5 @@
+python setup.py test [With possible additional options like --addopts='-k select_tests -v -s']
+
+python
+import Ska.tdb
+Ska.tdb.test()

Ska/tdb/__init__.py

@@ -1 +1,7 @@
 from .tdb import *
+
+
+def test(*args):
+    """Run self tests"""
+    import pytest
+    pytest.main(list(args))

Ska/tdb/tests/test_tdb.py

@@ -0,0 +1,42 @@
+from .. import msids, tables, set_tdb_version, get_tdb_version
+
+# Set to fixed version for regression testing
+TDB_VERSION = 12
+set_tdb_version(TDB_VERSION)
+
+tmsrment_colnames = (
+    'MSID', 'TECHNICAL_NAME', 'DATA_TYPE', 'CALIBRATION_TYPE', 'ENG_UNIT', 'LOW_RAW_COUNT',
+    'HIGH_RAW_COUNT', 'TOTAL_LENGTH', 'PROP', 'COUNTER_MSID', 'RANGE_MSID',
+    'CALIBRATION_SWITCH_MSID', 'CALIBRATION_DEFAULT_SET_NUM', 'LIMIT_SWITCH_MSID',
+    'LIMIT_DEFAULT_SET_NUM', 'ES_SWITCH_MSID', 'ES_DEFAULT_SET_NUM', 'OWNER_ID',
+    'DESCRIPTION', 'EHS_HEADER_FLAG')
+
+
+def test_msids():
+    tephin = msids['tephin']
+    assert tephin.Tmsrment.colnames == tmsrment_colnames
+    vals = list(tephin.Tmsrment.data)
+    assert vals == ['TEPHIN', 'EPHIN SENSOR HOUSING TEMP', 'IUNS', 'PP', 'DEGF', 0, 255, 8,
+                    'N', '0', '0', '0', 1, '0', 1, '0', 0, 'THM', 'LR/15/PA/2', 'U']
+
+    assert tephin.Tpp.data.dtype.names == ('MSID', 'CALIBRATION_SET_NUM', 'SEQUENCE_NUM',
+                                           'RAW_COUNT', 'ENG_UNIT_VALUE')
+
+    vals = msids.find('teph')
+    assert [v.msid for v in vals] == ['TEPHIN', 'TEPHTRP1', 'TEPHTRP2', 'TEPHTRR1', 'TEPHTRR2']
+
+
+def test_tables():
+    assert sorted(tables.keys()) == ['tcntr', 'tes', 'tlmt', 'tloc', 'tmsrment', 'towner', 'tpc',
+                                     'tpp', 'tsc', 'tsmpl', 'tstream', 'ttdm', 'ttdm_fmt']
+
+    tm = tables['tmsrment']
+    assert tm.colnames == tmsrment_colnames
+
+
+def test_version():
+    assert list(msids['tephin'].Tlmt) == ['TEPHIN', 1, 10.0, 161.0, 5.0, 999.0, 0, 5, 'A']
+    set_tdb_version(8)
+    assert get_tdb_version() == 8
+    assert list(msids['tephin'].Tlmt) == ['TEPHIN', 1, 10.0, 81.0, 5.0, 86.0, 0, 5, 'A']
+    set_tdb_version(TDB_VERSION)

Ska/tdb/version.py

@@ -14,7 +14,7 @@
 therein are applicable.
 """
 
-version = '0.4.2'
+version = '0.5'
 
 _versplit = version.replace('dev', '').split('.')
 major = int(_versplit[0])

make_tdb.py

@@ -1,9 +1,18 @@
+#!/usr/bin/env python
 """
 Make the numpy data files that are used by Ska.tdb.  From the repo root directory:
 
-% python make_tdb.py
+$ ./make_tdb.py
 
-This creates files in ./Ska/tdb/data/p0<VERSION>/.
+This creates files in ./data/p0<VERSION>/.  These dirs get installed to $SKA/data/Ska.tdb
+via the Makefile with "make install".
+
+This requires that directories be in /proj/sot/ska/ops/TDB which are the '*.txt' files
+that have been created by CXCDS from the MSFC-1949 files.  These are normally supplied
+by DS (historically Ian Evans) following a TDB update.
+
+There is some normalization vs. the input 1949 files and my memory is that it isn't so
+trivial to make the .txt files (but this might be wrong).
 """
 
 from __future__ import print_function
@@ -21,10 +30,16 @@
 TDB_versions = [os.path.basename(x) for x in glob.glob(os.path.join(TDB_ROOT, 'p0??'))]
 
 for TDB_version in sorted(TDB_versions):
-    out_path = os.path.join('Ska', 'tdb', 'data', TDB_version)
+    out_path = os.path.join('data', TDB_version)
     if os.path.exists(out_path):
-        print('Skipping TDB version {}'.format(TDB_version))
+        print('Skipping TDB version {}: already processed'.format(TDB_version))
         continue
+
+    if any(not os.path.exists(os.path.join(TDB_ROOT, TDB_version, name + '.txt'))
+           for name in names):
+        print('Skipping TDB version {}: input file(s) missing'.format(TDB_version))
+        continue
+
     os.mkdir(out_path)
     print('Processing TDB version {}'.format(TDB_version))
 

setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

setup.py

@@ -8,6 +8,8 @@
       author_email='aldcroft@head.cfa.harvard.edu',
       version=version,
       zip_safe=False,
-      packages=['Ska', 'Ska.tdb'],
+      setup_requires=['pytest-runner'],
+      packages=['Ska', 'Ska.tdb', 'Ska.tdb.tests'],
       package_dir={'Ska.tdb': 'Ska/tdb'},
+      tests_require=['pytest'],
       )