diff --git a/kadi/commands/commands.py b/kadi/commands/commands.py index c369487e..f26bab16 100644 --- a/kadi/commands/commands.py +++ b/kadi/commands/commands.py @@ -5,7 +5,7 @@ import numpy as np from astropy.table import Table, Row, Column, vstack -from Chandra.Time import DateTime +from Chandra.Time import DateTime, date2secs import pickle from ..paths import IDX_CMDS_PATH, PARS_DICT_PATH @@ -90,9 +90,11 @@ def get_cmds(start=None, stop=None, inclusive_stop=False, **kwargs): >>> print(cmds) :param start: DateTime format (optional) Start time, defaults to beginning - of available commands (2002:001) :param stop: DateTime format (optional) - Stop time, defaults to end of available commands :param kwargs: key=val - keyword argument pairs + of available commands (2002:001) + :param stop: DateTime format (optional) Stop time, defaults to end of available + commands + :param inclusive_stop: bool, include commands at exactly ``stop`` if True. + :param kwargs: key=val keyword argument pairs for filtering :returns: :class:`~kadi.commands.commands.CommandTable` of commands """ @@ -100,16 +102,23 @@ def get_cmds(start=None, stop=None, inclusive_stop=False, **kwargs): out = CommandTable(cmds) out['params'] = None if len(out) > 0 else Column([], dtype=object) + # Convert 'date' from bytestring to unicode. This allows + # date2secs(out['date']) to work and will generally reduce weird problems. + out.convert_bytestring_to_unicode() + + out.add_column(date2secs(out['date']), name='time', index=6) + out['time'].info.format = '.3f' + return out -def get_cmds_from_backstop(backstop, remove_starcat=True): +def get_cmds_from_backstop(backstop, remove_starcat=False): """ Initialize a ``CommandTable`` from ``backstop``, which can either be a string file name or a backstop table from ``parse_cm.read_backstop``. :param backstop: str or Table - :param remove_starcat: remove star catalog command parameters (default=True) + :param remove_starcat: remove star catalog command parameters (default=False) :returns: :class:`~kadi.commands.commands.CommandTable` of commands """ if isinstance(backstop, Path): @@ -133,6 +142,7 @@ def get_cmds_from_backstop(backstop, remove_starcat=True): out['tlmsid'] = np.chararray.encode(bs['tlmsid']) out['scs'] = bs['scs'].astype(np.uint8) out['step'] = bs['step'].astype(np.uint16) + out['time'] = date2secs(bs['date']) # Set timeline_id to 0, does not match any real timeline id out['timeline_id'] = np.zeros(n_bs, dtype=np.uint32) out['vcdu'] = bs['vcdu'].astype(np.int32) @@ -154,7 +164,14 @@ def get_cmds_from_backstop(backstop, remove_starcat=True): params['event_type'] = params['type'] del params['type'] - return CommandTable(out) + out = CommandTable(out) + out['time'].info.format = '.3f' + + # Convert 'date' from bytestring to unicode. This allows + # date2secs(out['date']) to work and will generally reduce weird problems. + out.convert_bytestring_to_unicode() + + return out def _find(start=None, stop=None, inclusive_stop=False, **kwargs): @@ -217,6 +234,7 @@ def _find(start=None, stop=None, inclusive_stop=False, **kwargs): par_ok |= (idx_cmds['idx'] == idx) ok &= par_ok cmds = idx_cmds[ok] + return cmds @@ -258,7 +276,7 @@ def __str__(self): out = ('{} {} '.format(self['date'], self['type']) + ' '.join('{}={}'.format(key, self[key]) for key in keys - if key not in ('type', 'date'))) + if key not in ('type', 'date', 'time'))) return out def __sstr__(self): @@ -354,3 +372,17 @@ def add_cmds(self, cmds): out.sort(['date', 'step', 'scs']) return out + + def as_list_of_dict(self): + """Convert CommandTable to a list of dict (ala Ska.ParseCM) + + The command ``params`` are embedded as a dict for each command. + + :return: list of dict + """ + self.fetch_params() + + names = self.colnames + cmds_list = [{name: cmd[name] for name in names} for cmd in self] + + return cmds_list diff --git a/kadi/commands/states.py b/kadi/commands/states.py index b0e410f2..e16e02bc 100644 --- a/kadi/commands/states.py +++ b/kadi/commands/states.py @@ -13,8 +13,7 @@ from astropy.table import Table, Column -from Chandra.cmd_states import decode_power -from Chandra.Time import DateTime +from Chandra.Time import DateTime, date2secs, secs2date import Chandra.Maneuver from Quaternion import Quat import Ska.Sun @@ -920,6 +919,63 @@ def callback(cls, date, transitions, state, idx): ################################################################### # ACIS transitions ################################################################### +def decode_power(mnem): + """ + Decode number of chips and feps from a ACIS power command + Return a dictionary with the number of chips and their identifiers + + Example:: + + >>> decode_power("WSPOW08F3E") + {'ccd_count': 5, + 'ccds': 'I0 I1 I2 I3 S3 ', + 'clocking': 0, + 'fep_count': 5, + 'feps': '1 2 3 4 5 ', + 'vid_board': 1} + + :param mnem: power command string + + """ + fep_info = {'fep_count': 0, + 'ccd_count': 0, + 'feps': '', + 'ccds': '', + 'vid_board': 1, + 'clocking': 0} + + # Special case WSPOW000XX to turn off vid_board + if mnem.startswith('WSPOW000'): + fep_info['vid_board'] = 0 + + # the hex for the commanding is after the WSPOW + powstr = mnem[5:] + if (len(powstr) != 5): + raise ValueError("%s in unexpected format" % mnem) + + # convert the hex to decimal and "&" it with 63 (binary 111111) + fepkey = int(powstr, 16) & 63 + # count the true binary bits + for bit in range(0, 6): + if (fepkey & (1 << bit)): + fep_info['fep_count'] = fep_info['fep_count'] + 1 + fep_info['feps'] = fep_info['feps'] + str(bit) + ' ' + + # convert the hex to decimal and right shift by 8 places + vidkey = int(powstr, 16) >> 8 + + # count the true bits + for bit in range(0, 10): + if (vidkey & (1 << bit)): + fep_info['ccd_count'] = fep_info['ccd_count'] + 1 + # position indicates I or S chip + if (bit < 4): + fep_info['ccds'] = fep_info['ccds'] + 'I' + str(bit) + ' ' + else: + fep_info['ccds'] = fep_info['ccds'] + 'S' + str(bit - 4) + ' ' + + return fep_info + class ACISTransition(BaseTransition): """ @@ -1258,6 +1314,13 @@ def get_states(start=None, stop=None, state_keys=None, cmds=None, continuity=Non # Final datestop far in the future datestop[-1] = stop out.add_column(Column(datestop, name='datestop'), 1) + + # Add corresponding tstart, tstop + out.add_column(Column(date2secs(out['datestart']), name='tstart'), 2) + out.add_column(Column(date2secs(out['datestop']), name='tstop'), 3) + out['tstart'].info.format = '.3f' + out['tstop'].info.format = '.3f' + out['trans_keys'] = [st.trans_keys for st in states] if reduce: @@ -1313,9 +1376,10 @@ def reduce_states(states, state_keys, merge_identical=False): has_transition |= has_transitions[key] # Create output with only desired state keys and only states with a transition - out = states[['datestart', 'datestop'] + list(state_keys)][has_transition] - out['datestop'][:-1] = out['datestart'][1:] - out['datestop'][-1] = states['datestop'][-1] + out = states[['datestart', 'datestop', 'tstart', 'tstop'] + list(state_keys)][has_transition] + for dt in ('date', 't'): + out[f'{dt}stop'][:-1] = out[f'{dt}start'][1:] + out[f'{dt}stop'][-1] = states[f'{dt}stop'][-1] trans_keys_list = [TransKeysSet() for _ in range(len(out))] for key in state_keys: @@ -1406,7 +1470,8 @@ def get_continuity(date=None, state_keys=None, lookbacks=(7, 30, 180, 1000)): # the stop time and did not get processed. continuity_transitions.extend(states.meta['continuity_transitions']) - colnames = set(states.colnames) - set(['datestart', 'datestop', 'trans_keys']) + colnames = set(states.colnames) - set(['datestart', 'datestop', + 'tstart', 'tstop', 'trans_keys']) for colname in colnames: if states[colname][-1] is not None: # Reduce states to only the desired state_key @@ -1451,13 +1516,25 @@ def get_continuity(date=None, state_keys=None, lookbacks=(7, 30, 180, 1000)): def interpolate_states(states, times): """Interpolate ``states`` table at given times. - :param states: states (np.recarray) - :param times: times (np.array or list) + :param states: states (astropy states Table) + :param times: times (np.array or any DateTime compatible input) :returns: ``states`` view at ``times`` """ - indexes = np.searchsorted(states['tstop'], times) - return states[indexes] + from astropy.table import Column + if not isinstance(times, np.ndarray) or times.dtype.kind != 'f': + times = DateTime(times).secs + + try: + tstops = states['tstop'] + except (ValueError, KeyError): + tstops = date2secs(states['datestop']) + + indexes = np.searchsorted(tstops, times) + out = states[indexes] + out.add_column(Column(secs2date(times), name='date'), index=0) + + return out def _unique(seq): @@ -1515,5 +1592,7 @@ def get_chandra_states(main_args=None): state_keys = opt.state_keys.split(',') if opt.state_keys else None states = get_states(start, stop, state_keys, merge_identical=opt.merge_identical) del states['trans_keys'] + del states['tstart'] + del states['tstop'] ascii.write(states, output=opt.outfile, format='fixed_width', delimiter='') diff --git a/kadi/commands/tests/data/states_0ea3eafaf003c1af3157722188a5f23c.ecsv.gz b/kadi/commands/tests/data/states_0ea3eafaf003c1af3157722188a5f23c.ecsv.gz new file mode 100644 index 00000000..de3c3a7d Binary files /dev/null and b/kadi/commands/tests/data/states_0ea3eafaf003c1af3157722188a5f23c.ecsv.gz differ diff --git a/kadi/commands/tests/data/states_2d7f3c7508f7171e335e4f69512dfd61.ecsv.gz b/kadi/commands/tests/data/states_2d7f3c7508f7171e335e4f69512dfd61.ecsv.gz new file mode 100644 index 00000000..61aaa225 Binary files /dev/null and b/kadi/commands/tests/data/states_2d7f3c7508f7171e335e4f69512dfd61.ecsv.gz differ diff --git a/kadi/commands/tests/data/states_31383844083b88458cfc87547f979942.ecsv.gz b/kadi/commands/tests/data/states_31383844083b88458cfc87547f979942.ecsv.gz new file mode 100644 index 00000000..5fcce7c5 Binary files /dev/null and b/kadi/commands/tests/data/states_31383844083b88458cfc87547f979942.ecsv.gz differ diff --git a/kadi/commands/tests/data/states_51cbc3f16a3910206859e2fb95ab417c.ecsv.gz b/kadi/commands/tests/data/states_51cbc3f16a3910206859e2fb95ab417c.ecsv.gz new file mode 100644 index 00000000..1a8f481f Binary files /dev/null and b/kadi/commands/tests/data/states_51cbc3f16a3910206859e2fb95ab417c.ecsv.gz differ diff --git a/kadi/commands/tests/data/states_e7f30129f60747e1d27c4672b18a056b.ecsv.gz b/kadi/commands/tests/data/states_e7f30129f60747e1d27c4672b18a056b.ecsv.gz new file mode 100644 index 00000000..4e0e7c5b Binary files /dev/null and b/kadi/commands/tests/data/states_e7f30129f60747e1d27c4672b18a056b.ecsv.gz differ diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index b5bb5771..dbd81f80 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -6,6 +6,7 @@ # Use data file from parse_cm.test for get_cmds_from_backstop test. # This package is a dependency import parse_cm.tests +from Chandra.Time import secs2date # Import cmds module directly (not kadi.cmds package, which is from ... import cmds) from .. import commands @@ -65,7 +66,7 @@ def test_get_cmds_zero_length_result(): cmds = commands.get_cmds(date='2017:001:12:00:00') assert len(cmds) == 0 assert cmds.colnames == ['idx', 'date', 'type', 'tlmsid', 'scs', - 'step', 'timeline_id', 'vcdu', 'params'] + 'step', 'time', 'timeline_id', 'vcdu', 'params'] def test_get_cmds_inclusive_stop(): @@ -81,12 +82,22 @@ def test_get_cmds_inclusive_stop(): assert np.all(cmds['date'] == [start, stop]) +def test_cmds_as_list_of_dict(): + cmds = commands.get_cmds('2020:140', '2020:141') + cmds_list = cmds.as_list_of_dict() + assert isinstance(cmds_list, list) + assert isinstance(cmds_list[0], dict) + cmds_rt = commands.CommandTable(cmds) + assert set(cmds_rt.colnames) == set(cmds.colnames) + for name in cmds.colnames: + assert np.all(cmds_rt[name] == cmds[name]) + + def test_get_cmds_from_backstop_and_add_cmds(): bs_file = Path(parse_cm.tests.__file__).parent / 'data' / 'CR182_0803.backstop' - bs_cmds = commands.get_cmds_from_backstop(bs_file) + bs_cmds = commands.get_cmds_from_backstop(bs_file, remove_starcat=True) - cmds = commands.get_cmds(start='2018:182:00:00:00', - stop='2018:182:08:00:00') + cmds = commands.get_cmds(start='2018:182:00:00:00', stop='2018:182:08:00:00') assert len(bs_cmds) == 674 assert len(cmds) == 56 @@ -95,6 +106,9 @@ def test_get_cmds_from_backstop_and_add_cmds(): for bs_col, col in zip(bs_cmds.itercols(), cmds.itercols()): assert bs_col.dtype == col.dtype + assert np.all(secs2date(cmds['time']) == cmds['date']) + assert np.all(secs2date(bs_cmds['time']) == bs_cmds['date']) + new_cmds = cmds.add_cmds(bs_cmds) assert len(new_cmds) == len(cmds) + len(bs_cmds) @@ -104,7 +118,7 @@ def test_get_cmds_from_backstop_and_add_cmds(): assert np.all(bs_cmds['params'][ok] == {}) # Accept MP_STARCAT commands - bs_cmds = commands.get_cmds_from_backstop(bs_file, remove_starcat=False) + bs_cmds = commands.get_cmds_from_backstop(bs_file) ok = bs_cmds['type'] == 'MP_STARCAT' assert np.count_nonzero(ok) == 15 assert np.all(bs_cmds['params'][ok] != {}) diff --git a/kadi/commands/tests/test_states.py b/kadi/commands/tests/test_states.py index ddbd978f..6bf9afe2 100644 --- a/kadi/commands/tests/test_states.py +++ b/kadi/commands/tests/test_states.py @@ -1,10 +1,12 @@ import os +import hashlib +from pathlib import Path +import gzip import numpy as np from .. import commands, states import pytest -import Chandra.cmd_states as cmd_states from Chandra.Time import DateTime from Ska.engarchive import fetch from astropy.io import ascii @@ -17,6 +19,37 @@ HAS_PITCH = False +# Canonical state0 giving spacecraft state at beginning of timelines +# 2002:007:13 fetch --start 2002:007:13:00:00 --stop 2002:007:13:02:00 aoattqt1 +# aoattqt2 aoattqt3 aoattqt4 cobsrqid aopcadmd tscpos +STATE0 = {'ccd_count': 5, + 'clocking': 0, + 'datestart': '2002:007:13:00:00.000', + 'datestop': '2099:001:00:00:00.000', + 'dec': -11.500, + 'fep_count': 0, + 'hetg': 'RETR', + 'letg': 'RETR', + 'obsid': 61358, + 'pcad_mode': 'NPNT', + 'pitch': 61.37, + 'power_cmd': 'AA00000000', + 'q1': -0.568062, + 'q2': 0.121674, + 'q3': 0.00114141, + 'q4': 0.813941, + 'ra': 352.000, + 'roll': 289.37, + 'si_mode': 'undef', + 'simfa_pos': -468, + 'simpos': -99616, + 'trans_keys': 'undef', + 'tstart': 127020624.552, + 'tstop': 3187296066.184, + 'vid_board': 0, + 'dither': 'None'} + + def assert_all_close_states(rc, rk, keys): """ Compare all ``key`` columns of the commanded states table ``rc`` and @@ -37,7 +70,7 @@ def get_states_test(start, stop, state_keys, continuity=None): start = DateTime(start) stop = DateTime(stop) - cstates = Table(cmd_states.fetch_states(start, stop)) + cstates = cmd_states_fetch_states(start.date, stop.date) trans_keys = [set(val.split(',')) for val in cstates['trans_keys']] cstates.remove_column('trans_keys') # Necessary for older astropy cstates['trans_keys'] = trans_keys @@ -123,6 +156,9 @@ def test_quick(): # Now test using start/stop pair with start/stop and no supplied cmds or continuity. # This also tests the API kwarg order: datestart, datestop, state_keys, ..) sts = states.get_states('2018:235:12:00:00', '2018:245:12:00:00', state_keys, reduce=False) + assert np.all(DateTime(sts['tstart']).date == sts['datestart']) + assert np.all(DateTime(sts['tstop']).date == sts['datestop']) + rk = states.reduce_states(sts, state_keys, merge_identical=True) assert len(rc) == len(rk) @@ -199,8 +235,8 @@ def test_pitch_2017(): rkstates['tstop'] = DateTime(rkstates['datestop']).secs times = np.arange(rcstates['tstop'][0], rcstates['tstop'][-2], 200.0) - rci = cmd_states.interpolate_states(rcstates, times) - rki = cmd_states.interpolate_states(rkstates, times) + rci = states.interpolate_states(rcstates, times) + rki = states.interpolate_states(rkstates, times) dp = np.abs(rci['pitch'] - rki['pitch']) assert np.all(dp < 0.5) @@ -420,19 +456,23 @@ def test_get_continuity_fail(): def test_reduce_states_merge_identical(): - datestart = DateTime(np.arange(0, 5)).date - datestop = DateTime(np.arange(1, 6)).date + tstart = np.arange(0, 5) + tstop = np.arange(1, 6) + datestart = DateTime(tstart).date + datestop = DateTime(tstop).date + dat0 = Table([datestart, datestop, tstart, tstop], + names=['datestart', 'datestop', 'tstart', 'tstop']) # Table with something that changes every time - vals = np.arange(5) - dat = Table([datestart, datestop, vals], names=['datestart', 'datestop', 'vals']) + dat = dat0.copy() + dat['vals'] = np.arange(5) dat['val1'] = 1 dr = states.reduce_states(dat, ['vals', 'val1'], merge_identical=True) assert np.all(dr[dat.colnames] == dat) # Table with nothing that changes - vals = np.ones(5) - dat = Table([datestart, datestop, vals], names=['datestart', 'datestop', 'vals']) + dat = dat0.copy() + dat['vals'] = 1 dat['val1'] = 1 dr = states.reduce_states(dat, ['vals', 'val1'], merge_identical=True) assert len(dr) == 1 @@ -440,8 +480,8 @@ def test_reduce_states_merge_identical(): assert dr['datestop'][0] == dat['datestop'][-1] # Table with edge changes - vals = [1, 0, 0, 0, 1] - dat = Table([datestart, datestop, vals], names=['datestart', 'datestop', 'vals']) + dat = dat0.copy() + dat['vals'] = [1, 0, 0, 0, 1] dr = states.reduce_states(dat, ['vals'], merge_identical=True) assert len(dr) == 3 assert np.all(dr['datestart'] == dat['datestart'][[0, 1, 4]]) @@ -449,9 +489,9 @@ def test_reduce_states_merge_identical(): assert np.all(dr['vals'] == [1, 0, 1]) # Table with multiple changes - val1 = [1, 0, 1, 1, 1] - val2 = [1, 1, 1, 1, 0] - dat = Table([datestart, datestop, val1, val2], names=['datestart', 'datestop', 'val1', 'val2']) + dat = dat0.copy() + dat['val1'] = [1, 0, 1, 1, 1] + dat['val2'] = [1, 1, 1, 1, 0] dr = states.reduce_states(dat, ['val1', 'val2'], merge_identical=True) assert len(dr) == 4 assert np.all(dr['datestart'] == dat['datestart'][[0, 1, 2, 4]]) @@ -465,15 +505,49 @@ def test_reduce_states_merge_identical(): assert dr['trans_keys'][3] == set(['val2']) +def cmd_states_fetch_states(*args, **kwargs): + """Generate regression data files for states using Chandra.cmd_states. + + Once files have been created they are included in the package distribution + and Chandra.cmd_states is no longer needed. From this point kadi will be + the definitive reference for states. + """ + md5 = hashlib.md5() + md5.update(repr(args).encode('utf8')) + md5.update(repr(kwargs).encode('utf8')) + digest = md5.hexdigest() + datafile = Path(__file__).parent / 'data' / f'states_{digest}.ecsv' + datafile_gz = datafile.parent / (datafile.name + '.gz') + + if datafile_gz.exists(): + cs = Table.read(datafile_gz, format='ascii.ecsv') + else: + # Prevent accidentally writing data to flight in case of some packaging problem. + if 'KADI_WRITE_TEST_DATA' not in os.environ: + raise RuntimeError('cannot find test data. Define KADI_WRITE_TEST_DATA ' + 'env var to create it.') + import Chandra.cmd_states as cmd_states + cs = cmd_states.fetch_states(*args, **kwargs) + cs = Table(cs) + print(f'Writing {datafile_gz} for args={args} kwargs={kwargs}') + cs.write(datafile, format='ascii.ecsv') + + # Gzip the file + with open(datafile, 'rb') as f_in, gzip.open(datafile_gz, 'wb') as f_out: + f_out.writelines(f_in) + datafile.unlink() + + return cs + + def test_reduce_states_cmd_states(): """ Test that simple get_states() call with defaults gives the same results as calling cmd_states.fetch_states(). """ - cs = cmd_states.fetch_states('2018:235:12:00:00', '2018:245:12:00:00', allow_identical=True) - cs = Table(cs) + cs = cmd_states_fetch_states('2018:235:12:00:00', '2018:245:12:00:00', allow_identical=True) - state_keys = (set(cmd_states.STATE0) + state_keys = (set(STATE0) - set(['datestart', 'datestop', 'trans_keys', 'tstart', 'tstop'])) # Default setting is reduce states with merge_identical=False, which is the same @@ -1229,11 +1303,16 @@ def test_continuity_with_transitions_SPM(): '__transitions__': [{'date': '2017:087:08:21:35.838', 'sun_pos_mon': 'ENAB'}], 'sun_pos_mon': 'DISA'} - exp = [' datestart datestop sun_pos_mon trans_keys', - '--------------------- --------------------- ----------- -----------', - '2017:087:08:20:35.838 2017:087:08:21:35.838 DISA ', - '2017:087:08:21:35.838 2017:087:08:30:50.891 ENAB sun_pos_mon', - '2017:087:08:30:50.891 2017:087:10:20:35.838 DISA sun_pos_mon'] + exp = [' datestart datestop tstart tstop ' + 'sun_pos_mon trans_keys', + '--------------------- --------------------- ------------- ------------- ' + '----------- -----------', + '2017:087:08:20:35.838 2017:087:08:21:35.838 607076505.022 ' + '607076565.022 DISA ', + '2017:087:08:21:35.838 2017:087:08:30:50.891 607076565.022 ' + '607077120.075 ENAB sun_pos_mon', + '2017:087:08:30:50.891 2017:087:10:20:35.838 607077120.075 ' + '607083705.022 DISA sun_pos_mon'] sts = states.get_states(start, stop, state_keys=['sun_pos_mon']) assert sts.pformat(max_lines=-1, max_width=-1) == exp @@ -1277,6 +1356,9 @@ def test_get_pitch_from_mid_maneuver(): def test_acisfp_setpoint_state(): sts = states.get_states('1999-01-01 12:00:00', '2004-01-01 12:00:00', state_keys='acisfp_setpoint') + del sts['tstart'] + del sts['tstop'] + assert repr(sts).splitlines() == [ '', ' datestart datestop acisfp_setpoint trans_keys ', @@ -1290,6 +1372,8 @@ def test_acisfp_setpoint_state(): sts = states.get_states('2018-01-01 12:00:00', '2020-03-01 12:00:00', state_keys='acisfp_setpoint') + del sts['tstart'] + del sts['tstop'] assert repr(sts).splitlines() == [ '
', ' datestart datestop acisfp_setpoint trans_keys ', diff --git a/kadi/events/__init__.py b/kadi/events/__init__.py index e246ef22..a9efe45e 100644 --- a/kadi/events/__init__.py +++ b/kadi/events/__init__.py @@ -49,7 +49,12 @@ # For WSGI server the env var is set in wsgi.py. # For the dev server it is set in manage.py. +# In addition, set DJANGO_ALLOW_ASYNC_UNSAFE, to avoid exception seen running in +# Jupyter notebook: SynchronousOnlyOperation: You cannot call this from an async +# context. See: https://stackoverflow.com/questions/59119396 + if 'DJANGO_SETTINGS_MODULE' not in os.environ: os.environ['DJANGO_SETTINGS_MODULE'] = 'kadi.settings' + os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true" django.setup() from .query import * # noqa diff --git a/kadi/update_cmds.py b/kadi/update_cmds.py index 4ff9f4fe..7f0aeaf3 100644 --- a/kadi/update_cmds.py +++ b/kadi/update_cmds.py @@ -12,7 +12,6 @@ import Ska.DBI import Ska.File from Chandra.Time import DateTime -from Chandra.cmd_states.cmd_states import _tl_to_bs_cmds from ska_helpers.run_info import log_run_info from .paths import IDX_CMDS_PATH, PARS_DICT_PATH @@ -116,6 +115,39 @@ def fix_nonload_cmds(nl_cmds): return new_cmds +def _tl_to_bs_cmds(tl_cmds, tl_id, db): + """ + Convert the commands ``tl_cmds`` (numpy recarray) that occur in the + timeline ``tl_id'' to a format mimicking backstop commands from + Ska.ParseCM.read_backstop(). This includes reading parameter values + from the ``db``. + + :param tl_cmds: numpy recarray of commands from timeline load segment + :param tl_id: timeline id + :param db: Ska.DBI db object + + :returns: list of command dicts + """ + bs_cmds = [dict((col, row[col]) for col in tl_cmds.dtype.names) + for row in tl_cmds] + cmd_index = dict((x['id'], x) for x in bs_cmds) + + # Add 'params' dict of command parameter key=val pairs to each tl_cmd + for par_table in ('cmd_intpars', 'cmd_fltpars'): + tl_params = db.fetchall("SELECT * FROM %s WHERE timeline_id %s" % + (par_table, + '= %d' % tl_id if tl_id else 'IS NULL')) + + # Build up the params dict for each command in timeline load segment + for par in tl_params: + # I.e. cmd_index[par.cmd_id]['params'][par.name] = par.value + # but create the ['params'] dict as needed. + if par.cmd_id in cmd_index: + cmd_index[par.cmd_id].setdefault('params', {})[par.name] = par.value + + return bs_cmds + + def get_cmds(start, stop, mp_dir='/data/mpcrit1/mplogs'): """ Get backstop commands corresponding to the supplied timeline load segments. diff --git a/setup.py b/setup.py index 97339d7a..d5179893 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,8 @@ package_data={'kadi.events': ['templates/*/*.html', 'templates/*.html'], 'kadi': foundation_files + ['templates/*/*.html', 'templates/*.html', 'static/images/*', 'static/*.css', - 'GIT_VERSION']}, + 'GIT_VERSION'], + 'kadi.commands.tests': ['data/*.ecsv.gz']}, tests_require=['pytest'], data_files=data_files, cmdclass=cmdclass,