diff --git a/kadi/commands/command_sets.py b/kadi/commands/command_sets.py index 6be01c18..c74f262f 100644 --- a/kadi/commands/command_sets.py +++ b/kadi/commands/command_sets.py @@ -1,9 +1,9 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -import re from pathlib import Path import astropy.units as u from cxotime import CxoTime +from parse_cm.backstop import parse_backstop_params from Quaternion import Quat from ska_helpers.utils import convert_to_int_float_str @@ -173,22 +173,17 @@ def cmd_set_hrc_not_run(load_name, date=None): def cmd_set_command(*args, date=None): - params_str = args[0] - cmd_type, args_str = params_str.split("|", 1) - cmd = {"type": cmd_type.strip().upper()} + """Parse Command or Command not run params string and return a command dict. - # Strip spaces around equals signs and uppercase args (note that later the - # keys are lowercased). - args_str = re.sub(r"\s*=\s*", "=", args_str).upper() + The format follows Backstop ``" | PARAM1=VAL1, PARAM2=VAL2, .."``. + This code follows the key steps in parse_cm.backstop.read_backstop_as_list(). + """ + params_str = args[0].strip().replace(" ", "").upper() - params = {} - for param in args_str.split(): - key, val = param.split("=") - if key == "TLMSID": - cmd["tlmsid"] = val - else: - params[key] = convert_to_int_float_str(val) - cmd["params"] = params + cmd_type, args_str = params_str.split("|", 1) + params = parse_backstop_params(args_str) + tlmsid = params.pop("tlmsid", "None") + cmd = {"type": cmd_type, "tlmsid": tlmsid, "params": params} return (cmd,) @@ -204,6 +199,8 @@ def cmd_set_end_scs(*args, date=None): def cmd_set_command_not_run(*args, date=None): (cmd,) = cmd_set_command(*args, date=date) + # Save original type which gets used later in CommandTable.remove_not_run_cmds(). + cmd["params"]["__type__"] = cmd["type"] cmd["type"] = "NOT_RUN" return (cmd,) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 1a90b35c..b48c4264 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -402,7 +402,7 @@ def update_archive_and_get_cmds_recent( start = CxoTime(min(loads["cmd_start"])) stop = CxoTime(max(loads["cmd_stop"])) # Allow for variations in input format of date - dates = np.array([CxoTime(date).date for date in cmd_events["Date"]]) + dates = np.array([CxoTime(date).date for date in cmd_events["Date"]], dtype=str) bad = (dates < (start - 14 * u.day).date) | (dates > stop.date) cmd_events = cmd_events[~bad] cmd_events_ids = [evt["Event"] + " at " + evt["Date"] for evt in cmd_events] diff --git a/kadi/commands/core.py b/kadi/commands/core.py index fead77bb..c2e9c57c 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -868,15 +868,44 @@ def remove_not_run_cmds(self): the "Command not run" event in the Command Events sheet, e.g. the LETG retract command in the loads after the LETG insert anomaly. """ - idxs_remove = set() idxs_not_run = np.where(self["type"] == "NOT_RUN")[0] + if len(idxs_not_run) == 0: + return + + idxs_remove = set() for idx in idxs_not_run: - cmd = self[idx] - ok = (self["date"] == cmd["date"]) & (self["tlmsid"] == cmd["tlmsid"]) - idxs_remove.update(np.where(ok)[0]) - if idxs_remove: - logger.info(f"Removing {len(idxs_remove)} NOT_RUN cmds") - self.remove_rows(list(idxs_remove)) + cmd_not_run = self[idx] + ok = ( + (self["date"] == cmd_not_run["date"]) + & (self["type"] == cmd_not_run["__type__"]) + & (self["tlmsid"] == cmd_not_run["tlmsid"]) + ) + for key in ("scs", "step"): + if cmd_not_run[key] != 0: + ok &= self[key] == cmd_not_run[key] + + # Indexes of self commands that *might* match cmd_not_run. + idxs = np.arange(len(self))[ok].tolist() + + # Now check that the params match. + idxs_match = [] + for idx in idxs: + # Get the intersection of the keys in cmd_not_run["params"] and self["params"][idx] + self_params = self["params"][idx] + cmd_not_run_params = cmd_not_run["params"] + keys = set(cmd_not_run_params) & set(self_params) + + # Check that the values match for all common keys + match = all(cmd_not_run_params[key] == self_params[key] for key in keys) + if match: + idxs_match.append(idx) + + idxs_remove.update(idxs_match) + + logger.info(f"Removing {len(idxs_remove)} NOT_RUN cmds") + for idx in sorted(idxs_remove, reverse=True): + logger.debug(f" {self[idx]}") + self.remove_rows(list(idxs_remove) + list(idxs_not_run)) def get_par_idx_update_pars_dict(pars_dict, cmd, params=None, rev_pars_dict=None): diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index dfb732bb..ea7eee0d 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -892,9 +892,9 @@ def test_get_starcats_as_table(): @pytest.mark.parametrize( "par_str", [ - "ACISPKT| TLmSID= aa0000000 par1 = 1 par2=-1.0", - "AcisPKT|TLmSID=AA0000000 par1=1 par2=-1.0", - "ACISPKT| TLmSID = aa0000000 par1 =1 par2 = -1.0", + "ACISPKT| TLmSID= aa0000000, par1 = 1 , par2=-1.0", + "AcisPKT|TLmSID=AA0000000 ,par1=1, par2=-1.0", + "ACISPKT| TLmSID = aa0000000 , par1 =1, par2 = -1.0", ], ) def test_get_cmds_from_event_case(par_str): @@ -914,8 +914,8 @@ def test_get_cmds_from_event_case(par_str): Event,Params Observing not run,FEB1422A Load not run,OCT2521A - Command,ACISPKT | TLMSID=AA00000000 - Command not run,COMMAND_SW | TLMSID=4OHETGIN + Command,"ACISPKT | TLMSID= AA00000000, CMDS= 3, WORDS= 3, PACKET(40)= D80000300030603001300" + Command not run,"COMMAND_SW | TLMSID=4OHETGIN, HEX= 8050300, MSID= 4OHETGIN" RTS,"RTSLOAD,1_4_CTI,NUM_HOURS=39:00:00,SCS_NUM=135" Obsid,65527 Maneuver,0.70546907 0.32988307 0.53440901 0.32847766 @@ -934,10 +934,10 @@ def test_get_cmds_from_event_case(par_str): "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2020:001:00:00:00, event_type=LOAD_NOT_RUN, load=OCT2521A, scs=0" # noqa ], [ - "2020:001:00:00:00.000 | ACISPKT | AA00000000 | CMD_EVT | event=Command, event_date=2020:001:00:00:00, scs=0" # noqa + "2020:001:00:00:00.000 | ACISPKT | AA00000000 | CMD_EVT | event=Command, event_date=2020:001:00:00:00, cmds=3, words=3, scs=0" # noqa ], [ - "2020:001:00:00:00.000 | NOT_RUN | 4OHETGIN | CMD_EVT | event=Command_not_run, event_date=2020:001:00:00:00, scs=0" # noqa + "2020:001:00:00:00.000 | NOT_RUN | 4OHETGIN | CMD_EVT | event=Command_not_run, event_date=2020:001:00:00:00, hex=8050300, msid=4OHETGIN, __type__=COMMAND_SW, scs=0" # noqa ], [ "2020:001:00:00:00.000 | COMMAND_SW | OORMPEN | CMD_EVT | event=RTS," @@ -1417,3 +1417,95 @@ def test_hrc_not_run_scenario(stop_date_2023200): # noqa: ARG001 assert states_out == states_exp commands_v2.clear_caches() + + +test_command_not_run_cases = [ + { + # Matches multiple commands + "event": { + "date": "2023:351:13:30:33.849", + "event": "Command not run", + "params_str": "COMMAND_SW | TLMSID= COACTSX", + }, + "removed": [3, 4], + }, + { + # Matches one command with multiple criteria + "event": { + "date": "2023:351:13:30:33.849", + "event": "Command not run", + "params_str": ( + "COMMAND_SW | TLMSID= COACTSX, HEX= 840B100, " + "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 690" + ), + }, + "removed": [3], + }, + { + # Wrong TLMSID + "event": { + "date": "2023:351:13:30:33.849", + "event": "Command not run", + "params_str": ( + "COMMAND_SW | TLMSID= XXXXXXX, HEX= 840B100, " + "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 690" + ), + }, + "removed": [], + }, + { + # Wrong SCS + "event": { + "date": "2023:351:13:30:33.849", + "event": "Command not run", + "params_str": ( + "COMMAND_SW | TLMSID= XXXXXXX, HEX= 840B100, " + "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 133, STEP= 690" + ), + }, + "removed": [], + }, + { + # Wrong Step + "event": { + "date": "2023:351:13:30:33.849", + "event": "Command not run", + "params_str": ( + "COMMAND_SW | TLMSID= XXXXXXX, HEX= 840B100, " + "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 111" + ), + }, + "removed": [], + }, + { + # No TLMSID + "event": { + "date": "2023:351:19:38:41.550", + "event": "Command not run", + "params_str": "SIMTRANS | POS= 92904, SCS= 131, STEP= 1191", + }, + "removed": [6], + }, +] + + +@pytest.mark.parametrize("case", test_command_not_run_cases) +def test_command_not_run(case): + backstop_text = """ +2023:351:13:30:32.824 | 0 0 | COMMAND_SW | TLMSID= AOMANUVR, HEX= 8034101, MSID= AOMANUVR, SCS= 128, STEP= 686 +2023:351:13:30:33.849 | 1 0 | COMMAND_SW | TLMSID= AOACRSTE, HEX= 8032001, MSID= AOACRSTE, SCS= 128, STEP= 688 +2023:351:13:30:33.849 | 2 0 | COMMAND_SW | TLMSID= COENASX, HEX= 844B100, MSID= COENASX, COENAS1=177 , SCS= 128, STEP= 689 +2023:351:13:30:33.849 | 3 0 | COMMAND_SW | TLMSID= COACTSX, HEX= 840B100, MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 690 +2023:351:13:30:33.849 | 4 0 | COMMAND_SW | TLMSID= COACTSX, HEX= 8402600, MSID= COACTSX, COACTS1=38 , COACTS2=0 , SCS= 128, STEP= 691 +2023:351:13:30:55.373 | 5 0 | COMMAND_HW | TLMSID= 4MC5AEN, HEX= 4800012, MSID= 4MC5AEN, SCS= 131, STEP= 892 +2023:351:19:38:41.550 | 6 0 | SIMTRANS | POS= 92904, SCS= 131, STEP= 1191 + """ # noqa + cmds = commands.read_backstop(backstop_text.strip().splitlines()) + cmds["source"] = "DEC1123A" + cmds_exp = cmds.copy() + cmds_exp.remove_rows(case["removed"]) + cmds_from_event = get_cmds_from_event(**case["event"]) + cmds_with_event = cmds.add_cmds(cmds_from_event) + cmds_with_event.sort_in_backstop_order() + cmds_with_event.remove_not_run_cmds() + assert cmds_with_event.pformat_like_backstop() == cmds_exp.pformat_like_backstop() diff --git a/utils/make_hrc_disable_events.py b/utils/make_hrc_disable_events.py new file mode 100644 index 00000000..58951f83 --- /dev/null +++ b/utils/make_hrc_disable_events.py @@ -0,0 +1,129 @@ +"""Create a scenario to handle HRC commands not run to to HRC being disabled. + +This removes HRC state-impacting commands:: + + {"tlmsid": "224PCAON"}, + {"tlmsid": "215PCAON"}, + {"tlmsid": "COACTSX", "coacts1": 134}, + {"tlmsid": "COENASX", "coenas1": 89}, + {"tlmsid": "COENASX", "coenas1": 90}, + +In practice the hardware commands are not in loads since the HRC return to science. + +This script creates a scenario ~/.kadi/ that can be used to remove these +commands from the flight kadi commands database. It then tests that by setting +``KADI_SCENARIO=`` and getting kadi states over the time period of interest. + +The simplest way to do import to the flight sheet is to import the CSV file into a +temporary Google Sheet, then copy/paste that table into the flight Chandra Command +Events Google Sheet. Remember to create empty rows for the copy/paste. +""" + +import argparse + +import kadi.paths +from astropy import table +from cxotime import CxoTime +from kadi.commands import get_cmds +from kadi.commands.core import CommandTable +from kadi.commands.states import get_states +from ska_helpers.utils import temp_env_var + + +# When was F_HRC_SAFING script run in late 2023 +hrc_safing_date = "2023:343:02:00:00" + + +def get_parser(): + parser = argparse.ArgumentParser( + description="Print HRC state-impacting commands that were not run due to HRC being disabled." + ) + parser.add_argument( + "--start", + default=hrc_safing_date, + help=f"Start time for searching for commands (default={hrc_safing_date}))", + ) + parser.add_argument( + "--stop", + help="Stop time for searching for commands (default=NOW)", + ) + parser.add_argument( + "--status", + default="Definitive", + help="Status of command events (Definitive or In-work)", + ) + parser.add_argument( + "--scenario", + default="hrc_disable", + help="Scenario name (default=hrc_disable). This creates ~/.kadi//cmd_events.csv.", + ) + return parser + + +def main(): + opt = get_parser().parse_args() + make_cmd_events(opt) + test_cmd_events(opt) + + +def test_cmd_events(opt): + def get_states_local(): + states = get_states( + start=opt.start, + stop=opt.stop, + state_keys=["hrc_15v", "hrc_24v", "hrc_i", "hrc_s"], + merge_identical=True, + ) + return states + + print("Current flight states:") + get_states_local().pprint_all() + print() + print(f"States with HRC disable scenario {opt.scenario}:") + with temp_env_var("KADI_SCENARIO", opt.scenario): + get_states_local().pprint_all() + + +def make_cmd_events(opt): + cmd_kwargs_list = [ + {"tlmsid": "224PCAON"}, + {"tlmsid": "215PCAON"}, + {"tlmsid": "COACTSX", "coacts1": 134}, + {"tlmsid": "COENASX", "coenas1": 89}, + {"tlmsid": "COENASX", "coenas1": 90}, + ] + + rows = [] + start = CxoTime(opt.start) + stop = CxoTime(opt.stop) + + for cmd_kwargs in cmd_kwargs_list: + cmds: CommandTable = get_cmds(start=start, stop=stop, **cmd_kwargs) + print(f"{len(cmds)} cmd(s) found for {cmd_kwargs}") + params_str = ", ".join([f"{k.upper()}={v}" for k, v in cmd_kwargs.items()]) + for cmd in cmds: + row = ( + opt.status, + cmd["date"], + "Command not run", + f"{cmd['type']} | {params_str}", + "Tom Aldcroft", + "Jean Connelly", + f"Not run due to F_HRC_SAFING at {hrc_safing_date}", + ) + rows.append(row) + + names = "State Date Event Params Author Reviewer Comment".split() + cmd_events = table.Table(rows=rows, names=names) + cmd_events.sort("Date", reverse=True) + cmd_events.pprint_all() + + cmd_events_path = kadi.paths.CMD_EVENTS_PATH(opt.scenario) + cmd_events_path.parent.mkdir(parents=True, exist_ok=True) + print() + print(f"Writing {len(cmd_events)} events to {cmd_events_path}") + cmd_events.write(cmd_events_path, format="ascii.csv", overwrite=True) + + +if __name__ == "__main__": + main()