From ff7a3dead12f1ed0196aeecf2805c13fd75c4059 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 22 Aug 2022 20:27:44 -0400 Subject: [PATCH 01/16] Fix bug where RLTT from not-run load is still applied --- kadi/commands/commands_v2.py | 12 +++++++++-- kadi/commands/tests/test_commands.py | 31 +++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 4db31800..e55005e0 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -328,7 +328,10 @@ def update_archive_and_get_cmds_recent( # Command Events sheet). cmds = interrupt_load_commands(load, cmds) if len(cmds) > 0: - logger.info(f'Load {load["name"]} has {len(cmds)} commands') + logger.info( + f'Load {load["name"]} has {len(cmds)} commands' + f" with rltt {load['rltt']}" + ) cmds_list.append(cmds) rltts.append(load["rltt"]) else: @@ -373,7 +376,7 @@ def update_archive_and_get_cmds_recent( for jj in range(0, ii): prev_cmds = cmds_list[jj] # First check for any overlap since prev_cmds is sorted by date. - if prev_cmds["date"][-1] > rltt: + if len(prev_cmds) > 0 and prev_cmds["date"][-1] > rltt: # Cut commands EXCEPT for ORBPOINT ones, which we leave as a # special case to ensure availability of orbit events from # commands. Otherwise RLTT can cut ORBPOINT commands that @@ -986,8 +989,13 @@ def get_load_dict_from_cmds(load_name, cmds, cmd_events): logger.info( f'{cmd_event["Event"]} at {cmd_event_date} found for {load_name}' ) + # Stop the loads before they start load["observing_stop"] = "1998:001" load["vehicle_stop"] = "1998:001" + # Not-run load does not terminate other loads or commands or have a + # scheduled stop time. + load["rltt"] = None + load["scheduled_stop_time"] = None if ( cmd_event["Event"] == "Observing not run" diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index 59990da9..2f546ed2 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -1047,6 +1047,9 @@ def test_scenario_with_rts(monkeypatch): 2021:297:23:00:42.886 | ORBPOINT | None | OCT2521A | event_type=XQF005M, scs=0 2021:298:00:12:42.886 | ORBPOINT | None | OCT2521A | event_type=XQF003M, scs=0 2021:298:00:12:42.886 | ORBPOINT | None | OCT2521A | event_type=XQF013M, scs=0 +2021:298:01:01:39.000 | COMMAND_HW | 2S2HVOF | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, msid=2S2HVOF, scs=1 +2021:298:01:01:39.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, msid=OORMPDS, scs=1 +2021:298:01:01:40.000 | COMMAND_HW | 2S2STHV | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, 2s2sthv2=0 , msid=2 2021:298:01:57:00.000 | LOAD_EVENT | None | OCT2521B | event_type=RUNNING_LOAD_TERMINATION_TIME, scs=0 2021:298:01:57:00.000 | COMMAND_SW | AOACRSTD | OCT2521B | msid=AOACRSTD, scs=128 2021:298:01:57:00.000 | ACISPKT | AA00000000 | OCT2521B | cmds=3, words=3, scs=131 @@ -1062,4 +1065,30 @@ def test_scenario_with_rts(monkeypatch): # TODO: (someday?) instead of the RLTT notice the disable SCS 135 command # CODISASX. ok = cmds["event"] == "RTS" - assert np.count_nonzero(ok) == 11 + assert np.count_nonzero(ok) == 14 + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_no_rltt_for_not_run_load(): + """The AUG2122A loads were never run but they contain an RLTT that had been + stopping the 2022:232:03:09 ACIS ECS that was previously running. This tests + the fix. + """ + exp = [ + " date tlmsid scs", + "--------------------- ---------- ---", + "2022:232:03:09:00.000 AA00000000 135", + "2022:232:03:09:04.000 WSPOW00000 135", + "2022:232:03:09:28.000 WSPOW08E1E 135", + "2022:232:03:10:31.000 WT00C62014 135", + "2022:232:03:10:35.000 XTZ0000005 135", + "2022:232:03:10:39.000 RS_0000001 135", + "2022:232:03:10:43.000 RH_0000001 135", + "2022:233:18:10:43.000 AA00000000 135", # <== After the AUG2122A RLTT + "2022:233:18:10:53.000 AA00000000 135", + "2022:233:18:10:57.000 WSPOW0002A 135", + "2022:233:18:12:00.000 RS_0000001 135", + ] + cmds = commands_v2.get_cmds("2022:232:03:00:00", "2022:233:18:30:00") + cmds = cmds[cmds["type"] == "ACISPKT"] + assert cmds["date", "tlmsid", "scs"].pformat() == exp From c2a5b4c1e0d0bae2952c467ce849f9cd3f6f857e Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 23 Aug 2022 10:25:23 -0400 Subject: [PATCH 02/16] Add command sets to interrupt SCS, vehicle, observing --- kadi/commands/command_sets.py | 46 +++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/kadi/commands/command_sets.py b/kadi/commands/command_sets.py index e03613b0..7b814e32 100644 --- a/kadi/commands/command_sets.py +++ b/kadi/commands/command_sets.py @@ -59,6 +59,16 @@ def cmd_set_aciscti(date=None): ) +def cmd_set_end_vehicle(date=None): + cmds = cmd_set_end_scs(128) + cmd_set_end_scs(129) + cmd_set_end_scs(130) + return cmds + + +def cmd_set_end_observing(date=None): + cmds = cmd_set_end_scs(131) + cmd_set_end_scs(132) + cmd_set_end_scs(133) + return cmds + + def cmd_set_scs107(date=None): # SCS-106 (which is called by 107) was patched around 2021-Jun-08 if date is not None and date > CxoTime("2021-06-08").date: @@ -66,7 +76,8 @@ def cmd_set_scs107(date=None): else: pow_cmd = "WSPOW00000" # 0-FEPS - return ( + cmds = cmd_set_end_observing() + cmds += ( dict(type="COMMAND_SW", dur=1.025, tlmsid="OORMPDS"), dict( type="COMMAND_HW", @@ -79,6 +90,7 @@ def cmd_set_scs107(date=None): dict(type="ACISPKT", tlmsid="AA00000000", dur=10.25), dict(type="ACISPKT", tlmsid=pow_cmd), ) + return cmds def cmd_set_dither(state, date=None): @@ -94,13 +106,16 @@ def cmd_set_dither(state, date=None): def cmd_set_bright_star_hold(date=None): - out = cmd_set_scs107(date=date) + out = cmd_set_end_vehicle() + out += cmd_set_scs107(date=date) return out def cmd_set_nsm(date=None): nsm_cmd = dict(type="COMMAND_SW", tlmsid="AONSMSAF") - out = (nsm_cmd,) + cmd_set_scs107(date=date) + cmd_set_dither("OFF", date=date) + out = (nsm_cmd,) + out += cmd_set_end_vehicle() + out += cmd_set_scs107(date=date) return out @@ -118,11 +133,19 @@ def cmd_set_safe_mode(date=None): def cmd_set_load_not_run(load_name, date=None): - return None + cmd = { + "type": "LOAD_EVENT", + "params": {"TYPE": "LOAD_NOT_RUN", "LOAD": load_name}, + } + return (cmd,) def cmd_set_observing_not_run(load_name, date=None): - return None + cmd = { + "type": "LOAD_EVENT", + "params": {"TYPE": "OBSERVING_NOT_RUN", "LOAD": load_name}, + } + return (cmd,) def cmd_set_command(*args, date=None): @@ -146,6 +169,15 @@ def cmd_set_command(*args, date=None): return (cmd,) +def cmd_set_end_scs(*args, date=None): + cmd = {"type": "COMMAND_SW", "tlmsid": "CODISASX"} + cmd["params"] = { + "MSID": "CODISASX", + "CODISAS1": args[0], + } + return (cmd,) + + def cmd_set_command_not_run(*args, date=None): (cmd,) = cmd_set_command(*args, date=date) cmd["type"] = "NOT_RUN" @@ -167,9 +199,7 @@ def get_cmds_from_event(date, event, params_str): raise ValueError(f"unknown event {event!r}") if isinstance(params_str, str): - if event == "RTS": - args = [params_str] - elif event in ("Command", "Command not run"): + if event in ("RTS", "Command", "Command not run"): # Delegate parsing to cmd_set_command args = [params_str] else: From 8851275c333ed07b9f39729d61c8dd37b9d64322 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 23 Aug 2022 11:20:51 -0400 Subject: [PATCH 03/16] Revert removing NSM dither disable --- kadi/commands/command_sets.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kadi/commands/command_sets.py b/kadi/commands/command_sets.py index 7b814e32..450b04cf 100644 --- a/kadi/commands/command_sets.py +++ b/kadi/commands/command_sets.py @@ -116,6 +116,9 @@ def cmd_set_nsm(date=None): out = (nsm_cmd,) out += cmd_set_end_vehicle() out += cmd_set_scs107(date=date) + # Disable dither. Looking at a few NSMs this seems to be the case, but + # not for the 2022:232:18:36 NSM from a CTU reset. ?? + out += cmd_set_dither("OFF", date=date) return out From f0e0b2b17cf683b099818295e95309dd719132ea Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 23 Aug 2022 11:21:12 -0400 Subject: [PATCH 04/16] Bit of type annotation --- kadi/commands/commands_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index e55005e0..b75f0aea 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -950,7 +950,7 @@ def clean_loads_dir(loads): file.unlink() -def get_load_dict_from_cmds(load_name, cmds, cmd_events): +def get_load_dict_from_cmds(load_name: str, cmds: CommandTable, cmd_events: Table): """Update ``load`` dict in place from the backstop commands.""" vehicle_stop_events = ("NSM", "Safe mode", "Bright star hold") observing_stop_events = vehicle_stop_events + ("SCS-107",) From dbd827b4ff3043216d22ec9fc415051789b6b938 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 23 Aug 2022 11:21:50 -0400 Subject: [PATCH 05/16] Fix one test --- kadi/commands/tests/test_commands.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index 2f546ed2..2a2364cb 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -423,8 +423,23 @@ def test_get_cmds_nsm_2021(stop_date_2021_10_24): "hex=7800056, msid=CPABON, scs=128", "2021:296:10:35:01.285 | COMMAND_HW | CTXBON | OCT1821A | " "hex=7800044, msid=CTXBON, scs=128", + "2021:296:10:41:57.000 | LOAD_EVENT | None | CMD_EVT | " + "event=Load_not_run, event_date=2021:296:10:41:57, type=LOAD_NOT_RUN, " + "load=OCT2521A, scs=0", "2021:296:10:41:57.000 | COMMAND_SW | AONSMSAF | CMD_EVT | " "event=NSM, event_date=2021:296:10:41:57, scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=128 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=129 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=130 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=131 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=132 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=133 , scs=0", "2021:296:10:41:57.000 | COMMAND_SW | OORMPDS | CMD_EVT | " "event=NSM, event_date=2021:296:10:41:57, scs=0", "2021:296:10:41:58.025 | COMMAND_HW | AFIDP | CMD_EVT | " From 0524f1c5a401953b4f0452039dba36820e179dec Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 23 Aug 2022 13:03:34 -0400 Subject: [PATCH 06/16] Fix tests --- kadi/commands/tests/test_commands.py | 117 ++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 4 deletions(-) diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index 2a2364cb..a00de7f5 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -286,8 +286,28 @@ def test_commands_create_archive_regress(tmpdir, version_env): assert len(cmds_empty) == 0 cmds_local = commands.get_cmds(start + 3, stop - 3) + + # FIXME: workaround that flight archive does not have these non-load + # commands added in #248. If flight archive is regenerated, this + # should be removed. + ignore = ( + (cmds_local["type"] == "LOAD_EVENT") + & (cmds_local["source"] == "CMD_EVT") + & (cmds_local["tlmsid"] == "None") + ) + ignore |= (cmds_local["tlmsid"] == "CODISASX") & ( + cmds_local["source"] == "CMD_EVT" + ) + cmds_local = cmds_local[~ignore] + cmds_local.fetch_params() - assert len(cmds_flight) == len(cmds_local) + if len(cmds_flight) != len(cmds_local): + for ii, (cmd_flight, cmd_local) in enumerate( + zip(cmds_flight, cmds_local) + ): + if cmd_flight["tlmsid"] != cmd_local["tlmsid"]: + err = f"First mismatch at line {ii}\n{cmd_flight}\n{cmd_local}" + raise AssertionError(err) # 'starcat_idx' param in OBS cmd does not match since the pickle files # are different, so remove it. @@ -517,6 +537,24 @@ def test_cmds_scenario(stop_date_2020_12_03): def test_command_set_bsh(): cmds = get_cmds_from_event("2000:001", "Bright star hold", "") exp = [ + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " + "codisas1=12", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " + "codisas1=12", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " + "codisas1=13", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " + "codisas1=13", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " + "codisas1=13", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " + "codisas1=13", "2000:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | " "event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0", "2000:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | " @@ -544,6 +582,24 @@ def test_command_set_safe_mode(): "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", "2000:001:00:00:00.000 | COMMAND_SW | AONSMSAF | CMD_EVT | " "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=128 , " + "scs", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=129 , " + "scs", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=130 , " + "scs", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=131 , " + "scs", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=132 , " + "scs", + "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=133 , " + "scs", "2000:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | " "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", "2000:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | " @@ -595,6 +651,24 @@ def test_bright_star_hold_event(cmds_dir, stop_date_2020_12_03): "2020:336:21:55:23.061 | COMMAND_SW | AOFUNCEN | NOV3020A | " "hex=8030315, msid=AOFUNCEN, aopcadse=21 , scs=128", # BSH interrupt at 2020:337 + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=12", + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=12", + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=13", + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=13", + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=13", + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=13", "2020:337:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | " "event=Bright_star_hold, event_date=2020:337:00:00:00, scs=0", "2020:337:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | " @@ -876,8 +950,12 @@ def test_get_cmds_from_event_case(par_str): """ cmd_events_all = Table.read(cmd_events_all_text, format="ascii.csv") cmd_events_all_exps = [ - None, - None, + [ + "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Observing_not_run, event_date=2020:001:00:00:00, type=OBSERVING_NOT_RUN, load=FEB1422A, scs=0" # noqa + ], + [ + "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2020:001:00:00:00, type=LOAD_NOT_RUN, load=OCT2521A, scs=0" # noqa + ], [ "2020:001:00:00:00.000 | ACISPKT | AA00000000 | CMD_EVT | event=Command, event_date=2020:001:00:00:00, scs=0" # noqa ], @@ -917,6 +995,12 @@ def test_get_cmds_from_event_case(par_str): "2020:001:00:00:00.000 | COMMAND_SW | ACPCSFSU | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, scs=0", # noqa "2020:001:00:00:00.000 | COMMAND_SW | CSELFMT5 | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, scs=0", # noqa "2020:001:00:00:00.000 | COMMAND_SW | AONSMSAF | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=128 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=129 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=130 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=131 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=132 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=133 , scs=0", # noqa "2020:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, scs=0", # noqa "2020:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, msid=AFLCRSET, scs=0", # noqa "2020:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | event=Safe_mode, event_date=2020:001:00:00:00, pos=-99616, scs=0", # noqa @@ -927,6 +1011,12 @@ def test_get_cmds_from_event_case(par_str): ], # noqa [ "2020:001:00:00:00.000 | COMMAND_SW | AONSMSAF | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=128 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=129 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=130 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=131 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=132 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=133 , scs=0", # noqa "2020:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, scs=0", # noqa "2020:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, msid=AFLCRSET, scs=0", # noqa "2020:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, pos=-99616, scs=0", # noqa @@ -936,6 +1026,9 @@ def test_get_cmds_from_event_case(par_str): "2020:001:00:01:17.960 | COMMAND_SW | AODSDITH | CMD_EVT | event=NSM, event_date=2020:001:00:00:00, scs=0", # noqa ], # noqa [ + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=SCS-107, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=131 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=SCS-107, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=132 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=SCS-107, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=133 , scs=0", # noqa "2020:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=SCS-107, event_date=2020:001:00:00:00, scs=0", # noqa "2020:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | event=SCS-107, event_date=2020:001:00:00:00, msid=AFLCRSET, scs=0", # noqa "2020:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | event=SCS-107, event_date=2020:001:00:00:00, pos=-99616, scs=0", # noqa @@ -944,6 +1037,12 @@ def test_get_cmds_from_event_case(par_str): "2020:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=SCS-107, event_date=2020:001:00:00:00, scs=0", # noqa ], # noqa [ + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=128 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=129 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=130 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=131 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=132 , scs=0", # noqa + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=133 , scs=0", # noqa "2020:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=Bright_star_hold, event_date=2020:001:00:00:00, scs=0", # noqa "2020:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | event=Bright_star_hold, event_date=2020:001:00:00:00, msid=AFLCRSET, scs=0", # noqa "2020:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | event=Bright_star_hold, event_date=2020:001:00:00:00, pos=-99616, scs=0", # noqa @@ -1016,7 +1115,14 @@ def test_scenario_with_rts(monkeypatch): 2021:296:10:35:00.771 | COMMAND_HW | CTXBOF | OCT1821A | msid=CTXBOF, scs=128 2021:296:10:35:01.028 | COMMAND_HW | CPABON | OCT1821A | msid=CPABON, scs=128 2021:296:10:35:01.285 | COMMAND_HW | CTXBON | OCT1821A | msid=CTXBON, scs=128 +2021:296:10:41:57.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2021:296:10:41:57, type=LOAD_ 2021:296:10:41:57.000 | COMMAND_SW | AONSMSAF | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi 2021:296:10:41:57.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 2021:296:10:41:58.025 | COMMAND_HW | AFIDP | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=AFLCRSET, scs= 2021:296:10:41:58.025 | SIMTRANS | None | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, pos=-99616, scs=0 @@ -1083,8 +1189,11 @@ def test_scenario_with_rts(monkeypatch): assert np.count_nonzero(ok) == 14 +stop_date_2022_236 = stop_date_fixture_factory("2022:236") + + @pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") -def test_no_rltt_for_not_run_load(): +def test_no_rltt_for_not_run_load(stop_date_2022_236): """The AUG2122A loads were never run but they contain an RLTT that had been stopping the 2022:232:03:09 ACIS ECS that was previously running. This tests the fix. From 67cb28b58916bffb81097f828ecca9decab83693 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 24 Aug 2022 06:42:46 -0400 Subject: [PATCH 07/16] Major refactor/improvement in command interrupt handling --- kadi/commands/command_sets.py | 4 +- kadi/commands/commands_v2.py | 107 +++++++++++++++++++-------- kadi/commands/tests/test_commands.py | 37 +++++---- 3 files changed, 99 insertions(+), 49 deletions(-) diff --git a/kadi/commands/command_sets.py b/kadi/commands/command_sets.py index 450b04cf..1945605d 100644 --- a/kadi/commands/command_sets.py +++ b/kadi/commands/command_sets.py @@ -138,7 +138,7 @@ def cmd_set_safe_mode(date=None): def cmd_set_load_not_run(load_name, date=None): cmd = { "type": "LOAD_EVENT", - "params": {"TYPE": "LOAD_NOT_RUN", "LOAD": load_name}, + "params": {"EVENT_TYPE": "LOAD_NOT_RUN", "LOAD": load_name}, } return (cmd,) @@ -146,7 +146,7 @@ def cmd_set_load_not_run(load_name, date=None): def cmd_set_observing_not_run(load_name, date=None): cmd = { "type": "LOAD_EVENT", - "params": {"TYPE": "OBSERVING_NOT_RUN", "LOAD": load_name}, + "params": {"EVENT_TYPE": "OBSERVING_NOT_RUN", "LOAD": load_name}, } return (cmd,) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index b75f0aea..a8fa6af6 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -87,6 +87,7 @@ def clear_caches(): OBSERVATIONS.clear() +# NOT USED! def interrupt_load_commands(load, cmds): """Cut commands beyond observing or vehicle stop times. @@ -198,7 +199,9 @@ def get_matching_block_idx_simple(cmds_recent, cmds_arch, min_match): return i0_arch -def get_cmds(start=None, stop=None, inclusive_stop=False, scenario=None, **kwargs): +def get_cmds( + start=None, stop=None, inclusive_stop=False, scenario=None, **kwargs +) -> CommandTable: """Get commands using loads table, relying entirely on RLTT. :param start: CxoTime-like @@ -307,35 +310,53 @@ def update_archive_and_get_cmds_recent( :returns: CommandTable """ - cmds_list = [] # List of CommandTable objects from loads and cmd_events - rltts = [] # Corresponding list of RLTTs, where cmd_events use None for RLTT + # List of CommandTable objects from loads and cmd_events + cmds_list = [] # Update local cmds_events.csv from Google Sheets cmd_events = update_cmd_events(scenario) + # Get load names that were not run at all or where observing was not run. + # E.g. an SCS-107 near end of loads where next week vehicle loads only were + # uplinked. + not_run_loads = {} + for not_run_type in ("Load", "Observing"): + not_run_loads[not_run_type] = set( + cmd_event["Params"] + for cmd_event in cmd_events + if cmd_event["Event"] == f"{not_run_type} not run" + ) + # Update loads table and download/archive backstop files from OCCweb loads = update_loads(scenario, cmd_events=cmd_events, lookback=lookback, stop=stop) logger.info(f'Including loads {", ".join(loads["name"])}') for load in loads: - loads_backstop_path = paths.LOADS_BACKSTOP_PATH(load["name"]) + load_name = load["name"] + loads_backstop_path = paths.LOADS_BACKSTOP_PATH(load_name) with gzip.open(loads_backstop_path, "rb") as fh: cmds = pickle.load(fh) - # Apply load interrupts (SCS-107, NSM) from the loads table to this - # command load. This assumes that loads.csv has been updated - # appropriately from cmd_events.csv (which might have come from the - # Command Events sheet). - cmds = interrupt_load_commands(load, cmds) + # Filter commands if loads (vehicle and/or observing) were approved but + # never uplinked + if load_name in not_run_loads["Load"]: + # Keep only the orbit points + cmds = cmds[cmds["type"] == "ORBPOINT"] + elif load_name in not_run_loads["Observing"]: + # Cut observing commands + bad = np.isin(cmds["scs"], [131, 132, 133]) + cmds = cmds[~bad] + if len(cmds) > 0: logger.info( - f'Load {load["name"]} has {len(cmds)} commands' + f"Load {load_name} has {len(cmds)} commands" f" with rltt {load['rltt']}" ) cmds_list.append(cmds) - rltts.append(load["rltt"]) + if load_name not in not_run_loads["Load"]: + cmds.meta["rltt"] = cmds.get_rltt() else: - logger.info(f'Load {load["name"]} has no commands, skipping') + logger.info(f"Load {load_name} has no commands, skipping") # Filter events outside the time interval, assuming command event cannot # last more than 2 weeks. @@ -356,43 +377,65 @@ def update_archive_and_get_cmds_recent( cmd_event["Date"], cmd_event["Event"], cmd_event["Params"] ) - # Events that do not generate commands (e.g. load interrupt) return - # None from get_cmds_from_event. + # Events that do not generate commands return None from + # get_cmds_from_event. if cmds is not None and len(cmds) > 0: cmds_list.append(cmds) - rltts.append(None) # Sort cmds_list and rltts by the date of the first cmd in each cmds Table cmds_starts = np.array([cmds["date"][0] for cmds in cmds_list]) idx_sort = np.argsort(cmds_starts) cmds_list = [cmds_list[ii] for ii in idx_sort] - rltts = [rltts[ii] for ii in idx_sort] - for ii, cmds, rltt in zip(itertools.count(), cmds_list, rltts): - if rltt is not None: - # Apply RLTT from this load to the current running loads (cmds_all). - # Remove commands with date greater than the RLTT date. In most + # Apply RLTT and any END SCS commands (CODISAXS) to loads. RLTT is applied + # by stopping SCS's 128-133. + for ii, cmds in enumerate(cmds_list): + print(f"Processing {cmds['source'][0]} with {len(cmds)} commands") + end_scs = {} + if rltt := cmds.meta.get("rltt"): + source = f'RLTT in {cmds["source"][0]}' + for scs in range(128, 134): + end_scs[scs] = (rltt, source) + + # Explicit END SCS commands. Most commonly these come from command events + # like SCS-107, but can also be in weekly loads e.g. disabling an ACIS + # ECS measurement in SCS-135. + ok = cmds["tlmsid"] == "CODISASX" + if np.any(ok): + for cmd in cmds[ok]: + if (scs := cmd["params"]["codisas1"]) >= 128: + source = f'DISABLE SCS {scs} in {cmd["source"]} at {cmd["date"]}' + end_scs[scs] = (cmd["date"], source) + + import pprint + + pprint.pprint(end_scs) + + for scs, (date_end, source) in end_scs.items(): + # Apply end SCS from these commands to the current running loads. + # Remove commands with date greater than end SCS date. In most # cases this does not cut anything. for jj in range(0, ii): prev_cmds = cmds_list[jj] # First check for any overlap since prev_cmds is sorted by date. - if len(prev_cmds) > 0 and prev_cmds["date"][-1] > rltt: - # Cut commands EXCEPT for ORBPOINT ones, which we leave as a - # special case to ensure availability of orbit events from - # commands. Otherwise RLTT can cut ORBPOINT commands that - # are not replaced by the subsquent loads. - bad = (prev_cmds["date"] > rltt) & (prev_cmds["type"] != "ORBPOINT") + print( + f'{scs=}, {jj=}, {len(prev_cmds)=}, {prev_cmds["date"][-1]=}, {date_end=}, {prev_cmds["date"][-1] > date_end=}' + ) + if len(prev_cmds) > 0 and prev_cmds["date"][-1] > date_end: + bad = (prev_cmds["date"] > date_end) & (prev_cmds["scs"] == scs) + print(f"{np.count_nonzero(bad)} commands to remove") if np.any(bad): n_bad = np.count_nonzero(bad) logger.info( - f'Removing {n_bad} cmds from {prev_cmds["source"][0]}' + f"Removing {n_bad} SCS={scs} cmds from " + f'{prev_cmds["source"][0]} due to {source}' ) cmds_list[jj] = prev_cmds[~bad] if len(cmds) > 0: logger.info(f'Adding {len(cmds)} commands from {cmds["source"][0]}') - cmds_recent = vstack_exact(cmds_list) + cmds_recent: CommandTable = vstack_exact(cmds_list) cmds_recent.sort_in_backstop_order() cmds_recent.deduplicate_orbit_cmds() cmds_recent.remove_not_run_cmds() @@ -828,7 +871,7 @@ def get_loads(scenario=None): return loads -def update_loads(scenario=None, *, cmd_events=None, lookback=None, stop=None): +def update_loads(scenario=None, *, cmd_events=None, lookback=None, stop=None) -> Table: """Update or create loads.csv and loads/ archive though ``lookback`` days CSV table file with column names in the first row and data in subsequent rows. @@ -867,7 +910,7 @@ def update_loads(scenario=None, *, cmd_events=None, lookback=None, stop=None): loads_rows = [] # Probably too complicated, but this bit of code generates a list of dates - # that are guaranteed to sample all the months in the lookback period with + # that are guaranteed too sample all the months in the lookback period with # two weeks of margin on the tail end. dt = 21 * u.day start = CxoTime(stop) - lookback * u.day @@ -918,6 +961,8 @@ def update_loads(scenario=None, *, cmd_events=None, lookback=None, stop=None): continue if load_date >= start and load_date <= stop: cmds = get_load_cmds_from_occweb_or_local(dir_year_month, load_name) + cmds.meta["rltt"] = cmds.get_rltt() + cmds.meta["scheduled_stop_time"] = cmds.get_scheduled_stop_time() load = get_load_dict_from_cmds(load_name, cmds, cmd_events) loads_rows.append(load) @@ -1011,7 +1056,7 @@ def get_load_dict_from_cmds(load_name: str, cmds: CommandTable, cmd_events: Tabl def get_load_cmds_from_occweb_or_local( dir_year_month=None, load_name=None, use_ska_dir=False -): +) -> CommandTable: """Get the load cmds (backstop) for ``load_name`` within ``dir_year_month`` If the backstop file is already available locally, use that. Otherwise, the diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index a00de7f5..57be8864 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -288,26 +288,31 @@ def test_commands_create_archive_regress(tmpdir, version_env): cmds_local = commands.get_cmds(start + 3, stop - 3) # FIXME: workaround that flight archive does not have these non-load - # commands added in #248. If flight archive is regenerated, this + # commands added in PR #248. If flight archive is regenerated, this # should be removed. - ignore = ( - (cmds_local["type"] == "LOAD_EVENT") - & (cmds_local["source"] == "CMD_EVT") - & (cmds_local["tlmsid"] == "None") - ) - ignore |= (cmds_local["tlmsid"] == "CODISASX") & ( - cmds_local["source"] == "CMD_EVT" - ) - cmds_local = cmds_local[~ignore] + def get_ok(cmds): + ignore = (cmds["type"] == "LOAD_EVENT") & ( + cmds["event_type"] == "SCHEDULED_STOP_TIME" + ) + ignore |= ( + (cmds["type"] == "LOAD_EVENT") + & (cmds["source"] == "CMD_EVT") + & np.isin(cmds["event_type"], ["LOAD_NOT_RUN", "OBSERVING_NOT_RUN"]) + ) + ignore |= (cmds["tlmsid"] == "CODISASX") & (cmds["source"] == "CMD_EVT") + return ~ignore + + cmds_local = cmds_local[get_ok(cmds_local)] + cmds_flight = cmds_flight[get_ok(cmds_flight)] cmds_local.fetch_params() if len(cmds_flight) != len(cmds_local): - for ii, (cmd_flight, cmd_local) in enumerate( - zip(cmds_flight, cmds_local) - ): - if cmd_flight["tlmsid"] != cmd_local["tlmsid"]: - err = f"First mismatch at line {ii}\n{cmd_flight}\n{cmd_local}" - raise AssertionError(err) + # Code to debug problems, leave commented for production + # out = "\n".join(cmds_flight.pformat_like_backstop()) + # Path("cmds_flight.txt").write_text(out) + # out = "\n".join(cmds_local.pformat_like_backstop()) + # Path("cmds_local.txt").write_text(out) + assert len(cmds_flight) == len(cmds_local) # 'starcat_idx' param in OBS cmd does not match since the pickle files # are different, so remove it. From 550fb3ff3d56072172281a80f1cb7f2af75f27d5 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 24 Aug 2022 08:30:22 -0400 Subject: [PATCH 08/16] Further improve interrupt handling and fix tests --- kadi/commands/commands_v2.py | 36 +++++++++++++--------------- kadi/commands/tests/test_commands.py | 11 +++++---- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index a8fa6af6..66372157 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1,9 +1,9 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import calendar +import collections import difflib import functools import gzip -import itertools import logging import math import os @@ -390,12 +390,14 @@ def update_archive_and_get_cmds_recent( # Apply RLTT and any END SCS commands (CODISAXS) to loads. RLTT is applied # by stopping SCS's 128-133. for ii, cmds in enumerate(cmds_list): - print(f"Processing {cmds['source'][0]} with {len(cmds)} commands") - end_scs = {} - if rltt := cmds.meta.get("rltt"): + cmds_source = cmds["source"][0] + if cmds_source == "CMD_EVT": + cmds_source = f"CMD_EVT {cmds[0]['event']} at {cmds[0]['date']}" + logger.info(f"Processing {cmds_source} with {len(cmds)} commands") + end_scs = collections.defaultdict(list) + if date_end := cmds.meta.get("rltt"): source = f'RLTT in {cmds["source"][0]}' - for scs in range(128, 134): - end_scs[scs] = (rltt, source) + end_scs[date_end, source].extend([128, 129, 130, 131, 132, 133]) # Explicit END SCS commands. Most commonly these come from command events # like SCS-107, but can also be in weekly loads e.g. disabling an ACIS @@ -404,36 +406,30 @@ def update_archive_and_get_cmds_recent( if np.any(ok): for cmd in cmds[ok]: if (scs := cmd["params"]["codisas1"]) >= 128: - source = f'DISABLE SCS {scs} in {cmd["source"]} at {cmd["date"]}' - end_scs[scs] = (cmd["date"], source) + source = f'DISABLE SCS in {cmd["source"]} at {cmd["date"]}' + end_scs[cmd["date"], source].append(scs) - import pprint - - pprint.pprint(end_scs) - - for scs, (date_end, source) in end_scs.items(): + for (date_end, source), scss in end_scs.items(): # Apply end SCS from these commands to the current running loads. # Remove commands with date greater than end SCS date. In most # cases this does not cut anything. for jj in range(0, ii): prev_cmds = cmds_list[jj] # First check for any overlap since prev_cmds is sorted by date. - print( - f'{scs=}, {jj=}, {len(prev_cmds)=}, {prev_cmds["date"][-1]=}, {date_end=}, {prev_cmds["date"][-1] > date_end=}' - ) if len(prev_cmds) > 0 and prev_cmds["date"][-1] > date_end: - bad = (prev_cmds["date"] > date_end) & (prev_cmds["scs"] == scs) - print(f"{np.count_nonzero(bad)} commands to remove") + bad = (prev_cmds["date"] > date_end) & np.isin( + prev_cmds["scs"], scss + ) if np.any(bad): n_bad = np.count_nonzero(bad) logger.info( - f"Removing {n_bad} SCS={scs} cmds from " + f"Removing {n_bad} cmds in SCS slots {scss} from " f'{prev_cmds["source"][0]} due to {source}' ) cmds_list[jj] = prev_cmds[~bad] if len(cmds) > 0: - logger.info(f'Adding {len(cmds)} commands from {cmds["source"][0]}') + logger.info(f"Adding {len(cmds)} commands from {cmds_source}") cmds_recent: CommandTable = vstack_exact(cmds_list) cmds_recent.sort_in_backstop_order() diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index 57be8864..fbc8a486 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -449,7 +449,7 @@ def test_get_cmds_nsm_2021(stop_date_2021_10_24): "2021:296:10:35:01.285 | COMMAND_HW | CTXBON | OCT1821A | " "hex=7800044, msid=CTXBON, scs=128", "2021:296:10:41:57.000 | LOAD_EVENT | None | CMD_EVT | " - "event=Load_not_run, event_date=2021:296:10:41:57, type=LOAD_NOT_RUN, " + "event=Load_not_run, event_date=2021:296:10:41:57, event_type=LOAD_NOT_RUN, " "load=OCT2521A, scs=0", "2021:296:10:41:57.000 | COMMAND_SW | AONSMSAF | CMD_EVT | " "event=NSM, event_date=2021:296:10:41:57, scs=0", @@ -504,6 +504,8 @@ def test_get_cmds_nsm_2021(stop_date_2021_10_24): "event_type=EQF013M, scs=0", "2021:297:13:59:39.602 | ORBPOINT | None | OCT1821A | " "event_type=EEF1000, scs=0", + "2021:297:14:01:00.000 | LOAD_EVENT | None | OCT1821A | " + "event_type=SCHEDULED_STOP_TIME, scs=0", ] assert cmds.pformat_like_backstop(max_params_width=200) == exp commands_v2.clear_caches() @@ -956,10 +958,10 @@ def test_get_cmds_from_event_case(par_str): cmd_events_all = Table.read(cmd_events_all_text, format="ascii.csv") cmd_events_all_exps = [ [ - "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Observing_not_run, event_date=2020:001:00:00:00, type=OBSERVING_NOT_RUN, load=FEB1422A, scs=0" # noqa + "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Observing_not_run, event_date=2020:001:00:00:00, event_type=OBSERVING_NOT_RUN, load=FEB1422A, scs=0" # noqa ], [ - "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2020:001:00:00:00, type=LOAD_NOT_RUN, load=OCT2521A, scs=0" # noqa + "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2020:001:00:00:00, event_type=LOAD_NOT_RUN, load=OCT2521A, scs=0" # noqa ], [ "2020:001:00:00:00.000 | ACISPKT | AA00000000 | CMD_EVT | event=Command, event_date=2020:001:00:00:00, scs=0" # noqa @@ -1120,7 +1122,7 @@ def test_scenario_with_rts(monkeypatch): 2021:296:10:35:00.771 | COMMAND_HW | CTXBOF | OCT1821A | msid=CTXBOF, scs=128 2021:296:10:35:01.028 | COMMAND_HW | CPABON | OCT1821A | msid=CPABON, scs=128 2021:296:10:35:01.285 | COMMAND_HW | CTXBON | OCT1821A | msid=CTXBON, scs=128 -2021:296:10:41:57.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2021:296:10:41:57, type=LOAD_ +2021:296:10:41:57.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2021:296:10:41:57, event_type 2021:296:10:41:57.000 | COMMAND_SW | AONSMSAF | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi 2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi @@ -1160,6 +1162,7 @@ def test_scenario_with_rts(monkeypatch): 2021:297:13:01:35.000 | ACISPKT | RS_0000001 | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, scs=135 2021:297:13:01:39.000 | ACISPKT | RH_0000001 | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, scs=135 2021:297:13:59:39.602 | ORBPOINT | None | OCT2521A | event_type=EEF1000, scs=0 +2021:297:14:01:00.000 | LOAD_EVENT | None | OCT1821A | event_type=SCHEDULED_STOP_TIME, scs=0 2021:297:14:37:39.602 | ORBPOINT | None | OCT2521A | event_type=EPF1000, scs=0 2021:297:15:01:42.681 | ORBPOINT | None | OCT2521A | event_type=EALT0, scs=0 2021:297:15:01:43.574 | ORBPOINT | None | OCT2521A | event_type=XALT0, scs=0 From 18a79add8067816ac934706d565e919b383a78ec Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 24 Aug 2022 08:32:47 -0400 Subject: [PATCH 09/16] Fix typo --- kadi/commands/commands_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 66372157..ccfa575f 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -906,7 +906,7 @@ def update_loads(scenario=None, *, cmd_events=None, lookback=None, stop=None) -> loads_rows = [] # Probably too complicated, but this bit of code generates a list of dates - # that are guaranteed too sample all the months in the lookback period with + # that are guaranteed to sample all the months in the lookback period with # two weeks of margin on the tail end. dt = 21 * u.day start = CxoTime(stop) - lookback * u.day From 3a3a6b517e886efb61eaddb0abd63830b940fc81 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 24 Aug 2022 09:02:10 -0400 Subject: [PATCH 10/16] Remove orphaned interrupt_load_commands function --- kadi/commands/commands_v2.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index ccfa575f..56ef423e 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -87,29 +87,6 @@ def clear_caches(): OBSERVATIONS.clear() -# NOT USED! -def interrupt_load_commands(load, cmds): - """Cut commands beyond observing or vehicle stop times. - - Orbit point commands are NOT cut so that in the case of a load stop the - orbit points are still available. This takes advantage of additional code - that de-duplicates orbit points. - """ - bad = np.zeros(len(cmds), dtype=bool) - if load["observing_stop"] != "": - bad |= (cmds["date"] > load["observing_stop"]) & (cmds["scs"] > 130) - if load["vehicle_stop"] != "": - bad |= ( - (cmds["date"] > load["vehicle_stop"]) - & (cmds["scs"] < 131) - & (cmds["type"] != "ORBPOINT") - ) - if np.any(bad): - logger.info(f'Cutting {bad.sum()} commands from {load["name"]}') - cmds = cmds[~bad] - return cmds - - def _merge_cmds_archive_recent(start, scenario): """Merge cmds archive from ``start`` onward with recent cmds for ``scenario`` From 02609cc96f71dfa5afb403112bebf6054b20d7bb Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 24 Aug 2022 11:31:19 -0400 Subject: [PATCH 11/16] Stop creating loads.csv/dat and cut a bunch of code --- kadi/commands/commands_v2.py | 104 ++++----------------------- kadi/commands/core.py | 18 ++--- kadi/commands/tests/test_commands.py | 19 +---- 3 files changed, 25 insertions(+), 116 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 56ef423e..ac29c164 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -312,7 +312,7 @@ def update_archive_and_get_cmds_recent( load_name = load["name"] loads_backstop_path = paths.LOADS_BACKSTOP_PATH(load_name) with gzip.open(loads_backstop_path, "rb") as fh: - cmds = pickle.load(fh) + cmds: CommandTable = pickle.load(fh) # Filter commands if loads (vehicle and/or observing) were approved but # never uplinked @@ -325,13 +325,14 @@ def update_archive_and_get_cmds_recent( cmds = cmds[~bad] if len(cmds) > 0: + rltt = cmds.meta["rltt"] = cmds.get_rltt() + if rltt is None and load_name not in not_run_loads["Load"]: + # This is unexpected but press ahead anyway + logger.error(f"No RLTT for {load_name=}") logger.info( - f"Load {load_name} has {len(cmds)} commands" - f" with rltt {load['rltt']}" + f"Load {load_name} has {len(cmds)} commands" f" with RLTT={rltt}" ) cmds_list.append(cmds) - if load_name not in not_run_loads["Load"]: - cmds.meta["rltt"] = cmds.get_rltt() else: logger.info(f"Load {load_name} has no commands, skipping") @@ -837,25 +838,8 @@ def get_cmd_events(scenario=None): return cmd_events -def get_loads(scenario=None): - loads_path = paths.LOADS_TABLE_PATH(scenario) - logger.info(f"Reading loads file {loads_path}") - loads = Table.read(str(loads_path), format="csv") - return loads - - def update_loads(scenario=None, *, cmd_events=None, lookback=None, stop=None) -> Table: - """Update or create loads.csv and loads/ archive though ``lookback`` days - - CSV table file with column names in the first row and data in subsequent rows. - - - load_name: name of load products containing this load segment (e.g. "MAY2217B") - - cmd_start: time of first command in load segment - - cmd_stop: time of last command in load segment - - rltt: running load termination time (terminates previous running loads). - - schedule_stop_observing: activity end time for loads (propagation goes to this point). - - schedule_stop_vehicle: activity end time for loads (propagation goes to this point). - """ + """Update local copy of approved command loads though ``lookback`` days.""" # For testing allow override of default `stop` value if stop is None: stop = os.environ.get("KADI_COMMANDS_DEFAULT_STOP") @@ -879,7 +863,6 @@ def update_loads(scenario=None, *, cmd_events=None, lookback=None, stop=None) -> # by load name and use those to avoid reading in the cmds table. # For now just get things working reliably. - loads_table_path = paths.LOADS_TABLE_PATH(scenario) loads_rows = [] # Probably too complicated, but this bit of code generates a list of dates @@ -934,9 +917,11 @@ def update_loads(scenario=None, *, cmd_events=None, lookback=None, stop=None) -> continue if load_date >= start and load_date <= stop: cmds = get_load_cmds_from_occweb_or_local(dir_year_month, load_name) - cmds.meta["rltt"] = cmds.get_rltt() - cmds.meta["scheduled_stop_time"] = cmds.get_scheduled_stop_time() - load = get_load_dict_from_cmds(load_name, cmds, cmd_events) + load = { + "name": load_name, + "cmd_start": cmds["date"][0], + "cmd_stop": cmds["date"][-1], + } loads_rows.append(load) if not loads_rows: @@ -944,12 +929,6 @@ def update_loads(scenario=None, *, cmd_events=None, lookback=None, stop=None) -> # Finally, save the table to file loads_table = Table(loads_rows) - logger.info(f"Saving {len(loads_table)} loads to {loads_table_path}") - loads_table.sort("cmd_start") - loads_table.write(loads_table_path, format="csv", overwrite=True) - loads_table.write( - loads_table_path.with_suffix(".dat"), format="ascii.fixed_width", overwrite=True - ) if conf.clean_loads_dir: clean_loads_dir(loads_table) @@ -968,65 +947,6 @@ def clean_loads_dir(loads): file.unlink() -def get_load_dict_from_cmds(load_name: str, cmds: CommandTable, cmd_events: Table): - """Update ``load`` dict in place from the backstop commands.""" - vehicle_stop_events = ("NSM", "Safe mode", "Bright star hold") - observing_stop_events = vehicle_stop_events + ("SCS-107",) - - load = { - "name": load_name, - "cmd_start": cmds["date"][0], - "cmd_stop": cmds["date"][-1], - "observing_stop": "", - "vehicle_stop": "", - } - - load["rltt"] = cmds.get_rltt() - load["scheduled_stop_time"] = cmds.get_scheduled_stop_time() - - # CHANGE THIS to use LOAD_EVENT entries in commands. Or NOT?? Probably - # provides good visibility into what's going on. But this hard-coding is - # annoying. - for cmd_event in cmd_events: - cmd_event_date = cmd_event["Date"] - - if ( - cmd_event_date >= load["cmd_start"] - and cmd_event_date <= load["cmd_stop"] - and cmd_event["Event"] in observing_stop_events - ): - logger.info( - f'{cmd_event["Event"]} at {cmd_event_date} found for {load_name}' - ) - load["observing_stop"] = cmd_event["Date"] - - if cmd_event["Event"] in vehicle_stop_events: - load["vehicle_stop"] = cmd_event["Date"] - - if cmd_event["Event"] == "Load not run" and cmd_event["Params"] == load_name: - logger.info( - f'{cmd_event["Event"]} at {cmd_event_date} found for {load_name}' - ) - # Stop the loads before they start - load["observing_stop"] = "1998:001" - load["vehicle_stop"] = "1998:001" - # Not-run load does not terminate other loads or commands or have a - # scheduled stop time. - load["rltt"] = None - load["scheduled_stop_time"] = None - - if ( - cmd_event["Event"] == "Observing not run" - and cmd_event["Params"] == load_name - ): - logger.info( - f'{cmd_event["Event"]} at {cmd_event_date} found for {load_name}' - ) - load["observing_stop"] = "1998:001" - - return load - - def get_load_cmds_from_occweb_or_local( dir_year_month=None, load_name=None, use_ska_dir=False ) -> CommandTable: diff --git a/kadi/commands/core.py b/kadi/commands/core.py index 722087b4..1d6c237d 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -63,7 +63,7 @@ def load_idx_cmds(version=None, file=None): ... File "H5FDsec2.c", line 941, in H5FD_sec2_lock unable to lock file, errno = 11, error message = 'Resource temporarily unavailable' - """ + """ # noqa: E501 if file is None: file = IDX_CMDS_PATH(version) with tables.open_file(file, mode="r") as h5: @@ -530,7 +530,7 @@ def get_rltt(self): ): return cmd["date"] else: - raise ValueError(f"No RLTT found") + return None def get_scheduled_stop_time(self): for idx in range(len(self), 0, -1): @@ -541,7 +541,7 @@ def get_scheduled_stop_time(self): ): return cmd["date"] else: - raise ValueError(f"No scheduled stop time found") + return None def add_cmds(self, cmds, rltt=None): """ @@ -645,7 +645,8 @@ def pformat_like_backstop( :param show_nonload_meta: bool, optional Show event and event_date for non-load commands (default=True) :param sort_orbit_events: bool, optional - Sort orbit events at same date by event_type (default=False, mostly for testing) + Sort orbit events at same date by event_type (default=False, mostly + for testing) :param max_params_width: int, optional Maximum width of parameter values string (default=80) :returns: list of lines @@ -737,7 +738,8 @@ def pprint_like_backstop(self, *, logger_func=None, logger_text="", **kwargs): :param show_nonload_meta: bool, optional Show event and event_date for non-load commands (default=True) :param sort_orbit_events: bool, optional - Sort orbit events at same date by event_type (default=False, mostly for testing) + Sort orbit events at same date by event_type (default=False, mostly + for testing) :param max_params_width: int, optional Maximum width of parameter values string (default=80) :returns: list of lines @@ -776,9 +778,9 @@ def deduplicate_orbit_cmds(self): cmd["params"]["event_type"] == last_cmd["params"]["event_type"] and abs(cmd["time"] - last_cmd["time"]) < 180 ): - # Same event as last (even if date is a bit different). Now if this one - # has a larger timeline_id that means it is from a more recent schedule, so - # use that one. + # Same event as last (even if date is a bit different). Now if + # this one has a larger timeline_id that means it is from a more + # recent schedule, so use that one. load_date = load_name_to_cxotime(cmd["source"]) load_date_last = load_name_to_cxotime(last_cmd["source"]) if load_date > load_date_last: diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index fbc8a486..a986e9bf 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -379,15 +379,6 @@ def test_get_cmds_v2_arch_recent(stop_date_2020_12_03): assert np.all(cmds["idx"] != -1) assert len(cmds) == 17640 - loads = commands_v2.get_loads() - assert loads.pformat_all() == [ - " name cmd_start cmd_stop observing_stop vehicle_stop rltt scheduled_stop_time ", # noqa - "-------- --------------------- --------------------- -------------- ------------ --------------------- ---------------------", # noqa - "NOV0920A 2020:314:12:13:00.000 2020:321:00:48:01.673 -- -- 2020:314:12:16:00.000 2020:321:00:48:01.673", # noqa - "NOV1620A 2020:321:00:45:01.673 2020:327:19:26:00.000 -- -- 2020:321:00:48:01.673 2020:327:19:26:00.000", # noqa - "NOV2320A 2020:327:19:23:00.000 2020:334:20:44:27.758 -- -- 2020:327:19:26:00.000 2020:334:20:44:27.758", # noqa - "NOV3020A 2020:334:20:41:27.758 2020:342:06:04:34.287 -- -- 2020:334:20:44:27.758 2020:342:06:04:34.287", # noqa - ] commands_v2.clear_caches() @@ -420,10 +411,6 @@ def test_get_cmds_v2_recent_only(stop_date_2020_12_03): assert len(cmds) == 1523 assert np.all(cmds["idx"] == -1) - # Sanity check on the loads - loads = commands_v2.get_loads() - assert np.all(loads["name"] == ["NOV0920A", "NOV1620A", "NOV2320A", "NOV3020A"]) - # zero-length query cmds = commands_v2.get_cmds(start="2020-12-01", stop="2020-12-01") assert len(cmds) == 0 @@ -878,7 +865,7 @@ def test_get_starcats_date(): Note: from https://icxc.harvard.edu//mp/mplogs/2006/DEC2506/oflsc/starcheck.html#obsid8008 MP_STARCAT at 2007:002:04:31:43.965 (VCDU count = 7477935) - """ + """ # noqa: E501 sc = get_starcats(obsid=8008, scenario="flight")[0] obs = get_observations(obsid=8008, scenario="flight")[0] assert sc.date == obs["starcat_date"] == "2007:002:04:31:43.965" @@ -888,8 +875,8 @@ def test_get_starcats_date(): def test_get_starcats_by_date(): - """Test that the getting a starcat using the starcat_date as argument returns the same catalog - as using the OBSID. + """Test that the getting a starcat using the starcat_date as argument + returns the same catalog as using the OBSID. """ sc = get_starcats(obsid=8008, scenario="flight")[0] sc_by_date = get_starcats(starcat_date="2007:002:04:31:43.965", scenario="flight")[ From 3d40dd6cd9294a5465252d0192d19fa6bb65cb09 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 28 Aug 2022 05:22:40 -0400 Subject: [PATCH 12/16] Doc changes --- kadi/commands/commands_v2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index ac29c164..ed1da119 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -12,6 +12,7 @@ import weakref from collections import defaultdict from pathlib import Path +from typing import List import astropy.units as u import numpy as np @@ -271,6 +272,9 @@ def update_archive_and_get_cmds_recent( ): """Update local loads table and downloaded loads and return all recent cmds. + This is the main entry point for getting recent commands and for updating + the archive HDF5 file. + This also caches the recent commands in the global CMDS_RECENT dict. This relies entirely on RLTT and load_events to assemble the commands. @@ -288,7 +292,7 @@ def update_archive_and_get_cmds_recent( :returns: CommandTable """ # List of CommandTable objects from loads and cmd_events - cmds_list = [] + cmds_list: List[CommandTable] = [] # Update local cmds_events.csv from Google Sheets cmd_events = update_cmd_events(scenario) From 7f69b527f0f865b772a8a85a6a35df6dede2e104 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 28 Aug 2022 06:55:53 -0400 Subject: [PATCH 13/16] Add validation notebook and other stuff --- .gitignore | 17 +- validate/compare-commands-acis-cti.ipynb | 271 ++++++++++++++++ .../validate-cmd-interrupt-refactor.ipynb | 301 ++++++++++++++++++ validate/validate-get-starcats.ipynb | 202 ++++++++++++ validate/write_events_cmds.py | 62 ++++ 5 files changed, 851 insertions(+), 2 deletions(-) create mode 100644 validate/compare-commands-acis-cti.ipynb create mode 100644 validate/validate-cmd-interrupt-refactor.ipynb create mode 100644 validate/validate-get-starcats.ipynb create mode 100644 validate/write_events_cmds.py diff --git a/.gitignore b/.gitignore index eaed9c4d..96e015b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,31 @@ test/ +DAWG-demo/ djangotutorial kadi/events/cache +kadi/events/migrations *.log web2py build/ _build/ dist/ -*.ipynb *.pkl *.db3 *.h5 MANIFEST -validate/ GIT_VERSION kadi.egg-info/ *.pyc *~ + +docs/.buildinfo +docs/_modules/ +docs/_sources/ +docs/_static/ + +.flake8 +.idea +.vscode +cmds*.dat +cmds*.txt + +.ipynb_checkpoints diff --git a/validate/compare-commands-acis-cti.ipynb b/validate/compare-commands-acis-cti.ipynb new file mode 100644 index 00000000..57c7279f --- /dev/null +++ b/validate/compare-commands-acis-cti.ipynb @@ -0,0 +1,271 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compare commands for ACIS CTI commands in 2020:145 safe mode recovery\n", + "\n", + "The V1 (legacy) commands archive is used as a reference standard to validate\n", + "the new V2 commands archive.\n", + "\n", + "The primary difference is that V1 non-load commands and load stoppages are \n", + "generated using the Chandra.cmd_states non-load commands table and timelines,\n", + "while V2 uses only the Flight Command Events from Google Sheets." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "from pathlib import Path\n", + "import difflib\n", + "import webbrowser\n", + "\n", + "import numpy as np\n", + "# Around 2020:145 safe mode\n", + "os.environ['KADI_COMMANDS_DEFAULT_STOP'] = '2020:152'\n", + "\n", + "# Using dev versions\n", + "sys.path.insert(0, '../')\n", + "sys.path.insert(0, str(Path.home() / 'git' / 'parse_cm'))\n", + "sys.path.insert(0, str(Path.home() / 'git' / 'testr'))\n", + "\n", + "from kadi.commands import get_cmds, conf" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from kadi import logger\n", + "# If needed for debugging...\n", + "logger.setLevel(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate diffs for commands from 2020-01-01 to 2022-01-01\n", + "\n", + "In this era there are real diffs, but all are acceptable. The diffs have been\n", + "reviewed individually by aspect team members." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "start = '2020:146'\n", + "stop = '2020:150'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-01-07 10:36:20,085 load_idx_cmds: Loaded /Users/aldcroft/ska/data/kadi/cmds.h5 with 1329878 commands\n" + ] + } + ], + "source": [ + "with conf.set_temp('commands_version', '1'):\n", + " cmds1 = get_cmds(start, stop)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-01-07 10:36:20,282 update_cmd_events: Getting cmd_events from https://docs.google.com/spreadsheets/d/19d6XqBhWoFjC-z1lS1nM6wLE_zjr4GYB1lOvrEGCbKQ/export?format=csv\n", + "2022-01-07 10:36:20,938 update_cmd_events: Writing 23 cmd_events to /Users/aldcroft/.kadi/cmd_events.csv\n", + "2022-01-07 10:36:20,947 get_occweb_page: Getting OCCweb FOT/mission_planning/PRODUCTS/APPR_LOADS/2020/MAY with cache=False\n", + "2022-01-07 10:36:21,203 get_load_cmds_from_occweb_or_local: Already have /Users/aldcroft/.kadi/loads/MAY0420A.pkl.gz\n", + "2022-01-07 10:36:21,210 get_load_cmds_from_occweb_or_local: Already have /Users/aldcroft/.kadi/loads/MAY1120B.pkl.gz\n", + "2022-01-07 10:36:21,216 get_load_cmds_from_occweb_or_local: Already have /Users/aldcroft/.kadi/loads/MAY1820A.pkl.gz\n", + "2022-01-07 10:36:21,222 get_load_dict_from_cmds: NSM at 2020:145:14:17:20 found for MAY1820A\n", + "2022-01-07 10:36:21,223 get_load_cmds_from_occweb_or_local: Already have /Users/aldcroft/.kadi/loads/MAY2420A.pkl.gz\n", + "2022-01-07 10:36:21,229 get_load_dict_from_cmds: NSM at 2020:145:14:17:20 found for MAY2420A\n", + "2022-01-07 10:36:21,231 get_load_cmds_from_occweb_or_local: Already have /Users/aldcroft/.kadi/loads/MAY2620B.pkl.gz\n", + "2022-01-07 10:36:21,238 get_load_cmds_from_occweb_or_local: Already have /Users/aldcroft/.kadi/loads/MAY2820A.pkl.gz\n", + "2022-01-07 10:36:21,243 update_loads: Saving 6 loads to /Users/aldcroft/.kadi/loads.csv\n", + "2022-01-07 10:36:21,247 update_archive_and_get_cmds_recent: Including loads MAY0420A, MAY1120B, MAY1820A, MAY2420A, MAY2620B, MAY2820A\n", + "2022-01-07 10:36:21,251 update_archive_and_get_cmds_recent: Load MAY0420A has 1263 commands\n", + "2022-01-07 10:36:21,254 update_archive_and_get_cmds_recent: Load MAY1120B has 1124 commands\n", + "2022-01-07 10:36:21,258 interrupt_load_commands: Cutting 130 commands from MAY1820A\n", + "2022-01-07 10:36:21,259 update_archive_and_get_cmds_recent: Load MAY1820A has 1457 commands\n", + "2022-01-07 10:36:21,263 interrupt_load_commands: Cutting 1404 commands from MAY2420A\n", + "2022-01-07 10:36:21,264 update_archive_and_get_cmds_recent: Load MAY2420A has 106 commands\n", + "2022-01-07 10:36:21,266 update_archive_and_get_cmds_recent: Load MAY2620B has 66 commands\n", + "2022-01-07 10:36:21,269 update_archive_and_get_cmds_recent: Load MAY2820A has 960 commands\n", + "2022-01-07 10:36:21,279 update_archive_and_get_cmds_recent: Including cmd_events:\n", + " RTS at 2020:148:14:16:00\n", + " Obsid at 2020:148:14:15:00\n", + " Maneuver at 2020:147:11:21:00\n", + " RTS at 2020:147:02:08:00\n", + " Obsid at 2020:147:02:05:00\n", + " Maneuver at 2020:147:01:55:00\n", + " Command at 2020:145:14:17:30\n", + " NSM at 2020:145:14:17:20\n", + "2022-01-07 10:36:21,289 get_occweb_page: Getting OCCweb FOT/configuration/products/rts/1_4_CTI.RTS with cache=True\n", + "2022-01-07 10:36:21,326 update_archive_and_get_cmds_recent: Adding 1263 commands from MAY0420A\n", + "2022-01-07 10:36:21,327 update_archive_and_get_cmds_recent: Adding 1124 commands from MAY1120B\n", + "2022-01-07 10:36:21,327 update_archive_and_get_cmds_recent: Adding 1457 commands from MAY1820A\n", + "2022-01-07 10:36:21,328 update_archive_and_get_cmds_recent: Removing 12 cmds from MAY1820A\n", + "2022-01-07 10:36:21,329 update_archive_and_get_cmds_recent: Adding 106 commands from MAY2420A\n", + "2022-01-07 10:36:21,330 update_archive_and_get_cmds_recent: Adding 8 commands from CMD_EVT\n", + "2022-01-07 10:36:21,330 update_archive_and_get_cmds_recent: Adding 1 commands from CMD_EVT\n", + "2022-01-07 10:36:21,331 update_archive_and_get_cmds_recent: Adding 4 commands from CMD_EVT\n", + "2022-01-07 10:36:21,331 update_archive_and_get_cmds_recent: Adding 1 commands from CMD_EVT\n", + "2022-01-07 10:36:21,332 update_archive_and_get_cmds_recent: Adding 18 commands from CMD_EVT\n", + "2022-01-07 10:36:21,333 update_archive_and_get_cmds_recent: Adding 4 commands from CMD_EVT\n", + "2022-01-07 10:36:21,333 update_archive_and_get_cmds_recent: Adding 66 commands from MAY2620B\n", + "2022-01-07 10:36:21,334 update_archive_and_get_cmds_recent: Adding 1 commands from CMD_EVT\n", + "2022-01-07 10:36:21,334 update_archive_and_get_cmds_recent: Adding 18 commands from CMD_EVT\n", + "2022-01-07 10:36:21,335 update_archive_and_get_cmds_recent: Removing 7 cmds from CMD_EVT\n", + "2022-01-07 10:36:21,336 update_archive_and_get_cmds_recent: Adding 960 commands from MAY2820A\n", + "2022-01-07 10:36:21,359 get_cmds: Getting commands from recent only scenario=None\n" + ] + } + ], + "source": [ + "with conf.set_temp('commands_version', '2'):\n", + " cmds2 = get_cmds(start, stop)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Commands V1 does not have starcat parameters or LOAD_EVENT commands\n", + "for cmds in cmds1, cmds2:\n", + " ok = cmds['type'] == 'MP_STARCAT'\n", + " cmds['params'][ok] = {}\n", + " ok = cmds['type'] == 'LOAD_EVENT'\n", + " cmds.remove_rows(np.where(ok)[0])\n", + " cmds.sort(['date', 'step', 'scs'])" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(340, 358)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# There are differences in the number of commands\n", + "len(cmds1), len(cmds2)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-01-07 10:36:21,518 load_pars_dict: Loaded /Users/aldcroft/ska/data/kadi/cmds.pkl with 71769 pars\n" + ] + } + ], + "source": [ + "lines1 = cmds1.pformat_like_backstop(show_source=False, show_nonload_meta=False, max_params_width=40,\n", + " sort_orbit_events=True)\n", + "lines2 = cmds2.pformat_like_backstop(show_source=False, show_nonload_meta=False, max_params_width=40, \n", + " sort_orbit_events=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "htmldiffer = difflib.HtmlDiff()\n", + "html = htmldiffer.make_file(lines1, lines2, 'kadi commands v1', 'kadi commands v2', context=True)\n", + "\n", + "out = Path('commands_v1_v2_diff.html')\n", + "out.write_text(html)\n", + "\n", + "url = 'file://' + str(out.absolute())\n", + "webbrowser.open(url)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "d2df0004ee630a46de2935730c9c65ee0c09bd3f3b85f07c44dd36ceff3dbd5e" + }, + "kernelspec": { + "display_name": "Python 3.8.3 64-bit ('ska3': conda)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.3" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/validate/validate-cmd-interrupt-refactor.ipynb b/validate/validate-cmd-interrupt-refactor.ipynb new file mode 100644 index 00000000..02bbbda1 --- /dev/null +++ b/validate/validate-cmd-interrupt-refactor.ipynb @@ -0,0 +1,301 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Validate PR #248 to refactor command interrupt handling\n", + "\n", + "This notebook compares the commands generated using PR #248 to those from\n", + "current flight (kadi 6.0.1). \n", + "\n", + "The notebook assumes that a new commands archive\n", + "(`cmds2.h5, cmds2.pkl`) has been created in the current directory using\n", + "`utils.migrate_cmds_to_cmds2.make_cmds2()`.\n", + "\n", + "All diffs are either expected or reveal issues in the current flight\n", + "processing to determine observation intervals (`OBS` `LOAD_EVT` commands)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "\n", + "from kadi import __version__, logger\n", + "from kadi.commands import commands_v2\n", + "from kadi.commands.commands_v2 import clear_caches, get_cmds" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6.0.1\n" + ] + } + ], + "source": [ + "print(__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "logger.setLevel(10)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def get_ok(cmds):\n", + " \"\"\"PR #248 includes new commands that are not in the old archive so we\n", + " filter them out here.\"\"\"\n", + " ignore = (cmds[\"type\"] == \"LOAD_EVENT\") & (\n", + " cmds[\"event_type\"] == \"SCHEDULED_STOP_TIME\"\n", + " )\n", + " ignore |= (\n", + " (cmds[\"type\"] == \"LOAD_EVENT\")\n", + " & (cmds[\"source\"] == \"CMD_EVT\")\n", + " & np.isin(cmds[\"event_type\"], [\"LOAD_NOT_RUN\", \"OBSERVING_NOT_RUN\"])\n", + " )\n", + " ignore |= (cmds[\"tlmsid\"] == \"CODISASX\") & (cmds[\"source\"] == \"CMD_EVT\")\n", + " return ~ignore" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "start = \"2000:001\"\n", + "stop = \"2022:240\"" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-08-26 06:18:05,561 get_cmds: Getting commands from archive only\n", + "2022-08-26 06:18:05,743 load_idx_cmds: Loaded /Users/aldcroft/ska/data/kadi/cmds2.h5 with 1416611 commands\n", + "2022-08-26 06:18:06,172 load_pars_dict: Loaded /Users/aldcroft/ska/data/kadi/cmds2.pkl with 146263 pars\n", + "146263\n", + "146263\n" + ] + } + ], + "source": [ + "cmds_flight = get_cmds(start, stop, scenario='flight')\n", + "ok = get_ok(cmds_flight)\n", + "cmds_flight_ok = cmds_flight[ok]\n", + "print(len(commands_v2.PARS_DICT))\n", + "print(len(commands_v2.REV_PARS_DICT))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "clear_caches()\n", + "del commands_v2.PARS_DICT._val\n", + "del commands_v2.REV_PARS_DICT._val\n", + "\n", + "os.environ['KADI'] = os.path.abspath('..')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-08-26 06:18:23,822 get_cmds: Getting commands from archive only\n", + "2022-08-26 06:18:23,977 load_idx_cmds: Loaded /Users/aldcroft/git/kadi/cmds2.h5 with 1416689 commands\n", + "2022-08-26 06:18:24,304 load_pars_dict: Loaded /Users/aldcroft/git/kadi/cmds2.pkl with 146150 pars\n", + "146150\n", + "146150\n" + ] + } + ], + "source": [ + "cmds_local = get_cmds(start, stop, scenario='flight')\n", + "ok = get_ok(cmds_local)\n", + "cmds_local_ok = cmds_local[ok]\n", + "print(len(commands_v2.PARS_DICT))\n", + "print(len(commands_v2.REV_PARS_DICT))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1416396 1416400\n", + "1416507 1416585\n" + ] + } + ], + "source": [ + "print(len(cmds_flight_ok), len(cmds_local_ok))\n", + "print(len(cmds_flight), len(cmds_local))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "159237187" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Code to debug problems, leave commented for production\n", + "out = \"\\n\".join(cmds_flight_ok.pformat_like_backstop(max_params_width=200))\n", + "Path(\"cmds_flight.txt\").write_text(out)\n", + "out = \"\\n\".join(cmds_local_ok.pformat_like_backstop(max_params_width=200))\n", + "Path(\"cmds_local.txt\").write_text(out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Results of `diff cmds_flight.txt cmds_local.txt`\n", + "\n", + "Change in `obs_stop` for an `OBS` load event. The new (local) version has \n", + "`obs_stop` at the time of the commanded maneuver at 224:02:25, so that is correct for\n", + "defining the end of the observation.\n", + "```\n", + "1413286c1413286\n", + "< 2022:223:13:20:43.345 | LOAD_EVENT | OBS | AUG0822A | obsid=45339, simpos=-99616, obs_stop=2022:223:14:15:03.094, manvr_start=2022:223:12:40:44.686, targ_att=(-0.12752005, 0.556391476, -0.351473717, 0.742046756), npnt_enab=True, obs_start=2022:223:13:20:\n", + "---\n", + "> 2022:223:13:20:43.345 | LOAD_EVENT | OBS | AUG0822A | obsid=45339, simpos=-99616, obs_stop=2022:224:02:25:00.000, manvr_start=2022:223:12:40:44.686, targ_att=(-0.12752005, 0.556391476, -0.351473717, 0.742046756), npnt_enab=True, obs_start=2022:223:13:20:\n", + "```\n", + "This `AODSDITH` command did not happen, so `cmds_local` is correct:\n", + "```\n", + "1413295d1413294\n", + "< 2022:223:13:50:40.960 | COMMAND_SW | AODSDITH | CMD_EVT | event=Bright_star_hold, event_date=2022:223:13:49:23, scs=0\n", + "```\n", + "The new version is correct. The AUG1322A loads have a maneuver starting at 2022:225:11:40:00.\n", + "```\n", + "1413309c1413308\n", + "< 2022:224:02:40:48.033 | LOAD_EVENT | OBS | CMD_EVT | obsid=45339, simpos=-99616, obs_stop=2022:226:19:15:48.867, manvr_start=2022:224:02:25:10.250, targ_att=(-0.24849372, 0.40954561, -0.31440262, 0.81955735), npnt_enab=True, obs_start=2022:224:02:40:48.\n", + "---\n", + "> 2022:224:02:40:48.033 | LOAD_EVENT | OBS | CMD_EVT | obsid=45339, simpos=-99616, obs_stop=2022:225:11:40:00.000, manvr_start=2022:224:02:25:10.250, targ_att=(-0.24849372, 0.40954561, -0.31440262, 0.81955735), npnt_enab=True, obs_start=2022:224:02:40:48.\n", + "```\n", + "New version is correct, NSM at 2022:231:18:36:00 stops the observation.\n", + "```\n", + "1415054c1415053\n", + "< 2022:231:17:04:23.363 | LOAD_EVENT | OBS | AUG1322A | obsid=45317, simpos=-99616, obs_stop=2022:231:19:36:57.259, manvr_start=2022:231:16:44:24.407, targ_att=(0.649105589, -0.0318838109, -0.225906169, 0.725680205), npnt_enab=True, obs_start=2022:231:17:0\n", + "---\n", + "> 2022:231:17:04:23.363 | LOAD_EVENT | OBS | AUG1322A | obsid=45317, simpos=-99616, obs_stop=2022:231:18:36:00.000, manvr_start=2022:231:16:44:24.407, targ_att=(0.649105589, -0.0318838109, -0.225906169, 0.725680205), npnt_enab=True, obs_start=2022:231:17:0\n", + "```\n", + "Tiny difference in the target attitude due to timing difference.\n", + "```\n", + "1415082c1415081\n", + "< 2022:231:19:05:05.999 | LOAD_EVENT | OBS | CMD_EVT | obsid=45317, simpos=-99616, obs_stop=2022:232:03:07:00.000, manvr_start=2022:231:18:36:00.000, targ_att=(0.6377368264106795, 0.4497793420951543, 0.2522209483727346, 0.5721668262453579), npnt_enab=Fals\n", + "---\n", + "> 2022:231:19:05:05.999 | LOAD_EVENT | OBS | CMD_EVT | obsid=45317, simpos=-99616, obs_stop=2022:232:03:07:00.000, manvr_start=2022:231:18:36:00.000, targ_att=(0.6377368264106797, 0.44977934209515424, 0.2522209483727345, 0.572166826245358), npnt_enab=Fals\n", + "```\n", + "Tiny difference in the target attitude due to timing difference.\n", + "```\n", + "1415089c1415088\n", + "< 2022:232:03:07:00.000 | LOAD_EVENT | OBS | CMD_EVT | obsid=62624, simpos=-99616, obs_stop=2022:232:20:07:21.000, manvr_start=2022:231:18:36:00.000, targ_att=(0.6377368264106795, 0.4497793420951543, 0.2522209483727346, 0.5721668262453579), npnt_enab=Fals\n", + "---\n", + "> 2022:232:03:07:00.000 | LOAD_EVENT | OBS | CMD_EVT | obsid=62624, simpos=-99616, obs_stop=2022:232:20:07:21.000, manvr_start=2022:231:18:36:00.000, targ_att=(0.6377368264106797, 0.44977934209515424, 0.2522209483727345, 0.572166826245358), npnt_enab=Fals\n", + "```\n", + "New version is correct, obsid change to 62623 at 234:22:12:12.000. Flight version has\n", + "`obs_stop` at `2022:233:17:40:00` but there is really nothing happening then,\n", + "so that just seems like a bug.\n", + "```\n", + "1415105c1415104\n", + "< 2022:232:20:32:28.838 | LOAD_EVENT | OBS | CMD_EVT | obsid=62624, simpos=-99616, obs_stop=2022:233:17:40:00.000, manvr_start=2022:232:20:07:31.250, targ_att=(0.70443765, 0.06647616, -0.10211985, 0.69922818), npnt_enab=True, obs_start=2022:232:20:32:28.8\n", + "---\n", + "> 2022:232:20:32:28.838 | LOAD_EVENT | OBS | CMD_EVT | obsid=62624, simpos=-99616, obs_stop=2022:234:22:12:12.000, manvr_start=2022:232:20:07:31.250, targ_att=(0.70443765, 0.06647616, -0.10211985, 0.69922818), npnt_enab=True, obs_start=2022:232:20:32:28.8\n", + "```\n", + "New version is correct, flight version was erroneously cutting the ACIS CTI\n", + "for the RLTT of command loads that were not run.\n", + "```\n", + "1415110a1415110,1415114\n", + "> 2022:233:18:10:43.000 | ACISPKT | AA00000000 | CMD_EVT | event=RTS, event_date=2022:232:03:09:00, scs=135\n", + "> 2022:233:18:10:53.000 | ACISPKT | AA00000000 | CMD_EVT | event=RTS, event_date=2022:232:03:09:00, scs=135\n", + "> 2022:233:18:10:57.000 | ACISPKT | WSPOW0002A | CMD_EVT | event=RTS, event_date=2022:232:03:09:00, scs=135\n", + "> 2022:233:18:12:00.000 | ACISPKT | RS_0000001 | CMD_EVT | event=RTS, event_date=2022:232:03:09:00, scs=135\n", + "> 2022:233:18:12:04.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=RTS, event_date=2022:232:03:09:00, msid=OORMPDS, scs=135\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.12 ('ska3')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "d2df0004ee630a46de2935730c9c65ee0c09bd3f3b85f07c44dd36ceff3dbd5e" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/validate/validate-get-starcats.ipynb b/validate/validate-get-starcats.ipynb new file mode 100644 index 00000000..e474327e --- /dev/null +++ b/validate/validate-get-starcats.ipynb @@ -0,0 +1,202 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from kadi.commands import get_starcats, get_observations\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "STARCATS = {}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def test_get_starcats_each_year(year):\n", + " starcats = get_starcats(start=f'{year}:001', stop=f'{year+1}:001')\n", + " assert len(starcats) > 2\n", + " for starcat in starcats:\n", + " # Make sure fids and stars are all ID'd\n", + " ok = (starcat['type'] != 'MON') & (starcat['type'] != 'FID')\n", + " if np.any(starcat['id'][ok] == -999):\n", + " print('Bad stars', starcat.obsid)\n", + " STARCATS[starcat.obsid] = starcat\n", + " ok = starcat['type'] == 'FID'\n", + " if np.any(starcat['id'][ok] == -999):\n", + " print('Bad fids', starcat.obsid, starcat.sim_offset, starcat.detector)\n", + " STARCATS[starcat.obsid] = starcat\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "year=2003\n", + "Bad fids 3778 -1177 ACIS-I\n", + "Bad fids 4000 -1177 ACIS-I\n", + "Bad fids 4912 -4 HRC-S\n", + "year=2004\n", + "Bad fids 4937 -4 HRC-S\n", + "Bad fids 60116 -4 HRC-S\n", + "year=2005\n", + "Bad fids 5587 1 HRC-I\n", + "year=2006\n", + "Bad fids 6723 -1 ACIS-I\n", + "Bad fids 6432 -4 HRC-S\n", + "Bad fids 58647 -4 HRC-S\n", + "year=2007\n", + "Bad fids 7763 -2444 ACIS-S\n", + "Bad fids 7525 -2444 ACIS-S\n", + "year=2008\n", + "year=2009\n", + "year=2010\n", + "year=2011\n", + "year=2012\n", + "year=2013\n", + "year=2014\n", + "year=2015\n", + "year=2016\n", + "year=2017\n", + "year=2018\n", + "year=2019\n", + "year=2020\n", + "year=2021\n" + ] + } + ], + "source": [ + "for year in range(2003, 2023):\n", + " print(f'{year=}')\n", + " test_get_starcats_each_year(year)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "test_get_starcats_each_year(2022)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'obsid': 3778,\n", + " 'simpos': 91728,\n", + " 'obs_stop': '2003:279:05:39:43.813',\n", + " 'manvr_start': '2003:278:20:50:30.377',\n", + " 'targ_att': (-0.575413844, 0.226031366, -0.718979674, 0.31761133),\n", + " 'npnt_enab': True,\n", + " 'obs_start': '2003:278:21:06:12.378',\n", + " 'prev_att': (-0.424640337, 0.241997971, -0.84008342, 0.235324062),\n", + " 'starcat_idx': 139840,\n", + " 'source': 'OCT0503B'}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_observations(obsid=3778)[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "from kadi import events" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Table length=0\n", + "\n", + "\n", + "\n", + "
startdatetstartdescrnotesource
float64float64float64float64float64float64
" + ], + "text/plain": [ + "\n", + " start date tstart descr note source\n", + "float64 float64 float64 float64 float64 float64\n", + "------- ------- ------- ------- ------- -------" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "events.major_events.filter(start='2003:277', stop='2003:280').table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "d2df0004ee630a46de2935730c9c65ee0c09bd3f3b85f07c44dd36ceff3dbd5e" + }, + "kernelspec": { + "display_name": "Python 3.8.12 ('ska3')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/validate/write_events_cmds.py b/validate/write_events_cmds.py new file mode 100644 index 00000000..9d87d79b --- /dev/null +++ b/validate/write_events_cmds.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +import argparse +import os + +from Chandra.Time import DateTime + +import kadi.cmds +import kadi.events +import kadi.paths + + +def get_opt(args=None): + parser = argparse.ArgumentParser(description="Write commands and events ") + parser.add_argument("--stop", help="Processing stop date") + parser.add_argument("--start", help=("Processing start date")) + parser.add_argument("--data-root", default=".", help="Output data directory") + opt = parser.parse_args(args) + return opt + + +def write_events(start, stop): + print("Using events file {}".format(kadi.paths.EVENTS_DB_PATH())) + + for attr in dir(kadi.events): + evt = getattr(kadi.events, attr) + if type(evt) is not kadi.events.EventQuery: + continue + + # These event types are frequently updated after ingest. See + # https://github.com/sot/kadi/issues/85. For now just ignore. + if attr in ("caps", "dsn_comms", "major_events"): + continue + + dat = evt.filter(start, stop).table + if attr != "obsid" and "obsid" in dat.colnames: + del dat["obsid"] + + filename = os.path.join(opt.data_root, attr + ".ecsv") + print("Writing event {}".format(filename)) + dat.write(filename, format="ascii.ecsv") + + +def write_cmds(start, stop): + print("Using commands file {}".format(kadi.paths.IDX_CMDS_PATH())) + cmds = kadi.cmds.filter(start, stop) + out = repr(cmds) + filename = os.path.join(opt.data_root, "cmds.txt") + print("Writing commands {}".format(filename)) + with open(filename, "w") as fh: + fh.write(out) + + +if __name__ == "__main__": + opt = get_opt() + if not os.path.exists(opt.data_root): + os.makedirs(opt.data_root) + start = DateTime(opt.start) + stop = DateTime(opt.stop) + + write_events(start, stop) + # write_cmds(start, stop) From a75f34c84b8aafeefbd730b13127e2153c0cca89 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 31 Aug 2022 13:24:18 -0400 Subject: [PATCH 14/16] Compare command event outputs with full table --- kadi/commands/tests/test_commands.py | 107 ++++++++------------------- 1 file changed, 32 insertions(+), 75 deletions(-) diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index a986e9bf..59773b5c 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -530,86 +530,43 @@ def test_cmds_scenario(stop_date_2020_12_03): def test_command_set_bsh(): cmds = get_cmds_from_event("2000:001", "Bright star hold", "") - exp = [ - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " - "codisas1=12", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " - "codisas1=12", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " - "codisas1=13", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " - "codisas1=13", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " - "codisas1=13", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, " - "codisas1=13", - "2000:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0", - "2000:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, msid=AFLCRSET, scs=0", - "2000:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, pos=-99616, scs=0", - "2000:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0", - "2000:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0", - "2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | " - "event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0", - ] - - assert cmds.pformat_like_backstop() == exp + exp = """\ +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=128 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=129 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=130 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=131 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=132 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=133 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=AFLCRSET, scs=0 +2000:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, pos=-99616, scs=0 +2000:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0""" # noqa + + assert cmds.pformat_like_backstop(max_params_width=None) == exp.splitlines() commands_v2.clear_caches() def test_command_set_safe_mode(): cmds = get_cmds_from_event("2000:001", "Safe mode", "") - exp = [ - "2000:001:00:00:00.000 | COMMAND_SW | ACPCSFSU | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", - "2000:001:00:00:00.000 | COMMAND_SW | CSELFMT5 | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", - "2000:001:00:00:00.000 | COMMAND_SW | AONSMSAF | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=128 , " - "scs", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=129 , " - "scs", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=130 , " - "scs", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=131 , " - "scs", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=132 , " - "scs", - "2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=133 , " - "scs", - "2000:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", - "2000:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, msid=AFLCRSET, scs=0", - "2000:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, pos=-99616, scs=0", - "2000:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", - "2000:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", - "2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", - "2000:001:00:01:17.960 | COMMAND_SW | AODSDITH | CMD_EVT | " - "event=Safe_mode, event_date=2000:001:00:00:00, scs=0", - ] - + exp = """\ +2000:001:00:00:00.000 | COMMAND_SW | ACPCSFSU | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CSELFMT5 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:00:00.000 | COMMAND_SW | AONSMSAF | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=128 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=129 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=130 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=131 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=132 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=133 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=AFLCRSET, scs=0 +2000:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, pos=-99616, scs=0 +2000:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:17.960 | COMMAND_SW | AODSDITH | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0""" # noqa assert cmds.pformat_like_backstop() == exp commands_v2.clear_caches() From 41c0da3114f67f8252ec0a390973de40b055d67e Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 31 Aug 2022 15:42:44 -0400 Subject: [PATCH 15/16] Fix test --- kadi/commands/tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index 59773b5c..0e21054e 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -567,7 +567,7 @@ def test_command_set_safe_mode(): 2000:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 2000:001:00:01:17.960 | COMMAND_SW | AODSDITH | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0""" # noqa - assert cmds.pformat_like_backstop() == exp + assert cmds.pformat_like_backstop(max_params_width=None) == exp.splitlines() commands_v2.clear_caches() From bf2c6a8d462f684edb8f0c44a4c1e845bf2c4e2e Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 31 Aug 2022 15:49:40 -0400 Subject: [PATCH 16/16] Fix a test that failed for archive generated with #248 --- kadi/commands/tests/test_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index 0e21054e..cbe5e75b 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -377,7 +377,8 @@ def test_get_cmds_v2_arch_recent(stop_date_2020_12_03): # how the matching block is used (commands come from arch up through the end # of the matching block). assert np.all(cmds["idx"] != -1) - assert len(cmds) == 17640 + # PR #248: made this change from 17640 to 17644 + assert 17640 <= len(cmds) <= 17644 commands_v2.clear_caches()