From 1957e473b0f62e3a16aad55318e1aec044641a8f Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 8 Apr 2023 08:15:32 -0400 Subject: [PATCH 1/2] Refactor for cheta improvements --- kadi/commands/utils.py | 78 +-------------------------------- kadi/commands/validate.py | 30 ++++++++----- kadi/scripts/validate_states.py | 1 + 3 files changed, 21 insertions(+), 88 deletions(-) diff --git a/kadi/commands/utils.py b/kadi/commands/utils.py index 6124dafe..0396f8fd 100644 --- a/kadi/commands/utils.py +++ b/kadi/commands/utils.py @@ -1,25 +1,19 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -import functools import logging from dataclasses import dataclass from typing import List, Optional -import cheta.fetch_eng as fetch import numpy as np import plotly.graph_objects as pgo -from astropy.table import Table -from cxotime import CxoTime, CxoTimeLike, units as u +from cxotime import CxoTime, CxoTimeLike __all__ = [ "add_figure_regions", "compress_time_series", "convert_state_code_to_raw_val", - "get_telem_values", "fill_gaps_with_nan", - "NoTelemetryError", - "get_ofp_states", "get_time_series_chunks", "TimeSeriesChunk", "TimeSeriesPoint", @@ -28,10 +22,6 @@ logger = logging.getLogger(__name__) -class NoTelemetryError(Exception): - """No telemetry available for the specified interval""" - - @dataclass class TimeSeriesPoint: time: float @@ -67,72 +57,6 @@ def __repr__(self): return out -def get_telem_values(msids: list, stop, days: float = 14) -> Table: - """ - Fetch last ``days`` of available ``msids`` telemetry values before - time ``tstart``. - - :param msids: fetch msids list - :param stop: stop time for telemetry (CxoTime-like) - :param days: length of telemetry request before ``tstart`` - :returns: Table of requested telemetry values from fetch - """ - stop = CxoTime(stop) - start = stop - days * u.day - logger.info(f"Fetching telemetry for {msids} between {start.date} and {stop.date}") - - with fetch.data_source("cxc", "maude allow_subset=False"): - msidset = fetch.MSIDset(msids, start.date, stop.date) - - # Use the first MSID as the primary one to set the time base - msid0 = msidset[msids[0]] - - if len(msids) == 1: - # Only one MSID so just filter any bad values - msid0.filter_bad() - times = msid0.times - else: - # Multiple MSIDs so interpolate all to the same time base The assumption - # here is that all MSIDs have the same basic time base, e.g. AOCMDQT1-3. - msidset.interpolate(times=msid0.times, bad_union=True) - times = msidset.times - - # Finished when we found at least 10 good records (5 mins) - if len(times) < 10: - raise NoTelemetryError( - f"Found no telemetry for {msids!r} within {days} days of {stop}" - ) - - names = ["time"] + msids - out = Table([times] + [msidset[x].vals for x in msids], names=names) - return out - - -@functools.lru_cache(maxsize=1) -def get_ofp_states(stop, days): - """Get the Onboard Flight Program states for ``stop`` and ``days`` lookback - - This is normally "NRML" but in safe mode it is "SAFE" or other values. State codes: - ['NNRM' 'STDB' 'STBS' 'NRML' 'NSTB' 'SUOF' 'SYON' 'DPLY' 'SYSF' 'STUP' 'SAFE'] - """ - import astropy.table as tbl - from cheta.utils import logical_intervals - - msid = "conlofp" - tlm = get_telem_values([msid], stop, days) - states_list = [] - for state_code in np.unique(tlm[msid]): - states = logical_intervals( - tlm["time"], tlm[msid] == state_code, max_gap=2.1, complete_intervals=False - ) - states["val"] = state_code - states_list.append(states) - states = tbl.vstack(states_list) - states.sort("datestart") - - return states - - def add_figure_regions( fig: pgo.Figure, figure_start: CxoTimeLike, diff --git a/kadi/commands/validate.py b/kadi/commands/validate.py index c4dac401..430be1ab 100644 --- a/kadi/commands/validate.py +++ b/kadi/commands/validate.py @@ -25,7 +25,12 @@ import Ska.Shell import Ska.tdb from astropy.table import Table -from cheta.utils import logical_intervals +from cheta.utils import ( + get_ofp_states, + get_telem_table, + logical_intervals, + NoTelemetryError, +) from cxotime import CxoTime import kadi @@ -33,12 +38,9 @@ from kadi.commands.states import interpolate_states, reduce_states from kadi.commands.utils import ( CxoTimeLike, - NoTelemetryError, add_figure_regions, compress_time_series, convert_state_code_to_raw_val, - get_ofp_states, - get_telem_values, ) __all__ = [ @@ -52,7 +54,6 @@ "ValidatePcadMode", "ValidateLETG", "ValidateHETG", - "NoTelemetryError", "get_command_sheet_exclude_intervals", ] @@ -75,9 +76,9 @@ class PlotAttrs: :param title: (str): Plot title. :param ylabel: (str): Y-axis label. :param range: (list): Y-axis range (optional). - :param max_delta_time: (float): Maximum time delta before a new data point is plotted. - :param max_delta_val: (float): Maximum value delta before a new data point is plotted. - :param max_gap_time: (float): Maximum gap in time before a plot gap is inserted. + :param max_delta_time: (float): Maximum time delta before new data point is plotted. + :param max_delta_val: (float): Maximum value delta before new data point is plotted. + :param max_gap_time: (float): Maximum gap in time before plot gap is inserted. """ title: str @@ -135,9 +136,16 @@ def __init_subclass__(cls, **kwargs): @property def tlm(self): if not hasattr(self, "_tlm"): - self._tlm = get_telem_values( - msids=self.msids, stop=self.stop, days=self.days + logger.info( + f"Fetching telemetry for {self.msids} between {self.start.date} and" + f" {self.stop.date}" ) + self._tlm = get_telem_table(self.msids, self.start, self.stop) + if len(self._tlm) == 0: + raise NoTelemetryError( + f"No telemetry for {self.msids} between {self.start.date} and" + f" {self.stop.date}" + ) self.update_tlm() self.add_exclude_intervals() return self._tlm @@ -256,7 +264,7 @@ def exclude_ofp_intervals_except(self, states_expected: List[str]): This includes a padding of 30 minutes after SAFE mode and 5 minutes for non-NRML states other than SAFE like STUP, SYON, SYSF etc. """ - ofp_states = get_ofp_states(self.stop.date, self.days) + ofp_states = get_ofp_states(self.start, self.stop) for state in ofp_states: if state["val"] not in states_expected: pad_stop = 30 * u.min if state["val"] == "SAFE" else 5 * u.min diff --git a/kadi/scripts/validate_states.py b/kadi/scripts/validate_states.py index c390ac93..9b4f6f84 100644 --- a/kadi/scripts/validate_states.py +++ b/kadi/scripts/validate_states.py @@ -62,6 +62,7 @@ def main(args=None): # Enable logging in relevant packages logging.getLogger("kadi").setLevel(opt.log_level) fetch.add_logging_handler(level=opt.log_level) + fetch.data_source.set("cxc", "maude allow_subset=False") maude.set_logger_level(opt.log_level) maude.conf.cache_msid_queries = True From 97726551c0b563ea01d7554a0c54528d37a33aa4 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 9 Apr 2023 06:56:30 -0400 Subject: [PATCH 2/2] Remove local pytest.ini --- pytest.ini | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index bdaa7034..00000000 --- a/pytest.ini +++ /dev/null @@ -1,5 +0,0 @@ -[pytest] -filterwarnings = - ignore:the imp module is deprecated in favour of importlib:DeprecationWarning - ignore:'soft_unicode' has been renamed to 'soft_str' - ignore:`np.object` is a deprecated alias for the builtin `object`