From c624aef1634fcb73553633adf08813c33832f638 Mon Sep 17 00:00:00 2001 From: James Kristoff Date: Fri, 11 Oct 2024 11:48:44 -0400 Subject: [PATCH 1/4] - Changed default "/proj/sot/ska/" since this only exists on linux. - added functions/methods to get files/paths remotely if necessary --- ska_tdb/tdb.py | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/ska_tdb/tdb.py b/ska_tdb/tdb.py index 901960a..e86384b 100644 --- a/ska_tdb/tdb.py +++ b/ska_tdb/tdb.py @@ -6,16 +6,17 @@ """ import os import re -import glob import numpy as np import six +from cheta.fetch import local_or_remote_function + __all__ = ['msids', 'tables', 'set_tdb_version', 'get_tdb_version', 'TableView', 'MsidView'] -SKA = os.environ.get('SKA', os.path.join(os.sep, 'proj', 'sot', 'ska')) +SKA = os.environ.get('SKA', '/proj/sot/ska') # Set None values for module globals that are set in set_tdb_version TDB_VERSIONS = None @@ -52,6 +53,24 @@ """.lower().split() +@local_or_remote_function("Get TDB Versions from remote server...") +def get_versions(SKA): + import os + import glob + + version_dirs = glob.glob(os.path.join(SKA, 'data', 'Ska.tdb', 'p0??')) + tdb_versions = sorted([int(os.path.basename(vdir)[2:]) for vdir in version_dirs]) + return tdb_versions + + +@local_or_remote_function("Get TDB Data Directory from remote server...") +def get_data_dir(tdb_version, SKA=SKA): + import os + + data_dir = os.path.join(SKA, 'data', 'Ska.tdb', 'p{:03d}'.format(tdb_version)) + return data_dir + + def set_tdb_version(version=None): """ Set the version of the TDB which is used. @@ -66,8 +85,7 @@ def set_tdb_version(version=None): global DATA_DIR global tables global msids - version_dirs = glob.glob(os.path.join(SKA, 'data', 'Ska.tdb', 'p0??')) - TDB_VERSIONS = sorted([int(os.path.basename(vdir)[2:]) for vdir in version_dirs]) + TDB_VERSIONS = get_versions(SKA) if version is None: if TDB_VERSIONS: @@ -78,7 +96,7 @@ def set_tdb_version(version=None): raise ValueError('TDB version must be one of the following: {}'.format(TDB_VERSIONS)) TDB_VERSION = version - DATA_DIR = os.path.join(SKA, 'data', 'Ska.tdb', 'p{:03d}'.format(TDB_VERSION)) + DATA_DIR = get_data_dir(TDB_VERSION, SKA) tables = TableDict() msids = MsidView() @@ -91,18 +109,28 @@ def get_tdb_version(): class TableDict(dict): + @local_or_remote_function("Loading file from remote server...") + def _read_npy_file(item, data_dir): + filename = os.path.join(data_dir, item + '.npy') + tv = TableView(np.load(filename)) + return filename, tv + + @local_or_remote_function("Finding *.npy files on remote server...") + def _get_all_npy_files(data_dir): + import glob + files = glob.glob(os.path.join(data_dir, '*.npy')) + return files + def __getitem__(self, item): if item not in self: try: - filename = os.path.join(DATA_DIR, item + '.npy') - self[item] = TableView(np.load(filename)) + filename, self[item] = TableDict._read_npy_file(item, DATA_DIR) except IOError: raise KeyError("Table {} not in TDB files (no file {})".format(item, filename)) return dict.__getitem__(self, item) def keys(self): - import glob - files = glob.glob(os.path.join(DATA_DIR, '*.npy')) + files = TableDict._get_all_npy_files(DATA_DIR) return [os.path.basename(x)[:-4] for x in files] From 9ae2bd28e15e1b253f06ec4a5bc61fe53dbf77dd Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 14 Oct 2024 11:54:21 -0400 Subject: [PATCH 2/4] Changes to make things work for TA --- ska_tdb/tdb.py | 52 +++++++++++++++++++-------------------- ska_tdb/tests/test_tdb.py | 9 +++++++ 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/ska_tdb/tdb.py b/ska_tdb/tdb.py index e86384b..3615acd 100644 --- a/ska_tdb/tdb.py +++ b/ska_tdb/tdb.py @@ -16,8 +16,6 @@ 'TableView', 'MsidView'] -SKA = os.environ.get('SKA', '/proj/sot/ska') - # Set None values for module globals that are set in set_tdb_version TDB_VERSIONS = None TDB_VERSION = None @@ -54,21 +52,32 @@ @local_or_remote_function("Get TDB Versions from remote server...") -def get_versions(SKA): +def get_versions(): import os - import glob - - version_dirs = glob.glob(os.path.join(SKA, 'data', 'Ska.tdb', 'p0??')) - tdb_versions = sorted([int(os.path.basename(vdir)[2:]) for vdir in version_dirs]) + from pathlib import Path # noqa + version_dirs = (Path(os.environ["SKA"]) / "data" / "Ska.tdb").glob("p0??") + tdb_versions = sorted(int(vdir.name[2:]) for vdir in version_dirs) return tdb_versions -@local_or_remote_function("Get TDB Data Directory from remote server...") -def get_data_dir(tdb_version, SKA=SKA): - import os +@local_or_remote_function("Get TDB Versions from remote server...") +def get_data_path(): + from pathlib import Path # noqa + return Path(os.environ["SKA"]) / "data" / "Ska.tdb" - data_dir = os.path.join(SKA, 'data', 'Ska.tdb', 'p{:03d}'.format(tdb_version)) - return data_dir + +@local_or_remote_function("Loading file from remote server...") +def _read_npy_file(item, data_dir): + filename = os.path.join(data_dir, item + '.npy') + tv = TableView(np.load(filename)) + return filename, tv + + +@local_or_remote_function("Finding *.npy files on remote server...") +def _get_all_npy_files(data_dir): + import glob + files = glob.glob(os.path.join(data_dir, '*.npy')) + return files def set_tdb_version(version=None): @@ -85,7 +94,7 @@ def set_tdb_version(version=None): global DATA_DIR global tables global msids - TDB_VERSIONS = get_versions(SKA) + TDB_VERSIONS = get_versions() if version is None: if TDB_VERSIONS: @@ -96,7 +105,7 @@ def set_tdb_version(version=None): raise ValueError('TDB version must be one of the following: {}'.format(TDB_VERSIONS)) TDB_VERSION = version - DATA_DIR = get_data_dir(TDB_VERSION, SKA) + DATA_DIR = get_data_path() / f"p{version:03d}" tables = TableDict() msids = MsidView() @@ -109,28 +118,17 @@ def get_tdb_version(): class TableDict(dict): - @local_or_remote_function("Loading file from remote server...") - def _read_npy_file(item, data_dir): - filename = os.path.join(data_dir, item + '.npy') - tv = TableView(np.load(filename)) - return filename, tv - - @local_or_remote_function("Finding *.npy files on remote server...") - def _get_all_npy_files(data_dir): - import glob - files = glob.glob(os.path.join(data_dir, '*.npy')) - return files def __getitem__(self, item): if item not in self: try: - filename, self[item] = TableDict._read_npy_file(item, DATA_DIR) + filename, self[item] = _read_npy_file(item, DATA_DIR) except IOError: raise KeyError("Table {} not in TDB files (no file {})".format(item, filename)) return dict.__getitem__(self, item) def keys(self): - files = TableDict._get_all_npy_files(DATA_DIR) + files = _get_all_npy_files(DATA_DIR) return [os.path.basename(x)[:-4] for x in files] diff --git a/ska_tdb/tests/test_tdb.py b/ska_tdb/tests/test_tdb.py index 8477174..b4da6a7 100644 --- a/ska_tdb/tests/test_tdb.py +++ b/ska_tdb/tests/test_tdb.py @@ -1,6 +1,11 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +import os +import pytest + from .. import msids, tables, set_tdb_version, get_tdb_version +from ska_tdb.tdb import get_data_path +SKA_ACCESS_REMOTELY = os.environ.get("SKA_ACCESS_REMOTELY") == "True" # Set to fixed version for regression testing TDB_VERSION = 14 set_tdb_version(TDB_VERSION) @@ -13,6 +18,10 @@ 'DESCRIPTION', 'EHS_HEADER_FLAG') +@pytest.mark.skipif(not SKA_ACCESS_REMOTELY, reason="remote access not being tested") +def test_remote_access(): + assert str(get_data_path()) == "/proj/sot/ska3/flight/data/Ska.tdb" + def test_msids(): tephin = msids['tephin'] assert tephin.Tmsrment.colnames == tmsrment_colnames From cf1f145383ed7c58e596c91c3da5094110faa723 Mon Sep 17 00:00:00 2001 From: James Kristoff Date: Mon, 14 Oct 2024 16:24:31 -0400 Subject: [PATCH 3/4] - returned path as string to avoid issues trying to instantiate PosixPath objects on Windows. - passed version into remote function `get_data_path` to avoid path manipulations outside of remote functions. --- ska_tdb/tdb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ska_tdb/tdb.py b/ska_tdb/tdb.py index 3615acd..23a9311 100644 --- a/ska_tdb/tdb.py +++ b/ska_tdb/tdb.py @@ -60,10 +60,10 @@ def get_versions(): return tdb_versions -@local_or_remote_function("Get TDB Versions from remote server...") -def get_data_path(): +@local_or_remote_function("Get TDB Data Directory from remote server...") +def get_data_path(version): from pathlib import Path # noqa - return Path(os.environ["SKA"]) / "data" / "Ska.tdb" + return str(Path(os.environ["SKA"]) / "data" / "Ska.tdb" / f"p{version:03d}") @local_or_remote_function("Loading file from remote server...") @@ -105,7 +105,7 @@ def set_tdb_version(version=None): raise ValueError('TDB version must be one of the following: {}'.format(TDB_VERSIONS)) TDB_VERSION = version - DATA_DIR = get_data_path() / f"p{version:03d}" + DATA_DIR = get_data_path(version) tables = TableDict() msids = MsidView() From d5a23def2832d4f713b4f049c1a854364ebd5c65 Mon Sep 17 00:00:00 2001 From: James Kristoff Date: Mon, 14 Oct 2024 16:31:57 -0400 Subject: [PATCH 4/4] - updated test to expect a version input --- ska_tdb/tests/test_tdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ska_tdb/tests/test_tdb.py b/ska_tdb/tests/test_tdb.py index b4da6a7..05c2e46 100644 --- a/ska_tdb/tests/test_tdb.py +++ b/ska_tdb/tests/test_tdb.py @@ -20,7 +20,8 @@ @pytest.mark.skipif(not SKA_ACCESS_REMOTELY, reason="remote access not being tested") def test_remote_access(): - assert str(get_data_path()) == "/proj/sot/ska3/flight/data/Ska.tdb" + assert str(get_data_path(10)) == "/proj/sot/ska3/flight/data/Ska.tdb/p010" + def test_msids(): tephin = msids['tephin']