Changes to support rms-filecache (#138)

* Changes to support filecache * Minor comment * Remove os.path.join, remove Python 3.8 support * Fix up main oops tests * Fix golden tests * More gold master work * Cleanup * Fix summary, logs, tests, add spicedb automated test * Fix tests * Specify filecache version * Update requirements.txt
SETI · Oct 31, 2024 · 0c9a7be · 0c9a7be
1 parent b60ad3a
commit 0c9a7be
Show file tree

Hide file tree

Showing 48 changed files with 813 additions and 651 deletions.
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -18,8 +18,6 @@ jobs:
       matrix:
         # MacOS: Python 3.8-3.10 does not currently work on MacOS.
         include:
-          - os: self-hosted-linux
-            python-version: "3.8"
           - os: self-hosted-linux
             python-version: "3.9"
           - os: self-hosted-linux
@@ -32,8 +30,6 @@ jobs:
             python-version: "3.11"
           - os: self-hosted-macos
             python-version: "3.12"
-          - os: self-hosted-windows
-            python-version: "3.8"
           - os: self-hosted-windows
             python-version: "3.9"
           - os: self-hosted-windows

diff --git a/README.md b/README.md
@@ -24,3 +24,56 @@
 # rms-oops
 
 This package is under development. Use with extreme caution.
+
+# Environment Variables
+
+- `OOPS_RESOURCES`: The top-level directory containing all files needed by OOPS. Unless
+  overriden as described below, this environment variable is the only one that needs to be
+  set. It is expected that the specified directory will contain the subdirectories:
+  - `SPICE`: SPICE kernels and associated database.
+  - `HST`: Reference and calibration files required for HST.
+  - `JWST`: Reference and calibration files required for JWST.
+  - `gold_master`: Gold master files for host tests.
+  - `test_data`: Test input files.
+- `SPICE_PATH`: The location of the SPICE kernel files; defaults to
+  `${OOPS_RESOURCES}/SPICE`.
+- `SPICE_SQLITE_DB_NAME`: The full path and filename of the SPICE SQlite database;
+  defaults to `${SPICE_PATH}/SPICE.db`.
+- `OOPS_TEST_DATA_PATH`: The location of the oops test files; defaults to
+  `${OOPS_RESOURCES}/test_data`.
+- `OOPS_GOLD_MASTER_PATH`: The location of the oops gold master test files; defaults to
+  `${OOPS_RESOURCES}/gold_master`.
+- `OOPS_BACKPLANE_OUTPUT_PATH`: The output path to use when writing backplanes
+  for gold master tests; defaults to the current directory.
+- `HST_IDC_PATH`: The location of HST IDC files; defaults to
+  `${OOPS_RESOURCES}/HST/IDC`.
+- `HST_SYN_PATH`: The location of HST SYN files; defaults to
+  `${OOPS_RESOURCES}/HST/SYN`.
+
+# Running Tests
+
+- To run the main oops unit tests:
+
+```sh
+python -m unittest tests/unittester.py
+```
+
+- To run the host tests including golden master tests:
+
+```sh
+python -m unittest tests/hosts/unittester.py
+```
+
+- To run the main oops unit tests and the host tests:
+
+```sh
+python -m unittest tests/unittester_with_hosts.py
+```
+
+- To run the gold master tests for one instrument with the ability to specify command
+  line options:
+
+```sh
+python tests/hosts/cassini/iss/gold_master.py --help
+python tests/hosts/galileo/ssi/gold_master.py --help
+```
diff --git a/oops/gold_master/__init__.py b/oops/gold_master/__init__.py
diff --git a/oops/gold_master/test_support.py b/oops/gold_master/test_support.py
@@ -3,39 +3,45 @@
 ################################################################################
 
 import os
-from oops.unittester_support import OOPS_RESOURCES, OOPS_TEST_DATA_PATH
+from oops.unittester_support import (OOPS_RESOURCES,
+                                     TEST_DATA_FILECACHE,
+                                     TEST_DATA_PREFIX)
 
-# Attributes ending with underscore contain a trailing "/"; others do not
+# import filecache
+# filecache.set_easy_logger()
 
-# gold_master directory, converted to a physical path
-# Default is $OOPS_RESOURCES/gold_master, but this can be changed by the user
-# by defining the environment variable OOPS_GOLD_MASTER_PATH.
+# Environment variables used to support oops "gold master" testing:
+#
+# - $OOPS_RESOURCES is the top-level directory. It is expected to have the
+#   subdirectory "gold_master".
+# - $OOPS_GOLD_MASTER_PATH will override the location of the "gold_master"
+#   directory.
+# - $OOPS_BACKPLANE_OUTPUT_PATH specifies the location in which generated
+#   backplanes should be written.
+#
+# Any environment variable may be a URI for a cloud resource such as
+#   gs://rms-oops-resources/gold_master
 
 try:
     OOPS_GOLD_MASTER_PATH = os.environ['OOPS_GOLD_MASTER_PATH']
 except KeyError:
     if OOPS_RESOURCES:
-        OOPS_GOLD_MASTER_PATH = os.path.join(OOPS_RESOURCES, 'gold_master')
-        OOPS_GOLD_MASTER_PATH_ = OOPS_GOLD_MASTER_PATH + '/'
+        OOPS_GOLD_MASTER_PATH = f'{OOPS_RESOURCES}/gold_master'
     else:
-        OOPS_GOLD_MASTER_PATH = ''
-        OOPS_GOLD_MASTER_PATH_ = ''
-else:
-    OOPS_GOLD_MASTER_PATH = os.path.realpath(OOPS_GOLD_MASTER_PATH)
-    OOPS_GOLD_MASTER_PATH = os.path.abspath(OOPS_GOLD_MASTER_PATH)
-    OOPS_GOLD_MASTER_PATH = OOPS_GOLD_MASTER_PATH.rstrip('/')
-    OOPS_GOLD_MASTER_PATH_ = OOPS_GOLD_MASTER_PATH + '/'
+        OOPS_GOLD_MASTER_PATH = None
 
-# Local user path for backplane testing
-# This is defined by the environment variable OOPS_BACKPLANE_OUTPUT_PATH.
 try:
     OOPS_BACKPLANE_OUTPUT_PATH = os.environ['OOPS_BACKPLANE_OUTPUT_PATH']
 except KeyError:
-    OOPS_BACKPLANE_OUTPUT_PATH = ''
-    OOPS_BACKPLANE_OUTPUT_PATH_ = ''
+    OOPS_BACKPLANE_OUTPUT_PATH = os.getcwd()
+
+
+if OOPS_GOLD_MASTER_PATH:
+    GOLD_MASTER_PREFIX = TEST_DATA_FILECACHE.new_prefix(OOPS_GOLD_MASTER_PATH)
 else:
-    OOPS_BACKPLANE_OUTPUT_PATH = os.path.normpath(OOPS_BACKPLANE_OUTPUT_PATH)
-    OOPS_BACKPLANE_OUTPUT_PATH = OOPS_BACKPLANE_OUTPUT_PATH.rstrip('/')
-    OOPS_BACKPLANE_OUTPUT_PATH_ = OOPS_BACKPLANE_OUTPUT_PATH + '/'
+    GOLD_MASTER_PREFIX = None
+
+BACKPLANE_OUTPUT_PREFIX = TEST_DATA_FILECACHE.new_prefix(
+    OOPS_BACKPLANE_OUTPUT_PATH)
 
 ################################################################################
diff --git a/oops/hosts/cassini/__init__.py b/oops/hosts/cassini/__init__.py
@@ -6,11 +6,8 @@
 ################################################################################
 
 import numpy as np
-import unittest
-import os.path
 
 import julian
-import textkernel
 import spicedb
 import cspyce
 import oops

diff --git a/oops/hosts/cassini/iss.py b/oops/hosts/cassini/iss.py
@@ -128,8 +128,7 @@ def from_index(filespec, **parameters):
 
         item.spice_kernels = Cassini.used_kernels(item.time, 'iss')
 
-        item.filespec = os.path.join(row_dict['VOLUME_ID'],
-                                     row_dict['FILE_SPECIFICATION_NAME'])
+        item.filespec = f"{row_dict['VOLUME_ID']}/{row_dict['FILE_SPECIFICATION_NAME']}"
         item.basename = row_dict['FILE_NAME']
 
         snapshots.append(item)

diff --git a/oops/hosts/galileo/__init__.py b/oops/hosts/galileo/__init__.py
@@ -5,7 +5,6 @@
 # sets.
 ################################################################################
 import numpy as np
-import os.path
 
 import julian
 import spicedb
@@ -140,26 +139,84 @@ def reset():
     #===========================================================================
     @staticmethod
     def load_kernels():
-        from oops.unittester_support import TESTDATA_PARENT_DIRECTORY
-        import glob
-
-        kdir = os.path.join(TESTDATA_PARENT_DIRECTORY, '../SPICE/Galileo/')
-        gkdir = os.path.join(TESTDATA_PARENT_DIRECTORY, '../SPICE/General/')
-
-        cspyce.furnsh(gkdir + 'LSK/naif0012.tls')
-        cspyce.furnsh(kdir + 'SCLK/mk00062a.tsc')
-        cspyce.furnsh(kdir + 'IK/gll36001.ti')
-        cspyce.furnsh(kdir + 'FK/gll_v0.tf')
-        cspyce.furnsh(kdir + 'SPK/de421.bsp')
-        cspyce.furnsh(kdir + 'SPK/de432s.bsp')
-
-        for ckfile in glob.glob(kdir + 'CK/*.bc'):
-            cspyce.furnsh(ckfile)
-
-        for spkfile in glob.glob(kdir + 'SPK/*.bsp'):
-            cspyce.furnsh(spkfile)
-
-        return
+        from spicedb import get_spice_filecache_prefix
+
+        SPICE_FILECACHE_PFX = get_spice_filecache_prefix()
+
+        paths = SPICE_FILECACHE_PFX.retrieve([
+            'General/LSK/naif0012.tls',
+            'Galileo/SCLK/mk00062a.tsc',
+            'Galileo/IK/gll36001.ti',
+            'Galileo/FK/gll_v0.tf',
+            'Galileo/SPK/de421.bsp',
+            'Galileo/SPK/de432s.bsp',
+            'Galileo/CK/ckc03b_plt.bc',
+            'Galileo/CK/ckc09b_plt.bc',
+            'Galileo/CK/ckc10b_plt.bc',
+            'Galileo/CK/ckc20f_plt.bc',
+            'Galileo/CK/ckc21f_plt.bc',
+            'Galileo/CK/ckc22f_plt.bc',
+            'Galileo/CK/ckc23f_plt.bc',
+            'Galileo/CK/ckc30f_plt.bc',
+            'Galileo/CK/cke04b_plt.bc',
+            'Galileo/CK/cke06b_plt.bc',
+            'Galileo/CK/cke11b_plt.bc',
+            'Galileo/CK/cke12f_plt.bc',
+            'Galileo/CK/cke14f_plt.bc',
+            'Galileo/CK/cke15f_plt.bc',
+            'Galileo/CK/cke16f_plt.bc',
+            'Galileo/CK/cke17f_plt.bc',
+            'Galileo/CK/cke18f_plt.bc',
+            'Galileo/CK/cke19f_plt.bc',
+            'Galileo/CK/cke26f_plt.bc',
+            'Galileo/CK/ckg01b_plt.bc',
+            'Galileo/CK/ckg02b_plt.bc',
+            'Galileo/CK/ckg07b_plt.bc',
+            'Galileo/CK/ckg08b_plt.bc',
+            'Galileo/CK/ckg28f_plt.bc',
+            'Galileo/CK/ckg29f_plt.bc',
+            'Galileo/CK/cki24f_plt.bc',
+            'Galileo/CK/cki25f_plt.bc',
+            'Galileo/CK/cki27f_plt.bc',
+            'Galileo/CK/cki31f_plt.bc',
+            'Galileo/CK/cki32f_plt.bc',
+            'Galileo/CK/ckj0cav3_plt.bc',
+            'Galileo/CK/ckj0cduh_plt.bc',
+            'Galileo/CK/ckj0cv3_plt.bc',
+            'Galileo/CK/ckj0eav3_plt.bc',
+            'Galileo/CK/ckj0ebv3_plt.bc',
+            'Galileo/CK/ckj0ecv3_plt.bc',
+            'Galileo/CK/ckjaap_plt.bc',
+            'Galileo/CK/ckjaav3_plt.bc',
+            'Galileo/CK/ckjabp_plt.bc',
+            'Galileo/CK/ckjabv3_plt.bc',
+            'Galileo/CK/gll_plt_pre_1990_v00.bc',
+            'Galileo/CK/gll_plt_pre_1991_v00.bc',
+            'Galileo/CK/gll_plt_pre_1992_v00.bc',
+            'Galileo/CK/gll_plt_pre_1993_v00.bc',
+            'Galileo/CK/gll_plt_pre_1994_v00.bc',
+            'Galileo/CK/gll_plt_pre_1995_v00.bc',
+            'Galileo/CK/gll_plt_pre_1996_v00.bc',
+            'Galileo/CK/gll_plt_pre_1997_v00.bc',
+            'Galileo/CK/gll_plt_pre_1998_v00.bc',
+            'Galileo/CK/gll_plt_pre_1999_v00.bc',
+            'Galileo/CK/gll_plt_pre_2000_v00.bc',
+            'Galileo/CK/gll_plt_pre_2001_v00.bc',
+            'Galileo/SPK/de421.bsp',
+            'Galileo/SPK/de432s.bsp',
+            'Galileo/SPK/gll_951120_021126_raj2007.bsp',
+            'Galileo/SPK/gll_951120_021126_raj2021.bsp',
+            'Galileo/SPK/s000131a.bsp',
+            'Galileo/SPK/s000615a.bsp',
+            'Galileo/SPK/s020128a.bsp',
+            'Galileo/SPK/s030916a.bsp',
+            'Galileo/SPK/s960730a.bsp',
+            'Galileo/SPK/s970311a.bsp',
+            'Galileo/SPK/s971125a.bsp',
+            'Galileo/SPK/s980326a.bsp',
+        ])
+        for path in paths:
+            cspyce.furnsh(path)
 
     ############################################################################
     # Initialize the kernel lists

diff --git a/oops/hosts/galileo/ssi/__init__.py b/oops/hosts/galileo/ssi/__init__.py
@@ -138,8 +138,7 @@ def from_index(filespec, supplemental_filespec=None, full_fov=False, **parameter
 
         item.spice_kernels = Galileo.used_kernels(item.time, 'ssi')
 
-        item.filespec = os.path.join(row_dict['VOLUME_ID'],
-                                     row_dict['FILE_SPECIFICATION_NAME'])
+        item.filespec = f"{row_dict['VOLUME_ID']}/{row_dict['FILE_SPECIFICATION_NAME']}"
         item.basename = basename
 
         snapshots.append(item)