From c2919182f5b553d0e799a38800ffed69eaf03f01 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:25:41 -0400 Subject: [PATCH] don't require pyessv-archive to run whole package --- .gitmodules | 3 --- Makefile | 8 +++---- README.md | 39 +++++++++++++++++++++++++++---- STACpopulator/cli.py | 8 ++++++- STACpopulator/exceptions.py | 5 ++++ STACpopulator/extensions/cmip6.py | 7 +++++- pyessv-archive | 1 - 7 files changed, 57 insertions(+), 14 deletions(-) delete mode 100644 .gitmodules create mode 100644 STACpopulator/exceptions.py delete mode 160000 pyessv-archive diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 09bf040..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "pyessv-archive"] - path = pyessv-archive - url = https://github.com/dchandan/pyessv-archive diff --git a/Makefile b/Makefile index d061580..1e31d6d 100644 --- a/Makefile +++ b/Makefile @@ -18,15 +18,15 @@ STAC_HOST ?= http://localhost:8880/stac # CATALOG = https://daccs.cs.toronto.edu/twitcher/ows/proxy/thredds/catalog/datasets/CMIP6/CMIP/NUIST/catalog.html CATALOG = https://daccs.cs.toronto.edu/twitcher/ows/proxy/thredds/catalog/datasets/CMIP6/CMIP/MIROC/catalog.html -PYESSV_ARCHIVE_DIR ?= ~/.esdoc/pyessv-archive +PYESSV_ARCHIVE_HOME ?= ~/.esdoc/pyessv-archive PYESSV_ARCHIVE_REF ?= https://github.com/ES-DOC/pyessv-archive ## -- Testing targets -------------------------------------------------------------------------------------------- ## setup-pyessv-archive: - @echo "Updating pyessv archive [$(shell realpath $(PYESSV_ARCHIVE_DIR))]..." - @[ -d $(PYESSV_ARCHIVE_DIR) ] || git clone "$(PYESSV_ARCHIVE_REF)" $(PYESSV_ARCHIVE_DIR) - @cd $(PYESSV_ARCHIVE_DIR) && git pull + @echo "Updating pyessv archive [$(shell realpath $(PYESSV_ARCHIVE_HOME))]..." + @[ -d $(PYESSV_ARCHIVE_HOME) ] || git clone "$(PYESSV_ARCHIVE_REF)" $(PYESSV_ARCHIVE_HOME) + @cd $(PYESSV_ARCHIVE_HOME) && git pull test-cmip6: python $(IMP_DIR)/CMIP6_UofT/add_CMIP6.py $(STAC_HOST) $(CATALOG) diff --git a/README.md b/README.md index 272154f..8cad8a9 100644 --- a/README.md +++ b/README.md @@ -49,18 +49,49 @@ You should then be able to call the STAC populator CLI with following commands: ```shell # obtain the installed version of the STAC populator -stac-popultaor --version +stac-populator --version # obtain general help about available commands -stac-popultaor --help +stac-populator --help # obtain general help about available STAC populator implementations -stac-popultaor run --help +stac-populator run --help # obtain help specifically for the execution of a STAC populator implementation -stac-popultaor run [implementation] --help +stac-populator run [implementation] --help ``` +### CMIP6 extension: extra requirements + +The CMIP6 stac-populator extension requires that the [pyessv-archive](https://github.com/ES-DOC/pyessv-archive) data +files be installed. To install this package to the default location in your home directory at `~/.esdoc/pyessv-archive`: + +```shell +git clone https://github.com/ES-DOC/pyessv-archive ~/.esdoc/pyessv-archive +# OR +make setup-pyessv-archive +``` + +You can also choose to install them to a location on disk other than the default: + +```shell +git clone https://github.com/ES-DOC/pyessv-archive /some/other/place +# OR +PYESSV_ARCHIVE_HOME=/some/other/place make setup-pyessv-archive +``` + +*Note*:
+If you have installed the [pyessv-archive](https://github.com/ES-DOC/pyessv-archive) data files to a non-default +location, you need to specify that location with the `PYESSV_ARCHIVE_HOME` environment variable. For example, +if you've installed the pyessv-archive files to `/some/other/place` then run the following before executing +any of the example commands above: + +```shell +export PYESSV_ARCHIVE_HOME=/some/other/place +``` + +### Docker + You can also employ the pre-built Docker, which can be called as follows, where `[command]` corresponds to any of the above example operations. diff --git a/STACpopulator/cli.py b/STACpopulator/cli.py index 7cde7b6..cc4a9fe 100644 --- a/STACpopulator/cli.py +++ b/STACpopulator/cli.py @@ -4,6 +4,7 @@ import logging import os import sys +import warnings from datetime import datetime from http import cookiejar from typing import Callable, Optional @@ -13,6 +14,7 @@ from requests.sessions import Session from STACpopulator import __version__ +from STACpopulator.exceptions import STACPopulatorError from STACpopulator.logging import setup_logging POPULATORS = {} @@ -151,7 +153,11 @@ def make_run_command_parser(parent) -> argparse.ArgumentParser: populator_name, pop_mod_file = populator_py_mod.rsplit(".", 1) populator_root = f"STACpopulator.{populators_impl}.{populator_name}" pop_mod_file_loc = f"{populator_root}.{pop_mod_file}" - populator_module = importlib.import_module(pop_mod_file_loc, populator_root) + try: + populator_module = importlib.import_module(pop_mod_file_loc, populator_root) + except STACPopulatorError as e: + warnings.warn(f"Could not load extension {populator_name} because of error {e}") + continue parser_maker: Callable[[], argparse.ArgumentParser] = getattr(populator_module, "make_parser", None) populator_runner = getattr(populator_module, "runner", None) # optional, call main directly if not available populator_caller = getattr(populator_module, "main", None) diff --git a/STACpopulator/exceptions.py b/STACpopulator/exceptions.py new file mode 100644 index 0000000..e97d3ea --- /dev/null +++ b/STACpopulator/exceptions.py @@ -0,0 +1,5 @@ +class STACPopulatorError(Exception): + pass + +class ExtensionLoadError(STACPopulatorError): + pass diff --git a/STACpopulator/extensions/cmip6.py b/STACpopulator/extensions/cmip6.py index 46f4edb..4f848c4 100644 --- a/STACpopulator/extensions/cmip6.py +++ b/STACpopulator/extensions/cmip6.py @@ -15,7 +15,6 @@ get_args, ) -import pyessv import pystac from pydantic import ( AnyHttpUrl, @@ -34,6 +33,7 @@ SummariesExtension, ) +from STACpopulator.exceptions import ExtensionLoadError from STACpopulator.models import AnyGeometry from STACpopulator.stac_utils import ( ServiceType, @@ -42,6 +42,11 @@ ncattrs_to_geometry, ) +try: + import pyessv +except OSError as e: + raise ExtensionLoadError(str(e)) from e + T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) SchemaName = Literal["cmip6"] diff --git a/pyessv-archive b/pyessv-archive deleted file mode 160000 index 32175dc..0000000 --- a/pyessv-archive +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 32175dc565d7e25e9dd780ff0306d2873ce9f84b